<commit_before>"""
Module containing manifolds of fixed rank matrices.
"""

import numpy as np

from pymanopt.manifolds.manifold import Manifold
from pymanopt.manifolds.stiefel import Stiefel
from pymanopt.tools import ndarraySequenceMixin

class FixedRankEmbedded(Manifold):
    """
    Note: Currently not compatible with the second order TrustRegions solver.
    Should be fixed soon.

    Manifold of m-by-n real matrices of fixed rank k. This follows the
    embedded geometry described in Bart Vandereycken's 2013 paper:
    "Low-rank matrix completion by Riemannian optimization".

    Paper link: http://arxiv.org/pdf/1209.3834.pdf

    For efficiency purposes, Pymanopt does not represent points on this
    manifold explicitly using m x n matrices, but instead implicitly using
    a truncated singular value decomposition. Specifically, a point is
    represented by a tuple (u, s, vt) of three numpy arrays. The arrays u,
    s and vt have shapes (m, k), (k,) and (k, n) respectively, and the low
    rank matrix which they represent can be recovered by the matrix product
    u * diag(s) * vt.

    For example, to optimize over the space of 5 by 4 matrices with rank 3,
    we would need to
    >>> import pymanopt.manifolds
    >>> manifold = pymanopt.manifolds.FixedRankEmbedded(5, 4, 3)

    Then the shapes will be as follows:
    >>> x = manifold.rand()
    >>> x[0].shape
    (5, 3)
    >>> x[1].shape
    (3,)
    >>> x[2].shape
    (3, 4)

    and the full matrix can be recovered using the matrix product
    x[0] * diag(x[1]) * x[2]:
    >>> import numpy as np
    >>> X = x[0].dot(np.diag(x[1])).dot(x[2])

    Tangent vectors are represented as a tuple (Up, M Vp). The matrices Up
    (mxk) and Vp (nxk) obey Up'*U = 0 and Vp'*V = 0.
    The matrix M (kxk) is arbitrary. Such a structure corresponds to the
    following tangent vector in the ambient space of mxn matrices:
      Z = U*M*V' + Up*V' + U*Vp'
    where (U, S, V) is the current point and (Up, M, Vp) is the tangent
    vector at that point.

    Vectors in the ambient space are best represented as mxn matrices. If
    these are low-rank, they may also be represented as structures with
    U, S, V fields, such that Z = U*S*V'. There are no restrictions on what
    U, S and V are, as long as their product as indicated yields a real, mxn
    matrix.

    The chosen geometry yields a Riemannian submanifold of the embedding
    space R^(mxn) equipped with the usual trace (Frobenius) inner product.

    Please cite the Pymanopt paper as well as the research paper:
        @Article{vandereycken2013lowrank,
          Title   = {Low-rank matrix completion by {Riemannian} optimization},
          Author  = {Vandereycken, B.},
          Journal = {SIAM Journal on Optimization},
          Year    = {2013},
          Number  = {2},
          Pages   = {1214--1236},
          Volume  = {23},
          Doi     = {10.1137/110845768}
        }

    This file is based on fixedrankembeddedfactory from Manopt: www.manopt.org.
    Ported by: Jamie Townsend, Sebastian Weichwald
    Original author: Nicolas Boumal, Dec. 30, 2012.
    """

    def __init__(self, m, n, k):
        self._m = m
        self._n = n
        self._k = k

        self._name = ("Manifold of {m}-by-{n} matrices with rank {k} and "
                      "embedded geometry".format(m=m, n=n, k=k))

        self._stiefel_m = Stiefel(m, k)
        self._stiefel_n = Stiefel(n, k)

    def __str__(self):
        return self._name

    @property
    def dim(self):
        return (self._m + self._n - self._k) * self._k

    @property
    def typicaldist(self):
        return self.dim

    def dist(self, X, Y):
        raise NotImplementedError

    def inner(self, X, G, H):
        return np.sum(np.tensordot(a, b) for (a, b) in zip(G, H))

    def _apply_ambient(self, Z, W):
        """
        For a given ambient vector Z, given as a tuple (U, S, V) such that
        Z = U*S*V', applies it to a matrix W to calculate the matrix product
        ZW.
        """
        if isinstance(Z, tuple):
            return np.dot(Z[0], np.dot(Z[1], np.dot(Z[2].T, W)))
        else:
            return np.dot(Z, W)

    def _apply_ambient_transpose(self, Z, W):
        """
        Same as apply_ambient, but applies Z' to W.
        """
        if isinstance(Z, tuple):
            return np.dot(Z[2], np.dot(Z[1], np.dot(Z[0].T, W)))
        else:
            return np.dot(Z.T, W)

    def proj(self, X, Z):
        """
        Note that Z must either be an m x n matrix from the ambient space, or
        else a tuple (Uz, Sz, Vz), where Uz * Sz * Vz is in the ambient space
        (of low-rank matrices).

        This function then returns a tangent vector parameterized as
        (Up, M, Vp), as described in the class docstring.
        """
        ZV = self._apply_ambient(Z, X[2].T)
        UtZV = np.dot(X[0].T, ZV)
        ZtU = self._apply_ambient_transpose(Z, X[0])

        Up = ZV - np.dot(X[0], UtZV)
        M = UtZV
        Vp = ZtU - np.dot(X[2].T, UtZV.T)

        return _FixedRankTangentVector((Up, M, Vp))

    def egrad2rgrad(self, x, egrad):
        """
        Assuming that the cost function being optimized has been defined
        in terms of the low-rank singular value decomposition of X, the
        gradient returned by the autodiff backends will have three components
        and will be in the form of a tuple egrad = (df/dU, df/dS, df/dV).

        This function correctly maps a gradient of this form into the tangent
        space. See https://j-towns.github.io/papers/svd-derivative.pdf for a
        derivation.
        """
        utdu = np.dot(x[0].T, egrad[0])
        uutdu = np.dot(x[0], utdu)
        Up = (egrad[0] - uutdu) / x[1]

        vtdv = np.dot(x[2], egrad[2].T)
        vvtdv = np.dot(x[2].T, vtdv)
        Vp = (egrad[2].T - vvtdv) / x[1]

        i = np.eye(self._k)
        f = 1 / (x[1][np.newaxis, :]**2 - x[1][:, np.newaxis]**2 + i)

        M = (f * (utdu - utdu.T) * x[1] +
             x[1][:, np.newaxis] * f * (vtdv - vtdv.T) + np.diag(egrad[1]))

        return _FixedRankTangentVector((Up, M, Vp))

    def ehess2rhess(self, X, egrad, ehess, H):
        raise NotImplementedError

    # This retraction is second order, following general results from
    # Absil, Malick, "Projection-like retractions on matrix manifolds",
    # SIAM J. Optim., 22 (2012), pp. 135-158.
    def retr(self, X, Z):
        Qu, Ru = np.linalg.qr(Z[0])
        Qv, Rv = np.linalg.qr(Z[2])

        T = np.vstack((np.hstack((np.diag(X[1]) + Z[1], Rv.T)),
                      np.hstack((Ru, np.zeros((self._k, self._k))))))

        # Numpy svd outputs St as a 1d vector, not a matrix.
        Ut, St, Vt = np.linalg.svd(T, full_matrices=False)

        # Transpose because numpy outputs it the wrong way.
        Vt = Vt.T

        U = np.dot(np.hstack((X[0], Qu)), Ut[:, :self._k])
        V = np.dot(np.hstack((X[2].T, Qv)), Vt[:, :self._k])
        S = St[:self._k] + np.spacing(1)
        return (U, S, V.T)

    def norm(self, X, G):
        return np.sqrt(self.inner(X, G, G))

    def rand(self):
        u = self._stiefel_m.rand()
        s = np.sort(np.random.rand(self._k))[::-1]
        vt = self._stiefel_n.rand().T
        return (u, s, vt)

    def _tangent(self, X, Z):
        """
        Given Z in tangent vector format, projects the components Up and Vp
        such that they satisfy the tangent space constraints up to numerical
        errors. If Z was indeed a tangent vector at X, this should barely
        affect Z (it would not at all if we had infinite numerical accuracy).
        """
        Up = Z[0] - np.dot(X[0], np.dot(X[0].T, Z[0]))
        Vp = Z[2] - np.dot(X[2].T, np.dot(X[2], Z[2]))

        return _FixedRankTangentVector((Up, Z[1], Vp))

    def randvec(self, X):
        Up = np.random.randn(self._m, self._k)
        Vp = np.random.randn(self._n, self._k)
        M = np.random.randn(self._k, self._k)

        Z = self._tangent(X, (Up, M, Vp))

        nrm = self.norm(X, Z)

        return _FixedRankTangentVector((Z[0]/nrm, Z[1]/nrm, Z[2]/nrm))

    def tangent2ambient(self, X, Z):
        """
        Transforms a tangent vector Z represented as a structure (Up, M, Vp)
        into a structure with fields (U, S, V) that represents that same
        tangent vector in the ambient space of mxn matrices, as U*S*V'.
        This matrix is equal to X.U*Z.M*X.V' + Z.Up*X.V' + X.U*Z.Vp'. The
        latter is an mxn matrix, which could be too large to build
        explicitly, and this is why we return a low-rank representation
        instead. Note that there are no guarantees on U, S and V other than
        that USV' is the desired matrix. In particular, U and V are not (in
        general) orthonormal and S is not (in general) diagonal.
        (In this implementation, S is identity, but this might change.)
        """
        U = np.hstack((np.dot(X[0], Z[1]) + Z[0], X[0]))
        S = np.eye(2 * self._k)
        V = np.hstack(([X[2].T, Z[2]]))
        return (U, S, V)

    # Comment from Manopt:
    # New vector transport on June 24, 2014 (as indicated by Bart)
    # Reference: Absil, Mahony, Sepulchre 2008 section 8.1.3:
    # For Riemannian submanifolds of a Euclidean space, it is acceptable to
    # transport simply by orthogonal projection of the tangent vector
    # translated in the ambient space.
    def transp(self, X1, X2, G):
        return self.proj(X2, self.tangent2ambient(X1, G))

    def exp(self, X, U):
        raise NotImplementedError

    def log(self, X, Y):
        raise NotImplementedError

    def pairmean(self, X, Y):
        raise NotImplementedError

    def zerovec(self, X):
        return _FixedRankTangentVector((np.zeros((self._m, self._k)),
                                        np.zeros((self._k, self._k)),
                                        np.zeros((self._n, self._k))))

class _FixedRankTangentVector(tuple, ndarraySequenceMixin):
    def __repr__(self):
        repr_ = super(_FixedRankTangentVector, self).__repr__()
        return "_FixedRankTangentVector: " + repr_

    def to_ambient(self, x):
        Z1 = x[0].dot(self[1].dot(x[2]))
        Z2 = self[0].dot(x[2])
        Z3 = x[0].dot(self[2].T)
        return Z1 + Z2 + Z3

    def __add__(self, other):
        return _FixedRankTangentVector((s + o for (s, o) in zip(self, other)))

    def __sub__(self, other):
        return _FixedRankTangentVector((s - o for (s, o) in zip(self, other)))

    def __mul__(self, other):
        return _FixedRankTangentVector((other * s for s in self))

    __rmul__ = __mul__

    def __div__(self, other):
        return _FixedRankTangentVector((val / other for val in self))

    def __neg__(self):
        return _FixedRankTangentVector((-val for val in self))
<commit_msg>Adapt 'FixedRankEmbedded' class to new base class<commit_after>"""
Module containing manifolds of fixed rank matrices.
"""

import numpy as np

from pymanopt.manifolds.manifold import EuclideanEmbeddedSubmanifold
from pymanopt.manifolds.stiefel import Stiefel
from pymanopt.tools import ndarraySequenceMixin

class FixedRankEmbedded(EuclideanEmbeddedSubmanifold):
    """
    Note: Currently not compatible with the second order TrustRegions solver.
    Should be fixed soon.

    Manifold of m-by-n real matrices of fixed rank k. This follows the
    embedded geometry described in Bart Vandereycken's 2013 paper:
    "Low-rank matrix completion by Riemannian optimization".

    Paper link: http://arxiv.org/pdf/1209.3834.pdf

    For efficiency purposes, Pymanopt does not represent points on this
    manifold explicitly using m x n matrices, but instead implicitly using
    a truncated singular value decomposition. Specifically, a point is
    represented by a tuple (u, s, vt) of three numpy arrays. The arrays u,
    s and vt have shapes (m, k), (k,) and (k, n) respectively, and the low
    rank matrix which they represent can be recovered by the matrix product
    u * diag(s) * vt.

    For example, to optimize over the space of 5 by 4 matrices with rank 3,
    we would need to
    >>> import pymanopt.manifolds
    >>> manifold = pymanopt.manifolds.FixedRankEmbedded(5, 4, 3)

    Then the shapes will be as follows:
    >>> x = manifold.rand()
    >>> x[0].shape
    (5, 3)
    >>> x[1].shape
    (3,)
    >>> x[2].shape
    (3, 4)

    and the full matrix can be recovered using the matrix product
    x[0] * diag(x[1]) * x[2]:
    >>> import numpy as np
    >>> X = x[0].dot(np.diag(x[1])).dot(x[2])

    Tangent vectors are represented as a tuple (Up, M Vp). The matrices Up
    (mxk) and Vp (nxk) obey Up'*U = 0 and Vp'*V = 0.
    The matrix M (kxk) is arbitrary. Such a structure corresponds to the
    following tangent vector in the ambient space of mxn matrices:
      Z = U*M*V' + Up*V' + U*Vp'
    where (U, S, V) is the current point and (Up, M, Vp) is the tangent
    vector at that point.

    Vectors in the ambient space are best represented as mxn matrices. If
    these are low-rank, they may also be represented as structures with
    U, S, V fields, such that Z = U*S*V'. There are no restrictions on what
    U, S and V are, as long as their product as indicated yields a real, mxn
    matrix.

    The chosen geometry yields a Riemannian submanifold of the embedding
    space R^(mxn) equipped with the usual trace (Frobenius) inner product.

    Please cite the Pymanopt paper as well as the research paper:
        @Article{vandereycken2013lowrank,
          Title   = {Low-rank matrix completion by {Riemannian} optimization},
          Author  = {Vandereycken, B.},
          Journal = {SIAM Journal on Optimization},
          Year    = {2013},
          Number  = {2},
          Pages   = {1214--1236},
          Volume  = {23},
          Doi     = {10.1137/110845768}
        }

    This file is based on fixedrankembeddedfactory from Manopt: www.manopt.org.
    Ported by: Jamie Townsend, Sebastian Weichwald
    Original author: Nicolas Boumal, Dec. 30, 2012.
    """

    def __init__(self, m, n, k):
        self._m = m
        self._n = n
        self._k = k
        self._stiefel_m = Stiefel(m, k)
        self._stiefel_n = Stiefel(n, k)

        name = ("Manifold of {m}-by-{n} matrices with rank {k} and embedded "
                "geometry".format(m=m, n=n, k=k))
        dimension = (m + n - k) * k
        super().__init__(name, dimension)

    @property
    def typicaldist(self):
        return self.dim

    def dist(self, X, Y):
        raise NotImplementedError

    def inner(self, X, G, H):
        return np.sum(np.tensordot(a, b) for (a, b) in zip(G, H))

    def _apply_ambient(self, Z, W):
        """
        For a given ambient vector Z, given as a tuple (U, S, V) such that
        Z = U*S*V', applies it to a matrix W to calculate the matrix product
        ZW.
        """
        if isinstance(Z, tuple):
            return np.dot(Z[0], np.dot(Z[1], np.dot(Z[2].T, W)))
        else:
            return np.dot(Z, W)

    def _apply_ambient_transpose(self, Z, W):
        """
        Same as apply_ambient, but applies Z' to W.
        """
        if isinstance(Z, tuple):
            return np.dot(Z[2], np.dot(Z[1], np.dot(Z[0].T, W)))
        else:
            return np.dot(Z.T, W)

    def proj(self, X, Z):
        """
        Note that Z must either be an m x n matrix from the ambient space, or
        else a tuple (Uz, Sz, Vz), where Uz * Sz * Vz is in the ambient space
        (of low-rank matrices).

        This function then returns a tangent vector parameterized as
        (Up, M, Vp), as described in the class docstring.
        """
        ZV = self._apply_ambient(Z, X[2].T)
        UtZV = np.dot(X[0].T, ZV)
        ZtU = self._apply_ambient_transpose(Z, X[0])

        Up = ZV - np.dot(X[0], UtZV)
        M = UtZV
        Vp = ZtU - np.dot(X[2].T, UtZV.T)

        return _FixedRankTangentVector((Up, M, Vp))

    def egrad2rgrad(self, x, egrad):
        """
        Assuming that the cost function being optimized has been defined
        in terms of the low-rank singular value decomposition of X, the
        gradient returned by the autodiff backends will have three components
        and will be in the form of a tuple egrad = (df/dU, df/dS, df/dV).

        This function correctly maps a gradient of this form into the tangent
        space. See https://j-towns.github.io/papers/svd-derivative.pdf for a
        derivation.
        """
        utdu = np.dot(x[0].T, egrad[0])
        uutdu = np.dot(x[0], utdu)
        Up = (egrad[0] - uutdu) / x[1]

        vtdv = np.dot(x[2], egrad[2].T)
        vvtdv = np.dot(x[2].T, vtdv)
        Vp = (egrad[2].T - vvtdv) / x[1]

        i = np.eye(self._k)
        f = 1 / (x[1][np.newaxis, :]**2 - x[1][:, np.newaxis]**2 + i)

        M = (f * (utdu - utdu.T) * x[1] +
             x[1][:, np.newaxis] * f * (vtdv - vtdv.T) + np.diag(egrad[1]))

        return _FixedRankTangentVector((Up, M, Vp))

    # TODO(nkoep): Implement the 'weingarten' method to support the
    # trust-region solver, cf.
    # https://sites.uclouvain.be/absil/2013-01/Weingarten_07PA_techrep.pdf

    # This retraction is second order, following general results from
    # Absil, Malick, "Projection-like retractions on matrix manifolds",
    # SIAM J. Optim., 22 (2012), pp. 135-158.
    def retr(self, X, Z):
        Qu, Ru = np.linalg.qr(Z[0])
        Qv, Rv = np.linalg.qr(Z[2])

        T = np.vstack((np.hstack((np.diag(X[1]) + Z[1], Rv.T)),
                      np.hstack((Ru, np.zeros((self._k, self._k))))))

        # Numpy svd outputs St as a 1d vector, not a matrix.
        Ut, St, Vt = np.linalg.svd(T, full_matrices=False)

        # Transpose because numpy outputs it the wrong way.
        Vt = Vt.T

        U = np.dot(np.hstack((X[0], Qu)), Ut[:, :self._k])
        V = np.dot(np.hstack((X[2].T, Qv)), Vt[:, :self._k])
        S = St[:self._k] + np.spacing(1)
        return (U, S, V.T)

    def norm(self, X, G):
        return np.sqrt(self.inner(X, G, G))

    def rand(self):
        u = self._stiefel_m.rand()
        s = np.sort(np.random.rand(self._k))[::-1]
        vt = self._stiefel_n.rand().T
        return (u, s, vt)

    def _tangent(self, X, Z):
        """
        Given Z in tangent vector format, projects the components Up and Vp
        such that they satisfy the tangent space constraints up to numerical
        errors. If Z was indeed a tangent vector at X, this should barely
        affect Z (it would not at all if we had infinite numerical accuracy).
        """
        Up = Z[0] - np.dot(X[0], np.dot(X[0].T, Z[0]))
        Vp = Z[2] - np.dot(X[2].T, np.dot(X[2], Z[2]))

        return _FixedRankTangentVector((Up, Z[1], Vp))

    def randvec(self, X):
        Up = np.random.randn(self._m, self._k)
        Vp = np.random.randn(self._n, self._k)
        M = np.random.randn(self._k, self._k)

        Z = self._tangent(X, (Up, M, Vp))

        nrm = self.norm(X, Z)

        return _FixedRankTangentVector((Z[0]/nrm, Z[1]/nrm, Z[2]/nrm))

    def tangent2ambient(self, X, Z):
        """
        Transforms a tangent vector Z represented as a structure (Up, M, Vp)
        into a structure with fields (U, S, V) that represents that same
        tangent vector in the ambient space of mxn matrices, as U*S*V'.
        This matrix is equal to X.U*Z.M*X.V' + Z.Up*X.V' + X.U*Z.Vp'. The
        latter is an mxn matrix, which could be too large to build
        explicitly, and this is why we return a low-rank representation
        instead. Note that there are no guarantees on U, S and V other than
        that USV' is the desired matrix. In particular, U and V are not (in
        general) orthonormal and S is not (in general) diagonal.
        (In this implementation, S is identity, but this might change.)
        """
        U = np.hstack((np.dot(X[0], Z[1]) + Z[0], X[0]))
        S = np.eye(2 * self._k)
        V = np.hstack(([X[2].T, Z[2]]))
        return (U, S, V)

    # Comment from Manopt:
    # New vector transport on June 24, 2014 (as indicated by Bart)
    # Reference: Absil, Mahony, Sepulchre 2008 section 8.1.3:
    # For Riemannian submanifolds of a Euclidean space, it is acceptable to
    # transport simply by orthogonal projection of the tangent vector
    # translated in the ambient space.
    def transp(self, X1, X2, G):
        return self.proj(X2, self.tangent2ambient(X1, G))

    def zerovec(self, X):
        return _FixedRankTangentVector((np.zeros((self._m, self._k)),
                                        np.zeros((self._k, self._k)),
                                        np.zeros((self._n, self._k))))

class _FixedRankTangentVector(tuple, ndarraySequenceMixin):
    def __repr__(self):
        repr_ = super(_FixedRankTangentVector, self).__repr__()
        return "_FixedRankTangentVector: " + repr_

    def to_ambient(self, x):
        Z1 = x[0].dot(self[1].dot(x[2]))
        Z2 = self[0].dot(x[2])
        Z3 = x[0].dot(self[2].T)
        return Z1 + Z2 + Z3

    def __add__(self, other):
        return _FixedRankTangentVector((s + o for (s, o) in zip(self, other)))

    def __sub__(self, other):
        return _FixedRankTangentVector((s - o for (s, o) in zip(self, other)))

    def __mul__(self, other):
        return _FixedRankTangentVector((other * s for s in self))

    __rmul__ = __mul__

    def __div__(self, other):
        return _FixedRankTangentVector((val / other for val in self))

    def __neg__(self):
        return _FixedRankTangentVector((-val for val in self))
<|endoftext|>


<commit_before>#!/usr/bin/env python
__version__ = "0.2"

import sys
from os.path import expanduser, splitext, split
from os import remove, stat
import errno
from subprocess import check_call
import yaml
import argparse

from jinja2 import Environment

from texhelpers import escape_tex, TeXLoader

class Jekkish():

    def __init__(self, target_file, job_name=False):
        self.target_file = target_file
        fullpath, ext = splitext(self.target_file.name)
        path, filename = split(fullpath)
        self.temp_file = filename + '._' + ext[1:]
        self.job_name = job_name if job_name else filename
        self.variables = self.load_variables()
        self.home = expanduser("~")
        self.template_dir = self.home + '/.jekkish'
        self.default_template = self.template_dir + '/default.tex'

    def load_variables(self, division_string="---\n"):
        """ Converts the file to YAML and returns the parsed data.

        Ignores any content above the YAML header (start_yaml),
        Loads everything after the YAML as part of the 'content' variable """

        start_yaml = False
        end_yaml = False
        variables = ""
        content = "content: >"

        for line in self.target_file:

            if str(line) == division_string:
                if start_yaml:
                    end_yaml = True
                else:
                    start_yaml = True
            else:
                if start_yaml:
                    if not end_yaml:
                        variables += line
                    else:
                        if line == "\n":
                            content += "  {}".format(line)
                        else:
                            content += "  {}\n".format(line)

        variables += content
        return yaml.load(variables)

    def make_file(self):
        texenv = Environment(loader=TeXLoader(self.home))
        texenv.block_start_string = '((*'
        texenv.block_end_string = '*))'
        texenv.variable_start_string = '((('
        texenv.variable_end_string = ')))'
        texenv.comment_start_string = '((='
        texenv.comment_end_string = '=))'
        texenv.filters['escape_tex'] = escape_tex

        if self.variables["template"]:
            template_file = self.template_dir + \
                '/' + self.variables["template"] + '.tex'
        else:
            template_file = self.template_dir + \
                '/' + self.default

        template = texenv.get_template(template_file)

        f = open(self.temp_file, "w")
        f.write(template.render(self.variables))
        f.close()

        print("Temporary LaTeX file created ({})\n---".format(self.temp_file))

    def make_pdf(self, clean=True):
        print("Generating PDF\n---")

        command = 'pdflatex --jobname=' + self.job_name + ' ' + self.temp_file
        check_call(command, shell=True)

        if clean:
            for ext in ['aux', 'log', 'out', 'ent']:
                try:
                    remove(self.job_name + '.' + ext)
                except (OSError, IOError) as e:
                    # Use FileNotFoundError when python 2 is dropped
                    if e.errno != errno.ENOENT:
                        raise

    def render(self):
        self.make_file()
        self.make_pdf()

    def watch(self):
        print("---\nJekkish running in watch mode\n")
        print("---\nPerforming initial rendering\n---")

        last_time = False
        while True:
            if last_time != stat(self.target_file.name).st_mtime:
                last_time = stat(self.target_file.name).st_mtime
                self.render()
                print("---\nWatching {}\n---".format(self.target_file.name))

def main():
    parser = argparse.ArgumentParser(prog="Jekkish", description="A template-based pdftex CLI frontend inspired by Jekyll")
    parser.add_argument('filename', type=argparse.FileType('r'), default=sys.stdin, help='The file to process')
    parser.add_argument('jobname', nargs="?", default=False, help='Job name for pdftex output')
    # parser.add_argument('--watch', action='store_const', const=True, help='Watch <filename> for changes')
    parser.add_argument('--version', action='version', version='%(prog)s 0.1.8py3fix')
    args = parser.parse_args()

    new_file = Jekkish(args.filename, args.jobname)

    # if args.watch:
    #     new_file.watch()
    # else:
    new_file.render()

if __name__ == "__main__":
    main()
<commit_msg>Improved template handling.<commit_after>#!/usr/bin/env python
__version__ = "0.2"

import sys
from os.path import expanduser, splitext, split, exists, realpath, dirname, join
from os import remove, stat
import errno
from subprocess import check_call
import yaml
import argparse

from jinja2 import Environment
from texhelpers import escape_tex, TeXLoader

class Jekkish():

    def __init__(self, target_file, job_name=False):
        self.target_file = target_file
        fullpath, ext = splitext(self.target_file.name)
        path, filename = split(fullpath)
        self.temp_file = filename + '._' + ext[1:]
        self.job_name = job_name if job_name else filename
        self.variables = self.load_variables()
        self.home = expanduser("~")
        self.template_dir = self.home + '/.jekkish'
        self.default_template = 'default'

    def load_variables(self, division_string="---\n"):
        """ Converts the file to YAML and returns the parsed data.

        Ignores any content above the YAML header (start_yaml),
        Loads everything after the YAML as part of the 'content' variable """

        start_yaml = False
        end_yaml = False
        variables = ""
        content = "content: >"

        for line in self.target_file:
            if str(line) == division_string:
                if start_yaml:
                    end_yaml = True
                else:
                    start_yaml = True
            else:
                if start_yaml:
                    if not end_yaml:
                        variables += line
                    else:
                        if line == "\n":
                            content += "  {}".format(line)
                        else:
                            content += "  {}\n".format(line)

        variables += content
        return yaml.load(variables)

    def make_file(self):
        texenv = Environment(loader=TeXLoader(self.home))
        texenv.block_start_string = '((*'
        texenv.block_end_string = '*))'
        texenv.variable_start_string = '((('
        texenv.variable_end_string = ')))'
        texenv.comment_start_string = '((='
        texenv.comment_end_string = '=))'
        texenv.filters['escape_tex'] = escape_tex

        if "template" not in self.variables:
            self.variables["template"] = self.default_template

        template_file = self.template_dir + \
            '/' + self.variables["template"] + '.tex'

        if not exists(template_file):
            template_file = join(
                dirname(realpath(__file__)),
                self.variables["template"] + '.tex')

        template = texenv.get_template(template_file)

        f = open(self.temp_file, "w")
        f.write(template.render(self.variables))
        f.close()

        print("Temporary LaTeX file created ({})\n---".format(self.temp_file))

    def make_pdf(self, clean=True):
        print("Generating PDF\n---")

        command = 'pdflatex --jobname=' + self.job_name + ' ' + self.temp_file
        check_call(command, shell=True)

        if clean:
            for ext in ['aux', 'log', 'out', 'ent']:
                try:
                    remove(self.job_name + '.' + ext)
                except (OSError, IOError) as e:
                    # Use FileNotFoundError when python 2 is dropped
                    if e.errno != errno.ENOENT:
                        raise

    def render(self):
        self.make_file()
        self.make_pdf()

    def watch(self):
        print("---\nJekkish running in watch mode\n")
        print("---\nPerforming initial rendering\n---")

        last_time = False
        while True:
            if last_time != stat(self.target_file.name).st_mtime:
                last_time = stat(self.target_file.name).st_mtime
                self.render()
                print("---\nWatching {}\n---".format(self.target_file.name))

def main():
    parser = argparse.ArgumentParser(prog="Jekkish", description="A template-based pdftex CLI frontend inspired by Jekyll")
    parser.add_argument('filename', type=argparse.FileType('r'), default=sys.stdin, help='The file to process')
    parser.add_argument('jobname', nargs="?", default=False, help='Job name for pdftex output')
    # parser.add_argument('--watch', action='store_const', const=True, help='Watch <filename> for changes')
    parser.add_argument('--version', action='version', version='%(prog)s 0.1.8py3fix')
    args = parser.parse_args()

    new_file = Jekkish(args.filename, args.jobname)

    # if args.watch:
    #     new_file.watch()
    # else:
    new_file.render()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import cPickle

import numpy
from blocks.filter import VariableFilter
from fuel.datasets.hdf5 import H5PYDataset
from fuel.streams import DataStream
from fuel.schemes import SequentialScheme
from scipy.stats import pearsonr
from theano import function

from custom_blocks import PadAndAddMasks
from network import NetworkType, Network
from util import StateComputer, mark_seq_len, mark_word_boundaries

numpy.set_printoptions(precision=8, suppress=True)

# TODO get these into util.py, maybe in a prettier form
def mark_seq_len_batch(seq_batch, mask_batch):
    # get markers separately, then reshape
    padded_markers = numpy.array([numpy.arange(len(seq)) for seq in seq_batch])
    padded_markers = padded_markers.flatten(order="C")
    # throw away padding
    return padded_markers[mask_batch.flatten(order="C") == 1]

def mark_word_boundaries_batch(seq_batch, mask_batch):
    padded_markers = numpy.array([mark_word_boundaries([map_ind_2_chr[ind] for ind in seq]) for seq in seq_batch])
    padded_markers = padded_markers.flatten(order="C")
    return padded_markers[mask_batch.flatten(order="C") == 1]

def mark_letter(seq_batch, mask_batch, letter):
    padded_markers = 1*numpy.array([[map_ind_2_chr[char] == letter for char in seq] for seq in seq_batch])
    padded_markers = padded_markers.flatten(order="C")
    return padded_markers[mask_batch.flatten(order="C") == 1]

lstm_net = Network(NetworkType.LSTM)
lstm_net.set_parameters('seqgen_lstm_512_512_512.pkl')
map_chr_2_ind = cPickle.load(open("char_to_ind.pkl"))
map_ind_2_chr = cPickle.load(open("ind_to_char.pkl"))

# having a look at connectioneros from the cellsinas to the outputsos
params = lstm_net.cost_model.get_parameter_values()
for param in params:
    print param

# this section deals with prediction probabilities
"""
readouts = VariableFilter(theano_name="readout_readout_output_0")(lstm_net.cost_model.variables)[0]
char_probs = lstm_net.generator.readout.emitter.probs(readouts)

prob_function = function([lstm_net.x, lstm_net.mask], char_probs)

lord_original = "3:15 And the LORD came."
lord = [map_chr_2_ind[char] for char in lord_original]
print lord
zaza = prob_function([lord], numpy.ones((1, len(lord)), dtype="int8"))[:, 0, :]
print zaza
print zaza.shape
raw_input()
for (ey, row) in enumerate(zaza):
    print "PREDICTION PROBABILITIES FOR POSITION", ey, "LETTER", lord_original[ey]
    for (ind, prob) in enumerate(row):
        print repr(map_ind_2_chr[ind]), ":", prob
    print "\n"
"""

# define a function that gets the overall "sum of scores" at a given time step
readouts = VariableFilter(theano_name="readout_readout_output_0")(lstm_net.cost_model.variables)[0]
score_function = function([lstm_net.x, lstm_net.mask], readouts.sum(axis=2))

# this section of the playground has some fun rides that revolve around various correlation stuff. uncomment to access
# =)
sc = StateComputer(lstm_net.cost_model, map_chr_2_ind)
# storage for the correlations at the very end
correlation_dict = dict()
for name in sc.state_var_names:
    correlation_dict[name] = numpy.zeros(512, dtype=float)  # TODO NOT VERY GENERAL!!!

# get validation data to run over
valid_data = H5PYDataset("bible.hdf5", which_sets=("valid",), load_in_memory=True)
data_stream = PadAndAddMasks(
    DataStream.default_stream(dataset=valid_data, iteration_scheme=SequentialScheme(valid_data.num_examples,
                                                                                batch_size=128)),
    produces_examples=False)
iterator = data_stream.get_epoch_iterator()

# storage for the "supersequences" concatenated over all sequences
state_super_dict = dict()
for name in sc.state_var_names:
    state_super_dict[name] = numpy.empty(shape=(0, 512))  # TODO NOT VERY GENERAL AGAIN
super_marker = numpy.empty(shape=(0,))

# storage for the connections from states to output (softmax)
# this later allows easier connection between each layer's states and the corresponding output connection
connection_dict = dict()
standard_name = "/sequencegenerator/readout/merge/transform_states"
for name in sc.state_var_names:
    if name[-1] == "2":
        name_here = standard_name + "#2.W"
    elif name[-1] == "1":
        name_here = standard_name + "#1.W"
    else:
        name_here = standard_name + ".W"
    connection_dict[name] = params[name_here][:, map_chr_2_ind["O"]]

# if this is true, each state will be aligned with the character (or event derived from it) that it is used to *predict*
# if false, each state will be aligned with the character that was most recently read
prediction_alignment = False
try:
    while iterator:
        seq_batch, mask_batch = next(iterator)
        if not prediction_alignment:
            # "remove" last element of each sequence by modifying mask
            mask_batch[numpy.arange(mask_batch.shape[0]), mask_batch.sum(axis=1) - 1] = 0
        state_batch_dict = sc.read_sequence_batch(seq_batch, mask_batch)
        # reshape mask: we only need to do this once per batch, not again for each state_type
        # mask is in shape batch_size x seq_len, so NOT transposed, so it is flattened in C order
        mask_reshaped = mask_batch.flatten(order="C")
        # get marker (very preliminary...)
        seq_len_correlator = mark_letter(seq_batch, mask_batch, "L")
        super_marker = numpy.append(super_marker, seq_len_correlator)
        # TESTING total score thingy -- should be 2D, seq_len x batch_size
        overall_scores = score_function(seq_batch, mask_batch)
        for state_type in state_batch_dict:
            state_batch = state_batch_dict[state_type]
            if not prediction_alignment:
                # "throw away" initial state by rolling array backwards -- hacky, but sidesteps problems with needing
                # different masks for the sequences (the modified one further above) and for states (the "regular" one)
                state_batch = numpy.roll(state_batch, shift=-1, axis=0)
            state_batch *= connection_dict[state_type][None, None, :]
            state_batch *= overall_scores[:, :, None]
            # note: order of reshape is Fortran because states are "transposed" into seq_len x batch_size x dim
            state_reshaped = state_batch.reshape((state_batch.shape[0]*state_batch.shape[1], state_batch.shape[2]),
                                                 order="F")
            # throw away padding
            state_reshaped = state_reshaped[mask_reshaped == 1, :]
            # vstack will be very slow; here's hoping the relatively small number of operations will make it bearable...
            state_super_dict[state_type] = numpy.vstack((state_super_dict[state_type], state_reshaped))
        print "MADE IT THROUGH BATCH"
except StopIteration:
    pass
# do correlations between super long sequences...
for state_name in correlation_dict:
    for dim in xrange(correlation_dict[state_name].shape[0]):
        correlation_dict[state_name][dim] = pearsonr(state_super_dict[state_name][:, dim], super_marker)[0]
    print state_name
    print correlation_dict[state_name]
    print "LARGEST:", max(correlation_dict[state_name]), min(correlation_dict[state_name])
    print "\n\n"
<commit_msg>division instead of multiplication, whoops<commit_after>import cPickle

import numpy
from blocks.filter import VariableFilter
from fuel.datasets.hdf5 import H5PYDataset
from fuel.streams import DataStream
from fuel.schemes import SequentialScheme
from scipy.stats import pearsonr
from theano import function

from custom_blocks import PadAndAddMasks
from network import NetworkType, Network
from util import StateComputer, mark_seq_len, mark_word_boundaries

numpy.set_printoptions(precision=8, suppress=True)

# TODO get these into util.py, maybe in a prettier form
def mark_seq_len_batch(seq_batch, mask_batch):
    # get markers separately, then reshape
    padded_markers = numpy.array([numpy.arange(len(seq)) for seq in seq_batch])
    padded_markers = padded_markers.flatten(order="C")
    # throw away padding
    return padded_markers[mask_batch.flatten(order="C") == 1]

def mark_word_boundaries_batch(seq_batch, mask_batch):
    padded_markers = numpy.array([mark_word_boundaries([map_ind_2_chr[ind] for ind in seq]) for seq in seq_batch])
    padded_markers = padded_markers.flatten(order="C")
    return padded_markers[mask_batch.flatten(order="C") == 1]

def mark_letter(seq_batch, mask_batch, letter):
    padded_markers = 1*numpy.array([[map_ind_2_chr[char] == letter for char in seq] for seq in seq_batch])
    padded_markers = padded_markers.flatten(order="C")
    return padded_markers[mask_batch.flatten(order="C") == 1]

lstm_net = Network(NetworkType.LSTM)
lstm_net.set_parameters('seqgen_lstm_512_512_512.pkl')
map_chr_2_ind = cPickle.load(open("char_to_ind.pkl"))
map_ind_2_chr = cPickle.load(open("ind_to_char.pkl"))

# having a look at connectioneros from the cellsinas to the outputsos
params = lstm_net.cost_model.get_parameter_values()
for param in params:
    print param

# this section deals with prediction probabilities
"""
readouts = VariableFilter(theano_name="readout_readout_output_0")(lstm_net.cost_model.variables)[0]
char_probs = lstm_net.generator.readout.emitter.probs(readouts)

prob_function = function([lstm_net.x, lstm_net.mask], char_probs)

lord_original = "3:15 And the LORD came."
lord = [map_chr_2_ind[char] for char in lord_original]
print lord
zaza = prob_function([lord], numpy.ones((1, len(lord)), dtype="int8"))[:, 0, :]
print zaza
print zaza.shape
raw_input()
for (ey, row) in enumerate(zaza):
    print "PREDICTION PROBABILITIES FOR POSITION", ey, "LETTER", lord_original[ey]
    for (ind, prob) in enumerate(row):
        print repr(map_ind_2_chr[ind]), ":", prob
    print "\n"
"""

# define a function that gets the overall "sum of scores" at a given time step
readouts = VariableFilter(theano_name="readout_readout_output_0")(lstm_net.cost_model.variables)[0]
score_function = function([lstm_net.x, lstm_net.mask], readouts.sum(axis=2))

# this section of the playground has some fun rides that revolve around various correlation stuff. uncomment to access
# =)
sc = StateComputer(lstm_net.cost_model, map_chr_2_ind)
# storage for the correlations at the very end
correlation_dict = dict()
for name in sc.state_var_names:
    correlation_dict[name] = numpy.zeros(512, dtype=float)  # TODO NOT VERY GENERAL!!!

# get validation data to run over
valid_data = H5PYDataset("bible.hdf5", which_sets=("valid",), load_in_memory=True)
data_stream = PadAndAddMasks(
    DataStream.default_stream(dataset=valid_data, iteration_scheme=SequentialScheme(valid_data.num_examples,
                                                                                batch_size=128)),
    produces_examples=False)
iterator = data_stream.get_epoch_iterator()

# storage for the "supersequences" concatenated over all sequences
state_super_dict = dict()
for name in sc.state_var_names:
    state_super_dict[name] = numpy.empty(shape=(0, 512))  # TODO NOT VERY GENERAL AGAIN
super_marker = numpy.empty(shape=(0,))

# storage for the connections from states to output (softmax)
# this later allows easier connection between each layer's states and the corresponding output connection
connection_dict = dict()
standard_name = "/sequencegenerator/readout/merge/transform_states"
for name in sc.state_var_names:
    if name[-1] == "2":
        name_here = standard_name + "#2.W"
    elif name[-1] == "1":
        name_here = standard_name + "#1.W"
    else:
        name_here = standard_name + ".W"
    connection_dict[name] = params[name_here][:, map_chr_2_ind["O"]]

# if this is true, each state will be aligned with the character (or event derived from it) that it is used to *predict*
# if false, each state will be aligned with the character that was most recently read
prediction_alignment = False
try:
    while iterator:
        seq_batch, mask_batch = next(iterator)
        if not prediction_alignment:
            # "remove" last element of each sequence by modifying mask
            mask_batch[numpy.arange(mask_batch.shape[0]), mask_batch.sum(axis=1) - 1] = 0
        state_batch_dict = sc.read_sequence_batch(seq_batch, mask_batch)
        # reshape mask: we only need to do this once per batch, not again for each state_type
        # mask is in shape batch_size x seq_len, so NOT transposed, so it is flattened in C order
        mask_reshaped = mask_batch.flatten(order="C")
        # get marker (very preliminary...)
        seq_len_correlator = mark_letter(seq_batch, mask_batch, "L")
        super_marker = numpy.append(super_marker, seq_len_correlator)
        # TESTING total score thingy -- should be 2D, seq_len x batch_size
        overall_scores = score_function(seq_batch, mask_batch)
        for state_type in state_batch_dict:
            state_batch = state_batch_dict[state_type]
            if not prediction_alignment:
                # "throw away" initial state by rolling array backwards -- hacky, but sidesteps problems with needing
                # different masks for the sequences (the modified one further above) and for states (the "regular" one)
                state_batch = numpy.roll(state_batch, shift=-1, axis=0)
            state_batch *= connection_dict[state_type][None, None, :]
            state_batch /= overall_scores[:, :, None]
            # note: order of reshape is Fortran because states are "transposed" into seq_len x batch_size x dim
            state_reshaped = state_batch.reshape((state_batch.shape[0]*state_batch.shape[1], state_batch.shape[2]),
                                                 order="F")
            # throw away padding
            state_reshaped = state_reshaped[mask_reshaped == 1, :]
            # vstack will be very slow; here's hoping the relatively small number of operations will make it bearable...
            state_super_dict[state_type] = numpy.vstack((state_super_dict[state_type], state_reshaped))
        print "MADE IT THROUGH BATCH"
except StopIteration:
    pass
# do correlations between super long sequences...
for state_name in correlation_dict:
    for dim in xrange(correlation_dict[state_name].shape[0]):
        correlation_dict[state_name][dim] = pearsonr(state_super_dict[state_name][:, dim], super_marker)[0]
    print state_name
    print correlation_dict[state_name]
    print "LARGEST:", max(correlation_dict[state_name]), min(correlation_dict[state_name])
    print "\n\n"
<|endoftext|>


<commit_before>import itertools
from checks import *
from pymc import *
from numpy import array, inf
import numpy

from scipy import integrate
from knownfailure import *

class Domain(object):
    def __init__(self, vals, dtype = None, edges=None, shape = None):
        avals = array(vals)

        if edges is None:
            edges = array(vals[0]), array(vals[-1])
            vals = vals[1:-1]
        if shape is None:
            shape = avals[0].shape

        self.vals = vals
        self.shape = shape

        self.lower, self.upper = edges
        self.dtype = avals.dtype

    def __neg__(self):
        return Domain([-v for v in self.vals], self.dtype, (-self.lower, -self.upper), self.shape)

def product(domains):
    return itertools.product(*[d.vals for d in domains])

R = Domain([-inf, -2.1, -1, -.01, .0, .01, 1, 2.1, inf])
Rplus = Domain([0, .01, .1, .9, .99, 1, 1.5, 2, 100, inf])
Rplusbig = Domain([0, .5, .9, .99, 1, 1.5, 2, 20, inf])
Unit = Domain([0, .001, .1, .5, .75, .99, 1])

Runif = Domain([-1, -.4, 0, .4, 1])
Rdunif = Domain([-10, 0, 10.])
Rplusunif = Domain([0, .5, inf])
Rplusdunif = Domain([2, 10, 100], 'int64')

I = Domain([-1000, -3, -2, -1, 0, 1, 2, 3, 1000], 'int64')

NatSmall = Domain([0, 3, 4, 5, 1000], 'int64')
Nat = Domain([0, 1, 2, 3, 2000], 'int64')
NatBig = Domain([0, 1, 2, 3, 5000, 50000], 'int64')

Bool = Domain([0, 0, 1, 1], 'int64')

Vec2small = Domain([
    [.1, 0.0],
    [-2.3, .1],
    [-2.3, 1.5],
    ],
    edges = ([ -25, -25], [25, 25]))

Vec3small = Domain([
    [.1, 0.0, 0],
    [-2.3, .1,1],
    [-2.3, 1.5, 2],
    ],
    edges = ([ -12, -12, -12], [12, 12,12]))

PdMatrix2 = Domain([
    np.eye(2),
    [[.1, .05],
     [.05, 4.5]]
    ],
    edges = (None,None))

PdMatrix3 = Domain([
    np.eye(3),
    [[.1, .05,0],
     [.05, 1, 0],
     [0, 0, 2.5]]
    ],
    edges = (None,None))

def test_unif():
    checkd(Uniform, Runif, {'lower': -Rplusunif, 'upper': Rplusunif})

def test_discrete_unif():
    checkd(DiscreteUniform, Rdunif,
           {'lower': -Rplusdunif, 'upper': Rplusdunif})

def test_flat():
    checkd(Flat, Runif, {}, checks = [check_dlogp])

def test_normal():
    checkd(Normal, R, {'mu': R, 'tau': Rplus})

def test_beta():
    # TODO this fails with `Rplus`
    checkd(Beta, Unit, {'alpha': Rplusbig, 'beta': Rplusbig})

def test_exponential():
    checkd(Exponential, Rplus, {'lam': Rplus})

def test_geometric():
    checkd(Geometric, NatBig, {'p': Unit})

def test_negative_binomial():
    checkd(NegativeBinomial, Nat, {'mu': Rplusbig, 'alpha': Rplusbig})

def test_laplace():
    checkd(Laplace, R, {'mu': R, 'b': Rplus})

def test_lognormal():
    checkd(Lognormal, Rplus, {'mu': R, 'tau': Rplus}, False)

def test_t():
    checkd(T, R, {'nu': Rplus, 'mu': R, 'lam': Rplus})

def test_cauchy():
    checkd(Cauchy, R, {'alpha': R, 'beta': Rplusbig})

def test_gamma():
    checkd(Gamma, Rplus, {'alpha': Rplusbig, 'beta': Rplusbig})

def test_tpos():
    checkd(Tpos, Rplus, {'nu': Rplus, 'mu': R, 'lam': Rplus}, checks = [check_dlogp])

def test_binomial():
    checkd(Binomial, Nat, {'n': NatSmall, 'p': Unit})

def test_betabin():
    checkd(BetaBin, Nat, {'alpha': Rplus, 'beta': Rplus, 'n': NatSmall})

def test_bernoulli():
    checkd(Bernoulli, Bool, {'p': Unit})

def test_poisson():
    checkd(Poisson, Nat, {'mu': Rplus})

def test_constantdist():
    checkd(ConstantDist, I, {'c': I})

def test_zeroinflatedpoisson():
    checkd(ZeroInflatedPoisson, Nat, {'theta': Rplus, 'z': Bool})

def test_mvnormal():
    checkd(MvNormal, Vec2small, {'mu': R, 'tau': PdMatrix2}, checks = [check_dlogp, check_int_to_1])
    checkd(MvNormal, Vec3small, {'mu': R, 'tau': PdMatrix3}, checks = [check_dlogp, check_int_to_1])

def test_densitydist():
    def logp(x):
        return -log(2 * .5) - abs(x - .5) / .5

    checkd(DensityDist, R, {}, extra_args={'logp': logp})

def test_addpotential():
    with Model() as model:
        x = Normal('x', 1, 1)
        model.AddPotential(-x ** 2)

        check_dlogp(model, x, [R])

def test_wishart_initialization():
    with Model() as model:
        x = Wishart('wishart_test', n=3, p=2, V=numpy.eye(2), shape = [2,2])

def test_bound():
    with Model() as model:
        PositiveNormal = Bound(Normal, -.2)
        x = PositiveNormal('x', 1, 1)

        check_dlogp(model, x, [Rplus - .2])

def check_int_to_1(model, value, domain, paramdomains):
    pdf = compilef(exp(model.logp))
    names = map(str, model.vars)

    for a in product(paramdomains):
        a = a + (value.tag.test_value,)
        pt = Point(zip(names, a), model=model)

        bij = DictToVarBijection(value, (), pt)

        pdfx = bij.mapf(pdf)

        area = integrate_nd(pdfx, domain, value.dshape, value.dtype)

        assert_almost_equal(area, 1, err_msg=str(pt))

def integrate_nd(f, domain, shape, dtype):

    if shape == () or shape == (1,):
        if dtype in continuous_types:
            return integrate.quad(f, domain.lower, domain.upper, epsabs=1e-8)[0]
        else:
            return np.sum(map(f, np.arange(domain.lower, domain.upper + 1)))
    elif shape == (2,):
        def f2(a,b):
            return f([a,b])

        return integrate.dblquad(f2,
                domain.lower[0], domain.upper[0],
                lambda a: domain.lower[1], lambda a: domain.upper[1])[0]
    elif shape == (3,):
        def f3(a,b,c):
            return f([a,b,c])

        return integrate.tplquad(f3,
                domain.lower[0], domain.upper[0],
                lambda a: domain.lower[1], lambda a: domain.upper[1],
                lambda a, b: domain.lower[2], lambda a,b: domain.upper[2])[0]
    else:
        raise ValueError("Dont know how to integrate shape: " + str(shape))

def check_dlogp(model, value, domain, paramdomains):
    try:
        from numdifftools import Gradient
    except ImportError:
        return

    domains = paramdomains + [domain]
    bij = DictToArrayBijection(
        ArrayOrdering(model.cont_vars), model.test_point)

    if not model.cont_vars:
        return

    dlogp = bij.mapf(model.dlogpc())
    logp = bij.mapf(model.logpc)

    ndlogp = Gradient(logp)
    names = map(str, model.vars)

    for a in product(domains):
        pt = Point(zip(names, a), model=model)

        pt = bij.map(pt)

        assert_almost_equal(dlogp(pt), ndlogp(pt),
                            decimal=6, err_msg=str(pt))

def build_model(distfam, valuedomain, vardomains, extra_args={}):
    with Model() as m:
        vars = dict((v, Flat(
            v, dtype=dom.dtype, shape=dom.shape, testval=dom.vals[0])) for v, dom in vardomains.iteritems())
        vars.update(extra_args)

        value = distfam(
            'value', shape=valuedomain.shape, **vars)
    return m

def checkd(distfam, valuedomain, vardomains,
           checks = [check_int_to_1, check_dlogp], extra_args={}):

        m = build_model(distfam, valuedomain, vardomains, extra_args={})

        domains = [vardomains[str(v)] for v in m.vars[:-1]]

        for check in checks:
            check(m, m.named_vars['value'], valuedomain, domains)

<commit_msg>fixed tests<commit_after>import itertools
from checks import *
from pymc import *
from numpy import array, inf
import numpy

from scipy import integrate
from knownfailure import *

class Domain(object):
    def __init__(self, vals, dtype = None, edges=None, shape = None):
        avals = array(vals)

        if edges is None:
            edges = array(vals[0]), array(vals[-1])
            vals = vals[1:-1]
        if shape is None:
            shape = avals[0].shape

        self.vals = vals
        self.shape = shape

        self.lower, self.upper = edges
        self.dtype = avals.dtype

    def __neg__(self):
        return Domain([-v for v in self.vals], self.dtype, (-self.lower, -self.upper), self.shape)

def product(domains):
    return itertools.product(*[d.vals for d in domains])

R = Domain([-inf, -2.1, -1, -.01, .0, .01, 1, 2.1, inf])
Rplus = Domain([0, .01, .1, .9, .99, 1, 1.5, 2, 100, inf])
Rplusbig = Domain([0, .5, .9, .99, 1, 1.5, 2, 20, inf])
Unit = Domain([0, .001, .1, .5, .75, .99, 1])

Runif = Domain([-1, -.4, 0, .4, 1])
Rdunif = Domain([-10, 0, 10.])
Rplusunif = Domain([0, .5, inf])
Rplusdunif = Domain([2, 10, 100], 'int64')

I = Domain([-1000, -3, -2, -1, 0, 1, 2, 3, 1000], 'int64')

NatSmall = Domain([0, 3, 4, 5, 1000], 'int64')
Nat = Domain([0, 1, 2, 3, 2000], 'int64')
NatBig = Domain([0, 1, 2, 3, 5000, 50000], 'int64')

Bool = Domain([0, 0, 1, 1], 'int64')

Vec2small = Domain([
    [.1, 0.0],
    [-2.3, .1],
    [-2.3, 1.5],
    ],
    edges = ([ -25, -25], [25, 25]))

Vec3small = Domain([
    [.1, 0.0, 0],
    [-2.3, .1,1],
    [-2.3, 1.5, 2],
    ],
    edges = ([ -12, -12, -12], [12, 12,12]))

PdMatrix2 = Domain([
    np.eye(2),
    [[.5, .05],
     [.05, 4.5]]
    ],
    edges = (None,None))

PdMatrix3 = Domain([
    np.eye(3),
    [[.5, .1,0],
     [.1, 1, 0],
     [0, 0, 2.5]]
    ],
    edges = (None,None))

def test_unif():
    checkd(Uniform, Runif, {'lower': -Rplusunif, 'upper': Rplusunif})

def test_discrete_unif():
    checkd(DiscreteUniform, Rdunif,
           {'lower': -Rplusdunif, 'upper': Rplusdunif})

def test_flat():
    checkd(Flat, Runif, {}, checks = [check_dlogp])

def test_normal():
    checkd(Normal, R, {'mu': R, 'tau': Rplus})

def test_beta():
    # TODO this fails with `Rplus`
    checkd(Beta, Unit, {'alpha': Rplusbig, 'beta': Rplusbig})

def test_exponential():
    checkd(Exponential, Rplus, {'lam': Rplus})

def test_geometric():
    checkd(Geometric, NatBig, {'p': Unit})

def test_negative_binomial():
    checkd(NegativeBinomial, Nat, {'mu': Rplusbig, 'alpha': Rplusbig})

def test_laplace():
    checkd(Laplace, R, {'mu': R, 'b': Rplus})

def test_lognormal():
    checkd(Lognormal, Rplus, {'mu': R, 'tau': Rplus}, False)

def test_t():
    checkd(T, R, {'nu': Rplus, 'mu': R, 'lam': Rplus})

def test_cauchy():
    checkd(Cauchy, R, {'alpha': R, 'beta': Rplusbig})

def test_gamma():
    checkd(Gamma, Rplus, {'alpha': Rplusbig, 'beta': Rplusbig})

def test_tpos():
    checkd(Tpos, Rplus, {'nu': Rplus, 'mu': R, 'lam': Rplus}, checks = [check_dlogp])

def test_binomial():
    checkd(Binomial, Nat, {'n': NatSmall, 'p': Unit})

def test_betabin():
    checkd(BetaBin, Nat, {'alpha': Rplus, 'beta': Rplus, 'n': NatSmall})

def test_bernoulli():
    checkd(Bernoulli, Bool, {'p': Unit})

def test_poisson():
    checkd(Poisson, Nat, {'mu': Rplus})

def test_constantdist():
    checkd(ConstantDist, I, {'c': I})

def test_zeroinflatedpoisson():
    checkd(ZeroInflatedPoisson, Nat, {'theta': Rplus, 'z': Bool})

def test_mvnormal():
    checkd(MvNormal, Vec2small, {'mu': R, 'tau': PdMatrix2}, checks = [check_dlogp, check_int_to_1])
    checkd(MvNormal, Vec3small, {'mu': R, 'tau': PdMatrix3}, checks = [check_dlogp, check_int_to_1])

def test_densitydist():
    def logp(x):
        return -log(2 * .5) - abs(x - .5) / .5

    checkd(DensityDist, R, {}, extra_args={'logp': logp})

def test_addpotential():
    with Model() as model:
        x = Normal('x', 1, 1)
        model.AddPotential(-x ** 2)

        check_dlogp(model, x, R, [])

def test_wishart_initialization():
    with Model() as model:
        x = Wishart('wishart_test', n=3, p=2, V=numpy.eye(2), shape = [2,2])

def test_bound():
    with Model() as model:
        PositiveNormal = Bound(Normal, -.2)
        x = PositiveNormal('x', 1, 1)

        Rplus2 = Domain([-.2, -.19,-.1, 0, .5, 1, inf])

        check_dlogp(model, x, Rplus2, [])

def check_int_to_1(model, value, domain, paramdomains):
    pdf = compilef(exp(model.logp))
    names = map(str, model.vars)

    for a in product(paramdomains):
        a = a + (value.tag.test_value,)
        pt = Point(zip(names, a), model=model)

        bij = DictToVarBijection(value, (), pt)

        pdfx = bij.mapf(pdf)

        area = integrate_nd(pdfx, domain, value.dshape, value.dtype)

        assert_almost_equal(area, 1, err_msg=str(pt))

def integrate_nd(f, domain, shape, dtype):

    if shape == () or shape == (1,):
        if dtype in continuous_types:
            return integrate.quad(f, domain.lower, domain.upper, epsabs=1e-8)[0]
        else:
            return np.sum(map(f, np.arange(domain.lower, domain.upper + 1)))
    elif shape == (2,):
        def f2(a,b):
            return f([a,b])

        return integrate.dblquad(f2,
                domain.lower[0], domain.upper[0],
                lambda a: domain.lower[1], lambda a: domain.upper[1])[0]
    elif shape == (3,):
        def f3(a,b,c):
            return f([a,b,c])

        return integrate.tplquad(f3,
                domain.lower[0], domain.upper[0],
                lambda a: domain.lower[1], lambda a: domain.upper[1],
                lambda a, b: domain.lower[2], lambda a,b: domain.upper[2])[0]
    else:
        raise ValueError("Dont know how to integrate shape: " + str(shape))

def check_dlogp(model, value, domain, paramdomains):
    try:
        from numdifftools import Gradient
    except ImportError:
        return

    domains = paramdomains + [domain]
    bij = DictToArrayBijection(
        ArrayOrdering(model.cont_vars), model.test_point)

    if not model.cont_vars:
        return

    dlogp = bij.mapf(model.dlogpc())
    logp = bij.mapf(model.logpc)

    ndlogp = Gradient(logp)
    names = map(str, model.vars)

    for a in product(domains):
        pt = Point(zip(names, a), model=model)

        pt = bij.map(pt)

        assert_almost_equal(dlogp(pt), ndlogp(pt),
                            decimal=6, err_msg=str(pt))

def build_model(distfam, valuedomain, vardomains, extra_args={}):
    with Model() as m:
        vars = dict((v, Flat(
            v, dtype=dom.dtype, shape=dom.shape, testval=dom.vals[0])) for v, dom in vardomains.iteritems())
        vars.update(extra_args)

        value = distfam(
            'value', shape=valuedomain.shape, **vars)
    return m

def checkd(distfam, valuedomain, vardomains,
           checks = [check_int_to_1, check_dlogp], extra_args={}):

        m = build_model(distfam, valuedomain, vardomains, extra_args=extra_args)

        domains = [vardomains[str(v)] for v in m.vars[:-1]]

        for check in checks:
            check(m, m.named_vars['value'], valuedomain, domains)

<|endoftext|>


<commit_before>
#IMPORT YOUR LIBRARIES
try :
   from tkinter import *
except :
   from Tkinter import *
from random import randint
from time import sleep, time
import uuid
from math import sqrt

#CONSTANTS
HEIGHT = 400
WIDTH = 800
r = 5
MIN_DISTANCE = 5
MAX_DISTANCE = 60
#CONTROL VARIABLES
#starting ratio of predator and prey
start_ratio = 0.5
#the probabilities of killing and reproduction on collision
kill_chance = 85
repro_chance_prey = 70
repro_chance_predator = 45
#time
starvation_predator = 5000
predator_lifespan = 7000
prey_lifespan = 4500
#number of prey and predator
num_predator = 3
num_prey = int(num_predator/start_ratio)
speed = {
   'prey': 3,
   'predator': 3
}

color = {
   'prey':'blue',
   'predator':'red'
}

spawn = {
   'prey': 1,
   'predator': 2
}

#DEPENDENT VARIABLE
#the difference between the starting ratio and ending ratio

#INDEPENDENT VARIABLE
#the ratio of predator and prey

#VARIABLES
animal_list = list()
animal_motion = {
   'prey': {
      'change': {
         'max': 20,
         'min': 2
      }
   },
   'predator': {
      'change': {
         'max': 23,
         'min': 2
      }
   }
}

window = Tk()
c = Canvas(window, width=WIDTH, height=HEIGHT, bg='darkgreen')
step = 0

#FUNCTIONS
def create_animal(animal_type):
    x = randint(155, WIDTH)
    y = randint(35, HEIGHT)
    animal = c.create_oval(x - r, y - r, x +r, y + r, fill= color[animal_type])
    animal_list.append({
       'type': animal_type,
       'oval': animal,
       'name': str(uuid.uuid4()),
       'x': randint(-speed[animal_type],speed[animal_type]),
       'y': randint(-speed[animal_type],speed[animal_type]),
       'direction_count':randint(animal_motion[animal_type]['change']['min'],animal_motion[animal_type]['change']['max'])
    })
    return animal

def walk(animal_motion,step,animal_list):
   for i in range(len(animal_list)):
      animal = animal_list[i]
      animal_type = animal['type']
      if step % animal['direction_count'] == 0:
         animal['x'] = randint(-speed[animal_type],speed[animal_type])
         animal['y'] = randint(-speed[animal_type],speed[animal_type])
         animal['direction_count'] = randint(animal_motion[animal_type]['change']['min'],animal_motion[animal_type]['change']['max'])
      c.move(animal['oval'], animal['x'], animal['y'])
      handle_boundary(animal['oval'])

def difference(start_ratio, num_predator, num_prey):
   return start_ratio - num_predator/num_prey

def del_prey(animal):
   try:
      c.delete(animal['oval'])
      animal_list.remove(animal)
      del animal
   except:
      pass

def handle_boundary(animal):
   x,y = get_coords(animal)
   if x <= 0:
      c.move(animal,WIDTH, 0)
   elif x >= WIDTH:
      c.move(animal,-WIDTH, 0)
   if y <= 0:
      c.move(animal,0, HEIGHT)
   elif y >= WIDTH:
      c.move(animal,0, -HEIGHT)

def get_coords(id_num):
    pos = c.coords(id_num)
    x =  (pos[0] + pos[2])/2
    y =  (pos[1] + pos[3])/2
    return x, y

def distance(id1, id2):
    x1, y1 = get_coords(id1)
    x2, y2 = get_coords(id2)
    return sqrt((x2 - x1)**2 + (y2 - y1)**2)

def collision():
    for animal_i in animal_list:
       for animal_j in animal_list:
          if animal_i['name'] == animal_j['name']:
             continue
          try:
                if distance(animal_i['oval'], animal_j['oval']) < 2*r:
                   if animal_i['type']== 'predator' and animal_j['type']== 'prey':
                      del_prey(animal_j)
                   elif animal_i['type']== 'prey' and animal_j['type']== 'prey' and randint(1,100) <= spawn['prey']:
                      create_animal('prey')
                   if animal_i['type']== 'predator' and animal_j['type']== 'predator' and randint(1,100) <= spawn['predator']:
                      create_animal('predator')
          except:
             pass

    return

#ENTRY POINT
window.title('Predator Prey Simulation')
c.pack()
c.create_text(50, 30, text='# of Predators', fill='red')
c.create_text(150, 30, text='# of Prey', fill='blue')
predator_text = c.create_text(50, 50, fill='red' )
prey_text = c.create_text(150, 50, fill='blue' )

for i in range(num_prey):
    create_animal('prey')

for i in range(num_predator):
    create_animal('predator')

while True:
    walk(animal_motion, step, animal_list)
    window.update()
    sleep(0.01)
    step += 1
    collision()

<commit_msg>Added lifespan<commit_after>"""
TO DO:
   1)Lifespans
   2)Ratio count
   3)Equilibrium
   4)Humans
"""

#IMPORT YOUR LIBRARIES
try :
   from tkinter import *
except :
   from Tkinter import *
from random import randint
from time import sleep, time
import uuid
from math import sqrt

#CONSTANTS
HEIGHT = 400
WIDTH = 800
r = 8
MIN_DISTANCE = 5
MAX_DISTANCE = 60
#CONTROL VARIABLES
#starting ratio of predator and prey
start_ratio = 0.5
#the probabilities of killing and reproduction on collision
kill_chance = 85
repro_chance_prey = 70
repro_chance_predator = 45
#time
starvation_predator = 5000
predator_lifespan = 6200
prey_lifespan = 5600
#number of prey and predator
num_predator = 5
num_prey = int(num_predator/start_ratio - 1)
speed = {
   'prey': 3,
   'predator': 3
}

color = {
   'prey':'blue',
   'predator':'red'
}

spawn = {
   'prey': 7,
   'predator': 6
}

lifespan = {
   'prey': 900,
   'predator': 900
}

chance_of_death = {
   'prey': 1,
   'predator': 1
}

#DEPENDENT VARIABLE
#the difference between the starting ratio and ending ratio

#INDEPENDENT VARIABLE
#the ratio of predator and prey

#VARIABLES
animal_list = list()
animal_motion = {
   'prey': {
      'change': {
         'max': 20,
         'min': 2
      }
   },
   'predator': {
      'change': {
         'max': 23,
         'min': 2
      }
   }
}

window = Tk()
c = Canvas(window, width=WIDTH, height=HEIGHT, bg='darkgreen')
step = 0

#FUNCTIONS
def create_animal(animal_type):
    x = randint(155, WIDTH)
    y = randint(35, HEIGHT)
    animal = c.create_oval(x - r, y - r, x +r, y + r, fill= color[animal_type])
    animal_list.append({
       'type': animal_type,
       'oval': animal,
       'name': str(uuid.uuid4()),
       'x': randint(-speed[animal_type],speed[animal_type]),
       'y': randint(-speed[animal_type],speed[animal_type]),
       'direction_count':randint(animal_motion[animal_type]['change']['min'],animal_motion[animal_type]['change']['max']),
       'age_count': 0,
       'hunger': 0
    })
    return animal

def walk(animal_motion,step,animal_list):
   animal_count = {
      'prey': 0,
      'predator': 0
   }
   for i in range(len(animal_list)):
      try:
         animal = animal_list[i]
         animal_type = animal['type']
         animal_count[animal_type] += 1
         animal['age_count'] += 1
         animal['hunger'] += 1
         if step % animal['direction_count'] == 0:
            animal['x'] = randint(-speed[animal_type],speed[animal_type])
            animal['y'] = randint(-speed[animal_type],speed[animal_type])
            animal['direction_count'] = randint(animal_motion[animal_type]['change']['min'],animal_motion[animal_type]['change']['max'])
         c.move(animal['oval'], animal['x'], animal['y'])
         handle_boundary(animal['oval'])
         handle_death(animal)
      except:
            pass
   print(','.join([str(step)] + [str(v) for v in animal_count.values()]))

def difference(start_ratio, num_predator, num_prey):
   return start_ratio - num_predator/num_prey

def del_animal(animal):
   try:
      c.delete(animal['oval'])
      animal_list.remove(animal)
      del animal
   except:
      pass

def handle_boundary(animal):
   x,y = get_coords(animal)
   buffer = 10
   if x <= buffer:
      c.move(animal,WIDTH - buffer, 0)
   elif x >= WIDTH - buffer:
      c.move(animal,buffer - WIDTH, 0)
   if y <= buffer:
      c.move(animal,0, HEIGHT - buffer)
   elif y >= HEIGHT - buffer:
      c.move(animal,0, buffer - HEIGHT)

def handle_death(animal):
   if lifespan[animal['type']] < animal['age_count'] and randint(1,1000) <= chance_of_death[animal['type']]:
      del_animal(animal)

def get_coords(id_num):
    pos = c.coords(id_num)
    x =  (pos[0] + pos[2])/2
    y =  (pos[1] + pos[3])/2
    return x, y

def distance(id1, id2):
    x1, y1 = get_coords(id1)
    x2, y2 = get_coords(id2)
    return sqrt((x2 - x1)**2 + (y2 - y1)**2)

def collision():
    for animal_i in animal_list:
       for animal_j in animal_list:
          if animal_i['name'] == animal_j['name']:
             continue
          try:
                if distance(animal_i['oval'], animal_j['oval']) < 2*r:
                   if animal_i['type']== 'predator' and animal_j['type']== 'prey':
                      del_animal(animal_j)
                   elif animal_i['type']== 'prey' and animal_j['type']== 'prey' and randint(1,100) <= spawn['prey']:
                      create_animal('prey')
                   if animal_i['type']== 'predator' and animal_j['type']== 'predator' and randint(1,100) <= spawn['predator']:
                      create_animal('predator')
          except:
             pass

    return

#ENTRY POINT
window.title('Predator Prey Simulation')
c.pack()

for i in range(num_prey):
    create_animal('prey')

for i in range(num_predator):
    create_animal('predator')

while True:
    walk(animal_motion, step, animal_list)
    window.update()
    sleep(0.01)
    step += 1
    collision()

<|endoftext|>


<commit_before># writer.py Implements the Writer class.
# V0.22 Peter Hinch 3rd Jan 2018: Supports new SSD1306 driver.

# The MIT License (MIT)
#
# Copyright (c) 2016 Peter Hinch
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

# A Writer supports rendering text to a Display instance in a given font.
# Multiple Writer instances may be created, each rendering a font to the
# same Display object.

import framebuf

class Writer(object):
    text_row = 0        # attributes common to all Writer instances
    text_col = 0
    row_clip = False    # Clip or scroll when screen full
    col_clip = False    # Clip or new line when row is full

    @classmethod
    def set_textpos(cls, row, col):
        cls.text_row = row
        cls.text_col = col

    @classmethod
    def set_clip(cls, row_clip, col_clip):
        cls.row_clip = row_clip
        cls.col_clip = col_clip

    def __init__(self, device, font, verbose=True):
        super().__init__()
        self.device = device
        self.font = font
        # Allow to work with any font mapping
        if font.hmap():
            self.map = framebuf.MONO_HMSB if font.reverse() else framebuf.MONO_HLSB
        else:
            raise ValueError('Font must be horizontally mapped.')
        if verbose:
            print('Orientation: {} Reversal: {}'.format('horiz' if font.hmap() else 'vert', font.reverse()))
        self.screenwidth = device.width  # In pixels
        self.screenheight = device.height

    def _newline(self):
        height = self.font.height()
        Writer.text_row += height
        Writer.text_col = 0
        margin = self.screenheight - (Writer.text_row + height)
        if margin < 0:
            if not Writer.row_clip:
                self.device.scroll(0, margin)
                Writer.text_row += margin

    def printstring(self, string):
        for char in string:
            self._printchar(char)

    # Method using blitting. Efficient rendering for monochrome displays.
    # Tested on SSD1306.
    def _printchar(self, char):
        if char == '\n':
            self._newline()
            return
        glyph, char_height, char_width = self.font.get_ch(char)
        if Writer.text_row + char_height > self.screenheight:
            if Writer.row_clip:
                return
            self._newline()
        if Writer.text_col + char_width > self.screenwidth:
            if Writer.col_clip:
                return
            else:
                self._newline()
        buf = bytearray(glyph)
        fbc = framebuf.FrameBuffer(buf, char_width, char_height, self.map)
        self.device.blit(fbc, Writer.text_col, Writer.text_row)
        Writer.text_col += char_width

    # Bitwise rendering. Currently this is required for colour displays
    # because the framebuf blit method does not have an effective means of
    # colour mapping single bit framebufs onto n-bit ones
    def _printchar_bitwise(self, char):
        if char == '\n':
            self._newline()
            return
        glyph, char_height, char_width = self.font.get_ch(char)
        if Writer.text_row + char_height > self.screenheight:
            if Writer.row_clip:
                return
            self._newline()
        if Writer.text_col + char_width > self.screenwidth:
            if Writer.col_clip:
                return
            else:
                self._newline()

        div, mod = divmod(char_height, 8)
        gbytes = div + 1 if mod else div    # No. of bytes per column of glyph
        device = self.device
        for scol in range(char_width):      # Source column
            dcol = scol + Writer.text_col   # Destination column
            drow = Writer.text_row          # Destination row
            for srow in range(char_height): # Source row
                gbyte, gbit = divmod(srow, 8)
                if drow >= self.screenheight:
                    break
                if gbit == 0:               # Next glyph byte
                    data = glyph[scol * gbytes + gbyte]
                device.pixel(dcol, drow, data & (1 << gbit))
                drow += 1
        Writer.text_col += char_width

<commit_msg>SSD1306/writer now has invert option.<commit_after># writer.py Implements the Writer class.
# V0.22 Peter Hinch 3rd Jan 2018: Supports new SSD1306 driver.

# The MIT License (MIT)
#
# Copyright (c) 2016 Peter Hinch
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

# A Writer supports rendering text to a Display instance in a given font.
# Multiple Writer instances may be created, each rendering a font to the
# same Display object.

import framebuf

class Writer(object):
    text_row = 0        # attributes common to all Writer instances
    text_col = 0
    row_clip = False    # Clip or scroll when screen full
    col_clip = False    # Clip or new line when row is full

    @classmethod
    def set_textpos(cls, row, col):
        cls.text_row = row
        cls.text_col = col

    @classmethod
    def set_clip(cls, row_clip, col_clip):
        cls.row_clip = row_clip
        cls.col_clip = col_clip

    def __init__(self, device, font, verbose=True):
        super().__init__()
        self.device = device
        self.font = font
        # Allow to work with any font mapping
        if font.hmap():
            self.map = framebuf.MONO_HMSB if font.reverse() else framebuf.MONO_HLSB
        else:
            raise ValueError('Font must be horizontally mapped.')
        if verbose:
            print('Orientation: {} Reversal: {}'.format('horiz' if font.hmap() else 'vert', font.reverse()))
        self.screenwidth = device.width  # In pixels
        self.screenheight = device.height

    def _newline(self):
        height = self.font.height()
        Writer.text_row += height
        Writer.text_col = 0
        margin = self.screenheight - (Writer.text_row + height)
        if margin < 0:
            if not Writer.row_clip:
                self.device.scroll(0, margin)
                Writer.text_row += margin

    def printstring(self, string):
        for char in string:
            self._printchar(char)

    # Method using blitting. Efficient rendering for monochrome displays.
    # Tested on SSD1306. Invert is for black-on-white rendering.
    def _printchar(self, char, invert=False):
        if char == '\n':
            self._newline()
            return
        glyph, char_height, char_width = self.font.get_ch(char)
        if Writer.text_row + char_height > self.screenheight:
            if Writer.row_clip:
                return
            self._newline()
        if Writer.text_col + char_width > self.screenwidth:
            if Writer.col_clip:
                return
            else:
                self._newline()
        buf = bytearray(glyph)
        if invert:
            for i, v in enumerate(buf):
                buf[i] = 0xFF & ~ v
        fbc = framebuf.FrameBuffer(buf, char_width, char_height, self.map)
        self.device.blit(fbc, Writer.text_col, Writer.text_row)
        Writer.text_col += char_width

    # Bitwise rendering. Currently this is required for colour displays
    # because the framebuf blit method does not have an effective means of
    # colour mapping single bit framebufs onto n-bit ones
    def _printchar_bitwise(self, char):
        if char == '\n':
            self._newline()
            return
        glyph, char_height, char_width = self.font.get_ch(char)
        if Writer.text_row + char_height > self.screenheight:
            if Writer.row_clip:
                return
            self._newline()
        if Writer.text_col + char_width > self.screenwidth:
            if Writer.col_clip:
                return
            else:
                self._newline()

        div, mod = divmod(char_height, 8)
        gbytes = div + 1 if mod else div    # No. of bytes per column of glyph
        device = self.device
        for scol in range(char_width):      # Source column
            dcol = scol + Writer.text_col   # Destination column
            drow = Writer.text_row          # Destination row
            for srow in range(char_height): # Source row
                gbyte, gbit = divmod(srow, 8)
                if drow >= self.screenheight:
                    break
                if gbit == 0:               # Next glyph byte
                    data = glyph[scol * gbytes + gbyte]
                device.pixel(dcol, drow, data & (1 << gbit))
                drow += 1
        Writer.text_col += char_width

<|endoftext|>


<commit_before>#ebtel_dem_analysis_main.py

#Will Barnes
#15 May 2015

#Import needed modules
import sys
import os
import argparse
import numpy as np
sys.path.append('/home/wtb2/Documents/EBTEL_analysis/src/')
import ebtel_dem as ebd
import ebtel_plot_em as ebpe
import ebtel_plot as ebp

#Declare parser object
parser = argparse.ArgumentParser(description='Script that performs DEM analysis for EBTEL-2fluid runs.')
#Add arguments to parser
parser.add_argument("-s","--species",help="Species to which the heating was applied for particular run.")
parser.add_argument("--root_dir",help="Root directory from which EBTEL output will be read.")
parser.add_argument("--root_dir_figs",help="Root directory to which figures will be output.")
#Declare the parser dictionary
args = parser.parse_args()

#set root directory
if args.root_dir:
    root_dir = args.root_dir
else:
    root_dir = '/data/datadrive2/EBTEL-2fluid_runs/'

if args.root_dir_figs:
    root_dir_figs = args.root_dir_figs
else:
    root_dir_figs = '/data/datadrive1/EBTEL-2fluid_figs/'

#set figure file format
figdir = '%s_heating_runs/alpha%s/'
figname = 'ebtel_L%.1f_tpulse%.1f_alpha%s_%s_heating'

#make parameter vectors
loop_length = np.array([20.0,40.0,60.0,80.0,100.0,120.0])
alpha = ['uniform',1.5,2.0,2.5]
#make waiting time vector
Tn = np.arange(250,5250,250)

#set limits on hot/cool slope calculation
slope_limits = {'cool_lower':6.0,'cool_upper':6.6,'hot_lower':6.8,'hot_upper':7.25}
t_cool_static = np.linspace(slope_limits['cool_lower'],slope_limits['cool_upper'],1000)
t_hot_static = np.linspace(slope_limits['hot_lower'],slope_limits['hot_upper'],1000)

#function to calculate variable hot slopes
def calc_upper_hot_lim(mean_temp,mean_em,delta_lim):
    ninf_i = np.where(np.isinf(mean_em) == False) #find non-inf indices
    max_i = np.argmax(mean_em) #find index corresponding to max value
    hot_i = ninf_i[0][np.where(ninf_i[0]>max_i)] #indices for hot branch
    em_hot = mean_em[hot_i] #hot branch em
    delta_em_hot = np.fabs(np.diff(em_hot)) #delta(em) of hot branch
    delta_i = np.where(delta_em_hot>0.5)[0][0]
    lim_i = hot_i[delta_i - 2]
    t_upper = mean_temp[lim_i]
    t_lower = t_upper - delta_lim
    return t_lower,t_upper

#set static parameters
tpulse = 100.0
solver = 'rka4'
if args.species == 'ion':
    delta_lim = 0.2
else:
    delta_lim = 0.4

#declare instance of Plotter class
surf_plot = ebp.Plotter()

#iterate over variable parameters
for i in range(len(alpha)):
    temp_max_save = []
    em_max_save = []
    for j in  range(len(loop_length)):
        #print status
        print("Processing L = %.1f, alpha = %s"%(loop_length[j],str(alpha[i])))
        #get data
        if loop_length[j] == 20.0 or alpha[i] == 'uniform':
            solver = 'euler'
        else:
            solver = 'rka4'

        dema = ebd.DEMAnalyzer(root_dir,args.species,alpha[i],loop_length[j],tpulse,solver,Tn=Tn,slope_limits=slope_limits)
        dema.process_raw()
        dema.em_statistics()
        #variable limit configuration and temperature fit array
        hot_lower,hot_upper = [],[]
        t_cool,t_hot = [],[]
        for i in range(len(dema.temp_mean)):
            l,u = calc_upper_hot_lim(dema.temp_mean[i],dema.em_mean[i],delta_lim)
            hot_lower.append(l),hot_upper.append(u)
            t_cool.append(t_cool_static)
            t_hot.append(np.linspace(l,u,len(t_cool_static)))

        dema.many_slopes(slope_limits={'cool_lower':6.0,'cool_upper':6.6,'hot_lower':hot_lower,'hot_upper':hot_upper})
        dema.slope_statistics()
        dema.find_em_max()
        temp_max_save.append([np.mean(tmax) for tmax in dema.temp_max])
        em_max_save.append([np.mean(emmax) for emmax in dema.em_max])
        #build figure names and make sure directories exist
        if not os.path.exists(root_dir_figs + figdir%(args.species,str(alpha[i]))):
            os.makedirs(figdir%(root_dir_figs + args.species,str(alpha[i])))

        figname_temp = figdir%(args.species,str(alpha[i]))+figname%(loop_length[j],tpulse,str(alpha[i]),args.species)
        #plot data
        demp = ebpe.DEMPlotter(dema.temp_em,dema.em,alpha[i],Tn=Tn,format='eps',fs=22.0,figsize=(12,12),dpi=1000)
        demp.plot_em_max(dema.temp_max,dema.em_max,print_fig_filename=root_dir_figs + figname_temp + '_TmaxVTn')
        demp.plot_em_slopes(dema.a_cool_mean,dema.a_cool_std,dema.a_hot_mean,dema.a_hot_std,print_fig_filename=root_dir_figs + figname_temp + '_hs_compare')
        fit_lines = {'t_cool':t_cool,'a_cool':dema.a_cool_mean,'b_cool':dema.b_cool_mean,'t_hot':t_hot,'a_hot':dema.a_hot_mean,'b_hot':dema.b_hot_mean}
        demp.plot_em_curves(dema.temp_mean,dema.em_mean,fit_lines=fit_lines,print_fig_filename=root_dir_figs+figname_temp+'_dem')
        #plot all em curves for given tn
        if alpha[i] is not 'uniform':
            if not os.path.exists(root_dir_figs+figname_temp+'_dem_mc/'):
                os.makedirs(root_dir_figs+figname_temp+'_dem_mc/')

            for k in range(len(Tn)):
                demp.plot_em_curve(k,print_fig_filename=root_dir_figs + figname_temp + '_dem_mc/' + figname%(loop_length[j],tpulse,str(alpha[i]),args.species) + '_'+str(k) + '_dem')

    #build surface plot
    surf_plot.plot_surface(Tn,loop_length,temp_max_save,vmin=6.0,vmax=6.8,ylab=r'$L$ (Mm)',xlab=r'$T_n$ (s)',plot_title=r'$T_{max}$ Surface, $\alpha=$' + str(alpha[i]),print_fig_filename=root_dir_figs + figdir%(args.species,str(alpha[i])) + 't_max_surface_' + args.species + '_alpha' + str(alpha[i]) + '_tpulse' + str(tpulse) + '_' + solver)
    surf_plot.plot_surface(Tn,loop_length,em_max_save,vmin=26.0,vmax=30.0,ylab=r'$L$ (Mm)',xlab=r'$T_n$ (s)',plot_title=r'EM$_{max}$ Surface, $\alpha=$' + str(alpha[i]),print_fig_filename=root_dir_figs + figdir%(args.species,str(alpha[i])) + 'em_max_surface_' + args.species + '_alpha' + str(alpha[i]) + '_tpulse' + str(tpulse) + '_' + solver)
<commit_msg>debugging em stats in main script<commit_after>#ebtel_dem_analysis_main.py

#Will Barnes
#15 May 2015

#Import needed modules
import sys
import os
import argparse
import numpy as np
sys.path.append('/home/wtb2/Documents/EBTEL_analysis/src/')
import ebtel_dem as ebd
import ebtel_plot_em as ebpe
import ebtel_plot as ebp

#Declare parser object
parser = argparse.ArgumentParser(description='Script that performs DEM analysis for EBTEL-2fluid runs.')
#Add arguments to parser
parser.add_argument("-s","--species",help="Species to which the heating was applied for particular run.")
parser.add_argument("--root_dir",help="Root directory from which EBTEL output will be read.")
parser.add_argument("--root_dir_figs",help="Root directory to which figures will be output.")
#Declare the parser dictionary
args = parser.parse_args()

#set root directory
if args.root_dir:
    root_dir = args.root_dir
else:
    root_dir = '/data/datadrive2/EBTEL-2fluid_runs/'

if args.root_dir_figs:
    root_dir_figs = args.root_dir_figs
else:
    root_dir_figs = '/data/datadrive1/EBTEL-2fluid_figs/'

#set figure file format
figdir = '%s_heating_runs/alpha%s/'
figname = 'ebtel_L%.1f_tpulse%.1f_alpha%s_%s_heating'

#make parameter vectors
loop_length = np.array([20.0,40.0,60.0,80.0,100.0,120.0])
alpha = ['uniform',1.5,2.0,2.5]
#make waiting time vector
Tn = np.arange(250,5250,250)

#set limits on hot/cool slope calculation
slope_limits = {'cool_lower':6.0,'cool_upper':6.6,'hot_lower':6.8,'hot_upper':7.25}
t_cool_static = np.linspace(slope_limits['cool_lower'],slope_limits['cool_upper'],1000)
t_hot_static = np.linspace(slope_limits['hot_lower'],slope_limits['hot_upper'],1000)

#function to calculate variable hot slopes
def calc_upper_hot_lim(mean_temp,mean_em,delta_lim):
    ninf_i = np.where(np.isinf(mean_em) == False) #find non-inf indices
    max_i = np.argmax(mean_em) #find index corresponding to max value
    hot_i = ninf_i[0][np.where(ninf_i[0]>max_i)] #indices for hot branch
    em_hot = mean_em[hot_i] #hot branch em
    delta_em_hot = np.fabs(np.diff(em_hot)) #delta(em) of hot branch
    delta_i = np.where(delta_em_hot>0.5)[0][0]
    lim_i = hot_i[delta_i - 2]
    t_upper = mean_temp[lim_i]
    t_lower = t_upper - delta_lim
    return t_lower,t_upper

#set static parameters
tpulse = 100.0
solver = 'rka4'
if args.species == 'ion':
    delta_lim = 0.2
else:
    delta_lim = 0.4

#declare instance of Plotter class
surf_plot = ebp.Plotter()

#iterate over variable parameters
for i in range(len(alpha)):
    temp_max_save = []
    em_max_save = []
    for j in  range(len(loop_length)):
        #print status
        print("Processing L = %.1f, alpha = %s"%(loop_length[j],str(alpha[i])))
        #get data
        if loop_length[j] == 20.0 or alpha[i] == 'uniform':
            solver = 'euler'
        else:
            solver = 'rka4'

        dema = ebd.DEMAnalyzer(root_dir,args.species,alpha[i],loop_length[j],tpulse,solver,Tn=Tn,slope_limits=slope_limits)
        dema.process_raw()
        dema.em_statistics()
        #DEBUG--print shape of mean arrays
        print(np.shape(np.array(dema.em_mean)))
        print(np.shape(np.array(dema.temp_mean)))
        #variable limit configuration and temperature fit array
        hot_lower,hot_upper = [],[]
        t_cool,t_hot = [],[]
        for i in range(len(dema.temp_mean)):
            l,u = calc_upper_hot_lim(dema.temp_mean[i],dema.em_mean[i],delta_lim)
            hot_lower.append(l),hot_upper.append(u)
            t_cool.append(t_cool_static)
            t_hot.append(np.linspace(l,u,len(t_cool_static)))

        dema.many_slopes(slope_limits={'cool_lower':6.0,'cool_upper':6.6,'hot_lower':hot_lower,'hot_upper':hot_upper})
        dema.slope_statistics()
        dema.find_em_max()
        temp_max_save.append([np.mean(tmax) for tmax in dema.temp_max])
        em_max_save.append([np.mean(emmax) for emmax in dema.em_max])
        #build figure names and make sure directories exist
        if not os.path.exists(root_dir_figs + figdir%(args.species,str(alpha[i]))):
            os.makedirs(figdir%(root_dir_figs + args.species,str(alpha[i])))

        figname_temp = figdir%(args.species,str(alpha[i]))+figname%(loop_length[j],tpulse,str(alpha[i]),args.species)
        #plot data
        demp = ebpe.DEMPlotter(dema.temp_em,dema.em,alpha[i],Tn=Tn,format='eps',fs=22.0,figsize=(12,12),dpi=1000)
        demp.plot_em_max(dema.temp_max,dema.em_max,print_fig_filename=root_dir_figs + figname_temp + '_TmaxVTn')
        demp.plot_em_slopes(dema.a_cool_mean,dema.a_cool_std,dema.a_hot_mean,dema.a_hot_std,print_fig_filename=root_dir_figs + figname_temp + '_hs_compare')
        fit_lines = {'t_cool':t_cool,'a_cool':dema.a_cool_mean,'b_cool':dema.b_cool_mean,'t_hot':t_hot,'a_hot':dema.a_hot_mean,'b_hot':dema.b_hot_mean}
        demp.plot_em_curves(dema.temp_mean,dema.em_mean,fit_lines=fit_lines,print_fig_filename=root_dir_figs+figname_temp+'_dem')
        #plot all em curves for given tn
        if alpha[i] is not 'uniform':
            if not os.path.exists(root_dir_figs+figname_temp+'_dem_mc/'):
                os.makedirs(root_dir_figs+figname_temp+'_dem_mc/')

            for k in range(len(Tn)):
                demp.plot_em_curve(k,print_fig_filename=root_dir_figs + figname_temp + '_dem_mc/' + figname%(loop_length[j],tpulse,str(alpha[i]),args.species) + '_'+str(k) + '_dem')

    #build surface plot
    surf_plot.plot_surface(Tn,loop_length,temp_max_save,vmin=6.0,vmax=6.8,ylab=r'$L$ (Mm)',xlab=r'$T_n$ (s)',plot_title=r'$T_{max}$ Surface, $\alpha=$' + str(alpha[i]),print_fig_filename=root_dir_figs + figdir%(args.species,str(alpha[i])) + 't_max_surface_' + args.species + '_alpha' + str(alpha[i]) + '_tpulse' + str(tpulse) + '_' + solver)
    surf_plot.plot_surface(Tn,loop_length,em_max_save,vmin=26.0,vmax=30.0,ylab=r'$L$ (Mm)',xlab=r'$T_n$ (s)',plot_title=r'EM$_{max}$ Surface, $\alpha=$' + str(alpha[i]),print_fig_filename=root_dir_figs + figdir%(args.species,str(alpha[i])) + 'em_max_surface_' + args.species + '_alpha' + str(alpha[i]) + '_tpulse' + str(tpulse) + '_' + solver)
<|endoftext|>