<commit_before># -*- coding: utf-8 -*-

from __future__ import division, print_function

__all__ = ["GP"]

try:
    from itertools import izip
except ImportError:
    izip = zip

import numpy as np
import scipy.optimize as op
from scipy.linalg import cholesky, cho_solve, LinAlgError

from .utils import multivariate_gaussian_samples, nd_sort_samples

# MAGIC: tiny epsilon to add on the diagonal of the matrices in the absence
# of observational uncertainties. Needed for computational stability.
TINY = 1.25e-12

class GP(object):
    """
    The basic Gaussian Process object.

    :param kernel:
        An instance of a subclass of :class:`kernels.Kernel`.

    :param mean: (optional)
        A description of the mean function; can be a callable or a scalar. If
        scalar, the mean is assumed constant. Otherwise, the function will be
        called with the array of independent coordinates as the only argument.
        (default: ``0.0``)

    """

    def __init__(self, kernel, mean=None):
        self.kernel = kernel
        self._computed = False
        self.mean = mean

    @property
    def mean(self):
        return self._mean

    @mean.setter
    def mean(self, mean):
        if mean is None:
            self._mean = _default_mean(0.)
        else:
            try:
                val = float(mean)
            except TypeError:
                self._mean = mean
            else:
                self._mean = _default_mean(val)

    @property
    def computed(self):
        """
        Has the processes been computed since the last update of the kernel?

        """
        return self._computed and not self.kernel.dirty

    @computed.setter
    def computed(self, v):
        self._computed = v
        if v:
            self.kernel.dirty = False

    def parse_samples(self, t, sort=False):
        """
        Parse a list of samples to make sure that it has the correct
        dimensions and optionally sort it. In one dimension, the samples will
        be sorted in the logical order. In higher dimensions, a kd-tree is
        built and the samples are sorted in increasing distance from the
        *first* sample.

        :param t: ``(nsamples,)`` or ``(nsamples, ndim)``
            The list of samples. If 1-D, this is assumed to be a list of
            one-dimensional samples otherwise, the size of the second
            dimension is assumed to be the dimension of the input space.

        :param sort:
            A boolean flag indicating whether or not the samples should be
            sorted.

        Returns a tuple ``(samples, inds)`` where

        * **samples** is an array with shape ``(nsamples, ndim)`` and if
          ``sort`` was ``True``, it will also be sorted, and
        * **inds** is an ``(nsamples,)`` list of integer permutations used to
          sort the list of samples.

        Raises a ``RuntimeError`` if the input dimension doesn't match the
        dimension of the kernel.

        """
        t = np.atleast_1d(t)
        if len(t.shape) == 1:
            # Deal with one-dimensional data.
            if sort:
                inds = np.argsort(t)
            else:
                inds = np.arange(len(t), dtype=int)
            t = np.atleast_2d(t).T
        elif sort:
            # Sort the data using a KD-tree.
            inds = nd_sort_samples(t)
        else:
            # Otherwise, assume that the samples are sorted.
            inds = np.arange(t.shape[0], dtype=int)

        # Double check the dimensions against the kernel.
        if len(t.shape) != 2 or t.shape[1] != self.kernel.ndim:
            raise ValueError("Dimension mismatch")

        return t[inds], inds

    def _check_dimensions(self, y):
        n, ndim = self._x.shape
        y = np.atleast_1d(y)
        if len(y.shape) > 1:
            raise ValueError("The predicted dimension must be 1-D")
        if len(y) != n:
            raise ValueError("Dimension mismatch")
        return y

    def compute(self, x, yerr=TINY, sort=True, **kwargs):
        """
        Pre-compute the covariance matrix and factorize it for a set of times
        and uncertainties.

        :param x: ``(nsamples,)`` or ``(nsamples, ndim)``
            The independent coordinates of the data points.

        :param yerr: (optional) ``(nsamples,)`` or scalar
            The Gaussian uncertainties on the data points at coordinates
            ``x``. These values will be added in quadrature to the diagonal of
            the covariance matrix.

        :param sort: (optional)
            Should the samples be sorted before computing the covariance
            matrix? This can lead to more numerically stable results and with
            some linear algebra libraries this can more computationally
            efficient. Either way, this flag is passed directly to
            :func:`parse_samples`. (default: ``True``)

        """
        # Parse the input coordinates.
        self._x, self.inds = self.parse_samples(x, sort)
        try:
            self._yerr = float(yerr) * np.ones(len(x))
        except TypeError:
            self._yerr = self._check_dimensions(yerr)[self.inds]
        self._do_compute(**kwargs)

    def _do_compute(self, _scale=0.5*np.log(2*np.pi)):
        # Compute the kernel matrix.
        K = self.kernel(self._x[:, None], self._x[None, :])
        K[np.diag_indices_from(K)] += self._yerr ** 2

        # Factor the matrix and compute the log-determinant.
        factor, _ = self._factor = (cholesky(K, overwrite_a=True,
                                             lower=False), False)
        self._const = -(np.sum(np.log(np.diag(factor))) + _scale*len(self._x))

        # Save the computed state.
        self.computed = True

    def recompute(self, quiet=False, **kwargs):
        """
        Re-compute a previously computed model. You might want to do this if
        the kernel parameters change and the kernel is labeled as ``dirty``.

        """
        if not self.computed:
            if not (hasattr(self, "_x") and hasattr(self, "_yerr")):
                raise RuntimeError("You need to compute the model first")
            try:
                # Update the model making sure that we store the original
                # ordering of the points.
                initial_order = np.array(self.inds)
                self.compute(self._x, self._yerr, sort=False, **kwargs)
                self.inds = initial_order
            except (ValueError, LinAlgError):
                if quiet:
                    return False
                raise
        return True

    def _compute_lnlike(self, r):
        return self._const - 0.5*np.dot(r.T, cho_solve(self._factor, r))

    def lnlikelihood(self, y, quiet=False):
        """
        Compute the ln-likelihood of a set of observations under the Gaussian
        process model. You must call ``compute`` before this function.

        :param y: ``(nsamples, )``
            The observations at the coordinates provided in the ``compute``
            step.

        :param quiet:
            If ``True`` return negative infinity instead of raising an
            exception when there is an invalid kernel or linear algebra
            failure. (default: ``False``)

        """
        if not self.recompute(quiet=quiet):
            return -np.inf
        r = self._check_dimensions(y)[self.inds] - self.mean(self._x)
        ll = self._compute_lnlike(r)
        return ll if np.isfinite(ll) else -np.inf

    def grad_lnlikelihood(self, y, dims=None, quiet=False):
        """
        Compute the gradient of the ln-likelihood function as a function of
        the kernel parameters.

        :param y: ``(nsamples,)``
            The list of observations at coordinates ``x`` provided to the
            :func:`compute` function.

        :param dims: (optional)
            If you only want to compute the gradient in some dimensions,
            list them here.

        :param quiet:
            If ``True`` return a gradient of zero instead of raising an
            exception when there is an invalid kernel or linear algebra
            failure. (default: ``False``)

        """
        # By default, compute the gradient in all dimensions.
        if dims is None:
            dims = np.ones(len(self.kernel), dtype=bool)

        # Make sure that the model is computed and try to recompute it if it's
        # dirty.
        if not self.recompute(quiet=quiet):
            return np.zeros_like(dims, dtype=float)

        # Parse the input sample list.
        r = self._check_dimensions(y)[self.inds] - self.mean(self._x)

        # Pre-compute some factors.
        alpha = cho_solve(self._factor, r)
        Kg = self.kernel.grad(self._x[:, None], self._x[None, :])[dims]

        # Loop over dimensions and compute the gradient in each one.
        g = np.empty(len(Kg))
        for i, k in enumerate(Kg):
            d = sum(map(lambda r: np.dot(alpha, r), alpha[:, None] * k))
            d -= np.sum(np.diag(cho_solve(self._factor, k)))
            g[i] = 0.5 * d

        return g

    def predict(self, y, t):
        """
        Compute the conditional predictive distribution of the model.

        :param y: ``(nsamples,)``
            The observations to condition the model on.

        :param t: ``(ntest,)`` or ``(ntest, ndim)``
            The coordinates where the predictive distribution should be
            computed.

        Returns a tuple ``(mu, cov)`` where

        * **mu** ``(ntest,)`` is the mean of the predictive distribution, and
        * **cov** ``(ntest, ntest)`` is the predictive covariance.

        """
        self.recompute()
        r = self._check_dimensions(y)[self.inds] - self.mean(self._x)
        xs, i = self.parse_samples(t, False)
        alpha = cho_solve(self._factor, r)

        # Compute the predictive mean.
        Kxs = self.kernel(self._x[None, :], xs[:, None])
        mu = np.dot(Kxs, alpha) + self.mean(xs)

        # Compute the predictive covariance.
        cov = self.kernel(xs[:, None], xs[None, :])
        cov -= np.dot(Kxs, cho_solve(self._factor, Kxs.T))

        return mu, cov

    def sample_conditional(self, y, t, size=1):
        """
        Draw samples from the predictive conditional distribution.

        :param y: ``(nsamples, )``
            The observations to condition the model on.

        :param t: ``(ntest, )`` or ``(ntest, ndim)``
            The coordinates where the predictive distribution should be
            computed.

        :param size: (optional)
            The number of samples to draw. (default: ``1``)

        Returns **samples** ``(N, ntest)``, a list of predictions at
        coordinates given by ``t``.

        """
        mu, cov = self.predict(y, t)
        return multivariate_gaussian_samples(cov, size, mean=mu)

    def sample(self, t=None, size=1):
        """
        Draw samples from the prior distribution.

        :param t: ``(ntest, )`` or ``(ntest, ndim)`` (optional)
            The coordinates where the model should be sampled. If no
            coordinates are given, the precomputed coordinates and
            factorization are used.

        :param size: (optional)
            The number of samples to draw. (default: ``1``)

        Returns **samples** ``(size, ntest)``, a list of predictions at
        coordinates given by ``t``. If ``size == 1``, the result is a single
        sample with shape ``(ntest,)``.

        """
        if t is None:
            self.recompute()
            n, _ = self._x.shape

            # Generate samples using the precomputed factorization.
            samples = np.dot(np.random.randn(size, n), self._factor[0])
            samples += self.mean(self._x)

            # Reorder the samples correctly.
            results = np.empty_like(samples)
            results[:, self.inds] = samples
            return results[0] if size == 1 else results

        x, _ = self.parse_samples(t, False)
        cov = self.get_matrix(x)
        return multivariate_gaussian_samples(cov, size, mean=self.mean(x))

    def get_matrix(self, t):
        """
        Get the covariance matrix at a given set of independent coordinates.

        :param t: ``(nsamples,)`` or ``(nsamples, ndim)``
            The list of samples.

        """
        r, _ = self.parse_samples(t, False)
        return self.kernel(r[:, None], r[None, :])

    def optimize(self, x, y, yerr=TINY, sort=True, dims=None, in_log=True,
                 verbose=True, **kwargs):
        """
        A simple and not terribly robust non-linear optimization algorithm for
        the kernel hyperpararmeters.

        :param x: ``(nsamples,)`` or ``(nsamples, ndim)``
            The independent coordinates of the data points.

        :param y: ``(nsamples, )``
            The observations at the coordinates ``x``.

        :param yerr: (optional) ``(nsamples,)`` or scalar
            The Gaussian uncertainties on the data points at coordinates
            ``x``. These values will be added in quadrature to the diagonal of
            the covariance matrix.

        :param sort: (optional)
            Should the samples be sorted before computing the covariance
            matrix?

        :param dims: (optional)
            If you only want to optimize over some parameters, list their
            indices here.

        :param in_log: (optional) ``(len(kernel),)``, ``(len(dims),)`` or bool
            If you want to fit the parameters in the log (this can be useful
            for parameters that shouldn't go negative) specify that here. This
            can be a single boolean---in which case it is assumed to apply to
            every dimension---or it can be an array of booleans, one for each
            dimension.

        :param verbose: (optional)
            Display the results of the call to :func:`scipy.optimize.minimize`?
            (default: ``True``)

        Returns ``(pars, results)`` where ``pars`` is the list of optimized
        parameters and ``results`` is the results object returned by
        :func:`scipy.optimize.minimize`.

        """
        self.compute(x, yerr, sort=sort)

        # By default, optimize all the hyperparameters.
        if dims is None:
            dims = np.ones(len(self.kernel), dtype=bool)
        dims = np.arange(len(self.kernel))[dims]

        # Deal with conversion functions.
        try:
            len(in_log)
        except TypeError:
            in_log = in_log * np.ones_like(dims, dtype=bool)
        else:
            if len(in_log) != len(dims):
                raise RuntimeError("Dimension list and log mask mismatch")

        # Build the conversion functions.
        conv = np.array([lambda x: x for i in range(len(dims))])
        iconv = np.array([lambda x: x for i in range(len(dims))])
        conv[in_log] = np.exp
        iconv[in_log] = np.log

        # Define the objective function and gradient.
        def nll(pars):
            for i, f, p in izip(dims, conv, pars):
                self.kernel[i] = f(p)

            ll = self.lnlikelihood(y, quiet=True)
            if not np.isfinite(ll):
                return 1e25  # The optimizers can't deal with infinities.
            return -ll

        def grad_nll(pars):
            for i, f, p in izip(dims, conv, pars):
                self.kernel[i] = f(p)
            return -self.grad_lnlikelihood(y, dims=dims, quiet=True)

        # Run the optimization.
        p0 = [f(p) for f, p in izip(iconv, self.kernel.pars[dims])]
        results = op.minimize(nll, p0, jac=grad_nll, **kwargs)

        if verbose:
            print(results.message)

        # Update the kernel.
        for i, f, p in izip(dims, conv, results.x):
            self.kernel[i] = f(p)

        return self.kernel.pars[dims], results

class _default_mean(object):

    def __init__(self, value):
        self.value = value

    def __call__(self, t):
        return self.value + np.zeros(len(t), dtype=float)

    def __len__(self):
        return 1

    @property
    def vector(self):
        return np.array([self.value])

    @vector.setter
    def vector(self, value):
        self.value = float(value)

    def lnprior(self):
        return 0.0
<commit_msg>updating optimize to use vector instead of pars<commit_after># -*- coding: utf-8 -*-

from __future__ import division, print_function

__all__ = ["GP"]

try:
    from itertools import izip
except ImportError:
    izip = zip

import numpy as np
import scipy.optimize as op
from scipy.linalg import cholesky, cho_solve, LinAlgError

from .utils import multivariate_gaussian_samples, nd_sort_samples

# MAGIC: tiny epsilon to add on the diagonal of the matrices in the absence
# of observational uncertainties. Needed for computational stability.
TINY = 1.25e-12

class GP(object):
    """
    The basic Gaussian Process object.

    :param kernel:
        An instance of a subclass of :class:`kernels.Kernel`.

    :param mean: (optional)
        A description of the mean function; can be a callable or a scalar. If
        scalar, the mean is assumed constant. Otherwise, the function will be
        called with the array of independent coordinates as the only argument.
        (default: ``0.0``)

    """

    def __init__(self, kernel, mean=None):
        self.kernel = kernel
        self._computed = False
        self.mean = mean

    @property
    def mean(self):
        return self._mean

    @mean.setter
    def mean(self, mean):
        if mean is None:
            self._mean = _default_mean(0.)
        else:
            try:
                val = float(mean)
            except TypeError:
                self._mean = mean
            else:
                self._mean = _default_mean(val)

    @property
    def computed(self):
        """
        Has the processes been computed since the last update of the kernel?

        """
        return self._computed and not self.kernel.dirty

    @computed.setter
    def computed(self, v):
        self._computed = v
        if v:
            self.kernel.dirty = False

    def parse_samples(self, t, sort=False):
        """
        Parse a list of samples to make sure that it has the correct
        dimensions and optionally sort it. In one dimension, the samples will
        be sorted in the logical order. In higher dimensions, a kd-tree is
        built and the samples are sorted in increasing distance from the
        *first* sample.

        :param t: ``(nsamples,)`` or ``(nsamples, ndim)``
            The list of samples. If 1-D, this is assumed to be a list of
            one-dimensional samples otherwise, the size of the second
            dimension is assumed to be the dimension of the input space.

        :param sort:
            A boolean flag indicating whether or not the samples should be
            sorted.

        Returns a tuple ``(samples, inds)`` where

        * **samples** is an array with shape ``(nsamples, ndim)`` and if
          ``sort`` was ``True``, it will also be sorted, and
        * **inds** is an ``(nsamples,)`` list of integer permutations used to
          sort the list of samples.

        Raises a ``RuntimeError`` if the input dimension doesn't match the
        dimension of the kernel.

        """
        t = np.atleast_1d(t)
        if len(t.shape) == 1:
            # Deal with one-dimensional data.
            if sort:
                inds = np.argsort(t)
            else:
                inds = np.arange(len(t), dtype=int)
            t = np.atleast_2d(t).T
        elif sort:
            # Sort the data using a KD-tree.
            inds = nd_sort_samples(t)
        else:
            # Otherwise, assume that the samples are sorted.
            inds = np.arange(t.shape[0], dtype=int)

        # Double check the dimensions against the kernel.
        if len(t.shape) != 2 or t.shape[1] != self.kernel.ndim:
            raise ValueError("Dimension mismatch")

        return t[inds], inds

    def _check_dimensions(self, y):
        n, ndim = self._x.shape
        y = np.atleast_1d(y)
        if len(y.shape) > 1:
            raise ValueError("The predicted dimension must be 1-D")
        if len(y) != n:
            raise ValueError("Dimension mismatch")
        return y

    def compute(self, x, yerr=TINY, sort=True, **kwargs):
        """
        Pre-compute the covariance matrix and factorize it for a set of times
        and uncertainties.

        :param x: ``(nsamples,)`` or ``(nsamples, ndim)``
            The independent coordinates of the data points.

        :param yerr: (optional) ``(nsamples,)`` or scalar
            The Gaussian uncertainties on the data points at coordinates
            ``x``. These values will be added in quadrature to the diagonal of
            the covariance matrix.

        :param sort: (optional)
            Should the samples be sorted before computing the covariance
            matrix? This can lead to more numerically stable results and with
            some linear algebra libraries this can more computationally
            efficient. Either way, this flag is passed directly to
            :func:`parse_samples`. (default: ``True``)

        """
        # Parse the input coordinates.
        self._x, self.inds = self.parse_samples(x, sort)
        try:
            self._yerr = float(yerr) * np.ones(len(x))
        except TypeError:
            self._yerr = self._check_dimensions(yerr)[self.inds]
        self._do_compute(**kwargs)

    def _do_compute(self, _scale=0.5*np.log(2*np.pi)):
        # Compute the kernel matrix.
        K = self.kernel(self._x[:, None], self._x[None, :])
        K[np.diag_indices_from(K)] += self._yerr ** 2

        # Factor the matrix and compute the log-determinant.
        factor, _ = self._factor = (cholesky(K, overwrite_a=True,
                                             lower=False), False)
        self._const = -(np.sum(np.log(np.diag(factor))) + _scale*len(self._x))

        # Save the computed state.
        self.computed = True

    def recompute(self, quiet=False, **kwargs):
        """
        Re-compute a previously computed model. You might want to do this if
        the kernel parameters change and the kernel is labeled as ``dirty``.

        """
        if not self.computed:
            if not (hasattr(self, "_x") and hasattr(self, "_yerr")):
                raise RuntimeError("You need to compute the model first")
            try:
                # Update the model making sure that we store the original
                # ordering of the points.
                initial_order = np.array(self.inds)
                self.compute(self._x, self._yerr, sort=False, **kwargs)
                self.inds = initial_order
            except (ValueError, LinAlgError):
                if quiet:
                    return False
                raise
        return True

    def _compute_lnlike(self, r):
        return self._const - 0.5*np.dot(r.T, cho_solve(self._factor, r))

    def lnlikelihood(self, y, quiet=False):
        """
        Compute the ln-likelihood of a set of observations under the Gaussian
        process model. You must call ``compute`` before this function.

        :param y: ``(nsamples, )``
            The observations at the coordinates provided in the ``compute``
            step.

        :param quiet:
            If ``True`` return negative infinity instead of raising an
            exception when there is an invalid kernel or linear algebra
            failure. (default: ``False``)

        """
        if not self.recompute(quiet=quiet):
            return -np.inf
        r = self._check_dimensions(y)[self.inds] - self.mean(self._x)
        ll = self._compute_lnlike(r)
        return ll if np.isfinite(ll) else -np.inf

    def grad_lnlikelihood(self, y, dims=None, quiet=False):
        """
        Compute the gradient of the ln-likelihood function as a function of
        the kernel parameters.

        :param y: ``(nsamples,)``
            The list of observations at coordinates ``x`` provided to the
            :func:`compute` function.

        :param dims: (optional)
            If you only want to compute the gradient in some dimensions,
            list them here.

        :param quiet:
            If ``True`` return a gradient of zero instead of raising an
            exception when there is an invalid kernel or linear algebra
            failure. (default: ``False``)

        """
        # By default, compute the gradient in all dimensions.
        if dims is None:
            dims = np.ones(len(self.kernel), dtype=bool)

        # Make sure that the model is computed and try to recompute it if it's
        # dirty.
        if not self.recompute(quiet=quiet):
            return np.zeros_like(dims, dtype=float)

        # Parse the input sample list.
        r = self._check_dimensions(y)[self.inds] - self.mean(self._x)

        # Pre-compute some factors.
        alpha = cho_solve(self._factor, r)
        Kg = self.kernel.grad(self._x[:, None], self._x[None, :])[dims]

        # Loop over dimensions and compute the gradient in each one.
        g = np.empty(len(Kg))
        for i, k in enumerate(Kg):
            d = sum(map(lambda r: np.dot(alpha, r), alpha[:, None] * k))
            d -= np.sum(np.diag(cho_solve(self._factor, k)))
            g[i] = 0.5 * d

        return g

    def predict(self, y, t):
        """
        Compute the conditional predictive distribution of the model.

        :param y: ``(nsamples,)``
            The observations to condition the model on.

        :param t: ``(ntest,)`` or ``(ntest, ndim)``
            The coordinates where the predictive distribution should be
            computed.

        Returns a tuple ``(mu, cov)`` where

        * **mu** ``(ntest,)`` is the mean of the predictive distribution, and
        * **cov** ``(ntest, ntest)`` is the predictive covariance.

        """
        self.recompute()
        r = self._check_dimensions(y)[self.inds] - self.mean(self._x)
        xs, i = self.parse_samples(t, False)
        alpha = cho_solve(self._factor, r)

        # Compute the predictive mean.
        Kxs = self.kernel(self._x[None, :], xs[:, None])
        mu = np.dot(Kxs, alpha) + self.mean(xs)

        # Compute the predictive covariance.
        cov = self.kernel(xs[:, None], xs[None, :])
        cov -= np.dot(Kxs, cho_solve(self._factor, Kxs.T))

        return mu, cov

    def sample_conditional(self, y, t, size=1):
        """
        Draw samples from the predictive conditional distribution.

        :param y: ``(nsamples, )``
            The observations to condition the model on.

        :param t: ``(ntest, )`` or ``(ntest, ndim)``
            The coordinates where the predictive distribution should be
            computed.

        :param size: (optional)
            The number of samples to draw. (default: ``1``)

        Returns **samples** ``(N, ntest)``, a list of predictions at
        coordinates given by ``t``.

        """
        mu, cov = self.predict(y, t)
        return multivariate_gaussian_samples(cov, size, mean=mu)

    def sample(self, t=None, size=1):
        """
        Draw samples from the prior distribution.

        :param t: ``(ntest, )`` or ``(ntest, ndim)`` (optional)
            The coordinates where the model should be sampled. If no
            coordinates are given, the precomputed coordinates and
            factorization are used.

        :param size: (optional)
            The number of samples to draw. (default: ``1``)

        Returns **samples** ``(size, ntest)``, a list of predictions at
        coordinates given by ``t``. If ``size == 1``, the result is a single
        sample with shape ``(ntest,)``.

        """
        if t is None:
            self.recompute()
            n, _ = self._x.shape

            # Generate samples using the precomputed factorization.
            samples = np.dot(np.random.randn(size, n), self._factor[0])
            samples += self.mean(self._x)

            # Reorder the samples correctly.
            results = np.empty_like(samples)
            results[:, self.inds] = samples
            return results[0] if size == 1 else results

        x, _ = self.parse_samples(t, False)
        cov = self.get_matrix(x)
        return multivariate_gaussian_samples(cov, size, mean=self.mean(x))

    def get_matrix(self, t):
        """
        Get the covariance matrix at a given set of independent coordinates.

        :param t: ``(nsamples,)`` or ``(nsamples, ndim)``
            The list of samples.

        """
        r, _ = self.parse_samples(t, False)
        return self.kernel(r[:, None], r[None, :])

    def optimize(self, x, y, yerr=TINY, sort=True, dims=None, verbose=True,
                 **kwargs):
        """
        A simple and not terribly robust non-linear optimization algorithm for
        the kernel hyperpararmeters.

        :param x: ``(nsamples,)`` or ``(nsamples, ndim)``
            The independent coordinates of the data points.

        :param y: ``(nsamples, )``
            The observations at the coordinates ``x``.

        :param yerr: (optional) ``(nsamples,)`` or scalar
            The Gaussian uncertainties on the data points at coordinates
            ``x``. These values will be added in quadrature to the diagonal of
            the covariance matrix.

        :param sort: (optional)
            Should the samples be sorted before computing the covariance
            matrix?

        :param dims: (optional)
            If you only want to optimize over some parameters, list their
            indices here.

        :param verbose: (optional)
            Display the results of the call to :func:`scipy.optimize.minimize`?
            (default: ``True``)

        Returns ``(pars, results)`` where ``pars`` is the list of optimized
        parameters and ``results`` is the results object returned by
        :func:`scipy.optimize.minimize`.

        """
        self.compute(x, yerr, sort=sort)

        # By default, optimize all the hyperparameters.
        if dims is None:
            dims = np.ones(len(self.kernel), dtype=bool)
        dims = np.arange(len(self.kernel))[dims]

        # Define the objective function and gradient.
        def nll(pars):
            self.kernel.vector[dims] = pars
            ll = self.lnlikelihood(y, quiet=True)
            if not np.isfinite(ll):
                return 1e25  # The optimizers can't deal with infinities.
            return -ll

        def grad_nll(pars):
            self.kernel.vector[dims] = pars
            return -self.grad_lnlikelihood(y, dims=dims, quiet=True)

        # Run the optimization.
        p0 = self.kernel.vector[dims]
        results = op.minimize(nll, p0, jac=grad_nll, **kwargs)

        if verbose:
            print(results.message)

        return self.kernel.vector[dims], results

class _default_mean(object):

    def __init__(self, value):
        self.value = value

    def __call__(self, t):
        return self.value + np.zeros(len(t), dtype=float)

    def __len__(self):
        return 1

    @property
    def vector(self):
        return np.array([self.value])

    @vector.setter
    def vector(self, value):
        self.value = float(value)

    def lnprior(self):
        return 0.0
<|endoftext|>


<commit_before>#
# Refer to LICENSE file and README file for licensing information.
#
"""
Uses daily bhavcopy to download historical data for all stocks.
One challenge is, bhavcopy has the symbol name for that day. In case of NSE,
the symbol names change and that's a problem, so we need a way to track
changes in symbol names as well.

All this data is stored in an SQLite database as we download, as some of the
updates (eg. update data corresponding to a symbol with a new name are easier
to handle in SQLite than dealing with files).
"""

import requests
import sys
from datetime import datetime as dt
from datetime import timedelta as td
if sys.version_info.major < 3:
    from StringIO import StringIO as bio
    from itertools import izip
else:
    from io import BytesIO as bio
    izip = zip

import random
import time

from zipfile import ZipFile

from nse_utils import get_name_change_tuples, scrip_ohlcvd_info
import utils

import sqlite3

# SQLite database for stocks data. There are following tables
# bhav_downloads_info - columns "date","success/failure"
# deliv_downloads_info - columns "date","success/failure"
# scrip_info columns - "name","date","open","high","low","close","vol", "deliv"
_DEF_SQLIITE_FNAME = '.nse_all_data.sqlite'

_bhav_dload_info_cr_stmt = '''CREATE TABLE IF NOT EXISTS nse_bhav_downloads_info
                            (date FLOAT, success boolean
                            check(success in (0,1)))'''
_deliv_dload_info_cr_stmt = '''CREATE TABLE IF NOT EXISTS nse_deliv_downloads_info
                            (date FLOAT, success boolean
                            check(success in (0,1)))'''
_scrip_info_cr_stmt = '''CREATE TABLE IF NOT EXISTS nse_hist_data
                    (name VARCHAR(64), date TEXT,
                        open FLOAT, high FLOAT, low FLOAT, close FLOAT,
                        volume INTEGER, delivered INTEGER)'''

_create_stmts = {   'bhav_dload_info' : _bhav_dload_info_cr_stmt,
                    'deliv_dload_info' : _deliv_dload_info_cr_stmt,
                    'scrip_info' : _scrip_info_cr_stmt
                }

_date_fmt = '%d-%m-%Y'

def get_bhavcopy(date='01-01-2002'):
    """Downloads a bhavcopy for a given date and returns a dictionary of rows
    where each row stands for a traded scrip. The scripname is key. If the
    bhavcopy for a given day is already downloaded, returns None. date is in
    DD-MM-YYYY format"""

    global _date_fmt
    if isinstance(date, str):
        d2 = dt.strptime(date, _date_fmt)
        strdate = date
    elif isinstance(date, dt):
        d2 = date
        strdate = dt.strftime(date, _date_fmt)
    else:
        return None
    yr = d2.strftime('%Y')
    mon = d2.strftime('%b').upper()
    mm = d2.strftime('%0m')
    dd = d2.strftime('%0d')

    print yr, mm, dd
    fdate = utils.get_ts_for_datestr(strdate, _date_fmt)
    if _bhavcopy_downloaded(fdate): ## already downloaded
        return None

    bhav_url = 'http://nseindia.com/content/historical/EQUITIES/' \
                '%(year)s/%(mon)s/cm%(dd)s%(mon)s%(year)sbhav.csv.zip' % \
                        ({'year':yr, 'mon':mon, 'dd':dd})

    deliv_url = 'http://nseindia.com/archives/equities/mto/' \
                'MTO_%(dd)s%(mm)s%(year)s.DAT' % \
                        ({'year':yr, 'mm':mm, 'dd':dd})

    x = requests.get(bhav_url)

    y = requests.get(deliv_url)
    stocks_dict = {}
    _update_dload_success(fdate, x.ok, y.ok)
    if x.ok and y.ok:
        z = ZipFile(bio(x.content))
        for name in z.namelist():
            csv_name = name
        delivery = bio(y.text)
        with z.open(csv_name) as bhav:
            i = 0
            for line in bhav:
                if i == 0:
                    i += 1
                    continue
                l = line.split(',')
                sym, o, h, l, c, v, d = l[0], l[2], l[3], l[4], \
                                                l[5], l[8], l[8]
                stocks_dict[sym] = [float(o), float(h), float(l), float(c),
                                    int(v), int(d)]
        i = 0
        for line in delivery:
            if not line.startswith('20'):
                i += 1
                continue
            l = line.split(',')
            if (len(l)) == 4:
                sym, d = l[1].strip(), l[3].strip()
            elif len(l) == 7:
                sym, d = l[2].strip(), l[5].strip()
            stocks_dict[sym][-1] = int(d)
            i += 1
        for sym in stocks_dict.keys():
            stocks_dict[sym] = scrip_ohlcvd_info(*stocks_dict[sym])
        return stocks_dict

def _update_dload_success(fdate, bhav_ok, deliv_ok, fname=None):
    """ Update whether bhavcopy download and delivery data download for given
    date is successful"""
    fname = fname or _DEF_SQLIITE_FNAME
    with sqlite3.connect(fname) as con:
        cursor = con.cursor()
        insert_stmt = 'INSERT INTO %(table)s VALUES (%(date)f, %(success)d);'
        insert_stmt_final = insert_stmt % {'table':'nse_bhav_downloads_info',
                                    'date' : fdate, 'success':int(bhav_ok)}
        result = cursor.execute(insert_stmt_final)
        print insert_stmt_final
        insert_stmt_final = insert_stmt % {'table':'nse_deliv_downloads_info',
                                    'date' : fdate, 'success':int(bhav_ok)}
        cursor.execute(insert_stmt_final)
        print insert_stmt_final
        con.commit()

def _update_bhavcopy(strdate, stocks_dict, fname=None):
    """update bhavcopy Database date in DD-MM-YYYY format."""
    fname = fname or _DEF_SQLIITE_FNAME
    with sqlite3.connect(fname) as con:
        cur = con.cursor()

        fdate = utils.get_ts_for_datestr(strdate, _date_fmt)
        # just to be safe
        delete_stmt = '''DELETE FROM nse_hist_data where date = %(date)f'''
        delete_stmt_final = delete_stmt % { 'date': fdate}
        cur.execute(delete_stmt_final)
        insert_stmt = '''INSERT INTO nse_hist_data VALUES("%(sym)s", %(date)f,
                        %(o)f, %(h)f, %(l)f, %(c)f, %(v)d, %(d)d);'''
        for key, val in stocks_dict.iteritems():
            insert_stmt_final = insert_stmt % { 'sym' : key, 'date': fdate,
                                                'o' : val.open, 'h': val.high,
                                                'l' : val.low, 'c': val.close,
                                                'v' : val.volume, 'd': val.deliv
                                                }
            print "Executing:", insert_stmt_final
            cur.execute(insert_stmt_final)

        con.commit()

def _bhavcopy_downloaded(fdate, fname=None):
    fname = fname or _DEF_SQLIITE_FNAME
    with sqlite3.connect(fname) as con:
        cur = con.cursor()
        select_stmt = '''select count(*) from nse_bhav_downloads_info where
                     date = %(date)f;''' % {'date':fdate}
        cur.execute(select_stmt)
    x = cur.fetchone()

    return x[0] > 0

def _create_tables(fname=None):
    fname = fname or _DEF_SQLIITE_FNAME
    with sqlite3.connect(fname) as cursor:
        for stmt in _create_stmts.values():
            cursor.execute(stmt)

def _apply_name_changes_to_db(syms, fname=None):
    """Changes security names in nse_hist_data table so the name of the security
    is always the latest."""
    fname = fname or _DEF_SQLIITE_FNAME
    update_stmt = '''update nse_hist_data set name = '{}' where name in ({})'''
    with sqlite3.connect(fname) as con:
        cur = con.cursor()
        for sym in syms:
            old = sym[:-1]
            new = sym[-1]
            olds = ','.join(["'%s'" % x for x in old])
            update_stmt_final = update_stmt.format(new, olds)
            cur.execute(update_stmt_final)
        con.commit()

if __name__ == '__main__':
    # We run the full program
    import argparse
    parser = argparse.ArgumentParser()

    # --full option
    parser.add_argument("--full",
                        help="download full data from 1 Jan 2002",
                        action="store_true")

    # --from option
    parser.add_argument("--from",
                        help="From Date in DD-MM-YYYY format. " \
                                "Default is 01-01-2002",
                        dest='fromdate',
                        default="01-01-2002")
    # --to option
    parser.add_argument("--to",
                        help="From Date in DD-MM-YYYY format. " \
                                "Default is Today.",
                        dest='todate',
                        default="today")
    args = parser.parse_args()
    try:
        _ = dt.strptime(args.fromdate, _date_fmt)
        if args.todate.lower() == 'today':
            args.todate = dt.now().strftime(_date_fmt)
        _ = dt.strptime(args.todate, _date_fmt)
    except ValueError:
        print parser.format_usage()
        sys.exit(-1)

    # We are now ready to download data
    from_date = dt.strptime(args.fromdate, _date_fmt)
    to_date = dt.strptime(args.todate, _date_fmt)
    if from_date > to_date:
        print parser.format_usage()
        sys.exit(-1)

    _create_tables()

    tdelta = to_date - from_date
    if tdelta.days > 50:
        "Downloading data for {0} days".format(tdelta.days)

    cur_date = from_date
    while cur_date <= to_date:
        scrips_dict = get_bhavcopy(cur_date)
        str_curdate = cur_date.strftime(_date_fmt)
        if scrips_dict is not None:
            _update_bhavcopy(str_curdate, scrips_dict)

        time.sleep(random.randrange(1,10))

        cur_date += td(1)

    # Apply the name changes to the DB
    sym_change_tuples = get_name_change_tuples()
    if len(sym_change_tuples) == 0:
        print "No name change tuples found..."
        sys.exit(-1)
    _apply_name_changes_to_db(sym_change_tuples)
<commit_msg> - moved around a lot of code and got rid of messy sql statements inside functions    - all sql functions are now global. Time we start looking at SQL Alchemy?<commit_after>#
# Refer to LICENSE file and README file for licensing information.
#
"""
Uses daily bhavcopy to download historical data for all stocks.
One challenge is, bhavcopy has the symbol name for that day. In case of NSE,
the symbol names change and that's a problem, so we need a way to track
changes in symbol names as well.

All this data is stored in an SQLite database as we download, as some of the
updates (eg. update data corresponding to a symbol with a new name are easier
to handle in SQLite than dealing with files).
"""

# BIG FIXME: There are sql statements littered all over the place, sqlalchemy?

import requests
import sys
from datetime import datetime as dt
from datetime import timedelta as td
if sys.version_info.major < 3:
    from StringIO import StringIO as bio
    from itertools import izip
else:
    from io import BytesIO as bio
    izip = zip

import random
import time

from zipfile import ZipFile

from nse_utils import get_name_change_tuples, scrip_ohlcvd_info
import utils

import sqlite3

_DEF_SQLIITE_FNAME = '.nse_all_data.sqlite'
# SQLite database for stocks data. There are following tables
# nse_bhav_downloads_info - columns "date","success/failure"
# nse_deliv_downloads_info - columns "date","success/failure"
# nse_historical_data columns - "name","date","open","high","low","close","vol", "deliv"

_bhav_dload_info_cr_stmt = '''CREATE TABLE IF NOT EXISTS nse_bhav_downloads_info
                            (date FLOAT,
                            success BOOLEAN check(success in (0,1)))'''
_deliv_dload_info_cr_stmt = '''CREATE TABLE IF NOT EXISTS nse_deliv_downloads_info
                            (date FLOAT,
                            success BOOLEAN check(success in (0,1)))'''
_scrip_info_cr_stmt = '''CREATE TABLE IF NOT EXISTS nse_hist_data
                    (name VARCHAR(64), date TEXT,
                    open FLOAT, high FLOAT, low FLOAT, close FLOAT,
                    volume INTEGER, delivered INTEGER)'''

_create_stmts = {   'bhav_dload_info' : _bhav_dload_info_cr_stmt,
                    'deliv_dload_info' : _deliv_dload_info_cr_stmt,
                    'scrip_info' : _scrip_info_cr_stmt
                }

_insert_dload_stmt = 'INSERT INTO %(table)s VALUES (%(date)f, %(success)d);'

_del_for_date_stmt = '''DELETE FROM nse_hist_data where date = %(date)f'''

_nsedata_insert_stmt = '''INSERT INTO nse_hist_data VALUES("%(sym)s", %(date)f,
                        %(o)f, %(h)f, %(l)f, %(c)f, %(v)d, %(d)d);'''

_update_name_changes_stmt = '''update nse_hist_data set name = '{}' where name in ({})'''

_date_fmt = '%d-%m-%Y'

_bhav_url_base = 'http://nseindia.com/content/historical/EQUITIES/' \
                '%(year)s/%(mon)s/cm%(dd)s%(mon)s%(year)sbhav.csv.zip'

_deliv_url_base = 'http://nseindia.com/archives/equities/mto/' \
                'MTO_%(dd)s%(mm)s%(year)s.DAT'

def get_bhavcopy(date='01-01-2002'):
    """Downloads a bhavcopy for a given date and returns a dictionary of rows
    where each row stands for a traded scrip. The scripname is key. If the
    bhavcopy for a given day is already downloaded, returns None. date is in
    DD-MM-YYYY format"""

    global _date_fmt
    if isinstance(date, str):
        d2 = dt.strptime(date, _date_fmt)
        strdate = date
    elif isinstance(date, dt):
        d2 = date
        strdate = dt.strftime(date, _date_fmt)
    else:
        return None
    yr = d2.strftime('%Y')
    mon = d2.strftime('%b').upper()
    mm = d2.strftime('%0m')
    dd = d2.strftime('%0d')

    print yr, mm, dd
    fdate = utils.get_ts_for_datestr(strdate, _date_fmt)
    if _bhavcopy_downloaded(fdate): ## already downloaded
        return None

    global _bhav_url_base
    global _deliv_url_base

    bhav_url = _bhav_url_base % ({'year':yr, 'mon':mon, 'dd':dd})
    deliv_url = _deliv_url_base % ({'year':yr, 'mm':mm, 'dd':dd})

    x = requests.get(bhav_url)

    y = requests.get(deliv_url)
    stocks_dict = {}
    _update_dload_success(fdate, x.ok, y.ok)
    if x.ok and y.ok:
        z = ZipFile(bio(x.content))
        for name in z.namelist():
            csv_name = name
        delivery = bio(y.text)
        with z.open(csv_name) as bhav:
            i = 0
            for line in bhav:
                if i == 0:
                    i += 1
                    continue
                l = line.split(',')
                sym, o, h, l, c, v, d = l[0], l[2], l[3], l[4], \
                                                l[5], l[8], l[8]
                stocks_dict[sym] = [float(o), float(h), float(l), float(c),
                                    int(v), int(d)]
        i = 0
        for line in delivery:
            if not line.startswith('20'):
                i += 1
                continue
            l = line.split(',')
            if (len(l)) == 4:
                sym, d = l[1].strip(), l[3].strip()
            elif len(l) == 7:
                sym, d = l[2].strip(), l[5].strip()
            stocks_dict[sym][-1] = int(d)
            i += 1
        for sym in stocks_dict.keys():
            stocks_dict[sym] = scrip_ohlcvd_info(*stocks_dict[sym])
        return stocks_dict

def _update_dload_success(fdate, bhav_ok, deliv_ok, fname=None):
    """ Update whether bhavcopy download and delivery data download for given
    date is successful"""
    fname = fname or _DEF_SQLIITE_FNAME
    with sqlite3.connect(fname) as con:
        cursor = con.cursor()
        global _insert_dload_stmt
        insert_stmt_final = _insert_dload_stmt % {'table':'nse_bhav_downloads_info',
                                    'date' : fdate, 'success':int(bhav_ok)}
        result = cursor.execute(insert_stmt_final)
        print insert_stmt_final
        insert_stmt_final = _insert_dload_stmt % {'table':'nse_deliv_downloads_info',
                                    'date' : fdate, 'success':int(bhav_ok)}
        cursor.execute(insert_stmt_final)
        print insert_stmt_final
        con.commit()

def _update_bhavcopy(strdate, stocks_dict, fname=None):
    """update bhavcopy Database date in DD-MM-YYYY format."""
    fname = fname or _DEF_SQLIITE_FNAME
    with sqlite3.connect(fname) as con:
        cur = con.cursor()

        fdate = utils.get_ts_for_datestr(strdate, _date_fmt)
        # just to be safe
        global _del_for_date_stmt
        delete_stmt_final = _del_for_date_stmt % { 'date': fdate}
        cur.execute(delete_stmt_final)
        for key, val in stocks_dict.iteritems():
            insert_stmt_final = _nsedata_insert_stmt % { 'sym' : key,
                                                'date': fdate,
                                                'o' : val.open, 'h': val.high,
                                                'l' : val.low, 'c': val.close,
                                                'v' : val.volume, 'd': val.deliv
                                                }
            print "Executing:", insert_stmt_final
            cur.execute(insert_stmt_final)

        con.commit()

def _bhavcopy_downloaded(fdate, fname=None):
    fname = fname or _DEF_SQLIITE_FNAME
    with sqlite3.connect(fname) as con:
        cur = con.cursor()
        select_stmt = '''select count(*) from nse_bhav_downloads_info where
                     date = %(date)f;''' % {'date':fdate}
        cur.execute(select_stmt)
    x = cur.fetchone()

    return x[0] > 0

def _create_tables(fname=None):
    fname = fname or _DEF_SQLIITE_FNAME
    with sqlite3.connect(fname) as cursor:
        for stmt in _create_stmts.values():
            cursor.execute(stmt)

def _apply_name_changes_to_db(syms, fname=None):
    """Changes security names in nse_hist_data table so the name of the security
    is always the latest."""
    fname = fname or _DEF_SQLIITE_FNAME
    global _update_name_changes_stmt
    with sqlite3.connect(fname) as con:
        cur = con.cursor()
        for sym in syms:
            old = sym[:-1]
            new = sym[-1]
            olds = ','.join(["'%s'" % x for x in old])
            update_stmt_final = _update_name_changes_stmt.format(new, olds)
            cur.execute(update_stmt_final)
        con.commit()

if __name__ == '__main__':
    # We run the full program
    import argparse
    parser = argparse.ArgumentParser()

    # --full option
    parser.add_argument("--full",
                        help="download full data from 1 Jan 2002",
                        action="store_true")

    # --from option
    parser.add_argument("--from",
                        help="From Date in DD-MM-YYYY format. " \
                                "Default is 01-01-2002",
                        dest='fromdate',
                        default="01-01-2002")
    # --to option
    parser.add_argument("--to",
                        help="From Date in DD-MM-YYYY format. " \
                                "Default is Today.",
                        dest='todate',
                        default="today")
    args = parser.parse_args()
    try:
        _ = dt.strptime(args.fromdate, _date_fmt)
        if args.todate.lower() == 'today':
            args.todate = dt.now().strftime(_date_fmt)
        _ = dt.strptime(args.todate, _date_fmt)
    except ValueError:
        print parser.format_usage()
        sys.exit(-1)

    # We are now ready to download data
    from_date = dt.strptime(args.fromdate, _date_fmt)
    to_date = dt.strptime(args.todate, _date_fmt)
    if from_date > to_date:
        print parser.format_usage()
        sys.exit(-1)

    _create_tables()

    tdelta = to_date - from_date
    if tdelta.days > 50:
        "Downloading data for {0} days".format(tdelta.days)

    cur_date = from_date
    while cur_date <= to_date:
        scrips_dict = get_bhavcopy(cur_date)
        str_curdate = cur_date.strftime(_date_fmt)
        if scrips_dict is not None:
            _update_bhavcopy(str_curdate, scrips_dict)

        time.sleep(random.randrange(1,10))

        cur_date += td(1)

    # Apply the name changes to the DB
    sym_change_tuples = get_name_change_tuples()
    if len(sym_change_tuples) == 0:
        print "No name change tuples found..."
        sys.exit(-1)
    _apply_name_changes_to_db(sym_change_tuples)
<|endoftext|>


<commit_before>#!/usr/local/python/bin/python
###############################################################################
#                                                                             #
#          Script to get the last image from NGTS and display on webpage      #
#                                    v1.0                                     #
#                               James McCormac                                #
#                                                                             #
# Version History:                                                            #
#	20150408	v1.0	Code written and tested on par-ds                     #
#                                                                             #
###############################################################################
#
# process:
#	grab all the actions from 1 day ago, split according to camera
#	go to last action and get the last image
#	only make pngs and upload to webserver if needed
#
# to do:
#  test in morning
#

import pymysql
import os, sys
import glob as g
from create_movie import create_movie, logger
import getpass

me=getpass.getuser()

if me == "ops":
	topdir="/ngts"
	convert_loc="/usr/local/bin"
	cron_dir="/usr/local/cron/work"
	web_dir="/home/ops/ngts/prism/monitor/img"
else:
	print "Whoami!?"
	sys.exit(1)

# empty dictionary for the actions for each camera
cams={801:[],
	802:[],
	803:[],
	804:[],
	805:[],
	806:[],
	807:[],
	808:[],
	809:[],
	810:[],
	811:[],
	812:[],
	813:[],
	899:[]}

# this should really come from the db...
# where is the camera now, use that das machine?
# cam/das map
das={801:None,
	802:None,
	803:None,
	804:None,
	805:None,
	806:None,
	807:None,
	808:None,
	809:None,
	810:None,
	811:None,
	812:None,
	813:None,
	899:None}

def getDasLoc():
	for i in das:
		s=os.popen('/usr/local/paladin/bin/ngwhereis %d' % (i)).readline()
		try:
			das[i]=s.split()[0]
		except IndexError:
			das[i]=None
		print s

getDasLoc()

# connect to database
conn=pymysql.connect(host='ds',db='ngts_ops')
cur=conn.cursor()

# get the actions from yesterday
qry="SELECT action_id,camera_id,action FROM action_list WHERE schedule_start_utc BETWEEN date_sub(now(), INTERVAL 1 DAY) AND now()"
cur.execute(qry)

# get the action ids for each camera (and dome 899)
for row in cur:
	if row[2] != 'stow':
		cams[row[1]].append("action%s_%s" % (row[0],row[2]))

os.chdir(topdir)
for cam in cams:
	if len(cams[cam]) > 0 and cam != 899:
		# go into the last action directory
		if das[cam] != None:
			os.chdir("%s/%s" % (das[cam],cams[cam][-1]))
			logger.info("Moving to %s/%s" % (das[cam],cams[cam][-1]))
			# get the last image
			t=sorted(g.glob('*.fits'))

			if len(t)>0:
			 	pngfile="%s.png" % (t[-1])
				logger.info("PNG file to make is %s.png" % (t[-1]))
			 	if pngfile not in os.listdir('%s/last_imgs/%s/' % (cron_dir,cam)):
					create_movie([t[-1]],images_directory='%s/last_imgs/%s' % (cron_dir,cam),
						no_time_series=True,include_increment=False,clobber_images_directory=False,resize_factor=4)

					here=os.getcwd()
					os.chdir("%s/last_imgs/%s" % (cron_dir,cam))
					logger.info("Moving to %s/last_imgs/%s" % (cron_dir,cam))

					try:
						f=open('last_img.log').readline()
					except IOError:
						f="XXX"
					logger.info("Last image: %s" % (f))

					if f != pngfile:
						os.system('cp %s %s/cam_%s.png' % (pngfile,web_dir,cam))
						logger.info("Copying %s to %s/cam_%s.png" % (pngfile,web_dir,cam))
						f3=open('last_img.log','w')
						f3.write(pngfile)
						f3.close()
						logger.info('last_img.log updated with %s' % pngfile)

					else:
						print "Last image already up to date, skipping..."
						logger.info('Last image up to date')

					os.chdir(here)
					logger.info('Moving to %s' % (here))

				else:
					logger.info('%s exists already, skipping...' % (pngfile))

			else:
				logger.info("No new fits images to convert, skipping %s..." % (das[cam]))

			os.chdir('../../')
			logger.info('Moving to ../../')
<commit_msg>updated query to find real last image<commit_after>#!/usr/local/python/bin/python
###############################################################################
#                                                                             #
#          Script to get the last image from NGTS and display on webpage      #
#                                    v1.0                                     #
#                               James McCormac                                #
#                                                                             #
# Version History:                                                            #
#	20150408	v1.0	Code written and tested on par-ds                     #
#                                                                             #
###############################################################################
#
# process:
#	grab all the actions from 1 day ago, split according to camera
#	go to last action and get the last image
#	only make pngs and upload to webserver if needed
#
# to do:
#  test in morning
#

import pymysql
import os, sys
import glob as g
from create_movie import create_movie, logger
import getpass

me=getpass.getuser()

if me == "ops":
	topdir="/ngts"
	convert_loc="/usr/local/bin"
	cron_dir="/usr/local/cron/work"
	web_dir="/home/ops/ngts/prism/monitor/img"
else:
	print "Whoami!?"
	sys.exit(1)

# empty dictionary for the actions for each camera
cams={801:[],
	802:[],
	803:[],
	804:[],
	805:[],
	806:[],
	807:[],
	808:[],
	809:[],
	810:[],
	811:[],
	812:[],
	813:[],
	899:[]}

# this should really come from the db...
# where is the camera now, use that das machine?
# cam/das map
das={801:None,
	802:None,
	803:None,
	804:None,
	805:None,
	806:None,
	807:None,
	808:None,
	809:None,
	810:None,
	811:None,
	812:None,
	813:None,
	899:None}

def getDasLoc():
	for i in das:
		s=os.popen('/usr/local/paladin/bin/ngwhereis %d' % (i)).readline()
		try:
			das[i]=s.split()[0]
		except IndexError:
			das[i]=None
		print s

getDasLoc()

# connect to database
conn=pymysql.connect(host='ds',db='ngts_ops')
cur=conn.cursor()

# get the actions from yesterday
#qry="SELECT action_id,camera_id,action FROM action_list WHERE schedule_start_utc BETWEEN date_sub(now(), INTERVAL 1 DAY) AND now()"
#cur.execute(qry)

qry="SELECT image_id,camera_id,raw_image_list.action_id,action FROM raw_image_list LEFT JOIN action_list USING (action_id) WHERE camera_id=%s ORDER BY image_id DESC LIMIT 1 "
cur.execute(qry)

# get the action ids for each camera (and dome 899)
for row in cur:
	if row[3] != 'stow':
		cams[row[1]].append("action%s_%s" % (row[2],row[3]))

os.chdir(topdir)
for cam in cams:
	if len(cams[cam]) > 0 and cam != 899:
		# go into the last action directory
		if das[cam] != None:
			os.chdir("%s/%s" % (das[cam],cams[cam][-1]))
			logger.info("Moving to %s/%s" % (das[cam],cams[cam][-1]))
			# get the last image
			t=sorted(g.glob('*.fits'))

			if len(t)>0:
			 	pngfile="%s.png" % (t[-1])
				logger.info("PNG file to make is %s.png" % (t[-1]))
			 	if pngfile not in os.listdir('%s/last_imgs/%s/' % (cron_dir,cam)):
					create_movie([t[-1]],images_directory='%s/last_imgs/%s' % (cron_dir,cam),
						no_time_series=True,include_increment=False,clobber_images_directory=False,resize_factor=4)

					here=os.getcwd()
					os.chdir("%s/last_imgs/%s" % (cron_dir,cam))
					logger.info("Moving to %s/last_imgs/%s" % (cron_dir,cam))

					try:
						f=open('last_img.log').readline()
					except IOError:
						f="XXX"
					logger.info("Last image: %s" % (f))

					if f != pngfile:
						os.system('cp %s %s/cam_%s.png' % (pngfile,web_dir,cam))
						logger.info("Copying %s to %s/cam_%s.png" % (pngfile,web_dir,cam))
						f3=open('last_img.log','w')
						f3.write(pngfile)
						f3.close()
						logger.info('last_img.log updated with %s' % pngfile)

					else:
						print "Last image already up to date, skipping..."
						logger.info('Last image up to date')

					os.chdir(here)
					logger.info('Moving to %s' % (here))

				else:
					logger.info('%s exists already, skipping...' % (pngfile))

			else:
				logger.info("No new fits images to convert, skipping %s..." % (das[cam]))

			os.chdir('../../')
			logger.info('Moving to ../../')
<|endoftext|>


<commit_before>from pathlib import Path

from django import template
from django.template.defaultfilters import stringfilter

register = template.Library()

@register.filter
@stringfilter
def filename(path):
    """Removes traceback lines from a string (if any). It has no effect when
    no 'Traceback' pattern has been found.

    Returns: raws before the 'Traceback' pattern
    """
    return Path(path).name
<commit_msg>Fix filename template tag docstring<commit_after>from pathlib import Path

from django import template
from django.template.defaultfilters import stringfilter

register = template.Library()

@register.filter
@stringfilter
def filename(path):
    """Removes parent path from a relative or absolute filename

    Returns: the filename
    """
    return Path(path).name
<|endoftext|>


<commit_before>#!/usr/bin/env python
import time
import argparse
import os.path
import cStringIO
import gzip
import sys
import rfc822
import dateutil.parser
import threading
from operator import itemgetter
import datetime
import csv
import email
import json

import nntplib
import magic
from internetarchive import get_item
import futures

COLLECTION_DATE = time.strftime('%Y%m%d')
LIST_FILE = 'giganews_listfile_{date}.txt'.format(date=COLLECTION_DATE)
MBOX_LOCK = threading.RLock()
IDX_LOCK = threading.RLock()

def is_binary(text_buffer):
    """Check a given text_buffer for binary data.

    :type text_buffer: str
    :param text_buffer: The text buffer to be tested for binary data.

    :rtype: bool
    :returns: True if `text_buffer` is binary, else False.

    """
    data_type = magic.from_buffer(text_buffer)
    return True if data_type == 'data' else False

def inline_compress_chunk(chunk, level=9):
    """Join a list with '\n' and compress using gzip.

    :type chunk: list
    :param chunk: A list of strings to be compressed by gzip.

    :rtype: str
    :returns: The gzip compressed chunk represented as a string.
    """
    b = cStringIO.StringIO()
    g = gzip.GzipFile(fileobj=b, mode='wb', compresslevel=level)
    g.write(chunk)
    #g.write('\n')
    g.close()
    return b.getvalue()

def compress_and_sort_index(group):
    idx_fname = '{group}.mbox.{date}.csv'.format(group=group, date=COLLECTION_DATE)
    reader = csv.reader(open(idx_fname), dialect='excel-tab')
    index = [x for x in reader if x]
    sorted_index = sorted(index, key=itemgetter(0))
    gzip_idx_fname = idx_fname + '.gz'

    header = ['#date', 'msg_id', 'from', 'newsgroups', 'subject', 'start', 'length']
    s = cStringIO.StringIO()
    writer = csv.writer(s, dialect='excel-tab')
    writer.writerow(header)
    for line in sorted_index:
        writer.writerow(line)
    compressed_index = inline_compress_chunk(s.getvalue())

    with open(gzip_idx_fname, 'ab') as fp:
        fp.write(compressed_index)
    os.remove(idx_fname)

def get_utc_iso_date(date_str):
    utc_tuple = dateutil.parser.parse(date_str).utctimetuple()
    date_object = datetime.datetime.fromtimestamp(time.mktime(utc_tuple))
    utc_date_str = ''.join([x for x in date_object.isoformat() if x not in '-T:'])
    return utc_date_str

def save_article(article_number, group):
    try:
        _s = nntplib.NNTP('news.giganews.com')
        _s.group(group)
        resp, _, msg_id, msg_list = _s.article(article_number)
        _s.quit()
        if is_binary('\n'.join(msg_list)):
            return None
    except nntplib.NNTPTemporaryError as e:
        sys.stderr.write(
            'error downloading article #{0}: {1}\n'.format(article_number, e))
        return None
    mbox = email.message_from_string('\n'.join(msg_list))
    mbox = mbox.as_string(unixfrom=True)

    # Compress chunk and append to gzip file.
    gzip_fname = '{group}.mbox.{date}.gz'.format(group=group, date=COLLECTION_DATE)
    compressed_chunk = inline_compress_chunk(mbox)
    length = sys.getsizeof(compressed_chunk)
    with MBOX_LOCK:
        with open(gzip_fname, 'a') as fp:
            start = fp.tell()
            fp.write(compressed_chunk)
            #end = fp.tell()

    # Append index information to idx file.
    index_article(msg_list, article_number, start, length)
    print(' saved article #{a}'.format(a=article_number))
    return (int(article_number) + 1)

def index_article(msg_list, article_number, start, length):
    text = '\n'.join(msg_list)
    f = cStringIO.StringIO(text)
    message = rfc822.Message(f)

    # Replace header dict None values with '', and any tabs or
    # newlines with ' '.
    h = dict()
    for key in message.dict:
        if not message.dict[key]:
            h[key] = ''
        else:
            h[key] = message.dict[key].replace('\t', '')
            h[key] = message.dict[key].replace('\n', '')

    date = h.get('date', '')
    if date:
        date = get_utc_iso_date(date)
    idx_line = (date, h.get('message-id'), h.get('from'), h.get('newsgroups'),
                h.get('subject'), start, length)
    idx_fname = '{group}.mbox.{date}.csv'.format(group=group, date=COLLECTION_DATE)
    s = cStringIO.StringIO()
    writer = csv.writer(s, dialect='excel-tab')
    writer.writerow(idx_line)
    with IDX_LOCK:
        with open(idx_fname, 'a') as fp:
            fp.write(s.getvalue())
    return idx_line

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Archive Giganews.')
    parser.add_argument('--list-file', type=str, default=LIST_FILE)
    args = parser.parse_args()

    s = nntplib.NNTP('news.giganews.com')
    if not os.path.exists(LIST_FILE):
        s.list(file=LIST_FILE)

    for line in open(LIST_FILE):
        group, last, _first, flag = line.strip().split()
        identifier = 'usenet-{0}'.format('.'.join(group.split('.')[:2]))
        item = get_item(identifier)
        state = item.get_metadata(target='state')
        first = state.get(group, _first)

        # Exclude all groups with binaries (is this too much?).
        if 'binaries' in group:
            print('Skipping {group}, appears to be a binaries group'.format(group=group))
            continue

        # Check for new articles.
        print('Archiving {group}'.format(group=group))
        count = int(last) - int(first)
        if count <= 0:
            print(' no new articles found'.format(c=count))
            continue
        print(' {c} new articles found'.format(c=count))

        # Archive new articles.
        s.group(group)
        resp, article_list = s.xover(first, last)
        articles_archived = []
        article_numbers = tuple(a[0] for a in article_list)
        try:
            with futures.ThreadPoolExecutor(max_workers=30) as e:
                future_to_article = {
                    e.submit(save_article, a, group): a for a in article_numbers
                }
                for future in futures.as_completed(future_to_article):
                    article_number = future_to_article[future]
                    result = future.result()
                    if result:
                        articles_archived.append(result)
        except (KeyboardInterrupt, SystemExit):
            e.shutdown()
        finally:
            compress_and_sort_index(group)
            state[group] = max(articles_archived)
            idx_fname = '{group}.mbox.{date}.csv.gz'.format(group=group,
                                                            date=COLLECTION_DATE)
            mbox_fname = '{group}.mbox.{date}.gz'.format(group=group,
                                                         date=COLLECTION_DATE)
            item.upload([idx_fname, mbox_fname], verbose=True)
            item.modify_metadata(state, target='state')
<commit_msg>organized imports<commit_after>#!/usr/bin/env python
import time
import threading
import os.path
import sys
import email
import gzip
import cStringIO
import rfc822
import dateutil.parser
import datetime
import csv
from operator import itemgetter

import nntplib
from internetarchive import get_item
import futures
import magic

COLLECTION_DATE = time.strftime('%Y%m%d')
LIST_FILE = 'giganews_listfile_{date}.txt'.format(date=COLLECTION_DATE)
MBOX_LOCK = threading.RLock()
IDX_LOCK = threading.RLock()

def is_binary(text_buffer):
    """Check a given text_buffer for binary data.

    :type text_buffer: str
    :param text_buffer: The text buffer to be tested for binary data.

    :rtype: bool
    :returns: True if `text_buffer` is binary, else False.

    """
    data_type = magic.from_buffer(text_buffer)
    return True if data_type == 'data' else False

def inline_compress_chunk(chunk, level=9):
    """Join a list with '\n' and compress using gzip.

    :type chunk: list
    :param chunk: A list of strings to be compressed by gzip.

    :rtype: str
    :returns: The gzip compressed chunk represented as a string.
    """
    b = cStringIO.StringIO()
    g = gzip.GzipFile(fileobj=b, mode='wb', compresslevel=level)
    g.write(chunk)
    #g.write('\n')
    g.close()
    return b.getvalue()

def compress_and_sort_index(group):
    idx_fname = '{group}.mbox.{date}.csv'.format(group=group, date=COLLECTION_DATE)
    reader = csv.reader(open(idx_fname), dialect='excel-tab')
    index = [x for x in reader if x]
    sorted_index = sorted(index, key=itemgetter(0))
    gzip_idx_fname = idx_fname + '.gz'

    header = ['#date', 'msg_id', 'from', 'newsgroups', 'subject', 'start', 'length']
    s = cStringIO.StringIO()
    writer = csv.writer(s, dialect='excel-tab')
    writer.writerow(header)
    for line in sorted_index:
        writer.writerow(line)
    compressed_index = inline_compress_chunk(s.getvalue())

    with open(gzip_idx_fname, 'ab') as fp:
        fp.write(compressed_index)
    os.remove(idx_fname)

def get_utc_iso_date(date_str):
    utc_tuple = dateutil.parser.parse(date_str).utctimetuple()
    date_object = datetime.datetime.fromtimestamp(time.mktime(utc_tuple))
    utc_date_str = ''.join([x for x in date_object.isoformat() if x not in '-T:'])
    return utc_date_str

def save_article(article_number, group):
    try:
        _s = nntplib.NNTP('news.giganews.com')
        _s.group(group)
        resp, _, msg_id, msg_list = _s.article(article_number)
        _s.quit()
        if is_binary('\n'.join(msg_list)):
            return None
    except nntplib.NNTPTemporaryError as e:
        sys.stderr.write(
            'error downloading article #{0}: {1}\n'.format(article_number, e))
        return None
    mbox = email.message_from_string('\n'.join(msg_list))
    mbox = mbox.as_string(unixfrom=True)

    # Compress chunk and append to gzip file.
    gzip_fname = '{group}.mbox.{date}.gz'.format(group=group, date=COLLECTION_DATE)
    compressed_chunk = inline_compress_chunk(mbox)
    length = sys.getsizeof(compressed_chunk)
    with MBOX_LOCK:
        with open(gzip_fname, 'a') as fp:
            start = fp.tell()
            fp.write(compressed_chunk)
            #end = fp.tell()

    # Append index information to idx file.
    index_article(msg_list, article_number, start, length)
    print(' saved article #{a}'.format(a=article_number))
    return (int(article_number) + 1)

def index_article(msg_list, article_number, start, length):
    text = '\n'.join(msg_list)
    f = cStringIO.StringIO(text)
    message = rfc822.Message(f)

    # Replace header dict None values with '', and any tabs or
    # newlines with ' '.
    h = dict()
    for key in message.dict:
        if not message.dict[key]:
            h[key] = ''
        else:
            h[key] = message.dict[key].replace('\t', '')
            h[key] = message.dict[key].replace('\n', '')

    date = h.get('date', '')
    if date:
        date = get_utc_iso_date(date)
    idx_line = (date, h.get('message-id'), h.get('from'), h.get('newsgroups'),
                h.get('subject'), start, length)
    idx_fname = '{group}.mbox.{date}.csv'.format(group=group, date=COLLECTION_DATE)
    s = cStringIO.StringIO()
    writer = csv.writer(s, dialect='excel-tab')
    writer.writerow(idx_line)
    with IDX_LOCK:
        with open(idx_fname, 'a') as fp:
            fp.write(s.getvalue())
    return idx_line

if __name__ == '__main__':
    s = nntplib.NNTP('news.giganews.com')
    if not os.path.exists(LIST_FILE):
        s.list(file=LIST_FILE)

    for line in open(LIST_FILE):
        group, last, _first, flag = line.strip().split()
        identifier = 'usenet-{0}'.format('.'.join(group.split('.')[:2]))
        item = get_item(identifier)
        state = item.get_metadata(target='state')
        first = state.get(group, _first)

        # Exclude all groups with binaries (is this too much?).
        if 'binaries' in group:
            print('Skipping {group}, appears to be a binaries group'.format(group=group))
            continue

        # Check for new articles.
        print('Archiving {group}'.format(group=group))
        count = int(last) - int(first)
        if count <= 0:
            print(' no new articles found'.format(c=count))
            continue
        print(' {c} new articles found'.format(c=count))

        # Archive new articles.
        s.group(group)
        resp, article_list = s.xover(first, last)
        articles_archived = []
        article_numbers = tuple(a[0] for a in article_list)
        try:
            with futures.ThreadPoolExecutor(max_workers=30) as e:
                future_to_article = {
                    e.submit(save_article, a, group): a for a in article_numbers
                }
                for future in futures.as_completed(future_to_article):
                    article_number = future_to_article[future]
                    result = future.result()
                    if result:
                        articles_archived.append(result)
        except (KeyboardInterrupt, SystemExit):
            e.shutdown()
        finally:
            compress_and_sort_index(group)
            state[group] = max(articles_archived)
            idx_fname = '{group}.mbox.{date}.csv.gz'.format(group=group,
                                                            date=COLLECTION_DATE)
            mbox_fname = '{group}.mbox.{date}.gz'.format(group=group,
                                                         date=COLLECTION_DATE)
            item.upload([idx_fname, mbox_fname], verbose=True)
            item.modify_metadata(state, target='state')
<|endoftext|>