<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
import numpy as np
from astropy.stats import sigma_clip
from astropy.utils import lazyproperty

__all__ = ['Background']

__doctest_requires__ = {('Background'): ['scipy']}

class Background(object):
    """
    Class to estimate a 2D background and background rms noise in an
    image.

    The background is estimated using sigma-clipped statistics in each
    mesh of a grid that covers the input ``data`` to create a
    low-resolution background map.  The final background map is the
    bicubic spline interpolation of the low-resolution map.

    The exact method used to estimate the background in each mesh can be
    set with the ``method`` parameter.  The background rms in each mesh
    is estimated by the sigma-clipped standard deviation.
    """

    def __init__(self, data, box_shape, filter_shape=(3, 3),
                 filter_threshold=None, mask=None, method='sextractor',
                 backfunc=None, sigclip_sigma=3., sigclip_iters=None):
        """
        Parameters
        ----------
        data : array_like
            The 2D array from which to estimate the background and/or
            background rms map.

        box_shape : 2-tuple of int
            The ``(ny, nx)`` shape of the boxes in which to estimate the
            background.  For best results, the box shape should be
            chosen such that the ``data`` are covered by an integer
            number of boxes in both dimensions.

        filter_shape : 2-tuple of int, optional
            The ``(ny, nx)`` shape of the median filter to apply to the
            low-resolution background map.  A filter shape of ``(1, 1)``
            means no filtering.

        filter_threshold : int, optional
            The threshold value for used for selective median filtering
            of the low-resolution background map.  If not `None`, then
            the median filter will be applied to only the background
            meshes with values larger than ``filter_threshold``.

        mask : array_like (bool), optional
            A boolean mask, with the same shape as ``data``, where a
            `True` value indicates the corresponding element of ``data``
            is masked.  Masked data are excluded from all calculations.

        method : {'mean', 'median', 'sextractor', 'mode_estimate'}, optional
            The method use to estimate the background in the meshes.
            For all methods, the statistics are calculated from the
            sigma-clipped ``data`` values in each mesh.

            * 'mean':  Mean.
            * 'median':  Median.
            * 'sextractor':  The method used by `SExtractor`_.  The
              background in each mesh is a mode estimator: ``(2.5 *
              median) - (1.5 * mean)``.  If ``(mean - median) / std >
              0.3`` then the median is used instead.  Despite what the
              `SExtractor`_ User's Manual says, this is the method it
              *always* uses.
            * 'mode_estimate':  An alternative mode estimator:
              ``(3 * median) - (2 * mean)``.
            * 'custom': Use this method in combination with the
              ``backfunc`` parameter to specific a custom function to
              calculate the background in each mesh.

        backfunc : callable
            The function to compute the background in each mesh.  Must
            be a callable that takes in a 3D `~numpy.ma.MaskedArray` of
            size ``MxNxZ``, where the ``Z`` axis (axis=2) contains the
            pixels in each background mesh, and outputs a 2D
            `~numpy.ndarray` low-resolution background map of size
            ``MxN``.  ``backfunc`` is used only if ``method='custom'``.

        sigclip_sigma : float, optional
            The number of standard deviations to use as the clipping limit
            when calculating the image background statistics.

        sigclip_iters : int, optional
           The number of iterations to perform sigma clipping, or `None` to
           clip until convergence is achieved (i.e., continue until the last
           iteration clips nothing) when calculating the image background
           statistics.

        Notes
        -----
        Bicubic interpolation of the low-resolution background map will
        be used along a particular dimension only if there are more than
        3 meshes along that dimension.  If there are 3 or fewer meshes
        along a particular dimension in the low-resolution background
        map, then a bivariate spline of order ``size - 1`` will be used
        along that dimension.  If there is only 1 background mesh
        element (i.e., ``box_shape`` is the same size as the ``data``),
        then the background map will simply be a constant image with the
        value in the background mesh.

        Limiting ``sigclip_iters`` will speed up the calculations,
        especially for large images, at the cost of some precision.

        .. _SExtractor: http://www.astromatic.net/software/sextractor
        """

        if mask is not None:
            if mask.shape != data.shape:
                raise ValueError('mask shape must match data shape')
        valid_methods = ['mean', 'median', 'sextractor', 'mode_estimate',
                         'custom']
        if method not in valid_methods:
            raise ValueError('method "{0}" is not valid'.format(method))
        self.box_shape = (min(box_shape[0], data.shape[0]),
                          min(box_shape[1], data.shape[1]))
        self.filter_shape = filter_shape
        self.filter_threshold = filter_threshold
        self.mask = mask
        self.method = method
        self.backfunc = backfunc
        self.sigclip_sigma = sigclip_sigma
        self.sigclip_iters = sigclip_iters
        self.yextra = data.shape[0] % box_shape[0]
        self.xextra = data.shape[1] % box_shape[1]
        self.data_shape = data.shape
        if (self.yextra > 0) or (self.xextra > 0):
            self.padded = True
            data_ma = self._pad_data(data, mask)
        else:
            self.padded = False
            data_ma = np.ma.masked_array(data, mask=mask)
        self._sigclip_data(data_ma)

    def _pad_data(self, data, mask=None):
        """
        Pad the ``data`` and ``mask`` on the right and top with zeros if
        necessary to have a integer number of background meshes of size
        ``box_shape``.
        """
        ypad, xpad = 0, 0
        if self.yextra > 0:
            ypad = self.box_shape[0] - self.yextra
        if self.xextra > 0:
            xpad = self.box_shape[1] - self.xextra
        pad_width = ((0, ypad), (0, xpad))
        mode = str('constant')
        padded_data = np.pad(data, pad_width, mode=mode,
                             constant_values=[np.nan])
        padded_mask = np.isnan(padded_data)
        if mask is not None:
            mask_pad = np.pad(mask, pad_width, mode=mode,
                              constant_values=[False])
            padded_mask = np.logical_or(padded_mask, mask_pad)
        return np.ma.masked_array(padded_data, mask=padded_mask)

    def _sigclip_data(self, data_ma):
        """
        Perform sigma clipping on the data in regions of size
        ``box_shape``.
        """
        ny, nx = data_ma.shape
        ny_box, nx_box = self.box_shape
        y_nbins = int(ny / ny_box)   # always integer because data were padded
        x_nbins = int(nx / nx_box)   # always integer because data were padded
        data_rebin = np.ma.swapaxes(data_ma.reshape(
            y_nbins, ny_box, x_nbins, nx_box), 1, 2).reshape(y_nbins, x_nbins,
                                                             ny_box * nx_box)
        del data_ma
        self.data_sigclip = sigma_clip(
            data_rebin, sig=self.sigclip_sigma, axis=2,
            iters=self.sigclip_iters, cenfunc=np.ma.median, varfunc=np.ma.var)
        del data_rebin

    def _filter_meshes(self, mesh):
        """
        Apply a 2d median filter to the background meshes, including
        only pixels inside the image at the borders.
        """
        from scipy.ndimage import generic_filter
        if self.filter_threshold is None:
            return generic_filter(mesh, np.nanmedian, size=self.filter_shape,
                                  mode='constant', cval=np.nan)
        else:
            mesh_out = np.copy(mesh)
            for i, j in zip(*np.nonzero(mesh > self.filter_threshold)):
                yfs, xfs = self.filter_shape
                hyfs, hxfs = yfs // 2, xfs // 2
                y0, y1 = max(i - hyfs, 0), min(i - hyfs + yfs, mesh.shape[0])
                x0, x1 = max(j - hxfs, 0), min(j - hxfs + xfs, mesh.shape[1])
                mesh_out[i, j] = np.median(mesh[y0:y1, x0:x1])
            return mesh_out

    def _resize_meshes(self, mesh):
        """
        Resize the background meshes to the original data size using
        bicubic interpolation.
        """
        if np.min(mesh) == np.max(mesh):    # constant image (or only 1 mesh)
            return np.zeros(self.data_shape) + np.min(mesh)
        else:
            from scipy.interpolate import RectBivariateSpline
            ny, nx = mesh.shape
            x = np.arange(nx)
            y = np.arange(ny)
            xx = np.linspace(x.min() - 0.5, x.max() + 0.5, self.data_shape[1])
            yy = np.linspace(y.min() - 0.5, y.max() + 0.5, self.data_shape[0])
            ky, kx = min(nx - 1, 3), min(ny - 1, 3)   # axes reversed
            return RectBivariateSpline(y, x, mesh, kx=kx, ky=ky, s=0)(yy, xx)

    @lazyproperty
    def background_mesh(self):
        """
        A 2D `~numpy.ndarray` containing the background estimate in each
        of the meshes of size ``box_shape``.

        This low-resolution background map is equivalent to the
        low-resolution "MINIBACKGROUND" background map in `SExtractor`_.
        """
        if self.method == 'mean':
            bkg_mesh = np.ma.mean(self.data_sigclip, axis=2)
        elif self.method == 'median':
            bkg_mesh = np.ma.median(self.data_sigclip, axis=2)
        elif self.method == 'sextractor':
            box_mean = np.ma.mean(self.data_sigclip, axis=2)
            box_median = np.ma.median(self.data_sigclip, axis=2)
            box_std = np.ma.std(self.data_sigclip, axis=2)
            condition = (np.abs(box_mean - box_median) / box_std) < 0.3
            bkg_est = (2.5 * box_median) - (1.5 * box_mean)
            bkg_mesh = np.ma.where(condition, bkg_est, box_median)
            bkg_mesh = np.ma.where(box_std == 0, box_mean, bkg_mesh)
        elif self.method == 'mode_estimate':
            bkg_mesh = (3. * np.ma.median(self.data_sigclip, axis=2) -
                        2. * np.ma.mean(self.data_sigclip, axis=2))
        elif self.method == 'custom':
            bkg_mesh = self.backfunc(self.data_sigclip)
            if not isinstance(bkg_mesh, np.ndarray):   # np.ma will pass
                raise ValueError('"backfunc" must return a numpy.ndarray.')
            if isinstance(bkg_mesh, np.ma.MaskedArray):
                raise ValueError('"backfunc" must return a numpy.ndarray.')
            if bkg_mesh.shape != (self.data_sigclip.shape[0],
                                  self.data_sigclip.shape[1]):
                raise ValueError('The shape of the array returned by '
                                 '"backfunc" is not correct.')
        if self.method != 'custom':
            bkg_mesh = np.ma.filled(bkg_mesh,
                                    fill_value=np.ma.median(bkg_mesh))
        if self.filter_shape != (1, 1):
            bkg_mesh = self._filter_meshes(bkg_mesh)
        return bkg_mesh

    @lazyproperty
    def background_rms_mesh(self):
        """
        A 2D `~numpy.ndarray` containing the background rms estimate in
        each of the meshes of size ``box_shape``.

        This low-resolution background rms map is equivalent to the
        low-resolution "MINIBACK_RMS" background rms map in
        `SExtractor`_.
        """
        bkgrms_mesh = np.ma.std(self.data_sigclip, axis=2)
        bkgrms_mesh = np.ma.filled(bkgrms_mesh,
                                   fill_value=np.ma.median(bkgrms_mesh))
        if self.filter_shape != (1, 1):
            bkgrms_mesh = self._filter_meshes(bkgrms_mesh)
        return bkgrms_mesh

    @lazyproperty
    def background(self):
        """
        A 2D `~numpy.ndarray` containing the background estimate.

        This is equivalent to the low-resolution "BACKGROUND" background
        map in `SExtractor`_.
        """
        bkg = self._resize_meshes(self.background_mesh)
        if self.padded:
            y0, x0 = self.data_shape
            bkg = bkg[0:y0, 0:x0]
        if self.mask is not None:
            bkg[self.mask] = 0.
        return bkg

    @lazyproperty
    def background_rms(self):
        """
        A 2D `~numpy.ndarray` containing the background rms estimate.

        This is equivalent to the low-resolution "BACKGROUND_RMS"
        background rms map in `SExtractor`_.
        """
        bkgrms = self._resize_meshes(self.background_rms_mesh)
        if self.padded:
            y0, x0 = self.data_shape
            bkgrms = bkgrms[0:y0, 0:x0]
        if self.mask is not None:
            bkgrms[self.mask] = 0.
        return bkgrms

    @lazyproperty
    def background_median(self):
        """
        The median value of all the background meshes.

        This is equivalent to the value `SExtractor`_ prints to stdout
        (i.e., "(M+D) Background: <value>").
        """
        return np.median(self.background_mesh)

    @lazyproperty
    def background_rms_median(self):
        """
        The median value of all the background rms meshes.

        This is equivalent to the value `SExtractor`_ prints to stdout
        (i.e., "(M+D) RMS: <value>").
        """
        return np.median(self.background_rms_mesh)
<commit_msg>Use zoom instead of RectBivariateSpline in Background<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
import numpy as np
from astropy.stats import sigma_clip
from astropy.utils import lazyproperty

__all__ = ['Background']

__doctest_requires__ = {('Background'): ['scipy']}

class Background(object):
    """
    Class to estimate a 2D background and background rms noise in an
    image.

    The background is estimated using sigma-clipped statistics in each
    mesh of a grid that covers the input ``data`` to create a
    low-resolution background map.  The final background map is the
    bicubic spline interpolation of the low-resolution map.

    The exact method used to estimate the background in each mesh can be
    set with the ``method`` parameter.  The background rms in each mesh
    is estimated by the sigma-clipped standard deviation.
    """

    def __init__(self, data, box_shape, filter_shape=(3, 3),
                 filter_threshold=None, mask=None, method='sextractor',
                 backfunc=None, sigclip_sigma=3., sigclip_iters=None):
        """
        Parameters
        ----------
        data : array_like
            The 2D array from which to estimate the background and/or
            background rms map.

        box_shape : 2-tuple of int
            The ``(ny, nx)`` shape of the boxes in which to estimate the
            background.  For best results, the box shape should be
            chosen such that the ``data`` are covered by an integer
            number of boxes in both dimensions.

        filter_shape : 2-tuple of int, optional
            The ``(ny, nx)`` shape of the median filter to apply to the
            low-resolution background map.  A filter shape of ``(1, 1)``
            means no filtering.

        filter_threshold : int, optional
            The threshold value for used for selective median filtering
            of the low-resolution background map.  If not `None`, then
            the median filter will be applied to only the background
            meshes with values larger than ``filter_threshold``.

        mask : array_like (bool), optional
            A boolean mask, with the same shape as ``data``, where a
            `True` value indicates the corresponding element of ``data``
            is masked.  Masked data are excluded from all calculations.

        method : {'mean', 'median', 'sextractor', 'mode_estimate'}, optional
            The method use to estimate the background in the meshes.
            For all methods, the statistics are calculated from the
            sigma-clipped ``data`` values in each mesh.

            * 'mean':  Mean.
            * 'median':  Median.
            * 'sextractor':  The method used by `SExtractor`_.  The
              background in each mesh is a mode estimator: ``(2.5 *
              median) - (1.5 * mean)``.  If ``(mean - median) / std >
              0.3`` then the median is used instead.  Despite what the
              `SExtractor`_ User's Manual says, this is the method it
              *always* uses.
            * 'mode_estimate':  An alternative mode estimator:
              ``(3 * median) - (2 * mean)``.
            * 'custom': Use this method in combination with the
              ``backfunc`` parameter to specific a custom function to
              calculate the background in each mesh.

        backfunc : callable
            The function to compute the background in each mesh.  Must
            be a callable that takes in a 3D `~numpy.ma.MaskedArray` of
            size ``MxNxZ``, where the ``Z`` axis (axis=2) contains the
            pixels in each background mesh, and outputs a 2D
            `~numpy.ndarray` low-resolution background map of size
            ``MxN``.  ``backfunc`` is used only if ``method='custom'``.

        sigclip_sigma : float, optional
            The number of standard deviations to use as the clipping limit
            when calculating the image background statistics.

        sigclip_iters : int, optional
           The number of iterations to perform sigma clipping, or `None` to
           clip until convergence is achieved (i.e., continue until the last
           iteration clips nothing) when calculating the image background
           statistics.

        Notes
        -----
        If there is only 1 background mesh element (i.e., ``box_shape``
        is the same size as the ``data``), then the background map will
        simply be a constant image with the value in the background
        mesh.

        Limiting ``sigclip_iters`` will speed up the calculations,
        especially for large images, at the cost of some precision.

        .. _SExtractor: http://www.astromatic.net/software/sextractor
        """

        if mask is not None:
            if mask.shape != data.shape:
                raise ValueError('mask shape must match data shape')
        valid_methods = ['mean', 'median', 'sextractor', 'mode_estimate',
                         'custom']
        if method not in valid_methods:
            raise ValueError('method "{0}" is not valid'.format(method))
        self.box_shape = (min(box_shape[0], data.shape[0]),
                          min(box_shape[1], data.shape[1]))
        self.filter_shape = filter_shape
        self.filter_threshold = filter_threshold
        self.mask = mask
        self.method = method
        self.backfunc = backfunc
        self.sigclip_sigma = sigclip_sigma
        self.sigclip_iters = sigclip_iters
        self.yextra = data.shape[0] % box_shape[0]
        self.xextra = data.shape[1] % box_shape[1]
        self.data_shape = data.shape
        if (self.yextra > 0) or (self.xextra > 0):
            self.padded = True
            data_ma = self._pad_data(data, mask)
        else:
            self.padded = False
            data_ma = np.ma.masked_array(data, mask=mask)
        self.data_ma_shape = data_ma.shape
        self._sigclip_data(data_ma)

    def _pad_data(self, data, mask=None):
        """
        Pad the ``data`` and ``mask`` on the right and top with zeros if
        necessary to have a integer number of background meshes of size
        ``box_shape``.
        """
        ypad, xpad = 0, 0
        if self.yextra > 0:
            ypad = self.box_shape[0] - self.yextra
        if self.xextra > 0:
            xpad = self.box_shape[1] - self.xextra
        pad_width = ((0, ypad), (0, xpad))
        mode = str('constant')
        padded_data = np.pad(data, pad_width, mode=mode,
                             constant_values=[np.nan])
        padded_mask = np.isnan(padded_data)
        if mask is not None:
            mask_pad = np.pad(mask, pad_width, mode=mode,
                              constant_values=[False])
            padded_mask = np.logical_or(padded_mask, mask_pad)
        return np.ma.masked_array(padded_data, mask=padded_mask)

    def _sigclip_data(self, data_ma):
        """
        Perform sigma clipping on the data in regions of size
        ``box_shape``.
        """
        ny, nx = data_ma.shape
        ny_box, nx_box = self.box_shape
        y_nbins = int(ny / ny_box)   # always integer because data were padded
        x_nbins = int(nx / nx_box)   # always integer because data were padded
        data_rebin = np.ma.swapaxes(data_ma.reshape(
            y_nbins, ny_box, x_nbins, nx_box), 1, 2).reshape(y_nbins, x_nbins,
                                                             ny_box * nx_box)
        del data_ma
        self.data_sigclip = sigma_clip(
            data_rebin, sig=self.sigclip_sigma, axis=2,
            iters=self.sigclip_iters, cenfunc=np.ma.median, varfunc=np.ma.var)
        del data_rebin

    def _filter_meshes(self, mesh):
        """
        Apply a 2d median filter to the background meshes, including
        only pixels inside the image at the borders.
        """
        from scipy.ndimage import generic_filter
        if self.filter_threshold is None:
            return generic_filter(mesh, np.nanmedian, size=self.filter_shape,
                                  mode='constant', cval=np.nan)
        else:
            mesh_out = np.copy(mesh)
            for i, j in zip(*np.nonzero(mesh > self.filter_threshold)):
                yfs, xfs = self.filter_shape
                hyfs, hxfs = yfs // 2, xfs // 2
                y0, y1 = max(i - hyfs, 0), min(i - hyfs + yfs, mesh.shape[0])
                x0, x1 = max(j - hxfs, 0), min(j - hxfs + xfs, mesh.shape[1])
                mesh_out[i, j] = np.median(mesh[y0:y1, x0:x1])
            return mesh_out

    def _resize_meshes(self, mesh):
        """
        Resize the background meshes to the original data size using
        bicubic interpolation.
        """
        if np.min(mesh) == np.max(mesh):    # constant image (or only 1 mesh)
            return np.zeros(self.data_shape) + np.min(mesh)
        else:
            from scipy.ndimage import zoom
            zoom_factor = (int(self.data_ma_shape[0] / mesh.shape[0]),
                           int(self.data_ma_shape[1] / mesh.shape[1]))
            return zoom(mesh, zoom_factor, order=3, mode='reflect')

    @lazyproperty
    def background_mesh(self):
        """
        A 2D `~numpy.ndarray` containing the background estimate in each
        of the meshes of size ``box_shape``.

        This low-resolution background map is equivalent to the
        low-resolution "MINIBACKGROUND" background map in `SExtractor`_.
        """
        if self.method == 'mean':
            bkg_mesh = np.ma.mean(self.data_sigclip, axis=2)
        elif self.method == 'median':
            bkg_mesh = np.ma.median(self.data_sigclip, axis=2)
        elif self.method == 'sextractor':
            box_mean = np.ma.mean(self.data_sigclip, axis=2)
            box_median = np.ma.median(self.data_sigclip, axis=2)
            box_std = np.ma.std(self.data_sigclip, axis=2)
            condition = (np.abs(box_mean - box_median) / box_std) < 0.3
            bkg_est = (2.5 * box_median) - (1.5 * box_mean)
            bkg_mesh = np.ma.where(condition, bkg_est, box_median)
            bkg_mesh = np.ma.where(box_std == 0, box_mean, bkg_mesh)
        elif self.method == 'mode_estimate':
            bkg_mesh = (3. * np.ma.median(self.data_sigclip, axis=2) -
                        2. * np.ma.mean(self.data_sigclip, axis=2))
        elif self.method == 'custom':
            bkg_mesh = self.backfunc(self.data_sigclip)
            if not isinstance(bkg_mesh, np.ndarray):   # np.ma will pass
                raise ValueError('"backfunc" must return a numpy.ndarray.')
            if isinstance(bkg_mesh, np.ma.MaskedArray):
                raise ValueError('"backfunc" must return a numpy.ndarray.')
            if bkg_mesh.shape != (self.data_sigclip.shape[0],
                                  self.data_sigclip.shape[1]):
                raise ValueError('The shape of the array returned by '
                                 '"backfunc" is not correct.')
        if self.method != 'custom':
            bkg_mesh = np.ma.filled(bkg_mesh,
                                    fill_value=np.ma.median(bkg_mesh))
        if self.filter_shape != (1, 1):
            bkg_mesh = self._filter_meshes(bkg_mesh)
        return bkg_mesh

    @lazyproperty
    def background_rms_mesh(self):
        """
        A 2D `~numpy.ndarray` containing the background rms estimate in
        each of the meshes of size ``box_shape``.

        This low-resolution background rms map is equivalent to the
        low-resolution "MINIBACK_RMS" background rms map in
        `SExtractor`_.
        """
        bkgrms_mesh = np.ma.std(self.data_sigclip, axis=2)
        bkgrms_mesh = np.ma.filled(bkgrms_mesh,
                                   fill_value=np.ma.median(bkgrms_mesh))
        if self.filter_shape != (1, 1):
            bkgrms_mesh = self._filter_meshes(bkgrms_mesh)
        return bkgrms_mesh

    @lazyproperty
    def background(self):
        """
        A 2D `~numpy.ndarray` containing the background estimate.

        This is equivalent to the low-resolution "BACKGROUND" background
        map in `SExtractor`_.
        """
        bkg = self._resize_meshes(self.background_mesh)
        if self.padded:
            y0, x0 = self.data_shape
            bkg = bkg[0:y0, 0:x0]
        if self.mask is not None:
            bkg[self.mask] = 0.
        return bkg

    @lazyproperty
    def background_rms(self):
        """
        A 2D `~numpy.ndarray` containing the background rms estimate.

        This is equivalent to the low-resolution "BACKGROUND_RMS"
        background rms map in `SExtractor`_.
        """
        bkgrms = self._resize_meshes(self.background_rms_mesh)
        if self.padded:
            y0, x0 = self.data_shape
            bkgrms = bkgrms[0:y0, 0:x0]
        if self.mask is not None:
            bkgrms[self.mask] = 0.
        return bkgrms

    @lazyproperty
    def background_median(self):
        """
        The median value of all the background meshes.

        This is equivalent to the value `SExtractor`_ prints to stdout
        (i.e., "(M+D) Background: <value>").
        """
        return np.median(self.background_mesh)

    @lazyproperty
    def background_rms_median(self):
        """
        The median value of all the background rms meshes.

        This is equivalent to the value `SExtractor`_ prints to stdout
        (i.e., "(M+D) RMS: <value>").
        """
        return np.median(self.background_rms_mesh)
<|endoftext|>


<commit_before>import datetime
import dateutil.parser
import json
import pymongo
# import urllib

import girder.api.rest
from girder import logger
from girder.api import access
from girder.api.describe import Description

class Scraper(girder.api.rest.Resource):
    def __init__(self):
        self.resourceName = 'scrape'
        self.route('GET', (), self.scrape)

    @access.public
    def scrape(self, params):
        result = []
        url = params.get('url', None)
        if url:
            import requests
            import BeautifulSoup as bs4
            page = requests.get(url)
            #tree = html.fromstring(page.text)
            soup = bs4.BeautifulSoup(page.text)
            result = soup.findAll('img')
            result = [a.get('src') for a in result]
        return result

class Geospace(girder.api.rest.Resource):
    """API endpoint for Geospace data."""

    def __init__(self):
        self.resourceName = 'data'
        self.route('GET', (), self.find)

    @access.public
    def find(self, params):
        limit, offset, sort = self.getPagingParameters(params)
        result = {}

        db = pymongo.MongoClient('mongodb://localhost:27017/ist')
        database = db.get_default_database()
        coll = database['ads']

        # Get valid time range
        time_range = params.get('duration', None)
        if (time_range is None):
            result['duration'] = {
                "start": [i for i in coll.find(
                    {"field4": { "$exists": True, "$nin": ["None"] } } ).sort(
                        "field4",  1).limit(1)][0],
                "end":   [i for i in coll.find(
                    {"field4": { "$exists": True, "$nin": ["None"] } } ).sort(
                        "field4", -1).limit(1)][0]
            };
            return result;

        # Get location if provided
        location = params.get('location', None)
        geospatial_query = None
        use_location = False

        if (location is not None) and (len(location) > 0):
            use_location = True

            # Reverse geocode location
            from geopy.geocoders import Nominatim
            geolocator = Nominatim()
            loc = geolocator.geocode(location)

            # Hard-coded to 10 units for now
            geospatial_query = {   "loc" : {"$geoWithin" :
                    {
                        "$center" : [[loc.latitude, loc.longitude], 1]
                    }
                }
            }

        # Find all of the datasets within time range.
        time_range = eval(time_range)

        start = time_range[0]
        end = time_range[1]

        start_time = datetime.datetime.fromtimestamp(start).isoformat()
        end_time = datetime.datetime.fromtimestamp(end).isoformat()

        if not use_location:
            query_result = coll.find({"field4":
                {"$gte": start_time, "$lt": end_time}},
                skip=offset, limit=limit, sort=sort)
        else:
            # When using location do not limit by time for now
            query_result = coll.find(geospatial_query,
                skip=offset, limit=limit, sort=sort)
        result = [row for row in query_result]
        return result

    find.description = (
        Description('Get a set of data.')
        .param('source', 'Database source (default mongo).', required=False,
               enum=['mongo', 'mongofull', 'tangelo'])
        .param('limit', 'Result set size limit (default=50).', required=False,
               dataType='int')
        .param('offset', 'Offset into result set (default=0).', required=False,
               dataType='int')
        .param('sort', 'Field to sort the user list by (default='
               'pickup_datetime)', required=False)
        .param('sortdir', '1 for ascending, -1 for descending (default=1)',
               required=False, dataType='int'))

def load(info):
    info['apiRoot'].data = Geospace()
    info['apiRoot'].scrape = Scraper()<commit_msg>Enabled query by time and location together<commit_after>import datetime
import dateutil.parser
import json
import pymongo
# import urllib

import girder.api.rest
from girder import logger
from girder.api import access
from girder.api.describe import Description

class Scraper(girder.api.rest.Resource):
    def __init__(self):
        self.resourceName = 'scrape'
        self.route('GET', (), self.scrape)

    @access.public
    def scrape(self, params):
        result = []
        url = params.get('url', None)
        if url:
            import requests
            import BeautifulSoup as bs4
            page = requests.get(url)
            #tree = html.fromstring(page.text)
            soup = bs4.BeautifulSoup(page.text)
            result = soup.findAll('img')
            result = [a.get('src') for a in result]
        return result

class Geospace(girder.api.rest.Resource):
    """API endpoint for Geospace data."""

    def __init__(self):
        self.resourceName = 'data'
        self.route('GET', (), self.find)

    @access.public
    def find(self, params):
        limit, offset, sort = self.getPagingParameters(params)
        result = {}

        db = pymongo.MongoClient('mongodb://localhost:27017/ist')
        database = db.get_default_database()
        coll = database['ads']

        # Get valid time range
        time_range = params.get('duration', None)
        if (time_range is None):
            result['duration'] = {
                "start": [i for i in coll.find(
                    {"field4": { "$exists": True, "$nin": ["None"] } } ).sort(
                        "field4",  1).limit(1)][0],
                "end":   [i for i in coll.find(
                    {"field4": { "$exists": True, "$nin": ["None"] } } ).sort(
                        "field4", -1).limit(1)][0]
            };
            return result;

        # Get location if provided
        location = params.get('location', None)
        geospatial_query = None
        use_location = False

        if (location is not None) and (len(location) > 0):
            use_location = True

            # Reverse geocode location
            from geopy.geocoders import Nominatim
            geolocator = Nominatim()
            loc = geolocator.geocode(location)

            # Hard-coded to 10 units for now
            geospatial_query = {   "loc" : {"$geoWithin" :
                    {
                        "$center" : [[loc.latitude, loc.longitude], 1]
                    }
                }
            }

        # Find all of the datasets within time range.
        time_range = eval(time_range)

        start = time_range[0]
        end = time_range[1]

        start_time = datetime.datetime.fromtimestamp(start).isoformat()
        end_time = datetime.datetime.fromtimestamp(end).isoformat()

        if not use_location:
            query_result = coll.find({"field4":
                {"$gte": start_time, "$lt": end_time}},
                skip=offset, limit=limit, sort=sort)
        else:
            # When using location do not limit by time for now
            query_result = coll.find( {"$and":[
                    geospatial_query,
                    {"field4":
                        {"$gte": start_time, "$lt": end_time}
                    }]
                },
                skip=offset, limit=limit, sort=sort)
        result = [row for row in query_result]
        return result

    find.description = (
        Description('Get a set of data.')
        .param('source', 'Database source (default mongo).', required=False,
               enum=['mongo', 'mongofull', 'tangelo'])
        .param('limit', 'Result set size limit (default=50).', required=False,
               dataType='int')
        .param('offset', 'Offset into result set (default=0).', required=False,
               dataType='int')
        .param('sort', 'Field to sort the user list by (default='
               'pickup_datetime)', required=False)
        .param('sortdir', '1 for ascending, -1 for descending (default=1)',
               required=False, dataType='int'))

def load(info):
    info['apiRoot'].data = Geospace()
    info['apiRoot'].scrape = Scraper()<|endoftext|>


<commit_before>from httpie.compat import urlsplit, str

class HTTPMessage(object):
    """Abstract class for HTTP messages."""

    def __init__(self, orig):
        self._orig = orig

    def iter_body(self, chunk_size):
        """Return an iterator over the body."""
        raise NotImplementedError()

    def iter_lines(self, chunk_size):
        """Return an iterator over the body yielding (`line`, `line_feed`)."""
        raise NotImplementedError()

    @property
    def headers(self):
        """Return a `str` with the message's headers."""
        raise NotImplementedError()

    @property
    def encoding(self):
        """Return a `str` with the message's encoding, if known."""
        raise NotImplementedError()

    @property
    def body(self):
        """Return a `bytes` with the message's body."""
        raise NotImplementedError()

    @property
    def content_type(self):
        """Return the message content type."""
        ct = self._orig.headers.get('Content-Type', '')
        if not isinstance(ct, str):
            ct = ct.decode('utf8')
        return ct

class HTTPResponse(HTTPMessage):
    """A :class:`requests.models.Response` wrapper."""

    def iter_body(self, chunk_size=1):
        return self._orig.iter_content(chunk_size=chunk_size)

    def iter_lines(self, chunk_size):
        return ((line, b'\n') for line in self._orig.iter_lines(chunk_size))

    #noinspection PyProtectedMember
    @property
    def headers(self):
        original = self._orig.raw._original_response
        version = {9: '0.9', 10: '1.0', 11: '1.1', 20: '2.0'}[original.version]
        status_line = 'HTTP/{version} {status} {reason}'.format(
            version=version,
            status=original.status,
            reason=original.reason
        )
        headers = [status_line]
        try:
            # `original.msg` is a `http.client.HTTPMessage` on Python 3
            # `_headers` is a 2-tuple
            headers.extend(
                '%s: %s' % header for header in original.msg._headers)
        except AttributeError:
            # and a `httplib.HTTPMessage` on Python 2.x
            # `headers` is a list of `name: val<CRLF>`.
            headers.extend(h.strip() for h in original.msg.headers)

        return '\r\n'.join(headers)

    @property
    def encoding(self):
        return self._orig.encoding or 'utf8'

    @property
    def body(self):
        # Only now the response body is fetched.
        # Shouldn't be touched unless the body is actually needed.
        return self._orig.content

class HTTPRequest(HTTPMessage):
    """A :class:`requests.models.Request` wrapper."""

    def iter_body(self, chunk_size):
        yield self.body

    def iter_lines(self, chunk_size):
        yield self.body, b''

    @property
    def headers(self):
        url = urlsplit(self._orig.url)

        request_line = '{method} {path}{query} HTTP/1.1'.format(
            method=self._orig.method,
            path=url.path or '/',
            query='?' + url.query if url.query else ''
        )

        headers = dict(self._orig.headers)
        if 'Host' not in self._orig.headers:
            headers['Host'] = url.netloc.split('@')[-1]

        headers = [
            '%s: %s' % (
                name,
                value if isinstance(value, str) else value.decode('utf8')
            )
            for name, value in headers.items()
        ]

        headers.insert(0, request_line)
        headers = '\r\n'.join(headers).strip()

        if isinstance(headers, bytes):
            # Python < 3
            headers = headers.decode('utf8')
        return headers

    @property
    def encoding(self):
        return 'utf8'

    @property
    def body(self):
        body = self._orig.body
        if isinstance(body, str):
            # Happens with JSON/form request data parsed from the command line.
            body = body.encode('utf8')
        return body or b''
<commit_msg>HTTP/2 has no minor versions.<commit_after>from httpie.compat import urlsplit, str

class HTTPMessage(object):
    """Abstract class for HTTP messages."""

    def __init__(self, orig):
        self._orig = orig

    def iter_body(self, chunk_size):
        """Return an iterator over the body."""
        raise NotImplementedError()

    def iter_lines(self, chunk_size):
        """Return an iterator over the body yielding (`line`, `line_feed`)."""
        raise NotImplementedError()

    @property
    def headers(self):
        """Return a `str` with the message's headers."""
        raise NotImplementedError()

    @property
    def encoding(self):
        """Return a `str` with the message's encoding, if known."""
        raise NotImplementedError()

    @property
    def body(self):
        """Return a `bytes` with the message's body."""
        raise NotImplementedError()

    @property
    def content_type(self):
        """Return the message content type."""
        ct = self._orig.headers.get('Content-Type', '')
        if not isinstance(ct, str):
            ct = ct.decode('utf8')
        return ct

class HTTPResponse(HTTPMessage):
    """A :class:`requests.models.Response` wrapper."""

    def iter_body(self, chunk_size=1):
        return self._orig.iter_content(chunk_size=chunk_size)

    def iter_lines(self, chunk_size):
        return ((line, b'\n') for line in self._orig.iter_lines(chunk_size))

    #noinspection PyProtectedMember
    @property
    def headers(self):
        original = self._orig.raw._original_response

        version = {
            9: '0.9',
            10: '1.0',
            11: '1.1',
            20: '2',
        }[original.version]

        status_line = 'HTTP/{version} {status} {reason}'.format(
            version=version,
            status=original.status,
            reason=original.reason
        )
        headers = [status_line]
        try:
            # `original.msg` is a `http.client.HTTPMessage` on Python 3
            # `_headers` is a 2-tuple
            headers.extend(
                '%s: %s' % header for header in original.msg._headers)
        except AttributeError:
            # and a `httplib.HTTPMessage` on Python 2.x
            # `headers` is a list of `name: val<CRLF>`.
            headers.extend(h.strip() for h in original.msg.headers)

        return '\r\n'.join(headers)

    @property
    def encoding(self):
        return self._orig.encoding or 'utf8'

    @property
    def body(self):
        # Only now the response body is fetched.
        # Shouldn't be touched unless the body is actually needed.
        return self._orig.content

class HTTPRequest(HTTPMessage):
    """A :class:`requests.models.Request` wrapper."""

    def iter_body(self, chunk_size):
        yield self.body

    def iter_lines(self, chunk_size):
        yield self.body, b''

    @property
    def headers(self):
        url = urlsplit(self._orig.url)

        request_line = '{method} {path}{query} HTTP/1.1'.format(
            method=self._orig.method,
            path=url.path or '/',
            query='?' + url.query if url.query else ''
        )

        headers = dict(self._orig.headers)
        if 'Host' not in self._orig.headers:
            headers['Host'] = url.netloc.split('@')[-1]

        headers = [
            '%s: %s' % (
                name,
                value if isinstance(value, str) else value.decode('utf8')
            )
            for name, value in headers.items()
        ]

        headers.insert(0, request_line)
        headers = '\r\n'.join(headers).strip()

        if isinstance(headers, bytes):
            # Python < 3
            headers = headers.decode('utf8')
        return headers

    @property
    def encoding(self):
        return 'utf8'

    @property
    def body(self):
        body = self._orig.body
        if isinstance(body, str):
            # Happens with JSON/form request data parsed from the command line.
            body = body.encode('utf8')
        return body or b''
<|endoftext|>


<commit_before>import time

from lettuce import step
from lettuce import world

from lettuce_webdriver.util import assert_true
from lettuce_webdriver.util import assert_false

from selenium.common.exceptions import WebDriverException

import logging
log = logging.getLogger(__name__)

def wait_for_elem(browser, sel, timeout=15):
    start = time.time()
    elems = []
    while time.time() - start < timeout:
        elems = find_elements_by_jquery(browser, sel)
        if elems:
            return elems
        time.sleep(0.2)
    return elems

def load_script(browser, url):
    """Ensure that JavaScript at a given URL is available to the browser."""
    browser.execute_script("""
    var script_tag = document.createElement("script");
    script_tag.setAttribute("type", "text/javascript");
    script_tag.setAttribute("src", arguments[0]);
    document.getElementsByTagName("head")[0].appendChild(script_tag);
    """, url)

def find_elements_by_jquery(browser, selector):
    """Find HTML elements using jQuery-style selectors.

    Ensures that jQuery is available to the browser; if it gets a
    WebDriverException that looks like jQuery is not available, it attempts to
    include it and reexecute the script."""
    try:
        return browser.execute_script("""return $(arguments[0]).get();""", selector)
    except WebDriverException as e:
        if e.msg.startswith(u'$ is not defined'):
            load_script(browser, "//ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js")
            return browser.execute_script("""return $(arguments[0]).get();""", selector)
        else:
            raise

def find_parents_by_jquery(browser, selector):
    """Find HTML elements' parents using jQuery-style selectors.

    In addition to reliably including jQuery, this also finds the pa"""
    try:
        return browser.execute_script("""return $(arguments[0]).parent().get();""", selector)
    except WebDriverException as e:
        if e.msg.startswith(u'$ is not defined'):
            load_script(browser, "//ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js")
            return browser.execute_script("""return $(arguments[0]).parent().get();""", selector)
        else:
            raise

@step(r'There should be an element matching \$\("(.*?)"\)$')
def check_element_by_selector(step, selector):
    elems = find_elements_by_jquery(world.browser, selector)
    assert_true(step, elems)

@step(r'There should be an element matching \$\("(.*?)"\) within (\d+) seconds?$')
def wait_for_element_by_selector(step, selector, seconds):
    elems = wait_for_elem(world.browser, selector, int(seconds))
    assert_true(step, elems)

@step(r'I fill in \$\("(.*?)"\) with "(.*?)"$')
def fill_in_by_selector(step, selector, value):
    elem = find_elements_by_jquery(world.browser, selector)[0]
    elem.clear()
    elem.send_keys(value)

@step(r'I submit \$\("(.*?)"\)')
def submit_by_selector(step, selector):
    elem = find_elements_by_jquery(world.browser, selector)[0]
    elem.submit()

@step(r'I check \$\("(.*?)"\)$')
def check_by_selector(step, selector):
    elem = find_elements_by_jquery(world.browser, selector)[0]
    if not elem.is_selected():
        elem.click()

@step(r'I click \$\("(.*?)"\)$')
def click_by_selector(step, selector):
    # No need for separate button press step with selector style.
    elem = find_elements_by_jquery(world.browser, selector)[0]
    elem.click()

@step(r'I follow the link \$\("(.*?)"\)$')
def click_by_selector(step, selector):
    elem = find_elements_by_jquery(world.browser, selector)[0]
    href = elem.get_attribute('href')
    world.browser.get(href)

@step(r'\$\("(.*?)"\) should be selected$')
def click_by_selector(step, selector):
    # No need for separate button press step with selector style.
    elem = find_elements_by_jquery(world.browser, selector)[0]
    assert_true(step, elem.is_selected())

@step(r'I select \$\("(.*?)"\)$')
def select_by_selector(step, selector):
    option = find_elements_by_jquery(world.browser, selector)[0]
    selector = find_parents_by_jquery(world.browser, selector)[0]
    selector.click()
    time.sleep(0.3)
    option.click()
    assert_true(step, option.is_selected())

__all__ = [
    'wait_for_element_by_selector',
    'fill_in_by_selector',
    'check_by_selector',
    'click_by_selector',
    'check_element_by_selector',
]
<commit_msg>Count elements exactly by CSS selector.<commit_after>import time

from lettuce import step
from lettuce import world

from lettuce_webdriver.util import assert_true
from lettuce_webdriver.util import assert_false

from selenium.common.exceptions import WebDriverException

import logging
log = logging.getLogger(__name__)

def wait_for_elem(browser, sel, timeout=15):
    start = time.time()
    elems = []
    while time.time() - start < timeout:
        elems = find_elements_by_jquery(browser, sel)
        if elems:
            return elems
        time.sleep(0.2)
    return elems

def load_script(browser, url):
    """Ensure that JavaScript at a given URL is available to the browser."""
    browser.execute_script("""
    var script_tag = document.createElement("script");
    script_tag.setAttribute("type", "text/javascript");
    script_tag.setAttribute("src", arguments[0]);
    document.getElementsByTagName("head")[0].appendChild(script_tag);
    """, url)

def find_elements_by_jquery(browser, selector):
    """Find HTML elements using jQuery-style selectors.

    Ensures that jQuery is available to the browser; if it gets a
    WebDriverException that looks like jQuery is not available, it attempts to
    include it and reexecute the script."""
    try:
        return browser.execute_script("""return $(arguments[0]).get();""", selector)
    except WebDriverException as e:
        if e.msg.startswith(u'$ is not defined'):
            load_script(browser, "//ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js")
            return browser.execute_script("""return $(arguments[0]).get();""", selector)
        else:
            raise

def find_parents_by_jquery(browser, selector):
    """Find HTML elements' parents using jQuery-style selectors.

    In addition to reliably including jQuery, this also finds the pa"""
    try:
        return browser.execute_script("""return $(arguments[0]).parent().get();""", selector)
    except WebDriverException as e:
        if e.msg.startswith(u'$ is not defined'):
            load_script(browser, "//ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js")
            return browser.execute_script("""return $(arguments[0]).parent().get();""", selector)
        else:
            raise

@step(r'There should be an element matching \$\("(.*?)"\)$')
def check_element_by_selector(step, selector):
    elems = find_elements_by_jquery(world.browser, selector)
    assert_true(step, elems)

@step(r'There should be an element matching \$\("(.*?)"\) within (\d+) seconds?$')
def wait_for_element_by_selector(step, selector, seconds):
    elems = wait_for_elem(world.browser, selector, int(seconds))
    assert_true(step, elems)

@step(r'There should be exactly (\d+) elements matching \$\("(.*?)"\)$')
def count_elements_exactly_by_selector(step, number, selector):
    elems = find_elements_by_jquery(world.browser, selector)
    assert_true(step, len(elems) == int(number))

@step(r'I fill in \$\("(.*?)"\) with "(.*?)"$')
def fill_in_by_selector(step, selector, value):
    elem = find_elements_by_jquery(world.browser, selector)[0]
    elem.clear()
    elem.send_keys(value)

@step(r'I submit \$\("(.*?)"\)')
def submit_by_selector(step, selector):
    elem = find_elements_by_jquery(world.browser, selector)[0]
    elem.submit()

@step(r'I check \$\("(.*?)"\)$')
def check_by_selector(step, selector):
    elem = find_elements_by_jquery(world.browser, selector)[0]
    if not elem.is_selected():
        elem.click()

@step(r'I click \$\("(.*?)"\)$')
def click_by_selector(step, selector):
    # No need for separate button press step with selector style.
    elem = find_elements_by_jquery(world.browser, selector)[0]
    elem.click()

@step(r'I follow the link \$\("(.*?)"\)$')
def click_by_selector(step, selector):
    elem = find_elements_by_jquery(world.browser, selector)[0]
    href = elem.get_attribute('href')
    world.browser.get(href)

@step(r'\$\("(.*?)"\) should be selected$')
def click_by_selector(step, selector):
    # No need for separate button press step with selector style.
    elem = find_elements_by_jquery(world.browser, selector)[0]
    assert_true(step, elem.is_selected())

@step(r'I select \$\("(.*?)"\)$')
def select_by_selector(step, selector):
    option = find_elements_by_jquery(world.browser, selector)[0]
    selector = find_parents_by_jquery(world.browser, selector)[0]
    selector.click()
    time.sleep(0.3)
    option.click()
    assert_true(step, option.is_selected())

__all__ = [
    'wait_for_element_by_selector',
    'fill_in_by_selector',
    'check_by_selector',
    'click_by_selector',
    'check_element_by_selector',
]
<|endoftext|>


<commit_before>###############################################################################
# Caleydo - Visualization for Molecular Biology - http://caleydo.org
# Copyright (c) The Caleydo Team. All rights reserved.
# Licensed under the new BSD license, available at http://caleydo.org/license
###############################################################################

from builtins import str
import logging

_log = logging.getLogger(__name__)

# extend a dictionary recursivly
def extend(target, w):
  if w is None:
    return target
  for k, v in w.items():
    if isinstance(v, dict):
      if k not in target:
        target[k] = extend({}, v)
      else:
        target[k] = extend(target[k], v)
    else:
      target[k] = v
  return target

def replace_variables_f(s, lookup):
  import re
  s = str(s)
  if re.match(r'^\$\{([^}]+)\}$', s):  # full string is a pattern
    s = s[2:len(s) - 1]
    v = lookup(s)
    if v is None:
      _log.error('cant resolve ' + s)
      return '$unresolved$'
    return v

  def match(m):
    v = lookup(m.group(1))
    if v is None:
      _log.error('cant resolve ' + m.group(1))
      return '$unresolved$'
    return str(v)

  return re.sub(r'\$\{([^}]+)\}', match, s)

def replace_variables(s, variables):
  return replace_variables_f(s, lambda x: variables.get(x, None))

def replace_nested_variables(obj, lookup):
  if isinstance(obj, list):
    return [replace_nested_variables(o, lookup) for o in obj]
  elif isinstance(obj, str):
    return replace_variables_f(obj, lookup)
  elif isinstance(obj, dict):
    return {k: replace_nested_variables(v, lookup) for k, v in obj.items()}
  return obj
<commit_msg>removed check for NoneType<commit_after>###############################################################################
# Caleydo - Visualization for Molecular Biology - http://caleydo.org
# Copyright (c) The Caleydo Team. All rights reserved.
# Licensed under the new BSD license, available at http://caleydo.org/license
###############################################################################

from builtins import str
import logging

_log = logging.getLogger(__name__)

# extend a dictionary recursively
def extend(target, w):
  for k, v in w.items():
    if isinstance(v, dict):
      if k not in target:
        target[k] = extend({}, v)
      else:
        target[k] = extend(target[k], v)
    else:
      target[k] = v
  return target

def replace_variables_f(s, lookup):
  import re
  s = str(s)
  if re.match(r'^\$\{([^}]+)\}$', s):  # full string is a pattern
    s = s[2:len(s) - 1]
    v = lookup(s)
    if v is None:
      _log.error('cant resolve ' + s)
      return '$unresolved$'
    return v

  def match(m):
    v = lookup(m.group(1))
    if v is None:
      _log.error('cant resolve ' + m.group(1))
      return '$unresolved$'
    return str(v)

  return re.sub(r'\$\{([^}]+)\}', match, s)

def replace_variables(s, variables):
  return replace_variables_f(s, lambda x: variables.get(x, None))

def replace_nested_variables(obj, lookup):
  if isinstance(obj, list):
    return [replace_nested_variables(o, lookup) for o in obj]
  elif isinstance(obj, str):
    return replace_variables_f(obj, lookup)
  elif isinstance(obj, dict):
    return {k: replace_nested_variables(v, lookup) for k, v in obj.items()}
  return obj
<|endoftext|>


<commit_before>#!/usr/bin/env python

# AKA resonator local curriculum resolver/worker

import beanstalkc
from pattern.web import URL, plaintext, URLError, MIMETYPE_WEBPAGE, MIMETYPE_PLAINTEXT
from urlparse import urlparse as parse

beanstalk = beanstalkc.Connection(host='localhost', port=14711)
while beanstalk.peek_ready():
    job = beanstalk.reserve()
    output = open('test_output_redis.txt','w')
    # print url
    url = URL(job.body)
    try:
        # s = url.download(timeout=2500)
        s = url.download(cache=True,timeout=100)
        the_type = url.mimetype
        print the_type
        if url.mimetype==MIMETYPE_WEBPAGE or url.mimetype==MIMETYPE_PLAINTEXT:
            s = plaintext(s)
            output.write(s.encode('ascii','ignore'))
            print s
            c = c+1
            print c
    except URLError, e:
        # print 'URLError on ', url
        print e
    output.close()
    job.delete()
<commit_msg>mimetype validation is failing<commit_after>#!/usr/bin/env python

# AKA resonator local curriculum resolver/worker

import beanstalkc
from pattern.web import URL, plaintext, URLError, MIMETYPE_WEBPAGE, MIMETYPE_PLAINTEXT
from urlparse import urlparse as parse

beanstalk = beanstalkc.Connection(host='localhost', port=14711)
while beanstalk.peek_ready():
    job = beanstalk.reserve()
    output = open('test_output_redis.txt','w')
    # print url
    url = URL(job.body)
    try:
        # s = url.download(timeout=2500)
        s = url.download(cache=True,timeout=100)
        the_type = url.mimetype
        print the_type
        if url.mimetype==MIMETYPE_WEBPAGE or url.mimetype==MIMETYPE_PLAINTEXT:
            s = plaintext(s)
            output.write(s.encode('ascii','ignore'))
            print s
            c = c+1
            print c
    # except URLError, e:
    except:
        e = sys.exc_info()[0]
        # print 'URLError on ', url
        print e
    output.close()
    job.delete()
<|endoftext|>


<commit_before>###############################################################################
# Caleydo - Visualization for Molecular Biology - http://caleydo.org
# Copyright (c) The Caleydo Team. All rights reserved.
# Licensed under the new BSD license, available at http://caleydo.org/license
###############################################################################
from __future__ import print_function
import logging.config

# set configured registry
def _get_config():
  from . import config
  return config.view('phovea_server')

cc = _get_config()
_log = logging.getLogger(__name__)

def enable_dev_mode():
  _log.info('enabling development mode')
  cc.set('env', 'development')
  cc.set('debug', True)
  cc.set('error_stack_trace', True)
  cc.set('nocache', True)

def enable_prod_mode():
  _log.info('enabling production mode')
  cc.set('env', 'production')
  cc.set('debug', False)
  cc.set('error_stack_trace', False)
  cc.set('nocache', False)

def _config_files():
  """
  list all known config files
  :return:
  """
  from .plugin import plugins
  return [p for p in (p.config_file() for p in plugins()) if p is not None]

def _resolve_launcher(launcher):
  """
  resolves the launcher if it is a string it will load the module and determine the function to use
  :param launcher:
  :return: launcher function
  """
  import six
  import os
  import importlib

  if isinstance(launcher, six.string_types):
    module_name, function_name = os.path.splitext(launcher)
    m = importlib.import_module(module_name)
    return m[function_name]

  return launcher

def set_default_subparser(parser, name, args=None):
  """default subparser selection. Call after setup, just before parse_args()
  name: is the name of the subparser to call by default
  args: if set is the argument list handed to parse_args()
  see http://stackoverflow.com/questions/5176691/argparse-how-to-specify-a-default-subcommand

  , tested with 2.7, 3.2, 3.3, 3.4
  it works with 2.6 assuming argparse is installed
  """
  import sys
  import argparse

  subparser_found = False
  for arg in sys.argv[1:]:
    if arg in ['-h', '--help']:  # global help if no subparser
      break
  else:
    for x in parser._subparsers._actions:
      if not isinstance(x, argparse._SubParsersAction):
        continue
      for sp_name in x._name_parser_map.keys():
        if sp_name in sys.argv[1:]:
          subparser_found = True
    if not subparser_found:
      # insert default in first position, this implies no
      # global options without a sub_parsers specified
      if args is None:
        sys.argv.insert(1, name)
      else:
        args.insert(0, name)

def _resolve_commands(parser):
  from .plugin import list as list_plugins
  subparsers = parser.add_subparsers(dest='cmd')
  default_command = None
  for command in list_plugins('command'):
    _log.info('add command ' + command.id)
    if hasattr(command, 'isDefault') and command.isDefault:
      default_command = command.id
    cmdparser = subparsers.add_parser(command.id)
    instance = command.load().factory(cmdparser)
    cmdparser.set_defaults(launcher=instance)
  return default_command

def run():
  import argparse
  import sys

  parser = argparse.ArgumentParser(description='Phovea Server')
  parser.add_argument('--use_reloader', action='store_true', help='whether to automatically reload the server')
  parser.add_argument('--env', default=cc.get('env'), help='environment mode (dev or prod)')

  # parse before to enable correct plugin discovery
  args = parser.parse_known_args()[0]
  if args.env.startswith('dev'):
    enable_dev_mode()
  else:
    enable_prod_mode()

  default_command = _resolve_commands(parser)
  if default_command is not None:
    set_default_subparser(parser, default_command)

  args = parser.parse_args()

  main = args.launcher(args)

  if args.use_reloader:
    _log.info('start using reloader...')
    from werkzeug._reloader import run_with_reloader
    run_with_reloader(main, extra_files=_config_files())
  else:
    _log.info('start...')
    main()

def create_embedded():
  from .server import create_application
  return create_application()
<commit_msg>fix flake8 error<commit_after>###############################################################################
# Caleydo - Visualization for Molecular Biology - http://caleydo.org
# Copyright (c) The Caleydo Team. All rights reserved.
# Licensed under the new BSD license, available at http://caleydo.org/license
###############################################################################
from __future__ import print_function
import logging.config

# set configured registry
def _get_config():
  from . import config
  return config.view('phovea_server')

cc = _get_config()
_log = logging.getLogger(__name__)

def enable_dev_mode():
  _log.info('enabling development mode')
  cc.set('env', 'development')
  cc.set('debug', True)
  cc.set('error_stack_trace', True)
  cc.set('nocache', True)

def enable_prod_mode():
  _log.info('enabling production mode')
  cc.set('env', 'production')
  cc.set('debug', False)
  cc.set('error_stack_trace', False)
  cc.set('nocache', False)

def _config_files():
  """
  list all known config files
  :return:
  """
  from .plugin import plugins
  return [p for p in (p.config_file() for p in plugins()) if p is not None]

def _resolve_launcher(launcher):
  """
  resolves the launcher if it is a string it will load the module and determine the function to use
  :param launcher:
  :return: launcher function
  """
  import six
  import os
  import importlib

  if isinstance(launcher, six.string_types):
    module_name, function_name = os.path.splitext(launcher)
    m = importlib.import_module(module_name)
    return m[function_name]

  return launcher

def set_default_subparser(parser, name, args=None):
  """default subparser selection. Call after setup, just before parse_args()
  name: is the name of the subparser to call by default
  args: if set is the argument list handed to parse_args()
  see http://stackoverflow.com/questions/5176691/argparse-how-to-specify-a-default-subcommand

  , tested with 2.7, 3.2, 3.3, 3.4
  it works with 2.6 assuming argparse is installed
  """
  import sys
  import argparse

  subparser_found = False
  for arg in sys.argv[1:]:
    if arg in ['-h', '--help']:  # global help if no subparser
      break
  else:
    for x in parser._subparsers._actions:
      if not isinstance(x, argparse._SubParsersAction):
        continue
      for sp_name in x._name_parser_map.keys():
        if sp_name in sys.argv[1:]:
          subparser_found = True
    if not subparser_found:
      # insert default in first position, this implies no
      # global options without a sub_parsers specified
      if args is None:
        sys.argv.insert(1, name)
      else:
        args.insert(0, name)

def _resolve_commands(parser):
  from .plugin import list as list_plugins
  subparsers = parser.add_subparsers(dest='cmd')
  default_command = None
  for command in list_plugins('command'):
    _log.info('add command ' + command.id)
    if hasattr(command, 'isDefault') and command.isDefault:
      default_command = command.id
    cmdparser = subparsers.add_parser(command.id)
    instance = command.load().factory(cmdparser)
    cmdparser.set_defaults(launcher=instance)
  return default_command

def run():
  import argparse

  parser = argparse.ArgumentParser(description='Phovea Server')
  parser.add_argument('--use_reloader', action='store_true', help='whether to automatically reload the server')
  parser.add_argument('--env', default=cc.get('env'), help='environment mode (dev or prod)')

  # parse before to enable correct plugin discovery
  args = parser.parse_known_args()[0]
  if args.env.startswith('dev'):
    enable_dev_mode()
  else:
    enable_prod_mode()

  default_command = _resolve_commands(parser)
  if default_command is not None:
    set_default_subparser(parser, default_command)

  args = parser.parse_args()

  main = args.launcher(args)

  if args.use_reloader:
    _log.info('start using reloader...')
    from werkzeug._reloader import run_with_reloader
    run_with_reloader(main, extra_files=_config_files())
  else:
    _log.info('start...')
    main()

def create_embedded():
  from .server import create_application
  return create_application()
<|endoftext|>


<commit_before>from django.shortcuts import render

from data_center.models import Announcement

def index(request):
    announcements = Announcement.objects.all().order_by('time')
    return render(request, 'index.html', {'announcements': announcements})
<commit_msg>change sort<commit_after>from django.shortcuts import render

from data_center.models import Announcement

def index(request):
    announcements = Announcement.objects.all().order_by('-time')
    return render(request, 'index.html', {'announcements': announcements})
<|endoftext|>


<commit_before>"""setupEXOSIMS.py is designed to setup the EXOSIMS module for operation

Must be run from top file of EXOSIMS
Written by: Dean Keithly
Written on: 9/9/2018
"""
import os
import shutil
#import pip
from setuptools.command.easy_install import main as install

# def install(package):
#     #Checks to see if the package can be imported. If not, then use pip to install the package... (might not work)
#     try:
#         __import__(package)
#     except ImportError:
#         if hasattr(pip, 'main'):
#             pip.main(['install', package])
#         else:
#             pip._internal.main(['install', package])

if __name__ == '__main__':
    #Check to ensure files exist in the correct places
    files = ['makeSimilar.json','makeSimilar2.json','deleteMe100.json']
    for file in files:
        assert os.path.isfile('./EXOSIMS/Scripts/' + file), "%s is not a valid file" %file
    for folder in ['Scripts','run']:
        assert os.path.isdir('./EXOSIMS/' + folder), "%s is not a valid folder" %folder

    #### Part 1 makeSimilarScripts.py ##########################
    if not os.path.isdir('../Scripts'):#Check if the folder exists otherwise make it
        os.mkdir('../Scripts')
    for file in files:#Copy makeSimilarScripts to top level Scripts
        shutil.copy2('./EXOSIMS/Scripts/' + file,'../Scripts/' + file)
    shutil.copy2('./EXOSIMS/util/makeSimilarScripts.py','../Scripts/' + 'makeSimilarScripts.py')

    #### Part 2 cache #########################################
    #This folder is temporarily created to house files generated by EXOSIMS which are designed to be continually appended to or overwritten
    if not os.path.isdir('../cache'):#Check if the folder exists otherwise make it
        os.mkdir('../cache')

    #### Part 3 FIT FILES #####################################
    #This folder is designed *in the future* to house fit files for telescope instruments
    if not os.path.isdir('../fitFilesFolder'):#Check if the folder exists otherwise make it
        os.mkdir('../fitFilesFolder')

    #### If packages not installed, install them
    #pip.main(['install','-r','./requirements.txt'])
    with open('./requirements.txt') as g:
        reqs = g.readlines()
    for line in reqs:
        install(line.split())

    ### Set Python Path #############
    ### Download h5 fitting parameters file to planetPhysical Model

    #Download HLC fit files from IPAC WFIRST website
    #Download de432.bsp file fromJPL<commit_msg>setupEXOSIMS downloads .h5 file<commit_after>"""setupEXOSIMS.py is designed to setup the EXOSIMS module for operation

Must be run from top file of EXOSIMS
Written by: Dean Keithly
Written on: 9/9/2018
"""
import os
import shutil
import urllib
#import pip
from setuptools.command.easy_install import main as install

# def install(package):
#     #Checks to see if the package can be imported. If not, then use pip to install the package... (might not work)
#     try:
#         __import__(package)
#     except ImportError:
#         if hasattr(pip, 'main'):
#             pip.main(['install', package])
#         else:
#             pip._internal.main(['install', package])

if __name__ == '__main__':
    #Check to ensure files exist in the correct places
    files = ['makeSimilar.json','makeSimilar2.json','deleteMe100.json']
    for file in files:
        assert os.path.isfile('./EXOSIMS/Scripts/' + file), "%s is not a valid file" %file
    for folder in ['Scripts','run']:
        assert os.path.isdir('./EXOSIMS/' + folder), "%s is not a valid folder" %folder

    #### Part 1 makeSimilarScripts.py ##########################
    if not os.path.isdir('../Scripts'):#Check if the folder exists otherwise make it
        os.mkdir('../Scripts')
    for file in files:#Copy makeSimilarScripts to top level Scripts
        shutil.copy2('./EXOSIMS/Scripts/' + file,'../Scripts/' + file)
    shutil.copy2('./EXOSIMS/util/makeSimilarScripts.py','../Scripts/' + 'makeSimilarScripts.py')

    #### Part 2 cache #########################################
    #This folder is temporarily created to house files generated by EXOSIMS which are designed to be continually appended to or overwritten
    if not os.path.isdir('../cache'):#Check if the folder exists otherwise make it
        os.mkdir('../cache')

    #### Part 3 FIT FILES #####################################
    #This folder is designed *in the future* to house fit files for telescope instruments
    if not os.path.isdir('../fitFilesFolder'):#Check if the folder exists otherwise make it
        os.mkdir('../fitFilesFolder')

    #### If packages not installed, install them
    #pip.main(['install','-r','./requirements.txt'])
    with open('./requirements.txt') as g:
        reqs = g.readlines()
    for line in reqs:
        install(line.split())

    ### Set Python Path #############
    ### Download h5 fitting parameters file to planetPhysical Model

    testfile = urllib.URLopener()
    testfile.retrieve("https://github.com/chenjj2/forecaster/blob/master/fitting_parameters.h5", "./EXOSIMS/PlanetPhysicalModel/fitting_parameters.h5")

    #Download HLC fit files from IPAC WFIRST website
    #Download de432.bsp file fromJPL
<|endoftext|>


<commit_before>#!/usr/bin/env python3
import argparse
import collections
import datetime
import http.cookies
import logging
import os
import re
import sys
import time

import bs4
import requests
import setproctitle

class Shoutbox:
    base_url = ""
    inferno_url = ""
    s = None
    lines = []
    read = collections.deque(maxlen=21)

    def __init__(self, base_url, cookie={}, inferno_path="/infernoshout.php", base_path="/index.php"):
        self.base_url = base_url
        self.inferno_url = self.base_url + inferno_path
        self.s = requests.Session()
        self.s.headers.update({
            "User-Agent": "Mozilla/5.0 (Windows NT 6.1; rv:45.0) Gecko/20100101 Firefox/45.0",
            "X-Requested-With": "XMLHttpRequest",
            "Referer": self.base_url + base_path,
        })
        if cookie:
            self.s.cookies.update(cookie)

    def _parse(self, html):
        MAGIC = "<<~!PARSE_SHOUT!~>>"

        active_users = atoi(html)
        logging.info("%d active users" % active_users)
        html = html[len(str(active_users)):]
        if not html.startswith(MAGIC):
            logging.warning("ignoring bogus html: %s" % html)
            return

        html = html.lstrip(MAGIC)
        h = bs4.BeautifulSoup(html)

        # put the full URL after the text inside the "a" tag
        for a in h.find_all("a"):
            if "href" not in a.attrs:
                continue

            if a['href'] == "#":
                continue

            a.string.replace_with("%s (%s)" % (a.string, a['href']))

        for br in h.find_all("br"):
            br.string = "\n"

        chat = h.get_text()

        # remove timestamps - they're relative, and thus they make read lines appear as unread when the day changes
        chat = re.sub("^\[[^\]]*\] ", "", chat, flags=re.MULTILINE)
        return chat

    def _get(self):
        params = {
            "action": "getshouts",
            "timestamp": datetime.datetime.now().strftime("%s200"),
        }

        try:
            r = self.s.get(self.inferno_url, params=params)
            return r.text
        except requests.exceptions.ConnectionError as e:
            logging.warn("connection error: %s" % e)
            return ""

    def update(self):
        l = self._parse(self._get()).rstrip("\n").split('\n')
        self.lines.extend(l)

    def print_new(self):
        for i in self.lines:
            if i not in self.read:
                print(i, flush=True)
                self.read.append(i)
            else:
                logging.debug("skipping line " + i)

        self.lines = []

    def initial_fetch(self):
        self.update()
        for i in self.lines:
            self.read.append(i)
        self.lines = []

def atoi(string):
    s = []
    for i in string:
        try:
            int(i)
            s.append(i)
        except ValueError:
            break

    return int(''.join(s))

def dict_from_cookie_str(cookie_str):
    c = http.cookies.SimpleCookie()
    d = dict()

    c.load(cookie_str)
    for k, m in c.items():
        d[k] = m.value

    return d

def main():
    logging.basicConfig(level=logging.INFO)

    os.environ["SPT_NOENV"] = "true"
    setproctitle.setproctitle(sys.argv[0])

    parser = argparse.ArgumentParser(description="Command line feed for Inferno Shoutbox")
    parser.add_argument("-b", "--backlog", action="store_true", help="Display the backlog after connecting")
    parser.add_argument("url", help="Base URL of the forum")
    parser.add_argument("cookies", help="Cookies in the standard Cookie header format (RFC 6265, section 4.1.1)")
    args = parser.parse_args()

    s = Shoutbox(args.url, dict_from_cookie_str(args.cookies))

    if args.backlog:
        s.update()
        s.print_new()
    else:
        s.initial_fetch()

    while True:
        time.sleep(5)
        s.update()
        s.print_new()

if __name__ == "__main__":
   main()
<commit_msg>Don't crash on responses not starting with a number<commit_after>#!/usr/bin/env python3
import argparse
import collections
import datetime
import http.cookies
import logging
import os
import re
import sys
import time

import bs4
import requests
import setproctitle

class Shoutbox:
    base_url = ""
    inferno_url = ""
    s = None
    lines = []
    read = collections.deque(maxlen=21)

    def __init__(self, base_url, cookie={}, inferno_path="/infernoshout.php", base_path="/index.php"):
        self.base_url = base_url
        self.inferno_url = self.base_url + inferno_path
        self.s = requests.Session()
        self.s.headers.update({
            "User-Agent": "Mozilla/5.0 (Windows NT 6.1; rv:45.0) Gecko/20100101 Firefox/45.0",
            "X-Requested-With": "XMLHttpRequest",
            "Referer": self.base_url + base_path,
        })
        if cookie:
            self.s.cookies.update(cookie)

    def _parse(self, html):
        MAGIC = "<<~!PARSE_SHOUT!~>>"

        try:
            active_users = atoi(html)
            logging.info("%d active users" % active_users)
            html = html[len(str(active_users)):]
        except ValueError:
            pass

        if not html.startswith(MAGIC):
            logging.warning("ignoring bogus html: %s" % html)
            return

        html = html.lstrip(MAGIC)
        h = bs4.BeautifulSoup(html)

        # put the full URL after the text inside the "a" tag
        for a in h.find_all("a"):
            if "href" not in a.attrs:
                continue

            if a['href'] == "#":
                continue

            a.string.replace_with("%s (%s)" % (a.string, a['href']))

        for br in h.find_all("br"):
            br.string = "\n"

        chat = h.get_text()

        # remove timestamps - they're relative, and thus they make read lines appear as unread when the day changes
        chat = re.sub("^\[[^\]]*\] ", "", chat, flags=re.MULTILINE)
        return chat

    def _get(self):
        params = {
            "action": "getshouts",
            "timestamp": datetime.datetime.now().strftime("%s200"),
        }

        try:
            r = self.s.get(self.inferno_url, params=params)
            return r.text
        except requests.exceptions.ConnectionError as e:
            logging.warn("connection error: %s" % e)
            return ""

    def update(self):
        l = self._parse(self._get()).rstrip("\n").split('\n')
        self.lines.extend(l)

    def print_new(self):
        for i in self.lines:
            if i not in self.read:
                print(i, flush=True)
                self.read.append(i)
            else:
                logging.debug("skipping line " + i)

        self.lines = []

    def initial_fetch(self):
        self.update()
        for i in self.lines:
            self.read.append(i)
        self.lines = []

def atoi(string):
    s = []
    for i in string:
        try:
            int(i)
            s.append(i)
        except ValueError:
            break

    return int(''.join(s))

def dict_from_cookie_str(cookie_str):
    c = http.cookies.SimpleCookie()
    d = dict()

    c.load(cookie_str)
    for k, m in c.items():
        d[k] = m.value

    return d

def main():
    logging.basicConfig(level=logging.INFO)

    os.environ["SPT_NOENV"] = "true"
    setproctitle.setproctitle(sys.argv[0])

    parser = argparse.ArgumentParser(description="Command line feed for Inferno Shoutbox")
    parser.add_argument("-b", "--backlog", action="store_true", help="Display the backlog after connecting")
    parser.add_argument("url", help="Base URL of the forum")
    parser.add_argument("cookies", help="Cookies in the standard Cookie header format (RFC 6265, section 4.1.1)")
    args = parser.parse_args()

    s = Shoutbox(args.url, dict_from_cookie_str(args.cookies))

    if args.backlog:
        s.update()
        s.print_new()
    else:
        s.initial_fetch()

    while True:
        time.sleep(5)
        s.update()
        s.print_new()

if __name__ == "__main__":
   main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
import cv2
import go_vncdriver
import tensorflow as tf
import argparse
import logging
import time
import os
import universe.utils
from a3c import A3C
from envs import create_env

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

# Disables write_meta_graph argument, which freezes entire process and is mostly useless.
class FastSaver(tf.train.Saver):
    def save(self, sess, save_path, global_step=None, latest_filename=None,
             meta_graph_suffix="meta", write_meta_graph=True):
        super(FastSaver, self).save(sess, save_path, global_step, latest_filename,
                                    meta_graph_suffix, False)

def run(args, server):
    env = create_env(args.env_id, client_id=str(args.task), remotes=args.remotes)
    trainer = A3C(env, args.task)

    # Variable names that start with "local" are not saved in checkpoints.
    variables_to_save = [v for v in tf.global_variables() if not v.name.startswith("local")]
    init_op = tf.variables_initializer(variables_to_save)
    init_all_op = tf.global_variables_initializer()
    saver = FastSaver(variables_to_save)

    def init_fn(ses):
        logger.info("Initializing all parameters.")
        ses.run(init_all_op)

    config = tf.ConfigProto(device_filters=["/job:ps", "/job:worker/task:{}/cpu:0".format(args.task)])
    logdir = os.path.join(args.log_dir, 'train')
    summary_writer = tf.summary.FileWriter(logdir + "_%d" % args.task)
    logger.info("Events directory: %s_%s", logdir, args.task)
    sv = tf.train.Supervisor(is_chief=(args.task == 0),
                             logdir=logdir,
                             saver=saver,
                             summary_op=None,
                             init_op=init_op,
                             init_fn=init_fn,
                             summary_writer=summary_writer,
                             ready_op=tf.report_uninitialized_variables(variables_to_save),
                             global_step=trainer.global_step,
                             save_model_secs=30,
                             save_summaries_secs=30)

    num_global_steps = 100000000

    logger.info(
        "Starting session. If this hangs, we're mostly likely waiting to connect to the parameter server. " +
        "One common cause is that the parameter server DNS name isn't resolving yet, or is misspecified.")
    with sv.managed_session(server.target, config=config) as sess, sess.as_default():
        trainer.start(sess, summary_writer)
        global_step = sess.run(trainer.global_step)
        logger.info("Starting training at step=%d", global_step)
        while not sv.should_stop() and (not num_global_steps or global_step < num_global_steps):
            trainer.process(sess)
            global_step = sess.run(trainer.global_step)

    # Ask for all the services to stop.
    sv.stop()
    logger.info('reached %s steps. worker stopped.', global_step)

def cluster_spec(num_workers, num_ps):
    """
More tensorflow setup for data parallelism
"""
    cluster = {}
    port = 12222

    all_ps = []
    host = '127.0.0.1'
    for _ in range(num_ps):
        all_ps.append('{}:{}'.format(host, port))
        port += 1
    cluster['ps'] = all_ps

    all_workers = []
    for _ in range(num_workers):
        all_workers.append('{}:{}'.format(host, port))
        port += 1
    cluster['worker'] = all_workers
    return cluster

def main(_):
    """
Setting up Tensorflow for data parallel work
"""

    parser = argparse.ArgumentParser(description=None)
    parser.add_argument('-v', '--verbose', action='count', dest='verbosity', default=0, help='Set verbosity.')
    parser.add_argument('--task', default=0, type=int, help='Task index')
    parser.add_argument('--job-name', default="worker", help='worker or ps')
    parser.add_argument('--num-workers', default=1, type=int, help='Number of workers')
    parser.add_argument('--log-dir', default="/tmp/pong", help='Log directory path')
    parser.add_argument('--env-id', default="PongDeterministic-v3", help='Environment id')
    parser.add_argument('-r', '--remotes', default=None,
                        help='References to environments to create (e.g. -r 20), '
                             'or the address of pre-existing VNC servers and '
                             'rewarders to use (e.g. -r vnc://localhost:5900+15900,vnc://localhost:5901+15901)')

    args = parser.parse_args()
    spec = cluster_spec(args.num_workers, 1)
    cluster = tf.train.ClusterSpec(spec).as_cluster_def()

    universe.utils.exit_on_signal()

    if args.job_name == "worker":
        server = tf.train.Server(cluster, job_name="worker", task_index=args.task,
                                 config=tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=2))
        run(args, server)
    else:
        server = tf.train.Server(cluster, job_name="ps", task_index=args.task,
                                 config=tf.ConfigProto(device_filters=["/job:ps"]))
        while True:
            time.sleep(1000)

if __name__ == "__main__":
    tf.app.run()
<commit_msg>Show trainable vars<commit_after>#!/usr/bin/env python
import cv2
import go_vncdriver
import tensorflow as tf
import argparse
import logging
import time
import os
import universe.utils
from a3c import A3C
from envs import create_env

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

# Disables write_meta_graph argument, which freezes entire process and is mostly useless.
class FastSaver(tf.train.Saver):
    def save(self, sess, save_path, global_step=None, latest_filename=None,
             meta_graph_suffix="meta", write_meta_graph=True):
        super(FastSaver, self).save(sess, save_path, global_step, latest_filename,
                                    meta_graph_suffix, False)

def run(args, server):
    env = create_env(args.env_id, client_id=str(args.task), remotes=args.remotes)
    trainer = A3C(env, args.task)

    # Variable names that start with "local" are not saved in checkpoints.
    variables_to_save = [v for v in tf.global_variables() if not v.name.startswith("local")]
    init_op = tf.variables_initializer(variables_to_save)
    init_all_op = tf.global_variables_initializer()
    saver = FastSaver(variables_to_save)

    var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, tf.get_variable_scope().name)
    logger.info('Trainable vars:')
    for v in var_list:
        logger.info('  %s %s', v.name, v.get_shape())

    def init_fn(ses):
        logger.info("Initializing all parameters.")
        ses.run(init_all_op)

    config = tf.ConfigProto(device_filters=["/job:ps", "/job:worker/task:{}/cpu:0".format(args.task)])
    logdir = os.path.join(args.log_dir, 'train')
    summary_writer = tf.summary.FileWriter(logdir + "_%d" % args.task)
    logger.info("Events directory: %s_%s", logdir, args.task)
    sv = tf.train.Supervisor(is_chief=(args.task == 0),
                             logdir=logdir,
                             saver=saver,
                             summary_op=None,
                             init_op=init_op,
                             init_fn=init_fn,
                             summary_writer=summary_writer,
                             ready_op=tf.report_uninitialized_variables(variables_to_save),
                             global_step=trainer.global_step,
                             save_model_secs=30,
                             save_summaries_secs=30)

    num_global_steps = 100000000

    logger.info(
        "Starting session. If this hangs, we're mostly likely waiting to connect to the parameter server. " +
        "One common cause is that the parameter server DNS name isn't resolving yet, or is misspecified.")
    with sv.managed_session(server.target, config=config) as sess, sess.as_default():
        trainer.start(sess, summary_writer)
        global_step = sess.run(trainer.global_step)
        logger.info("Starting training at step=%d", global_step)
        while not sv.should_stop() and (not num_global_steps or global_step < num_global_steps):
            trainer.process(sess)
            global_step = sess.run(trainer.global_step)

    # Ask for all the services to stop.
    sv.stop()
    logger.info('reached %s steps. worker stopped.', global_step)

def cluster_spec(num_workers, num_ps):
    """
More tensorflow setup for data parallelism
"""
    cluster = {}
    port = 12222

    all_ps = []
    host = '127.0.0.1'
    for _ in range(num_ps):
        all_ps.append('{}:{}'.format(host, port))
        port += 1
    cluster['ps'] = all_ps

    all_workers = []
    for _ in range(num_workers):
        all_workers.append('{}:{}'.format(host, port))
        port += 1
    cluster['worker'] = all_workers
    return cluster

def main(_):
    """
Setting up Tensorflow for data parallel work
"""

    parser = argparse.ArgumentParser(description=None)
    parser.add_argument('-v', '--verbose', action='count', dest='verbosity', default=0, help='Set verbosity.')
    parser.add_argument('--task', default=0, type=int, help='Task index')
    parser.add_argument('--job-name', default="worker", help='worker or ps')
    parser.add_argument('--num-workers', default=1, type=int, help='Number of workers')
    parser.add_argument('--log-dir', default="/tmp/pong", help='Log directory path')
    parser.add_argument('--env-id', default="PongDeterministic-v3", help='Environment id')
    parser.add_argument('-r', '--remotes', default=None,
                        help='References to environments to create (e.g. -r 20), '
                             'or the address of pre-existing VNC servers and '
                             'rewarders to use (e.g. -r vnc://localhost:5900+15900,vnc://localhost:5901+15901)')

    args = parser.parse_args()
    spec = cluster_spec(args.num_workers, 1)
    cluster = tf.train.ClusterSpec(spec).as_cluster_def()

    universe.utils.exit_on_signal()

    if args.job_name == "worker":
        server = tf.train.Server(cluster, job_name="worker", task_index=args.task,
                                 config=tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=2))
        run(args, server)
    else:
        server = tf.train.Server(cluster, job_name="ps", task_index=args.task,
                                 config=tf.ConfigProto(device_filters=["/job:ps"]))
        while True:
            time.sleep(1000)

if __name__ == "__main__":
    tf.app.run()
<|endoftext|>


<commit_before>from django.contrib import admin
from .models import OurStory, OurTeam, EduProgram
# Register your models here.

class EduProgramAdmin(admin.ModelAdmin):
    fields = ['category', 'order', 'title', 'img', 'color', 'text']
    list_display = ('title','category', 'order')

class OurStoryAdmin(admin.ModelAdmin):
    fields = ['id','order','text']
    list_display=('id', 'order', 'text')

admin.site.register(OurStory, OurStoryAdmin)
admin.site.register(OurTeam)
admin.site.register(EduProgram, EduProgramAdmin)
<commit_msg>admin.py: add OurTeamAdmin<commit_after>from django.contrib import admin
from .models import OurStory, OurTeam, EduProgram
# Register your models here.

class EduProgramAdmin(admin.ModelAdmin):
    fields = ['category', 'order', 'title', 'img', 'color', 'text']
    list_display = ('title','category', 'order')

class OurStoryAdmin(admin.ModelAdmin):
    fields = ['id','order','text']
    list_display=('id', 'order', 'text')

class OurTeamAdmin(admin.ModelAdmin):
    fields = ['id', 'text']
    list_display = ('id', 'text')

admin.site.register(OurStory, OurStoryAdmin)
admin.site.register(OurTeam, OurTeamAdmin)
admin.site.register(EduProgram, EduProgramAdmin)
<|endoftext|>


<commit_before>#!/usr/bin/env python2
import io
import os.path
from lxml import etree
import pyreqif.extractOleData
#import openpyxl
#from openpyxl.drawing.image import Image
import xlsxwriter
from PIL import Image

def write_excel_line(worksheet, item, row, cols, depth, basepath, format):
    for col, value in item.items():
        files = []
        if col not in cols:
            continue
        if type(value) is not bytes:
            value = value.encode("utf8")
        if value is not None:
            if b"<" in value and b">" in value:
                try:
                    tree = etree.parse(io.BytesIO(value))
                    root = tree.getroot()
                    for element in root.iter("object"):
                        rtfFilename = os.path.join(basepath, element.attrib["data"])
                        if rtfFilename.endswith(".ole"):
                            files += pyreqif.extractOleData.extractOleData(rtfFilename)
                        else:
                            files += [rtfFilename]
                        if len(files) > 0 and files[0][-3:].lower() not in ["png","jpeg","jpg","bmp","wmf","emf"]:
                            for key in element.attrib:
                                del element.attrib[key]
                            element.tag = "a"
                            element.set("href", files[0])
                            element.text = "linked file: " + files[0]
                    value = "".join(root.itertext())
                    value = value.encode("utf8")
                except:
                    pass
#        worksheet.cell(row=row, column=cols.index(col)+1). value=value.decode("utf-8")
#        worksheet.row_dimensions[row].outlineLevel = depth
        worksheet.write(row, cols.index(col), value.decode("utf-8"), format)
        worksheet.set_row(row, None, None, {'level': depth})
        last_height = 0
        for file in files:
            if file[-3:].lower() in ["png", "jpeg", "jpg", "bmp", "wmf", "emf"]:
                worksheet.insert_image(row, cols.index(col), file)
                im = Image.open(file)
                im.close()
                _, height = im.size
                worksheet.set_row(row, max(height, last_height))
                last_height = max(height, last_height)

def dump(myDoc, outfile, basepath = None):
    if basepath is None:
        basepath = os.path.dirname(outfile)
    #workbook = openpyxl.Workbook()
    #worksheet = workbook.active
    #workbook.title = "Export"
    workbook = xlsxwriter.Workbook(outfile)
    worksheet = workbook.add_worksheet("Export")

    cell_format = workbook.add_format()
    cell_format.set_text_wrap()

    cols = myDoc.fields
    colNr = 0
#    colNr = 1
    for col in cols:
        worksheet.write(0, colNr, col)
#        worksheet.cell(row=1, column=colNr).value = col
        colNr += 1
    cols = myDoc.fields

    row = 0
#    row = 1

    for child in myDoc.hierarchy:
        for item, depth in  myDoc.hierach_iterator(child, cols):
            row += 1
            write_excel_line(worksheet, item, row, cols, depth, basepath, cell_format)
#    workbook.save(filename=outfile)
    workbook.close()
<commit_msg>support for row sizes as large es image and hopefully support for multiple images #9<commit_after>#!/usr/bin/env python2
import io
import os.path
from lxml import etree
import pyreqif.extractOleData
#import openpyxl
#from openpyxl.drawing.image import Image
import xlsxwriter
from PIL import Image

def write_excel_line(worksheet, item, row, cols, depth, basepath, format):
    max_height = 0

    for col, value in item.items():
        files = []
        if col not in cols:
            continue
        if type(value) is not bytes:
            value = value.encode("utf8")
        if value is not None:
            if b"<" in value and b">" in value:
                try:
                    tree = etree.parse(io.BytesIO(value))
                    root = tree.getroot()
                    for element in root.iter("object"):
                        rtfFilename = os.path.join(basepath, element.attrib["data"])
                        if rtfFilename.endswith(".ole"):
                            files += pyreqif.extractOleData.extractOleData(rtfFilename)
                        else:
                            files += [rtfFilename]
                        if len(files) > 0 and files[0][-3:].lower() not in ["png","jpeg","jpg","bmp","wmf","emf"]:
                            for key in element.attrib:
                                del element.attrib[key]
                            element.tag = "a"
                            element.set("href", files[0])
                            element.text = "linked file: " + files[0]
                    value = "".join(root.itertext())
                    value = value.encode("utf8")
                except:
                    pass
#        worksheet.cell(row=row, column=cols.index(col)+1). value=value.decode("utf-8")
#        worksheet.row_dimensions[row].outlineLevel = depth
        worksheet.write(row, cols.index(col), value.decode("utf-8"))
        for file in files:
            if file[-3:].lower() in ["png", "jpeg", "jpg", "bmp", "wmf", "emf"]:
                im = Image.open(file)
                im.close()
                _, height = im.size
                max_height = max(height, max_height)
                worksheet.set_row(row, max_height, None, {'level': depth})

            if file[-3:].lower() in ["png", "jpeg", "jpg", "bmp", "wmf", "emf"]:
                worksheet.insert_image(row, cols.index(col), file)
    if max_height == 0:
        worksheet.set_row(row, None, format, {'level': depth})

def dump(myDoc, outfile, basepath = None):
    if basepath is None:
        basepath = os.path.dirname(outfile)
    #workbook = openpyxl.Workbook()
    #worksheet = workbook.active
    #workbook.title = "Export"
    workbook = xlsxwriter.Workbook(outfile)
    worksheet = workbook.add_worksheet("Export")

    cell_format = workbook.add_format()
    cell_format.set_text_wrap()

    cols = myDoc.fields
    colNr = 0
#    colNr = 1
    for col in cols:
        worksheet.write(0, colNr, col)
#        worksheet.cell(row=1, column=colNr).value = col
        colNr += 1
    cols = myDoc.fields

    row = 0
#    row = 1

    for child in myDoc.hierarchy:
        for item, depth in  myDoc.hierach_iterator(child, cols):
            row += 1
            write_excel_line(worksheet, item, row, cols, depth, basepath, cell_format)
#    workbook.save(filename=outfile)
    workbook.close()
<|endoftext|>