<commit_before>
import numpy as np
import scipy.ndimage as nd
from signal_id import RadioMask, Noise
from radio_beam import Beam
import astropy.units as u
from astropy.io import fits
from astropy.extern import six
import astropy
from skimage.morphology import reconstruction

'''
Routines for constructing a robust clean mask.

1) Pick two sigma levels, then dilate the higher into the lower.
2) Pick two sigma levels, remove any components in the lower cut if it
   doesn't contain any pixels in the higher cut mask.
'''

class CleanMask(object):
    """
    Creates a robust CLEAN mask.

    Parameters
    ----------
    cube : numpy.ndarray or astropy PrimaryHDU

    low_cut : float or int
        Lower sigma cut.
    high_cut : float or int
        Higher sigma cut.
    beam : Beam
        Object defining the beam.
    pbcoverage : numpy.ndarray
        Defines the beam coverage over the image for mosaics.
    pb_thresh : float
        Defines a threshold between 0 and 1 to remove regions with low beam
        coverage in the image.

    """
    def __init__(self, cube, low_cut, high_cut, beam=None, pbcoverage=None,
                 pb_thresh=0.7):
        super(CleanMask, self).__init__()
        self._cube = cube
        self.low_cut = low_cut
        self.high_cut = high_cut

        if isinstance(beam, Beam):
            self.beam = beam
        elif beam is None:
            self.beam = None
        else:
            raise TypeError("beam must be a Beam object or None.")

        if pbcoverage is not None:
            if isinstance(pbcoverage, six.string_types):
                pbcoverage = fits.getdata(pbcoverage)

            if not isinstance(pbcoverage, np.ndarray):
                raise TypeError("pbcoverage must be a numpy array.")

            if pb_thresh < 0.0 or pb_thresh > 1.0:
                raise Warning("pb_thresh must be between 0 and 1.")

            self.pb_mask = pbcoverage > pb_thresh
            self.pb_flag = True
        else:
            self.pb_mask = np.ones_like(cube, dtype=bool)
            self.pb_flag = False

        self.vel_slices = self.cube.shape[0]  # Generalize with WCS object

        self._low_mask = None
        self._high_mask = None

        self._mask = None

    @property
    def cube(self):
        return Cube(self._cube)

    def make_initial_masks(self, compute_slicewise=False):
        '''
        Calculate the initial masks.
        '''

        if compute_slicewise or self.cube.huge_flag:
            sums = 0.0
            num_finite = 0

            for i in range(self.vel_slices):

                sums += np.nansum(self.cube[i, :, :])

                num_finite += np.isfinite(self.cube[i, :, :]).sum()

            mean = sums / num_finite

            var = 0.0

            for i in range(self.vel_slices):
                plane = self.cube[i, :, :]
                var += np.nansum(np.power(plane - mean, 2), axis=None)

            std = np.sqrt(var) / (num_finite - 1)

            low_thresh = mean + self.low_cut * std
            high_thresh = mean + self.high_cut * std

            self._low_mask = np.zeros_like(self.cube, dtype=bool)
            self._high_mask = np.zeros_like(self.cube, dtype=bool)

            for i in range(self.vel_slices):
                self._low_mask[i, :, :] = self.cube[i, :, :] > low_thresh
                self._high_mask[i, :, :] = self.cube[i, :, :] > high_thresh

        else:
            mean = np.nanmean(self.cube[:])
            std = np.nanstd(self.cube[:])

            low_thresh = mean + self.low_cut * std
            high_thresh = mean + self.high_cut * std

            self._low_mask = self.cube > low_thresh
            self._high_mask = self.cube > high_thresh

        return self

    @property
    def low_mask(self):
        return self._low_mask

    @property
    def high_mask(self):
        return self._high_mask

    @property
    def mask(self):
        return self._mask

    def to_RadioMask(self, which_mask='final'):

        if which_mask is 'final':
            return RadioMask(self._mask, wcs=None)  # Load in WCS somehow

        elif which_mask is 'low':
            return RadioMask(self._low_mask, wcs=None)  # Load in WCS somehow

        elif which_mask is 'high':
            return RadioMask(self._high_mask, wcs=None)  # Load in WCS somehow

        else:
            raise TypeError("which_mask must be 'final', 'low', or 'high'.")

    def dilate_into_low(self, verbose=False):
        '''
        Dilates the high mask into the low using morphological reconstruction.
        '''

        dilate_struct = nd.generate_binary_structure(2, 3)

        for i in range(self.vel_slices):

            # Skip empty channels
            if self._high_mask[i, :, :].max() is False:
                continue

            if verbose:
                print "Iteration %s of %s" % (str(i+1), self.vel_slices)

            self.high_mask[i, :, :] = \
                reconstruction(self.high_mask[i, :, :],
                               self.low_mask[i, :, :], selem=dilate_struct)

        self._mask = self._high_mask

        return self

    def remove_high_components(self, min_pix=10, beam_check=False,
                               pixscale=None, verbose=False):
        '''
        Remove components in the low mask which are not
        contained in the high mask.

        The criteria is set by min_pix, or is based off of the beam area.
        Note that if min_pix < beam area, min_pix has no effect.
        '''

        # 8-connectivity
        connect = np.ones((3, 3))

        # Objects must be at least the beam area to be kept.
        if beam_check:

            # Remove this when WCS object is added.
            if pixscale is None:
                raise TypeError("pixscale must be specified to use beamarea")

            major = self.major.to(u.deg).value/pixscale
            minor = self.minor.to(u.deg).value/pixscale

            # Round down by default?
            # Should this be made into an optional input?
            beam_pix_area = np.floor(np.pi * major * minor)

        else:
            beam_pix_area = 0

        for i in range(self.vel_slices):

            if verbose:
                print "Iteration %s of %s" % (str(i+1), self.vel_slices)

            # Skip empty channels
            if self.high_mask[i, :, :].max is False:
                continue

            low_labels, low_num = nd.label(self._low_mask[i, :, :], connect)

            for j in range(1, low_num+1):

                low_pix = zip(*np.where(low_labels == j))

                high_pix = zip(*np.where(self._high_mask[i, :, :] > 0))

                # Now check for overlap

                matches = list(set(low_pix) & set(high_pix))

                # Add in some check to make sure region is at least the beam size.
                if len(matches) >= min_pix:
                    continue

                if len(matches) > beam_pix_area:
                    continue

                x_pos = [x for x, y in low_pix]
                y_pos = [y for x, y in low_pix]

                # If less than match threshold, remove region in the low mask
                self._low_mask[i, :, :][x_pos, y_pos] = 0

        self._mask = self._low_mask

        return self

    def _smooth_it(self, kern_size='beam', pixscale=None):
        '''
        Apply median filter to smooth the edges of the mask.
        '''

        if kern_size is 'beam':
            if pixscale is None:
                raise TypeError("pixscale must be specified to use beamarea")

            footprint = self.beam.as_tophat_kernel(pixscale)

        elif isinstance(kern_size, float) or isinstance(kern_size, int):
            major = kern_size
            minor = kern_size
            footprint = np.ones((major, minor))
        else:
            Warning("kern_size must be 'beam', or a float or integer.")

        from scipy.ndimage import median_filter

        for i in range(self.vel_slices):
            self._mask[i, :, :] = \
                median_filter(self._mask[i, :, :],
                              footprint=footprint)

        return self

    def save_to_fits(self, filename, header=None, append_comments=True):
        '''
        Save the final mask as a FITS file. Optionally append the parameters
        used to create the mask.
        '''
        pass

    def make_mask(self, method="dilate", compute_slicewise=False,
                  smooth=False, kern_size='beam', pixscale=None,
                  verbose=False):

        self.make_initial_masks(compute_slicewise=compute_slicewise)

        if method == "dilate":
            self.dilate_into_low(verbose=verbose)
        elif method == "remove small":
            self.remove_high_components(pixscale=pixscale, verbose=verbose)
        else:
            raise TypeError("method must be 'dilate' or 'remove small'.")

        if smooth:
            self._smooth_it(kern_size=kern_size, pixscale=pixscale)

        if self.pb_flag:
            self._mask *= self.pb_mask

        return self

class Cube(object):
    """
    Cube attempts to handle numpy arrays and FITS HDUs transparently. This
    is useful for massive datasets, in particular. The data is loaded in only
    for the requested slice.

    It is certainly *NOT* robust or complete, but handles what is needed for
    creating CLEAN masks.
    """
    def __init__(self, cube, huge_flag=None, huge_thresh=1e8,
                 squeeze=True):

        self.cube = cube

        if huge_flag is not None:
            self.huge_flag = huge_flag
        else:
            self.huge_flag = self.size > huge_thresh

    @property
    def cube(self):
        return self._cube

    @cube.setter
    def cube(self, input_cube):

        if isinstance(input_cube, six.string_types):
            input_cube = self._load_fits(input_cube)

        is_array = isinstance(input_cube, np.ndarray)
        is_hdu = isinstance(input_cube, astropy.io.fits.hdu.image.PrimaryHDU)

        if not is_array and not is_hdu:
            raise TypeError("cube must be a numpy array or an astropy "
                            "PrimaryHDU. Input was of type " +
                            str(type(input_cube)))

        self._cube = input_cube

    def __getitem__(self, view):
        if self.is_hdu:
            return self.cube.data[view]
        else:
            return self.cube[view]

    def _load_fits(self, fitsfile, ext=0):
        return fits.open(fitsfile)[ext]

    def _is_hdu(self):
        if hasattr(self.cube, 'header'):
            return True
        return False

    @property
    def is_hdu(self):
        return self._is_hdu()

    @property
    def shape(self):
        if self.is_hdu:
            return self.cube.data.shape
        else:
            return self.cube.shape

    @property
    def size(self):
        if self.is_hdu:
            return self.cube.data.size
        else:
            return self.cube.size

    def close(self):
        '''
        If an HDU, close it.
        '''
        if self.is_hdu:
            self.cube.close()

    def __gt__(self, value):
        return self[:] > value

    def __lt__(self, value):
        return self[:] < value

    def __ge__(self, value):
        return self[:] >= value

    def __le__(self, value):
        return self[:] <= value
<commit_msg>Don't need checks here<commit_after>
import numpy as np
import scipy.ndimage as nd
from signal_id import RadioMask, Noise
from radio_beam import Beam
import astropy.units as u
from astropy.io import fits
from astropy.extern import six
import astropy
from skimage.morphology import reconstruction

'''
Routines for constructing a robust clean mask.

1) Pick two sigma levels, then dilate the higher into the lower.
2) Pick two sigma levels, remove any components in the lower cut if it
   doesn't contain any pixels in the higher cut mask.
'''

class CleanMask(object):
    """
    Creates a robust CLEAN mask.

    Parameters
    ----------
    cube : numpy.ndarray or astropy PrimaryHDU

    low_cut : float or int
        Lower sigma cut.
    high_cut : float or int
        Higher sigma cut.
    beam : Beam
        Object defining the beam.
    pbcoverage : numpy.ndarray
        Defines the beam coverage over the image for mosaics.
    pb_thresh : float
        Defines a threshold between 0 and 1 to remove regions with low beam
        coverage in the image.

    """
    def __init__(self, cube, low_cut, high_cut, beam=None, pbcoverage=None,
                 pb_thresh=0.7):
        super(CleanMask, self).__init__()
        self._cube = cube
        self.low_cut = low_cut
        self.high_cut = high_cut

        if isinstance(beam, Beam):
            self.beam = beam
        elif beam is None:
            self.beam = None
        else:
            raise TypeError("beam must be a Beam object or None.")

        if pbcoverage is not None:
            if isinstance(pbcoverage, six.string_types):
                pbcoverage = fits.getdata(pbcoverage)

            if not isinstance(pbcoverage, np.ndarray):
                raise TypeError("pbcoverage must be a numpy array.")

            if pb_thresh < 0.0 or pb_thresh > 1.0:
                raise Warning("pb_thresh must be between 0 and 1.")

            self.pb_mask = pbcoverage > pb_thresh
            self.pb_flag = True
        else:
            self.pb_mask = np.ones_like(cube, dtype=bool)
            self.pb_flag = False

        self.vel_slices = self.cube.shape[0]  # Generalize with WCS object

        self._low_mask = None
        self._high_mask = None

        self._mask = None

    @property
    def cube(self):
        return Cube(self._cube)

    def make_initial_masks(self, compute_slicewise=False):
        '''
        Calculate the initial masks.
        '''

        if compute_slicewise or self.cube.huge_flag:
            sums = 0.0
            num_finite = 0

            for i in range(self.vel_slices):

                sums += np.nansum(self.cube[i, :, :])

                num_finite += np.isfinite(self.cube[i, :, :]).sum()

            mean = sums / num_finite

            var = 0.0

            for i in range(self.vel_slices):
                plane = self.cube[i, :, :]
                var += np.nansum(np.power(plane - mean, 2), axis=None)

            std = np.sqrt(var) / (num_finite - 1)

            low_thresh = mean + self.low_cut * std
            high_thresh = mean + self.high_cut * std

            self._low_mask = np.zeros_like(self.cube, dtype=bool)
            self._high_mask = np.zeros_like(self.cube, dtype=bool)

            for i in range(self.vel_slices):
                self._low_mask[i, :, :] = self.cube[i, :, :] > low_thresh
                self._high_mask[i, :, :] = self.cube[i, :, :] > high_thresh

        else:
            mean = np.nanmean(self.cube[:])
            std = np.nanstd(self.cube[:])

            low_thresh = mean + self.low_cut * std
            high_thresh = mean + self.high_cut * std

            self._low_mask = self.cube > low_thresh
            self._high_mask = self.cube > high_thresh

        return self

    @property
    def low_mask(self):
        return self._low_mask

    @property
    def high_mask(self):
        return self._high_mask

    @property
    def mask(self):
        return self._mask

    def to_RadioMask(self, which_mask='final'):

        if which_mask is 'final':
            return RadioMask(self._mask, wcs=None)  # Load in WCS somehow

        elif which_mask is 'low':
            return RadioMask(self._low_mask, wcs=None)  # Load in WCS somehow

        elif which_mask is 'high':
            return RadioMask(self._high_mask, wcs=None)  # Load in WCS somehow

        else:
            raise TypeError("which_mask must be 'final', 'low', or 'high'.")

    def dilate_into_low(self, verbose=False):
        '''
        Dilates the high mask into the low using morphological reconstruction.
        '''

        dilate_struct = nd.generate_binary_structure(2, 3)

        for i in range(self.vel_slices):

            # Skip empty channels
            if self._high_mask[i, :, :].max() is False:
                continue

            if verbose:
                print "Iteration %s of %s" % (str(i+1), self.vel_slices)

            self.high_mask[i, :, :] = \
                reconstruction(self.high_mask[i, :, :],
                               self.low_mask[i, :, :], selem=dilate_struct)

        self._mask = self._high_mask

        return self

    def remove_high_components(self, min_pix=10, beam_check=False,
                               pixscale=None, verbose=False):
        '''
        Remove components in the low mask which are not
        contained in the high mask.

        The criteria is set by min_pix, or is based off of the beam area.
        Note that if min_pix < beam area, min_pix has no effect.
        '''

        # 8-connectivity
        connect = np.ones((3, 3))

        # Objects must be at least the beam area to be kept.
        if beam_check:

            # Remove this when WCS object is added.
            if pixscale is None:
                raise TypeError("pixscale must be specified to use beamarea")

            major = self.major.to(u.deg).value/pixscale
            minor = self.minor.to(u.deg).value/pixscale

            # Round down by default?
            # Should this be made into an optional input?
            beam_pix_area = np.floor(np.pi * major * minor)

        else:
            beam_pix_area = 0

        for i in range(self.vel_slices):

            if verbose:
                print "Iteration %s of %s" % (str(i+1), self.vel_slices)

            # Skip empty channels
            if self.high_mask[i, :, :].max is False:
                continue

            low_labels, low_num = nd.label(self._low_mask[i, :, :], connect)

            for j in range(1, low_num+1):

                low_pix = zip(*np.where(low_labels == j))

                high_pix = zip(*np.where(self._high_mask[i, :, :] > 0))

                # Now check for overlap

                matches = list(set(low_pix) & set(high_pix))

                # Add in some check to make sure region is at least the beam size.
                if len(matches) >= min_pix:
                    continue

                if len(matches) > beam_pix_area:
                    continue

                x_pos = [x for x, y in low_pix]
                y_pos = [y for x, y in low_pix]

                # If less than match threshold, remove region in the low mask
                self._low_mask[i, :, :][x_pos, y_pos] = 0

        self._mask = self._low_mask

        return self

    def _smooth_it(self, kern_size='beam', pixscale=None):
        '''
        Apply median filter to smooth the edges of the mask.
        '''

        if kern_size is 'beam':
            if pixscale is None:
                raise TypeError("pixscale must be specified to use beamarea")

            footprint = self.beam.as_tophat_kernel(pixscale)

        elif isinstance(kern_size, float) or isinstance(kern_size, int):
            major = kern_size
            minor = kern_size
            footprint = np.ones((major, minor))
        else:
            Warning("kern_size must be 'beam', or a float or integer.")

        from scipy.ndimage import median_filter

        for i in range(self.vel_slices):
            self._mask[i, :, :] = \
                median_filter(self._mask[i, :, :],
                              footprint=footprint)

        return self

    def save_to_fits(self, filename, header=None, append_comments=True):
        '''
        Save the final mask as a FITS file. Optionally append the parameters
        used to create the mask.
        '''
        pass

    def make_mask(self, method="dilate", compute_slicewise=False,
                  smooth=False, kern_size='beam', pixscale=None,
                  verbose=False):

        self.make_initial_masks(compute_slicewise=compute_slicewise)

        if method == "dilate":
            self.dilate_into_low(verbose=verbose)
        elif method == "remove small":
            self.remove_high_components(pixscale=pixscale, verbose=verbose)
        else:
            raise TypeError("method must be 'dilate' or 'remove small'.")

        if smooth:
            self._smooth_it(kern_size=kern_size, pixscale=pixscale)

        if self.pb_flag:
            self._mask *= self.pb_mask

        return self

class Cube(object):
    """
    Cube attempts to handle numpy arrays and FITS HDUs transparently. This
    is useful for massive datasets, in particular. The data is loaded in only
    for the requested slice.

    It is certainly *NOT* robust or complete, but handles what is needed for
    creating CLEAN masks.
    """
    def __init__(self, cube, huge_flag=None, huge_thresh=1e8,
                 squeeze=True):

        self.cube = cube

        if huge_flag is not None:
            self.huge_flag = huge_flag
        else:
            self.huge_flag = self.size > huge_thresh

    @property
    def cube(self):
        return self._cube

    @cube.setter
    def cube(self, input_cube):

        if isinstance(input_cube, six.string_types):
            input_cube = self._load_fits(input_cube)

        is_array = isinstance(input_cube, np.ndarray)
        is_hdu = isinstance(input_cube, astropy.io.fits.hdu.image.PrimaryHDU)

        if not is_array and not is_hdu:
            raise TypeError("cube must be a numpy array or an astropy "
                            "PrimaryHDU. Input was of type " +
                            str(type(input_cube)))

        self._cube = input_cube

    def __getitem__(self, view):
        if self.is_hdu:
            return self.cube.data[view]
        else:
            return self.cube[view]

    def _load_fits(self, fitsfile, ext=0):
        return fits.open(fitsfile)[ext]

    def _is_hdu(self):
        if hasattr(self.cube, 'header'):
            return True
        return False

    @property
    def is_hdu(self):
        return self._is_hdu()

    @property
    def shape(self):
        return self.cube.shape

    @property
    def size(self):
        return self.cube.size

    def close(self):
        '''
        If an HDU, close it.
        '''
        if self.is_hdu:
            self.cube.close()

    def __gt__(self, value):
        return self[:] > value

    def __lt__(self, value):
        return self[:] < value

    def __ge__(self, value):
        return self[:] >= value

    def __le__(self, value):
        return self[:] <= value
<|endoftext|>


<commit_before>import pandas as pd
import numpy as np
import datetime

from . import SignalDecomposition_Time as tsti
from . import SignalDecomposition_Perf as tsperf
from . import SignalDecomposition_Plots as tsplot

import sklearn as skl
import sklearn.preprocessing as preprocessing
import sklearn.linear_model as linear_model
from sklearn.feature_selection import RFE
from sklearn.linear_model import LinearRegression

class cAbstractAR:
    def __init__(self , cycle_residue_name):
        self.mTimeInfo = tsti.cTimeInfo()
        self.mCycleFrame = pd.DataFrame()
        self.mARFrame = pd.DataFrame()
        self.mCycleResidueName = cycle_residue_name

    def plot(self):
        tsplot.decomp_plot(self.mARFrame, self.mTimeInfo.mNormalizedTimeColumn,
                           self.mCycleResidueName, self.mOutName , self.mOutName + '_residue');

    def computePerf(self):
        self.mARFitPerf= tsperf.cPerf();
        self.mARForecastPerf= tsperf.cPerf();
        (lFrameFit, lFrameForecast, lFrameTest) = self.mTimeInfo.cutFrame(self.mARFrame);
        self.mARFitPerf.compute(
            lFrameFit[self.mCycleResidueName], lFrameFit[self.mOutName], self.mOutName)
        self.mARForecastPerf.compute(
            lFrameForecast[self.mCycleResidueName], lFrameForecast[self.mOutName], self.mOutName)

class cZeroAR(cAbstractAR):
    def __init__(self , cycle_residue_name):
        super().__init__(cycle_residue_name)
        self.mOutName = self.mCycleResidueName +  '_NoAR'
        self.mFormula = "NoAR";

    def fit(self):
        series = self.mCycleResidueName;
        self.mTime = self.mTimeInfo.mTime;
        self.mSignal = self.mTimeInfo.mSignal;
        self.mTimeInfo.addVars(self.mARFrame);
        self.mARFrame[series] = self.mCycleFrame[series]
        self.mARFrame[self.mOutName] = self.mARFrame[series] * 0.0;
        self.mARFrame[self.mOutName + '_residue'] = self.mARFrame[series];

    def transformDataset(self, df):
        series = self.mCycleResidueName;
        df[self.mOutName] = 0.0;
        target = df[series].values
        df[self.mOutName + '_residue'] = target - df[self.mOutName].values
        return df;

class cAutoRegressiveModel(cAbstractAR):
    def __init__(self , cycle_residue_name):
        super().__init__(cycle_residue_name)
        self.mARRidge = linear_model.Ridge()
        self.mARLagNames = [];

    def generateLags(self, df, P):
        lag_df = pd.DataFrame()
        series = self.mCycleResidueName;
        lag_df[series] = df[series]
        for i in range(1,P):
            name = series+'_Lag' + str(i);
            lag_df[name] = df[series].shift(i)
        return lag_df;

    def fit(self):
        series = self.mCycleResidueName;
        self.mTime = self.mTimeInfo.mTime;
        self.mSignal = self.mTimeInfo.mSignal;
        self.mNbLags = int(self.mCycleFrame.shape[0] / 4)
        if(self.mNbLags > self.mOptions.mMaxAROrder):
            self.mNbLags = self.mOptions.mMaxAROrder;
        self.mOutName = self.mCycleResidueName +  '_AR(' + str(self.mNbLags) + ")";
        self.mARFrame = self.generateLags(self.mCycleFrame, self.mNbLags);
        self.mAREstimFrame = self.mTimeInfo.getEstimPart(self.mARFrame)
        self.mDefaultValue = self.mAREstimFrame[series].mean()
        self.mARFrame.fillna(self.mDefaultValue , inplace=True)
        self.mAREstimFrame.fillna(self.mDefaultValue , inplace=True)
        self.mARLagNames = self.mARFrame.columns;

        lARInputs = self.mAREstimFrame.drop([series]  , axis=1).values
        lARTarget = self.mAREstimFrame[series].values
        self.mARRidge.fit(lARInputs, lARTarget)

        lARInputsFull = self.mARFrame.drop([series]  , axis=1).values
        self.mARFrame[self.mOutName] = self.mARRidge.predict(lARInputsFull)
        self.mARFrame[self.mOutName + '_residue'] =  self.mARFrame[series] - self.mARFrame[self.mOutName]
        self.mFormula = "AR(" + str(self.mNbLags) + ")";

    def transformDataset(self, df):
        series = self.mCycleResidueName;
        lag_df = self.generateLags(df, self.mNbLags);
        lag_df.fillna(self.mDefaultValue , inplace=True)
        inputs = lag_df.drop([series]  , axis=1).values
        pred = self.mARRidge.predict(inputs)
        df[self.mOutName] = pred;
        target = df[series].values
        df[self.mOutName + '_residue'] = target - df[self.mOutName].values
        return df;

class cAutoRegressiveEstimator:
    def __init__(self):
        self.mTimeInfo = tsti.cTimeInfo()
        self.mCycleFrame = pd.DataFrame()
        self.mARFrame = pd.DataFrame()
        self.mARList = {}

    def plotAR(self):
        for trend in self.mTrendList:
            for cycle in self.mCycleList[trend]:
                cycle_residue = cycle.getCycleResidueName();
                for autoreg in self.mARList[cycle_residue]:
                    autoreg.plot();

    def estimate(self):
        mARList = {}
        for trend in self.mTrendList:
            for cycle in self.mCycleList[trend]:
                cycle_residue = cycle.getCycleResidueName();
                self.mARList[cycle_residue] = [ cZeroAR(cycle_residue)];
                if(self.mOptions.mEnableARModels):
                    if((self.mCycleFrame[cycle_residue].shape[0] > 12) and (self.mCycleFrame[cycle_residue].std() > 0.00001)):
                        self.mARList[cycle_residue] = self.mARList[cycle_residue] + [cAutoRegressiveModel(cycle_residue)];
                for autoreg in self.mARList[cycle_residue]:
                    autoreg.mOptions = self.mOptions;
                    autoreg.mCycleFrame = self.mCycleFrame;
                    autoreg.mTimeInfo = self.mTimeInfo;
                    autoreg.fit();
                    autoreg.computePerf();

<commit_msg>Corrected lag names.<commit_after>import pandas as pd
import numpy as np
import datetime

from . import SignalDecomposition_Time as tsti
from . import SignalDecomposition_Perf as tsperf
from . import SignalDecomposition_Plots as tsplot

import sklearn as skl
import sklearn.preprocessing as preprocessing
import sklearn.linear_model as linear_model
from sklearn.feature_selection import RFE
from sklearn.linear_model import LinearRegression

class cAbstractAR:
    def __init__(self , cycle_residue_name):
        self.mTimeInfo = tsti.cTimeInfo()
        self.mCycleFrame = pd.DataFrame()
        self.mARFrame = pd.DataFrame()
        self.mCycleResidueName = cycle_residue_name

    def plot(self):
        tsplot.decomp_plot(self.mARFrame, self.mTimeInfo.mNormalizedTimeColumn,
                           self.mCycleResidueName, self.mOutName , self.mOutName + '_residue');

    def computePerf(self):
        self.mARFitPerf= tsperf.cPerf();
        self.mARForecastPerf= tsperf.cPerf();
        (lFrameFit, lFrameForecast, lFrameTest) = self.mTimeInfo.cutFrame(self.mARFrame);
        self.mARFitPerf.compute(
            lFrameFit[self.mCycleResidueName], lFrameFit[self.mOutName], self.mOutName)
        self.mARForecastPerf.compute(
            lFrameForecast[self.mCycleResidueName], lFrameForecast[self.mOutName], self.mOutName)

class cZeroAR(cAbstractAR):
    def __init__(self , cycle_residue_name):
        super().__init__(cycle_residue_name)
        self.mOutName = self.mCycleResidueName +  '_NoAR'
        self.mFormula = "NoAR";

    def fit(self):
        series = self.mCycleResidueName;
        self.mTime = self.mTimeInfo.mTime;
        self.mSignal = self.mTimeInfo.mSignal;
        self.mTimeInfo.addVars(self.mARFrame);
        self.mARFrame[series] = self.mCycleFrame[series]
        self.mARFrame[self.mOutName] = self.mARFrame[series] * 0.0;
        self.mARFrame[self.mOutName + '_residue'] = self.mARFrame[series];

    def transformDataset(self, df):
        series = self.mCycleResidueName;
        df[self.mOutName] = 0.0;
        target = df[series].values
        df[self.mOutName + '_residue'] = target - df[self.mOutName].values
        return df;

class cAutoRegressiveModel(cAbstractAR):
    def __init__(self , cycle_residue_name):
        super().__init__(cycle_residue_name)
        self.mARRidge = linear_model.Ridge()
        self.mARLagNames = [];

    def generateLags(self, df, P):
        lag_df = pd.DataFrame()
        series = self.mCycleResidueName;
        lag_df[series] = df[series]
        for i in range(1,P):
            name = series+'_Lag' + str(i);
            lag_df[name] = df[series].shift(i)
        return lag_df;

    def fit(self):
        series = self.mCycleResidueName;
        self.mTime = self.mTimeInfo.mTime;
        self.mSignal = self.mTimeInfo.mSignal;
        self.mNbLags = int(self.mCycleFrame.shape[0] / 4)
        if(self.mNbLags > self.mOptions.mMaxAROrder):
            self.mNbLags = self.mOptions.mMaxAROrder;
        self.mOutName = self.mCycleResidueName +  '_AR(' + str(self.mNbLags) + ")";
        self.mARFrame = self.generateLags(self.mCycleFrame, self.mNbLags);
        self.mAREstimFrame = self.mTimeInfo.getEstimPart(self.mARFrame)
        self.mDefaultValue = self.mAREstimFrame[series].mean()
        self.mARFrame.fillna(self.mDefaultValue , inplace=True)
        self.mAREstimFrame.fillna(self.mDefaultValue , inplace=True)
        self.mARLagNames = self.mARFrame.columns[1:];

        lARInputs = self.mAREstimFrame.drop([series]  , axis=1).values
        lARTarget = self.mAREstimFrame[series].values
        self.mARRidge.fit(lARInputs, lARTarget)

        lARInputsFull = self.mARFrame.drop([series]  , axis=1).values
        self.mARFrame[self.mOutName] = self.mARRidge.predict(lARInputsFull)
        self.mARFrame[self.mOutName + '_residue'] =  self.mARFrame[series] - self.mARFrame[self.mOutName]
        self.mFormula = "AR(" + str(self.mNbLags) + ")";

    def transformDataset(self, df):
        series = self.mCycleResidueName;
        lag_df = self.generateLags(df, self.mNbLags);
        lag_df.fillna(self.mDefaultValue , inplace=True)
        inputs = lag_df.drop([series]  , axis=1).values
        pred = self.mARRidge.predict(inputs)
        df[self.mOutName] = pred;
        target = df[series].values
        df[self.mOutName + '_residue'] = target - df[self.mOutName].values
        return df;

class cAutoRegressiveEstimator:
    def __init__(self):
        self.mTimeInfo = tsti.cTimeInfo()
        self.mCycleFrame = pd.DataFrame()
        self.mARFrame = pd.DataFrame()
        self.mARList = {}

    def plotAR(self):
        for trend in self.mTrendList:
            for cycle in self.mCycleList[trend]:
                cycle_residue = cycle.getCycleResidueName();
                for autoreg in self.mARList[cycle_residue]:
                    autoreg.plot();

    def estimate(self):
        mARList = {}
        for trend in self.mTrendList:
            for cycle in self.mCycleList[trend]:
                cycle_residue = cycle.getCycleResidueName();
                self.mARList[cycle_residue] = [ cZeroAR(cycle_residue)];
                if(self.mOptions.mEnableARModels):
                    if((self.mCycleFrame[cycle_residue].shape[0] > 12) and (self.mCycleFrame[cycle_residue].std() > 0.00001)):
                        self.mARList[cycle_residue] = self.mARList[cycle_residue] + [cAutoRegressiveModel(cycle_residue)];
                for autoreg in self.mARList[cycle_residue]:
                    autoreg.mOptions = self.mOptions;
                    autoreg.mCycleFrame = self.mCycleFrame;
                    autoreg.mTimeInfo = self.mTimeInfo;
                    autoreg.fit();
                    autoreg.computePerf();

<|endoftext|>


<commit_before>import xbmc
import xbmcaddon
import xbmcgui
import resources.lib.vfs as vfs
import os

class FileManager:
    walk_path = ''
    addonDir = ''
    fileArray = None

    def __init__(self,path,addon_dir):
        self.walk_path = path
        self.addonDir = addon_dir

        #create the addon folder if it doesn't exist
        if(not os.path.exists(unicode(xbmc.translatePath(self.addonDir),'utf-8'))):
            os.makedirs(unicode(xbmc.translatePath(self.addonDir),'utf-8'))

    def createFileList(self,Addon):
        self.fileArray = []

        #figure out which syncing options to run
        if(Addon.getSetting('backup_addons') == 'true'):
            self.addFile("-addons")
            self.walkTree(self.walk_path + "addons/")

        self.addFile("-userdata")

        if(Addon.getSetting('backup_addon_data') == 'true'):
            self.addFile("-userdata/addon_data")
            self.walkTree(self.walk_path + "userdata/addon_data/")

        if(Addon.getSetting('backup_database') == 'true'):
	    self.addFile("-userdata/Database")
            self.walkTree(self.walk_path + "userdata/Database")

        if(Addon.getSetting("backup_playlists") == 'true'):
	    self.addFile("-userdata/playlists")
	    self.walkTree(self.walk_path + "userdata/playlists")

        if(Addon.getSetting("backup_thumbnails") == "true"):
	    self.addFile("-userdata/Thumbnails")
	    self.walkTree(self.walk_path + "userdata/Thumbnails")

        if(Addon.getSetting("backup_config") == "true"):
            self.addFile("-userdata/keymaps")
            self.walkTree(self.walk_path + "userdata/keymaps")

            self.addFile("-userdata/peripheral_data")
            self.walkTree(self.walk_path + "userdata/peripheral_data")

	    #this part is an oddity
            configFiles = vfs.listdir(self.walk_path + "userdata/",extra_metadata=True)
	    for aFile in configFiles:
		if(aFile['file'].endswith(".xml")):
		    self.addFile("userdata/" + aFile['file'])

    def walkTree(self,directory):
        for (path, dirs, files) in vfs.walk(directory):

            #create all the subdirs first
            for aDir in dirs:
                self.addFile("-" + aDir[len(self.walk_path):])
            #copy all the files
            for aFile in files:
                filePath = aFile[len(self.walk_path):]
                self.addFile(filePath)

    def addFile(self,filename):
        #write the full remote path name of this file
        self.fileArray.append(filename)

    def getFileList(self):
       return self.fileArray

class XbmcBackup:
    __addon_id__ = 'script.xbmcbackup'
    Addon = xbmcaddon.Addon(__addon_id__)
    local_path = ''
    remote_path = ''
    restoreFile = None

    #for the progress bar
    progressBar = None
    filesLeft = 0
    filesTotal = 1

    fileManager = None

    def __init__(self):
        self.local_path = xbmc.translatePath("special://home")

	if(self.Addon.getSetting('remote_selection') == '1' and vfs.exists(self.Addon.getSetting('remote_path_2'))):
            xbmc.log(str(self.Addon.getSetting('remote_path_2')))
	    self.remote_path = self.Addon.getSetting('remote_path_2')
	    self.Addon.setSetting("remote_path","")
        elif(self.Addon.getSetting('remote_selection') == '0' and vfs.exists(self.Addon.getSetting("remote_path"))):
            self.remote_path = self.Addon.getSetting("remote_path")

	if(self.Addon.getSetting("backup_name") != '' and self.remote_path != ''):
	    self.remote_path = self.remote_path + self.Addon.getSetting("backup_name") + "/"
	else:
	    self.remote_path = ""

        self.log("Starting")
        self.log('Local Dir: ' + self.local_path)
        self.log('Remote Dir: ' + self.remote_path)

    def run(self):
	#check if we should use the progress bar
        if(self.Addon.getSetting('run_silent') == 'false'):
            self.progressBar = xbmcgui.DialogProgress()
            self.progressBar.create('XBMC Backup','Gathering file list.....')

        #check what mode were are in
        if(int(self.Addon.getSetting('addon_mode')) == 0):
            self.fileManager = FileManager(self.local_path,self.Addon.getAddonInfo('profile'))
            self.syncFiles()
        else:
            self.fileManager = FileManager(self.remote_path,self.Addon.getAddonInfo('profile'))
            self.restoreFiles()

    def syncFiles(self):
        if(vfs.exists(self.remote_path)):
            #this will fail - need a disclaimer here
            self.log("Remote Path exists - may have old files in it!")

        #make the remote directory
        vfs.mkdir(self.remote_path)

        self.fileManager.createFileList(self.Addon)

        allFiles = self.fileManager.getFileList()

        #write list from local to remote
        self.writeFiles(allFiles,self.local_path,self.remote_path)

    def restoreFiles(self):
        self.fileManager.createFileList(self.Addon)

        allFiles = self.fileManager.getFileList()

        #write list from remote to local
        self.writeFiles(allFiles,self.remote_path,self.local_path)

        #call update addons to refresh everything
        xbmc.executebuiltin('UpdateLocalAddons')

    def writeFiles(self,fileList,source,dest):
        self.filesTotal = len(fileList)
        self.filesLeft = self.filesTotal

        #write each file from source to destination
        for aFile in fileList:
            if(not self.checkCancel()):
                self.updateProgress(aFile)
                if (aFile.startswith("-")):
                    vfs.mkdir(dest + aFile[1:])
                else:
                    vfs.copy(source + aFile,dest + aFile)

        if(self.Addon.getSetting('run_silent') == 'false'):
            self.progressBar.close()

    def updateProgress(self,message=''):
        self.filesLeft = self.filesLeft - 1

        #update the progress bar
        if(self.progressBar != None):
            self.progressBar.update(int((float(self.filesTotal - self.filesLeft)/float(self.filesTotal)) * 100),message)

    def checkCancel(self):
        result = False

        if(self.progressBar != None):
            result = self.progressBar.iscanceled()

        return result

    def log(self,message):
        xbmc.log(self.__addon_id__ + ": " + message)

    def isReady(self):
        return True if self.remote_path != '' else False

#run the profile backup
backup = XbmcBackup()

if(backup.isReady()):
    backup.run()
else:
    xbmcgui.Dialog().ok('XBMC Backup','Error: Remote path cannot be empty')
<commit_msg>when using vfs module need to cut off full file path<commit_after>import xbmc
import xbmcaddon
import xbmcgui
import resources.lib.vfs as vfs
import os

class FileManager:
    walk_path = ''
    addonDir = ''
    fileArray = None

    def __init__(self,path,addon_dir):
        self.walk_path = path
        self.addonDir = addon_dir

        #create the addon folder if it doesn't exist
        if(not os.path.exists(unicode(xbmc.translatePath(self.addonDir),'utf-8'))):
            os.makedirs(unicode(xbmc.translatePath(self.addonDir),'utf-8'))

    def createFileList(self,Addon):
        self.fileArray = []

        #figure out which syncing options to run
        if(Addon.getSetting('backup_addons') == 'true'):
            self.addFile("-addons")
            self.walkTree(self.walk_path + "addons/")

        self.addFile("-userdata")

        if(Addon.getSetting('backup_addon_data') == 'true'):
            self.addFile("-userdata/addon_data")
            self.walkTree(self.walk_path + "userdata/addon_data/")

        if(Addon.getSetting('backup_database') == 'true'):
	    self.addFile("-userdata/Database")
            self.walkTree(self.walk_path + "userdata/Database")

        if(Addon.getSetting("backup_playlists") == 'true'):
	    self.addFile("-userdata/playlists")
	    self.walkTree(self.walk_path + "userdata/playlists")

        if(Addon.getSetting("backup_thumbnails") == "true"):
	    self.addFile("-userdata/Thumbnails")
	    self.walkTree(self.walk_path + "userdata/Thumbnails")

        if(Addon.getSetting("backup_config") == "true"):
            self.addFile("-userdata/keymaps")
            self.walkTree(self.walk_path + "userdata/keymaps")

            self.addFile("-userdata/peripheral_data")
            self.walkTree(self.walk_path + "userdata/peripheral_data")

	    #this part is an oddity
            configFiles = vfs.listdir(self.walk_path + "userdata/",extra_metadata=True)
	    for aFile in configFiles:
                xbmc.log(aFile['file'][len(self.walk_path):])
		if(aFile['file'].endswith(".xml")):
		    self.addFile(aFile['file'][len(self.walk_path):])

    def walkTree(self,directory):
        for (path, dirs, files) in vfs.walk(directory):

            #create all the subdirs first
            for aDir in dirs:
                self.addFile("-" + aDir[len(self.walk_path):])
            #copy all the files
            for aFile in files:
                filePath = aFile[len(self.walk_path):]
                self.addFile(filePath)

    def addFile(self,filename):
        #write the full remote path name of this file
        self.fileArray.append(filename)

    def getFileList(self):
       return self.fileArray

class XbmcBackup:
    __addon_id__ = 'script.xbmcbackup'
    Addon = xbmcaddon.Addon(__addon_id__)
    local_path = ''
    remote_path = ''
    restoreFile = None

    #for the progress bar
    progressBar = None
    filesLeft = 0
    filesTotal = 1

    fileManager = None

    def __init__(self):
        self.local_path = xbmc.translatePath("special://home")

	if(self.Addon.getSetting('remote_selection') == '1' and vfs.exists(self.Addon.getSetting('remote_path_2'))):
            xbmc.log(str(self.Addon.getSetting('remote_path_2')))
	    self.remote_path = self.Addon.getSetting('remote_path_2')
	    self.Addon.setSetting("remote_path","")
        elif(self.Addon.getSetting('remote_selection') == '0' and vfs.exists(self.Addon.getSetting("remote_path"))):
            self.remote_path = self.Addon.getSetting("remote_path")

	if(self.Addon.getSetting("backup_name") != '' and self.remote_path != ''):
	    self.remote_path = self.remote_path + self.Addon.getSetting("backup_name") + "/"
	else:
	    self.remote_path = ""

        self.log("Starting")
        self.log('Local Dir: ' + self.local_path)
        self.log('Remote Dir: ' + self.remote_path)

    def run(self):
	#check if we should use the progress bar
        if(self.Addon.getSetting('run_silent') == 'false'):
            self.progressBar = xbmcgui.DialogProgress()
            self.progressBar.create('XBMC Backup','Gathering file list.....')

        #check what mode were are in
        if(int(self.Addon.getSetting('addon_mode')) == 0):
            self.fileManager = FileManager(self.local_path,self.Addon.getAddonInfo('profile'))
            self.syncFiles()
        else:
            self.fileManager = FileManager(self.remote_path,self.Addon.getAddonInfo('profile'))
            self.restoreFiles()

    def syncFiles(self):
        if(vfs.exists(self.remote_path)):
            #this will fail - need a disclaimer here
            self.log("Remote Path exists - may have old files in it!")

        #make the remote directory
        vfs.mkdir(self.remote_path)

        self.fileManager.createFileList(self.Addon)

        allFiles = self.fileManager.getFileList()

        #write list from local to remote
        self.writeFiles(allFiles,self.local_path,self.remote_path)

    def restoreFiles(self):
        self.fileManager.createFileList(self.Addon)

        allFiles = self.fileManager.getFileList()

        #write list from remote to local
        self.writeFiles(allFiles,self.remote_path,self.local_path)

        #call update addons to refresh everything
        xbmc.executebuiltin('UpdateLocalAddons')

    def writeFiles(self,fileList,source,dest):
        self.filesTotal = len(fileList)
        self.filesLeft = self.filesTotal

        #write each file from source to destination
        for aFile in fileList:
            if(not self.checkCancel()):
                self.updateProgress(aFile)
                if (aFile.startswith("-")):
                    vfs.mkdir(dest + aFile[1:])
                else:
                    vfs.copy(source + aFile,dest + aFile)

        if(self.Addon.getSetting('run_silent') == 'false'):
            self.progressBar.close()

    def updateProgress(self,message=''):
        self.filesLeft = self.filesLeft - 1

        #update the progress bar
        if(self.progressBar != None):
            self.progressBar.update(int((float(self.filesTotal - self.filesLeft)/float(self.filesTotal)) * 100),message)

    def checkCancel(self):
        result = False

        if(self.progressBar != None):
            result = self.progressBar.iscanceled()

        return result

    def log(self,message):
        xbmc.log(self.__addon_id__ + ": " + message)

    def isReady(self):
        return True if self.remote_path != '' else False

#run the profile backup
backup = XbmcBackup()

if(backup.isReady()):
    backup.run()
else:
    xbmcgui.Dialog().ok('XBMC Backup','Error: Remote path cannot be empty')
<|endoftext|>


<commit_before>"""
Django settings for photologue_custom project.

Generated by 'django-admin startproject' using Django 1.8.

For more information on this file, see
https://docs.djangoproject.com/en/1.8/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.8/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '(v)-p((^ko^)=mstt=r+=82+xz&d5-neuq2b*&eq3slz$doov8'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'photologue_custom',
)

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'django.middleware.security.SecurityMiddleware',
)

ROOT_URLCONF = 'photologue_custom.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'photologue_custom.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.8/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Internationalization
# https://docs.djangoproject.com/en/1.8/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'Europe/Berlin'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.8/howto/static-files/

STATIC_URL = '/static/'
STATIC_ROOT = os.path.join(BASE_DIR, 'public', 'static')
<commit_msg>Delete settings.py<commit_after><|endoftext|>


<commit_before>from docopt import docopt
import numpy as np
from collections import defaultdict
from representations.embedding import SVDEmbedding
from representations.explicit import PositiveExplicit
from os.path import join, basename, normpath

def main():
    args = docopt("""
    Usage:
        vectors2similarity.py <target> <limit> <folders>...
""")
    id2freq, ppmi, chi, svd_similarity = [], [], [], []
    folders = args["<folders>"]
    limit = int(args["<limit>"])

    word2id = read_vocabs(limit, folders)
    for folder in folders:
        name = basename(normpath(folder))
        if "_" in name:
            year = name.split("_")[1]
        else:
            year = name
        id2freq.append((year, read_freq(folder, word2id)))
        ppmi.append((year, read_generic(folder2ppmi(folder), word2id)))
        chi.append((year, read_generic(folder2chi(folder), word2id)))
        svd_similarity.append(
            (year, read_generic(folder2svd(folder), word2id, True)))

    store_results(args["<target>"], ("WORDIDS", iterate(word2id, True)), ("FREQUENCY", iterate(
        id2freq)), ("PPMI", iterate(ppmi)), ("CHI", iterate(chi)), ("SIMILARITY", iterate(svd_similarity)))

def iterate(mapping, is_word2id=False):
    if is_word2id:
        for word, value in mapping.items():
            yield [str(x) for x in word, value]
    else:
        for (year, info) in mapping:
            for word, value in info.items():
                if isinstance(value, dict):
                    for word2, innervalue in value.items():
                        yield [str(x) for x in word, word2, year, innervalue]
                else:
                    yield [str(x) for x in word, year, value]

def store_results(path, *mappings):
    for name, mapping in mappings:
        with open(join(path, name + ".csv"), "w") as f:
            for l in mapping:
                print >>f, ",".join(l)

def folder2ppmi(folder):
    return PositiveExplicit(join(folder, "pmi")).similarity_first_order

def folder2chi(folder):
    return PositiveExplicit(join(folder, "chi")).similarity_first_order

def folder2svd(folder):
    return SVDEmbedding(join(folder, "svd_pmi")).similarity

def folder2vocab(folder):
    return join(folder, "counts.words.vocab")

def intersect(words):
    if len(words) == 0:
        return set()
    if len(words) == 1:
        return words[0]
    intersection = words[0]
    for w in words[1:]:
        intersection = intersection.intersection(w)
    return intersection

def read_vocabs(limit, folders):
    words = []
    for vocab in [folder2vocab(folder) for folder in folders]:
        w = set()
        i = 0
        with open(vocab, "r") as v:
            for line in v:
                if i == limit:
                    break
                i += 1
                word, freq = line.strip().split()
                w.add(word)
        words.append(w)
    return {y: x for x, y in enumerate(intersect(words))}

def read_freq(folder, word2id):
    id2freq = {}
    total = 0
    with open(folder2vocab(folder), "r") as v:
        for line in v:
            word, freq = line.strip().split()
            if word in word2id:
                id2freq[word2id[word]] = float(freq)
            total += float(freq)
    total = total / 100
    for i, f in id2freq.items():
        id2freq[i] = f / total
    return id2freq

def read_generic(method, word2id, remove_duplicates=False):
    mapping = defaultdict(dict)
    for word1, id1 in word2id.items():
        for word2, id2 in word2id.items():
            if word1 != word2:
                if not remove_duplicates or (remove_duplicates and id1 < id2):
                    m = method(word1, word2)
                    if m != 0.0:
                        mapping[id1][id2] = m
    return mapping

if __name__ == "__main__":
    main()
<commit_msg>from master<commit_after>from docopt import docopt
import numpy as np
from collections import defaultdict
from representations.embedding import SVDEmbedding
from representations.explicit import PositiveExplicit
from os.path import join, basename, normpath

def main():
    args = docopt("""
    Usage:
        vectors2similarity.py <target> <limit> <folders>...
""")
    id2freq, ppmi, chi, svd_similarity = [], [], [], []
    folders = args["<folders>"]
    limit = int(args["<limit>"])

    word2id = read_vocabs(limit, folders)
    for folder in folders:
        name = basename(normpath(folder))
        if "_" in name:
            year = name.split("_")[1]
        else:
            year = name
        id2freq.append((year, read_freq(folder, word2id)))
        ppmi.append((year, read_generic(folder2ppmi(folder), word2id)))
        chi.append((year, read_generic(folder2chi(folder), word2id)))
        svd_similarity.append(
            (year, read_generic(folder2svd(folder), word2id, True)))
    store_results(args["<target>"], ("WORDIDS", iterate(word2id, True)), ("FREQUENCY", iterate(
        id2freq)), ("PPMI", iterate(ppmi)), ("CHI", iterate(chi)), ("SIMILARITY", iterate(svd_similarity)))

def iterate(mapping, is_word2id=False, is_frequency=False):
    if is_word2id and is_frequency:
        raise Exception("Not allowed")
    elif is_word2id:
        for word, value in mapping.items():
            yield [str(x) for x in word, value]
    elif is_frequency:
        for year, info in mapping:
            for id1, id2, m in info.items():
                yield [str(x) for x in word, year, value]
    else:
        for year, generator in mapping:
            for word, word2, metric in generator:
                yield [str(x) for x in word, word2, year, metric]

def store_results(path, *mappings):
    for name, mapping in mappings:
        with open(join(path, name + ".csv"), "w") as f:
            for l in mapping:
                print >>f, ",".join(l)

def folder2ppmi(folder):
    return PositiveExplicit(join(folder, "pmi")).similarity_first_order

def folder2chi(folder):
    return PositiveExplicit(join(folder, "chi")).similarity_first_order

def folder2chi(folder):
    return PositiveExplicit(join(folder, "chi")).similarity_first_order

def folder2svd(folder):
    return SVDEmbedding(join(folder, "svd_pmi")).similarity

def folder2vocab(folder):
    return join(folder, "counts.words.vocab")

def intersect(words):
    if len(words) == 0:
        return set()
    if len(words) == 1:
        return words[0]
    intersection = words[0]
    for w in words[1:]:
        intersection = intersection.intersection(w)
    return intersection

def read_vocabs(limit, folders):
    words = []
    for vocab in [folder2vocab(folder) for folder in folders]:
        w = set()
        i = 0
        with open(vocab, "r") as v:
            for line in v:
                if i == limit:
                    break
                i += 1
                word, freq = line.strip().split()
                w.add(word)
        words.append(w)
    return {y: x for x, y in enumerate(intersect(words))}

def read_freq(folder, word2id):
    id2freq = {}
    total = 0
    with open(folder2vocab(folder), "r") as v:
        for line in v:
            word, freq = line.strip().split()
            if word in word2id:
                id2freq[word2id[word]] = float(freq)
            total += float(freq)
    total = total / 100
    for i, f in id2freq.items():
        id2freq[i] = f / total
    return id2freq

def read_generic(method, word2id, remove_duplicates=False):
    mapping = defaultdict(dict)
    for word1, id1 in word2id.items():
        for word2, id2 in word2id.items():
            if word1 != word2:
                if not remove_duplicates or (remove_duplicates and id1 < id2):
                    metric = method(word1, word2)
                    if metric != 0.0:
                        yield id1, id2, metric

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>"""
62-character classifier:
0-9
a-z
A-Z
"""
import time
import os
import warnings
import numpy as np
import keras
from keras.layers.core import Dense, Activation, Dropout, Flatten, Reshape
from keras.layers.recurrent import LSTM
from keras.layers.convolutional import Conv2D
from keras.layers.normalization import BatchNormalization
from keras.layers.pooling import MaxPooling2D
from keras.models import Sequential

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
warnings.filterwarnings('ignore')

class CharacterClassifier:
    def __init__(self, img_height, img_width):
        self.dropout = 0.5
        self.strides = 2
        self.height = img_height
        self.width = img_width
        self.kernel_size = [5,5]
        self.kernel_size_2 = [3,3]
        self.pool_size = (2,2)
        self.conv1_filters = 32
        self.conv2_filters = self.conv1_filters*2
        self.conv3_filters = self.conv2_filters*2
        self.conv4_filters = self.conv3_filters*2
        self.conv5_filters = self.conv3_filters*2
        self.num_units = 128
        self.num_units_2 = int(self.num_units/2)
        self.num_labels = 62
        self.model = self.build_model()

    def build_model(self):
        model = Sequential()

        # CONVOLUTION LAYER 1
        model.add(Conv2D(
            input_shape=(self.height, self.width, 1),
            filters=self.conv1_filters,
            kernel_size=self.kernel_size,
            padding="same"))
        BatchNormalization(axis=-1)
        model.add(Activation('relu'))

        model.add(MaxPooling2D(
            pool_size=self.pool_size,
            strides=self.strides))

        # CONVOLUTION LAYER 2
        model.add(Conv2D(
            filters=self.conv2_filters,
            kernel_size=self.kernel_size,
            padding="same"))
        BatchNormalization(axis=-1)
        model.add(Activation('relu'))

        model.add(MaxPooling2D(
            pool_size=self.pool_size,
            strides=self.strides))
        #"""
        # CONVOLUTION LAYER 3
        model.add(Conv2D(
            filters=self.conv3_filters,
            kernel_size=self.kernel_size_2,
            padding="same"))
        BatchNormalization(axis=-1)
        model.add(Activation('relu'))

        model.add(MaxPooling2D(
            pool_size=self.pool_size,
            strides=self.strides))
        #"""
        """
        # CONVOLUTION LAYER 4
        model.add(Conv2D(
            filters=self.conv4_filters,
            kernel_size=self.kernel_size_2,
            padding="same"))
        BatchNormalization(axis=-1)
        model.add(Activation('relu'))

        model.add(MaxPooling2D(
            pool_size=self.pool_size,
            strides=self.strides))
        """
        """
        # CONVOLUTION LAYER 5
        model.add(Conv2D(
            filters=self.conv5_filters,
            kernel_size=self.kernel_size_2,
            padding="same"))
        BatchNormalization(axis=-1)
        model.add(Activation('relu'))

        model.add(MaxPooling2D(
            pool_size=self.pool_size,
            strides=self.strides))
        """
        model.add(Flatten())

        # DENSE LAYER 1
        model.add(Dense(
            units=self.num_units,
            activation='relu'))

        model.add(Dropout(
            rate=self.dropout/2))

        # DENSE LAYER 2
        model.add(Dense(
            units=self.num_units_2,
            activation='relu'))

        model.add(Dropout(
            rate=self.dropout))

        # CLASSIFICATION LAYER
        model.add(Dense(
            units=self.num_labels,
            activation='softmax'))

        start = time.time()
        model.compile(loss=keras.losses.categorical_crossentropy,
                optimizer=keras.optimizers.Adadelta(),
                metrics=['accuracy'])
        print('> Compilation Time: {}'.format(time.time() - start))
        return model
<commit_msg>Started work on classifier methods<commit_after>"""
62-character classifier:
0-9
a-z
A-Z
"""
import time
import os
import warnings
import numpy as np
import keras
from keras.layers.core import Dense, Activation, Dropout, Flatten, Reshape
from keras.layers.recurrent import LSTM
from keras.layers.convolutional import Conv2D
from keras.layers.normalization import BatchNormalization
from keras.layers.pooling import MaxPooling2D
from keras.models import Sequential
from dataload import convert_to_pixel_array

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
warnings.filterwarnings('ignore')

class CharacterClassifier:
    def __init__(self, img_height, img_width, modelPath=None):
        self.height = img_height
        self.width = img_width
        if (modelPath):
            self.model = keras.models.load_model(modelPath)
        else:
            self.dropout = 0.5
            self.strides = 2
            self.kernel_size = [5,5]
            self.kernel_size_2 = [3,3]
            self.pool_size = (2,2)
            self.conv1_filters = 64
            self.conv2_filters = self.conv1_filters*2
            self.conv3_filters = self.conv2_filters*2
            self.conv4_filters = self.conv3_filters*2
            self.conv5_filters = self.conv3_filters*2
            self.num_units = 40
            self.num_units_2 = 40
            self.num_labels = 62
            self.model = self.build_model()

    def classify_image(self, image_path):
        try:
            image_pixels = convert_to_pixel_array(image_path, self.width, self.height)
            image_pixels = np.array(image_pixels)
            inp = np.array([image_pixels])
            inp = inp.reshape(inp.shape[0], 32, 100, 1)

            outp = self.model.predict(inp)[0]
            outp = np.array(outp)

            top5_idx = outp.argsort()[-5:]

            top5_words = [(k, outp[v['id']]) for k, v in self.word_ids.items() if v['id'] in top5_idx]
            top5_words = sorted(top5_words, key=lambda x: x[1], reverse=True)

            return top5_words
        except FileNotFoundError:
            print('Image not found at path {}'.format(image_path))

    def build_model(self):
        model = Sequential()

        # CONVOLUTION LAYER 1
        model.add(Conv2D(
            input_shape=(self.height, self.width, 1),
            filters=self.conv1_filters,
            kernel_size=self.kernel_size,
            padding="same"))
        BatchNormalization(axis=-1)
        model.add(Activation('relu'))

        model.add(MaxPooling2D(
            pool_size=self.pool_size,
            strides=self.strides))

        # CONVOLUTION LAYER 2
        model.add(Conv2D(
            filters=self.conv2_filters,
            kernel_size=self.kernel_size,
            padding="same"))
        BatchNormalization(axis=-1)
        model.add(Activation('relu'))

        model.add(MaxPooling2D(
            pool_size=self.pool_size,
            strides=self.strides))
        #"""
        # CONVOLUTION LAYER 3
        model.add(Conv2D(
            filters=self.conv3_filters,
            kernel_size=self.kernel_size_2,
            padding="same"))
        BatchNormalization(axis=-1)
        model.add(Activation('relu'))

        model.add(MaxPooling2D(
            pool_size=self.pool_size,
            strides=self.strides))
        #"""
        #"""
        # CONVOLUTION LAYER 4
        model.add(Conv2D(
            filters=self.conv4_filters,
            kernel_size=self.kernel_size_2,
            padding="same"))
        BatchNormalization(axis=-1)
        model.add(Activation('relu'))

        model.add(MaxPooling2D(
            pool_size=self.pool_size,
            strides=self.strides))
        #"""
        """
        # CONVOLUTION LAYER 5
        model.add(Conv2D(
            filters=self.conv5_filters,
            kernel_size=self.kernel_size_2,
            padding="same"))
        BatchNormalization(axis=-1)
        model.add(Activation('relu'))

        model.add(MaxPooling2D(
            pool_size=self.pool_size,
            strides=self.strides))
        """
        model.add(Flatten())

        # DENSE LAYER 1
        model.add(Dense(
            units=self.num_units,
            activation='relu'))

        model.add(Dropout(
            rate=self.dropout/2))

        # DENSE LAYER 2
        model.add(Dense(
            units=self.num_units_2,
            activation='relu'))

        model.add(Dropout(
            rate=self.dropout))

        # CLASSIFICATION LAYER
        model.add(Dense(
            units=self.num_labels,
            activation='softmax'))

        start = time.time()
        model.compile(loss=keras.losses.categorical_crossentropy,
                optimizer=keras.optimizers.Adadelta(),
                metrics=['accuracy'])
        print('> Compilation Time: {}'.format(time.time() - start))
        return model
<|endoftext|>


<commit_before>__author__ = 'ajtag'

import subprocess as sp
import glob
import pygame
import logging
from Constants import *
import os
import math
import random

pygame.font.init()
FONT = pygame.font.Font(None, 24)

_fps = None
def get_fps():
    return _fps

random_seed = str(random.random())
def new_random(name):
    return random.Random(random_seed + name)

class Trigger(object):
    """Create a new Group, or run a method on an existing group"""
    def __init__(self, scene, method=None, *args):
        self.scene = scene
        self.method = method
        self.args = args

    def __repr__(self):
        return "Trigger(%s,%s,%s)" % (self.scene, self.method, self.args)

def clean_images():
    # delete any files saved from previous runs
    [os.unlink(i) for i in glob.glob(os.path.join('images', '*.bmp'))]

class Player:
    log = logging.getLogger('Player')

    def __init__(self, title, width, height, display_scale=1, fps=24, args=()):
        pygame.init()
        global _fps
        _fps = fps
        try:
            global random_seed
            random_seed = args.random_seed
        except AttributeError:
            pass
        self.title = title
        self.width = width
        self.height = height
        self.size = (width, height)
        self.display_scale = display_scale
        self.lightmask = args.mask
        self.mask = pygame.Surface(self.size)
        self.mask.fill(dark_grey)
        self.mask.set_colorkey(white)
        for x, y in ceiling.lamps:
            self.mask.set_at((x, y), white)
        self.screen = pygame.Surface(self.size)
        self.display = pygame.display.set_mode((display_scale * width, display_scale * height))
        self.clock = pygame.time.Clock()
        self.fps = fps
        self.pause = False
        self.step = False
        self.objects = {}
        self.ticks = 0
        self.background = black
        self.save_images = args.save_images
        self.save_video = args.save_video
        self.cursor_loc_start = None
        self.cursor_loc_end = None
        self.warp = int(self.fps * args.warp)
        if self.warp != 0 and (self.save_images or self.save_video):
            raise Exception("Can not save when warping")
        self.quick = args.quick
        self.sparse = args.sparse
        if self.sparse > self.display_scale:
            raise Exception("Pixels bigger than screen")

        self.scene_data = {}
        self.scene_layer = {}

        self.key_triggers = {}
        self.timed_events = {}

        self.log.info('done init')

    def set_key_triggers(self, key, trig):
        self.key_triggers[key] = trig

    def load_sprite(self, name, layer, sprite):
        self.scene_layer[name] = layer
        self.objects[name] = sprite

    def load_scene(self, scene_name, layer, *scene_data):
        self.scene_data[scene_name] = scene_data
        self.scene_layer[scene_name] = layer

    def load_timed_event(self, time, events):
        ticks = int(time * self.fps)
        current_events = self.timed_events.get(ticks, [])
        if isinstance(events, Trigger):
            current_events.append(events)
        else:
            for e in events:
                current_events.append(e)
        self.timed_events[ticks] = current_events

    def export_video(self, x, y, ffmpeg_exe='ffmpeg'):
        command = [ffmpeg_exe,
               '-y',  # (optional) overwrite output file if it exists
               '-r', '{}'.format(self.fps),  # frames per second
               '-i', os.path.join('images', '{}_%d.bmp'.format(self.title)),
               '-s', '{}x{}'.format(x, y),
               '-an',  # Tells FFMPEG not to expect any audio
                '-c:v', 'qtrle',
               '-tune', 'animation',
                '-q', '0',
               '-s', '{}x{}'.format(x, y),  # size of one frame
               '{}.mov'.format(self.title)
               ]
        self.log.info(' '.join(command))

        if not self.save_video:
            return

        sp.call(command)

    def run_trigger(self, trigger):
        if trigger.method is None:
            try:
                d = self.scene_data[trigger.scene]
            except:
                self.log.error("No such scene '%s'" % trigger.scene)
                return
            try:
                self.objects[trigger.scene] = d[0](*d[1:])
            except:
                self.log.error("Failed to create '%s' %s" % (trigger.scene, d))
                raise
        else:
            try:
                try:
                    o = self.objects[trigger.scene]
                except KeyError:
                    self.log.error("Scene '%s' not running" % trigger.scene)
                    return
                getattr(o, trigger.method)(*trigger.args)
            except StopIteration:
                del self.objects[trigger.scene]
            except:
                self.log.error("%s" % (trigger))
                raise

    def sparse_blit(self):
        self.display.fill(0)
        scale = int(self.display_scale)
        width = self.sparse
        offset = (scale - width) // 2
        dest = pygame.PixelArray(self.display)
        src = pygame.PixelArray(self.screen)
        for lamp in ceiling.lamps:
            base_x = lamp.x * scale + offset
            base_y = lamp.y * scale + offset
            for x in range(self.sparse):
                for y in range(self.sparse):
                    dest[base_x + x, base_y + y] = src[lamp.x, lamp.y]

    def run(self):
        for event in pygame.event.get():

            # Check for quit
            if event.type == pygame.QUIT:
                return False
            # Mouse events
            if event.type == pygame.MOUSEBUTTONDOWN and event.button == 1:  # left start click
                self.cursor_loc_start = event.pos
                self.cursor_loc_end = None

            if event.type == pygame.MOUSEBUTTONUP and event.button == 1:  # left finish click
                self.cursor_loc_end = event.pos
                print('pygame.Rect({}, {}, {}, {})'.format(
                        math.floor(min(self.cursor_loc_start[0], self.cursor_loc_end[0])/self.display_scale),
                        math.floor(min(self.cursor_loc_start[1], self.cursor_loc_end[1])/self.display_scale),
                        math.floor((max(self.cursor_loc_end[0], self.cursor_loc_start[0]) - min(self.cursor_loc_end[0], self.cursor_loc_start[0]))/self.display_scale),
                        math.floor((max(self.cursor_loc_start[1], self.cursor_loc_end[1]) - min(self.cursor_loc_start[1], self.cursor_loc_end[1]))/self.display_scale)
                     )
             )

            if event.type == pygame.MOUSEBUTTONUP and event.button == 3:  # right click
                self.cursor_loc_start = None
                self.cursor_loc_end = None

            # Check Keys
            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_ESCAPE:
                    return False

                elif event.key == pygame.K_HASH:
                    if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
                        self.log.info('Verbose Output Off')
                        logging.getLogger().setLevel(logging.INFO)
                    else:
                        self.log.info('Verbose Output On')
                        logging.getLogger().setLevel(logging.DEBUG)

                elif event.key == pygame.K_SLASH:
                    self.log.info('''
/  - help
#  - print key triggers
F1 - save video on exit
F2 - view mask
F3 - toggle fps limiter
F4 - play/pause
F5 - step frame

esc - quit

========================================================
''')
                    for k,t in self.key_triggers.items():
                        self.log.info('{} -  {}'.format(chr(k), t))

                elif event.key == pygame.K_F4:
                    self.pause = not(self.pause)
                    self.log.info('pause video: {}'.format(self.pause))

                elif event.key == pygame.K_F5:
                    self.step = True

                elif event.key == pygame.K_F1:
                    self.save_video = not self.save_video
                    self.log.info('save video: {}'.format(self.save_video))

                elif event.key == pygame.K_F2:
                    self.lightmask = not self.lightmask
                    self.log.info('Mask: {}'.format(self.lightmask))

                elif event.key == pygame.K_F3:
                    self.quick = not self.quick
                    self.log.info('FPS de-limiter: {}'.format(self.quick))

                if event.key in self.key_triggers:
                    self.log.debug('pressed {}'.format(event.key))
                    self.run_trigger(self.key_triggers[event.key])

        if not (self.pause) or self.step:
            self.background = black
            self.screen.fill(self.background)
            for e in self.timed_events.get(self.ticks, []):
                self.run_trigger(e)

            draw = (self.ticks >= self.warp) or (self.ticks % self.fps == 0)
            remove = []
            items = [(k,v) for k,v in self.scene_layer.items() if k in self.objects.keys()]

            items.sort(key=lambda ele: ele[1])
            for name, layer in items:
                element = self.objects[name]
                try:
                    element.update()
                    if draw:
                        try:
                            drawfn = element.draw
                        except AttributeError:
                            self.screen.blit(element.image, element.rect.topleft)
                        else:
                            drawfn(self.screen)
                except StopIteration:
                    remove.append(name)
                except:
                    self.log.error('Error while drawing {}'.format(name))
                    raise
            for name in remove:
                del self.objects[name]
            self.step = False
            self.ticks += 1
        else:
            draw = True

        if draw:
            if self.sparse == 0:
                if self.lightmask:
                    pygame.Surface.blit(self.screen, self.mask, (0, 0))
                pygame.transform.scale(self.screen, self.display.get_size(), self.display)
            else:
                self.sparse_blit()

            #  draw a red rect overlay to the display surface by dragging the mouse
            if self.cursor_loc_start is not None:
                i, j = self.cursor_loc_start
                if self.cursor_loc_end is None:
                    x, y = pygame.mouse.get_pos()
                else:
                    x, y = self.cursor_loc_end
                r = pygame.Rect((min(i, x), min(j, y)), (max(i, x) - min(i, x), max(j, y) - min(j, y)))
                pygame.draw.rect(self.display, (255, 0, 0), r, 2)

            self.display.blit(FONT.render('{:.2f}/{:0} fps'.format(self.clock.get_fps(), self.fps), False, (255, 0, 0), ), (10,10))
            self.display.blit(FONT.render('{:.2f}'.format(
                    self.ticks/self.fps
                ), False, (255, 0, 0),), (10,45))

            pygame.display.flip()

        if self.save_images:
            savepath = os.path.join('images')

            if not (os.path.isdir(savepath)):
                os.mkdir(savepath)

            savefile = os.path.join('images', '{}_{}.bmp'.format(self.title, self.ticks))
            pygame.image.save(self.screen, savefile)

        if self.ticks == self.warp:
            self.log.info("Warp finished")

        if draw:
            if self.quick:
                self.clock.tick()
            else:
                self.clock.tick(self.fps)

        return True

    def end(self):
        pygame.quit()
<commit_msg>Add pause arg. Ignore pause while warp active<commit_after>__author__ = 'ajtag'

import subprocess as sp
import glob
import pygame
import logging
from Constants import *
import os
import math
import random

pygame.font.init()
FONT = pygame.font.Font(None, 24)

_fps = None
def get_fps():
    return _fps

random_seed = str(random.random())
def new_random(name):
    return random.Random(random_seed + name)

class Trigger(object):
    """Create a new Group, or run a method on an existing group"""
    def __init__(self, scene, method=None, *args):
        self.scene = scene
        self.method = method
        self.args = args

    def __repr__(self):
        return "Trigger(%s,%s,%s)" % (self.scene, self.method, self.args)

def clean_images():
    # delete any files saved from previous runs
    [os.unlink(i) for i in glob.glob(os.path.join('images', '*.bmp'))]

class Player:
    log = logging.getLogger('Player')

    def __init__(self, title, width, height, display_scale=1, fps=24, args=()):
        pygame.init()
        global _fps
        _fps = fps
        try:
            global random_seed
            random_seed = args.random_seed
        except AttributeError:
            pass
        self.title = title
        self.width = width
        self.height = height
        self.size = (width, height)
        self.display_scale = display_scale
        self.lightmask = args.mask
        self.mask = pygame.Surface(self.size)
        self.mask.fill(dark_grey)
        self.mask.set_colorkey(white)
        for x, y in ceiling.lamps:
            self.mask.set_at((x, y), white)
        self.screen = pygame.Surface(self.size)
        self.display = pygame.display.set_mode((display_scale * width, display_scale * height))
        self.clock = pygame.time.Clock()
        self.fps = fps
        self.pause = args.pause
        self.step = False
        self.objects = {}
        self.ticks = 0
        self.background = black
        self.save_images = args.save_images
        self.save_video = args.save_video
        self.cursor_loc_start = None
        self.cursor_loc_end = None
        self.warp = int(self.fps * args.warp)
        if self.warp >= 0 and (self.save_images or self.save_video):
            raise Exception("Can not save when warping")
        self.quick = args.quick
        self.sparse = args.sparse
        if self.sparse > self.display_scale:
            raise Exception("Pixels bigger than screen")

        self.scene_data = {}
        self.scene_layer = {}

        self.key_triggers = {}
        self.timed_events = {}

        self.log.info('done init')

    def set_key_triggers(self, key, trig):
        self.key_triggers[key] = trig

    def load_sprite(self, name, layer, sprite):
        self.scene_layer[name] = layer
        self.objects[name] = sprite

    def load_scene(self, scene_name, layer, *scene_data):
        self.scene_data[scene_name] = scene_data
        self.scene_layer[scene_name] = layer

    def load_timed_event(self, time, events):
        ticks = int(time * self.fps)
        current_events = self.timed_events.get(ticks, [])
        if isinstance(events, Trigger):
            current_events.append(events)
        else:
            for e in events:
                current_events.append(e)
        self.timed_events[ticks] = current_events

    def export_video(self, x, y, ffmpeg_exe='ffmpeg'):
        command = [ffmpeg_exe,
               '-y',  # (optional) overwrite output file if it exists
               '-r', '{}'.format(self.fps),  # frames per second
               '-i', os.path.join('images', '{}_%d.bmp'.format(self.title)),
               '-s', '{}x{}'.format(x, y),
               '-an',  # Tells FFMPEG not to expect any audio
                '-c:v', 'qtrle',
               '-tune', 'animation',
                '-q', '0',
               '-s', '{}x{}'.format(x, y),  # size of one frame
               '{}.mov'.format(self.title)
               ]
        self.log.info(' '.join(command))

        if not self.save_video:
            return

        sp.call(command)

    def run_trigger(self, trigger):
        if trigger.method is None:
            try:
                d = self.scene_data[trigger.scene]
            except:
                self.log.error("No such scene '%s'" % trigger.scene)
                return
            try:
                self.objects[trigger.scene] = d[0](*d[1:])
            except:
                self.log.error("Failed to create '%s' %s" % (trigger.scene, d))
                raise
        else:
            try:
                try:
                    o = self.objects[trigger.scene]
                except KeyError:
                    self.log.error("Scene '%s' not running" % trigger.scene)
                    return
                getattr(o, trigger.method)(*trigger.args)
            except StopIteration:
                del self.objects[trigger.scene]
            except:
                self.log.error("%s" % (trigger))
                raise

    def sparse_blit(self):
        self.display.fill(0)
        scale = int(self.display_scale)
        width = self.sparse
        offset = (scale - width) // 2
        dest = pygame.PixelArray(self.display)
        src = pygame.PixelArray(self.screen)
        for lamp in ceiling.lamps:
            base_x = lamp.x * scale + offset
            base_y = lamp.y * scale + offset
            for x in range(self.sparse):
                for y in range(self.sparse):
                    dest[base_x + x, base_y + y] = src[lamp.x, lamp.y]

    def run(self):
        for event in pygame.event.get():

            # Check for quit
            if event.type == pygame.QUIT:
                return False
            # Mouse events
            if event.type == pygame.MOUSEBUTTONDOWN and event.button == 1:  # left start click
                self.cursor_loc_start = event.pos
                self.cursor_loc_end = None

            if event.type == pygame.MOUSEBUTTONUP and event.button == 1:  # left finish click
                self.cursor_loc_end = event.pos
                print('pygame.Rect({}, {}, {}, {})'.format(
                        math.floor(min(self.cursor_loc_start[0], self.cursor_loc_end[0])/self.display_scale),
                        math.floor(min(self.cursor_loc_start[1], self.cursor_loc_end[1])/self.display_scale),
                        math.floor((max(self.cursor_loc_end[0], self.cursor_loc_start[0]) - min(self.cursor_loc_end[0], self.cursor_loc_start[0]))/self.display_scale),
                        math.floor((max(self.cursor_loc_start[1], self.cursor_loc_end[1]) - min(self.cursor_loc_start[1], self.cursor_loc_end[1]))/self.display_scale)
                     )
             )

            if event.type == pygame.MOUSEBUTTONUP and event.button == 3:  # right click
                self.cursor_loc_start = None
                self.cursor_loc_end = None

            # Check Keys
            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_ESCAPE:
                    return False

                elif event.key == pygame.K_HASH:
                    if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
                        self.log.info('Verbose Output Off')
                        logging.getLogger().setLevel(logging.INFO)
                    else:
                        self.log.info('Verbose Output On')
                        logging.getLogger().setLevel(logging.DEBUG)

                elif event.key == pygame.K_SLASH:
                    self.log.info('''
/  - help
#  - print key triggers
F1 - save video on exit
F2 - view mask
F3 - toggle fps limiter
F4 - play/pause
F5 - step frame

esc - quit

========================================================
''')
                    for k,t in self.key_triggers.items():
                        self.log.info('{} -  {}'.format(chr(k), t))

                elif event.key == pygame.K_F4:
                    self.pause = not(self.pause)
                    self.log.info('pause video: {}'.format(self.pause))

                elif event.key == pygame.K_F5:
                    self.step = True

                elif event.key == pygame.K_F1:
                    self.save_video = not self.save_video
                    self.log.info('save video: {}'.format(self.save_video))

                elif event.key == pygame.K_F2:
                    self.lightmask = not self.lightmask
                    self.log.info('Mask: {}'.format(self.lightmask))

                elif event.key == pygame.K_F3:
                    self.quick = not self.quick
                    self.log.info('FPS de-limiter: {}'.format(self.quick))

                if event.key in self.key_triggers:
                    self.log.debug('pressed {}'.format(event.key))
                    self.run_trigger(self.key_triggers[event.key])

        running = not self.pause
        if self.step or self.ticks < self.warp:
            running = True
        if running:
            self.background = black
            self.screen.fill(self.background)
            for e in self.timed_events.get(self.ticks, []):
                self.run_trigger(e)

            draw = (self.ticks >= self.warp) or (self.ticks % self.fps == 0)
            remove = []
            items = [(k,v) for k,v in self.scene_layer.items() if k in self.objects.keys()]

            items.sort(key=lambda ele: ele[1])
            for name, layer in items:
                element = self.objects[name]
                try:
                    element.update()
                    if draw:
                        try:
                            drawfn = element.draw
                        except AttributeError:
                            self.screen.blit(element.image, element.rect.topleft)
                        else:
                            drawfn(self.screen)
                except StopIteration:
                    remove.append(name)
                except:
                    self.log.error('Error while drawing {}'.format(name))
                    raise
            for name in remove:
                del self.objects[name]

            if self.ticks == self.warp:
                self.log.info("Warp finished")

            self.step = False
            self.ticks += 1
        else:
            draw = True

        if draw:
            if self.sparse == 0:
                if self.lightmask:
                    pygame.Surface.blit(self.screen, self.mask, (0, 0))
                pygame.transform.scale(self.screen, self.display.get_size(), self.display)
            else:
                self.sparse_blit()

            #  draw a red rect overlay to the display surface by dragging the mouse
            if self.cursor_loc_start is not None:
                i, j = self.cursor_loc_start
                if self.cursor_loc_end is None:
                    x, y = pygame.mouse.get_pos()
                else:
                    x, y = self.cursor_loc_end
                r = pygame.Rect((min(i, x), min(j, y)), (max(i, x) - min(i, x), max(j, y) - min(j, y)))
                pygame.draw.rect(self.display, (255, 0, 0), r, 2)

            self.display.blit(FONT.render('{:.2f}/{:0} fps'.format(self.clock.get_fps(), self.fps), False, (255, 0, 0), ), (10,10))
            self.display.blit(FONT.render('{:.2f}'.format(
                    self.ticks/self.fps
                ), False, (255, 0, 0),), (10,45))

            pygame.display.flip()

        if self.save_images:
            savepath = os.path.join('images')

            if not (os.path.isdir(savepath)):
                os.mkdir(savepath)

            savefile = os.path.join('images', '{}_{}.bmp'.format(self.title, self.ticks))
            pygame.image.save(self.screen, savefile)

        if draw:
            if self.quick:
                self.clock.tick()
            else:
                self.clock.tick(self.fps)

        return True

    def end(self):
        pygame.quit()
<|endoftext|>