<commit_before># -*- coding: utf-8 -*-
"""
Created on Tue Nov 07 11:48:53 2017

@author: Suhas Somnath

"""

from __future__ import division, print_function, absolute_import, unicode_literals

import time as tm
import numpy as np
from collections import Iterable
from .process import Process, parallel_compute
from ..io.microdata import MicroDataset, MicroDataGroup
from ..io.hdf_utils import getH5DsetRefs, getAuxData, copyAttributes, link_as_main, linkRefs
from ..io.translators.utils import build_ind_val_dsets
from ..io.io_hdf5 import ioHDF5
from .fft import getNoiseFloor, are_compatible_filters, build_composite_freq_filter

class SignalFilter(Process):
    def __init__(self, h5_main, frequency_filters=None, noise_threshold=None, write_filtered=True,
                 write_condensed=False, num_pix=1, phase_rad=0,  **kwargs):

        super(SignalFilter, self).__init__(h5_main, **kwargs)

        if frequency_filters is None and noise_threshold is None:
            raise ValueError('Need to specify at least some noise thresholding / frequency filter')

        if noise_threshold is not None:
            if noise_threshold >= 1 or noise_threshold <= 0:
                raise ValueError('Noise threshold must be within (0 1)')

        self.composite_filter = 1
        if frequency_filters is not None:
            if not isinstance(frequency_filters, Iterable):
                frequency_filters = [frequency_filters]
            if not are_compatible_filters(frequency_filters):
                raise ValueError('frequency filters must be a single or list of FrequencyFilter objects')
            self.composite_filter = build_composite_freq_filter(frequency_filters)
        else:
            write_condensed = False

        if write_filtered is False and write_condensed is False:
            raise ValueError('You need to write the filtered and/or the condensed dataset to the file')

        num_effective_pix = h5_main.shape[0] * 1.0 / num_pix
        if num_effective_pix % 1 > 0:
            raise ValueError('Number of pixels not divisible by the number of pixels to use for FFT filter')

        self.num_effective_pix = int(num_effective_pix)
        self.phase_rad = phase_rad

        self.noise_threshold = noise_threshold
        self.frequency_filters = frequency_filters

        self.write_filtered = write_filtered
        self.write_condensed = write_condensed

        self.data = None
        self.filtered_data = None
        self.condensed_data = None
        self.h5_filtered = None
        self.h5_condensed = None
        self.h5_noise_floors = None

    def _set_memory_and_cores(self, cores=1, mem=1024):
        """
        Checks hardware limitations such as memory, # cpus and sets the recommended datachunk sizes and the
        number of cores to be used by analysis methods.

        Parameters
        ----------
        cores : uint, optional
            Default - 1
            How many cores to use for the computation
        mem : uint, optional
            Default - 1024
            The amount a memory in Mb to use in the computation
        """
        super(SignalFilter, self)._set_memory_and_cores(cores=cores, mem=mem)
        """
        Remember that the default number of pixels corresponds to only the raw data that can be held in memory
        In the case of signal filtering, the datasets that will occupy space are:
        1. Raw, 2. filtered (real + freq space copies), 3. Condensed (substantially lesser space)
        The actual scaling of memory depends on options:
        """
        scaling_factor = 1 + 2*self.write_filtered + 0.25*self.write_condensed
        self._max_pos_per_read = int(self._max_pos_per_read / scaling_factor)

    def _create_results_datasets(self):
        """
        Process specific call that will write the h5 group, guess dataset, corresponding spectroscopic datasets and also
        link the guess dataset to the spectroscopic datasets. It is recommended that the ancillary datasets be populated
        within this function.
        """

        grp_name = self.h5_main.name.split('/')[-1] + '-FFT_Filtering_'
        grp_filt = MicroDataGroup(grp_name, self.h5_main.parent.name)
        filter_parms = dict()
        if self.frequency_filters is not None:
            for filter in self.frequency_filters:
                filter_parms.update(filter.get_parms())
        filter_parms['algorithm'] = 'pycroscopy_SignalFilter'
        filter_parms['last_pixel'] = 0
        grp_filt.attrs = filter_parms

        if isinstance(self.composite_filter, np.ndarray):
            ds_comp_filt = MicroDataset('Composite_Filter', np.float32(self.composite_filter))
            grp_filt.addChildren([ds_comp_filt])

        if self.noise_threshold is not None:
            ds_noise_floors = MicroDataset('Noise_Floors',
                                           data=np.zeros(shape=(self.num_effective_pix, 1), dtype=np.float32))
            ds_noise_spec_inds, ds_noise_spec_vals = build_ind_val_dsets([1], is_spectral=True,
                                                                         labels=['arb'], units=[''],
                                                                         verbose=self.verbose)
            ds_noise_spec_inds.name = 'Noise_Spectral_Indices'
            ds_noise_spec_vals.name = 'Noise_Spectral_Values'
            grp_filt.addChildren([ds_noise_floors, ds_noise_spec_inds, ds_noise_spec_vals])

        if self.write_filtered:
            ds_filt_data = MicroDataset('Filtered_Data', data=[], maxshape=self.h5_main.maxshape,
                                        dtype=np.float32, chunking=self.h5_main.chunks, compression='gzip')
            grp_filt.addChildren([ds_filt_data])

        self.hot_inds = None

        h5_pos_inds = getAuxData(self.h5_main, auxDataName=['Position_Indices'])[0]
        h5_pos_vals = getAuxData(self.h5_main, auxDataName=['Position_Values'])[0]

        if self.write_condensed:
            self.hot_inds = np.where(self.composite_filter > 0)[0]
            self.hot_inds = np.uint(self.hot_inds[int(0.5 * len(self.hot_inds)):])  # only need to keep half the data
            ds_spec_inds, ds_spec_vals = build_ind_val_dsets([int(0.5 * len(self.hot_inds))], is_spectral=True,
                                                             labels=['hot_frequencies'], units=[''],
                                                             verbose=self.verbose)
            ds_spec_vals.data = np.atleast_2d(self.hot_inds)  # The data generated above varies linearly. Override.
            ds_cond_data = MicroDataset('Condensed_Data', data=[],
                                        maxshape=(self.num_effective_pix, len(self.hot_inds)),
                                        dtype=np.complex, chunking=(1, len(self.hot_inds)), compression='gzip')
            grp_filt.addChildren([ds_spec_inds, ds_spec_vals, ds_cond_data])
            if self.num_effective_pix > 1:
                # need to make new position datasets by taking every n'th index / value:
                new_pos_vals = np.atleast_2d(h5_pos_vals[slice(0, None, self.num_effective_pix), :])
                ds_pos_inds, ds_pos_vals = build_ind_val_dsets([int(np.unique(h5_pos_inds[:, dim_ind]).size /
                                                                    self.num_effective_pix)
                                                                for dim_ind in range(h5_pos_inds.shape[1])],
                                                               is_spectral=False,
                                                               labels=h5_pos_inds.attrs['labels'],
                                                               units=h5_pos_inds.attrs['units'], verbose=self.verbose)
                h5_pos_vals.data = np.atleast_2d(new_pos_vals)  # The data generated above varies linearly. Override.
                grp_filt.addChildren([ds_pos_inds, ds_pos_vals])

        if self.verbose:
            grp_filt.showTree()
        hdf = ioHDF5(self.h5_main.file)
        h5_filt_refs = hdf.writeData(grp_filt, print_log=self.verbose)

        if isinstance(self.composite_filter, np.ndarray):
            h5_comp_filt = getH5DsetRefs(['Composite_Filter'], h5_filt_refs)[0]

        if self.noise_threshold is not None:
            self.h5_noise_floors = getH5DsetRefs(['Noise_Floors'], h5_filt_refs)[0]
            self.h5_results_grp = self.h5_noise_floors.parent
            link_as_main(self.h5_noise_floors, h5_pos_inds, h5_pos_vals,
                         getH5DsetRefs(['Noise_Spectral_Indices'], h5_filt_refs)[0],
                         getH5DsetRefs(['Noise_Spectral_Values'], h5_filt_refs)[0])

        # Now need to link appropriately:
        if self.write_filtered:
            self.h5_filtered = getH5DsetRefs(['Filtered_Data'], h5_filt_refs)[0]
            self.h5_results_grp = self.h5_filtered.parent
            copyAttributes(self.h5_main, self.h5_filtered, skip_refs=False)
            if isinstance(self.composite_filter, np.ndarray):
                linkRefs(self.h5_filtered, [h5_comp_filt])

            """link_as_main(self.h5_filtered, h5_pos_inds, h5_pos_vals,
                         getAuxData(h5_main, auxDataName=['Spectroscopic_Indices'])[0],
                         getAuxData(h5_main, auxDataName=['Spectroscopic_Values'])[0])"""

        if self.write_condensed:
            self.h5_condensed = getH5DsetRefs(['Condensed_Data'], h5_filt_refs)[0]
            self.h5_results_grp = self.h5_condensed.parent
            if isinstance(self.composite_filter, np.ndarray):
                linkRefs(self.h5_condensed, [h5_comp_filt])
            if self.noise_threshold is not None:
                linkRefs(self.h5_condensed, [self.h5_noise_floors])

            if self.num_effective_pix > 1:
                h5_pos_inds = getH5DsetRefs(['Position_Indices'], h5_filt_refs)[0]
                h5_pos_vals = getH5DsetRefs(['Position_Values'], h5_filt_refs)[0]

            link_as_main(self.h5_condensed, h5_pos_inds, h5_pos_vals,
                         getH5DsetRefs(['Spectroscopic_Indices'], h5_filt_refs)[0],
                         getH5DsetRefs(['Spectroscopic_Values'], h5_filt_refs)[0])

    def _write_results_chunk(self):

        pos_slice = slice(self._start_pos, self._end_pos)

        if self.write_condensed:
            self.h5_condensed[pos_slice] = self.condensed_data
        if self.noise_threshold is not None:
            self.h5_noise_floors[pos_slice] = np.atleast_2d(self.noise_floors)
        if self.write_filtered:
            self.h5_filtered[pos_slice] = self.filtered_data

        # Leaving in this provision that will allow restarting of processes
        self.h5_results_grp.attrs['last_pixel'] = self._end_pos

        self.hdf.flush()

        if self.verbose:
            print('Finished processing upto pixel ' + str(self._end_pos) + ' of ' + str(self.h5_main.shape[0]))

        # Now update the start position
        self._start_pos = self._end_pos

    @staticmethod
    def _unit_function():
        return getNoiseFloor

    def compute(self, *args, **kwargs):
        self._create_results_datasets()

        time_per_pix = 0

        num_pos = self.h5_main.shape[0]

        self._read_data_chunk()
        while self.data is not None:

            t_start = tm.time()

            # get FFT of the entire data chunk
            self.data = np.fft.fftshift(np.fft.fft(self.data, axis=1), axes=1)

            if self.noise_threshold is not None:
                self.noise_floors = parallel_compute(self.data, getNoiseFloor, cores=self._cores,
                                                     func_args=[self.noise_threshold])

            if isinstance(self.composite_filter, np.ndarray):
                # multiple fft of data with composite filter
                self.data *= self.composite_filter

            if self.noise_threshold is not None:
                # apply thresholding
                self.data[self.data < np.tile(np.atleast_2d(self.noise_floors), self.data.shape[1])] = 1E-16

            if self.write_condensed:
                # set self.condensed_data here
                self.condensed_data = self.data[:, self.hot_inds]

            if self.write_filtered:
                # take inverse FFT
                self.filtered_data = np.real(np.fft.ifft(np.fft.ifftshift(self.data, axes=1), axis=1))
                if self.phase_rad > 0:
                    # do np.roll on data
                    # self.data = np.roll(self.data, 0, axis=1)
                    pass

            tot_time = np.round(tm.time() - t_start, decimals=2)

            if self.verbose:
                print('Done parallel computing in {} sec or {} sec per pixel'.format(tot_time,
                                                                                tot_time / self.data.shape[0]))
            if self._start_pos == 0:
                time_per_pix = tot_time / self._end_pos  # in seconds
            else:
                print('Time remaining: {} mins'.format(np.round((num_pos - self._end_pos) * time_per_pix / 60, 2)))

            self._write_results_chunk()
            self._read_data_chunk()

        if self.verbose:
            print('Finished processing the dataset completely')

        # return self.h5_cap.parent
        return self.h5_results_grp<commit_msg>Added documentation<commit_after># -*- coding: utf-8 -*-
"""
Created on Tue Nov 07 11:48:53 2017

@author: Suhas Somnath

"""

from __future__ import division, print_function, absolute_import, unicode_literals

import time as tm
import numpy as np
from collections import Iterable
from .process import Process, parallel_compute
from ..io.microdata import MicroDataset, MicroDataGroup
from ..io.hdf_utils import getH5DsetRefs, getAuxData, copyAttributes, link_as_main, linkRefs
from ..io.translators.utils import build_ind_val_dsets
from ..io.io_hdf5 import ioHDF5
from .fft import getNoiseFloor, are_compatible_filters, build_composite_freq_filter
# TODO: implement phase compensation

class SignalFilter(Process):
    def __init__(self, h5_main, frequency_filters=None, noise_threshold=None, write_filtered=True,
                 write_condensed=False, num_pix=1, phase_rad=0,  **kwargs):
        """
        Filters the entire h5 dataset with the given filtering parameters.

        Parameters
        ----------
        h5_main : h5py.Dataset object
            Dataset to process
        frequency_filters : (Optional) single or list of pycroscopy.fft.FrequencyFilter objects
            Frequency (vertical) filters to apply to signal
        noise_threshold : (Optional) float. Default - None
            Noise tolerance to apply to data. Value must be within (0, 1)
        write_filtered : (Optional) bool. Default - True
            Whether or not to write the filtered data to file
        write_condensed : Optional) bool. Default - False
            Whether or not to write the condensed data in frequency space to file. Use this for datasets that are very
            large but sparse in frequency space.
        num_pix : (Optional) uint. Default - 1
            Number of pixels to use for filtering. More pixels means a lower noise floor and the ability to pick up
            weaker signals. Use only if absolutely necessary. This value must be a divisor of the number of pixels in
            the dataset
        phase_rad : (Optional). float
            Degrees by which the output is rotated with respect to the input to compensate for phase lag.
            This feature has NOT yet been implemented.
        kwargs : (Optional). dictionary
            Please see Process class for additional inputs
        """

        super(SignalFilter, self).__init__(h5_main, **kwargs)

        if frequency_filters is None and noise_threshold is None:
            raise ValueError('Need to specify at least some noise thresholding / frequency filter')

        if noise_threshold is not None:
            if noise_threshold >= 1 or noise_threshold <= 0:
                raise ValueError('Noise threshold must be within (0 1)')

        self.composite_filter = 1
        if frequency_filters is not None:
            if not isinstance(frequency_filters, Iterable):
                frequency_filters = [frequency_filters]
            if not are_compatible_filters(frequency_filters):
                raise ValueError('frequency filters must be a single or list of FrequencyFilter objects')
            self.composite_filter = build_composite_freq_filter(frequency_filters)
        else:
            write_condensed = False

        if write_filtered is False and write_condensed is False:
            raise ValueError('You need to write the filtered and/or the condensed dataset to the file')

        num_effective_pix = h5_main.shape[0] * 1.0 / num_pix
        if num_effective_pix % 1 > 0:
            raise ValueError('Number of pixels not divisible by the number of pixels to use for FFT filter')

        self.num_effective_pix = int(num_effective_pix)
        self.phase_rad = phase_rad

        self.noise_threshold = noise_threshold
        self.frequency_filters = frequency_filters

        self.write_filtered = write_filtered
        self.write_condensed = write_condensed

        self.data = None
        self.filtered_data = None
        self.condensed_data = None
        self.noise_floors = None
        self.h5_filtered = None
        self.h5_condensed = None
        self.h5_noise_floors = None

    def _set_memory_and_cores(self, cores=1, mem=1024):
        """
        Checks hardware limitations such as memory, # cpus and sets the recommended datachunk sizes and the
        number of cores to be used by analysis methods.

        Parameters
        ----------
        cores : uint, optional
            Default - 1
            How many cores to use for the computation
        mem : uint, optional
            Default - 1024
            The amount a memory in Mb to use in the computation
        """
        super(SignalFilter, self)._set_memory_and_cores(cores=cores, mem=mem)
        """
        Remember that the default number of pixels corresponds to only the raw data that can be held in memory
        In the case of signal filtering, the datasets that will occupy space are:
        1. Raw, 2. filtered (real + freq space copies), 3. Condensed (substantially lesser space)
        The actual scaling of memory depends on options:
        """
        scaling_factor = 1 + 2*self.write_filtered + 0.25*self.write_condensed
        self._max_pos_per_read = int(self._max_pos_per_read / scaling_factor)

    def _create_results_datasets(self):
        """
        Creates all the datasets necessary for holding all parameters + data.
        """

        grp_name = self.h5_main.name.split('/')[-1] + '-FFT_Filtering_'
        grp_filt = MicroDataGroup(grp_name, self.h5_main.parent.name)
        filter_parms = dict()
        if self.frequency_filters is not None:
            for filter in self.frequency_filters:
                filter_parms.update(filter.get_parms())
        filter_parms['algorithm'] = 'pycroscopy_SignalFilter'
        filter_parms['last_pixel'] = 0
        grp_filt.attrs = filter_parms

        if isinstance(self.composite_filter, np.ndarray):
            ds_comp_filt = MicroDataset('Composite_Filter', np.float32(self.composite_filter))
            grp_filt.addChildren([ds_comp_filt])

        if self.noise_threshold is not None:
            ds_noise_floors = MicroDataset('Noise_Floors',
                                           data=np.zeros(shape=(self.num_effective_pix, 1), dtype=np.float32))
            ds_noise_spec_inds, ds_noise_spec_vals = build_ind_val_dsets([1], is_spectral=True,
                                                                         labels=['arb'], units=[''],
                                                                         verbose=self.verbose)
            ds_noise_spec_inds.name = 'Noise_Spectral_Indices'
            ds_noise_spec_vals.name = 'Noise_Spectral_Values'
            grp_filt.addChildren([ds_noise_floors, ds_noise_spec_inds, ds_noise_spec_vals])

        if self.write_filtered:
            ds_filt_data = MicroDataset('Filtered_Data', data=[], maxshape=self.h5_main.maxshape,
                                        dtype=np.float32, chunking=self.h5_main.chunks, compression='gzip')
            grp_filt.addChildren([ds_filt_data])

        self.hot_inds = None

        h5_pos_inds = getAuxData(self.h5_main, auxDataName=['Position_Indices'])[0]
        h5_pos_vals = getAuxData(self.h5_main, auxDataName=['Position_Values'])[0]

        if self.write_condensed:
            self.hot_inds = np.where(self.composite_filter > 0)[0]
            self.hot_inds = np.uint(self.hot_inds[int(0.5 * len(self.hot_inds)):])  # only need to keep half the data
            ds_spec_inds, ds_spec_vals = build_ind_val_dsets([int(0.5 * len(self.hot_inds))], is_spectral=True,
                                                             labels=['hot_frequencies'], units=[''],
                                                             verbose=self.verbose)
            ds_spec_vals.data = np.atleast_2d(self.hot_inds)  # The data generated above varies linearly. Override.
            ds_cond_data = MicroDataset('Condensed_Data', data=[],
                                        maxshape=(self.num_effective_pix, len(self.hot_inds)),
                                        dtype=np.complex, chunking=(1, len(self.hot_inds)), compression='gzip')
            grp_filt.addChildren([ds_spec_inds, ds_spec_vals, ds_cond_data])
            if self.num_effective_pix > 1:
                # need to make new position datasets by taking every n'th index / value:
                new_pos_vals = np.atleast_2d(h5_pos_vals[slice(0, None, self.num_effective_pix), :])
                ds_pos_inds, ds_pos_vals = build_ind_val_dsets([int(np.unique(h5_pos_inds[:, dim_ind]).size /
                                                                    self.num_effective_pix)
                                                                for dim_ind in range(h5_pos_inds.shape[1])],
                                                               is_spectral=False,
                                                               labels=h5_pos_inds.attrs['labels'],
                                                               units=h5_pos_inds.attrs['units'], verbose=self.verbose)
                h5_pos_vals.data = np.atleast_2d(new_pos_vals)  # The data generated above varies linearly. Override.
                grp_filt.addChildren([ds_pos_inds, ds_pos_vals])

        if self.verbose:
            grp_filt.showTree()
        hdf = ioHDF5(self.h5_main.file)
        h5_filt_refs = hdf.writeData(grp_filt, print_log=self.verbose)

        if isinstance(self.composite_filter, np.ndarray):
            h5_comp_filt = getH5DsetRefs(['Composite_Filter'], h5_filt_refs)[0]

        if self.noise_threshold is not None:
            self.h5_noise_floors = getH5DsetRefs(['Noise_Floors'], h5_filt_refs)[0]
            self.h5_results_grp = self.h5_noise_floors.parent
            link_as_main(self.h5_noise_floors, h5_pos_inds, h5_pos_vals,
                         getH5DsetRefs(['Noise_Spectral_Indices'], h5_filt_refs)[0],
                         getH5DsetRefs(['Noise_Spectral_Values'], h5_filt_refs)[0])

        # Now need to link appropriately:
        if self.write_filtered:
            self.h5_filtered = getH5DsetRefs(['Filtered_Data'], h5_filt_refs)[0]
            self.h5_results_grp = self.h5_filtered.parent
            copyAttributes(self.h5_main, self.h5_filtered, skip_refs=False)
            if isinstance(self.composite_filter, np.ndarray):
                linkRefs(self.h5_filtered, [h5_comp_filt])

            """link_as_main(self.h5_filtered, h5_pos_inds, h5_pos_vals,
                         getAuxData(h5_main, auxDataName=['Spectroscopic_Indices'])[0],
                         getAuxData(h5_main, auxDataName=['Spectroscopic_Values'])[0])"""

        if self.write_condensed:
            self.h5_condensed = getH5DsetRefs(['Condensed_Data'], h5_filt_refs)[0]
            self.h5_results_grp = self.h5_condensed.parent
            if isinstance(self.composite_filter, np.ndarray):
                linkRefs(self.h5_condensed, [h5_comp_filt])
            if self.noise_threshold is not None:
                linkRefs(self.h5_condensed, [self.h5_noise_floors])

            if self.num_effective_pix > 1:
                h5_pos_inds = getH5DsetRefs(['Position_Indices'], h5_filt_refs)[0]
                h5_pos_vals = getH5DsetRefs(['Position_Values'], h5_filt_refs)[0]

            link_as_main(self.h5_condensed, h5_pos_inds, h5_pos_vals,
                         getH5DsetRefs(['Spectroscopic_Indices'], h5_filt_refs)[0],
                         getH5DsetRefs(['Spectroscopic_Values'], h5_filt_refs)[0])

    def _write_results_chunk(self):
        """
        Writes data chunks back to the file
        """

        pos_slice = slice(self._start_pos, self._end_pos)

        if self.write_condensed:
            self.h5_condensed[pos_slice] = self.condensed_data
        if self.noise_threshold is not None:
            self.h5_noise_floors[pos_slice] = np.atleast_2d(self.noise_floors)
        if self.write_filtered:
            self.h5_filtered[pos_slice] = self.filtered_data

        # Leaving in this provision that will allow restarting of processes
        self.h5_results_grp.attrs['last_pixel'] = self._end_pos

        self.hdf.flush()

        if self.verbose:
            print('Finished processing upto pixel ' + str(self._end_pos) + ' of ' + str(self.h5_main.shape[0]))

        # Now update the start position
        self._start_pos = self._end_pos

    @staticmethod
    def _unit_function():
        return getNoiseFloor

    def compute(self, *args, **kwargs):
        """
        Creates placeholders for the results, applies the filers to the data, and writes the output to the file.

        Parameters
        ----------
        None

        Returns
        -------
        h5_results_grp : h5py.Datagroup object
            Datagroup containing all the results
        """
        self._create_results_datasets()

        time_per_pix = 0

        num_pos = self.h5_main.shape[0]

        self._read_data_chunk()
        while self.data is not None:

            t_start = tm.time()

            # get FFT of the entire data chunk
            self.data = np.fft.fftshift(np.fft.fft(self.data, axis=1), axes=1)

            if self.noise_threshold is not None:
                self.noise_floors = parallel_compute(self.data, getNoiseFloor, cores=self._cores,
                                                     func_args=[self.noise_threshold])

            if isinstance(self.composite_filter, np.ndarray):
                # multiple fft of data with composite filter
                self.data *= self.composite_filter

            if self.noise_threshold is not None:
                # apply thresholding
                self.data[self.data < np.tile(np.atleast_2d(self.noise_floors), self.data.shape[1])] = 1E-16

            if self.write_condensed:
                # set self.condensed_data here
                self.condensed_data = self.data[:, self.hot_inds]

            if self.write_filtered:
                # take inverse FFT
                self.filtered_data = np.real(np.fft.ifft(np.fft.ifftshift(self.data, axes=1), axis=1))
                if self.phase_rad > 0:
                    # do np.roll on data
                    # self.data = np.roll(self.data, 0, axis=1)
                    pass

            tot_time = np.round(tm.time() - t_start, decimals=2)

            if self.verbose:
                print('Done parallel computing in {} sec or {} sec per pixel'.format(tot_time,
                                                                                tot_time / self.data.shape[0]))
            if self._start_pos == 0:
                time_per_pix = tot_time / self._end_pos  # in seconds
            else:
                print('Time remaining: {} mins'.format(np.round((num_pos - self._end_pos) * time_per_pix / 60, 2)))

            self._write_results_chunk()
            self._read_data_chunk()

        if self.verbose:
            print('Finished processing the dataset completely')

        return self.h5_results_grp
<|endoftext|>


<commit_before>#! /usr/bin/env python
# Square script
# Executed when asked to draw a square by the user
# Draws a 12 inch square

import serial
import create
from time import strftime

# Create robot
robot = create.Create("/dev/ttyUSB0")
robot.toFullMode()

# Record current position, at start of drawing
pose = robot.getPose()
now = strftime("%H:%M:%S %m-%d-%Y")
line = "On %s I started drawing a TRIANGLE at:\n" % now
line += "X: %s\n" % pose[0]
line += "Y: %s\n" % pose[1]
line += "Theta: %s\n" % pose[2]
with open('positions.txt','a') as f:
  f.write(line)
f.closed

# Draw square
robot.move(305, 30) #12" equivilent in millimeters, rounded up, 30cm/s
robot.turn(-90, 60) # 90 degrees clockwise, 60 degrees/s
robot.move(305, 30)
robot.turn(-90, 60)
robot.move(305, 30)
robot.turn(-90, 60)
robot.move(305, 30)

# Record position now, at end of drawing
new_pose = robot.getPose()
new_now = strftime("%H:%M:%S %m-%d-%Y")
line = "On %s I started drawing a TRIANGLE at:\n" % new_now
line += "X: %s\n" % new_pose[0]
line += "Y: %s\n" % new_pose[1]
line += "Theta: %s\n" % new_pose[2]
with open('positions.txt','a') as f:
  f.write(line)
f.closed
<commit_msg>Update square.py<commit_after>#! /usr/bin/env python
# Square script
# Executed when asked to draw a square by the user
# Draws a 12 inch square

import serial
import create
from time import strftime

# Create robot
robot = create.Create("/dev/ttyUSB0")
robot.toFullMode()

# Record current position, at start of drawing
pose = robot.getPose()
now = strftime("%H:%M:%S %m-%d-%Y")
line = "On %s I started drawing a TRIANGLE at:\n" % now
line += "X: %s\n" % pose[0]
line += "Y: %s\n" % pose[1]
line += "Theta: %s\n" % pose[2]
#with open('positions.txt','a') as f:
#  f.write(line)
#f.closed

# Draw square
robot.move(305, 30) #12" equivilent in millimeters, rounded up, 30cm/s
robot.turn(-90, 60) # 90 degrees clockwise, 60 degrees/s
robot.move(305, 30)
robot.turn(-90, 60)
robot.move(305, 30)
robot.turn(-90, 60)
robot.move(305, 30)

# Record position now, at end of drawing
new_pose = robot.getPose()
new_now = strftime("%H:%M:%S %m-%d-%Y")
line = "On %s I started drawing a TRIANGLE at:\n" % new_now
line += "X: %s\n" % new_pose[0]
line += "Y: %s\n" % new_pose[1]
line += "Theta: %s\n" % new_pose[2]
#with open('positions.txt','a') as f:
#  f.write(line)
#f.closed
<|endoftext|>


<commit_before>from django.contrib.auth.decorators     import login_required, staff_member_required
from django.core.urlresolvers           import reverse
from django.http                        import HttpResponse, HttpResponseRedirect
from django.shortcuts                   import render_to_response, get_object_or_404, redirect
from django.template                    import RequestContext
from django.views.generic.list_detail   import object_list, object_detail
from django.views.generic.simple        import direct_to_template
from django.http                        import Http404
from django.core.mail                   import EmailMessage
from django.db.models                   import Count

from tutordb.models      import Tutor, Centre
from certificates.models import Certificate
from certificates.forms  import CertificateForm, CertificateEmailForm

def check_user_may_see( request, certificate_tutor ):
    """Check that the current user may see the requested details"""
    req_tutor = request.user
    if req_tutor == certificate_tutor or req_tutor.is_head_office() or req_tutor.is_manager_of(certificate_tutor):
        pass
    else:
        raise Http404

@login_required
def index(request):
    """show the certificates nav page"""

    return object_list(
        request,
        template_name = 'certificates/index.html',
        paginate_by   = 40,
        allow_empty   = True,
        queryset      = request.user.certificate_set.order_by('-id'),
        extra_context = {},
    )

# TODO - factor out common access control tests from the display, display_as_pdf
# and tutor_list functions

@login_required
def display(request, certificate_id):
    """Display a certificate"""

    # load the certificate
    certificate = get_object_or_404( Certificate, pk=certificate_id )
    certificate_tutor = certificate.tutor

    check_user_may_see( request, certificate_tutor )

    return object_detail(
        request,
        queryset  = certificate_tutor.certificate_set,
        object_id = certificate_id,
    )

@login_required
def display_as_pdf(request, certificate_id):
    """Create a PDF for certificate"""

    # load the certificate
    certificate = get_object_or_404( Certificate, pk=certificate_id )
    certificate_tutor = certificate.tutor

    check_user_may_see( request, certificate_tutor )

    # create the pdf
    pdf = certificate.as_pdf()

    # set up the response for pdf
    response = HttpResponse(mimetype='application/pdf')
    response['Content-Disposition'] = 'filename=certificate_%s.pdf' % certificate_id
    response.write(pdf)
    return response

@login_required
def email_pdf(request, certificate_id):
    """Email the certificate to the email given"""

    # load the certificate
    certificate = get_object_or_404( Certificate, pk=certificate_id )
    certificate_tutor = certificate.tutor
    check_user_may_see( request, certificate_tutor )

    # set up the form
    form = CertificateEmailForm( request.POST or None, certificate=certificate )

    # send the email if we can
    if form.is_valid():
        data = form.cleaned_data

        # create the email
        email = EmailMessage(
            from_email = certificate_tutor.email,
            to         = [ data['to'] ],
            subject    = data['subject'],
            body       = data['message'],
        )

        # attach the PDF
        email.attach(
            'certificate_%s.pdf' % certificate_id,
            certificate.as_pdf(),
            'application/pdf',
        )

        # send the email (complain if there is a problem)
        email.send( fail_silently=False )

        email_sent = True
    else:
        email_sent = False

    # Show the user the results
    return render_to_response(
        'certificates/email_pdf.html',
        {
            'form': form,
            'email_sent': email_sent,
        },
        context_instance=RequestContext(request)
    )

@login_required
def select_centre(request):
    """Select a centre to use for the certificate"""
    tutor = request.user
    # base_on  = request.GET.get('base_on')
    postcode = request.GET.get('postcode')

    # If we have a postcode load nearby centres.
    if postcode:
        centres_qs = Centre.objects.all().near_postcode(postcode)[:8]
    else:
        centres_qs = Centre.objects.none()

    # Show the user the results
    return render_to_response(
        'certificates/select_centre.html',
        {
            'tutor':            tutor,
            'tutor_centres_qs': tutor.centres.all(),
            'postcode':         postcode,
            'centres_qs':       centres_qs,
        },
        context_instance=RequestContext(request)
    )

@login_required
def create(request):
    """create a new certificate"""
    tutor    = request.user

    # we can't (easily) do this with generics as we need to pre-populate the
    # form if we are cloning an existing certificate.

    # If given a base_on try to load that certificate
    if request.GET.get('base_on'):
        # FIXME - add access controls
        base_certificate = get_object_or_404( Certificate, pk=request.GET.get('base_on') )
    else:
        base_certificate=None

    # try to get the centre from parameters or the base certificate - if none
    # found prompt the user for one.

    centre_id = request.REQUEST.get( 'centre_id', None )
    if centre_id:
        centre = get_object_or_404( Centre, pk=centre_id )
    elif base_certificate:
        centre = base_certificate.centre
    else:
        return redirect( select_centre )

    if request.method == 'POST':
        form = CertificateForm(
            request.POST,
            tutor            = tutor,
            base_certificate = base_certificate,
            centre           = centre,
        )

        if form.is_valid():
            certificate = form.save(commit=False)
            certificate.tutor = tutor
            certificate.save()
            return HttpResponseRedirect(
                reverse(
                    display,
                    kwargs={'certificate_id': certificate.id},
                )
            )

    else:
        form = CertificateForm(
            tutor            = tutor,
            base_certificate = base_certificate,
            centre           = centre,
        )

    return render_to_response(
        'certificates/edit_certificate.html',
        {
            'form': form,
        },
        context_instance=RequestContext(request)
    )

@login_required
def tutor_list(request, tutor_id):
    """Show all certificates for a particular user"""

    # 404 if user is not found
    certificate_tutor = get_object_or_404( Tutor, pk=tutor_id )

    check_user_may_see( request, certificate_tutor )

    # create the queryset
    queryset = certificate_tutor.certificate_set.order_by('-id')

    return object_list(
        request,
        template_name = 'certificates/tutor_list.html',
        paginate_by   = 40,
        allow_empty   = True,
        queryset      = queryset,
        extra_context = {
            "tutor": certificate_tutor,
        },
    )

@staff_member_required
def summarise(request):

    return render_to_response(
        'admin/certificates/summarise.html',
        {
            'certificates': Certificate.objects.values('course_name').annotate(num_title=Count('id')).order_by(),
            'foo': 'bar',
        },
        context_instance=RequestContext(request)
    )
<commit_msg>remove staff_member_required decorator as not available on comet<commit_after>from django.contrib.auth.decorators     import login_required
from django.core.urlresolvers           import reverse
from django.http                        import HttpResponse, HttpResponseRedirect
from django.shortcuts                   import render_to_response, get_object_or_404, redirect
from django.template                    import RequestContext
from django.views.generic.list_detail   import object_list, object_detail
from django.views.generic.simple        import direct_to_template
from django.http                        import Http404
from django.core.mail                   import EmailMessage
from django.db.models                   import Count

from tutordb.models      import Tutor, Centre
from certificates.models import Certificate
from certificates.forms  import CertificateForm, CertificateEmailForm

def check_user_may_see( request, certificate_tutor ):
    """Check that the current user may see the requested details"""
    req_tutor = request.user
    if req_tutor == certificate_tutor or req_tutor.is_head_office() or req_tutor.is_manager_of(certificate_tutor):
        pass
    else:
        raise Http404

@login_required
def index(request):
    """show the certificates nav page"""

    return object_list(
        request,
        template_name = 'certificates/index.html',
        paginate_by   = 40,
        allow_empty   = True,
        queryset      = request.user.certificate_set.order_by('-id'),
        extra_context = {},
    )

# TODO - factor out common access control tests from the display, display_as_pdf
# and tutor_list functions

@login_required
def display(request, certificate_id):
    """Display a certificate"""

    # load the certificate
    certificate = get_object_or_404( Certificate, pk=certificate_id )
    certificate_tutor = certificate.tutor

    check_user_may_see( request, certificate_tutor )

    return object_detail(
        request,
        queryset  = certificate_tutor.certificate_set,
        object_id = certificate_id,
    )

@login_required
def display_as_pdf(request, certificate_id):
    """Create a PDF for certificate"""

    # load the certificate
    certificate = get_object_or_404( Certificate, pk=certificate_id )
    certificate_tutor = certificate.tutor

    check_user_may_see( request, certificate_tutor )

    # create the pdf
    pdf = certificate.as_pdf()

    # set up the response for pdf
    response = HttpResponse(mimetype='application/pdf')
    response['Content-Disposition'] = 'filename=certificate_%s.pdf' % certificate_id
    response.write(pdf)
    return response

@login_required
def email_pdf(request, certificate_id):
    """Email the certificate to the email given"""

    # load the certificate
    certificate = get_object_or_404( Certificate, pk=certificate_id )
    certificate_tutor = certificate.tutor
    check_user_may_see( request, certificate_tutor )

    # set up the form
    form = CertificateEmailForm( request.POST or None, certificate=certificate )

    # send the email if we can
    if form.is_valid():
        data = form.cleaned_data

        # create the email
        email = EmailMessage(
            from_email = certificate_tutor.email,
            to         = [ data['to'] ],
            subject    = data['subject'],
            body       = data['message'],
        )

        # attach the PDF
        email.attach(
            'certificate_%s.pdf' % certificate_id,
            certificate.as_pdf(),
            'application/pdf',
        )

        # send the email (complain if there is a problem)
        email.send( fail_silently=False )

        email_sent = True
    else:
        email_sent = False

    # Show the user the results
    return render_to_response(
        'certificates/email_pdf.html',
        {
            'form': form,
            'email_sent': email_sent,
        },
        context_instance=RequestContext(request)
    )

@login_required
def select_centre(request):
    """Select a centre to use for the certificate"""
    tutor = request.user
    # base_on  = request.GET.get('base_on')
    postcode = request.GET.get('postcode')

    # If we have a postcode load nearby centres.
    if postcode:
        centres_qs = Centre.objects.all().near_postcode(postcode)[:8]
    else:
        centres_qs = Centre.objects.none()

    # Show the user the results
    return render_to_response(
        'certificates/select_centre.html',
        {
            'tutor':            tutor,
            'tutor_centres_qs': tutor.centres.all(),
            'postcode':         postcode,
            'centres_qs':       centres_qs,
        },
        context_instance=RequestContext(request)
    )

@login_required
def create(request):
    """create a new certificate"""
    tutor    = request.user

    # we can't (easily) do this with generics as we need to pre-populate the
    # form if we are cloning an existing certificate.

    # If given a base_on try to load that certificate
    if request.GET.get('base_on'):
        # FIXME - add access controls
        base_certificate = get_object_or_404( Certificate, pk=request.GET.get('base_on') )
    else:
        base_certificate=None

    # try to get the centre from parameters or the base certificate - if none
    # found prompt the user for one.

    centre_id = request.REQUEST.get( 'centre_id', None )
    if centre_id:
        centre = get_object_or_404( Centre, pk=centre_id )
    elif base_certificate:
        centre = base_certificate.centre
    else:
        return redirect( select_centre )

    if request.method == 'POST':
        form = CertificateForm(
            request.POST,
            tutor            = tutor,
            base_certificate = base_certificate,
            centre           = centre,
        )

        if form.is_valid():
            certificate = form.save(commit=False)
            certificate.tutor = tutor
            certificate.save()
            return HttpResponseRedirect(
                reverse(
                    display,
                    kwargs={'certificate_id': certificate.id},
                )
            )

    else:
        form = CertificateForm(
            tutor            = tutor,
            base_certificate = base_certificate,
            centre           = centre,
        )

    return render_to_response(
        'certificates/edit_certificate.html',
        {
            'form': form,
        },
        context_instance=RequestContext(request)
    )

@login_required
def tutor_list(request, tutor_id):
    """Show all certificates for a particular user"""

    # 404 if user is not found
    certificate_tutor = get_object_or_404( Tutor, pk=tutor_id )

    check_user_may_see( request, certificate_tutor )

    # create the queryset
    queryset = certificate_tutor.certificate_set.order_by('-id')

    return object_list(
        request,
        template_name = 'certificates/tutor_list.html',
        paginate_by   = 40,
        allow_empty   = True,
        queryset      = queryset,
        extra_context = {
            "tutor": certificate_tutor,
        },
    )

@login_required
def summarise(request):

    return render_to_response(
        'admin/certificates/summarise.html',
        {
            'certificates': Certificate.objects.values('course_name').annotate(num_title=Count('id')).order_by(),
            'foo': 'bar',
        },
        context_instance=RequestContext(request)
    )
<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-
# Application: Vimba - CMS
# Module: SimpleNews
# Copyright (c) 2010 Vimba inc. All rights reserved.
# Created by Francois Lebel on 20-03-2010.

from django.conf.urls.defaults import *
from vcms.apps.simplenews.feeds import NewsCategoryRssFeed
from vcms.apps.simplenews.models import APP_SLUGS

urlpatterns_prefix = r'^%s/' % APP_SLUGS

urlpatterns = patterns('vcms.apps.simplenews.views',
    (r'^(?P<category_slug>.+)/page-(?P<page>\d+)/$', 'list_news'), # /news/finances/page-1/
    url(r'^page-(?P<page>\d+)/$', 'list_news', { "category_slug": None }), # /news/page-1/
    (r'^(?P<category_slug>.+)/(?P<news_slug>.+)/$', 'single_news'), # /news/finances/year-of-the-linux-desktop/
    (r'^(?P<category_slug>.+)/tag/(?P<category>.+)/$', 'news_category'), # /news/finances/tag/Technology/
    url(r'^tag/(?P<category>.)/$', 'news_category', { "category_slug": None }), # /news/tag/Technology/
    (r'^(?P<category_slug>.+)/tag/(?P<category>.+)/page-(?P<page>\d+)/$', 'news_category'), # /news/finances/tag/Technology/page-1/
    url(r'^tag/(?P<category>.)/page-(?P<page>\d+)/$', 'news_category', { "category_slug": None }), # /news/tag/Technology/page-1/
    #(r'^(?P<category_slug>.+)/tag/(?P<category>.+).atom$', 'TODO'), # /news/finances/tag/Technology.atom
    #url(r'^tag/(?P<category>.+).atom$', 'TODO', { "category_slug": None }), # /news/tag/Technology.atom
    #(r'^(?P<category_slug>.+)/tag/(?P<category>.+).rss$', 'TODO'), # /news/finances/tag/Technology.rss
    #url(r'^tag/(?P<category>.+).rss$', 'TODO', { "category_slug": None }), # /news/tag/Technology.rss
    (r'^(?P<category_slug>.+)/archives/(?P<month>\d{2})-(?P<year>\d{4})/$', 'news_archives'), # /news/finances/archives/12-2010/
    url(r'^archives/(?P<month>\d{2})-(?P<year>\d{4})/$', 'news_archives', { "category_slug": None }), # /news/archives/12-2010/
    (r'^(?P<category_slug>.+)/archives/(?P<month>\d{2})-(?P<year>\d{4})/page-(?P<page>\d+)/$', 'news_archives'), # /news/finances/archives/12-2010/page-1/
    url(r'^archives/(?P<month>\d{2})-(?P<year>\d{4})/page-(?P<page>\d+)/$', 'news_archives', { "category_slug": None }), # /news/archives/12-2010/page-1/
    #(r'^(?P<category_slug>.+)/archives/(?P<month>\d{2})-(?P<year>\d{4}).atom$', 'TODO'), # /news/finances/archives/12-2010.atom
    #url(r'^archives/(?P<month>\d{2})-(?P<year>\d{4}).atom$', 'TODO', { "category_slug": None }), # /news/archives/12-2010.atom
    #(r'^(?P<category_slug>.+)/archives/(?P<month>\d{2})-(?P<year>\d{4}).rss$', 'TODO'), # /news/finances/archives/12-2010.rss
    #url(r'^archives/(?P<month>\d{2})-(?P<year>\d{4}).rss$', 'TODO', { "category_slug": None }), # /news/archives/12-2010.rss
    #url(r'^recent.atom$', 'TODO', { "category_slug": None }), # /news/recent.atom
    #(r'^(?P<category_slug>.+)/recent.atom$', 'TODO'), # /news/finances/recent.atom
    #url(r'^recent.rss$', 'TODO', { "category_slug": None }), # /news/recent.rss
    (r'^(?P<category_slug>.+)/recent.rss$', NewsCategoryRssFeed()), # /news/finances/recent.rss
    (r'^(?P<category_slug>.+)/', 'list_news'), # Catchall index page for a category
    url(r'^', 'list_news', { "category_slug": None }), # Catchall index page
)
<commit_msg>Improved the url for the RSS feed of a NewsCategory.<commit_after># -*- coding: UTF-8 -*-
# Application: Vimba - CMS
# Module: SimpleNews
# Copyright (c) 2010 Vimba inc. All rights reserved.
# Created by Francois Lebel on 20-03-2010.

from django.conf.urls.defaults import *
from vcms.apps.simplenews.feeds import NewsCategoryRssFeed
from vcms.apps.simplenews.models import APP_SLUGS

urlpatterns_prefix = r'^%s/' % APP_SLUGS

urlpatterns = patterns('vcms.apps.simplenews.views',
    (r'^(?P<category_slug>.+)/page-(?P<page>\d+)/$', 'list_news'), # /news/finances/page-1/
    url(r'^page-(?P<page>\d+)/$', 'list_news', { "category_slug": None }), # /news/page-1/
    (r'^(?P<category_slug>.+)/(?P<news_slug>.+)/$', 'single_news'), # /news/finances/year-of-the-linux-desktop/
    (r'^(?P<category_slug>.+)/tag/(?P<category>.+)/$', 'news_category'), # /news/finances/tag/Technology/
    url(r'^tag/(?P<category>.)/$', 'news_category', { "category_slug": None }), # /news/tag/Technology/
    (r'^(?P<category_slug>.+)/tag/(?P<category>.+)/page-(?P<page>\d+)/$', 'news_category'), # /news/finances/tag/Technology/page-1/
    url(r'^tag/(?P<category>.)/page-(?P<page>\d+)/$', 'news_category', { "category_slug": None }), # /news/tag/Technology/page-1/
    #(r'^(?P<category_slug>.+)/tag/(?P<category>.+).atom$', 'TODO'), # /news/finances/tag/Technology.atom
    #url(r'^tag/(?P<category>.+).atom$', 'TODO', { "category_slug": None }), # /news/tag/Technology.atom
    #(r'^(?P<category_slug>.+)/tag/(?P<category>.+).rss$', 'TODO'), # /news/finances/tag/Technology.rss
    #url(r'^tag/(?P<category>.+).rss$', 'TODO', { "category_slug": None }), # /news/tag/Technology.rss
    (r'^(?P<category_slug>.+)/archives/(?P<month>\d{2})-(?P<year>\d{4})/$', 'news_archives'), # /news/finances/archives/12-2010/
    url(r'^archives/(?P<month>\d{2})-(?P<year>\d{4})/$', 'news_archives', { "category_slug": None }), # /news/archives/12-2010/
    (r'^(?P<category_slug>.+)/archives/(?P<month>\d{2})-(?P<year>\d{4})/page-(?P<page>\d+)/$', 'news_archives'), # /news/finances/archives/12-2010/page-1/
    url(r'^archives/(?P<month>\d{2})-(?P<year>\d{4})/page-(?P<page>\d+)/$', 'news_archives', { "category_slug": None }), # /news/archives/12-2010/page-1/
    #(r'^(?P<category_slug>.+)/archives/(?P<month>\d{2})-(?P<year>\d{4}).atom$', 'TODO'), # /news/finances/archives/12-2010.atom
    #url(r'^archives/(?P<month>\d{2})-(?P<year>\d{4}).atom$', 'TODO', { "category_slug": None }), # /news/archives/12-2010.atom
    #(r'^(?P<category_slug>.+)/archives/(?P<month>\d{2})-(?P<year>\d{4}).rss$', 'TODO'), # /news/finances/archives/12-2010.rss
    #url(r'^archives/(?P<month>\d{2})-(?P<year>\d{4}).rss$', 'TODO', { "category_slug": None }), # /news/archives/12-2010.rss
    #url(r'^recent.atom$', 'TODO', { "category_slug": None }), # /news/recent.atom
    #(r'^(?P<category_slug>.+)/recent.atom$', 'TODO'), # /news/finances/recent.atom
    #url(r'^recent.rss$', 'TODO', { "category_slug": None }), # /news/recent.rss
    (r'^(?P<category_slug>.+).rss$', NewsCategoryRssFeed()), # /news/finances/recent.rss
    (r'^(?P<category_slug>.+)/', 'list_news'), # Catchall index page for a category
    url(r'^', 'list_news', { "category_slug": None }), # Catchall index page
)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import os.path
import subprocess
import json
import re
import argparse

print " "

def outfileexists(outname):
    if os.path.isfile(outname):
        return True
    else:
        return False

def assembledict(mydict, mykeys, dockjson):
    userdict = {'UserParams': {'restart': '', 'rm': '' , 'dockercommand': '',
                             'sig-proxy':''
                            }}
    for desc in mykeys:
        newdict = {desc: {}}
        for keys in mykeys[desc]:
            newdict[desc][keys] = dockjson[desc][keys]
        mydict.append(newdict)
    if dockjson['Name'] != "":
        namedict = {'Name': dockjson['Name'] }
        mydict.append(namedict)
    mydict.append(userdict)
    return mydict

def checkcontaineruid(cuid):
    """Checks ID and returns valid containeruid. Accepts partial UID"""
    proc = subprocess.Popen(["docker ps -q"],
                            stdout=subprocess.PIPE, shell=True)
    out = proc.stdout.read()
    containeruids = out.split()
    if not len(cuid) >= 3:
        print "Container ID must be at least 3 characters"
        quit()
    else:
        for containeruid in containeruids:
            m = re.match(cuid, containeruid)
            if m:
                return containeruid
            else:
                print "Unable to find container ID '%s'. Try 'docker ps'." % cuid
                quit()

def writeoutput(outfile):
    exists = outfileexists(outname)
    if (not args.force) and (exists):
        print ("{0} already exists. You can pass \
               -f to override".format(outname))
        quit()
    with open(outname, "w") as outfile:
        json.dump(vals, outfile, indent=4)
    outfile.closed
    print ("Wrote {0}".format(outname))

parser = argparse.ArgumentParser()
parser.add_argument("cuid",
                    metavar="CONTAINER_ID",
                    help="Container ID. Provide at least 3 characters.")
parser.add_argument("-f", "--force", default=False,
                    action="store_true",
                    help="Force overwriting the output file")

args = parser.parse_args()

cuid = checkcontaineruid(args.cuid)

# Do I need to check if they are running?
# docker inspect works on images and contianers
# check if cuid is in the 'docker ps -a' list?

outname = ("{0}.json".format(cuid))

mycommand = "docker inspect %s" % cuid
containerproc = subprocess.Popen([mycommand],
                                 stdout=subprocess.PIPE, shell=True)
dockjson = json.loads(containerproc.stdout.read())[0]

newjson = dict
newconfig = []
newhost = dict

configkeys = {'Config': {'AttachStderr', 'AttachStdin', 'AttachStdout',
                         'CpuShares', 'Cpuset', 'Env', 'Hostname',
                         'Image', 'Memory', 'Tty', 'User', 'WorkingDir'
                         },
              'HostConfig': {'Binds', 'CapAdd', 'CapDrop', 'ContainerIDFile', 'Dns',
                             'DnsSearch', 'Links', 'LxcConf', 'NetworkMode',
                             'PortBindings', 'Privileged', 'PublishAllPorts'
                             }
              }

vals = assembledict(newconfig, configkeys, dockjson)
writeoutput(outname)
<commit_msg>add argument for output filename<commit_after>#!/usr/bin/env python

import os
import os.path
import subprocess
import json
import re
import argparse

print " "

def outfileexists():
    if os.path.isfile(outname):
        return True
    else:
        return False

def assembledict(mydict, mykeys, dockjson):
    userdict = {'UserParams': {'restart': '', 'rm': '' , 'dockercommand': '',
                             'sig-proxy':''
                            }}
    for desc in mykeys:
        newdict = {desc: {}}
        for keys in mykeys[desc]:
            newdict[desc][keys] = dockjson[desc][keys]
        mydict.append(newdict)
    if dockjson['Name'] != "":
        namedict = {'Name': dockjson['Name'] }
        mydict.append(namedict)
    mydict.append(userdict)
    return mydict

def checkcontaineruid(cuid):
    """Checks ID and returns valid containeruid. Accepts partial UID"""
    proc = subprocess.Popen(["docker ps -q"],
                            stdout=subprocess.PIPE, shell=True)
    out = proc.stdout.read()
    containeruids = out.split()
    if not len(cuid) >= 3:
        print "Container ID must be at least 3 characters"
        quit()
    else:
        for containeruid in containeruids:
            m = re.match(cuid, containeruid)
            if m:
                return containeruid
            else:
                print "Unable to find container ID '%s'. Try 'docker ps'." % cuid
                quit()

def writeoutput():
    #outname = outname()
    exists = outfileexists()
    if (not args.force) and (exists):
        print ("{0} already exists. You can pass \
               -f to override".format(outname))
        quit()
    with open(outname, "w") as outfile:
        json.dump(vals, outfile, indent=4)
    outfile.closed
    print ("Wrote {0}".format(outname))

parser = argparse.ArgumentParser()
parser.add_argument("cuid",
                    metavar="CONTAINER_ID",
                    help="Container ID. Provide at least 3 characters.")
parser.add_argument("-f", "--force", default=False,
                    action="store_true",
                    help="Force overwriting the output file")
parser.add_argument("-o", "--output",
                    metavar="OUTPUT.JSON",
                    help="Specify the output filename")

args = parser.parse_args()

def outname():
    if args.output:
        return args.output
    else:
        return "{0}.json".format(args.cuid)

outname = outname()

cuid = checkcontaineruid(args.cuid)

# Do I need to check if they are running?
# docker inspect works on images and contianers
# check if cuid is in the 'docker ps -a' list?

mycommand = "docker inspect %s" % cuid
containerproc = subprocess.Popen([mycommand],
                                 stdout=subprocess.PIPE, shell=True)
dockjson = json.loads(containerproc.stdout.read())[0]

newjson = dict
newconfig = []
newhost = dict

configkeys = {'Config': {'AttachStderr', 'AttachStdin', 'AttachStdout',
                         'CpuShares', 'Cpuset', 'Env', 'Hostname',
                         'Image', 'Memory', 'Tty', 'User', 'WorkingDir'
                         },
              'HostConfig': {'Binds', 'CapAdd', 'CapDrop', 'ContainerIDFile', 'Dns',
                             'DnsSearch', 'Links', 'LxcConf', 'NetworkMode',
                             'PortBindings', 'Privileged', 'PublishAllPorts'
                             }
              }

vals = assembledict(newconfig, configkeys, dockjson)
writeoutput()
<|endoftext|>


<commit_before>__author__ = 'deathowl'

from twilio.rest import TwilioRestClient

class TwilioSmsNotifier:

    def __init__(self, config):
        self.__config = config

    def notify(self, notification):
        max_length = 140
        message = (notification.message[:max_length-2] + '..') if len(notification.message) > max_length else notification.message
        client = TwilioRestClient(self.__config['SID'], self.__config['token'])
        try:
            client.sms.messages.create(body=message,
                to=notification.user_to_notify.profile.phone_number,
                from_=self.__config['sms_number'])
            print 'successfully sent the sms'
        except :
            print 'failed to send sms'<commit_msg>max length of sms is actually 160<commit_after>__author__ = 'deathowl'

from twilio.rest import TwilioRestClient

class TwilioSmsNotifier:

    def __init__(self, config):
        self.__config = config

    def notify(self, notification):
        max_length = 160
        message = (notification.message[:max_length-2] + '..') if len(notification.message) > max_length else notification.message
        client = TwilioRestClient(self.__config['SID'], self.__config['token'])
        try:
            client.sms.messages.create(body=message,
                to=notification.user_to_notify.profile.phone_number,
                from_=self.__config['sms_number'])
            print 'successfully sent the sms'
        except :
            print 'failed to send sms'<|endoftext|>


<commit_before>import os
import signal
import traceback
from time import sleep

from multiprocessing import Process
from subprocess import PIPE, Popen

from core import consts
from core import models

class TaskManager:
    def run(self):
        """ Tasks monitoring and check status"""
        while True:
            self.check_in_progress()
            self.start_wait_tasks()
            sleep(1)

    def check_in_progress(self):
        tasks = models.Task.objects.filter(status=consts.IN_PROGRESS, pid__isnull=False)
        not_run_tasks = []
        for task in tasks:
            if not task.pid:
                models.TaskLog.objects.create(
                    task=task,
                    status=consts.FAIL,
                    message='Error "pid is None"',
                )
                continue

            if not self._is_process_run(task.pid):
                not_run_tasks.append(task)

        for task in not_run_tasks:
            task.status = consts.FAIL
            task.save()

            models.TaskLog.objects.create(
                task=task,
                status=consts.FAIL,
                message='Task with pid %s is not running' % task.pid
            )

    def start_wait_tasks(self):
        tasks = models.Task.objects.filter(status=consts.WAIT)
        for task in tasks:
            self.run_task_process(task)

    @staticmethod
    def run_task(task_id):
        from django.db import connection
        connection.connection.close()
        connection.connection = None

        task = models.Task.objects.get(id=task_id)

        task.logs.create(
            status=consts.IN_PROGRESS,
            message='Run with pid %s in new process' % task.pid,
        )

        command = task.get_command(splited=True)
        inventory_file_path = command[2]
        task.logs.create(
            status=consts.IN_PROGRESS,
            message='Command: %s' % ' '.join(command),
        )

        try:
            proc = Popen(command, stdout=PIPE, stderr=PIPE)
            while proc.poll() is None:
                output = proc.stdout.readline()

                if output:
                    models.TaskLog.objects.create(
                        task=task,
                        status=consts.IN_PROGRESS,
                        output=output
                    )

            code = proc.returncode
            if code == 0:
                output = proc.stdout.readline()
                if output:
                    models.TaskLog.objects.create(
                        task=task,
                        status=consts.IN_PROGRESS,
                        output=output
                    )
                task.status = consts.COMPLETED
                task.save()
                models.TaskLog.objects.create(
                    task=task,
                    status=consts.COMPLETED,
                    message='Task complete'
                )
            else:
                task.status = consts.FAIL
                task.save()
                _, err = proc.communicate()
                models.TaskLog.objects.create(
                    task=task,
                    status=consts.FAIL,
                    output=err,
                    message='Failed with status code %s' % code
                )

        except Exception as e:
            task.status = consts.FAIL
            task.save()
            traceback_message = traceback.format_exc()
            models.TaskLog.objects.create(
                task=task,
                output=traceback_message,
                message='Progress error "%s"' % e,
                status=consts.FAIL
            )
        finally:
            os.remove(inventory_file_path)
            if os.path.exists("/proc/%s" % task.pid):
                os.kill(task.pid, signal.SIGTERM)

    @staticmethod
    def stop_task(task):
        assert task.status in consts.RUN_STATUSES

        if task.status == consts.IN_PROGRESS and task.pid:
            try:
                os.kill(task.pid, signal.SIGTERM)
            except Exception as e:
                traceback_message = traceback.format_exc()
                models.TaskLog.objects.create(
                    task=task,
                    output=traceback_message,
                    messgae='Stop error "%s"' % e,
                    status=consts.FAIL
                )

        task.status = consts.STOPPED
        task.save()
        models.TaskLog.objects.create(
            task=task,
            status=consts.STOPPED,
            message='Task stopped'
        )

    def run_task_process(self, task):
        proc = Process(target=self.run_task, args=(task.id,))
        proc.start()
        pid = proc.pid

        from django.db import connection
        connection.connection.close()
        connection.connection = None

        models.TaskLog.objects.create(
            task=task,
            status=consts.IN_PROGRESS,
            message='Start task with pid %s' % pid,
        )

        task.pid = pid
        task.status = consts.IN_PROGRESS
        task.save()

    @staticmethod
    def _is_process_run(pid):
        return os.path.exists("/proc/%s" % pid)

<commit_msg>fix #9 - переработал параллельную обработку вывода комманды<commit_after>import os
import signal
import traceback
from time import sleep
from multiprocessing import Process
import asyncio
from asyncio.subprocess import PIPE, create_subprocess_shell

from core import consts
from core import models

class TaskManager:
    def run(self):
        """ Tasks monitoring and check status"""
        while True:
            self.check_in_progress_tasks()
            self.start_waiting_tasks()
            sleep(1)

    def check_in_progress_tasks(self):
        tasks = models.Task.objects.filter(status=consts.IN_PROGRESS, pid__isnull=False)
        not_run_tasks = []
        for task in tasks:
            if not task.pid:
                task.logs.create(
                    status=consts.FAIL,
                    message='Error "pid is None"',
                )
                continue

            if not self._is_process_run(task.pid):
                not_run_tasks.append(task)

        for task in not_run_tasks:
            task.status = consts.FAIL
            task.save()

            task.logs.create(
                status=consts.FAIL,
                message='Task with pid %s is not running' % task.pid
            )

    def start_waiting_tasks(self):
        tasks = models.Task.objects.filter(status=consts.WAIT)
        for task in tasks:
            self.run_task_process(task)

    @classmethod
    def run_task(cls, task_id):
        from django.db import connection
        connection.connection.close()
        connection.connection = None

        task = models.Task.objects.get(id=task_id)

        command = task.get_command(splited=True)
        shell_command = ' '.join(command)
        inventory_file_path = command[2]
        task.logs.create(
            status=consts.IN_PROGRESS,
            message='Command: %s' % shell_command,
        )

        try:
            loop = asyncio.get_event_loop()

            create_proc = create_subprocess_shell(shell_command, stdout=PIPE, stderr=PIPE)
            proc = loop.run_until_complete(create_proc)

            tasks = [
                proc.wait(),
                cls._log_output(task, proc.stdout, consts.IN_PROGRESS),
                cls._log_output(task, proc.stderr, consts.FAIL),
            ]
            loop.run_until_complete(asyncio.wait(tasks))

            code = proc.returncode
            if code == 0:
                task.logs.create(
                    status=consts.COMPLETED,
                    message='Completed.'
                )
                task.status = consts.COMPLETED
                task.save()
                task.logs.create(
                    status=consts.COMPLETED,
                    message='Task complete'
                )
            else:
                task.status = consts.FAIL
                task.save()
                task.logs.create(
                    status=consts.FAIL,
                    message='Failed with status code %s' % code
                )

        except Exception as e:
            task.status = consts.FAIL
            task.save()
            traceback_message = traceback.format_exc()
            task.logs.create(
                output=traceback_message,
                message='Progress error "%s"' % e,
                status=consts.FAIL
            )
        finally:
            os.remove(inventory_file_path)
            if os.path.exists("/proc/%s" % task.pid):
                os.kill(task.pid, signal.SIGTERM)

    @staticmethod
    def stop_task(task):
        assert task.status in consts.RUN_STATUSES

        if task.status == consts.IN_PROGRESS and task.pid:
            try:
                os.kill(task.pid, signal.SIGTERM)
            except Exception as e:
                traceback_message = traceback.format_exc()
                task.logs.create(
                    output=traceback_message,
                    messgae='Stop error "%s"' % e,
                    status=consts.FAIL
                )

        task.status = consts.STOPPED
        task.save()
        task.logs.create(
            status=consts.STOPPED,
            message='Task stopped'
        )

    def run_task_process(self, task):
        proc = Process(target=self.run_task, args=(task.id,))
        proc.start()
        pid = proc.pid

        from django.db import connection
        connection.connection.close()
        connection.connection = None

        task.pid = pid
        task.status = consts.IN_PROGRESS
        task.save()

        task.logs.create(
            status=consts.IN_PROGRESS,
            message='Start task with pid %s' % pid,
        )

    @staticmethod
    def _is_process_run(pid):
        return os.path.exists("/proc/%s" % pid)

    @staticmethod
    async def _log_output(task, stream, status):
        while True:
            output = await stream.readline()
            if not output:
                return

            task.logs.create(
                status=status,
                output=output,
            )<|endoftext|>


<commit_before>"""
Module containing functions to differentiate functions using pytorch.
"""
import functools

try:
    import torch
except ImportError:
    torch = None
else:
    from torch import autograd

from ._backend import Backend
from .. import make_tracing_backend_decorator
from ...tools import flatten_arguments, group_return_values

class _PyTorchBackend(Backend):
    def __init__(self):
        super().__init__("PyTorch")

    @staticmethod
    def is_available():
        return torch is not None and torch.__version__ >= "0.4.1"

    @Backend._assert_backend_available
    def is_compatible(self, function, arguments):
        return callable(function)

    @Backend._assert_backend_available
    def compile_function(self, function, arguments):
        flattened_arguments = flatten_arguments(arguments)

        if len(flattened_arguments) == 1:
            @functools.wraps(function)
            def unary_function(argument):
                return function(torch.from_numpy(argument)).numpy()
            return unary_function

        @functools.wraps(function)
        def nary_function(arguments):
            return function(
                *map(torch.from_numpy, flatten_arguments(arguments))).numpy()
        return nary_function

    def _sanitize_gradient(self, tensor):
        if tensor.grad is None:
            return torch.zeros(tensor.shape, dtype=tensor.dtype).numpy()
        return tensor.grad.numpy()

    def _sanitize_gradients(self, tensors):
        return list(map(self._sanitize_gradient, tensors))

    @Backend._assert_backend_available
    def compute_gradient(self, function, arguments):
        flattened_arguments = flatten_arguments(arguments)

        if len(flattened_arguments) == 1:
            def unary_gradient(argument):
                torch_argument = torch.from_numpy(argument)
                torch_argument.requires_grad_()
                function(torch_argument).backward()
                return self._sanitize_gradient(torch_argument)
            return unary_gradient

        def nary_gradient(arguments):
            torch_arguments = []
            for argument in flatten_arguments(arguments):
                torch_argument = torch.from_numpy(argument)
                torch_argument.requires_grad_()
                torch_arguments.append(torch_argument)
            function(*torch_arguments).backward()
            return self._sanitize_gradients(torch_arguments)
        return group_return_values(nary_gradient, arguments)

    @Backend._assert_backend_available
    def compute_hessian(self, function, arguments):
        flattened_arguments = flatten_arguments(arguments)

        if len(flattened_arguments) == 1:
            def unary_hessian(point, vector):
                x = torch.from_numpy(point)
                v = torch.from_numpy(vector)
                x.requires_grad_()
                fx = function(x)
                (grad_fx,) = autograd.grad(fx, x, create_graph=True,
                                           allow_unused=True)
                (grad_fx * v).sum().backward()
                return self._sanitize_gradient(x)
            return unary_hessian

        def nary_hessian(points, vectors):
            xs = []
            for point in flatten_arguments(points):
                x = torch.from_numpy(point)
                x.requires_grad_()
                xs.append(x)
            vs = [torch.from_numpy(vector)
                  for vector in flatten_arguments(vectors)]
            fx = function(*xs)
            fx.requires_grad_()
            gradients = autograd.grad(fx, xs, create_graph=True,
                                      allow_unused=True)
            dot_product = 0
            for gradient, vector in zip(gradients, vs):
                dot_product += (gradient * vector).sum()
            dot_product.backward()
            return self._sanitize_gradients(xs)
        return group_return_values(nary_hessian, arguments)

PyTorch = make_tracing_backend_decorator(_PyTorchBackend)
<commit_msg>_pytorch.py: use 'tensordot' over manual version<commit_after>"""
Module containing functions to differentiate functions using pytorch.
"""
import functools

try:
    import torch
except ImportError:
    torch = None
else:
    from torch import autograd

from ._backend import Backend
from .. import make_tracing_backend_decorator
from ...tools import flatten_arguments, group_return_values

class _PyTorchBackend(Backend):
    def __init__(self):
        super().__init__("PyTorch")

    @staticmethod
    def is_available():
        return torch is not None and torch.__version__ >= "0.4.1"

    @Backend._assert_backend_available
    def is_compatible(self, function, arguments):
        return callable(function)

    @Backend._assert_backend_available
    def compile_function(self, function, arguments):
        flattened_arguments = flatten_arguments(arguments)

        if len(flattened_arguments) == 1:
            @functools.wraps(function)
            def unary_function(argument):
                return function(torch.from_numpy(argument)).numpy()
            return unary_function

        @functools.wraps(function)
        def nary_function(arguments):
            return function(
                *map(torch.from_numpy, flatten_arguments(arguments))).numpy()
        return nary_function

    def _sanitize_gradient(self, tensor):
        if tensor.grad is None:
            return torch.zeros(tensor.shape, dtype=tensor.dtype).numpy()
        return tensor.grad.numpy()

    def _sanitize_gradients(self, tensors):
        return list(map(self._sanitize_gradient, tensors))

    @Backend._assert_backend_available
    def compute_gradient(self, function, arguments):
        flattened_arguments = flatten_arguments(arguments)

        if len(flattened_arguments) == 1:
            def unary_gradient(argument):
                torch_argument = torch.from_numpy(argument)
                torch_argument.requires_grad_()
                function(torch_argument).backward()
                return self._sanitize_gradient(torch_argument)
            return unary_gradient

        def nary_gradient(arguments):
            torch_arguments = []
            for argument in flatten_arguments(arguments):
                torch_argument = torch.from_numpy(argument)
                torch_argument.requires_grad_()
                torch_arguments.append(torch_argument)
            function(*torch_arguments).backward()
            return self._sanitize_gradients(torch_arguments)
        return group_return_values(nary_gradient, arguments)

    @Backend._assert_backend_available
    def compute_hessian(self, function, arguments):
        flattened_arguments = flatten_arguments(arguments)

        if len(flattened_arguments) == 1:
            def unary_hessian(point, vector):
                x = torch.from_numpy(point)
                v = torch.from_numpy(vector)
                x.requires_grad_()
                fx = function(x)
                (grad_fx,) = autograd.grad(fx, x, create_graph=True,
                                           allow_unused=True)
                torch.tensordot(grad_fx, v, dims=grad_fx.dim()).backward()
                return self._sanitize_gradient(x)
            return unary_hessian

        def nary_hessian(points, vectors):
            xs = []
            for point in flatten_arguments(points):
                x = torch.from_numpy(point)
                x.requires_grad_()
                xs.append(x)
            vs = [torch.from_numpy(vector)
                  for vector in flatten_arguments(vectors)]
            fx = function(*xs)
            fx.requires_grad_()
            gradients = autograd.grad(fx, xs, create_graph=True,
                                      allow_unused=True)
            dot_product = 0
            for gradient, vector in zip(gradients, vs):
                dot_product += torch.tensordot(
                    gradient, vector, dims=gradient.dim())
            dot_product.backward()
            return self._sanitize_gradients(xs)
        return group_return_values(nary_hessian, arguments)

PyTorch = make_tracing_backend_decorator(_PyTorchBackend)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

"""
    pytimekr
    ~~~~~~~~~~~~~

    fork pytime for Korean.

    :copyright: (c) 2015 by Parkayun <iamparkayun@gmail.com>
    :license: MIT, see LICENSE for more details.
"""

import datetime
import calendar
from .filter import BaseParser, str_tuple
from .exception import CanNotFormatError, UnexpectedTypeError

bp = BaseParser.main
dp = BaseParser.parse_diff

def parse(value):
    return bp(value)

def count(value1, value2):
    _val1, _val2 = parse(value1), parse(value2)
    if type(_val1) == type(_val2):
        return _val1 - _val2
    else:
        _val1 = _val1 if isinstance(_val1, datetime.datetime) else midnight(_val1)
        _val2 = _val2 if isinstance(_val2, datetime.datetime) else midnight(_val2)
        return _val1 - _val2

# max, min

_date = datetime.date.today()
_datetime = datetime.datetime.now()
_year = _date.year
_month = _date.month
_day = _date.day

_SEVEN_DAYS = datetime.timedelta(days=7)
_ONE_DAY = datetime.timedelta(days=1)

def today(year=None):
    """this day, last year"""
    return datetime.date(int(year), _date.month, _date.day) if year else _date

def tomorrow(date=None):
    """tomorrow is another day"""
    if not date:
        return _date + datetime.timedelta(days=1)
    else:
        current_date = parse(date)
        return current_date + datetime.timedelta(days=1)

def yesterday(date=None):
    """yesterday once more"""
    if not date:
        return _date - datetime.timedelta(days=1)
    else:
        current_date = parse(date)
        return current_date - datetime.timedelta(days=1)

########################
# function method
########################

def daysrange(first=None, second=None, wipe=False):
    """
    get all days between first and second

    :param first: datetime, date or string
    :param second: datetime, date or string
    :return: list
    """
    _first, _second = parse(first), parse(second)
    (_start, _end) = (_second, _first) if _first > _second else (_first, _second)
    days_between = (_end - _start).days
    date_list = [_end - datetime.timedelta(days=x) for x in range(0, days_between + 1)]
    if wipe and len(date_list) >= 2:
        date_list = date_list[1:-1]
    return date_list

def lastday(year=_year, month=_month):
    """
    get the current month's last day
    :param year:  default to current year
    :param month:  default to current month
    :return: month's last day
    """
    last_day = calendar.monthrange(year, month)[1]
    return datetime.date(year=year, month=month, day=last_day)

def midnight(arg=None):
    """
    convert date to datetime as midnight or get current day's midnight
    :param arg: string or date/datetime
    :return: datetime at 00:00:00
    """
    if arg:
        _arg = parse(arg)
        if isinstance(_arg, datetime.date):
            return datetime.datetime.combine(_arg, datetime.datetime.min.time())
        elif isinstance(_arg, datetime.datetime):
            return datetime.datetime.combine(_arg.date(), datetime.datetime.min.time())
    else:
        return datetime.datetime.combine(_date, datetime.datetime.min.time())

def before(base=_datetime, diff=None):
    """
    count datetime before `base` time
    :param base:  minuend -> str/datetime/date
    :param diff:  str
    :return: datetime
    """
    _base = parse(base)
    if isinstance(_base, datetime.date):
        _base = midnight(_base)
    if not diff:
        return _base
    result_dict = dp(diff)
    # weeks already convert to days in diff_parse function(dp)
    for unit in result_dict:
        _val = result_dict[unit]
        if not _val:
            continue
        if unit == 'years':
            _base = _base.replace(year=(_base.year - _val))
        elif unit == 'months':
            if _base.month <= _val:
                _month_diff = 12 - (_val - _base.month)
                _base = _base.replace(year=_base.year - 1).replace(month=_month_diff)
            else:
                _base = _base.replace(month=_base.month - _val)
        elif unit in ['days', 'hours', 'minutes', 'seconds']:
            _base = _base - datetime.timedelta(**{unit: _val})
    return _base

def after(base=_datetime, diff=None):
    """
    count datetime after diff args
    :param base: str/datetime/date
    :param diff: str
    :return: datetime
    """
    _base = parse(base)
    if isinstance(_base, datetime.date):
        _base = midnight(_base)
    result_dict = dp(diff)
    for unit in result_dict:
        _val = result_dict[unit]
        if not _val:
            continue
        if unit == 'years':
            _base = _base.replace(year=(_base.year + _val))
        elif unit == 'months':
            if _base.month + _val <= 12:
                _base = _base.replace(month=_base.month + _val)
            else:
                _month_diff = (_base.month + _val) - 12
                _base = _base.replace(year=_base.year + 1).replace(month=_month_diff)
        elif unit in ['days', 'hours', 'minutes', 'seconds']:
            _base = _base + datetime.timedelta(**{unit: _val})
    return _base

def _datetime_to_date(arg):
    """
    convert datetime/str to date
    :param arg:
    :return:
    """
    _arg = parse(arg)
    if isinstance(_arg, datetime.datetime):
        _arg = _arg.date()
    return _arg

# Monday to Monday -> 00:00:00 to 00:00:00   month 1st -  next month 1st
def this_week(arg=_date, clean=False):
    _arg = _datetime_to_date(arg)
    return _arg - datetime.timedelta(days=_arg.weekday()), _arg + datetime.timedelta(
        days=6 - _arg.weekday()) if clean else _arg + datetime.timedelta(days=6 - _arg.weekday()) + _ONE_DAY

def last_week(arg=_date, clean=False):
    this_week_tuple = this_week(arg)
    return this_week_tuple[0] - _SEVEN_DAYS, this_week_tuple[1] - _SEVEN_DAYS if clean \
        else this_week_tuple[1] - _SEVEN_DAYS + _ONE_DAY

def next_week(arg=_date, clean=False):
    this_week_tuple = this_week(arg)
    return this_week_tuple[0] + _SEVEN_DAYS, this_week_tuple[1] + _SEVEN_DAYS if clean \
        else this_week_tuple[1] + _SEVEN_DAYS + _ONE_DAY

def this_month(arg=_date, clean=False):
    _arg = _datetime_to_date(arg)
    return datetime.date(_arg.year, _arg.month, 1), lastday(_arg.year, _arg.month) if clean \
        else lastday(_arg.year, _arg.month) + _ONE_DAY

def last_month(arg=_date, clean=False):
    _arg = _datetime_to_date(arg)
    this_month_first_day = datetime.date(_arg.year, _arg.month, 1)
    last_month_last_day = this_month_first_day - _ONE_DAY
    last_month_first_day = datetime.date(last_month_last_day.year, last_month_last_day.month, 1)
    return last_month_first_day, last_month_last_day if clean else this_month_first_day

def next_month(arg=_date, clean=False):
    _arg = _datetime_to_date(arg)
    this_month_last_day = lastday(_arg.year, _arg.month)
    next_month_first_day = this_month_last_day + _ONE_DAY
    next_month_last_day = lastday(next_month_first_day.year, next_month_first_day.month)
    return next_month_first_day, next_month_last_day if clean else next_month_last_day + _ONE_DAY

######################
# festival
######################

def newyear(year=None):
    return datetime.date(int(year), 1, 1) if year else datetime.date(_year, 1, 1)

def valentine(year=None):
    return datetime.date(int(year), 2, 14) if year else datetime.date(_year, 2, 14)

def fool(year=None):
    return datetime.date(int(year), 4, 1) if year else datetime.date(_year, 4, 1)

def christmas(year=None):
    return datetime.date(int(year), 12, 25) if year else datetime.date(_year, 12, 25)

def christeve(year=None):
    return yesterday(christmas(year))

def mother(year=None):
    """
    the 2nd Sunday in May
    :param year: int
    :return: Mother's day
    """
    may_first = datetime.date(_year, 5, 1) if not year else datetime.date(int(year), 5, 1)
    weekday_seq = may_first.weekday()
    return datetime.date(may_first.year, 5, (14 - weekday_seq))

def father(year=None):
    """
    the 3rd Sunday in June
    :param year: int
    :return: Father's day
    """
    june_first = datetime.date(_year, 6, 1) if not year else datetime.date(int(year), 6, 1)
    weekday_seq = june_first.weekday()
    return datetime.date(june_first.year, 6, (21 - weekday_seq))

def halloween(year=None):
    return lastday(month=10) if not year else lastday(year, 10)

def easter(year=None):
    """
    1900 - 2099 limit
    :param year: int
    :return: Easter day
    """
    y = int(year) if year else _year
    n = y - 1900
    a = n % 19
    q = n // 4
    b = (7 * a + 1) // 19
    m = (11 * a + 4 - b) % 29
    w = (n + q + 31 - m) % 7
    d = 25 - m - w
    if d > 0:
        return datetime.date(y, 4, d)
    else:
        return datetime.date(y, 3, (31 - d))

def thanks(year=None):
    """
    4rd Thursday in Nov
    :param year: int
    :return: Thanksgiving Day
    """
    nov_first = datetime.date(_year, 11, 1) if not year else datetime.date(int(year), 11, 1)
    weekday_seq = nov_first.weekday()
    if weekday_seq > 3:
        current_day = 32 - weekday_seq
    else:
        current_day = 25 - weekday_seq
    return datetime.date(nov_first.year, 11, current_day)

def independence(year=None):
    """
    15, August
    :param year: int
    :return: Korea's Independence Day
    """
    return datetime.date(int(year), 8, 15) if year else datetime.date(_year, 8, 15)

def hangul(year=None):
    """
    9, October
    :param yare: int
    :return: Hangul Day
    """
    return datetime.date(int(year), 10, 9) if year else datetime.date(_year, 10, 9)

def constitution(year=None):
    """
    17, July
    :param year: int
    :return: Korea's Constitution Day
    """
    return datetime.date(int(year), 7, 17) if year else datetime.date(_year, 7, 17)

if __name__ == '__main__':
    # _time_filter('2015-01-03')
    # print(calendar.monthrange(2015, 10))
    print(bp('2015-01-03'))
<commit_msg>Fix typo<commit_after>#!/usr/bin/env python
# encoding: utf-8

"""
    pytimekr
    ~~~~~~~~~~~~~

    fork pytime for Korean.

    :copyright: (c) 2015 by Parkayun <iamparkayun@gmail.com>
    :license: MIT, see LICENSE for more details.
"""

import datetime
import calendar
from .filter import BaseParser, str_tuple
from .exception import CanNotFormatError, UnexpectedTypeError

bp = BaseParser.main
dp = BaseParser.parse_diff

def parse(value):
    return bp(value)

def count(value1, value2):
    _val1, _val2 = parse(value1), parse(value2)
    if type(_val1) == type(_val2):
        return _val1 - _val2
    else:
        _val1 = _val1 if isinstance(_val1, datetime.datetime) else midnight(_val1)
        _val2 = _val2 if isinstance(_val2, datetime.datetime) else midnight(_val2)
        return _val1 - _val2

# max, min

_date = datetime.date.today()
_datetime = datetime.datetime.now()
_year = _date.year
_month = _date.month
_day = _date.day

_SEVEN_DAYS = datetime.timedelta(days=7)
_ONE_DAY = datetime.timedelta(days=1)

def today(year=None):
    """this day, last year"""
    return datetime.date(int(year), _date.month, _date.day) if year else _date

def tomorrow(date=None):
    """tomorrow is another day"""
    if not date:
        return _date + datetime.timedelta(days=1)
    else:
        current_date = parse(date)
        return current_date + datetime.timedelta(days=1)

def yesterday(date=None):
    """yesterday once more"""
    if not date:
        return _date - datetime.timedelta(days=1)
    else:
        current_date = parse(date)
        return current_date - datetime.timedelta(days=1)

########################
# function method
########################

def daysrange(first=None, second=None, wipe=False):
    """
    get all days between first and second

    :param first: datetime, date or string
    :param second: datetime, date or string
    :return: list
    """
    _first, _second = parse(first), parse(second)
    (_start, _end) = (_second, _first) if _first > _second else (_first, _second)
    days_between = (_end - _start).days
    date_list = [_end - datetime.timedelta(days=x) for x in range(0, days_between + 1)]
    if wipe and len(date_list) >= 2:
        date_list = date_list[1:-1]
    return date_list

def lastday(year=_year, month=_month):
    """
    get the current month's last day
    :param year:  default to current year
    :param month:  default to current month
    :return: month's last day
    """
    last_day = calendar.monthrange(year, month)[1]
    return datetime.date(year=year, month=month, day=last_day)

def midnight(arg=None):
    """
    convert date to datetime as midnight or get current day's midnight
    :param arg: string or date/datetime
    :return: datetime at 00:00:00
    """
    if arg:
        _arg = parse(arg)
        if isinstance(_arg, datetime.date):
            return datetime.datetime.combine(_arg, datetime.datetime.min.time())
        elif isinstance(_arg, datetime.datetime):
            return datetime.datetime.combine(_arg.date(), datetime.datetime.min.time())
    else:
        return datetime.datetime.combine(_date, datetime.datetime.min.time())

def before(base=_datetime, diff=None):
    """
    count datetime before `base` time
    :param base:  minuend -> str/datetime/date
    :param diff:  str
    :return: datetime
    """
    _base = parse(base)
    if isinstance(_base, datetime.date):
        _base = midnight(_base)
    if not diff:
        return _base
    result_dict = dp(diff)
    # weeks already convert to days in diff_parse function(dp)
    for unit in result_dict:
        _val = result_dict[unit]
        if not _val:
            continue
        if unit == 'years':
            _base = _base.replace(year=(_base.year - _val))
        elif unit == 'months':
            if _base.month <= _val:
                _month_diff = 12 - (_val - _base.month)
                _base = _base.replace(year=_base.year - 1).replace(month=_month_diff)
            else:
                _base = _base.replace(month=_base.month - _val)
        elif unit in ['days', 'hours', 'minutes', 'seconds']:
            _base = _base - datetime.timedelta(**{unit: _val})
    return _base

def after(base=_datetime, diff=None):
    """
    count datetime after diff args
    :param base: str/datetime/date
    :param diff: str
    :return: datetime
    """
    _base = parse(base)
    if isinstance(_base, datetime.date):
        _base = midnight(_base)
    result_dict = dp(diff)
    for unit in result_dict:
        _val = result_dict[unit]
        if not _val:
            continue
        if unit == 'years':
            _base = _base.replace(year=(_base.year + _val))
        elif unit == 'months':
            if _base.month + _val <= 12:
                _base = _base.replace(month=_base.month + _val)
            else:
                _month_diff = (_base.month + _val) - 12
                _base = _base.replace(year=_base.year + 1).replace(month=_month_diff)
        elif unit in ['days', 'hours', 'minutes', 'seconds']:
            _base = _base + datetime.timedelta(**{unit: _val})
    return _base

def _datetime_to_date(arg):
    """
    convert datetime/str to date
    :param arg:
    :return:
    """
    _arg = parse(arg)
    if isinstance(_arg, datetime.datetime):
        _arg = _arg.date()
    return _arg

# Monday to Monday -> 00:00:00 to 00:00:00   month 1st -  next month 1st
def this_week(arg=_date, clean=False):
    _arg = _datetime_to_date(arg)
    return _arg - datetime.timedelta(days=_arg.weekday()), _arg + datetime.timedelta(
        days=6 - _arg.weekday()) if clean else _arg + datetime.timedelta(days=6 - _arg.weekday()) + _ONE_DAY

def last_week(arg=_date, clean=False):
    this_week_tuple = this_week(arg)
    return this_week_tuple[0] - _SEVEN_DAYS, this_week_tuple[1] - _SEVEN_DAYS if clean \
        else this_week_tuple[1] - _SEVEN_DAYS + _ONE_DAY

def next_week(arg=_date, clean=False):
    this_week_tuple = this_week(arg)
    return this_week_tuple[0] + _SEVEN_DAYS, this_week_tuple[1] + _SEVEN_DAYS if clean \
        else this_week_tuple[1] + _SEVEN_DAYS + _ONE_DAY

def this_month(arg=_date, clean=False):
    _arg = _datetime_to_date(arg)
    return datetime.date(_arg.year, _arg.month, 1), lastday(_arg.year, _arg.month) if clean \
        else lastday(_arg.year, _arg.month) + _ONE_DAY

def last_month(arg=_date, clean=False):
    _arg = _datetime_to_date(arg)
    this_month_first_day = datetime.date(_arg.year, _arg.month, 1)
    last_month_last_day = this_month_first_day - _ONE_DAY
    last_month_first_day = datetime.date(last_month_last_day.year, last_month_last_day.month, 1)
    return last_month_first_day, last_month_last_day if clean else this_month_first_day

def next_month(arg=_date, clean=False):
    _arg = _datetime_to_date(arg)
    this_month_last_day = lastday(_arg.year, _arg.month)
    next_month_first_day = this_month_last_day + _ONE_DAY
    next_month_last_day = lastday(next_month_first_day.year, next_month_first_day.month)
    return next_month_first_day, next_month_last_day if clean else next_month_last_day + _ONE_DAY

######################
# festival
######################

def newyear(year=None):
    return datetime.date(int(year), 1, 1) if year else datetime.date(_year, 1, 1)

def valentine(year=None):
    return datetime.date(int(year), 2, 14) if year else datetime.date(_year, 2, 14)

def fool(year=None):
    return datetime.date(int(year), 4, 1) if year else datetime.date(_year, 4, 1)

def christmas(year=None):
    return datetime.date(int(year), 12, 25) if year else datetime.date(_year, 12, 25)

def christeve(year=None):
    return yesterday(christmas(year))

def mother(year=None):
    """
    the 2nd Sunday in May
    :param year: int
    :return: Mother's day
    """
    may_first = datetime.date(_year, 5, 1) if not year else datetime.date(int(year), 5, 1)
    weekday_seq = may_first.weekday()
    return datetime.date(may_first.year, 5, (14 - weekday_seq))

def father(year=None):
    """
    the 3rd Sunday in June
    :param year: int
    :return: Father's day
    """
    june_first = datetime.date(_year, 6, 1) if not year else datetime.date(int(year), 6, 1)
    weekday_seq = june_first.weekday()
    return datetime.date(june_first.year, 6, (21 - weekday_seq))

def halloween(year=None):
    return lastday(month=10) if not year else lastday(year, 10)

def easter(year=None):
    """
    1900 - 2099 limit
    :param year: int
    :return: Easter day
    """
    y = int(year) if year else _year
    n = y - 1900
    a = n % 19
    q = n // 4
    b = (7 * a + 1) // 19
    m = (11 * a + 4 - b) % 29
    w = (n + q + 31 - m) % 7
    d = 25 - m - w
    if d > 0:
        return datetime.date(y, 4, d)
    else:
        return datetime.date(y, 3, (31 - d))

def thanks(year=None):
    """
    4rd Thursday in Nov
    :param year: int
    :return: Thanksgiving Day
    """
    nov_first = datetime.date(_year, 11, 1) if not year else datetime.date(int(year), 11, 1)
    weekday_seq = nov_first.weekday()
    if weekday_seq > 3:
        current_day = 32 - weekday_seq
    else:
        current_day = 25 - weekday_seq
    return datetime.date(nov_first.year, 11, current_day)

def independence(year=None):
    """
    15, August
    :param year: int
    :return: Korea's Independence Day
    """
    return datetime.date(int(year), 8, 15) if year else datetime.date(_year, 8, 15)

def hangul(year=None):
    """
    9, October
    :param year: int
    :return: Hangul Day
    """
    return datetime.date(int(year), 10, 9) if year else datetime.date(_year, 10, 9)

def constitution(year=None):
    """
    17, July
    :param year: int
    :return: Korea's Constitution Day
    """
    return datetime.date(int(year), 7, 17) if year else datetime.date(_year, 7, 17)

if __name__ == '__main__':
    # _time_filter('2015-01-03')
    # print(calendar.monthrange(2015, 10))
    print(bp('2015-01-03'))
<|endoftext|>