<commit_before>"""
Created on 7/17/16 10:08 AM
@author: Suhas Somnath, Chris Smith
"""

from __future__ import division, print_function, absolute_import

import numpy as np
import psutil
import joblib
import time as tm

from ..io.hdf_utils import check_if_main, check_for_old, get_attributes
from ..io.pycro_data import PycroDataset
from ..io.io_utils import recommend_cpu_cores, get_available_memory, format_time

class Process(object):
    """
    Encapsulates the typical steps performed when applying a processing function to  a dataset.
    """

    def __init__(self, h5_main, cores=None, max_mem_mb=4*1024, verbose=False):
        """
        Parameters
        ----------
        h5_main : h5py.Dataset instance
            The dataset over which the analysis will be performed. This dataset should be linked to the spectroscopic
            indices and values, and position indices and values datasets.
        cores : uint, optional
            Default - all available cores - 2
            How many cores to use for the computation
        max_mem_mb : uint, optional
            How much memory to use for the computation.  Default 1024 Mb
        verbose : Boolean, (Optional, default = False)
            Whether or not to print debugging statements
        """

        if h5_main.file.mode != 'r+':
            raise TypeError('Need to ensure that the file is in r+ mode to write results back to the file')

        # Checking if dataset is "Main"
        if check_if_main(h5_main):
            self.h5_main = PycroDataset(h5_main)
        else:
            raise ValueError('Provided dataset is not a "Main" dataset with necessary ancillary datasets')

        self.verbose = verbose
        self._max_pos_per_read = None
        self._max_mem_mb = None

        self._start_pos = 0
        self._end_pos = self.h5_main.shape[0]

        # Determining the max size of the data that can be put into memory
        self._set_memory_and_cores(cores=cores, mem=max_mem_mb)
        self.duplicate_h5_groups = []
        self.partial_h5_groups = []
        self.process_name = None  # Reset this in the extended classes
        self.parms_dict = None

        self._results = None
        self.h5_results_grp = None

        print('Consider calling test() to check results before calling compute() which computes on the entire'
              ' dataset and writes back to the HDF5 file')

        # DON'T check for duplicates since parms_dict has not yet been initialized.
        # Sub classes will check by themselves if they are interested.

    def test(self, **kwargs):
        """
        Tests the process on a subset (for example a pixel) of the whole data. The class can be reinstantiated with
        improved parameters and tested repeatedly until the user is content, at which point the user can call
        compute() on the whole dataset.

        Parameters
        ----------
        kwargs - dict, optional
            keyword arguments to test the process

        Returns
        -------
        """
        raise NotImplementedError('test_on_subset has not yet been implemented')

    def _check_for_duplicates(self):
        """
        Checks for instances where the process was applied to the same dataset with the same parameters

        Returns
        -------
        duplicate_h5_groups : list of h5py.Datagroup objects
            List of groups satisfying the above conditions
        """
        if self.verbose:
            print('Checking for duplicates:')

        duplicate_h5_groups = check_for_old(self.h5_main, self.process_name, new_parms=self.parms_dict)
        partial_h5_groups = []

        # First figure out which ones are partially completed:
        if len(duplicate_h5_groups) > 0:
            for index, curr_group in enumerate(duplicate_h5_groups):
                if curr_group.attrs['last_pixel'] < self._end_pos:
                    # remove from duplicates and move to partial
                    partial_h5_groups.append(duplicate_h5_groups.pop(index))

        if len(duplicate_h5_groups) > 0:
            print('Note: ' + self.process_name + ' has already been performed with the same parameters before. '
                                                 'These results will be returned by compute() by default. '
                                                 'Set override to True to force fresh computation')
            print(duplicate_h5_groups)

        if partial_h5_groups:
            print('Note: ' + self.process_name + ' has already been performed PARTIALLY with the same parameters. '
                                                 'compute() will resuming computation in the last group below. '
                                                 'To choose a different group call use_patial_computation()'
                                                 'Set override to True to force fresh computation or resume from a '
                                                 'data group besides the last in the list.')
            print(partial_h5_groups)

        return duplicate_h5_groups, partial_h5_groups

    def use_partial_computation(self, h5_partial_group=None):
        """
        Extracts the necessary parameters from the provided h5 group to resume computation

        Parameters
        ----------
        h5_partial_group : h5py.Datagroup object
            Datagroup containing partially computed results
        """
        # Attempt to automatically take partial results
        if h5_partial_group is None:
            if len(self.partial_h5_groups) < 1:
                raise ValueError('No group was found with partial results and no such group was provided')
            h5_partial_group = self.partial_h5_groups[-1]
        else:
            # Make sure that this group is among the legal ones already discovered:
            if h5_partial_group not in self.partial_h5_groups:
                raise ValueError('Provided group does not appear to be in the list of discovered groups')

        self.parms_dict = get_attributes(h5_partial_group)
        self._start_pos = self.parms_dict.pop('last_pixel')
        if self._start_pos == self.h5_main.shape[0] - 1:
            raise ValueError('The last computed pixel shows that the computation was already complete')

        self.h5_results_grp = h5_partial_group

    def _set_memory_and_cores(self, cores=1, mem=1024):
        """
        Checks hardware limitations such as memory, # cpus and sets the recommended datachunk sizes and the
        number of cores to be used by analysis methods.

        Parameters
        ----------
        cores : uint, optional
            Default - 1
            How many cores to use for the computation
        mem : uint, optional
            Default - 1024
            The amount a memory in Mb to use in the computation
        """

        if cores is None:
            self._cores = psutil.cpu_count() - 2
        else:
            cores = int(abs(cores))
            self._cores = min(psutil.cpu_count(), max(1, cores))

        _max_mem_mb = get_available_memory() / 1E6  # in MB

        self._max_mem_mb = min(_max_mem_mb, mem)

        max_data_chunk = self._max_mem_mb / self._cores

        # Now calculate the number of positions that can be stored in memory in one go.
        mb_per_position = self.h5_main.dtype.itemsize * self.h5_main.shape[1] / 1e6
        self._max_pos_per_read = int(np.floor(max_data_chunk / mb_per_position))

        if self.verbose:
            print('Allowed to read {} pixels per chunk'.format(self._max_pos_per_read))
            print('Allowed to use up to', str(self._cores), 'cores and', str(self._max_mem_mb), 'MB of memory')

    @staticmethod
    def _map_function(*args, **kwargs):
        """
        The function that manipulates the data on a single instance (position). This will be used by _unit_computation()
        to process a chunk of data in parallel

        Parameters
        ----------
        args : list
            arguments to the function in the correct order
        kwargs : dictionary
            keyword arguments to the function

        Returns
        -------
        object
        """
        raise NotImplementedError('Please override the _unit_function specific to your process')

    def _read_data_chunk(self):
        """
        Reads a chunk of data for the intended computation into memory
        """
        if self._start_pos < self.h5_main.shape[0]:
            self._end_pos = int(min(self.h5_main.shape[0], self._start_pos + self._max_pos_per_read))
            self.data = self.h5_main[self._start_pos:self._end_pos, :]
            if self.verbose:
                print('Reading pixels {} to {} of {}'.format(self._start_pos, self._end_pos, self.h5_main.shape[0]))

            # DON'T update the start position

        else:
            if self.verbose:
                print('Finished reading all data!')
            self.data = None

    def _write_results_chunk(self):
        """
        Writes the computed results into appropriate datasets.

        This needs to be rewritten since the processed data is expected to be at least as large as the dataset
        """
        # Now update the start position
        self._start_pos = self._end_pos
        raise NotImplementedError('Please override the _set_results specific to your process')

    def _create_results_datasets(self):
        """
        Process specific call that will write the h5 group, guess dataset, corresponding spectroscopic datasets and also
        link the guess dataset to the spectroscopic datasets. It is recommended that the ancillary datasets be populated
        within this function.
        """
        raise NotImplementedError('Please override the _create_results_datasets specific to your process')

    def _get_existing_datasets(self):
        """
        The purpose of this function is to allow processes to resume from partly computed results

        Start with self.h5_results_grp
        """
        raise NotImplementedError('Please override the _get_existing_datasets specific to your process')

    def _unit_computation(self, *args, **kwargs):
        """
        The unit computation that is performed per data chunk. This allows room for any data pre / post-processing
        as well as multiple calls to parallel_compute if necessary
        """
        # TODO: Try to use the functools.partials to preconfigure the map function
        self._results = parallel_compute(self.data, self._map_function, cores=self._cores,
                                         lengthy_computation=False,
                                         func_args=args, func_kwargs=kwargs)

    def compute(self, *args, override=False, **kwargs):
        """
        Creates placeholders for the results, applies the unit computation to chunks of the dataset

        Parameters
        ----------
        override : bool, optional. default = False
            By default, compute will simply return duplicate results to avoid recomputing or resume computation on a
            group with partial results. Set to True to force fresh computation.
        args : list
            arguments to the mapped function in the correct order
        kwargs : dictionary
            keyword arguments to the mapped function

        Returns
        -------
        h5_results_grp : h5py.Datagroup object
            Datagroup containing all the results
        """
        if not override:
            if len(self.duplicate_h5_groups) > 0:
                print('Returned previously computed results at ' + self.duplicate_h5_groups[-1].name)
                return self.duplicate_h5_groups[-1]
            elif len(self.partial_h5_groups) > 0:
                print('Resuming computation in group: ' + self.partial_h5_groups[-1].name)
                self.use_partial_computation()

        if self.h5_results_grp is None:
            # starting fresh
            if self.verbose:
                print('Creating datagroup and datasets')
            self._create_results_datasets()
        else:
            # resuming from previous checkpoint
            if self.verbose:
                print('Resuming computation')
            self._get_existing_datasets()

        time_per_pix = 0
        num_pos = self.h5_main.shape[0] - self._start_pos
        orig_start_pos = self._start_pos

        print('You can abort this computation at any time and resume at a later time!\n'
              '\tIf you are operating in a python console, press Ctrl+C or Cmd+C to abort\n'
              '\tIf you are in a Jupyter notebook, click on "Kernel">>"Interrupt"')

        self._read_data_chunk()
        while self.data is not None:

            t_start = tm.time()

            self._unit_computation(*args, **kwargs)

            tot_time = np.round(tm.time() - t_start, decimals=2)
            if self.verbose:
                print('Done parallel computing in {} or {} per pixel'.format(format_time(tot_time),
                                                                             format_time(
                                                                                tot_time / self.data.shape[0])))
            if self._start_pos == orig_start_pos:
                time_per_pix = tot_time / self._end_pos  # in seconds
            else:
                time_remaining = (num_pos - self._end_pos) * time_per_pix  # in seconds
                print('Time remaining: ' + format_time(time_remaining))

            self._write_results_chunk()
            self._read_data_chunk()

        print('Completed computation on chunk. Writing to file.')

        return self.h5_results_grp

def parallel_compute(data, func, cores=1, lengthy_computation=False, func_args=None, func_kwargs=None):
    """
    Computes the guess function using multiple cores

    Parameters
    ----------
    data : numpy.ndarray
        Data to map function to. Function will be mapped to the first axis of data
    func : callable
        Function to map to data
    cores : uint, optional
        Number of logical cores to use to compute
        Default - 1 (serial computation)
    lengthy_computation : bool, optional
        Whether or not each computation is expected to take substantial time.
        Sometimes the time for adding more cores can outweigh the time per core
        Default - False
    func_args : list, optional
        arguments to be passed to the function
    func_kwargs : dict, optional
        keyword arguments to be passed onto function

    Returns
    -------
    results : list
        List of computational results
    """

    if not callable(func):
        raise TypeError('Function argument is not callable')
    if not isinstance(data, np.ndarray):
        raise TypeError('data must be a numpy array')
    if func_args is None:
        func_args = list()
    else:
        if isinstance(func_args, tuple):
            func_args = list(func_args)
        if not isinstance(func_args, list):
            raise TypeError('Arguments to the mapped function should be specified as a list')
    if func_kwargs is None:
        func_kwargs = dict()
    else:
        if not isinstance(func_kwargs, dict):
            raise TypeError('Keyword arguments to the mapped function should be specified via a dictionary')

    cores = recommend_cpu_cores(data.shape[0], requested_cores=cores,
                                lengthy_computation=lengthy_computation)

    if cores > 1:
        values = [joblib.delayed(func)(x, *func_args, **func_kwargs) for x in data]
        results = joblib.Parallel(n_jobs=cores)(values)

        # Finished reading the entire data set
        print('Finished parallel computation')

    else:
        print("Computing serially ...")
        results = [func(vector, *func_args, **func_kwargs) for vector in data]

    return results
<commit_msg>Added TODO About print statement regarding resuming the computation<commit_after>"""
Created on 7/17/16 10:08 AM
@author: Suhas Somnath, Chris Smith
"""

from __future__ import division, print_function, absolute_import

import numpy as np
import psutil
import joblib
import time as tm

from ..io.hdf_utils import check_if_main, check_for_old, get_attributes
from ..io.pycro_data import PycroDataset
from ..io.io_utils import recommend_cpu_cores, get_available_memory, format_time

class Process(object):
    """
    Encapsulates the typical steps performed when applying a processing function to  a dataset.
    """

    def __init__(self, h5_main, cores=None, max_mem_mb=4*1024, verbose=False):
        """
        Parameters
        ----------
        h5_main : h5py.Dataset instance
            The dataset over which the analysis will be performed. This dataset should be linked to the spectroscopic
            indices and values, and position indices and values datasets.
        cores : uint, optional
            Default - all available cores - 2
            How many cores to use for the computation
        max_mem_mb : uint, optional
            How much memory to use for the computation.  Default 1024 Mb
        verbose : Boolean, (Optional, default = False)
            Whether or not to print debugging statements
        """

        if h5_main.file.mode != 'r+':
            raise TypeError('Need to ensure that the file is in r+ mode to write results back to the file')

        # Checking if dataset is "Main"
        if check_if_main(h5_main):
            self.h5_main = PycroDataset(h5_main)
        else:
            raise ValueError('Provided dataset is not a "Main" dataset with necessary ancillary datasets')

        self.verbose = verbose
        self._max_pos_per_read = None
        self._max_mem_mb = None

        self._start_pos = 0
        self._end_pos = self.h5_main.shape[0]

        # Determining the max size of the data that can be put into memory
        self._set_memory_and_cores(cores=cores, mem=max_mem_mb)
        self.duplicate_h5_groups = []
        self.partial_h5_groups = []
        self.process_name = None  # Reset this in the extended classes
        self.parms_dict = None

        self._results = None
        self.h5_results_grp = None

        print('Consider calling test() to check results before calling compute() which computes on the entire'
              ' dataset and writes back to the HDF5 file')

        # DON'T check for duplicates since parms_dict has not yet been initialized.
        # Sub classes will check by themselves if they are interested.

    def test(self, **kwargs):
        """
        Tests the process on a subset (for example a pixel) of the whole data. The class can be reinstantiated with
        improved parameters and tested repeatedly until the user is content, at which point the user can call
        compute() on the whole dataset.

        Parameters
        ----------
        kwargs - dict, optional
            keyword arguments to test the process

        Returns
        -------
        """
        raise NotImplementedError('test_on_subset has not yet been implemented')

    def _check_for_duplicates(self):
        """
        Checks for instances where the process was applied to the same dataset with the same parameters

        Returns
        -------
        duplicate_h5_groups : list of h5py.Datagroup objects
            List of groups satisfying the above conditions
        """
        if self.verbose:
            print('Checking for duplicates:')

        duplicate_h5_groups = check_for_old(self.h5_main, self.process_name, new_parms=self.parms_dict)
        partial_h5_groups = []

        # First figure out which ones are partially completed:
        if len(duplicate_h5_groups) > 0:
            for index, curr_group in enumerate(duplicate_h5_groups):
                if curr_group.attrs['last_pixel'] < self._end_pos:
                    # remove from duplicates and move to partial
                    partial_h5_groups.append(duplicate_h5_groups.pop(index))

        if len(duplicate_h5_groups) > 0:
            print('Note: ' + self.process_name + ' has already been performed with the same parameters before. '
                                                 'These results will be returned by compute() by default. '
                                                 'Set override to True to force fresh computation')
            print(duplicate_h5_groups)

        if partial_h5_groups:
            print('Note: ' + self.process_name + ' has already been performed PARTIALLY with the same parameters. '
                                                 'compute() will resuming computation in the last group below. '
                                                 'To choose a different group call use_patial_computation()'
                                                 'Set override to True to force fresh computation or resume from a '
                                                 'data group besides the last in the list.')
            print(partial_h5_groups)

        return duplicate_h5_groups, partial_h5_groups

    def use_partial_computation(self, h5_partial_group=None):
        """
        Extracts the necessary parameters from the provided h5 group to resume computation

        Parameters
        ----------
        h5_partial_group : h5py.Datagroup object
            Datagroup containing partially computed results
        """
        # Attempt to automatically take partial results
        if h5_partial_group is None:
            if len(self.partial_h5_groups) < 1:
                raise ValueError('No group was found with partial results and no such group was provided')
            h5_partial_group = self.partial_h5_groups[-1]
        else:
            # Make sure that this group is among the legal ones already discovered:
            if h5_partial_group not in self.partial_h5_groups:
                raise ValueError('Provided group does not appear to be in the list of discovered groups')

        self.parms_dict = get_attributes(h5_partial_group)
        self._start_pos = self.parms_dict.pop('last_pixel')
        if self._start_pos == self.h5_main.shape[0] - 1:
            raise ValueError('The last computed pixel shows that the computation was already complete')

        self.h5_results_grp = h5_partial_group

    def _set_memory_and_cores(self, cores=1, mem=1024):
        """
        Checks hardware limitations such as memory, # cpus and sets the recommended datachunk sizes and the
        number of cores to be used by analysis methods.

        Parameters
        ----------
        cores : uint, optional
            Default - 1
            How many cores to use for the computation
        mem : uint, optional
            Default - 1024
            The amount a memory in Mb to use in the computation
        """

        if cores is None:
            self._cores = psutil.cpu_count() - 2
        else:
            cores = int(abs(cores))
            self._cores = min(psutil.cpu_count(), max(1, cores))

        _max_mem_mb = get_available_memory() / 1E6  # in MB

        self._max_mem_mb = min(_max_mem_mb, mem)

        max_data_chunk = self._max_mem_mb / self._cores

        # Now calculate the number of positions that can be stored in memory in one go.
        mb_per_position = self.h5_main.dtype.itemsize * self.h5_main.shape[1] / 1e6
        self._max_pos_per_read = int(np.floor(max_data_chunk / mb_per_position))

        if self.verbose:
            print('Allowed to read {} pixels per chunk'.format(self._max_pos_per_read))
            print('Allowed to use up to', str(self._cores), 'cores and', str(self._max_mem_mb), 'MB of memory')

    @staticmethod
    def _map_function(*args, **kwargs):
        """
        The function that manipulates the data on a single instance (position). This will be used by _unit_computation()
        to process a chunk of data in parallel

        Parameters
        ----------
        args : list
            arguments to the function in the correct order
        kwargs : dictionary
            keyword arguments to the function

        Returns
        -------
        object
        """
        raise NotImplementedError('Please override the _unit_function specific to your process')

    def _read_data_chunk(self):
        """
        Reads a chunk of data for the intended computation into memory
        """
        if self._start_pos < self.h5_main.shape[0]:
            self._end_pos = int(min(self.h5_main.shape[0], self._start_pos + self._max_pos_per_read))
            self.data = self.h5_main[self._start_pos:self._end_pos, :]
            if self.verbose:
                print('Reading pixels {} to {} of {}'.format(self._start_pos, self._end_pos, self.h5_main.shape[0]))

            # DON'T update the start position

        else:
            if self.verbose:
                print('Finished reading all data!')
            self.data = None

    def _write_results_chunk(self):
        """
        Writes the computed results into appropriate datasets.

        This needs to be rewritten since the processed data is expected to be at least as large as the dataset
        """
        # Now update the start position
        self._start_pos = self._end_pos
        raise NotImplementedError('Please override the _set_results specific to your process')

    def _create_results_datasets(self):
        """
        Process specific call that will write the h5 group, guess dataset, corresponding spectroscopic datasets and also
        link the guess dataset to the spectroscopic datasets. It is recommended that the ancillary datasets be populated
        within this function.
        """
        raise NotImplementedError('Please override the _create_results_datasets specific to your process')

    def _get_existing_datasets(self):
        """
        The purpose of this function is to allow processes to resume from partly computed results

        Start with self.h5_results_grp
        """
        raise NotImplementedError('Please override the _get_existing_datasets specific to your process')

    def _unit_computation(self, *args, **kwargs):
        """
        The unit computation that is performed per data chunk. This allows room for any data pre / post-processing
        as well as multiple calls to parallel_compute if necessary
        """
        # TODO: Try to use the functools.partials to preconfigure the map function
        self._results = parallel_compute(self.data, self._map_function, cores=self._cores,
                                         lengthy_computation=False,
                                         func_args=args, func_kwargs=kwargs)

    def compute(self, *args, override=False, **kwargs):
        """
        Creates placeholders for the results, applies the unit computation to chunks of the dataset

        Parameters
        ----------
        override : bool, optional. default = False
            By default, compute will simply return duplicate results to avoid recomputing or resume computation on a
            group with partial results. Set to True to force fresh computation.
        args : list
            arguments to the mapped function in the correct order
        kwargs : dictionary
            keyword arguments to the mapped function

        Returns
        -------
        h5_results_grp : h5py.Datagroup object
            Datagroup containing all the results
        """
        if not override:
            if len(self.duplicate_h5_groups) > 0:
                print('Returned previously computed results at ' + self.duplicate_h5_groups[-1].name)
                return self.duplicate_h5_groups[-1]
            elif len(self.partial_h5_groups) > 0:
                print('Resuming computation in group: ' + self.partial_h5_groups[-1].name)
                self.use_partial_computation()

        if self.h5_results_grp is None:
            # starting fresh
            if self.verbose:
                print('Creating datagroup and datasets')
            self._create_results_datasets()
        else:
            # resuming from previous checkpoint
            if self.verbose:
                print('Resuming computation')
            self._get_existing_datasets()

        time_per_pix = 0
        num_pos = self.h5_main.shape[0] - self._start_pos
        orig_start_pos = self._start_pos

        # TODO: Need to find a nice way of figuring out if a process has implemented the partial feature.
        print('You maybe able to abort this computation at any time and resume at a later time!\n'
              '\tIf you are operating in a python console, press Ctrl+C or Cmd+C to abort\n'
              '\tIf you are in a Jupyter notebook, click on "Kernel">>"Interrupt"')

        self._read_data_chunk()
        while self.data is not None:

            t_start = tm.time()

            self._unit_computation(*args, **kwargs)

            tot_time = np.round(tm.time() - t_start, decimals=2)
            if self.verbose:
                print('Done parallel computing in {} or {} per pixel'.format(format_time(tot_time),
                                                                             format_time(
                                                                                tot_time / self.data.shape[0])))
            if self._start_pos == orig_start_pos:
                time_per_pix = tot_time / self._end_pos  # in seconds
            else:
                time_remaining = (num_pos - self._end_pos) * time_per_pix  # in seconds
                print('Time remaining: ' + format_time(time_remaining))

            self._write_results_chunk()
            self._read_data_chunk()

        print('Completed computation on chunk. Writing to file.')

        return self.h5_results_grp

def parallel_compute(data, func, cores=1, lengthy_computation=False, func_args=None, func_kwargs=None):
    """
    Computes the guess function using multiple cores

    Parameters
    ----------
    data : numpy.ndarray
        Data to map function to. Function will be mapped to the first axis of data
    func : callable
        Function to map to data
    cores : uint, optional
        Number of logical cores to use to compute
        Default - 1 (serial computation)
    lengthy_computation : bool, optional
        Whether or not each computation is expected to take substantial time.
        Sometimes the time for adding more cores can outweigh the time per core
        Default - False
    func_args : list, optional
        arguments to be passed to the function
    func_kwargs : dict, optional
        keyword arguments to be passed onto function

    Returns
    -------
    results : list
        List of computational results
    """

    if not callable(func):
        raise TypeError('Function argument is not callable')
    if not isinstance(data, np.ndarray):
        raise TypeError('data must be a numpy array')
    if func_args is None:
        func_args = list()
    else:
        if isinstance(func_args, tuple):
            func_args = list(func_args)
        if not isinstance(func_args, list):
            raise TypeError('Arguments to the mapped function should be specified as a list')
    if func_kwargs is None:
        func_kwargs = dict()
    else:
        if not isinstance(func_kwargs, dict):
            raise TypeError('Keyword arguments to the mapped function should be specified via a dictionary')

    cores = recommend_cpu_cores(data.shape[0], requested_cores=cores,
                                lengthy_computation=lengthy_computation)

    if cores > 1:
        values = [joblib.delayed(func)(x, *func_args, **func_kwargs) for x in data]
        results = joblib.Parallel(n_jobs=cores)(values)

        # Finished reading the entire data set
        print('Finished parallel computation')

    else:
        print("Computing serially ...")
        results = [func(vector, *func_args, **func_kwargs) for vector in data]

    return results
<|endoftext|>


<commit_before># Copyright (c) 2006-2007 The Regents of The University of Michigan
# Copyright (c) 2009 Advanced Micro Devices, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Brad Beckmann

import math
import m5
from m5.objects import *
from m5.defines import buildEnv

def define_options(parser):
    # ruby network options
    parser.add_option("--topology", type="string", default="Crossbar",
                 help="check src/mem/ruby/network/topologies for complete set")
    parser.add_option("--mesh-rows", type="int", default=1,
                      help="the number of rows in the mesh topology")
    parser.add_option("--garnet-network", type="string", default=None,
                      help="'fixed'|'flexible'")

    # ruby mapping options
    parser.add_option("--numa-high-bit", type="int", default=None,
                      help="high order address bit to use for numa mapping. " \
                           "0 = highest bit, not specified = lowest bit")

    # ruby sparse memory options
    parser.add_option("--use-map", action="store_true", default=False)
    parser.add_option("--map-levels", type="int", default=4)

    # ruby debug cmd line options
    parser.add_option("--ruby-debug", action="store_true", default=False)
    parser.add_option("--ruby-debug-cycle", type="int", default=1)

    parser.add_option("--recycle-latency", type="int", default=10,
                      help="Recycle latency for ruby controller input buffers")

    protocol = buildEnv['PROTOCOL']
    exec "import %s" % protocol
    eval("%s.define_options(parser)" % protocol)

def create_system(options, system, piobus = None, dma_devices = []):

    protocol = buildEnv['PROTOCOL']
    exec "import %s" % protocol
    try:
        (cpu_sequencers, dir_cntrls, all_cntrls) = \
             eval("%s.create_system(options, system, piobus, dma_devices)" \
                  % protocol)
    except:
        print "Error: could not create sytem for ruby protocol %s" % protocol
        raise

    #
    # Important: the topology must be created before the network and after the
    # controllers.
    #
    exec "import %s" % options.topology
    try:
        net_topology = eval("%s.makeTopology(all_cntrls, options)" \
                            % options.topology)
    except:
        print "Error: could not create topology %s" % options.topology
        raise

    if options.garnet_network == "fixed":
        network = GarnetNetwork_d(topology = net_topology)
    elif options.garnet_network == "flexible":
        network = GarnetNetwork(topology = net_topology)
    else:
        network = SimpleNetwork(topology = net_topology)

    #
    # Loop through the directory controlers.
    # Determine the total memory size of the ruby system and verify it is equal
    # to physmem.  However, if Ruby memory is using sparse memory in SE
    # mode, then the system should not back-up the memory state with
    # the Memory Vector and thus the memory size bytes should stay at 0.
    # Also set the numa bits to the appropriate values.
    #
    total_mem_size = MemorySize('0B')

    dir_bits = int(math.log(options.num_dirs, 2))

    if options.numa_high_bit:
        numa_bit = options.numa_high_bit
    else:
        # if not specified, use the lowest bits above the block offest
        if dir_bits > 0:
            # add 5 because bits 0-5 are the block offset
            numa_bit = dir_bits + 5
        else:
            numa_bit = 6

    for dir_cntrl in dir_cntrls:
        total_mem_size.value += dir_cntrl.directory.size.value
        dir_cntrl.directory.numa_high_bit = numa_bit

    physmem_size = long(system.physmem.range.second) - \
                     long(system.physmem.range.first) + 1
    assert(total_mem_size.value == physmem_size)

    ruby_profiler = RubyProfiler(num_of_sequencers = len(cpu_sequencers))

    ruby = RubySystem(clock = options.clock,
                      network = network,
                      profiler = ruby_profiler,
                      tracer = RubyTracer(),
                      debug = RubyDebug(filter_string = 'none',
                                        verbosity_string = 'none',
                                        protocol_trace = options.ruby_debug,
                                        start_time = options.ruby_debug_cycle),
                      mem_size = total_mem_size)

    ruby.cpu_ruby_ports = cpu_sequencers

    return ruby
<commit_msg>Ruby: Add option for random seed to Ruby.py This patch adds an option to the script Ruby.py for setting the parameter m_random_seed used for randomizing delays in the memory system. The option can be specified as "--random_seed <seed value>".<commit_after># Copyright (c) 2006-2007 The Regents of The University of Michigan
# Copyright (c) 2009 Advanced Micro Devices, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Brad Beckmann

import math
import m5
from m5.objects import *
from m5.defines import buildEnv

def define_options(parser):
    # ruby network options
    parser.add_option("--topology", type="string", default="Crossbar",
                 help="check src/mem/ruby/network/topologies for complete set")
    parser.add_option("--mesh-rows", type="int", default=1,
                      help="the number of rows in the mesh topology")
    parser.add_option("--garnet-network", type="string", default=None,
                      help="'fixed'|'flexible'")

    # ruby mapping options
    parser.add_option("--numa-high-bit", type="int", default=None,
                      help="high order address bit to use for numa mapping. " \
                           "0 = highest bit, not specified = lowest bit")

    # ruby sparse memory options
    parser.add_option("--use-map", action="store_true", default=False)
    parser.add_option("--map-levels", type="int", default=4)

    # ruby debug cmd line options
    parser.add_option("--ruby-debug", action="store_true", default=False)
    parser.add_option("--ruby-debug-cycle", type="int", default=1)

    parser.add_option("--recycle-latency", type="int", default=10,
                      help="Recycle latency for ruby controller input buffers")

    parser.add_option("--random_seed", type="int", default=1234,
                      help="Used for seeding the random number generator")

    protocol = buildEnv['PROTOCOL']
    exec "import %s" % protocol
    eval("%s.define_options(parser)" % protocol)

def create_system(options, system, piobus = None, dma_devices = []):

    protocol = buildEnv['PROTOCOL']
    exec "import %s" % protocol
    try:
        (cpu_sequencers, dir_cntrls, all_cntrls) = \
             eval("%s.create_system(options, system, piobus, dma_devices)" \
                  % protocol)
    except:
        print "Error: could not create sytem for ruby protocol %s" % protocol
        raise

    #
    # Important: the topology must be created before the network and after the
    # controllers.
    #
    exec "import %s" % options.topology
    try:
        net_topology = eval("%s.makeTopology(all_cntrls, options)" \
                            % options.topology)
    except:
        print "Error: could not create topology %s" % options.topology
        raise

    if options.garnet_network == "fixed":
        network = GarnetNetwork_d(topology = net_topology)
    elif options.garnet_network == "flexible":
        network = GarnetNetwork(topology = net_topology)
    else:
        network = SimpleNetwork(topology = net_topology)

    #
    # Loop through the directory controlers.
    # Determine the total memory size of the ruby system and verify it is equal
    # to physmem.  However, if Ruby memory is using sparse memory in SE
    # mode, then the system should not back-up the memory state with
    # the Memory Vector and thus the memory size bytes should stay at 0.
    # Also set the numa bits to the appropriate values.
    #
    total_mem_size = MemorySize('0B')

    dir_bits = int(math.log(options.num_dirs, 2))

    if options.numa_high_bit:
        numa_bit = options.numa_high_bit
    else:
        # if not specified, use the lowest bits above the block offest
        if dir_bits > 0:
            # add 5 because bits 0-5 are the block offset
            numa_bit = dir_bits + 5
        else:
            numa_bit = 6

    for dir_cntrl in dir_cntrls:
        total_mem_size.value += dir_cntrl.directory.size.value
        dir_cntrl.directory.numa_high_bit = numa_bit

    physmem_size = long(system.physmem.range.second) - \
                     long(system.physmem.range.first) + 1
    assert(total_mem_size.value == physmem_size)

    ruby_profiler = RubyProfiler(num_of_sequencers = len(cpu_sequencers))

    ruby = RubySystem(clock = options.clock,
                      network = network,
                      profiler = ruby_profiler,
                      tracer = RubyTracer(),
                      debug = RubyDebug(filter_string = 'none',
                                        verbosity_string = 'none',
                                        protocol_trace = options.ruby_debug,
                                        start_time = options.ruby_debug_cycle),
                      mem_size = total_mem_size)

    ruby.cpu_ruby_ports = cpu_sequencers
    ruby.random_seed    = options.random_seed

    return ruby
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import cPickle
import json
import sys
import threading
import time

import requests
import webbrowser

class Fetcher(object):
    """Abstracts the twitter API.
    """

    def __init__(self):
        """Constructs the twitter fetcher.
        """
        self.data = []
        self.thread_lock = threading.Lock()

    def worker(self, query, page):
        """Thread worker which actually fetches a page of tweets.

        Args:
            query: the search query term that must be used to fetch tweets.
            num_page: number of pages to fetch.
        """
        try:
            r = requests.get('http://search.twitter.com/search.json',
                             params={'q': query, 'rpp': 100, 'page': page,
                                     'lang': 'en'})
            results = json.loads(r.text)
            if results:
                self.thread_lock.acquire()
                self.data.extend(results['results'])
                self.thread_lock.release()
        except requests.ConnectionError:
            pass

    def fetch(self, query, start_page=1, num_pages=10):
        """Method that performs the fetch of tweets asyncronously.

        This method is responsible for spawning threads for fetching tweets.

        Args:
            query: the search query term that must be used to fetch tweets.
            start_page: the starting page on twitter to fetch the tweets from.
            num_page: number of pages to fetch.
        """
        threads = []
        for page in range(start_page, num_pages):
            t = threading.Thread(target=self.worker, args=(query, page))
            threads.append(t)
            t.start()

        while threading.active_count() > 3:
            print threading.active_count()
            time.sleep(1)

        return self.data

    def userworker(self, query, page):
        try:
            r = requests.get('https://api.twitter.com/1/statuses/user_timeline.json',
                             params={'include_entities' : True, 'include_rts' : True,
                                     'screen_name' : query, 'page' : page, 'count' : 200})
            if r.status_code == 200:
                results = json.loads(r.text)
                if results:
                    self.thread_lock.acquire()
                    self.data.extend(results)
                    self.thread_lock.release()

        except requests.ConnectionError:
            pass

    def userfetch(self, query, start_page=1, num_pages=16):
        threads = []
        for page in range(start_page, num_pages+1):
            t = threading.Thread(target=self.userworker, args=(query, page))
            threads.append(t)
            t.start()

        while threading.active_count() > 3:
            print threading.active_count()
            time.sleep(1)

        return self.data

def main():
    threads = []
    for i in range(100):
        t = threading.Thread(target=fetch, args=(i + 1 + 1270, 'fail'))
        time.sleep(1)
        threads.append(t)
        t.start()
    return

<commit_msg>Print the API response status code to the debug logs while fetching the tweets.<commit_after># -*- coding: utf-8 -*-

import cPickle
import json
import sys
import threading
import time

import requests
import webbrowser

class Fetcher(object):
    """Abstracts the twitter API.
    """

    def __init__(self):
        """Constructs the twitter fetcher.
        """
        self.data = []
        self.thread_lock = threading.Lock()

    def worker(self, query, page):
        """Thread worker which actually fetches a page of tweets.

        Args:
            query: the search query term that must be used to fetch tweets.
            num_page: number of pages to fetch.
        """
        try:
            r = requests.get('http://search.twitter.com/search.json',
                             params={'q': query, 'rpp': 100, 'page': page,
                                     'lang': 'en'})
            results = json.loads(r.text)
            if results:
                self.thread_lock.acquire()
                self.data.extend(results['results'])
                self.thread_lock.release()
        except requests.ConnectionError:
            pass

    def fetch(self, query, start_page=1, num_pages=10):
        """Method that performs the fetch of tweets asyncronously.

        This method is responsible for spawning threads for fetching tweets.

        Args:
            query: the search query term that must be used to fetch tweets.
            start_page: the starting page on twitter to fetch the tweets from.
            num_page: number of pages to fetch.
        """
        threads = []
        for page in range(start_page, num_pages):
            t = threading.Thread(target=self.worker, args=(query, page))
            threads.append(t)
            t.start()

        while threading.active_count() > 3:
            print threading.active_count()
            time.sleep(1)

        return self.data

    def userworker(self, query, page):
        try:
            r = requests.get('https://api.twitter.com/1/statuses/user_timeline.json',
                             params={'include_entities' : True, 'include_rts' : True,
                                     'screen_name' : query, 'page' : page, 'count' : 200})
            print "User Fectch status code", r.status_code
            if r.status_code == 200:
                results = json.loads(r.text)
                if results:
                    self.thread_lock.acquire()
                    self.data.extend(results)
                    self.thread_lock.release()

        except requests.ConnectionError:
            pass

    def userfetch(self, query, start_page=1, num_pages=16):
        threads = []
        for page in range(start_page, num_pages+1):
            t = threading.Thread(target=self.userworker, args=(query, page))
            threads.append(t)
            t.start()

        while threading.active_count() > 3:
            print threading.active_count()
            time.sleep(1)

        return self.data

def main():
    threads = []
    for i in range(100):
        t = threading.Thread(target=fetch, args=(i + 1 + 1270, 'fail'))
        time.sleep(1)
        threads.append(t)
        t.start()
    return

<|endoftext|>


<commit_before>"""Django DDP app config."""

from __future__ import print_function

from django.apps import AppConfig
from django.conf import settings, ImproperlyConfigured
from django.db import DatabaseError
from django.db.models import signals

from dddp import autodiscover
from dddp.models import Connection

class DjangoDDPConfig(AppConfig):

    """Django app config for django-ddp."""

    api = None
    name = 'dddp'
    verbose_name = 'Django DDP'
    _in_migration = False

    def ready(self):
        """Initialisation for django-ddp (setup lookups and signal handlers)."""
        if not settings.DATABASES:
            raise ImproperlyConfigured('No databases configured.')
        for (alias, conf) in settings.DATABASES.items():
            if conf['ENGINE'] != 'django.db.backends.postgresql_psycopg2':
                raise ImproperlyConfigured(
                    '%r uses %r: django-ddp only works with PostgreSQL.' % (
                        alias, conf['backend'],
                    )
                )
        self.api = autodiscover()
        self.api.ready()
<commit_msg>Remove unused imports from AppConfig module.<commit_after>"""Django DDP app config."""

from __future__ import print_function

from django.apps import AppConfig
from django.conf import settings, ImproperlyConfigured

from dddp import autodiscover

class DjangoDDPConfig(AppConfig):

    """Django app config for django-ddp."""

    api = None
    name = 'dddp'
    verbose_name = 'Django DDP'
    _in_migration = False

    def ready(self):
        """Initialisation for django-ddp (setup lookups and signal handlers)."""
        if not settings.DATABASES:
            raise ImproperlyConfigured('No databases configured.')
        for (alias, conf) in settings.DATABASES.items():
            if conf['ENGINE'] != 'django.db.backends.postgresql_psycopg2':
                raise ImproperlyConfigured(
                    '%r uses %r: django-ddp only works with PostgreSQL.' % (
                        alias, conf['backend'],
                    )
                )
        self.api = autodiscover()
        self.api.ready()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function, division
import logging, json
from datetime import datetime, timedelta
from util import getnick

log = logging.getLogger('openweather')
url = 'http://api.wunderground.com/api/%s/conditions/forecast/q/%s.json'
defaults = {}

def init(bot):
    global defaults
    global api_key

    config = bot.config.get('module_openweather', {})
    default_location = config.get('default_location', 'YZF')
    api_key = config.get('wunderground_key')
    log.info('Using %s as default location' % default_location)
    with open('/usr/pyfibot/pyfibot/modules/module_openweather_conf.json') as configfile:
        defaults = json.load(configfile)

def command_weather(bot, user, channel, args):
    global defaults

    if not args:
        if user in defaults:
            return get_weather(bot, user, channel, defaults[user], True)
        else:
            return bot.say(channel,"No location specified, and no default found! Use '.weather set [LOC]' to set a default.")

    splut = args.split(' ', 1)
    cmd = splut[0]
    if cmd == "set":
        set_weather_default(bot, user, channel, splut[1])
    else:
        get_weather(bot, user, channel, args, True)

def set_weather_default(bot, user, channel, args):
    global defaults

    defaults[user] = args
    bot.say(channel,"Default location for {0} set to {1}".format(getnick.get(user), args))
    with open('/usr/pyfibot/pyfibot/modules/module_openweather_conf.json','w') as file:
        json.dump(defaults, file)

def command_fullweather(bot, user, channel, args):
    global defaults

    if not args:
        if user in defaults:
            parsed = get_weather(bot, user, channel, defaults[user], True)
        else:
            return bot.say(channel,"No location specified, and no default found! Use '.weather set [LOC]' to set a default.")
    else:
        parsed = get_weather(bot, user, channel, args, True)

    info = parsed['current_observation']

    wind = info['wind_mph']
    direction = info['wind_dir']
    pressure = info['pressure_mb']

    if info['pressure_trend'] == "-":
        pressuretext = "downward"
    else:
        pressuretext = "upward"

    bot.say(channel, "Wind: %smph from the %s | Pressure: %smb, trending %s" % (wind, direction, pressure, pressuretext))

    history = info['history_url']

    bot.say(channel, "Station history at: %s" % history)

def command_forecast(bot, user, channel, args):
    global defaults

    if not args:
        if user in defaults:
            parsed = get_weather(bot, user, channel, defaults[user], False)
        else:
            return bot.say(channel,"No location specified, and no default found! Use '.weather set [LOC]' to set a default.")
    else:
        parsed = get_weather(bot, user, channel, args, False)

    info = parsed['forecast']['txt_forecast']['forecastday']

    current = info[0]['title']
    currentfc = info[0]['fcttext']

    next = info[1]['title']
    nextfc = info[1]['fcttext']

    bot.say(channel, "Forecast for %s: %s" % (current, currentfc))
    bot.say(channel, "For %s: %s" % (next, nextfc))

def get_weather(bot, user, channel, args, output):
    global api_key

    location = args
    q = bot.get_url(url % (api_key, location))
    parsed = q.json()
    degree_sign= u'\N{DEGREE SIGN}'

    try:
        result = parsed['response']['results'][0]
        bestguess = result['zmw']
        guesscity = result['city']
        guessstate = result['state']
        if output:
            bot.say(channel, 'Assuming you meant ' + guesscity + ', ' + guessstate + ', heeeeere\'s the weather!')

        q = bot.get_url(url % bestguess)
        parsed = q.json()
    except KeyError:
        pass

    try:
        info = parsed['current_observation']

        location = info['observation_location']['full']
        temp = info['temp_f']
        tempc = info['temp_c']
        condition = info['weather']
        humidity = info['relative_humidity']

        if output:
            bot.say(channel, getnick.get(user) + ': [' + location + '] Temp: ' + str(temp) + degree_sign + 'F, ' + str(tempc) + degree_sign + 'C | Currently ' + condition + ' | Humidity of ' + humidity)
        return parsed
    except KeyError:
        error = parsed['response']['error']['description']
        bot.say(channel, "ERROR: %s [for query '%s']" % (error, location))
        pass
<commit_msg>Weather modules gets api key from config file<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function, division
import logging, json
from datetime import datetime, timedelta
from util import getnick

log = logging.getLogger('openweather')
defaults = {}
url = 'http://api.wunderground.com/api/%s/conditions/forecast/q/%s.json'

def init(bot):
    global defaults
    global api_key

    config = bot.config.get('module_openweather', {})
    default_location = config.get('default_location', 'YZF')
    api_key = config.get('wunderground_key')
    log.info('Using %s as default location' % default_location)
    with open('/usr/pyfibot/pyfibot/modules/module_openweather_conf.json') as configfile:
        defaults = json.load(configfile)

def command_weather(bot, user, channel, args):
    global defaults

    if not args:
        if user in defaults:
            return get_weather(bot, user, channel, defaults[user], True)
        else:
            return bot.say(channel,"No location specified, and no default found! Use '.weather set [LOC]' to set a default.")

    splut = args.split(' ', 1)
    cmd = splut[0]
    if cmd == "set":
        set_weather_default(bot, user, channel, splut[1])
    else:
        get_weather(bot, user, channel, args, True)

def set_weather_default(bot, user, channel, args):
    global defaults

    defaults[user] = args
    bot.say(channel,"Default location for {0} set to {1}".format(getnick.get(user), args))
    with open('/usr/pyfibot/pyfibot/modules/module_openweather_conf.json','w') as file:
        json.dump(defaults, file)

def command_fullweather(bot, user, channel, args):
    global defaults

    if not args:
        if user in defaults:
            parsed = get_weather(bot, user, channel, defaults[user], True)
        else:
            return bot.say(channel,"No location specified, and no default found! Use '.weather set [LOC]' to set a default.")
    else:
        parsed = get_weather(bot, user, channel, args, True)

    info = parsed['current_observation']

    wind = info['wind_mph']
    direction = info['wind_dir']
    pressure = info['pressure_mb']

    if info['pressure_trend'] == "-":
        pressuretext = "downward"
    else:
        pressuretext = "upward"

    bot.say(channel, "Wind: %smph from the %s | Pressure: %smb, trending %s" % (wind, direction, pressure, pressuretext))

    history = info['history_url']

    bot.say(channel, "Station history at: %s" % history)

def command_forecast(bot, user, channel, args):
    global defaults

    if not args:
        if user in defaults:
            parsed = get_weather(bot, user, channel, defaults[user], False)
        else:
            return bot.say(channel,"No location specified, and no default found! Use '.weather set [LOC]' to set a default.")
    else:
        parsed = get_weather(bot, user, channel, args, False)

    info = parsed['forecast']['txt_forecast']['forecastday']

    current = info[0]['title']
    currentfc = info[0]['fcttext']

    next = info[1]['title']
    nextfc = info[1]['fcttext']

    bot.say(channel, "Forecast for %s: %s" % (current, currentfc))
    bot.say(channel, "For %s: %s" % (next, nextfc))

def get_weather(bot, user, channel, args, output):
    global api_key

    location = args
    q = bot.get_url(url % (api_key, location))
    parsed = q.json()
    degree_sign= u'\N{DEGREE SIGN}'

    try:
        result = parsed['response']['results'][0]
        bestguess = result['zmw']
        guesscity = result['city']
        guessstate = result['state']
        if output:
            bot.say(channel, 'Assuming you meant ' + guesscity + ', ' + guessstate + ', heeeeere\'s the weather!')

        q = bot.get_url(url % bestguess)
        parsed = q.json()
    except KeyError:
        pass

    try:
        info = parsed['current_observation']

        location = info['observation_location']['full']
        temp = info['temp_f']
        tempc = info['temp_c']
        condition = info['weather']
        humidity = info['relative_humidity']

        if output:
            bot.say(channel, getnick.get(user) + ': [' + location + '] Temp: ' + str(temp) + degree_sign + 'F, ' + str(tempc) + degree_sign + 'C | Currently ' + condition + ' | Humidity of ' + humidity)
        return parsed
    except KeyError:
        error = parsed['response']['error']['description']
        bot.say(channel, "ERROR: %s [for query '%s']" % (error, location))
        pass
<|endoftext|>


<commit_before># Current Thread Scheduler
import time
import logging
import threading
from datetime import timedelta

from rx.internal import PriorityQueue

from .schedulerbase import SchedulerBase
from .scheduleditem import ScheduledItem

log = logging.getLogger('Rx')

class Trampoline(object):
    @classmethod
    def run(cls, queue):
        while queue:
            item = queue.dequeue()
            if not item.is_cancelled():
                diff = item.duetime - item.scheduler.now
                while diff > timedelta(0):
                    seconds = diff.seconds + diff.microseconds / 1E6 + diff.days * 86400
                    log.warning("Do not schedule blocking work!")
                    time.sleep(seconds)
                    diff = item.duetime - item.scheduler.now

                if not item.is_cancelled():
                    item.invoke()

class CurrentThreadScheduler(SchedulerBase):
    """Represents an object that schedules units of work on the current
    thread. You never want to schedule timeouts using the CurrentThreadScheduler
    since it will block the current thread while waiting."""

    def __init__(self):
        """Gets a scheduler that schedules work as soon as possible on the
        current thread."""

        self.queues = dict()
        self.lock = threading.RLock()

    def schedule(self, action, state=None):
        """Schedules an action to be executed."""

        #log.debug("CurrentThreadScheduler.schedule(state=%s)", state)
        return self.schedule_relative(timedelta(0), action, state)

    def schedule_relative(self, duetime, action, state=None):
        """Schedules an action to be executed after duetime."""

        duetime = self.to_timedelta(duetime)

        dt = self.now + SchedulerBase.normalize(duetime)
        si = ScheduledItem(self, state, action, dt)

        queue = self.queue
        if queue is None:
            queue = PriorityQueue(4)
            queue.enqueue(si)

            self.queue = queue
            try:
                Trampoline.run(queue)
            finally:
                self.queue = None
        else:
            queue.enqueue(si)

        return si.disposable

    def schedule_absolute(self, duetime, action, state=None):
        """Schedules an action to be executed at duetime."""

        duetime = self.to_datetime(duetime)
        return self.schedule_relative(duetime - self.now, action, state=None)

    def get_queue(self):
        ident = threading.current_thread().ident

        with self.lock:
            return self.queues.get(ident)

    def set_queue(self, queue):
        ident = threading.current_thread().ident

        with self.lock:
            self.queues[ident] = queue

    queue = property(get_queue, set_queue)

    def schedule_required(self):
        """Test if scheduling is required.

        Gets a value indicating whether the caller must call a
        schedule method. If the trampoline is active, then it returns
        False; otherwise, if  the trampoline is not active, then it
        returns True.
        """
        return self.queue is None

    def ensure_trampoline(self, action):
        """Method for testing the CurrentThreadScheduler."""
        if self.schedule_required():
            return self.schedule(action)
        else:
            return action(self, None)

current_thread_scheduler = CurrentThreadScheduler()
<commit_msg>Remove unnecessary else<commit_after># Current Thread Scheduler
import time
import logging
import threading
from datetime import timedelta

from rx.internal import PriorityQueue

from .schedulerbase import SchedulerBase
from .scheduleditem import ScheduledItem

log = logging.getLogger('Rx')

class Trampoline(object):
    @classmethod
    def run(cls, queue):
        while queue:
            item = queue.dequeue()
            if not item.is_cancelled():
                diff = item.duetime - item.scheduler.now
                while diff > timedelta(0):
                    seconds = diff.seconds + diff.microseconds / 1E6 + diff.days * 86400
                    log.warning("Do not schedule blocking work!")
                    time.sleep(seconds)
                    diff = item.duetime - item.scheduler.now

                if not item.is_cancelled():
                    item.invoke()

class CurrentThreadScheduler(SchedulerBase):
    """Represents an object that schedules units of work on the current
    thread. You never want to schedule timeouts using the CurrentThreadScheduler
    since it will block the current thread while waiting."""

    def __init__(self):
        """Gets a scheduler that schedules work as soon as possible on the
        current thread."""

        self.queues = dict()
        self.lock = threading.RLock()

    def schedule(self, action, state=None):
        """Schedules an action to be executed."""

        #log.debug("CurrentThreadScheduler.schedule(state=%s)", state)
        return self.schedule_relative(timedelta(0), action, state)

    def schedule_relative(self, duetime, action, state=None):
        """Schedules an action to be executed after duetime."""

        duetime = self.to_timedelta(duetime)

        dt = self.now + SchedulerBase.normalize(duetime)
        si = ScheduledItem(self, state, action, dt)

        queue = self.queue
        if queue is None:
            queue = PriorityQueue(4)
            queue.enqueue(si)

            self.queue = queue
            try:
                Trampoline.run(queue)
            finally:
                self.queue = None
        else:
            queue.enqueue(si)

        return si.disposable

    def schedule_absolute(self, duetime, action, state=None):
        """Schedules an action to be executed at duetime."""

        duetime = self.to_datetime(duetime)
        return self.schedule_relative(duetime - self.now, action, state=None)

    def get_queue(self):
        ident = threading.current_thread().ident

        with self.lock:
            return self.queues.get(ident)

    def set_queue(self, queue):
        ident = threading.current_thread().ident

        with self.lock:
            self.queues[ident] = queue

    queue = property(get_queue, set_queue)

    def schedule_required(self):
        """Test if scheduling is required.

        Gets a value indicating whether the caller must call a
        schedule method. If the trampoline is active, then it returns
        False; otherwise, if  the trampoline is not active, then it
        returns True.
        """
        return self.queue is None

    def ensure_trampoline(self, action):
        """Method for testing the CurrentThreadScheduler."""

        if self.schedule_required():
            return self.schedule(action)

        return action(self, None)

current_thread_scheduler = CurrentThreadScheduler()
<|endoftext|>


<commit_before>"""cx_Freeze extension

Extends:

- the 'build' command with the 'exe-command' option to allow using a
different command from 'build_exe' to build executables from Python scripts.

- the 'install' command with the 'skip-sub-commands' option to allow not
running a set of sub commands, e.g.:

    install --skip-sub-commands=install_lib,install_scripts,install_data
"""

import sys
import distutils.command.build
from cx_Freeze.dist import build as cx_build
from cx_Freeze.dist import install as cx_install
from cx_Freeze.dist import setup as cx_setup
from cx_Freeze.dist import _AddCommandClass

class build(cx_build):

    cx_build.user_options.append(
        ('exe-command=', None, "Python script executables command"))

    def initialize_options(self):
        cx_build.initialize_options(self)
        self.exe_command = 'build_exe'

    def get_sub_commands(self):
        subCommands = distutils.command.build.build.get_sub_commands(self)
        if self.distribution.executables:
            subCommands.append(self.exe_command)
        return subCommands

class install(cx_install):

    cx_install.user_options.append(
        ('skip-sub-commands=', None,
         "sub commands to ignore when running command"))

    def initialize_options(self):
        cx_install.initialize_options(self)
        self.skip_sub_commands = None

    def get_sub_commands(self):
        subCommands = cx_install.get_sub_commands(self)
        if self.skip_sub_commands:
            skip_sub_commands = self.skip_sub_commands.split(',')
            for cmd in skip_sub_commands:
                if cmd in subCommands:
                    subCommands.remove(cmd)
        return subCommands

# Override cx_Freeze setup to override build and install commands.
def setup(**attrs):
    commandClasses = attrs.setdefault("cmdclass", {})
    _AddCommandClass(commandClasses, "build", build)
    _AddCommandClass(commandClasses, "install", install)
    cx_setup(**attrs)
<commit_msg>NXP-13818: Remove unused import<commit_after>"""cx_Freeze extension

Extends:

- the 'build' command with the 'exe-command' option to allow using a
different command from 'build_exe' to build executables from Python scripts.

- the 'install' command with the 'skip-sub-commands' option to allow not
running a set of sub commands, e.g.:

    install --skip-sub-commands=install_lib,install_scripts,install_data
"""

import distutils.command.build
from cx_Freeze.dist import build as cx_build
from cx_Freeze.dist import install as cx_install
from cx_Freeze.dist import setup as cx_setup
from cx_Freeze.dist import _AddCommandClass

class build(cx_build):

    cx_build.user_options.append(
        ('exe-command=', None, "Python script executables command"))

    def initialize_options(self):
        cx_build.initialize_options(self)
        self.exe_command = 'build_exe'

    def get_sub_commands(self):
        subCommands = distutils.command.build.build.get_sub_commands(self)
        if self.distribution.executables:
            subCommands.append(self.exe_command)
        return subCommands

class install(cx_install):

    cx_install.user_options.append(
        ('skip-sub-commands=', None,
         "sub commands to ignore when running command"))

    def initialize_options(self):
        cx_install.initialize_options(self)
        self.skip_sub_commands = None

    def get_sub_commands(self):
        subCommands = cx_install.get_sub_commands(self)
        if self.skip_sub_commands:
            skip_sub_commands = self.skip_sub_commands.split(',')
            for cmd in skip_sub_commands:
                if cmd in subCommands:
                    subCommands.remove(cmd)
        return subCommands

# Override cx_Freeze setup to override build and install commands.
def setup(**attrs):
    commandClasses = attrs.setdefault("cmdclass", {})
    _AddCommandClass(commandClasses, "build", build)
    _AddCommandClass(commandClasses, "install", install)
    cx_setup(**attrs)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
###############################################################################
#
# Copyright (c) 2013-2014 Didotech srl (info at didotech.com)
#                          All Rights Reserved.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################

from openerp.osv import orm, fields

class stock_picking(orm.Model):
    _inherit = "stock.picking"

    def action_invoice_create(self, cr, uid, ids, journal_id=False,
                            group=False, type='out_invoice', context=None):

        if context is None:
            context = self.pool['res.users'].context_get(cr, uid)

        invoice_dict = super(stock_picking, self).action_invoice_create(cr, uid,
                            ids, journal_id, group, type, context=context)
        processed_invoice = []
        for picking_key in invoice_dict:
            if invoice_dict[picking_key] in processed_invoice:
                continue
            processed_invoice.append(invoice_dict[picking_key])
            invoice = self.pool['account.invoice'].browse(cr, uid, invoice_dict[picking_key], context=context)
            if not invoice.company_id.is_group_invoice_line:
                continue

            for line in invoice.invoice_line:
                if line.move_line_id and line.move_line_id.sale_line_id and line.move_line_id.sale_line_id.mrp_bom:

                    qty_delivery = 0
                    for bom_line in line.move_line_id.sale_line_id.mrp_bom:
                        qty_delivery += bom_line.product_uom_qty

                    new_line_vals = {
                        'price_unit': line.move_line_id.sale_line_id.price_unit,
                        'discount': line.move_line_id.sale_line_id.discount,
                        'quantity': line.quantity / qty_delivery,
                    }
                    self.pool['account.invoice.line'].write(cr, uid, line.id, new_line_vals, context=context)
            invoice.button_reset_taxes()
        return invoice_dict
<commit_msg>check if i'm on right sale.order.line<commit_after># -*- coding: utf-8 -*-
###############################################################################
#
# Copyright (c) 2013-2014 Didotech srl (info at didotech.com)
#                          All Rights Reserved.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################

from openerp.osv import orm, fields

class stock_picking(orm.Model):
    _inherit = "stock.picking"

    def action_invoice_create(self, cr, uid, ids, journal_id=False,
                            group=False, type='out_invoice', context=None):

        if context is None:
            context = self.pool['res.users'].context_get(cr, uid)

        invoice_dict = super(stock_picking, self).action_invoice_create(cr, uid,
                            ids, journal_id, group, type, context=context)
        processed_invoice = []
        for picking_key in invoice_dict:
            if invoice_dict[picking_key] in processed_invoice:
                continue
            processed_invoice.append(invoice_dict[picking_key])
            invoice = self.pool['account.invoice'].browse(cr, uid, invoice_dict[picking_key], context=context)
            if not invoice.company_id.is_group_invoice_line:
                continue

            for line in invoice.invoice_line:
                if line.move_line_id and line.move_line_id.sale_line_id and line.move_line_id.sale_line_id._columns.get('mrp_bom', False) and line.move_line_id.sale_line_id.mrp_bom:

                    qty_delivery = 0
                    for bom_line in line.move_line_id.sale_line_id.mrp_bom:
                        qty_delivery += bom_line.product_uom_qty

                    new_line_vals = {
                        'price_unit': line.move_line_id.sale_line_id.price_unit,
                        'discount': line.move_line_id.sale_line_id.discount,
                        'quantity': line.quantity / qty_delivery,
                    }
                    self.pool['account.invoice.line'].write(cr, uid, line.id, new_line_vals, context=context)
            invoice.button_reset_taxes()
        return invoice_dict
<|endoftext|>


<commit_before>import json
import requests

""" OANDA API wrapper for OANDA's REST API """

""" HTTPS Streaming """

class Streamer(object):
    """ Provides functionality for HTTPS Streaming
    Docs: http://developer.oanda.com/rest-live/streaming
    """

    def __init__(self, environment="practice", access_token=None):
        """Instantiates an instance of OandaPy's streaming API wrapper.
        :param environment: (optional) Provide the environment for oanda's
         REST api, either 'practice', or 'live'. Default: practice
        :param access_token: (optional) Provide a valid access token if you
         have one. This is required if the environment is not sandbox.
        """

        if environment == 'practice':
            self.api_url = 'https://stream-fxpractice.oanda.com/v1/prices'
        elif environment == 'live':
            self.api_url = 'https://stream-fxtrade.oanda.com/v1/prices'

        self.access_token = access_token
        self.client = requests.Session()
        self.client.stream = True
        self.connected = False

        # personal token authentication
        if self.access_token:
            self.client.headers['Authorization'] = 'Bearer '+self.access_token

    def start(self, ignore_heartbeat=True, **params):
        """ Starts the stream with the given parameters
        :param accountId: (Required) The account that prices are applicable for
        :param instruments: (Required) A (URL encoded) comma separated list of
         instruments to fetch prices for.
        :param ignore_heartbeat: (optional) Whether or not to display the
         heartbeat. Default: True
        """
        self.connected = True

        request_args = {}
        request_args['params'] = params

        while self.connected:
            response = self.client.get(self.api_url, **request_args)

            if response.status_code != 200:
                self.on_error(response.content)

            for line in response.iter_lines(90):
                if not self.connected:
                    break

                if line:
                    data = json.loads(line.decode("utf-8"))
                    if not (ignore_heartbeat and "heartbeat" in data):
                        self.on_success(data)

    def on_success(self, data):
        """ Called when data is successfully retrieved from the stream
        Override this to handle your streaming data.
        :param data: response object sent from stream
        """

        return True

    def on_error(self, data):
        """ Called when stream returns non-200 status code
        Override this to handle your streaming data.
        :param data: error response object sent from stream
        """

        return

    def disconnect(self):
        """ Manually disconnects the streaming client
        """
        self.connected = False
<commit_msg>The on_success and on_error methods are ment to be abstract methods. By using the abc module these methods are forced to be abstractmethods If they are not overridden an error occurs<commit_after>import json
import requests
from abc import ABCMeta, abstractmethod

""" OANDA API wrapper for OANDA's REST API """

""" HTTPS Streaming """

class Streamer(object):
    """ Provides functionality for HTTPS Streaming
    Docs: http://developer.oanda.com/rest-live/streaming
    """
    __metaclass__ = ABCMeta

    def __init__(self, environment="practice", access_token=None):
        """Instantiates an instance of OandaPy's streaming API wrapper.
        :param environment: (optional) Provide the environment for oanda's
         REST api, either 'practice', or 'live'. Default: practice
        :param access_token: (optional) Provide a valid access token if you
         have one. This is required if the environment is not sandbox.
        """

        if environment == 'practice':
            self.api_url = 'https://stream-fxpractice.oanda.com/v1/prices'
        elif environment == 'live':
            self.api_url = 'https://stream-fxtrade.oanda.com/v1/prices'

        self.access_token = access_token
        self.client = requests.Session()
        self.client.stream = True
        self.connected = False

        # personal token authentication
        if self.access_token:
            self.client.headers['Authorization'] = 'Bearer '+self.access_token

    def start(self, ignore_heartbeat=True, **params):
        """ Starts the stream with the given parameters
        :param accountId: (Required) The account that prices are applicable for
        :param instruments: (Required) A (URL encoded) comma separated list of
         instruments to fetch prices for.
        :param ignore_heartbeat: (optional) Whether or not to display the
         heartbeat. Default: True
        """
        self.connected = True

        request_args = {}
        request_args['params'] = params

        while self.connected:
            response = self.client.get(self.api_url, **request_args)

            if response.status_code != 200:
                self.on_error(response.content)

            for line in response.iter_lines(90):
                if not self.connected:
                    break

                if line:
                    data = json.loads(line.decode("utf-8"))
                    if not (ignore_heartbeat and "heartbeat" in data):
                        self.on_success(data)

    @abstractmethod
    def on_success(self, data):
        """ Called when data is successfully retrieved from the stream
        Override this to handle your streaming data.
        :param data: response object sent from stream
        """

        return True

    @abstractmethod
    def on_error(self, data):
        """ Called when stream returns non-200 status code
        Override this to handle your streaming data.
        :param data: error response object sent from stream
        """

        return

    def disconnect(self):
        """ Manually disconnects the streaming client
        """
        self.connected = False
<|endoftext|>