<commit_before># -*- coding: utf-8 -*-
from __future__ import division, print_function, absolute_import
from datetime import datetime
from itertools import product
import logging
import os
import os.path

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from ..utils import make_same_length, setdbspl, int2srt

log = logging.getLogger(__name__)

class Experiment(object):
    """
    Performs a speech intelligibility experiment.

    The masker is truncated to the length of the target, or is padded with
    zeros if it is shorter than the target.

    Parameters
    ----------
    models : single model, or list
        List of intelligibility models.
    materials : object
        object that implements a `next` interface that returns the next
        pair of target and maskers.
    snrs : array_like
        List of SNRs.
    name : str,
        name of experiment, will be appended to the date when writing to file.
    write : bool, optional
        Write the result to file, as CSV, the default is True.
    output_path : string, optional.
        Path where the results will be written if `write` is True. The
        default is './Data'.
    timestamp_format : str, optional
        Datetime timestamp format for the CSV file name. The default is of
        the form YYYYMMDD-HHMMSS.

    """

    def __init__(
            self,
            models,
            material,
            snrs,
            distortion=None,
            dist_params=(None,),
            fixed_level=65,
            fixed_target=True,
            name=None,
            write=True,
            output_path='./Data/',
            timestamp_format="%Y%m%d-%H%M%S"
    ):
        self.models = models
        self.material = material
        self.snrs = snrs
        self.distortion = distortion
        self.dist_params = dist_params
        self.fixed_level = fixed_level
        self.fixed_target = fixed_target
        self.name = name
        self.timestamp_format = timestamp_format
        self.write = write
        self.output_path = output_path
        self._key_full_pred = 'Full Prediction'
        self._key_value = 'Value'
        self._key_output = 'Output'
        self._key_dist_params = "Distortion params"
        self._key_models = 'Model'
        self._key_snr = 'SNR'
        self._key_sent = 'Sentence number'
        self._all_keys = [
            self._key_full_pred,
            self._key_value,
            self._key_output,
            self._key_dist_params,
            self._key_models,
            self._key_snr,
            self._key_sent
        ]

    def preprocessing(self, target, masker, snr, params):
        """
        Applies preprocessing to the target and masker before setting the
        levels. In this case, the masker is padded with zeros if it is longer
        than the target, or it is truncated to be the same length as the target.

        :param target:
        :param masker:
        :return:
        """

        # Make target and masker same length
        if target.shape[-1] != masker.shape[-1]:
            target, masker = make_same_length(target, masker,
                                              extend_first=False)

        if params:
            if isinstance(params, dict):
                target, masker = self.distortion(target, masker, **params)
            else:
                target, masker = self.distortion(target, masker, *params)

        target, masker = self.adjust_levels(target, masker, snr)
        return target, target + masker, masker

    def adjust_levels(self, target, masker, snr):
        """
        Adjusts level of target and maskers.

        Uses the `self.fixed_level` as the reference level for the target and
        masker. If `self.fixed_target` is True, the masker level is varied to
        set the required SNR, otherwise the target level is changed.

        :param target: ndarray
            Target signal.
        :param masker: ndarray
            Masker signal.
        :param snr: float
            SNR at which to set the target and masker.
        :return: tuple
            Level adjusted `target` and `masker`.
        """

        target_level = self.fixed_level
        masker_level = self.fixed_level
        if self.fixed_target:
            masker_level -= snr
        else:
            target_level += snr
        target = setdbspl(target, target_level)
        masker = setdbspl(masker, masker_level)
        return target, masker

    def next_masker(self, target):
        return self.material.ssn(target)

    def append_results(
            self,
            df,
            res,
            model,
            snr,
            i_target,
            params
    ):
        """
        Appends results to a DataFrame

        Parameters
        ----------
        df : dataframe
            DataFrame where the new results will be appended.
        res : dict
            Output dictionary from an intelligibility model.
        model: object
            Intelligibility model. Will use it's `name` attribute,
            if available, to add the source model to the DataFrame. Otherwise,
            the `__class__.__name__` attribute will be used.
        snr : float
            SNR at which the simulation was performed.
        i_target : int
            Number of the target sentence
        params : object
            Parameters that were passed to the distortion process.

        Returns
        -------
        df : dataframe
            DataFrame with new entry appended.
        """
        try:
            model_name = model.name
        except AttributeError:
            model_name = model.__class__.__name__
        try:
            material_name = self.material.name
        except AttributeError:
            material_name = self.material.__class__.__name__
        d = {
            'SNR': snr
            , 'Model': model_name
            , 'Sentence number': i_target
            , self._key_full_pred: res
            , 'Material': material_name
        }
        # If the distortion parameters are in a dictionary, put each value in
        # a different column. Otherwise, group everything in a single column.
        if isinstance(params, dict):
            for k, v in params.iteritems():
                d[k] = v
        else:
            # Make sure the values are hashable for later manipulation
            if isinstance(params, list):
                params = tuple(params)
            else:
                pass
            d['Distortion params'] = params

        for name, value in res['p'].iteritems():
            d['Output'] = name
            d['Value'] = value
            df = df.append(d, ignore_index=True)

        return df

    def run(self, n=None, seed=0):
        """ Run the experiment.

        Parameters
        ----------
        n : int
            Number of sentences to process.

        Returns
        -------
        df : pd.Dataframe
            Pandas dataframe with the experimental results.

        """
        if not seed:
            seed = 0
        np.random.seed(seed)

        try:
            self.models = iter(self.models)
        except TypeError:
            self.models = [self.models]

        targets = self.material.load_files(n)

        # Initialize the dataframe in which the results are saved.
        df = pd.DataFrame()

        for ii, ((i_target, target), params, snr, model) \
                in enumerate(product(
                    enumerate(targets),
                    self.dist_params,
                    self.snrs,
                    self.models
        )):
            masker = self.next_masker(target)

            target, mix, masker = self.preprocessing(
                target,
                masker,
                snr,
                params
            )
            log.info("Simulation # %s\t SNR: %s, sentence %s", ii, snr,
                     i_target)
            res = self.prediction(model, target, mix, masker)

            df = self.append_results(
                df,
                res,
                model,
                snr,
                i_target,
                params
            )

        if self.write:
            self._write_results(df)
        return df

    def _write_results(self, df):
        """Writes results to CSV file.

        Will drop the column where all the complete model output is stored
        before writing to disk.

        Parameters
        ----------
        df : dataframe

        Returns
        -------
        filepath : str
            Path to the CSV file.

        Raises
        ------
        IOError : Raise if the path where to write the CSV file is not
        accessible. Additionally, the function tries to save the CSV file to
        the current directory, in order not to loose the simulation data.

        """
        timestamp = datetime.now()
        date = timestamp.strftime(self.timestamp_format)
        if self.name:
            name = "-{}".format(self.name)
        else:
            name = ''
        filename = "{date}{name}.csv".format(date=date, name=name)

        if not os.path.isdir(self.output_path):
            try:
                os.mkdir(self.output_path)
                log.info('Created directory %s', self.output_path)
            except IOError as e:
                log.info("Could not create directory %s", self.output_path)
                log.error(e)

        output_file = os.path.join(self.output_path, filename)
        try:
            df.drop(self._key_full_pred, axis=1).to_csv(output_file)
            log.info('Saved CSV file to location: %s'.format(output_file))
        except IOError as e:
            try:
                alternate_path = os.path.join(os.getcwd(), filename)
                err_msg = 'Could not write CSV file to path: {}, tried to ' \
                          'save to ' \
                          '{} in order not to loose data.'.format(
                    output_file, alternate_path)
                log.error(err_msg)
                raise
            finally:
                try:
                    df.drop(self._key_full_pred, axis=1).to_csv(alternate_path)
                except:
                    pass
        else:
            return output_file

    @staticmethod
    def prediction(model, target, mix, masker):
        """
        Predicts intelligibility for a target and masker pair. The target and
        masker are simply added together to create the mixture.

        Parameters
        ----------
        model :
        target :
        masker :

        Returns
        -------
        :return:
        """
        return model.predict(target, mix, masker)

    def plot_results(self,
                     df,
                     var=None,
                     xlabel='SNR (dB)',
                     ylabel='% Intelligibility'):

        # Drop the column with the full prediction results
        if self._key_full_pred in df.columns:
            df = df.drop(self._key_full_pred, axis=1)

        # Use the single "Distortion params" column if available.
        if self._key_dist_params in df.columns:
            params = [self._key_dist_params]
        else:
            params = list(set(df.columns) - set(self._all_keys)
                          - {'Intelligibility'})
        log.debug("Found the following parameter keys %s", params)

        if np.unique(df[params]):
            groups = [self._key_snr, self._key_models]
        else:
            groups = params + [self._key_snr, self._key_models]

        grouped_cols = df.groupby(groups).mean().unstack(
            self._key_snr).T

        # Which column to plot?
        if not var:
            if "Intelligibility" in df.columns:
                var = 'Intelligibility'
            else:
                var = self._key_value

        grouped_cols.xs(var).plot()
        if var == 'Intelligibility':
            plt.ylim((0, 100))

        plt.legend(loc='best')
        plt.xlabel(xlabel)
        plt.ylabel(ylabel)

    def pred_to_pc(
            self,
            df,
            fc,
            col='Value',
            models=None,
            out_name='Intelligibility'
    ):
        """Converts the data in a given column to percent correct.

        Parameters
        ----------
        df : Dataframe
            Dataframe where the intelligibility predictions are stored.
        fc : function
            The function used to convert the model outputs to
            intelligibility. The function must take a float as input and
            returns a float.
        col : string
            Name of the column to convert to intelligibility. The default is
            "Value".
        models : string, list or dict
            This argument can either be a string, with the name of the model
            for which the output value will be transformed to
            intelligibility, or a list of model names. The argument can also
            be a dictionary where the keys are model names and the values are
            "output names", i.e. the name of the value output by the model.
            This is useful if a model has multiple prediction values. The
            default is `None`, all the rows will be converted with the same
            function.
        out_name : str
            Name of the output column (default: 'Intelligibility')

        Returns
        -------
        df : dataframe
            Dataframe with the new column column with intelligibility values.
        """
        if models:
            if isinstance(models, list):
                for model in models:
                    df[out_name] \
                        = df[df[self._key_models] == model][col].map(fc)
            elif isinstance(models, dict):
                for model, v in models.iteritems():
                    key = (df[self._key_models] == model) & (
                        df[self._key_output == v])
                    df[out_name] = df[key][col].map(fc)
            else:
                df[out_name] = df[df[self._key_models] == models][col].map(fc)
        else:
            df[out_name] = df[col].map(fc)
        return df

    @staticmethod
    def get_srt(df):
        """Converts SRTs to DeltaSRTs.

        :return: tuple, srts and DeltaSRTs
        """

        scores = []
        df_tn = df[df['Target distance'] == 0.5]
        for ii, (key, grp) in enumerate(df_tn.groupby(['Target distance',
                                                       'Masker distance'])):
            scores.append(grp.groupby('SNR')['Intelligibility_L'].mean())

        srts = np.zeros(4)
        for ii, score in enumerate(scores):
            srts[ii] = int2srt(score.index, score)

        print(srts)
        dsrts = srts[1] - srts
        print(dsrts)
        return srts, dsrts

def srt_dict_to_dataframe(d):
    df_srts = pd.DataFrame()
    for k, v in d.iteritems():
        model, material, tdist, mdist = k.split('_')

        try:
            srt = v[0]
        except TypeError:
            srt = np.nan

        df_srts = df_srts.append({'model': model,
                                  'tidst': tdist,
                                  'mdist': mdist,
                                  'srt': srt},
                                 ignore_index=True)
    df_srts = df_srts.convert_objects(convert_numeric=True, )
    return df_srts.sort(['model', 'mdist'])

def plot_srt_dataframe(df):
    for key, grp in df.groupby('model'):
        plt.plot(grp['mdist'], grp['srt'], label=key)
    plt.legend(loc='best')
    plt.xlabel('Masker distance')
    plt.ylabel('SRT (dB)')
<commit_msg>Add method to extract groups<commit_after># -*- coding: utf-8 -*-
from __future__ import division, print_function, absolute_import
from datetime import datetime
from itertools import product
import logging
import os
import os.path

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from ..utils import make_same_length, setdbspl, int2srt

log = logging.getLogger(__name__)

class Experiment(object):
    """
    Performs a speech intelligibility experiment.

    The masker is truncated to the length of the target, or is padded with
    zeros if it is shorter than the target.

    Parameters
    ----------
    models : single model, or list
        List of intelligibility models.
    materials : object
        object that implements a `next` interface that returns the next
        pair of target and maskers.
    snrs : array_like
        List of SNRs.
    name : str,
        name of experiment, will be appended to the date when writing to file.
    write : bool, optional
        Write the result to file, as CSV, the default is True.
    output_path : string, optional.
        Path where the results will be written if `write` is True. The
        default is './Data'.
    timestamp_format : str, optional
        Datetime timestamp format for the CSV file name. The default is of
        the form YYYYMMDD-HHMMSS.

    """

    def __init__(
            self,
            models,
            material,
            snrs,
            distortion=None,
            dist_params=(None,),
            fixed_level=65,
            fixed_target=True,
            name=None,
            write=True,
            output_path='./Data/',
            timestamp_format="%Y%m%d-%H%M%S"
    ):
        self.models = models
        self.material = material
        self.snrs = snrs
        self.distortion = distortion
        self.dist_params = dist_params
        self.fixed_level = fixed_level
        self.fixed_target = fixed_target
        self.name = name
        self.timestamp_format = timestamp_format
        self.write = write
        self.output_path = output_path
        self._key_full_pred = 'Full Prediction'
        self._key_value = 'Value'
        self._key_output = 'Output'
        self._key_dist_params = "Distortion params"
        self._key_models = 'Model'
        self._key_snr = 'SNR'
        self._key_sent = 'Sentence number'
        self._all_keys = [
            self._key_full_pred,
            self._key_value,
            self._key_dist_params,
            self._key_models,
            self._key_snr,
            self._key_sent
        ]

    def preprocessing(self, target, masker, snr, params):
        """
        Applies preprocessing to the target and masker before setting the
        levels. In this case, the masker is padded with zeros if it is longer
        than the target, or it is truncated to be the same length as the target.

        :param target:
        :param masker:
        :return:
        """

        # Make target and masker same length
        if target.shape[-1] != masker.shape[-1]:
            target, masker = make_same_length(target, masker,
                                              extend_first=False)

        if params:
            if isinstance(params, dict):
                target, masker = self.distortion(target, masker, **params)
            else:
                target, masker = self.distortion(target, masker, *params)

        target, masker = self.adjust_levels(target, masker, snr)
        return target, target + masker, masker

    def adjust_levels(self, target, masker, snr):
        """
        Adjusts level of target and maskers.

        Uses the `self.fixed_level` as the reference level for the target and
        masker. If `self.fixed_target` is True, the masker level is varied to
        set the required SNR, otherwise the target level is changed.

        :param target: ndarray
            Target signal.
        :param masker: ndarray
            Masker signal.
        :param snr: float
            SNR at which to set the target and masker.
        :return: tuple
            Level adjusted `target` and `masker`.
        """

        target_level = self.fixed_level
        masker_level = self.fixed_level
        if self.fixed_target:
            masker_level -= snr
        else:
            target_level += snr
        target = setdbspl(target, target_level)
        masker = setdbspl(masker, masker_level)
        return target, masker

    def next_masker(self, target):
        return self.material.ssn(target)

    def append_results(
            self,
            df,
            res,
            model,
            snr,
            i_target,
            params
    ):
        """
        Appends results to a DataFrame

        Parameters
        ----------
        df : dataframe
            DataFrame where the new results will be appended.
        res : dict
            Output dictionary from an intelligibility model.
        model: object
            Intelligibility model. Will use it's `name` attribute,
            if available, to add the source model to the DataFrame. Otherwise,
            the `__class__.__name__` attribute will be used.
        snr : float
            SNR at which the simulation was performed.
        i_target : int
            Number of the target sentence
        params : object
            Parameters that were passed to the distortion process.

        Returns
        -------
        df : dataframe
            DataFrame with new entry appended.
        """
        try:
            model_name = model.name
        except AttributeError:
            model_name = model.__class__.__name__
        try:
            material_name = self.material.name
        except AttributeError:
            material_name = self.material.__class__.__name__
        d = {
            'SNR': snr
            , 'Model': model_name
            , 'Sentence number': i_target
            , self._key_full_pred: res
            , 'Material': material_name
        }
        # If the distortion parameters are in a dictionary, put each value in
        # a different column. Otherwise, group everything in a single column.
        if isinstance(params, dict):
            for k, v in params.iteritems():
                d[k] = v
        else:
            # Make sure the values are hashable for later manipulation
            if isinstance(params, list):
                params = tuple(params)
            else:
                pass
            d['Distortion params'] = params

        for name, value in res['p'].iteritems():
            d['Output'] = name
            d['Value'] = value
            df = df.append(d, ignore_index=True)

        return df

    def run(self, n=None, seed=0):
        """ Run the experiment.

        Parameters
        ----------
        n : int
            Number of sentences to process.

        Returns
        -------
        df : pd.Dataframe
            Pandas dataframe with the experimental results.

        """
        if not seed:
            seed = 0
        np.random.seed(seed)

        try:
            self.models = iter(self.models)
        except TypeError:
            self.models = [self.models]

        targets = self.material.load_files(n)

        # Initialize the dataframe in which the results are saved.
        df = pd.DataFrame()

        for ii, ((i_target, target), params, snr, model) \
                in enumerate(product(
                    enumerate(targets),
                    self.dist_params,
                    self.snrs,
                    self.models
        )):
            masker = self.next_masker(target)

            target, mix, masker = self.preprocessing(
                target,
                masker,
                snr,
                params
            )
            log.info("Simulation # %s\t SNR: %s, sentence %s", ii, snr,
                     i_target)
            res = self.prediction(model, target, mix, masker)

            df = self.append_results(
                df,
                res,
                model,
                snr,
                i_target,
                params
            )

        if self.write:
            self._write_results(df)
        return df

    def _write_results(self, df):
        """Writes results to CSV file.

        Will drop the column where all the complete model output is stored
        before writing to disk.

        Parameters
        ----------
        df : dataframe

        Returns
        -------
        filepath : str
            Path to the CSV file.

        Raises
        ------
        IOError : Raise if the path where to write the CSV file is not
        accessible. Additionally, the function tries to save the CSV file to
        the current directory, in order not to loose the simulation data.

        """
        timestamp = datetime.now()
        date = timestamp.strftime(self.timestamp_format)
        if self.name:
            name = "-{}".format(self.name)
        else:
            name = ''
        filename = "{date}{name}.csv".format(date=date, name=name)

        if not os.path.isdir(self.output_path):
            try:
                os.mkdir(self.output_path)
                log.info('Created directory %s', self.output_path)
            except IOError as e:
                log.info("Could not create directory %s", self.output_path)
                log.error(e)

        output_file = os.path.join(self.output_path, filename)
        try:
            df.drop(self._key_full_pred, axis=1).to_csv(output_file)
            log.info('Saved CSV file to location: %s'.format(output_file))
        except IOError as e:
            try:
                alternate_path = os.path.join(os.getcwd(), filename)
                err_msg = 'Could not write CSV file to path: {}, tried to ' \
                          'save to ' \
                          '{} in order not to loose data.'.format(
                    output_file, alternate_path)
                log.error(err_msg)
                raise
            finally:
                try:
                    df.drop(self._key_full_pred, axis=1).to_csv(alternate_path)
                except:
                    pass
        else:
            return output_file

    @staticmethod
    def prediction(model, target, mix, masker):
        """
        Predicts intelligibility for a target and masker pair. The target and
        masker are simply added together to create the mixture.

        Parameters
        ----------
        model :
        target :
        masker :

        Returns
        -------
        :return:
        """
        return model.predict(target, mix, masker)

    def _get_groups(self, df, var=None):
        """Get of variables for plotting.

        Ignored variables should be:
        - SNR
        - Sentence number

        :param df:
        :param var:
        :return:
        """

        # Use the single "Distortion params" column if available and not
        # None. Else, consider all "extra columns" as parameters.
        if self._key_dist_params in df.columns:
            # Use the distortion parameters only if it's not None.
            if df[self._key_dist_params].unique().any():
                params = [self._key_dist_params]
            else:
                params = []
        else:
            params = list(set(df.columns) - set(self._all_keys)
                          - {'Intelligibility'})
        # If var is defined, remove it from the groups
        if var:
            params = list(set(params) - set([var]))
        log.debug("Found the following parameter keys %s.", params)
        if len(np.unique(df[params])):
            groups = params + [self._key_snr, self._key_models]
        else:
            groups = [self._key_snr, self._key_models]
        log.debug("The plotting groups are: %s.", groups)
        return groups

    def plot_results(self,
                     df,
                     var=None,
                     xlabel='SNR (dB)',
                     ylabel='% Intelligibility'):

        # Drop the column with the full prediction results
        if self._key_full_pred in df.columns:
            df = df.drop(self._key_full_pred, axis=1)

        groups = self._get_groups(df, var)

        grouped_cols = df.groupby(groups).mean().unstack(
            self._key_snr).T

        # Which column to plot?
        if not var:
            if "Intelligibility" in df.columns:
                var = 'Intelligibility'
            else:
                var = self._key_value

        grouped_cols.xs(var).plot()
        if var == 'Intelligibility':
            plt.ylim((0, 100))

        plt.legend(loc='best')
        plt.xlabel(xlabel)
        plt.ylabel(ylabel)

    def pred_to_pc(
            self,
            df,
            fc,
            col='Value',
            models=None,
            out_name='Intelligibility'
    ):
        """Converts the data in a given column to percent correct.

        Parameters
        ----------
        df : Dataframe
            Dataframe where the intelligibility predictions are stored.
        fc : function
            The function used to convert the model outputs to
            intelligibility. The function must take a float as input and
            returns a float.
        col : string
            Name of the column to convert to intelligibility. The default is
            "Value".
        models : string, list or dict
            This argument can either be a string, with the name of the model
            for which the output value will be transformed to
            intelligibility, or a list of model names. The argument can also
            be a dictionary where the keys are model names and the values are
            "output names", i.e. the name of the value output by the model.
            This is useful if a model has multiple prediction values. The
            default is `None`, all the rows will be converted with the same
            function.
        out_name : str
            Name of the output column (default: 'Intelligibility')

        Returns
        -------
        df : dataframe
            Dataframe with the new column column with intelligibility values.
        """
        if models:
            if isinstance(models, list):
                for model in models:
                    df[out_name] \
                        = df[df[self._key_models] == model][col].map(fc)
            elif isinstance(models, dict):
                for model, v in models.iteritems():
                    key = (df[self._key_models] == model) & (
                        df[self._key_output == v])
                    df[out_name] = df[key][col].map(fc)
            else:
                df[out_name] = df[df[self._key_models] == models][col].map(fc)
        else:
            df[out_name] = df[col].map(fc)
        return df

    @staticmethod
    def get_srt(df):
        """Converts SRTs to DeltaSRTs.

        :return: tuple, srts and DeltaSRTs
        """

        scores = []
        df_tn = df[df['Target distance'] == 0.5]
        for ii, (key, grp) in enumerate(df_tn.groupby(['Target distance',
                                                       'Masker distance'])):
            scores.append(grp.groupby('SNR')['Intelligibility_L'].mean())

        srts = np.zeros(4)
        for ii, score in enumerate(scores):
            srts[ii] = int2srt(score.index, score)

        print(srts)
        dsrts = srts[1] - srts
        print(dsrts)
        return srts, dsrts

def srt_dict_to_dataframe(d):
    df_srts = pd.DataFrame()
    for k, v in d.iteritems():
        model, material, tdist, mdist = k.split('_')

        try:
            srt = v[0]
        except TypeError:
            srt = np.nan

        df_srts = df_srts.append({'model': model,
                                  'tidst': tdist,
                                  'mdist': mdist,
                                  'srt': srt},
                                 ignore_index=True)
    df_srts = df_srts.convert_objects(convert_numeric=True, )
    return df_srts.sort(['model', 'mdist'])

def plot_srt_dataframe(df):
    for key, grp in df.groupby('model'):
        plt.plot(grp['mdist'], grp['srt'], label=key)
    plt.legend(loc='best')
    plt.xlabel('Masker distance')
    plt.ylabel('SRT (dB)')
<|endoftext|>


<commit_before>from django.utils import timezone
from django.test import TestCase
from facebook_pages.models import Page
from facebook_pages.factories import PageFactory

from models import PageStatistic

PAGE_FANS_ID = 501842786534856

class PageStatisticTest(TestCase):

    def setUp(self):
        self.t = timezone.now()
        self.p = PageFactory(graph_id=PAGE_FANS_ID, likes_count=10, talking_about_count=20)

    def test_page_statistic_create(self):
        t = self.t
        p = self.p

        ps = PageStatistic.objects.get(id=1)
        self.assertEqual(ps.page, p)
        self.assertEqual(ps.likes_count, p.likes_count)
        self.assertEqual(ps.talking_about_count, p.talking_about_count)
        self.assertGreater(ps.updated_at, t)

    def test_test_page_statistic_update(self):
        t = timezone.now()
        p = self.p
        p.likes_count += 100
        p.talking_about_count += 100
        p.save()

        ps = PageStatistic.objects.get(id=2)
        self.assertEqual(ps.page, p)
        self.assertEqual(ps.likes_count, p.likes_count)
        self.assertEqual(ps.talking_about_count, p.talking_about_count)
        self.assertGreater(ps.updated_at, t)

    def test_null_stats_test(self):
        p = PageFactory(graph_id=PAGE_FANS_ID, likes_count=None, talking_about_count=None)
        self.assertEqual(PageStatistic.objects.count(), 0)
<commit_msg>test fix<commit_after>from django.utils import timezone
from django.test import TestCase
from facebook_pages.models import Page
from facebook_pages.factories import PageFactory

from models import PageStatistic

PAGE_FANS_ID = 501842786534856

class PageStatisticTest(TestCase):

    def setUp(self):
        self.t = timezone.now()
        self.p = PageFactory(graph_id=PAGE_FANS_ID, likes_count=10, talking_about_count=20)

    def test_page_statistic_create(self):
        t = self.t
        p = self.p

        ps = PageStatistic.objects.get(id=1)
        self.assertEqual(ps.page, p)
        self.assertEqual(ps.likes_count, p.likes_count)
        self.assertEqual(ps.talking_about_count, p.talking_about_count)
        self.assertGreater(ps.updated_at, t)

    def test_test_page_statistic_update(self):
        t = timezone.now()
        p = self.p
        p.likes_count += 100
        p.talking_about_count += 100
        p.save()

        ps = PageStatistic.objects.get(id=2)
        self.assertEqual(ps.page, p)
        self.assertEqual(ps.likes_count, p.likes_count)
        self.assertEqual(ps.talking_about_count, p.talking_about_count)
        self.assertGreater(ps.updated_at, t)

    def test_null_stats_test(self):
        count = PageStatistic.objects.count()
        p = PageFactory(graph_id=123456, likes_count=None, talking_about_count=None)
        self.assertEqual(PageStatistic.objects.count(), count)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import cv2
import sys
import numpy as np

def resize(src, w_ratio, h_ratio):
    height = src.shape[0]
    width = src.shape[1]
    dst = cv2.resize(src,(width/100*w_ratio,height/100*h_ratio))
    return dst

if __name__ == '__main__':
    param = sys.argv
    if (len(param) != 4):
        print ("Usage: $ python " + param[0] + " sample.jpg wide_ratio height_ratio")
        quit()

    # open image file
    try:
        input_img = cv2.imread(param[1])
    except:
        print ('faild to load %s' % param[1])
        quit()

    if input_img is None:
        print ('faild to load %s' % param[1])
        quit()

    w_ratio = int(param[2])
    h_ratio = int(param[3])

    output_img = resize(input_img, w_ratio, h_ratio)
    cv2.imwrite(param[1], output_img)
<commit_msg>small modify<commit_after># -*- coding: utf-8 -*-
import cv2
import sys
import numpy as np

def resize(src, w_ratio, h_ratio):
    height = src.shape[0]
    width = src.shape[1]
    dst = cv2.resize(src,(width/100*w_ratio,height/100*h_ratio))
    return dst

if __name__ == '__main__':
    param = sys.argv
    if (len(param) != 4):
        print ("Usage: $ python " + param[0] + " sample.jpg wide_ratio height_ratio")
        quit()

    # open image file
    try:
        input_img = cv2.imread(param[1])
    except:
        print ('faild to load %s' % param[1])
        quit()

    if input_img is None:
        print ('faild to load %s' % param[1])
        quit()

    w_ratio = int(param[2])
    h_ratio = int(param[3])

    output_img = resize(input_img, w_ratio, h_ratio)
    cv2.imwrite(param[1], output_img)
<|endoftext|>


<commit_before>import datetime
from sqlalchemy import func, distinct, or_, desc
from sqlalchemy.dialects import mysql

from db import session
import model
import util

class TaskStatus:
	LOCKED = 'locked'
	BASE = 'base'
	CORRECTING = 'correcting'
	DONE = 'done'

def fully_submitted(user_id):
	if user_id is None:
		return []

	max_modules_count = { task.id: task.modules for task in session.query(model.Task.id, func.count(model.Module.id).label('modules')).outerjoin(model.Module).group_by(model.Task.id).all() }

	real_modules_count = { task.id: task.modules for task in session.query(model.Task.id, func.count(distinct(model.Module.id)).label('modules')).join(model.Module).join(model.Evaluation).filter(model.Evaluation.user == user_id, or_(model.Module.autocorrect != True, model.Module.max_points == model.Evaluation.points)).group_by(model.Task.id).all() }

	return { int(key): int(val) for key, val in real_modules_count.items() if max_modules_count[key] == val }

def after_deadline():
	return { int(task.id) for task in session.query(model.Task).filter(model.Task.time_deadline < datetime.datetime.now() ).all() }

def max_points(task_id):
	points = session.query(func.sum(model.Module.max_points).label('points')).\
		filter(model.Module.task == task_id).first().points

	return int(points) if points else 0

def max_points_dict():
	points_per_task = session.query(model.Module.task.label('id'), func.sum(model.Module.max_points).label('points')).\
		group_by(model.Module.task).all()

	return { task.id: int(task.points) for task in points_per_task }

def points_per_module(task_id, user_id):
	return session.query(model.Module, \
		func.max(model.Evaluation.points).label('points')).\
		join(model.Evaluation, model.Evaluation.module == model.Module.id).\
		filter(model.Module.task == task_id, model.Evaluation.user == user_id).\
		group_by(model.Evaluation.module).all()

def points(task_id, user_id):
	ppm = points_per_module(task_id, user_id)

	if len(ppm) == 0:
		return None

	return sum(module.points for module in ppm if module.points is not None)

def comment_thread(task_id, user_id):
	query = session.query(model.SolutionComment).filter(model.SolutionComment.task == task_id, model.SolutionComment.user == user_id).first()

	return query.thread if query is not None else None

def status(task, user, adeadline=None, fsubmitted=None):
	if user is None or user.id is None:
		return TaskStatus.BASE if task.prerequisite is None else TaskStatus.LOCKED

	if user.role in ('org', 'admin'):
		return TaskStatus.BASE

	if task.time_deadline < datetime.datetime.now():
		return TaskStatus.BASE

	if not fsubmitted:
		fsubmitted = fully_submitted(user.id)

	if task.id in fsubmitted:
		return TaskStatus.DONE

	if not adeadline:
		adeadline = after_deadline()

	currently_active = adeadline | set(fully_submitted(user.id).keys())

	if task.id in currently_active:
		return TaskStatus.BASE

	return TaskStatus.BASE if util.PrerequisitiesEvaluator(task.prerequisite_obj, currently_active).evaluate() else TaskStatus.LOCKED

def to_json(task, user=None, adeadline=None, fsubmitted=None):
	max_points = sum([ module.max_points for module in task.modules ])
	tstatus = status(task, user, adeadline, fsubmitted)

	return {
		'id': task.id,
		'title': task.title,
		'author': task.author,
		'category': task.category,
		'details': task.id,
		'intro': task.intro,
		'max_score': sum([ module.max_points for module in task.modules ]),
		'position': [ task.position_x, task.position_y ],
		'time_published': task.time_published.isoformat(),
		'time_deadline': task.time_deadline.isoformat(),
		'state': tstatus,
		'prerequisities': [] if not task.prerequisite_obj else util.prerequisite.to_json(task.prerequisite_obj),
		'picture_base': task.picture_base,
		'picture_suffix': '.svg'
	}

def details_to_json(task, status, achievements, best_scores, comment_thread=None):
	return {
		'id': task.id,
		'body': task.body,
		'thread': task.thread,
		'modules': [ module.id for module in task.modules ],
		'best_scores': [ best_score.User.id for best_score in best_scores ],
		'comment': comment_thread,
		'solution': task.solution if status == TaskStatus.DONE or task.time_deadline < datetime.datetime.now() else None,
		'achievements': [ achievement.id for achievement in achievements ]
	}

def best_scores(task_id):
	per_modules = session.query(model.User.id.label('user_id'), \
			func.max(model.Evaluation.points).label('points')).\
			join(model.Evaluation, model.Evaluation.user == model.User.id).\
			filter(model.Module.task == task_id, 'points' is not None).\
			filter(model.Evaluation.module == model.Module.id).\
			group_by(model.Evaluation.module, model.User).subquery()
	print per_modules

	tmp = session.query(model.User, func.sum(per_modules.c.points).label('sum')).join(per_modules, per_modules.c.user_id == model.User.id).slice(0, 5).all();
	print tmp

	#return session.query(model.User).slice(0, 5).all();
	ret = session.query(model.User, func.sum(per_modules.c.points).label('sum')).join(per_modules, per_modules.c.user_id == model.User.id).filter(model.User.role == 'participant').group_by(per_modules.c.user_id).order_by(desc('sum')).slice(0, 5).all()
	print ret
	return ret

def best_score_to_json(best_score):
	achievements = session.query(model.UserAchievement).filter(model.UserAchievement.user_id == best_score.User.id).all()

	return {
		'id': best_score.User.id,
		'user': best_score.User.id,
		'achievements': [ achievement.achievement_id for achievement in achievements ],
		'score': int(best_score.sum)
	}
<commit_msg>Statistics fixed.<commit_after>import datetime
from sqlalchemy import func, distinct, or_, desc
from sqlalchemy.dialects import mysql

from db import session
import model
import util

class TaskStatus:
	LOCKED = 'locked'
	BASE = 'base'
	CORRECTING = 'correcting'
	DONE = 'done'

def fully_submitted(user_id):
	if user_id is None:
		return []

	max_modules_count = { task.id: task.modules for task in session.query(model.Task.id, func.count(model.Module.id).label('modules')).outerjoin(model.Module).group_by(model.Task.id).all() }

	real_modules_count = { task.id: task.modules for task in session.query(model.Task.id, func.count(distinct(model.Module.id)).label('modules')).join(model.Module).join(model.Evaluation).filter(model.Evaluation.user == user_id, or_(model.Module.autocorrect != True, model.Module.max_points == model.Evaluation.points)).group_by(model.Task.id).all() }

	return { int(key): int(val) for key, val in real_modules_count.items() if max_modules_count[key] == val }

def after_deadline():
	return { int(task.id) for task in session.query(model.Task).filter(model.Task.time_deadline < datetime.datetime.now() ).all() }

def max_points(task_id):
	points = session.query(func.sum(model.Module.max_points).label('points')).\
		filter(model.Module.task == task_id).first().points

	return int(points) if points else 0

def max_points_dict():
	points_per_task = session.query(model.Module.task.label('id'), func.sum(model.Module.max_points).label('points')).\
		group_by(model.Module.task).all()

	return { task.id: int(task.points) for task in points_per_task }

def points_per_module(task_id, user_id):
	return session.query(model.Module, \
		func.max(model.Evaluation.points).label('points')).\
		join(model.Evaluation, model.Evaluation.module == model.Module.id).\
		filter(model.Module.task == task_id, model.Evaluation.user == user_id).\
		group_by(model.Evaluation.module).all()

def points(task_id, user_id):
	ppm = points_per_module(task_id, user_id)

	if len(ppm) == 0:
		return None

	return sum(module.points for module in ppm if module.points is not None)

def comment_thread(task_id, user_id):
	query = session.query(model.SolutionComment).filter(model.SolutionComment.task == task_id, model.SolutionComment.user == user_id).first()

	return query.thread if query is not None else None

def status(task, user, adeadline=None, fsubmitted=None):
	if user is None or user.id is None:
		return TaskStatus.BASE if task.prerequisite is None else TaskStatus.LOCKED

	if user.role in ('org', 'admin'):
		return TaskStatus.BASE

	if task.time_deadline < datetime.datetime.now():
		return TaskStatus.BASE

	if not fsubmitted:
		fsubmitted = fully_submitted(user.id)

	if task.id in fsubmitted:
		return TaskStatus.DONE

	if not adeadline:
		adeadline = after_deadline()

	currently_active = adeadline | set(fully_submitted(user.id).keys())

	if task.id in currently_active:
		return TaskStatus.BASE

	return TaskStatus.BASE if util.PrerequisitiesEvaluator(task.prerequisite_obj, currently_active).evaluate() else TaskStatus.LOCKED

def to_json(task, user=None, adeadline=None, fsubmitted=None):
	max_points = sum([ module.max_points for module in task.modules ])
	tstatus = status(task, user, adeadline, fsubmitted)

	return {
		'id': task.id,
		'title': task.title,
		'author': task.author,
		'category': task.category,
		'details': task.id,
		'intro': task.intro,
		'max_score': sum([ module.max_points for module in task.modules ]),
		'position': [ task.position_x, task.position_y ],
		'time_published': task.time_published.isoformat(),
		'time_deadline': task.time_deadline.isoformat(),
		'state': tstatus,
		'prerequisities': [] if not task.prerequisite_obj else util.prerequisite.to_json(task.prerequisite_obj),
		'picture_base': task.picture_base,
		'picture_suffix': '.svg'
	}

def details_to_json(task, status, achievements, best_scores, comment_thread=None):
	return {
		'id': task.id,
		'body': task.body,
		'thread': task.thread,
		'modules': [ module.id for module in task.modules ],
		'best_scores': [ best_score.User.id for best_score in best_scores ],
		'comment': comment_thread,
		'solution': task.solution if status == TaskStatus.DONE or task.time_deadline < datetime.datetime.now() else None,
		'achievements': [ achievement.id for achievement in achievements ]
	}

def best_scores(task_id):
	per_modules = session.query(model.User.id.label('user_id'), \
			func.max(model.Evaluation.points).label('points')).\
			join(model.Evaluation, model.Evaluation.user == model.User.id).\
			filter(model.Module.task == task_id, 'points' is not None).\
			filter(model.Evaluation.module == model.Module.id).\
			group_by(model.Evaluation.module, model.User).subquery()
	print per_modules

	return session.query(model.User, func.sum(per_modules.c.points).label('sum')).join(per_modules, per_modules.c.user_id == model.User.id).filter(model.User.role == 'participant').group_by(per_modules.c.user_id).order_by(desc('sum')).slice(0, 5).all()

def best_score_to_json(best_score):
	achievements = session.query(model.UserAchievement).filter(model.UserAchievement.user_id == best_score.User.id).all()

	return {
		'id': best_score.User.id,
		'user': best_score.User.id,
		'achievements': [ achievement.achievement_id for achievement in achievements ],
		'score': int(best_score.sum)
	}
<|endoftext|>


<commit_before>from django.db.models.fields import FieldDoesNotExist
from django.db.models.fields.related import ForeignObjectRel
from django.db.models.fields.related import OneToOneRel

from django_filters import FilterSet
from django_filters import NumberFilter
from django_filters import DateFilter
from django_filters import BooleanFilter
from django_filters import TypedChoiceFilter

from distutils.util import strtobool

from api.generics.filters import CommaSeparatedCharFilter
from api.generics.filters import TogetherFilterSet
from api.generics.filters import ToManyFilter

from rest_framework import filters

from iati.models import *
from iati.transaction.models import *

class ActivityFilter(TogetherFilterSet):

    activity_id = CommaSeparatedCharFilter(
        name='id',
        lookup_type='in')

    activity_scope = CommaSeparatedCharFilter(
        name='scope__code',
        lookup_type='in',)

    document_link_category = ToManyFilter(
        qs=DocumentLink,
        lookup_type='in',
        name='categories',
        fk='activity',
    )

    planned_start_date_lte = DateFilter(
        lookup_type='lte',
        name='planned_start')

    planned_start_date_gte = DateFilter(
        lookup_type='gte',
        name='planned_start')

    actual_start_date_lte = DateFilter(
        lookup_type='lte',
        name='actual_start')

    actual_start_date_gte = DateFilter(
        lookup_type='gte',
        name='actual_start')

    planned_end_date_lte = DateFilter(
        lookup_type='lte',
        name='planned_end')

    planned_end_date_gte = DateFilter(
        lookup_type='gte',
        name='planned_end')

    actual_end_date_lte = DateFilter(
        lookup_type='lte',
        name='actual_end')

    actual_end_date_gte = DateFilter(
        lookup_type='gte',
        name='actual_end')

    end_date_lte = DateFilter(
        lookup_type='lte',
        name='end_date')

    end_date_gte = DateFilter(
        lookup_type='gte',
        name='end_date')

    start_date_lte = DateFilter(
        lookup_type='lte',
        name='start_date')

    start_date_gte = DateFilter(
        lookup_type='gte',
        name='start_date')

    end_date_isnull = BooleanFilter(lookup_type='isnull', name='end_date')
    start_date_isnull = BooleanFilter(lookup_type='isnull', name='start_date')

    xml_source_ref = CommaSeparatedCharFilter(
        lookup_type='in',
        name='xml_source_ref',)

    activity_status = CommaSeparatedCharFilter(
        lookup_type='in',
        name='activity_status',)

    hierarchy = CommaSeparatedCharFilter(
        lookup_type='in',
        name='hierarchy',)

    collaboration_type = CommaSeparatedCharFilter(
        lookup_type='in',
        name='collaboration_type',)

    default_flow_type = CommaSeparatedCharFilter(
        lookup_type='in',
        name='default_flow_type',)

    default_aid_type = CommaSeparatedCharFilter(
        lookup_type='in',
        name='default_aid_type',)

    default_finance_type = CommaSeparatedCharFilter(
        lookup_type='in',
        name='default_finance_type',)

    default_tied_status = CommaSeparatedCharFilter(
        lookup_type='in',
        name='default_tied_status',)

    budget_period_start = DateFilter(
        lookup_type='gte',
        name='budget__period_start',)

    budget_period_end = DateFilter(
        lookup_type='lte',
        name='budget__period_end')

    humanitarian = TypedChoiceFilter(
        choices=(('0', 'False'), ('1', 'True')),
        coerce=strtobool)

    humanitarian_scope_type = ToManyFilter(
        qs=HumanitarianScope,
        lookup_type='in',
        name='type__code',
        fk='activity',
    )

    related_activity_id = ToManyFilter(
        qs=RelatedActivity,
        fk='current_activity',
        lookup_type='in',
        name='ref_activity__id',
    )

    related_activity_type = ToManyFilter(
        qs=RelatedActivity,
        lookup_type='in',
        name='type__code',
        fk='current_activity',
    )

    related_activity_recipient_country = ToManyFilter(
        qs=RelatedActivity,
        lookup_type='in',
        name='ref_activity__recipient_country',
        fk='current_activity',
    )

    related_activity_recipient_region = ToManyFilter(
        qs=RelatedActivity,
        lookup_type='in',
        name='ref_activity__recipient_region',
        fk='current_activity',
    )

    related_activity_sector = ToManyFilter(
        qs=RelatedActivity,
        lookup_type='in',
        name='ref_activity__sector',
        fk='current_activity',
    )

    related_activity_sector_category = ToManyFilter(
        qs=RelatedActivity,
        lookup_type='in',
        name='ref_activity__sector__category',
        fk='current_activity',
    )

    budget_currency = ToManyFilter(
        qs=Budget,
        lookup_type='in',
        name='currency__code',
        fk='activity',
    )

    recipient_country = ToManyFilter(
        qs=ActivityRecipientCountry,
        lookup_type='in',
        name='country__code',
        fk='activity',
    )

    recipient_region = ToManyFilter(
        qs=ActivityRecipientRegion,
        lookup_type='in',
        name='region__code',
        fk='activity',
    )

    sector = ToManyFilter(
        qs=ActivitySector,
        lookup_type='in',
        name='sector__code',
        fk='activity',
    )

    sector_category = ToManyFilter(
        qs=ActivitySector,
        lookup_type='in',
        name='sector__category__code',
        fk='activity',
    )

    policy_marker = ToManyFilter(
        qs=ActivityPolicyMarker,
        lookup_type='in',
        name='code',
        fk='activity',
    )

    participating_organisation = ToManyFilter(
        qs=ActivityParticipatingOrganisation,
        lookup_type='in',
        name='normalized_ref',
        fk='activity',
    )

    participating_organisation_name = ToManyFilter(
        qs=ActivityParticipatingOrganisation,
        lookup_type='in',
        name='primary_name',
        fk='activity',
    )

    participating_organisation_role = ToManyFilter(
        qs=ActivityParticipatingOrganisation,
        lookup_type='in',
        name='role__code',
        fk='activity',
    )

    participating_organisation_type = ToManyFilter(
        qs=ActivityParticipatingOrganisation,
        lookup_type='in',
        name='type__code',
        fk='activity',
    )

    reporting_organisation = ToManyFilter(
        qs=ActivityReportingOrganisation,
        lookup_type='in',
        name='normalized_ref',
        fk='activity',
    )

    reporting_organisation_startswith = ToManyFilter(
        qs=ActivityReportingOrganisation,
        lookup_type='startswith',
        name='normalized_ref',
        fk='activity',
    )

    result_title = ToManyFilter(
        qs=Result,
        lookup_type='in',
        name='resulttitle__narratives__content',
        fk='activity',
    )

    indicator_title = ToManyFilter(
        qs=ResultIndicatorTitle,
        lookup_type='in',
        name='primary_name',
        fk='result_indicator__result__activity')

    indicator_period_end_year = ToManyFilter(
        qs=ResultIndicatorPeriod,
        lookup_type='year',
        name='period_end',
        fk='result_indicator__result__activity')

    #
    # Transaction filters
    #

    transaction_type = ToManyFilter(
        qs=Transaction,
        lookup_type='in',
        name='transaction_type',
        fk='activity',
    )

    provider_organisation_primary_name = ToManyFilter(
        qs=Transaction,
        lookup_type='in',
        name='provider_organisation__primary_name',
        fk='activity',
    )

    receiver_organisation_primary_name = ToManyFilter(
        qs=Transaction,
        lookup_type='in',
        name='receiver_organisation__primary_name',
        fk='activity',
    )

    transaction_provider_organisation_name = ToManyFilter(
        qs=Transaction,
        lookup_type='in',
        name='provider_organisation__narratives__content',
        fk='activity',
    )

    transaction_receiver_organisation_name = ToManyFilter(
        qs=Transaction,
        lookup_type='in',
        name='receiver_organisation__narratives__content',
        fk='activity',
    )

    transaction_provider_activity = ToManyFilter(
        qs=Transaction,
        lookup_type='in',
        name='provider_organisation__provider_activity_ref',
        fk='activity',
    )

    transaction_receiver_activity = ToManyFilter(
        qs=Transaction,
        lookup_type='in',
        name='receiver_organisation__receiver_activity_ref',
        fk='activity',
    )

    transaction_currency = ToManyFilter(
        qs=Transaction,
        lookup_type='in',
        name='currency',
        fk='activity',
    )

    transaction_date_year = ToManyFilter(
        qs=Transaction,
        lookup_type='year',
        name='transaction_date',
        fk='activity'
    )

    #
    # Aggregated values filters
    #

    total_budget_lte = NumberFilter(
        lookup_type='lte',
        name='activity_aggregation__budget_value')

    total_budget_gte = NumberFilter(
        lookup_type='gte',
        name='activity_aggregation__budget_value')

    total_disbursement_lte = NumberFilter(
        lookup_type='lte',
        name='activity_aggregation__disbursement_value')

    total_disbursement_gte = NumberFilter(
        lookup_type='gte',
        name='activity_aggregation__disbursement_value')

    total_incoming_funds_lte = NumberFilter(
        lookup_type='lte',
        name='activity_aggregation__incoming_funds_value')

    total_incoming_funds_gte = NumberFilter(
        lookup_type='gte',
        name='activity_aggregation__incoming_funds_value')

    total_expenditure_lte = NumberFilter(
        lookup_type='lte',
        name='activity_aggregation__expenditure_value')

    total_expenditure_gte = NumberFilter(
        lookup_type='gte',
        name='activity_aggregation__expenditure_value')

    total_commitment_lte = NumberFilter(
        lookup_type='lte',
        name='activity_aggregation__commitment_value')

    total_commitment_gte = NumberFilter(
        lookup_type='gte',
        name='activity_aggregation__commitment_value')

    total_hierarchy_budget_lte = NumberFilter(
        lookup_type='lte',
        name='activity_plus_child_aggregation__budget_value')

    total_hierarchy_budget_gte = NumberFilter(
        lookup_type='gte',
        name='activity_plus_child_aggregation__budget_value')

    total_hierarchy_disbursement_lte = NumberFilter(
        lookup_type='lte',
        name='activity_plus_child_aggregation__disbursement_value')

    total_hierarchy_disbursement_gte = NumberFilter(
        lookup_type='gte',
        name='activity_plus_child_aggregation__disbursement_value')

    total_hierarchy_incoming_funds_lte = NumberFilter(
        lookup_type='lte',
        name='activity_plus_child_aggregation__incoming_funds_value')

    total_hierarchy_incoming_funds_gte = NumberFilter(
        lookup_type='gte',
        name='activity_plus_child_aggregation__incoming_funds_value')

    total_hierarchy_expenditure_lte = NumberFilter(
        lookup_type='lte',
        name='activity_plus_child_aggregation__expenditure_value')

    total_hierarchy_expenditure_gte = NumberFilter(
        lookup_type='gte',
        name='activity_plus_child_aggregation__expenditure_value')

    total_hierarchy_commitment_lte = NumberFilter(
        lookup_type='lte',
        name='activity_plus_child_aggregation__commitment_value')

    total_hierarchy_commitment_gte = NumberFilter(
        lookup_type='gte',
        name='activity_plus_child_aggregation__commitment_value')

    class Meta:
        model = Activity
        together_exclusive = [('budget_period_start', 'budget_period_end')]

class RelatedActivityFilter(FilterSet):

    related_activity_type = CommaSeparatedCharFilter(
        lookup_type='in',
        name='type__code')

    class Meta:
        model = RelatedActivity

class RelatedOrderingFilter(filters.OrderingFilter):
    """
    Extends OrderingFilter to support ordering by fields in related models
    using the Django ORM __ notation

    Also provides support for mapping of fields,
    in remove_invalid_fields a mapping is maintained
    to make 'user-friendly' names possible
    """

    def get_ordering(self, request, queryset, view):
        ordering = super(RelatedOrderingFilter, self).get_ordering(request, queryset, view)

        always_ordering = getattr(view, 'always_ordering', None)

        if ordering and always_ordering:
            ordering = ordering + [always_ordering]
            queryset.distinct(always_ordering)

        return ordering

    def filter_queryset(self, request, queryset, view):

        ordering = self.get_ordering(request, queryset, view)

        if ordering:
            ordering = [order.replace("-", "") for order in ordering]
            queryset = queryset.distinct(*ordering)

        return super(RelatedOrderingFilter, self).filter_queryset(request, queryset, view)

    def is_valid_field(self, model, field):
        """
        Return true if the field exists within the model (or in the related
        model specified using the Django ORM __ notation)
        """
        components = field.split('__', 1)
        try:
            field = model._meta.get_field(components[0])

            if isinstance(field, OneToOneRel):
                return self.is_valid_field(field.related_model, components[1])

            # reverse relation
            if isinstance(field, ForeignObjectRel):
                return self.is_valid_field(field.model, components[1])

            # foreign key
            if field.rel and len(components) == 2:
                return self.is_valid_field(field.rel.to, components[1])
            return True
        except FieldDoesNotExist:
            return False

    def remove_invalid_fields(self, queryset, ordering, view):

        mapped_fields = {
            'title': 'title__narratives__content',
            'recipient_country': 'recipient_country__name',
            'activity_budget_value': 'activity_aggregation__budget_value',
            'activity_incoming_funds_value': 'activity_aggregation__incoming_funds_value',
            'activity_commitment_value': 'activity_aggregation__commitment_value',
            'activity_disbursement_value': 'activity_aggregation__disbursement_value',
            'activity_expenditure_value': 'activity_aggregation__expenditure_value',
            'activity_plus_child_budget_value': 'activity_plus_child_aggregation__budget_value',
            'planned_start_date': 'planned_start',
            'actual_start_date': 'actual_start',
            'planned_end_date': 'planned_end',
            'actual_end_date': 'actual_end',
            'start_date': 'start_date',
            'end_date': 'end_date',
        }

        for i, term in enumerate(ordering):
            if term.lstrip('-') in mapped_fields:
                ordering[i] = ordering[i].replace(term.lstrip('-'), mapped_fields[term.lstrip('-')])

        return [term for term in ordering
                if self.is_valid_field(queryset.model, term.lstrip('-'))]
<commit_msg>provider/receiver org filter<commit_after>from django.db.models.fields import FieldDoesNotExist
from django.db.models.fields.related import ForeignObjectRel
from django.db.models.fields.related import OneToOneRel

from django_filters import FilterSet
from django_filters import NumberFilter
from django_filters import DateFilter
from django_filters import BooleanFilter
from django_filters import TypedChoiceFilter

from distutils.util import strtobool

from api.generics.filters import CommaSeparatedCharFilter
from api.generics.filters import TogetherFilterSet
from api.generics.filters import ToManyFilter

from rest_framework import filters

from iati.models import *
from iati.transaction.models import *

class ActivityFilter(TogetherFilterSet):

    activity_id = CommaSeparatedCharFilter(
        name='id',
        lookup_type='in')

    activity_scope = CommaSeparatedCharFilter(
        name='scope__code',
        lookup_type='in',)

    document_link_category = ToManyFilter(
        qs=DocumentLink,
        lookup_type='in',
        name='categories',
        fk='activity',
    )

    planned_start_date_lte = DateFilter(
        lookup_type='lte',
        name='planned_start')

    planned_start_date_gte = DateFilter(
        lookup_type='gte',
        name='planned_start')

    actual_start_date_lte = DateFilter(
        lookup_type='lte',
        name='actual_start')

    actual_start_date_gte = DateFilter(
        lookup_type='gte',
        name='actual_start')

    planned_end_date_lte = DateFilter(
        lookup_type='lte',
        name='planned_end')

    planned_end_date_gte = DateFilter(
        lookup_type='gte',
        name='planned_end')

    actual_end_date_lte = DateFilter(
        lookup_type='lte',
        name='actual_end')

    actual_end_date_gte = DateFilter(
        lookup_type='gte',
        name='actual_end')

    end_date_lte = DateFilter(
        lookup_type='lte',
        name='end_date')

    end_date_gte = DateFilter(
        lookup_type='gte',
        name='end_date')

    start_date_lte = DateFilter(
        lookup_type='lte',
        name='start_date')

    start_date_gte = DateFilter(
        lookup_type='gte',
        name='start_date')

    end_date_isnull = BooleanFilter(lookup_type='isnull', name='end_date')
    start_date_isnull = BooleanFilter(lookup_type='isnull', name='start_date')

    xml_source_ref = CommaSeparatedCharFilter(
        lookup_type='in',
        name='xml_source_ref',)

    activity_status = CommaSeparatedCharFilter(
        lookup_type='in',
        name='activity_status',)

    hierarchy = CommaSeparatedCharFilter(
        lookup_type='in',
        name='hierarchy',)

    collaboration_type = CommaSeparatedCharFilter(
        lookup_type='in',
        name='collaboration_type',)

    default_flow_type = CommaSeparatedCharFilter(
        lookup_type='in',
        name='default_flow_type',)

    default_aid_type = CommaSeparatedCharFilter(
        lookup_type='in',
        name='default_aid_type',)

    default_finance_type = CommaSeparatedCharFilter(
        lookup_type='in',
        name='default_finance_type',)

    default_tied_status = CommaSeparatedCharFilter(
        lookup_type='in',
        name='default_tied_status',)

    budget_period_start = DateFilter(
        lookup_type='gte',
        name='budget__period_start',)

    budget_period_end = DateFilter(
        lookup_type='lte',
        name='budget__period_end')

    humanitarian = TypedChoiceFilter(
        choices=(('0', 'False'), ('1', 'True')),
        coerce=strtobool)

    humanitarian_scope_type = ToManyFilter(
        qs=HumanitarianScope,
        lookup_type='in',
        name='type__code',
        fk='activity',
    )

    related_activity_id = ToManyFilter(
        qs=RelatedActivity,
        fk='current_activity',
        lookup_type='in',
        name='ref_activity__id',
    )

    related_activity_type = ToManyFilter(
        qs=RelatedActivity,
        lookup_type='in',
        name='type__code',
        fk='current_activity',
    )

    related_activity_recipient_country = ToManyFilter(
        qs=RelatedActivity,
        lookup_type='in',
        name='ref_activity__recipient_country',
        fk='current_activity',
    )

    related_activity_recipient_region = ToManyFilter(
        qs=RelatedActivity,
        lookup_type='in',
        name='ref_activity__recipient_region',
        fk='current_activity',
    )

    related_activity_sector = ToManyFilter(
        qs=RelatedActivity,
        lookup_type='in',
        name='ref_activity__sector',
        fk='current_activity',
    )

    related_activity_sector_category = ToManyFilter(
        qs=RelatedActivity,
        lookup_type='in',
        name='ref_activity__sector__category',
        fk='current_activity',
    )

    budget_currency = ToManyFilter(
        qs=Budget,
        lookup_type='in',
        name='currency__code',
        fk='activity',
    )

    recipient_country = ToManyFilter(
        qs=ActivityRecipientCountry,
        lookup_type='in',
        name='country__code',
        fk='activity',
    )

    recipient_region = ToManyFilter(
        qs=ActivityRecipientRegion,
        lookup_type='in',
        name='region__code',
        fk='activity',
    )

    sector = ToManyFilter(
        qs=ActivitySector,
        lookup_type='in',
        name='sector__code',
        fk='activity',
    )

    sector_category = ToManyFilter(
        qs=ActivitySector,
        lookup_type='in',
        name='sector__category__code',
        fk='activity',
    )

    policy_marker = ToManyFilter(
        qs=ActivityPolicyMarker,
        lookup_type='in',
        name='code',
        fk='activity',
    )

    participating_organisation = ToManyFilter(
        qs=ActivityParticipatingOrganisation,
        lookup_type='in',
        name='normalized_ref',
        fk='activity',
    )

    participating_organisation_name = ToManyFilter(
        qs=ActivityParticipatingOrganisation,
        lookup_type='in',
        name='primary_name',
        fk='activity',
    )

    participating_organisation_role = ToManyFilter(
        qs=ActivityParticipatingOrganisation,
        lookup_type='in',
        name='role__code',
        fk='activity',
    )

    participating_organisation_type = ToManyFilter(
        qs=ActivityParticipatingOrganisation,
        lookup_type='in',
        name='type__code',
        fk='activity',
    )

    reporting_organisation = ToManyFilter(
        qs=ActivityReportingOrganisation,
        lookup_type='in',
        name='normalized_ref',
        fk='activity',
    )

    reporting_organisation_startswith = ToManyFilter(
        qs=ActivityReportingOrganisation,
        lookup_type='startswith',
        name='normalized_ref',
        fk='activity',
    )

    result_title = ToManyFilter(
        qs=Result,
        lookup_type='in',
        name='resulttitle__narratives__content',
        fk='activity',
    )

    indicator_title = ToManyFilter(
        qs=ResultIndicatorTitle,
        lookup_type='in',
        name='primary_name',
        fk='result_indicator__result__activity')

    indicator_period_end_year = ToManyFilter(
        qs=ResultIndicatorPeriod,
        lookup_type='year',
        name='period_end',
        fk='result_indicator__result__activity')

    #
    # Transaction filters
    #

    transaction_type = ToManyFilter(
        qs=Transaction,
        lookup_type='in',
        name='transaction_type',
        fk='activity',
    )

    provider_organisation_primary_name = ToManyFilter(
        qs=Transaction,
        lookup_type='in',
        name='provider_organisation__primary_name',
        fk='activity',
    )

    receiver_organisation_primary_name = ToManyFilter(
        qs=Transaction,
        lookup_type='in',
        name='receiver_organisation__primary_name',
        fk='activity',
    )

    transaction_provider_organisation = ToManyFilter(
        qs=Transaction,
        lookup_type='in',
        name='provider_organisation__ref',
        fk='activity',
    )

    transaction_receiver_organisation = ToManyFilter(
        qs=Transaction,
        lookup_type='in',
        name='receiver_organisation__ref',
        fk='activity',
    )

    transaction_provider_organisation_name = ToManyFilter(
        qs=Transaction,
        lookup_type='in',
        name='provider_organisation__narratives__content',
        fk='activity',
    )

    transaction_receiver_organisation_name = ToManyFilter(
        qs=Transaction,
        lookup_type='in',
        name='receiver_organisation__narratives__content',
        fk='activity',
    )

    transaction_provider_activity = ToManyFilter(
        qs=Transaction,
        lookup_type='in',
        name='provider_organisation__provider_activity_ref',
        fk='activity',
    )

    transaction_receiver_activity = ToManyFilter(
        qs=Transaction,
        lookup_type='in',
        name='receiver_organisation__receiver_activity_ref',
        fk='activity',
    )

    transaction_currency = ToManyFilter(
        qs=Transaction,
        lookup_type='in',
        name='currency',
        fk='activity',
    )

    transaction_date_year = ToManyFilter(
        qs=Transaction,
        lookup_type='year',
        name='transaction_date',
        fk='activity'
    )

    #
    # Aggregated values filters
    #

    total_budget_lte = NumberFilter(
        lookup_type='lte',
        name='activity_aggregation__budget_value')

    total_budget_gte = NumberFilter(
        lookup_type='gte',
        name='activity_aggregation__budget_value')

    total_disbursement_lte = NumberFilter(
        lookup_type='lte',
        name='activity_aggregation__disbursement_value')

    total_disbursement_gte = NumberFilter(
        lookup_type='gte',
        name='activity_aggregation__disbursement_value')

    total_incoming_funds_lte = NumberFilter(
        lookup_type='lte',
        name='activity_aggregation__incoming_funds_value')

    total_incoming_funds_gte = NumberFilter(
        lookup_type='gte',
        name='activity_aggregation__incoming_funds_value')

    total_expenditure_lte = NumberFilter(
        lookup_type='lte',
        name='activity_aggregation__expenditure_value')

    total_expenditure_gte = NumberFilter(
        lookup_type='gte',
        name='activity_aggregation__expenditure_value')

    total_commitment_lte = NumberFilter(
        lookup_type='lte',
        name='activity_aggregation__commitment_value')

    total_commitment_gte = NumberFilter(
        lookup_type='gte',
        name='activity_aggregation__commitment_value')

    total_hierarchy_budget_lte = NumberFilter(
        lookup_type='lte',
        name='activity_plus_child_aggregation__budget_value')

    total_hierarchy_budget_gte = NumberFilter(
        lookup_type='gte',
        name='activity_plus_child_aggregation__budget_value')

    total_hierarchy_disbursement_lte = NumberFilter(
        lookup_type='lte',
        name='activity_plus_child_aggregation__disbursement_value')

    total_hierarchy_disbursement_gte = NumberFilter(
        lookup_type='gte',
        name='activity_plus_child_aggregation__disbursement_value')

    total_hierarchy_incoming_funds_lte = NumberFilter(
        lookup_type='lte',
        name='activity_plus_child_aggregation__incoming_funds_value')

    total_hierarchy_incoming_funds_gte = NumberFilter(
        lookup_type='gte',
        name='activity_plus_child_aggregation__incoming_funds_value')

    total_hierarchy_expenditure_lte = NumberFilter(
        lookup_type='lte',
        name='activity_plus_child_aggregation__expenditure_value')

    total_hierarchy_expenditure_gte = NumberFilter(
        lookup_type='gte',
        name='activity_plus_child_aggregation__expenditure_value')

    total_hierarchy_commitment_lte = NumberFilter(
        lookup_type='lte',
        name='activity_plus_child_aggregation__commitment_value')

    total_hierarchy_commitment_gte = NumberFilter(
        lookup_type='gte',
        name='activity_plus_child_aggregation__commitment_value')

    class Meta:
        model = Activity
        together_exclusive = [('budget_period_start', 'budget_period_end')]

class RelatedActivityFilter(FilterSet):

    related_activity_type = CommaSeparatedCharFilter(
        lookup_type='in',
        name='type__code')

    class Meta:
        model = RelatedActivity

class RelatedOrderingFilter(filters.OrderingFilter):
    """
    Extends OrderingFilter to support ordering by fields in related models
    using the Django ORM __ notation

    Also provides support for mapping of fields,
    in remove_invalid_fields a mapping is maintained
    to make 'user-friendly' names possible
    """

    def get_ordering(self, request, queryset, view):
        ordering = super(RelatedOrderingFilter, self).get_ordering(request, queryset, view)

        always_ordering = getattr(view, 'always_ordering', None)

        if ordering and always_ordering:
            ordering = ordering + [always_ordering]
            queryset.distinct(always_ordering)

        return ordering

    def filter_queryset(self, request, queryset, view):

        ordering = self.get_ordering(request, queryset, view)

        if ordering:
            ordering = [order.replace("-", "") for order in ordering]
            queryset = queryset.distinct(*ordering)

        return super(RelatedOrderingFilter, self).filter_queryset(request, queryset, view)

    def is_valid_field(self, model, field):
        """
        Return true if the field exists within the model (or in the related
        model specified using the Django ORM __ notation)
        """
        components = field.split('__', 1)
        try:
            field = model._meta.get_field(components[0])

            if isinstance(field, OneToOneRel):
                return self.is_valid_field(field.related_model, components[1])

            # reverse relation
            if isinstance(field, ForeignObjectRel):
                return self.is_valid_field(field.model, components[1])

            # foreign key
            if field.rel and len(components) == 2:
                return self.is_valid_field(field.rel.to, components[1])
            return True
        except FieldDoesNotExist:
            return False

    def remove_invalid_fields(self, queryset, ordering, view):

        mapped_fields = {
            'title': 'title__narratives__content',
            'recipient_country': 'recipient_country__name',
            'activity_budget_value': 'activity_aggregation__budget_value',
            'activity_incoming_funds_value': 'activity_aggregation__incoming_funds_value',
            'activity_commitment_value': 'activity_aggregation__commitment_value',
            'activity_disbursement_value': 'activity_aggregation__disbursement_value',
            'activity_expenditure_value': 'activity_aggregation__expenditure_value',
            'activity_plus_child_budget_value': 'activity_plus_child_aggregation__budget_value',
            'planned_start_date': 'planned_start',
            'actual_start_date': 'actual_start',
            'planned_end_date': 'planned_end',
            'actual_end_date': 'actual_end',
            'start_date': 'start_date',
            'end_date': 'end_date',
        }

        for i, term in enumerate(ordering):
            if term.lstrip('-') in mapped_fields:
                ordering[i] = ordering[i].replace(term.lstrip('-'), mapped_fields[term.lstrip('-')])

        return [term for term in ordering
                if self.is_valid_field(queryset.model, term.lstrip('-'))]
<|endoftext|>


<commit_before>import caffe
from caffe import layers as L
from caffe import params as P

def conv_bn_scale_relu(bottom, num_output=64, kernel_size=3, stride=1, pad=0):
    conv = L.Convolution(bottom, num_output=num_output, kernel_size=kernel_size, stride=stride, pad=pad,
                         param=[dict(lr_mult=1, decay_mult=1), dict(lr_mult=2, decay_mult=0)],
                         weight_filler=dict(type='xavier', std=0.01),
                         bias_filler=dict(type='constant', value=0))
    conv_bn = L.BatchNorm(conv, use_global_stats=False, in_place=True)
    conv_scale = L.Scale(conv, scale_param=dict(bias_term=True), in_place=True)
    conv_relu = L.ReLU(conv, in_place=True)

    return conv, conv_bn, conv_scale, conv_relu

def conv_bn_scale(bottom, num_output=64, kernel_size=3, stride=1, pad=0):
    conv = L.Convolution(bottom, num_output=num_output, kernel_size=kernel_size, stride=stride, pad=pad,
                         param=[dict(lr_mult=1, decay_mult=1), dict(lr_mult=2, decay_mult=0)],
                         weight_filler=dict(type='xavier', std=0.01),
                         bias_filler=dict(type='constant', value=0.2))
    conv_bn = L.BatchNorm(conv, use_global_stats=False, in_place=True)
    conv_scale = L.Scale(conv, scale_param=dict(bias_term=True), in_place=True)

    return conv, conv_bn, conv_scale

def eltwize_relu(bottom1, bottom2):
    residual_eltwise = L.Eltwise(bottom1, bottom2, eltwise_param=dict(operation=1))
    residual_eltwise_relu = L.ReLU(residual_eltwise, in_place=True)

    return residual_eltwise, residual_eltwise_relu

def residual_branch(bottom, base_output=64):
    """
    input:4*base_output x n x n
    output:4*base_output x n x n
    :param base_output: base num_output of branch2
    :param bottom: bottom layer
    :return: layers
    """
    branch2a, branch2a_bn, branch2a_scale, branch2a_relu = \
        conv_bn_scale_relu(bottom, num_output=base_output, kernel_size=1)  # base_output x n x n
    branch2b, branch2b_bn, branch2b_scale, branch2b_relu = \
        conv_bn_scale_relu(branch2a, num_output=base_output, kernel_size=3, pad=1)  # base_output x n x n
    branch2c, branch2c_bn, branch2c_scale = \
        conv_bn_scale(branch2b, num_output=4 * base_output, kernel_size=1)  # 4*base_output x n x n

    residual, residual_relu = \
        eltwize_relu(bottom, branch2c)  # 4*base_output x n x n

    return branch2a, branch2a_bn, branch2a_scale, branch2a_relu, branch2b, branch2b_bn, branch2b_scale, branch2b_relu, \
           branch2c, branch2c_bn, branch2c_scale, residual, residual_relu

def residual_branch_shortcut(bottom, stride=2, base_output=64):
    """

    :param stride: stride
    :param base_output: base num_output of branch2
    :param bottom: bottom layer
    :return: layers
    """
    branch1, branch1_bn, branch1_scale = \
        conv_bn_scale(bottom, num_output=4 * base_output, kernel_size=1, stride=stride)

    branch2a, branch2a_bn, branch2a_scale, branch2a_relu = \
        conv_bn_scale_relu(bottom, num_output=base_output, kernel_size=1, stride=stride)
    branch2b, branch2b_bn, branch2b_scale, branch2b_relu = \
        conv_bn_scale_relu(branch2a, num_output=base_output, kernel_size=3, pad=1)
    branch2c, branch2c_bn, branch2c_scale = \
        conv_bn_scale(branch2b, num_output=4 * base_output, kernel_size=1)

    residual, residual_relu = \
        eltwize_relu(branch1, branch2c)  # 4*base_output x n x n

    return branch1, branch1_bn, branch1_scale, branch2a, branch2a_bn, branch2a_scale, branch2a_relu, branch2b, \
           branch2b_bn, branch2b_scale, branch2b_relu, branch2c, branch2c_bn, branch2c_scale, residual, residual_relu

branch_shortcut_string = 'n.res(stage)a_branch1, n.res(stage)a_branch1_bn, n.res(stage)a_branch1_scale, \
        n.res(stage)a_branch2a, n.res(stage)a_branch2a_bn, n.res(stage)a_branch2a_scale, n.res(stage)a_branch2a_relu, \
        n.res(stage)a_branch2b, n.res(stage)a_branch2b_bn, n.res(stage)a_branch2b_scale, n.res(stage)a_branch2b_relu, \
        n.res(stage)a_branch2c, n.res(stage)a_branch2c_bn, n.res(stage)a_branch2c_scale, n.res(stage)a, n.res(stage)a_relu = \
            residual_branch_shortcut((bottom), stride=(stride), base_output=(num))'

branch_string = 'n.res(stage)b(order)_branch2a, n.res(stage)b(order)_branch2a_bn, n.res(stage)b(order)_branch2a_scale, \
        n.res(stage)b(order)_branch2a_relu, n.res(stage)b(order)_branch2b, n.res(stage)b(order)_branch2b_bn, \
        n.res(stage)b(order)_branch2b_scale, n.res(stage)b(order)_branch2b_relu, n.res(stage)b(order)_branch2c, \
        n.res(stage)b(order)_branch2c_bn, n.res(stage)b(order)_branch2c_scale, n.res(stage)b(order), n.res(stage)b(order)_relu = \
            residual_branch((bottom), base_output=(num))'

class ResNet(object):
    def __init__(self, lmdb_train, lmdb_test, num_output):
        self.train_data = lmdb_train
        self.test_data = lmdb_test
        self.classifier_num = num_output

    def resnet_layers_proto(self, batch_size, phase='TRAIN', stages=(3, 4, 6, 3)):
        """

        :param batch_size: the batch_size of train and test phase
        :param phase: TRAIN or TEST
        :param stages: the num of layers = 2 + 3*sum(stages), layers would better be chosen from [50, 101, 152]
                       {every stage is composed of 1 residual_branch_shortcut module and stage[i]-1 residual_branch
                       modules, each module consists of 3 conv layers}
                        (3, 4, 6, 3) for 50 layers; (3, 4, 23, 3) for 101 layers; (3, 8, 36, 3) for 152 layers
        """
        n = caffe.NetSpec()
        if phase == 'TRAIN':
            source_data = self.train_data
            mirror = True
        else:
            source_data = self.test_data
            mirror = False
        n.data, n.label = L.Data(source=source_data, backend=P.Data.LMDB, batch_size=batch_size, ntop=2,
                                 transform_param=dict(crop_size=224, mean_value=[104, 117, 123], mirror=mirror))

        n.conv1, n.conv1_bn, n.conv1_scale, n.conv1_relu = \
            conv_bn_scale_relu(n.data, num_output=64, kernel_size=7, stride=2, pad=3)  # 64x112x112
        n.pool1 = L.Pooling(n.conv1, kernel_size=3, stride=2, pool=P.Pooling.MAX)  # 64x56x56

        for num in xrange(len(stages)):  # num = 0, 1, 2, 3
            for i in xrange(stages[num]):
                if i == 0:
                    stage_string = branch_shortcut_string
                    bottom_string = ['n.pool1', 'n.res2b%s' % str(stages[0] - 1), 'n.res3b%s' % str(stages[1] - 1),
                                     'n.res4b%s' % str(stages[2] - 1)][num]
                else:
                    stage_string = branch_string
                    if i == 1:
                        bottom_string = 'n.res%sa' % str(num + 2)
                    else:
                        bottom_string = 'n.res%sb%s' % (str(num + 2), str(i - 1))
                exec (stage_string.replace('(stage)', str(num + 2)).replace('(bottom)', bottom_string).
                      replace('(num)', str(2 ** num * 64)).replace('(order)', str(i)).
                      replace('(stride)', str(int(num > 0) + 1)))

        exec 'n.pool5 = L.Pooling((bottom), pool=P.Pooling.AVE, global_pooling=True)'.\
            replace('(bottom)', 'n.res5b%s' % str(stages[3] - 1))
        n.classifier = L.InnerProduct(n.pool5, num_output=self.classifier_num,
                                      param=[dict(lr_mult=1, decay_mult=1), dict(lr_mult=2, decay_mult=0)],
                                      weight_filler=dict(type='xavier'),
                                      bias_filler=dict(type='constant', value=0))
        n.loss = L.SoftmaxWithLoss(n.classifier, n.label)
        if phase == 'TRAIN':
            pass
        else:
            n.accuracy_top1 = L.Accuracy(n.classifier, n.label, include=dict(phase=1))
            n.accuracy_top5 = L.Accuracy(n.classifier, n.label, include=dict(phase=1),
                                         accuracy_param=dict(top_k=5))

        return n.to_proto()
<commit_msg>Delete resnet.py<commit_after><|endoftext|>


<commit_before># modu
# Copyright (C) 2007-2008 Phil Christensen
#
# $Id$
#
# See LICENSE for details

"""
Twisted plugin to launch a modu web application container.
"""

from zope.interface import implements

from twisted.python import usage
from twisted.plugin import IPlugin
from twisted.application import internet, service
from twisted.internet import reactor
from twisted.web import server, resource

from modu.web import wsgi, app

import os

class Options(usage.Options):
	"""
	Implement usage parsing for the modu-web plugin.
	"""
	optParameters = [["port", "p", 8888, "Port to use for web server."],
					 ['interface', 'i', None, 'Interface to listen on.'],
					 ['logfile', 'l', 'twistd-access.log', 'Path to access log.']
					]

class ModuServiceMaker(object):
	"""
	Create a modu web service with twisted.web.
	"""
	implements(service.IServiceMaker, IPlugin)
	tapname = "modu-web"
	description = "Run a modu application server."
	options = Options

	def makeService(self, config):
		"""
		Instantiate the service.
		"""
		env = {'MODU_PATH': os.environ.get('MODU_PATH', '.')}

		if(config['logfile'] != '-'):
			site = server.Site(wsgi.WSGIResource(app.handler, env=env), logPath=config['logfile'])
		else:
			site = server.Site(wsgi.WSGIResource(app.handler, env=env))
		server.Site.requestFactory = app.UnparsedRequest

		if(config['interface'] is not None):
			web_service = internet.TCPServer(int(config['port']), site, interface=config['interface'])
		else:
			web_service = internet.TCPServer(int(config['port']), site)

		return web_service

serviceMaker = ModuServiceMaker()<commit_msg>changed default behavior to log to standard out<commit_after># modu
# Copyright (C) 2007-2008 Phil Christensen
#
# $Id$
#
# See LICENSE for details

"""
Twisted plugin to launch a modu web application container.
"""

from zope.interface import implements

from twisted.python import usage
from twisted.plugin import IPlugin
from twisted.application import internet, service
from twisted.internet import reactor
from twisted.web import server, resource

from modu.web import wsgi, app

import os

class Options(usage.Options):
	"""
	Implement usage parsing for the modu-web plugin.
	"""
	optParameters = [["port", "p", 8888, "Port to use for web server."],
					 ['interface', 'i', None, 'Interface to listen on.'],
					 ['logfile', 'l', '-', 'Path to access log.']
					]

class ModuServiceMaker(object):
	"""
	Create a modu web service with twisted.web.
	"""
	implements(service.IServiceMaker, IPlugin)
	tapname = "modu-web"
	description = "Run a modu application server."
	options = Options

	def makeService(self, config):
		"""
		Instantiate the service.
		"""
		env = {'MODU_PATH': os.environ.get('MODU_PATH', '.')}

		if(config['logfile'] != '-'):
			site = server.Site(wsgi.WSGIResource(app.handler, env=env), logPath=config['logfile'])
		else:
			site = server.Site(wsgi.WSGIResource(app.handler, env=env))
		server.Site.requestFactory = app.UnparsedRequest

		if(config['interface'] is not None):
			web_service = internet.TCPServer(int(config['port']), site, interface=config['interface'])
		else:
			web_service = internet.TCPServer(int(config['port']), site)

		return web_service

serviceMaker = ModuServiceMaker()<|endoftext|>