<commit_before># pylint: disable=line-too-long,invalid-name

import warnings

from allennlp import predictors
from allennlp.predictors import Predictor
from allennlp.models.archival import load_archive

class PretrainedModel:
    """
    A pretrained model is determined by both an archive file
    (representing the trained model)
    and a choice of predictor.
    """
    def __init__(self, archive_file: str, predictor_name: str) -> None:
        self.archive_file = archive_file
        self.predictor_name = predictor_name

    def predictor(self) -> Predictor:
        archive = load_archive(self.archive_file)
        return Predictor.from_archive(archive, self.predictor_name)

# TODO(Mark): Figure out a way to make PretrainedModel generic on Predictor, so we can remove these type ignores.

#### Models in the demo ####
def srl_with_elmo_luheng_2018() -> predictors.SemanticRoleLabelerPredictor:
    with warnings.catch_warnings():
        warnings.simplefilter(action="ignore", category=DeprecationWarning)
        model = PretrainedModel('https://s3-us-west-2.amazonaws.com/allennlp/models/srl-model-2018.05.25.tar.gz',
                                'semantic-role-labeling')
        return model.predictor() # type: ignore

def bidirectional_attention_flow_seo_2017() -> predictors.BidafPredictor:
    with warnings.catch_warnings():
        warnings.simplefilter(action="ignore", category=DeprecationWarning)
        model = PretrainedModel('https://s3-us-west-2.amazonaws.com/allennlp/models/bidaf-model-2017.09.15-charpad.tar.gz',
                                'machine-comprehension')
        return model.predictor() # type: ignore

def naqanet_dua_2019() -> predictors.BidafPredictor:
    with warnings.catch_warnings():
        warnings.simplefilter(action="ignore", category=DeprecationWarning)
        model = PretrainedModel('https://s3-us-west-2.amazonaws.com/allennlp/models/naqanet-2019.04.29-fixed-weight-names.tar.gz',
                                'machine-comprehension')
        return model.predictor()  # type: ignore

def open_information_extraction_stanovsky_2018() -> predictors.OpenIePredictor:
    model = PretrainedModel('https://s3-us-west-2.amazonaws.com/allennlp/models/openie-model.2018-08-20.tar.gz',
                            'open-information-extraction')
    return model.predictor() # type: ignore

def decomposable_attention_with_elmo_parikh_2017() -> predictors.DecomposableAttentionPredictor:
    with warnings.catch_warnings():
        warnings.simplefilter(action="ignore", category=DeprecationWarning)
        model = PretrainedModel('https://s3-us-west-2.amazonaws.com/allennlp/models/decomposable-attention-elmo-2018.02.19.tar.gz',
                                'textual-entailment')
        return model.predictor() # type: ignore

def neural_coreference_resolution_lee_2017() -> predictors.CorefPredictor:
    with warnings.catch_warnings():
        warnings.simplefilter(action="ignore", category=DeprecationWarning)
        model = PretrainedModel('https://s3-us-west-2.amazonaws.com/allennlp/models/coref-model-2018.02.05.tar.gz',
                                'coreference-resolution')
        return model.predictor() # type: ignore

def named_entity_recognition_with_elmo_peters_2018() -> predictors.SentenceTaggerPredictor:
    with warnings.catch_warnings():
        warnings.simplefilter(action="ignore", category=DeprecationWarning)
        model = PretrainedModel('https://s3-us-west-2.amazonaws.com/allennlp/models/ner-model-2018.12.18.tar.gz',
                                'sentence-tagger')
        predictor = model.predictor()
        # pylint: disable=protected-access
        predictor._dataset_reader._token_indexers['token_characters']._min_padding_length = 3  # type: ignore
        return predictor  # type: ignore

def fine_grained_named_entity_recognition_with_elmo_peters_2018() -> predictors.SentenceTaggerPredictor:
    model = PretrainedModel('https://s3-us-west-2.amazonaws.com/allennlp/models/fine-grained-ner-model-elmo-2018.08.31.tar.gz',
                            'sentence-tagger')
    predictor = model.predictor()
    # pylint: disable=protected-access
    predictor._dataset_reader._token_indexers['token_characters']._min_padding_length = 3  # type: ignore
    return predictor  # type: ignore

def span_based_constituency_parsing_with_elmo_joshi_2018() -> predictors.ConstituencyParserPredictor:
    with warnings.catch_warnings():
        warnings.simplefilter(action="ignore", category=DeprecationWarning)
        model = PretrainedModel('https://s3-us-west-2.amazonaws.com/allennlp/models/elmo-constituency-parser-2018.03.14.tar.gz',
                                'constituency-parser')
        return model.predictor() # type: ignore

def biaffine_parser_stanford_dependencies_todzat_2017() -> predictors.BiaffineDependencyParserPredictor:
    with warnings.catch_warnings():
        warnings.simplefilter(action="ignore", category=DeprecationWarning)
        model = PretrainedModel('https://s3-us-west-2.amazonaws.com/allennlp/models/biaffine-dependency-parser-ptb-2018.08.23.tar.gz',
                                'biaffine-dependency-parser')
        return model.predictor() # type: ignore

#### Models not in the demo ####
def biaffine_parser_universal_dependencies_todzat_2017() -> predictors.BiaffineDependencyParserPredictor:
    with warnings.catch_warnings():
        warnings.simplefilter(action="ignore", category=DeprecationWarning)
        model = PretrainedModel('https://s3-us-west-2.amazonaws.com/allennlp/models/biaffine-dependency-parser-ud-2018.08.23.tar.gz',
                                'biaffine-dependency-parser')
        return model.predictor() # type: ignore

def esim_nli_with_elmo_chen_2017() -> predictors.DecomposableAttentionPredictor:
    with warnings.catch_warnings():
        warnings.simplefilter(action="ignore", category=DeprecationWarning)
        model = PretrainedModel('https://s3-us-west-2.amazonaws.com/allennlp/models/esim-elmo-2018.05.17.tar.gz',
                                'textual-entailment')
        return model.predictor() # type: ignore
<commit_msg>updated fine-grained-ner-model-elmo URL (#2868)<commit_after># pylint: disable=line-too-long,invalid-name

import warnings

from allennlp import predictors
from allennlp.predictors import Predictor
from allennlp.models.archival import load_archive

class PretrainedModel:
    """
    A pretrained model is determined by both an archive file
    (representing the trained model)
    and a choice of predictor.
    """
    def __init__(self, archive_file: str, predictor_name: str) -> None:
        self.archive_file = archive_file
        self.predictor_name = predictor_name

    def predictor(self) -> Predictor:
        archive = load_archive(self.archive_file)
        return Predictor.from_archive(archive, self.predictor_name)

# TODO(Mark): Figure out a way to make PretrainedModel generic on Predictor, so we can remove these type ignores.

#### Models in the demo ####
def srl_with_elmo_luheng_2018() -> predictors.SemanticRoleLabelerPredictor:
    with warnings.catch_warnings():
        warnings.simplefilter(action="ignore", category=DeprecationWarning)
        model = PretrainedModel('https://s3-us-west-2.amazonaws.com/allennlp/models/srl-model-2018.05.25.tar.gz',
                                'semantic-role-labeling')
        return model.predictor() # type: ignore

def bidirectional_attention_flow_seo_2017() -> predictors.BidafPredictor:
    with warnings.catch_warnings():
        warnings.simplefilter(action="ignore", category=DeprecationWarning)
        model = PretrainedModel('https://s3-us-west-2.amazonaws.com/allennlp/models/bidaf-model-2017.09.15-charpad.tar.gz',
                                'machine-comprehension')
        return model.predictor() # type: ignore

def naqanet_dua_2019() -> predictors.BidafPredictor:
    with warnings.catch_warnings():
        warnings.simplefilter(action="ignore", category=DeprecationWarning)
        model = PretrainedModel('https://s3-us-west-2.amazonaws.com/allennlp/models/naqanet-2019.04.29-fixed-weight-names.tar.gz',
                                'machine-comprehension')
        return model.predictor()  # type: ignore

def open_information_extraction_stanovsky_2018() -> predictors.OpenIePredictor:
    model = PretrainedModel('https://s3-us-west-2.amazonaws.com/allennlp/models/openie-model.2018-08-20.tar.gz',
                            'open-information-extraction')
    return model.predictor() # type: ignore

def decomposable_attention_with_elmo_parikh_2017() -> predictors.DecomposableAttentionPredictor:
    with warnings.catch_warnings():
        warnings.simplefilter(action="ignore", category=DeprecationWarning)
        model = PretrainedModel('https://s3-us-west-2.amazonaws.com/allennlp/models/decomposable-attention-elmo-2018.02.19.tar.gz',
                                'textual-entailment')
        return model.predictor() # type: ignore

def neural_coreference_resolution_lee_2017() -> predictors.CorefPredictor:
    with warnings.catch_warnings():
        warnings.simplefilter(action="ignore", category=DeprecationWarning)
        model = PretrainedModel('https://s3-us-west-2.amazonaws.com/allennlp/models/coref-model-2018.02.05.tar.gz',
                                'coreference-resolution')
        return model.predictor() # type: ignore

def named_entity_recognition_with_elmo_peters_2018() -> predictors.SentenceTaggerPredictor:
    with warnings.catch_warnings():
        warnings.simplefilter(action="ignore", category=DeprecationWarning)
        model = PretrainedModel('https://s3-us-west-2.amazonaws.com/allennlp/models/ner-model-2018.12.18.tar.gz',
                                'sentence-tagger')
        predictor = model.predictor()
        # pylint: disable=protected-access
        predictor._dataset_reader._token_indexers['token_characters']._min_padding_length = 3  # type: ignore
        return predictor  # type: ignore

def fine_grained_named_entity_recognition_with_elmo_peters_2018() -> predictors.SentenceTaggerPredictor:
    model = PretrainedModel('https://s3-us-west-2.amazonaws.com/allennlp/models/fine-grained-ner-model-elmo-2018.12.21.tar.gz',
                            'sentence-tagger')
    predictor = model.predictor()
    # pylint: disable=protected-access
    predictor._dataset_reader._token_indexers['token_characters']._min_padding_length = 3  # type: ignore
    return predictor  # type: ignore

def span_based_constituency_parsing_with_elmo_joshi_2018() -> predictors.ConstituencyParserPredictor:
    with warnings.catch_warnings():
        warnings.simplefilter(action="ignore", category=DeprecationWarning)
        model = PretrainedModel('https://s3-us-west-2.amazonaws.com/allennlp/models/elmo-constituency-parser-2018.03.14.tar.gz',
                                'constituency-parser')
        return model.predictor() # type: ignore

def biaffine_parser_stanford_dependencies_todzat_2017() -> predictors.BiaffineDependencyParserPredictor:
    with warnings.catch_warnings():
        warnings.simplefilter(action="ignore", category=DeprecationWarning)
        model = PretrainedModel('https://s3-us-west-2.amazonaws.com/allennlp/models/biaffine-dependency-parser-ptb-2018.08.23.tar.gz',
                                'biaffine-dependency-parser')
        return model.predictor() # type: ignore

#### Models not in the demo ####
def biaffine_parser_universal_dependencies_todzat_2017() -> predictors.BiaffineDependencyParserPredictor:
    with warnings.catch_warnings():
        warnings.simplefilter(action="ignore", category=DeprecationWarning)
        model = PretrainedModel('https://s3-us-west-2.amazonaws.com/allennlp/models/biaffine-dependency-parser-ud-2018.08.23.tar.gz',
                                'biaffine-dependency-parser')
        return model.predictor() # type: ignore

def esim_nli_with_elmo_chen_2017() -> predictors.DecomposableAttentionPredictor:
    with warnings.catch_warnings():
        warnings.simplefilter(action="ignore", category=DeprecationWarning)
        model = PretrainedModel('https://s3-us-west-2.amazonaws.com/allennlp/models/esim-elmo-2018.05.17.tar.gz',
                                'textual-entailment')
        return model.predictor() # type: ignore
<|endoftext|>


<commit_before>from collections import defaultdict

LINK_PDWIKI = defaultdict(lambda: 'https://en.wikipedia.org/wiki/Public_domain', **{
    'en-US': 'https://en.wikipedia.org/wiki/Public_domain',
    'de': 'https://de.wikipedia.org/wiki/Gemeinfreiheit',
    'fr': 'https://fr.wikipedia.org/wiki/Domaine_public_%28propri%C3%A9t%C3%A9_intellectuelle%29',
    'es': 'https://es.wikipedia.org/wiki/Dominio_p%C3%BAblico',
    'nl': 'https://nl.wikipedia.org/wiki/Publiek_domein',
    'pl': 'https://pl.wikipedia.org/wiki/Domena_publiczna',
    'pt-BR': 'https://pt.wikipedia.org/wiki/Dom%C3%ADnio_p%C3%BAblico',
    'sl': 'https://sl.wikipedia.org/wiki/Javna_last',
    'sq': 'https://sq.wikipedia.org/wiki/Domen_publik',
    'zh-CN': 'https://zh.wikipedia.org/wiki/%E5%85%AC%E6%9C%89%E9%A2%86%E5%9F%9F',
    'zh-TW': 'https://zh.wikipedia.org/wiki/%E5%85%AC%E6%9C%89%E9%A2%86%E5%9F%9F',
    'it': 'https://it.wikipedia.org/wiki/Pubblico_dominio',
    'lij': 'https://it.wikipedia.org/wiki/Pubblico_dominio',
    'ja': 'https://ja.wikipedia.org/wiki/%E3%83%91%E3%83%96%E3%83%AA%E3%83%83%E3%82%AF%E3%83%89%E3%83%A1%E3%82%A4%E3%83%B3',
})

LINK_BRIEF = defaultdict(lambda: 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_en-US.pdf', **{
    'en-US': 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_en-US.pdf',
    'de': 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_de.pdf',
    'es': 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_es-ES.pdf',
    'nl': 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_nl.pdf',
    'pl': 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_pl.pdf',
    'sl': 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_sl.pdf',
    'fr': 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_fr.pdf',
    'zh-TW': 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_zh-TW.pdf',
    'pt-BR': 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_pt-BR.pdf',
    'it': 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_it.pdf',
    'lij': 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_lij.pdf',
    'ja': 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_ja.pdf',
    'sq': 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_sq.pdf',
})

LINK_CCLICENSE = defaultdict(lambda: 'http://creativecommons.org/licenses/by-nc-sa/2.5/', **{
    'en-US': 'http://creativecommons.org/licenses/by-nc-sa/2.5/',
    'de': 'http://creativecommons.org/licenses/by-nc-sa/2.5/deed.de',
    'fr': 'http://creativecommons.org/licenses/by-nc-sa/2.5/deed.fr',
    'es': 'http://creativecommons.org/licenses/by-nc-sa/2.5/deed.es',
    'nl': 'http://creativecommons.org/licenses/by-nc-sa/2.5/deed.nl',
    'pl': 'http://creativecommons.org/licenses/by-nc-sa/2.5/deed.pl',
    'pt-BR': 'http://creativecommons.org/licenses/by-nc-sa/2.5/deed.pt_BR',
    'sl': 'http://creativecommons.org/licenses/by-nc-sa/2.5/deed.sl',
    'sq': 'http://creativecommons.org/licenses/by-nc-sa/2.5/deed.sq',
    'zh-CN': 'http://creativecommons.org/licenses/by-nc-sa/2.5/deed.zh_CN',
    'zh-TW': 'http://creativecommons.org/licenses/by-nc-sa/2.5/deed.zh_TW',
})

VIDEO_IC = defaultdict(lambda: '59415650', **{
    'en-US': '59415650',
    'de': '59416761',
    'es': '59418049',
    'fr': '59419952',
    'pt-BR': '59421007',
})

VIDEO_SLAPSTICK = defaultdict(lambda: '59415651', **{
    'en-US': '59415651',
    'de': '59416762',
    'es': '59418050',
    'fr': '59419953',
    'pt-BR': '59421008',
})

VIDEO_ALLTHREE = defaultdict(lambda: '59415649', **{
    'en-US': '59415649',
    'de': '59416760',
    'es': '59418048',
    'fr': '59419955',
    'pt-BR': '59421006',
})
<commit_msg>added ja link for creative commens license<commit_after>from collections import defaultdict

LINK_PDWIKI = defaultdict(lambda: 'https://en.wikipedia.org/wiki/Public_domain', **{
    'en-US': 'https://en.wikipedia.org/wiki/Public_domain',
    'de': 'https://de.wikipedia.org/wiki/Gemeinfreiheit',
    'fr': 'https://fr.wikipedia.org/wiki/Domaine_public_%28propri%C3%A9t%C3%A9_intellectuelle%29',
    'es': 'https://es.wikipedia.org/wiki/Dominio_p%C3%BAblico',
    'nl': 'https://nl.wikipedia.org/wiki/Publiek_domein',
    'pl': 'https://pl.wikipedia.org/wiki/Domena_publiczna',
    'pt-BR': 'https://pt.wikipedia.org/wiki/Dom%C3%ADnio_p%C3%BAblico',
    'sl': 'https://sl.wikipedia.org/wiki/Javna_last',
    'sq': 'https://sq.wikipedia.org/wiki/Domen_publik',
    'zh-CN': 'https://zh.wikipedia.org/wiki/%E5%85%AC%E6%9C%89%E9%A2%86%E5%9F%9F',
    'zh-TW': 'https://zh.wikipedia.org/wiki/%E5%85%AC%E6%9C%89%E9%A2%86%E5%9F%9F',
    'it': 'https://it.wikipedia.org/wiki/Pubblico_dominio',
    'lij': 'https://it.wikipedia.org/wiki/Pubblico_dominio',
    'ja': 'https://ja.wikipedia.org/wiki/%E3%83%91%E3%83%96%E3%83%AA%E3%83%83%E3%82%AF%E3%83%89%E3%83%A1%E3%82%A4%E3%83%B3',
})

LINK_BRIEF = defaultdict(lambda: 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_en-US.pdf', **{
    'en-US': 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_en-US.pdf',
    'de': 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_de.pdf',
    'es': 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_es-ES.pdf',
    'nl': 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_nl.pdf',
    'pl': 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_pl.pdf',
    'sl': 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_sl.pdf',
    'fr': 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_fr.pdf',
    'zh-TW': 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_zh-TW.pdf',
    'pt-BR': 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_pt-BR.pdf',
    'it': 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_it.pdf',
    'lij': 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_lij.pdf',
    'ja': 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_ja.pdf',
    'sq': 'http://static.mozilla.com/firefoxflicks/pdf/Filmmakers_Creative_Brief_sq.pdf',
})

LINK_CCLICENSE = defaultdict(lambda: 'http://creativecommons.org/licenses/by-nc-sa/2.5/', **{
    'en-US': 'http://creativecommons.org/licenses/by-nc-sa/2.5/',
    'de': 'http://creativecommons.org/licenses/by-nc-sa/2.5/deed.de',
    'fr': 'http://creativecommons.org/licenses/by-nc-sa/2.5/deed.fr',
    'es': 'http://creativecommons.org/licenses/by-nc-sa/2.5/deed.es',
    'ja': 'http://creativecommons.org/licenses/by-nc-sa/2.5/deed.ja',
    'nl': 'http://creativecommons.org/licenses/by-nc-sa/2.5/deed.nl',
    'pl': 'http://creativecommons.org/licenses/by-nc-sa/2.5/deed.pl',
    'pt-BR': 'http://creativecommons.org/licenses/by-nc-sa/2.5/deed.pt_BR',
    'sl': 'http://creativecommons.org/licenses/by-nc-sa/2.5/deed.sl',
    'sq': 'http://creativecommons.org/licenses/by-nc-sa/2.5/deed.sq',
    'zh-CN': 'http://creativecommons.org/licenses/by-nc-sa/2.5/deed.zh_CN',
    'zh-TW': 'http://creativecommons.org/licenses/by-nc-sa/2.5/deed.zh_TW',
})

VIDEO_IC = defaultdict(lambda: '59415650', **{
    'en-US': '59415650',
    'de': '59416761',
    'es': '59418049',
    'fr': '59419952',
    'pt-BR': '59421007',
})

VIDEO_SLAPSTICK = defaultdict(lambda: '59415651', **{
    'en-US': '59415651',
    'de': '59416762',
    'es': '59418050',
    'fr': '59419953',
    'pt-BR': '59421008',
})

VIDEO_ALLTHREE = defaultdict(lambda: '59415649', **{
    'en-US': '59415649',
    'de': '59416760',
    'es': '59418048',
    'fr': '59419955',
    'pt-BR': '59421006',
})
<|endoftext|>


<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Utility functions for data (frame) transformations."""
from analysis import stat_tests
from common import environment

class EmptyDataError(ValueError):
    """An exception for when the data is empty."""

def validate_data(experiment_df):
    """Checks if the experiment data is valid."""
    if experiment_df.empty:
        raise EmptyDataError('Empty experiment data.')

    expected_columns = {
        'experiment', 'benchmark', 'fuzzer', 'trial_id', 'time_started',
        'time_ended', 'time', 'edges_covered'
    }
    missing_columns = expected_columns.difference(experiment_df.columns)
    if missing_columns:
        raise ValueError(
            'Missing columns in experiment data: {}'.format(missing_columns))

def drop_uninteresting_columns(experiment_df):
    """Returns table with only interesting columns."""
    return experiment_df[[
        'benchmark', 'fuzzer', 'trial_id', 'time', 'edges_covered'
    ]]

def clobber_experiments_data(df, experiments):
    """Clobber experiment data that is part of lower priority (generally
    earlier) versions of the same trials in |df|. For example in experiment-1 we
    may test fuzzer-a on benchmark-1. In experiment-2 we may again test fuzzer-a
    on benchmark-1 because fuzzer-a was updated. This function will remove the
    snapshots from fuzzer-a,benchmark-1,experiment-1 from |df| because we want
    the report to only contain the up-to-date data. Experiment priority is
    determined by order of each experiment in |experiments| with the highest
    priority experiment coming last in that list."""
    # We don't call |df| "experiment_df" because it is a misnomer and leads to
    # confusion in this case where it contains data from multiple experiments.

    # Include everything from the last experiment.
    experiments = experiments.copy()  # Copy so we dont mutate experiments.
    experiments.reverse()
    highest_rank_experiment = experiments[0]
    result = df[df.experiment == highest_rank_experiment]

    for experiment in experiments[1:]:
        # Include data for not yet covered benchmark/fuzzer pairs.
        covered_pairs = result[['benchmark', 'fuzzer']].drop_duplicates()
        covered_pairs = covered_pairs.apply(tuple, axis=1)
        experiment_data = df[df.experiment == experiment]
        experiment_pairs = experiment_data[['benchmark',
                                            'fuzzer']].apply(tuple, axis=1)
        to_include = experiment_data[~experiment_pairs.isin(covered_pairs)]
        result = result.append(to_include)
    return result

def filter_fuzzers(experiment_df, included_fuzzers):
    """Returns table with only rows where fuzzer is in |included_fuzzers|."""
    return experiment_df[experiment_df['fuzzer'].isin(included_fuzzers)]

def filter_benchmarks(experiment_df, included_benchmarks):
    """Returns table with only rows where benchmark is in
    |included_benchmarks|."""
    return experiment_df[experiment_df['benchmark'].isin(included_benchmarks)]

def label_fuzzers_by_experiment(experiment_df):
    """Returns table where every fuzzer is labeled by the experiment it
    was run in."""
    experiment_df['fuzzer'] = (experiment_df['fuzzer'] + '-' +
                               experiment_df['experiment'])

    return experiment_df

def filter_max_time(experiment_df, max_time):
    """Returns table with snapshots that have time less than or equal to
    |max_time|."""
    return experiment_df[experiment_df['time'] <= max_time]

# Creating "snapshots" (see README.md for definition).

_DEFAULT_BENCHMARK_SAMPLE_NUM_THRESHOLD = 0.8

def get_benchmark_snapshot(benchmark_df,
                           threshold=_DEFAULT_BENCHMARK_SAMPLE_NUM_THRESHOLD):
    """Finds the latest time where 80% of the trials were still running. In most
    cases, this is the end of the experiment. In this case, we won't consider
    the <20% of the trials that ended early for our analysis. If more than 20%
    of the trials ended early, it's better to pick an earlier snapshot time.
    The 80% can be overridden using the |threshold| argument. E.g., to find the
    latest time where each trials were running, set |threshold| to 1.0.

    Returns data frame that only contains the measurements of the picked
    snapshot time.
    """
    # Allow overriding threshold with environment variable as well.
    threshold = environment.get('BENCHMARK_SAMPLE_NUM_THRESHOLD', threshold)

    num_trials = benchmark_df.trial_id.nunique()
    trials_running_at_time = benchmark_df.time.value_counts()
    criteria = trials_running_at_time > threshold * num_trials
    ok_times = trials_running_at_time[criteria]
    latest_ok_time = ok_times.index.max()
    benchmark_snapshot_df = benchmark_df[benchmark_df.time == latest_ok_time]
    return benchmark_snapshot_df

_DEFAULT_FUZZER_SAMPLE_NUM_THRESHOLD = 0.8

def get_fuzzers_with_not_enough_samples(
        benchmark_snapshot_df, threshold=_DEFAULT_FUZZER_SAMPLE_NUM_THRESHOLD):
    """Returns fuzzers that didn't have enough trials running at snapshot time.
    It takes a benchmark snapshot and finds the fuzzers that have a sample size
    smaller than 80% of the largest sample size. Default threshold can be
    overridden.
    """
    # Allow overriding threshold with environment variable as well.
    threshold = environment.get('FUZZER_SAMPLE_NUM_THRESHOLD', threshold)

    samples_per_fuzzer = benchmark_snapshot_df.fuzzer.value_counts()
    max_samples = samples_per_fuzzer.max()
    few_sample_criteria = samples_per_fuzzer < threshold * max_samples
    few_sample_fuzzers = samples_per_fuzzer[few_sample_criteria].index
    return few_sample_fuzzers.tolist()

def get_experiment_snapshots(experiment_df):
    """Finds a good snapshot time for each benchmark in the experiment data.

    Returns the data frame that only contains the measurements made at these
    snapshot times.
    """
    benchmark_groups = experiment_df.groupby('benchmark')
    experiment_snapshots = benchmark_groups.apply(get_benchmark_snapshot)
    # We don't need the extra index added by the groupby('benchmark').
    experiment_snapshots.reset_index(drop=True, inplace=True)
    return experiment_snapshots

# Summary tables containing statistics on the samples.

def benchmark_summary(benchmark_snapshot_df):
    """Creates summary table for a benchmark snapshot with columns:
    |fuzzer|time||count|mean|std|min|25%|median|75%|max|
    """
    groups = benchmark_snapshot_df.groupby(['fuzzer', 'time'])
    summary = groups['edges_covered'].describe()
    summary.rename(columns={'50%': 'median'}, inplace=True)
    return summary.sort_values(('median'), ascending=False)

def experiment_summary(experiment_snapshots_df):
    """Creates summary table for all benchmarks in experiment, i.e. table like:
    |benchmark|| < benchmark level summary >
    """
    groups = experiment_snapshots_df.groupby('benchmark')
    summaries = groups.apply(benchmark_summary)
    return summaries

# Per-benchmark fuzzer ranking options.

def benchmark_rank_by_mean(benchmark_snapshot_df):
    """Returns ranking of fuzzers based on mean coverage."""
    assert benchmark_snapshot_df.time.nunique() == 1, 'Not a snapshot!'
    means = benchmark_snapshot_df.groupby('fuzzer')['edges_covered'].mean()
    means.rename('mean cov', inplace=True)
    return means.sort_values(ascending=False)

def benchmark_rank_by_median(benchmark_snapshot_df):
    """Returns ranking of fuzzers based on median coverage."""
    assert benchmark_snapshot_df.time.nunique() == 1, 'Not a snapshot!'
    medians = benchmark_snapshot_df.groupby('fuzzer')['edges_covered'].median()
    medians.rename('median cov', inplace=True)
    return medians.sort_values(ascending=False)

def benchmark_rank_by_average_rank(benchmark_snapshot_df):
    """Ranks all coverage measurements in the snapshot across fuzzers.

    Returns the average rank by fuzzer.
    """
    # Make a copy of the dataframe view, because we want to add a new column.
    measurements = benchmark_snapshot_df[['fuzzer', 'edges_covered']].copy()
    measurements['rank'] = measurements['edges_covered'].rank()
    avg_rank = measurements.groupby('fuzzer').mean()
    avg_rank.rename(columns={'rank': 'avg rank'}, inplace=True)
    avg_rank.sort_values('avg rank', ascending=False, inplace=True)
    return avg_rank['avg rank']

def benchmark_rank_by_stat_test_wins(benchmark_snapshot_df):
    """Carries out one-tailed statistical tests for each fuzzer pair.

    Returns ranking according to the number of statistical test wins.
    """
    p_values = stat_tests.one_sided_u_test(benchmark_snapshot_df)

    # Turn "significant" p-values into 1-s.
    better_than = p_values.applymap(
        lambda p: p < stat_tests.SIGNIFICANCE_THRESHOLD)
    better_than = better_than.applymap(int)

    score = better_than.sum(axis=1).sort_values(ascending=False)
    score.rename('stat wins', inplace=True)

    return score

def create_better_than_table(benchmark_snapshot_df):
    """Creates table showing whether fuzzer in row is statistically
    significantly better than the fuzzer in the column."""
    p_values = stat_tests.one_sided_u_test(benchmark_snapshot_df)

    # Turn "significant" p-values into 1-s.
    better_than = p_values.applymap(
        lambda p: p < stat_tests.SIGNIFICANCE_THRESHOLD)
    better_than = better_than.applymap(int)

    # Order rows and columns of matrix according to score ranking.
    score = better_than.sum(axis=1).sort_values(ascending=False)
    better_than = better_than.reindex(index=score.index,
                                      columns=score.index[::-1])
    return better_than

# Experiment level ranking of fuzzers (across-benchmarks).
# Experiment level ranking depends on the per-benchmark ranking method.

def experiment_pivot_table(experiment_snapshots_df,
                           benchmark_level_ranking_function):
    """Creates a pivot table according to a given per benchmark ranking, where
    the columns are the fuzzers, the rows are the benchmarks, and the values
    are the scores according to the per benchmark ranking."""
    benchmark_blocks = experiment_snapshots_df.groupby('benchmark')
    groups_ranked = benchmark_blocks.apply(benchmark_level_ranking_function)
    already_unstacked = groups_ranked.index.names == ['benchmark']
    pivot_df = groups_ranked if already_unstacked else groups_ranked.unstack()
    return pivot_df

def experiment_rank_by_average_rank(experiment_pivot_df):
    """Creates experiment level ranking of fuzzers.

    Takes a pivot table representing per benchmark ranking scores. Ranks
    fuzzers per benchmark, then takes the average rank across benchmarks
    (smaller is better).
    """
    # Rank fuzzers in each benchmark block.
    pivot_ranked = experiment_pivot_df.rank('columns',
                                            na_option='keep',
                                            ascending=False)
    average_ranks = pivot_ranked.mean().sort_values()
    return average_ranks.rename('average rank')

def experiment_rank_by_num_firsts(experiment_pivot_df):
    """Creates experiment level ranking by number of first places in per
    benchmark rankings (higher is better)."""
    # Rank fuzzers in each benchmark block.
    pivot_ranked = experiment_pivot_df.rank('columns',
                                            na_option='keep',
                                            ascending=False)
    # Count first places for each fuzzer.
    firsts = pivot_ranked[pivot_ranked == 1]
    num_firsts = firsts.sum().sort_values(ascending=False)
    return num_firsts.rename('number of wins')

def experiment_rank_by_average_normalized_score(experiment_pivot_df):
    """Creates experiment level ranking by taking the average of normalized per
    benchmark scores from 0 to 100, where 100 is the highest reach coverage."""
    # Normalize coverage values.
    benchmark_maximum = experiment_pivot_df.max(axis='columns')
    normalized_score = experiment_pivot_df.div(benchmark_maximum,
                                               axis='index').mul(100)

    average_score = normalized_score.mean().sort_values(ascending=False)
    return average_score.rename('average normalized score')

def experiment_level_ranking(experiment_snapshots_df,
                             benchmark_level_ranking_function,
                             experiment_level_ranking_function):
    """Returns an aggregate ranking of fuzzers across all benchmarks."""
    pivot_table = experiment_pivot_table(experiment_snapshots_df,
                                         benchmark_level_ranking_function)
    return experiment_level_ranking_function(pivot_table)
<commit_msg>Don't shorten reported experiment time when data is incomplete. (#483)<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Utility functions for data (frame) transformations."""
from analysis import stat_tests
from common import environment

class EmptyDataError(ValueError):
    """An exception for when the data is empty."""

def validate_data(experiment_df):
    """Checks if the experiment data is valid."""
    if experiment_df.empty:
        raise EmptyDataError('Empty experiment data.')

    expected_columns = {
        'experiment', 'benchmark', 'fuzzer', 'trial_id', 'time_started',
        'time_ended', 'time', 'edges_covered'
    }
    missing_columns = expected_columns.difference(experiment_df.columns)
    if missing_columns:
        raise ValueError(
            'Missing columns in experiment data: {}'.format(missing_columns))

def drop_uninteresting_columns(experiment_df):
    """Returns table with only interesting columns."""
    return experiment_df[[
        'benchmark', 'fuzzer', 'trial_id', 'time', 'edges_covered'
    ]]

def clobber_experiments_data(df, experiments):
    """Clobber experiment data that is part of lower priority (generally
    earlier) versions of the same trials in |df|. For example in experiment-1 we
    may test fuzzer-a on benchmark-1. In experiment-2 we may again test fuzzer-a
    on benchmark-1 because fuzzer-a was updated. This function will remove the
    snapshots from fuzzer-a,benchmark-1,experiment-1 from |df| because we want
    the report to only contain the up-to-date data. Experiment priority is
    determined by order of each experiment in |experiments| with the highest
    priority experiment coming last in that list."""
    # We don't call |df| "experiment_df" because it is a misnomer and leads to
    # confusion in this case where it contains data from multiple experiments.

    # Include everything from the last experiment.
    experiments = experiments.copy()  # Copy so we dont mutate experiments.
    experiments.reverse()
    highest_rank_experiment = experiments[0]
    result = df[df.experiment == highest_rank_experiment]

    for experiment in experiments[1:]:
        # Include data for not yet covered benchmark/fuzzer pairs.
        covered_pairs = result[['benchmark', 'fuzzer']].drop_duplicates()
        covered_pairs = covered_pairs.apply(tuple, axis=1)
        experiment_data = df[df.experiment == experiment]
        experiment_pairs = experiment_data[['benchmark',
                                            'fuzzer']].apply(tuple, axis=1)
        to_include = experiment_data[~experiment_pairs.isin(covered_pairs)]
        result = result.append(to_include)
    return result

def filter_fuzzers(experiment_df, included_fuzzers):
    """Returns table with only rows where fuzzer is in |included_fuzzers|."""
    return experiment_df[experiment_df['fuzzer'].isin(included_fuzzers)]

def filter_benchmarks(experiment_df, included_benchmarks):
    """Returns table with only rows where benchmark is in
    |included_benchmarks|."""
    return experiment_df[experiment_df['benchmark'].isin(included_benchmarks)]

def label_fuzzers_by_experiment(experiment_df):
    """Returns table where every fuzzer is labeled by the experiment it
    was run in."""
    experiment_df['fuzzer'] = (experiment_df['fuzzer'] + '-' +
                               experiment_df['experiment'])

    return experiment_df

def filter_max_time(experiment_df, max_time):
    """Returns table with snapshots that have time less than or equal to
    |max_time|."""
    return experiment_df[experiment_df['time'] <= max_time]

# Creating "snapshots" (see README.md for definition).

_DEFAULT_BENCHMARK_SAMPLE_NUM_THRESHOLD = 1.0

def get_benchmark_snapshot(benchmark_df,
                           threshold=_DEFAULT_BENCHMARK_SAMPLE_NUM_THRESHOLD):
    """Finds the latest time where all trials were still running
    (or |threshold| * trials were still running). In most cases, this is the end
    of the experiment. However, if some trials (or more than (1 - |threshold|) *
    trials) ended earlier, we will use an earlier "snapshot" time for comparing
    results.

    Returns data frame that only contains the measurements of the picked
    snapshot time.
    """
    # Allow overriding threshold with environment variable as well.
    threshold = environment.get('BENCHMARK_SAMPLE_NUM_THRESHOLD', threshold)

    num_trials = benchmark_df.trial_id.nunique()
    trials_running_at_time = benchmark_df.time.value_counts()
    criteria = trials_running_at_time >= threshold * num_trials
    ok_times = trials_running_at_time[criteria]
    latest_ok_time = ok_times.index.max()
    benchmark_snapshot_df = benchmark_df[benchmark_df.time == latest_ok_time]
    return benchmark_snapshot_df

_DEFAULT_FUZZER_SAMPLE_NUM_THRESHOLD = 0.8

def get_fuzzers_with_not_enough_samples(
        benchmark_snapshot_df, threshold=_DEFAULT_FUZZER_SAMPLE_NUM_THRESHOLD):
    """Returns fuzzers that didn't have enough trials running at snapshot time.
    It takes a benchmark snapshot and finds the fuzzers that have a sample size
    smaller than 80% of the largest sample size. Default threshold can be
    overridden.
    """
    # Allow overriding threshold with environment variable as well.
    threshold = environment.get('FUZZER_SAMPLE_NUM_THRESHOLD', threshold)

    samples_per_fuzzer = benchmark_snapshot_df.fuzzer.value_counts()
    max_samples = samples_per_fuzzer.max()
    few_sample_criteria = samples_per_fuzzer < threshold * max_samples
    few_sample_fuzzers = samples_per_fuzzer[few_sample_criteria].index
    return few_sample_fuzzers.tolist()

def get_experiment_snapshots(experiment_df):
    """Finds a good snapshot time for each benchmark in the experiment data.

    Returns the data frame that only contains the measurements made at these
    snapshot times.
    """
    benchmark_groups = experiment_df.groupby('benchmark')
    experiment_snapshots = benchmark_groups.apply(get_benchmark_snapshot)
    # We don't need the extra index added by the groupby('benchmark').
    experiment_snapshots.reset_index(drop=True, inplace=True)
    return experiment_snapshots

# Summary tables containing statistics on the samples.

def benchmark_summary(benchmark_snapshot_df):
    """Creates summary table for a benchmark snapshot with columns:
    |fuzzer|time||count|mean|std|min|25%|median|75%|max|
    """
    groups = benchmark_snapshot_df.groupby(['fuzzer', 'time'])
    summary = groups['edges_covered'].describe()
    summary.rename(columns={'50%': 'median'}, inplace=True)
    return summary.sort_values(('median'), ascending=False)

def experiment_summary(experiment_snapshots_df):
    """Creates summary table for all benchmarks in experiment, i.e. table like:
    |benchmark|| < benchmark level summary >
    """
    groups = experiment_snapshots_df.groupby('benchmark')
    summaries = groups.apply(benchmark_summary)
    return summaries

# Per-benchmark fuzzer ranking options.

def benchmark_rank_by_mean(benchmark_snapshot_df):
    """Returns ranking of fuzzers based on mean coverage."""
    assert benchmark_snapshot_df.time.nunique() == 1, 'Not a snapshot!'
    means = benchmark_snapshot_df.groupby('fuzzer')['edges_covered'].mean()
    means.rename('mean cov', inplace=True)
    return means.sort_values(ascending=False)

def benchmark_rank_by_median(benchmark_snapshot_df):
    """Returns ranking of fuzzers based on median coverage."""
    assert benchmark_snapshot_df.time.nunique() == 1, 'Not a snapshot!'
    medians = benchmark_snapshot_df.groupby('fuzzer')['edges_covered'].median()
    medians.rename('median cov', inplace=True)
    return medians.sort_values(ascending=False)

def benchmark_rank_by_average_rank(benchmark_snapshot_df):
    """Ranks all coverage measurements in the snapshot across fuzzers.

    Returns the average rank by fuzzer.
    """
    # Make a copy of the dataframe view, because we want to add a new column.
    measurements = benchmark_snapshot_df[['fuzzer', 'edges_covered']].copy()
    measurements['rank'] = measurements['edges_covered'].rank()
    avg_rank = measurements.groupby('fuzzer').mean()
    avg_rank.rename(columns={'rank': 'avg rank'}, inplace=True)
    avg_rank.sort_values('avg rank', ascending=False, inplace=True)
    return avg_rank['avg rank']

def benchmark_rank_by_stat_test_wins(benchmark_snapshot_df):
    """Carries out one-tailed statistical tests for each fuzzer pair.

    Returns ranking according to the number of statistical test wins.
    """
    p_values = stat_tests.one_sided_u_test(benchmark_snapshot_df)

    # Turn "significant" p-values into 1-s.
    better_than = p_values.applymap(
        lambda p: p < stat_tests.SIGNIFICANCE_THRESHOLD)
    better_than = better_than.applymap(int)

    score = better_than.sum(axis=1).sort_values(ascending=False)
    score.rename('stat wins', inplace=True)

    return score

def create_better_than_table(benchmark_snapshot_df):
    """Creates table showing whether fuzzer in row is statistically
    significantly better than the fuzzer in the column."""
    p_values = stat_tests.one_sided_u_test(benchmark_snapshot_df)

    # Turn "significant" p-values into 1-s.
    better_than = p_values.applymap(
        lambda p: p < stat_tests.SIGNIFICANCE_THRESHOLD)
    better_than = better_than.applymap(int)

    # Order rows and columns of matrix according to score ranking.
    score = better_than.sum(axis=1).sort_values(ascending=False)
    better_than = better_than.reindex(index=score.index,
                                      columns=score.index[::-1])
    return better_than

# Experiment level ranking of fuzzers (across-benchmarks).
# Experiment level ranking depends on the per-benchmark ranking method.

def experiment_pivot_table(experiment_snapshots_df,
                           benchmark_level_ranking_function):
    """Creates a pivot table according to a given per benchmark ranking, where
    the columns are the fuzzers, the rows are the benchmarks, and the values
    are the scores according to the per benchmark ranking."""
    benchmark_blocks = experiment_snapshots_df.groupby('benchmark')
    groups_ranked = benchmark_blocks.apply(benchmark_level_ranking_function)
    already_unstacked = groups_ranked.index.names == ['benchmark']
    pivot_df = groups_ranked if already_unstacked else groups_ranked.unstack()
    return pivot_df

def experiment_rank_by_average_rank(experiment_pivot_df):
    """Creates experiment level ranking of fuzzers.

    Takes a pivot table representing per benchmark ranking scores. Ranks
    fuzzers per benchmark, then takes the average rank across benchmarks
    (smaller is better).
    """
    # Rank fuzzers in each benchmark block.
    pivot_ranked = experiment_pivot_df.rank('columns',
                                            na_option='keep',
                                            ascending=False)
    average_ranks = pivot_ranked.mean().sort_values()
    return average_ranks.rename('average rank')

def experiment_rank_by_num_firsts(experiment_pivot_df):
    """Creates experiment level ranking by number of first places in per
    benchmark rankings (higher is better)."""
    # Rank fuzzers in each benchmark block.
    pivot_ranked = experiment_pivot_df.rank('columns',
                                            na_option='keep',
                                            ascending=False)
    # Count first places for each fuzzer.
    firsts = pivot_ranked[pivot_ranked == 1]
    num_firsts = firsts.sum().sort_values(ascending=False)
    return num_firsts.rename('number of wins')

def experiment_rank_by_average_normalized_score(experiment_pivot_df):
    """Creates experiment level ranking by taking the average of normalized per
    benchmark scores from 0 to 100, where 100 is the highest reach coverage."""
    # Normalize coverage values.
    benchmark_maximum = experiment_pivot_df.max(axis='columns')
    normalized_score = experiment_pivot_df.div(benchmark_maximum,
                                               axis='index').mul(100)

    average_score = normalized_score.mean().sort_values(ascending=False)
    return average_score.rename('average normalized score')

def experiment_level_ranking(experiment_snapshots_df,
                             benchmark_level_ranking_function,
                             experiment_level_ranking_function):
    """Returns an aggregate ranking of fuzzers across all benchmarks."""
    pivot_table = experiment_pivot_table(experiment_snapshots_df,
                                         benchmark_level_ranking_function)
    return experiment_level_ranking_function(pivot_table)
<|endoftext|>


<commit_before># -*- Mode: Python -*-
# vi:si:et:sw=4:sts=4:ts=4
#
# bin/flumotion-worker: Flumotion worker
#
# Flumotion - a streaming media server
# Copyright (C) 2004 Fluendo, S.L. (www.fluendo.com). All rights reserved.

# This file may be distributed and/or modified under the terms of
# the GNU General Public License version 2 as published by
# the Free Software Foundation.
# This file is distributed without any warranty; without even the implied
# warranty of merchantability or fitness for a particular purpose.
# See "LICENSE.GPL" in the source distribution for more information.

# Licensees having purchased or holding a valid Flumotion Advanced
# Streaming Server license may use this file in accordance with the
# Flumotion Advanced Streaming Server Commercial License Agreement.
# See "LICENSE.Flumotion" in the source distribution for more information.

# Headers in this file shall remain intact.

import flumotion.common.registry
import flumotion.common.log

def setup(root=None):
    flumotion.common.log.init()
    if root:
        flumotion.common.registry.registry.verify(root)

<commit_msg>pychecker: use from x import y instead of import x.y<commit_after># -*- Mode: Python -*-
# vi:si:et:sw=4:sts=4:ts=4
#
# bin/flumotion-worker: Flumotion worker
#
# Flumotion - a streaming media server
# Copyright (C) 2004 Fluendo, S.L. (www.fluendo.com). All rights reserved.

# This file may be distributed and/or modified under the terms of
# the GNU General Public License version 2 as published by
# the Free Software Foundation.
# This file is distributed without any warranty; without even the implied
# warranty of merchantability or fitness for a particular purpose.
# See "LICENSE.GPL" in the source distribution for more information.

# Licensees having purchased or holding a valid Flumotion Advanced
# Streaming Server license may use this file in accordance with the
# Flumotion Advanced Streaming Server Commercial License Agreement.
# See "LICENSE.Flumotion" in the source distribution for more information.

# Headers in this file shall remain intact.

from flumotion.common import log, registry

def setup(root=None):
    log.init()
    if root:
        registry.registry.verify(root)

<|endoftext|>


<commit_before>from __future__ import unicode_literals
from django.core.urlresolvers import reverse
from django.db import models
from django.core.urlresolvers import reverse_lazy
from django.conf import settings
import os.path as op
import os
import uuid

def get_creds_file_path(instance, filename):
	return os.path.join("/".join(["creds", filename]))

def get_data_file_path(instance, filename):
	return str(filename)

class Submission(models.Model):
	output_url = models.CharField(max_length=200, null=True, blank=True)
	STATE_CHOICES = (
		('participant', 'Participant analysis'),
		('group', 'Group analysis')
	)
	state = models.CharField(max_length=20, blank=False, default="Participant analysis")
	bucket = models.CharField(max_length=100, blank=False, default="placeholderbucket")
	bidsdir = models.CharField(max_length=100, blank=False, default="DC1-demo")
	jobdir = models.CharField(max_length=100, blank=False)
	creds_file = models.FileField(upload_to=get_creds_file_path, null=True, blank=False)
	data_file = models.FileField(upload_to=get_data_file_path, null=True, blank=False)
	datasetname = models.CharField(max_length=100, blank=True)
	MOD_CHOICES = (
		('func', 'Functional'),
		("dwi", 'DWI')
	)
	modality = models.CharField(max_length=20, blank=False, default="Functional")
	STC_CHOICES = (
		('None', 'None'),
		('up', 'Bottom Up Acquisition (standard)'),
		('down', 'Top Down Acquisition'),
		("interleaved", 'Interleaved Acquisition')
	)
	slice_timing = models.CharField(max_length=20, blank=False, default="Interleaved acquisition")
	UPLOAD_CHOICES = (
		('yes', 'Yes'),
		('no', 'No')
	)
	upload_data_or_not = models.CharField(max_length=20, blank=False, default="Yes")

	def add_output_url(self, url):
		output_url = models.TextField(url)

	def __str__(self):
		return str(self.bucket) + "/" + str(self.bidsdir)

	def __eq__(self, other):
		if isinstance(other, self.__class__):
			return (self.bucket == other.bucket) and (self.bidsdir == other.bidsdir)
		return False

	def __ne__(self, other):
		return not self.__eq__(other)
<commit_msg>Update models.py<commit_after>from __future__ import unicode_literals
from django.core.urlresolvers import reverse
from django.db import models
from django.core.urlresolvers import reverse_lazy
from django.conf import settings
import os.path as op
import os
import uuid

def get_creds_file_path(instance, filename):
	return os.path.join("/".join(["creds", filename]))

def get_data_file_path(instance, filename):
	return str(filename)

class Submission(models.Model):
	output_url = models.CharField(max_length=200, null=True, blank=True)
	STATE_CHOICES = (
		('participant', 'Participant analysis'),
		('group', 'Group analysis')
	)
	state = models.CharField(max_length=20, blank=False, default="Participant analysis")
	bucket = models.CharField(max_length=100, blank=False, default="fngstestbucket")
	bidsdir = models.CharField(max_length=100, blank=False, default="DC1-demo")
	jobdir = models.CharField(max_length=100, blank=False)
	creds_file = models.FileField(upload_to=get_creds_file_path, null=True, blank=False)
	data_file = models.FileField(upload_to=get_data_file_path, null=True, blank=False)
	datasetname = models.CharField(max_length=100, blank=True)
	MOD_CHOICES = (
		('func', 'Functional'),
		("dwi", 'DWI')
	)
	modality = models.CharField(max_length=20, blank=False, default="Functional")
	STC_CHOICES = (
		('None', 'None'),
		('up', 'Bottom Up Acquisition (standard)'),
		('down', 'Top Down Acquisition'),
		("interleaved", 'Interleaved Acquisition')
	)
	slice_timing = models.CharField(max_length=20, blank=False, default="Interleaved acquisition")
	UPLOAD_CHOICES = (
		('yes', 'Yes'),
		('no', 'No')
	)
	upload_data_or_not = models.CharField(max_length=20, blank=False, default="Yes")

	def add_output_url(self, url):
		output_url = models.TextField(url)

	def __str__(self):
		return str(self.bucket) + "/" + str(self.bidsdir)

	def __eq__(self, other):
		if isinstance(other, self.__class__):
			return (self.bucket == other.bucket) and (self.bidsdir == other.bidsdir)
		return False

	def __ne__(self, other):
		return not self.__eq__(other)
<|endoftext|>


<commit_before>import urlparse

from formspree.app import DB, redis_store
from formspree import settings, log
from formspree.utils import unix_time_for_12_months_from_now, next_url
from flask import url_for, render_template
from helpers import HASH, MONTHLY_COUNTER_KEY, http_form_to_dict, referrer_to_path, send_email
from formspree.utils import int2bigstring, bigstring2int
import datetime

CODE_TEMPLATE = '''
<form action="{action}" method="POST">
    <input type="text" name="_gotcha" style="display:none" />
    <input type="email" name="email" placeholder="Your email">
    <textarea name="message" rows="5" placeholder="Your message"></textarea>
    <input type="submit" value="Send">
</form>
'''

class Form(DB.Model):
    __tablename__ = 'forms'

    id = DB.Column(DB.Integer, primary_key=True)
    hash = DB.Column(DB.String(32), unique=True)
    email = DB.Column(DB.String(120))
    host = DB.Column(DB.String(300))
    confirm_sent = DB.Column(DB.Boolean)
    confirmed = DB.Column(DB.Boolean)
    counter = DB.Column(DB.Integer)
    owner_id = DB.Column(DB.Integer, DB.ForeignKey('users.id'))

    '''
    When the form is created by a spontaneous submission, it is added to
    the table with a `host`, an `email` and a `hash` made of these two
    (+ a secret nonce).

    `hash` is UNIQUE because it is used to query these spontaneous forms
    when the form is going to be confirmed and whenever a new submission arrives.

    When a registered user POSTs to /forms, a new form is added to the table
    with an `email` (provided by the user) and an `owner_id`. Later, when this
    form receives its first submission and confirmation, `host` is added, so
    we can ensure that no one will submit to this same form from another host.

    `hash` is never added to these forms, because they could conflict with other
    forms, created by the spontaneous process, with the same email and host. So
    for these forms a different confirmation method is used (see below).
    '''

    STATUS_EMAIL_SENT              = 0
    STATUS_EMAIL_EMPTY             = 1
    STATUS_EMAIL_FAILED            = 2

    STATUS_CONFIRMATION_SENT       = 10
    STATUS_CONFIRMATION_DUPLICATED = 11
    STATUS_CONFIRMATION_FAILED     = 12

    def __init__(self, email, host=None, owner=None):
        if host:
            self.hash = HASH(email, host)
        elif owner:
            self.owner_id = owner.id
        else:
            raise Exception('cannot create form without a host and a owner. provide one of these.')
        self.email = email
        self.host = host
        self.confirm_sent = False
        self.confirmed = False
        self.counter = 0

    def __repr__(self):
        return '<Form %s, email=%s, host=%s>' % (self.id, self.email, self.host)

    def get_random_like_string(self):
        if not self.id:
            raise Exception("this form doesn't have an id yet, commit it first.")
        return int2bigstring(self.id)

    @classmethod
    def get_form_by_random_like_string(cls, random_like_string):
        id = bigstring2int(random_like_string)
        return cls.query.get(id)

    def send(self, http_form, referrer):
        '''
        Sends form to user's email.
        Assumes sender's email has been verified.
        '''

        data, keys = http_form_to_dict(http_form)

        subject = data.get('_subject', 'New submission from %s' % referrer_to_path(referrer))
        reply_to = data.get('_replyto', data.get('email', data.get('Email', None)))
        cc = data.get('_cc', None)
        next = next_url(referrer, data.get('_next'))
        spam = data.get('_gotcha', None)

        # prevent submitting empty form
        if not any(data.values()):
            return { 'code': Form.STATUS_EMAIL_EMPTY }

        # return a fake success for spam
        if spam:
            return { 'code': Form.STATUS_EMAIL_SENT, 'next': next }

        # increment the forms counter
        self.counter = Form.counter + 1
        DB.session.add(self)
        DB.session.commit()

        # increase the monthly counter
        request_date = datetime.datetime.now()
        self.increase_monthly_counter(basedate=request_date)

        # check if the forms are over the counter and the user is not upgraded
        overlimit = False
        if self.get_monthly_counter(basedate=request_date) > settings.MONTHLY_SUBMISSIONS_LIMIT:
            if not self.owner or not self.owner.upgraded:
                overlimit = True

        now = datetime.datetime.utcnow().strftime('%I:%M %p UTC - %d %B %Y')
        if not overlimit:
            text = render_template('email/form.txt', data=data, host=self.host, keys=keys, now=now)
            html = render_template('email/form.html', data=data, host=self.host, keys=keys, now=now)
        else:
            text = render_template('email/overlimit-notification.txt', host=self.host)
            html = render_template('email/overlimit-notification.html', host=self.host)

        result = send_email(to=self.email,
                          subject=subject,
                          text=text,
                          html=html,
                          sender=settings.DEFAULT_SENDER,
                          reply_to=reply_to,
                          cc=cc)

        if not result[0]:
            return{ 'code': Form.STATUS_EMAIL_FAILED }

        return { 'code': Form.STATUS_EMAIL_SENT, 'next': next }

    def get_monthly_counter(self, basedate=None):
        basedate = basedate or datetime.datetime.now()
        month = basedate.month
        key = MONTHLY_COUNTER_KEY(form_id=self.id, month=month)
        counter = redis_store.get(key) or 0
        return int(counter)

    def increase_monthly_counter(self, basedate=None):
        basedate = basedate or datetime.datetime.now()
        month = basedate.month
        key = MONTHLY_COUNTER_KEY(form_id=self.id, month=month)
        redis_store.incr(key)
        redis_store.expireat(key, unix_time_for_12_months_from_now(basedate))

    def send_confirmation(self):
        '''
        Helper that actually creates confirmation nonce
        and sends the email to associated email. Renders
        different templates depending on the result
        '''

        log.debug('Sending confirmation')
        if self.confirm_sent:
            return { 'code': Form.STATUS_CONFIRMATION_DUPLICATED }

        # the nonce for email confirmation will be the hash when it exists
        # (whenever the form was created from a simple submission) or
        # a concatenation of HASH(email, id) + ':' + id (whenever the form
        # was created from the dashboard)
        id = str(self.id)
        nonce = self.hash or '%s:%s' % (HASH(self.email, id), id)
        link = url_for('confirm_email', nonce=nonce, _external=True)

        def render_content(type):
            return render_template('email/confirm.%s' % type,
                                      email=self.email,
                                      host=self.host,
                                      nonce_link=link)

        log.debug('Sending email')

        result = send_email(to=self.email,
                            subject='Confirm email for %s' % settings.SERVICE_NAME,
                            text=render_content('txt'),
                            html=render_content('html'),
                            sender=settings.DEFAULT_SENDER)

        log.debug('Sent')

        if not result[0]:
            return { 'code': Form.STATUS_CONFIRMATION_FAILED }

        self.confirm_sent = True
        DB.session.add(self)
        DB.session.commit()

        return { 'code': Form.STATUS_CONFIRMATION_SENT }

    @classmethod
    def confirm(cls, nonce):
        if ':' in nonce:
            # form created in the dashboard
            # nonce is another hash and the id
            # comes in the request
            nonce, id = nonce.split(':')
            form = cls.query.get(id)
            if HASH(form.email, str(form.id)) == nonce:
                pass
            else:
                form = None
        else:
            # normal form, nonce is HASH(email, host)
            form = cls.query.filter_by(hash=nonce).first()

        if form:
            form.confirmed = True
            DB.session.add(form)
            DB.session.commit()
            return form

    @property
    def status(self):
        if self.confirmed:
            return 'confirmed'

        if self.confirm_sent:
            return 'awaiting confirmation'

        return 'unknown'

    @property
    def action(self):
        return url_for('send', email_or_string=self.get_random_like_string(), _external=True)

    @property
    def code(self):
        return CODE_TEMPLATE.format(action=self.action)

    @property
    def is_new(self):
        return not self.host
<commit_msg>Remove status property, using template for that<commit_after>import urlparse

from formspree.app import DB, redis_store
from formspree import settings, log
from formspree.utils import unix_time_for_12_months_from_now, next_url
from flask import url_for, render_template
from helpers import HASH, MONTHLY_COUNTER_KEY, http_form_to_dict, referrer_to_path, send_email
from formspree.utils import int2bigstring, bigstring2int
import datetime

CODE_TEMPLATE = '''
<form action="{action}" method="POST">
    <input type="text" name="_gotcha" style="display:none" />
    <input type="email" name="email" placeholder="Your email">
    <textarea name="message" rows="5" placeholder="Your message"></textarea>
    <input type="submit" value="Send">
</form>
'''

class Form(DB.Model):
    __tablename__ = 'forms'

    id = DB.Column(DB.Integer, primary_key=True)
    hash = DB.Column(DB.String(32), unique=True)
    email = DB.Column(DB.String(120))
    host = DB.Column(DB.String(300))
    confirm_sent = DB.Column(DB.Boolean)
    confirmed = DB.Column(DB.Boolean)
    counter = DB.Column(DB.Integer)
    owner_id = DB.Column(DB.Integer, DB.ForeignKey('users.id'))

    '''
    When the form is created by a spontaneous submission, it is added to
    the table with a `host`, an `email` and a `hash` made of these two
    (+ a secret nonce).

    `hash` is UNIQUE because it is used to query these spontaneous forms
    when the form is going to be confirmed and whenever a new submission arrives.

    When a registered user POSTs to /forms, a new form is added to the table
    with an `email` (provided by the user) and an `owner_id`. Later, when this
    form receives its first submission and confirmation, `host` is added, so
    we can ensure that no one will submit to this same form from another host.

    `hash` is never added to these forms, because they could conflict with other
    forms, created by the spontaneous process, with the same email and host. So
    for these forms a different confirmation method is used (see below).
    '''

    STATUS_EMAIL_SENT              = 0
    STATUS_EMAIL_EMPTY             = 1
    STATUS_EMAIL_FAILED            = 2

    STATUS_CONFIRMATION_SENT       = 10
    STATUS_CONFIRMATION_DUPLICATED = 11
    STATUS_CONFIRMATION_FAILED     = 12

    def __init__(self, email, host=None, owner=None):
        if host:
            self.hash = HASH(email, host)
        elif owner:
            self.owner_id = owner.id
        else:
            raise Exception('cannot create form without a host and a owner. provide one of these.')
        self.email = email
        self.host = host
        self.confirm_sent = False
        self.confirmed = False
        self.counter = 0

    def __repr__(self):
        return '<Form %s, email=%s, host=%s>' % (self.id, self.email, self.host)

    def get_random_like_string(self):
        if not self.id:
            raise Exception("this form doesn't have an id yet, commit it first.")
        return int2bigstring(self.id)

    @classmethod
    def get_form_by_random_like_string(cls, random_like_string):
        id = bigstring2int(random_like_string)
        return cls.query.get(id)

    def send(self, http_form, referrer):
        '''
        Sends form to user's email.
        Assumes sender's email has been verified.
        '''

        data, keys = http_form_to_dict(http_form)

        subject = data.get('_subject', 'New submission from %s' % referrer_to_path(referrer))
        reply_to = data.get('_replyto', data.get('email', data.get('Email', None)))
        cc = data.get('_cc', None)
        next = next_url(referrer, data.get('_next'))
        spam = data.get('_gotcha', None)

        # prevent submitting empty form
        if not any(data.values()):
            return { 'code': Form.STATUS_EMAIL_EMPTY }

        # return a fake success for spam
        if spam:
            return { 'code': Form.STATUS_EMAIL_SENT, 'next': next }

        # increment the forms counter
        self.counter = Form.counter + 1
        DB.session.add(self)
        DB.session.commit()

        # increase the monthly counter
        request_date = datetime.datetime.now()
        self.increase_monthly_counter(basedate=request_date)

        # check if the forms are over the counter and the user is not upgraded
        overlimit = False
        if self.get_monthly_counter(basedate=request_date) > settings.MONTHLY_SUBMISSIONS_LIMIT:
            if not self.owner or not self.owner.upgraded:
                overlimit = True

        now = datetime.datetime.utcnow().strftime('%I:%M %p UTC - %d %B %Y')
        if not overlimit:
            text = render_template('email/form.txt', data=data, host=self.host, keys=keys, now=now)
            html = render_template('email/form.html', data=data, host=self.host, keys=keys, now=now)
        else:
            text = render_template('email/overlimit-notification.txt', host=self.host)
            html = render_template('email/overlimit-notification.html', host=self.host)

        result = send_email(to=self.email,
                          subject=subject,
                          text=text,
                          html=html,
                          sender=settings.DEFAULT_SENDER,
                          reply_to=reply_to,
                          cc=cc)

        if not result[0]:
            return{ 'code': Form.STATUS_EMAIL_FAILED }

        return { 'code': Form.STATUS_EMAIL_SENT, 'next': next }

    def get_monthly_counter(self, basedate=None):
        basedate = basedate or datetime.datetime.now()
        month = basedate.month
        key = MONTHLY_COUNTER_KEY(form_id=self.id, month=month)
        counter = redis_store.get(key) or 0
        return int(counter)

    def increase_monthly_counter(self, basedate=None):
        basedate = basedate or datetime.datetime.now()
        month = basedate.month
        key = MONTHLY_COUNTER_KEY(form_id=self.id, month=month)
        redis_store.incr(key)
        redis_store.expireat(key, unix_time_for_12_months_from_now(basedate))

    def send_confirmation(self):
        '''
        Helper that actually creates confirmation nonce
        and sends the email to associated email. Renders
        different templates depending on the result
        '''

        log.debug('Sending confirmation')
        if self.confirm_sent:
            return { 'code': Form.STATUS_CONFIRMATION_DUPLICATED }

        # the nonce for email confirmation will be the hash when it exists
        # (whenever the form was created from a simple submission) or
        # a concatenation of HASH(email, id) + ':' + id (whenever the form
        # was created from the dashboard)
        id = str(self.id)
        nonce = self.hash or '%s:%s' % (HASH(self.email, id), id)
        link = url_for('confirm_email', nonce=nonce, _external=True)

        def render_content(type):
            return render_template('email/confirm.%s' % type,
                                      email=self.email,
                                      host=self.host,
                                      nonce_link=link)

        log.debug('Sending email')

        result = send_email(to=self.email,
                            subject='Confirm email for %s' % settings.SERVICE_NAME,
                            text=render_content('txt'),
                            html=render_content('html'),
                            sender=settings.DEFAULT_SENDER)

        log.debug('Sent')

        if not result[0]:
            return { 'code': Form.STATUS_CONFIRMATION_FAILED }

        self.confirm_sent = True
        DB.session.add(self)
        DB.session.commit()

        return { 'code': Form.STATUS_CONFIRMATION_SENT }

    @classmethod
    def confirm(cls, nonce):
        if ':' in nonce:
            # form created in the dashboard
            # nonce is another hash and the id
            # comes in the request
            nonce, id = nonce.split(':')
            form = cls.query.get(id)
            if HASH(form.email, str(form.id)) == nonce:
                pass
            else:
                form = None
        else:
            # normal form, nonce is HASH(email, host)
            form = cls.query.filter_by(hash=nonce).first()

        if form:
            form.confirmed = True
            DB.session.add(form)
            DB.session.commit()
            return form

    @property
    def action(self):
        return url_for('send', email_or_string=self.get_random_like_string(), _external=True)

    @property
    def code(self):
        return CODE_TEMPLATE.format(action=self.action)

    @property
    def is_new(self):
        return not self.host
<|endoftext|>


<commit_before>"""
This module defines functions of time and tools to manipulate them.

The main class TimeFunction, is a function from (real) time to an arbitrary
number of (real) outputs.

These objects can be (coordinate-wised) multiplied, added, subtracted and
divided.

"""

import types

import numpy as N
from neuroimaging import traits

from scipy.sandbox.models.utils import StepFunction, recipr0

# Prototypical stimuli: "Event" (on/off) and "Stimuli" (step function)
# -Event inherits from Stimulus so most functionality is in Stimulus
# -changes are just in specifying parameters of self.fn

times = N.arange(0,50,0.1)

class TimeFunction(traits.HasTraits):

    nout = traits.Int(1)
    fn = traits.Any()

    windowed = traits.false
    window = traits.List([0.,0.])

    def __getitem__(self, j):

        def _f(time=None, obj=self, **extra):
            return obj(time=time, **extra)[int(j)]
        return TimeFunction(fn=_f)

    def __call__(self, time=None, **extra):
        columns = []

        if self.nout == 1:
            columns.append(self.fn(time=time))
        else:
            if type(self.fn) in [types.ListType, types.TupleType]:
                for fn in self.fn:
                    columns.append(fn(time=time))
            else:
                columns = self.fn(time=time)

        if self.windowed:
            _window = N.greater(time, self.window[0]) * N.less_equal(time, self.window[1])
            columns = [column * _window for column in columns]

        return N.squeeze(N.array(columns))

    def __mul__(self, other):
        if isinstance(other, TimeFunction):
            if other.nout == self.nout:
                def _f(time=None, _self=self, _other=other, **extra):
                    return N.squeeze(_self(time=time, **extra) * _other(time=time, **extra))
            else:
                raise ValueError, 'number of outputs of regressors do not match'
        elif type(other) in [types.FloatType, types.IntType]:
            def _f(time=None, _self=self, _other=other, **extra):
                return N.squeeze(_self(time=time, **extra) * other)
        elif type(other) in [types.ListType, types.TupleType, N.ndarray]:
            if type(other) is N.ndarray:
                if other.shape != (self.nout,):
                    raise 'shape does not much output, expecting (%d,)' % self.nout
            elif len(other) != self.nout:
                    raise 'length does not much output, expecting sequence of length %d' % self.nout
            def _f(time=None, _self=self, _other=N.array(other), **extra):
                v = _self(time=time, **extra)
                for i in range(_other.shape[0]):
                    v[i] *= _other[i]
                return N.squeeze(v)
        else:
            raise ValueError, 'unrecognized type'
        return TimeFunction(fn=_f, nout=self.nout)

    def __add__(self, other):
        if isinstance(other, TimeFunction):
            if other.nout == self.nout:
                def _f(time=None, _self=self, _other=other, **extra):
                    v = _self(time=time, **extra) + _other(time=time, **extra)
                    return N.squeeze(v)
            else:
                raise ValueError, 'number of outputs of regressors do not match'
        elif type(other) in [types.FloatType, types.IntType]:
            def _f(time=None, _self=self, _other=other, **extra):
                v = _self(time=time, **extra) + other
                return N.squeeze(v)
        elif type(other) in [types.ListType, types.TupleType, N.ndarray]:
            if type(other) is N.ndarray:
                if other.shape != (self.nout,):
                    raise 'shape does not much output, expecting (%d,)' % self.nout
            elif len(other) != self.nout:
                    raise 'length does not much output, expecting sequence of length %d' % self.nout
            def _f(time=None, _self=self, _other=N.array(other), **extra):
                v = _self(time=time, **extra)
                for i in range(_other.shape[0]):
                    v[i] += _other[i]
                return N.squeeze(v)
        else:
            raise ValueError, 'unrecognized type'
        return TimeFunction(fn=_f, nout=self.nout)

    def __sub__(self, other):
        if isinstance(other, TimeFunction):
            if other.nout == self.nout:
                def _f(time=None, _self=self, _other=other, **extra):
                    v = _self(time=time, **extra) - _other(time=time, **extra)
                    return N.squeeze(v)
            else:
                raise ValueError, 'number of outputs of regressors do not match'
        elif type(other) in [types.FloatType, types.IntType]:
            def _f(time=None, _self=self, _other=other, **extra):
                v = _self(time=time, **extra) - other
                return N.squeeze(v)
        elif type(other) in [types.ListType, types.TupleType, N.ndarray]:
            if type(other) is N.ndarray:
                if other.shape != (self.nout,):
                    raise 'shape does not much output, expecting (%d,)' % self.nout
            elif len(other) != self.nout:
                    raise 'length does not much output, expecting sequence of length %d' % self.nout
            def _f(time=None, _self=self, _other=N.array(other), **extra):
                v = _self(time=time, **extra)
                for i in range(_other.shape[0]):
                    v[i] -= _other[i]
                return N.squeeze(v)
        else:
            raise ValueError, 'unrecognized type'
        return TimeFunction(fn=_f, nout=self.nout)

    def __div__(self, other):
        if isinstance(other, TimeFunction):
            if other.nout == self.nout:
                def _f(time=None, _self=self, _other=other, **extra):
                    return N.squeeze(_self(time=time, **extra) * recipr0(_other(time=time, **extra)))
            else:
                raise ValueError, 'number of outputs of regressors do not match'
        elif type(other) in [types.FloatType, types.IntType]:
            def _f(time=None, _self=self, _other=other, **extra):
                return N.squeeze(_self(time=time, **extra) * recipr0(other))
        elif type(other) in [types.ListType, types.TupleType, N.ndarray]:
            if type(other) is N.ndarray:
                if other.shape != (self.nout,):
                    raise 'shape does not much output, expecting (%d,)' % self.nout
            elif len(other) != self.nout:
                    raise 'length does not much output, expecting sequence of length %d' % self.nout
            def _f(time=None, _self=self, _other=N.array(other), **extra):
                v = _self(time=time, **extra)
                for i in range(_other.shape[0]):
                    v[i] *= recipr0(_other[i])
                return N.squeeze(v)
        else:
            raise ValueError, 'unrecognized type'
        return TimeFunction(fn=_f, nout=self.nout)

<commit_msg>fix indentation and remove unused import<commit_after>"""
This module defines functions of time and tools to manipulate them.

The main class TimeFunction, is a function from (real) time to an arbitrary
number of (real) outputs.

These objects can be (coordinate-wised) multiplied, added, subtracted and
divided.

"""

import types

import numpy as N
from neuroimaging import traits

from scipy.sandbox.models.utils import recipr0

# Prototypical stimuli: "Event" (on/off) and "Stimuli" (step function)
# -Event inherits from Stimulus so most functionality is in Stimulus
# -changes are just in specifying parameters of self.fn

times = N.arange(0,50,0.1)

class TimeFunction(traits.HasTraits):

    nout = traits.Int(1)
    fn = traits.Any()

    windowed = traits.false
    window = traits.List([0.,0.])

    def __getitem__(self, j):

        def _f(time=None, obj=self, **extra):
            return obj(time=time, **extra)[int(j)]
        return TimeFunction(fn=_f)

    def __call__(self, time=None, **extra):
        columns = []

        if self.nout == 1:
            columns.append(self.fn(time=time))
        else:
            if type(self.fn) in [types.ListType, types.TupleType]:
                for fn in self.fn:
                    columns.append(fn(time=time))
            else:
                columns = self.fn(time=time)

        if self.windowed:
            _window = N.greater(time, self.window[0]) * N.less_equal(time, self.window[1])
            columns = [column * _window for column in columns]

        return N.squeeze(N.array(columns))

    def __mul__(self, other):
        if isinstance(other, TimeFunction):
            if other.nout == self.nout:
                def _f(time=None, _self=self, _other=other, **extra):
                    return N.squeeze(_self(time=time, **extra) * _other(time=time, **extra))
            else:
                raise ValueError, 'number of outputs of regressors do not match'
        elif type(other) in [types.FloatType, types.IntType]:
            def _f(time=None, _self=self, _other=other, **extra):
                return N.squeeze(_self(time=time, **extra) * other)
        elif type(other) in [types.ListType, types.TupleType, N.ndarray]:
            if type(other) is N.ndarray:
                if other.shape != (self.nout,):
                    raise 'shape does not much output, expecting (%d,)' % self.nout
            elif len(other) != self.nout:
                raise 'length does not much output, expecting sequence of length %d' % self.nout
            def _f(time=None, _self=self, _other=N.array(other), **extra):
                v = _self(time=time, **extra)
                for i in range(_other.shape[0]):
                    v[i] *= _other[i]
                return N.squeeze(v)
        else:
            raise ValueError, 'unrecognized type'
        return TimeFunction(fn=_f, nout=self.nout)

    def __add__(self, other):
        if isinstance(other, TimeFunction):
            if other.nout == self.nout:
                def _f(time=None, _self=self, _other=other, **extra):
                    v = _self(time=time, **extra) + _other(time=time, **extra)
                    return N.squeeze(v)
            else:
                raise ValueError, 'number of outputs of regressors do not match'
        elif type(other) in [types.FloatType, types.IntType]:
            def _f(time=None, _self=self, _other=other, **extra):
                v = _self(time=time, **extra) + other
                return N.squeeze(v)
        elif type(other) in [types.ListType, types.TupleType, N.ndarray]:
            if type(other) is N.ndarray:
                if other.shape != (self.nout,):
                    raise 'shape does not much output, expecting (%d,)' % self.nout
            elif len(other) != self.nout:
                raise 'length does not much output, expecting sequence of length %d' % self.nout
            def _f(time=None, _self=self, _other=N.array(other), **extra):
                v = _self(time=time, **extra)
                for i in range(_other.shape[0]):
                    v[i] += _other[i]
                return N.squeeze(v)
        else:
            raise ValueError, 'unrecognized type'
        return TimeFunction(fn=_f, nout=self.nout)

    def __sub__(self, other):
        if isinstance(other, TimeFunction):
            if other.nout == self.nout:
                def _f(time=None, _self=self, _other=other, **extra):
                    v = _self(time=time, **extra) - _other(time=time, **extra)
                    return N.squeeze(v)
            else:
                raise ValueError, 'number of outputs of regressors do not match'
        elif type(other) in [types.FloatType, types.IntType]:
            def _f(time=None, _self=self, _other=other, **extra):
                v = _self(time=time, **extra) - other
                return N.squeeze(v)
        elif type(other) in [types.ListType, types.TupleType, N.ndarray]:
            if type(other) is N.ndarray:
                if other.shape != (self.nout,):
                    raise 'shape does not much output, expecting (%d,)' % self.nout
            elif len(other) != self.nout:
                raise 'length does not much output, expecting sequence of length %d' % self.nout
            def _f(time=None, _self=self, _other=N.array(other), **extra):
                v = _self(time=time, **extra)
                for i in range(_other.shape[0]):
                    v[i] -= _other[i]
                return N.squeeze(v)
        else:
            raise ValueError, 'unrecognized type'
        return TimeFunction(fn=_f, nout=self.nout)

    def __div__(self, other):
        if isinstance(other, TimeFunction):
            if other.nout == self.nout:
                def _f(time=None, _self=self, _other=other, **extra):
                    return N.squeeze(_self(time=time, **extra) * recipr0(_other(time=time, **extra)))
            else:
                raise ValueError, 'number of outputs of regressors do not match'
        elif type(other) in [types.FloatType, types.IntType]:
            def _f(time=None, _self=self, _other=other, **extra):
                return N.squeeze(_self(time=time, **extra) * recipr0(other))
        elif type(other) in [types.ListType, types.TupleType, N.ndarray]:
            if type(other) is N.ndarray:
                if other.shape != (self.nout,):
                    raise 'shape does not much output, expecting (%d,)' % self.nout
            elif len(other) != self.nout:
                raise 'length does not much output, expecting sequence of length %d' % self.nout
            def _f(time=None, _self=self, _other=N.array(other), **extra):
                v = _self(time=time, **extra)
                for i in range(_other.shape[0]):
                    v[i] *= recipr0(_other[i])
                return N.squeeze(v)
        else:
            raise ValueError, 'unrecognized type'
        return TimeFunction(fn=_f, nout=self.nout)

<|endoftext|>


<commit_before>"""
Simple utility for running unit tests.
"""

import sys
from optparse import OptionParser, Option

from testutils import test_all, test_package, get_package_tests
from testutils import doctest_all, doctest_package
from neuroimaging import nontest_packages

# description = __doc__

class TestTool (OptionParser):
    """
    %prog [options] [<pkgname> [<testname>]]

    Tests will be run for the named package (or all packages).  If a test name
    is given, only that test will be run.  If the list option is given,
    avalailable tests will be listed for the given package (or all packages).
    """

    _usage = __doc__
    options = (
      Option('-l', '--list', dest="list_tests", action="store_true",
        default=False, help="list available tests"),
      Option('-d', '--doctest', dest="doctest", action="store_true",
        default=False, help="run available doctests"),)

    def __init__(self, *args, **kwargs):
        OptionParser.__init__(self, *args, **kwargs)
        self.set_usage(self._usage)
        self.add_options(self.options)
#        self.set_description("description:" + description)

    def list_tests(self, package=None):
        if package:
            packs = [package]
        else:
            packs = list(nontest_packages)
            packs.sort()
        for package in packs:
            tests = get_package_tests(package).keys()
            tests.sort()
            print package,":"
            if tests:
                for test in tests: print " ",test
            else:
                print "  No tests found for this package"

    def run_tests(self, package=None, testcase=None):
        if not package: test_all()
        else: test_package(package, testcase)

    def run_doctests(self, package=None):
        if not package: doctest_all()
        else: doctest_package(package)

    def run(self):
        options, args = self.parse_args()

        if options.list_tests:
            if len(args) > 1:
                self.print_help()
                sys.exit(0)
            self.list_tests(*args)
        else: self.run_tests(*args)

        if options.doctest and len(args)==1:
            self.run_doctests(*args)

if __name__ == "__main__": TestTool().run()
<commit_msg>added a flag to not run unittests so only doctests can be run<commit_after>"""
Simple utility for running unit tests.
"""

import sys
from optparse import OptionParser, Option

from testutils import test_all, test_package, get_package_tests
from testutils import doctest_all, doctest_package
from neuroimaging import nontest_packages

# description = __doc__

class TestTool (OptionParser):
    """
    %prog [options] [<pkgname> [<testname>]]

    Tests will be run for the named package (or all packages).  If a test name
    is given, only that test will be run.  If the list option is given,
    avalailable tests will be listed for the given package (or all packages).
    """

    _usage = __doc__
    options = (
      Option('-l', '--list', dest="list_tests", action="store_true",
        default=False, help="list available tests"),
      Option('-d', '--doctest', dest="doctest", action="store_true",
        default=False, help="run available doctests"),
      Option('', '--nounit', dest="unit", action="store_false",
        default=True, help="do not run unittests"),)

    def __init__(self, *args, **kwargs):
        OptionParser.__init__(self, *args, **kwargs)
        self.set_usage(self._usage)
        self.add_options(self.options)
#        self.set_description("description:" + description)

    def list_tests(self, package=None):
        if package:
            packs = [package]
        else:
            packs = list(nontest_packages)
            packs.sort()
        for package in packs:
            tests = get_package_tests(package).keys()
            tests.sort()
            print package,":"
            if tests:
                for test in tests: print " ",test
            else:
                print "  No tests found for this package"

    def run_tests(self, package=None, testcase=None):
        if not package: test_all()
        else: test_package(package, testcase)

    def run_doctests(self, package=None):
        if not package: doctest_all()
        else: doctest_package(package)

    def run(self):
        options, args = self.parse_args()

        if options.list_tests:
            if len(args) > 1:
                self.print_help()
                sys.exit(0)
            self.list_tests(*args)
        elif options.unit: self.run_tests(*args)

        if options.doctest and len(args)<=1:
            self.run_doctests(*args)

if __name__ == "__main__": TestTool().run()
<|endoftext|>