<commit_before>'''
Extractors that operate primarily or exclusively on Text stimuli.
'''
import sys

from pliers.stimuli.text import TextStim, ComplexTextStim
from pliers.extractors.base import Extractor, ExtractorResult
from pliers.support.exceptions import PliersError
from pliers.support.decorators import requires_nltk_corpus
from pliers.datasets.text import fetch_dictionary
from pliers.transformers import BatchTransformerMixin
from pliers.utils import attempt_to_import, verify_dependencies
import numpy as np
import pandas as pd
import nltk
from nltk.sentiment.vader import SentimentIntensityAnalyzer
import os
import re
import logging
from six import string_types

keyedvectors = attempt_to_import('gensim.models.keyedvectors', 'keyedvectors',
                                 ['KeyedVectors'])
sklearn_text = attempt_to_import('sklearn.feature_extraction.text', 'sklearn_text',
                                 ['VectorizerMixin', 'CountVectorizer'])
spacy = attempt_to_import('spacy')

class TextExtractor(Extractor):

    ''' Base Text Extractor class; all subclasses can only be applied to text.
    '''
    _input_type = TextStim

class ComplexTextExtractor(Extractor):

    ''' Base ComplexTextStim Extractor class; all subclasses can only be
    applied to ComplexTextStim instance.
    '''
    _input_type = ComplexTextStim

    def _extract(self, stim):
        ''' Returns all words. '''
        props = [(e.text, e.onset, e.duration) for e in stim.elements]
        vals, onsets, durations = map(list, zip(*props))
        return ExtractorResult(vals, stim, self, ['word'], onsets, durations)

class DictionaryExtractor(TextExtractor):

    ''' A generic dictionary-based extractor that supports extraction of
    arbitrary features contained in a lookup table.

    Args:
        dictionary (str, DataFrame): The dictionary containing the feature
            values. Either a string giving the path to the dictionary file,
            or a pandas DF. Format must be tab-delimited, with the first column
            containing the text key used for lookup. Subsequent columns each
            represent a single feature that can be used in extraction.
        variables (list): Optional subset of columns to keep from the
            dictionary.
        missing: Value to insert if no lookup value is found for a text token.
            Defaults to numpy's NaN.
    '''

    _log_attributes = ('dictionary', 'variables', 'missing')
    VERSION = '1.0'

    def __init__(self, dictionary, variables=None, missing=np.nan):
        if isinstance(dictionary, string_types):
            self.dictionary = dictionary  # for TranformationHistory logging
            dictionary = pd.read_csv(dictionary, sep='\t', index_col=0)
        else:
            self.dictionary = None
        self.data = dictionary
        if variables is None:
            variables = list(self.data.columns)
        else:
            self.data = self.data[variables]
        self.variables = variables
        # Set up response when key is missing
        self.missing = missing
        super(DictionaryExtractor, self).__init__()

    def _extract(self, stim):
        if stim.text not in self.data.index:
            vals = pd.Series(self.missing, self.variables)
        else:
            vals = self.data.loc[stim.text].fillna(self.missing)
        vals = vals.to_dict()
        return ExtractorResult(np.array([list(vals.values())]), stim, self,
                               features=list(vals.keys()))

class PredefinedDictionaryExtractor(DictionaryExtractor):

    ''' A generic Extractor that maps words onto values via one or more
    pre-defined dictionaries accessed via the web.

    Args:
        variables (list or dict): A specification of the dictionaries and
            column names to map the input TextStims onto. If a list, each
            element must be a string with the format 'dict/column', where the
            value before the slash gives the name of the dictionary, and the
            value after the slash gives the name of the column in that
            dictionary. These names can be found in the dictionaries.json
            specification file under the datasets submodule. Examples of
            valid values are 'affect/V.Mean.Sum' and
            'subtlexusfrequency/Lg10WF'. If a dict, the keys are the names of
            the dictionary files (e.g., 'affect'), and the values are lists
            of columns to use (e.g., ['V.Mean.Sum', 'V.SD.Sum']).
        missing (object): Value to use when an entry for a word is missing in
            a dictionary (defaults to numpy's NaN).
        case_sensitive (bool): If True, entries in the dictionary are treated
            as case-sensitive (e.g., 'John' and 'john' are different words).
        force_retrieve (bool): If True, the source dictionary will always be
            retrieved/download, even if it exists locally. If False, a cached
            local version will be used if it exists.
    '''

    _log_attributes = ('variables', 'missing', 'case_sensitive')
    VERSION = '1.0'

    def __init__(self, variables, missing=np.nan, case_sensitive=True,
                 force_retrieve=False):

        self.case_sensitive = case_sensitive

        if isinstance(variables, (list, tuple)):
            _vars = {}
            for v in variables:
                v = v.split('/')
                if v[0] not in _vars:
                    _vars[v[0]] = []
                if len(v) == 2:
                    _vars[v[0]].append(v[1])
            variables = _vars

        dicts = []
        for k, v in variables.items():
            d = fetch_dictionary(k, force_retrieve=force_retrieve)
            if not case_sensitive:
                d.index = d.index.str.lower()
            if v:
                d = d[v]
            d.columns = ['%s_%s' % (k, c) for c in d.columns]
            dicts.append(d)

        # Make sure none of the dictionaries have duplicate indices
        drop_dups = lambda d: d[~d.index.duplicated(keep='first')]
        dicts = [d if d.index.is_unique else drop_dups(d) for d in dicts]

        dictionary = pd.concat(dicts, axis=1, join='outer', sort=False)

        super(PredefinedDictionaryExtractor, self).__init__(
            dictionary, missing=missing)

class LengthExtractor(TextExtractor):

    ''' Extracts the length of the text in characters. '''

    VERSION = '1.0'

    def _extract(self, stim):
        return ExtractorResult(np.array([[len(stim.text.strip())]]), stim,
                               self, features=['text_length'])

class NumUniqueWordsExtractor(TextExtractor):

    ''' Extracts the number of unique words used in the text. '''

    _log_attributes = ('tokenizer',)
    VERSION = '1.0'

    def __init__(self, tokenizer=None):
        super(NumUniqueWordsExtractor, self).__init__()
        self.tokenizer = tokenizer

    @requires_nltk_corpus
    def _extract(self, stim):
        text = stim.text
        if self.tokenizer is None:
            if nltk is None:
                num_words = len(set(text.split()))
            else:
                num_words = len(set(nltk.word_tokenize(text)))
        else:
            num_words = len(set(self.tokenizer.tokenize(text)))

        return ExtractorResult(np.array([[num_words]]), stim, self,
                               features=['num_unique_words'])

class PartOfSpeechExtractor(BatchTransformerMixin, TextExtractor):

    ''' Tags parts of speech in text with nltk. '''

    _batch_size = sys.maxsize
    VERSION = '1.0'

    @requires_nltk_corpus
    def _extract(self, stims):
        words = [w.text for w in stims]
        pos = nltk.pos_tag(words)
        if len(words) != len(pos):
            raise PliersError(
                "The number of words does not match the number of tagged words"
                "returned by nltk's part-of-speech tagger.")

        results = []
        tagset = nltk.data.load('help/tagsets/upenn_tagset.pickle').keys()
        for i, s in enumerate(stims):
            pos_vector = dict.fromkeys(tagset, 0)
            pos_vector[pos[i][1]] = 1
            values = [list(pos_vector.values())]
            results.append(ExtractorResult(values, s, self,
                                           features=list(pos_vector.keys())))

        return results

class WordEmbeddingExtractor(TextExtractor):

    ''' An extractor that uses a word embedding file to look up embedding
    vectors for text.

    Args:
        embedding_file (str): Path to a word embedding file. Assumed to be in
            word2vec format compatible with gensim.
        binary (bool): Flag indicating whether embedding file is saved in a
            binary format.
        prefix (str): Prefix for feature names in the ExtractorResult.
        unk_vector (numpy array or str): Default vector to use for texts not
            found in the embedding file. If None is specified, uses a
            vector with all zeros. If 'random' is specified, uses a vector with
            random values between -1.0 and 1.0. Must have the same dimensions
            as the embeddings.
    '''

    _log_attributes = ('wvModel', 'prefix')

    def __init__(self, embedding_file, binary=False, prefix='embedding_dim',
                 unk_vector=None):
        verify_dependencies(['keyedvectors'])
        self.wvModel = keyedvectors.KeyedVectors.load_word2vec_format(embedding_file, binary=binary)
        self.prefix = prefix
        self.unk_vector = unk_vector
        super(WordEmbeddingExtractor, self).__init__()

    def _extract(self, stim):
        num_dims = self.wvModel.vector_size
        if stim.text in self.wvModel:
            embedding_vector = self.wvModel[stim.text]
        else:
            unk = self.unk_vector
            if hasattr(unk, 'shape') and unk.shape[0] == num_dims:
                embedding_vector = unk
            elif unk == 'random':
                embedding_vector = 2.0 * np.random.random(num_dims) - 1.0
            else:
                # By default, UNKs will have zeroed-out vectors
                embedding_vector = np.zeros(num_dims)

        features = ['%s%d' % (self.prefix, i) for i in range(num_dims)]
        return ExtractorResult([embedding_vector],
                               stim,
                               self,
                               features=features)

class TextVectorizerExtractor(BatchTransformerMixin, TextExtractor):

    ''' Uses a scikit-learn Vectorizer to extract bag-of-features
    from text.

    Args:
        vectorizer (sklearn Vectorizer or str): a scikit-learn Vectorizer
            (or the name in a string) to extract with. Will use the
            CountVectorizer by default. Uses supporting *args and **kwargs.
    '''

    _log_attributes = ('vectorizer',)
    _batch_size = sys.maxsize

    def __init__(self, vectorizer=None, *vectorizer_args, **vectorizer_kwargs):
        verify_dependencies(['sklearn_text'])
        if isinstance(vectorizer, sklearn_text.VectorizerMixin):
            self.vectorizer = vectorizer
        elif isinstance(vectorizer, str):
            vec = getattr(sklearn_text, vectorizer)
            self.vectorizer = vec(*vectorizer_args, **vectorizer_kwargs)
        else:
            self.vectorizer = sklearn_text.CountVectorizer(*vectorizer_args,
                                                           **vectorizer_kwargs)
        super(TextVectorizerExtractor, self).__init__()

    def _extract(self, stims):
        mat = self.vectorizer.fit_transform([s.text for s in stims]).toarray()
        results = []
        for i, row in enumerate(mat):
            results.append(
                ExtractorResult([row], stims[i], self,
                                features=self.vectorizer.get_feature_names()))
        return results

class VADERSentimentExtractor(TextExtractor):

    ''' Uses nltk's VADER lexicon to extract (0.0-1.0) values for the positve,
    neutral, and negative sentiment of a TextStim. Also returns a compound
    score ranging from -1 (very negative) to +1 (very positive). '''

    _log_attributes = ('analyzer',)
    VERSION = '1.0'

    def __init__(self):
        self.analyzer = SentimentIntensityAnalyzer()
        super(VADERSentimentExtractor, self).__init__()

    @requires_nltk_corpus
    def _extract(self, stim):
        scores = self.analyzer.polarity_scores(stim.text)
        features = ['sentiment_' + k for k in scores.keys()]
        return ExtractorResult([list(scores.values())], stim, self,
                               features=features)

class SpaCyExtractor(TextExtractor):

    ''' A generic class for Spacy Text extractors

    Uses SpaCy to extract features from text. Extracts features for every word
    (token) in a sentence.

    Args:
        extractor_type(str): The type of feature to extract. Must be one of
            'doc' (analyze an entire sentence/document) or 'token'
            (analyze each word).
        features(list): A list of strings giving the names of spaCy features to
            extract. See SpacY documentation for details. By default, returns
            all available features for the given extractor type.
        model (str): The name of the language model to use.
    '''

    def __init__(self, extractor_type='token', features=None,
                 model='en_core_web_sm'):

        verify_dependencies(['spacy'])

        try:
            self.model = spacy.load(model)
        except (ImportError, IOError, OSError) as e:
            logging.warning("Spacy Models ('{}') not found. Downloading and"
                            "installing".format(model))

            spacy.cli.download(model)
            self.model = spacy.load(model)

        logging.info('Loaded model: {}'.format(self.model))

        self.features = features
        self.extractor_type = extractor_type.lower()

        super(SpaCyExtractor, self).__init__()

    def _extract(self, stim):

        features_list = []
        elements = self.model(stim.text)
        order_list = []

        if self.extractor_type == 'token':
            if self.features is None:
                self.features = ['text', 'lemma_', 'pos_', 'tag_', 'dep_',
                                 'shape_', 'is_alpha', 'is_stop', 'is_punct',
                                 'sentiment', 'is_ascii', 'is_digit']

        elif self.extractor_type == 'doc':
             elements = [elem.as_doc() for elem in list(elements.sents)]
             if self.features is None:
                 self.features = ['text', 'is_tagged', 'is_parsed',
                                  'is_sentenced', 'sentiment']

        else:
            raise(ValueError("Invalid extractor_type; must be one of 'token'"
                             " or 'doc'."))

        features_list = []
        for elem in elements:
            arr = []
            for feat in self.features:
                arr.append(getattr(elem, feat))
            features_list.append(arr)

        order_list = list(range(1, len(elements) + 1))

        return ExtractorResult(features_list, stim, self,
                               features=self.features, orders=order_list)

class WordCounterExtractor(BatchTransformerMixin, TextExtractor):

    ''' Extracts number of times each unique word has occurred within text

    Args:
        lemmatize(bool): specifies if words are to be lemmatized before being counted.
        '''

    _log_attributes = ('lemmatize',)
    _batch_size = sys.maxsize

    def __init__(self, lemmatize=None):
        super(WordCounterExtractor, self).__init__()
        self.lemmatize = lemmatize

    @requires_nltk_corpus
    def _extract(self, stims):

        tokens = [s.text for s in stims]

        if self.lemmatize:
            lemmatizer=nltk.WordNetLemmatizer()
            pos_map = dict(zip(['ADJ', 'ADJ_SAT', 'ADV', 'NOUN', 'VERB'],
                               ['a', 's', 'r', 'n', 'v'])) # map tags to wordnet tagset
            tokens = dict(nltk.pos_tag(tokens, tagset='universal'))
            tokens = {k: pos_map[v] if v in pos_map else 'n' for k, v in tokens.items()}
            tokens = [lemmatizer.lemmatize(k, pos=v) for k, v in tokens.items()]

        word_counter = pd.Series(tokens).groupby(tokens).cumcount()

        results = []
        for i, count in enumerate(word_counter):
            results.append(ExtractorResult([count], stims[i], self,
                                features=['word_counter']))
        return results
<commit_msg>start count at 1 and add log scale option<commit_after>'''
Extractors that operate primarily or exclusively on Text stimuli.
'''
import sys

from pliers.stimuli.text import TextStim, ComplexTextStim
from pliers.extractors.base import Extractor, ExtractorResult
from pliers.support.exceptions import PliersError
from pliers.support.decorators import requires_nltk_corpus
from pliers.datasets.text import fetch_dictionary
from pliers.transformers import BatchTransformerMixin
from pliers.utils import attempt_to_import, verify_dependencies
import numpy as np
import pandas as pd
import nltk
from nltk.sentiment.vader import SentimentIntensityAnalyzer
import os
import re
import logging
from six import string_types

keyedvectors = attempt_to_import('gensim.models.keyedvectors', 'keyedvectors',
                                 ['KeyedVectors'])
sklearn_text = attempt_to_import('sklearn.feature_extraction.text', 'sklearn_text',
                                 ['VectorizerMixin', 'CountVectorizer'])
spacy = attempt_to_import('spacy')

class TextExtractor(Extractor):

    ''' Base Text Extractor class; all subclasses can only be applied to text.
    '''
    _input_type = TextStim

class ComplexTextExtractor(Extractor):

    ''' Base ComplexTextStim Extractor class; all subclasses can only be
    applied to ComplexTextStim instance.
    '''
    _input_type = ComplexTextStim

    def _extract(self, stim):
        ''' Returns all words. '''
        props = [(e.text, e.onset, e.duration) for e in stim.elements]
        vals, onsets, durations = map(list, zip(*props))
        return ExtractorResult(vals, stim, self, ['word'], onsets, durations)

class DictionaryExtractor(TextExtractor):

    ''' A generic dictionary-based extractor that supports extraction of
    arbitrary features contained in a lookup table.

    Args:
        dictionary (str, DataFrame): The dictionary containing the feature
            values. Either a string giving the path to the dictionary file,
            or a pandas DF. Format must be tab-delimited, with the first column
            containing the text key used for lookup. Subsequent columns each
            represent a single feature that can be used in extraction.
        variables (list): Optional subset of columns to keep from the
            dictionary.
        missing: Value to insert if no lookup value is found for a text token.
            Defaults to numpy's NaN.
    '''

    _log_attributes = ('dictionary', 'variables', 'missing')
    VERSION = '1.0'

    def __init__(self, dictionary, variables=None, missing=np.nan):
        if isinstance(dictionary, string_types):
            self.dictionary = dictionary  # for TranformationHistory logging
            dictionary = pd.read_csv(dictionary, sep='\t', index_col=0)
        else:
            self.dictionary = None
        self.data = dictionary
        if variables is None:
            variables = list(self.data.columns)
        else:
            self.data = self.data[variables]
        self.variables = variables
        # Set up response when key is missing
        self.missing = missing
        super(DictionaryExtractor, self).__init__()

    def _extract(self, stim):
        if stim.text not in self.data.index:
            vals = pd.Series(self.missing, self.variables)
        else:
            vals = self.data.loc[stim.text].fillna(self.missing)
        vals = vals.to_dict()
        return ExtractorResult(np.array([list(vals.values())]), stim, self,
                               features=list(vals.keys()))

class PredefinedDictionaryExtractor(DictionaryExtractor):

    ''' A generic Extractor that maps words onto values via one or more
    pre-defined dictionaries accessed via the web.

    Args:
        variables (list or dict): A specification of the dictionaries and
            column names to map the input TextStims onto. If a list, each
            element must be a string with the format 'dict/column', where the
            value before the slash gives the name of the dictionary, and the
            value after the slash gives the name of the column in that
            dictionary. These names can be found in the dictionaries.json
            specification file under the datasets submodule. Examples of
            valid values are 'affect/V.Mean.Sum' and
            'subtlexusfrequency/Lg10WF'. If a dict, the keys are the names of
            the dictionary files (e.g., 'affect'), and the values are lists
            of columns to use (e.g., ['V.Mean.Sum', 'V.SD.Sum']).
        missing (object): Value to use when an entry for a word is missing in
            a dictionary (defaults to numpy's NaN).
        case_sensitive (bool): If True, entries in the dictionary are treated
            as case-sensitive (e.g., 'John' and 'john' are different words).
        force_retrieve (bool): If True, the source dictionary will always be
            retrieved/download, even if it exists locally. If False, a cached
            local version will be used if it exists.
    '''

    _log_attributes = ('variables', 'missing', 'case_sensitive')
    VERSION = '1.0'

    def __init__(self, variables, missing=np.nan, case_sensitive=True,
                 force_retrieve=False):

        self.case_sensitive = case_sensitive

        if isinstance(variables, (list, tuple)):
            _vars = {}
            for v in variables:
                v = v.split('/')
                if v[0] not in _vars:
                    _vars[v[0]] = []
                if len(v) == 2:
                    _vars[v[0]].append(v[1])
            variables = _vars

        dicts = []
        for k, v in variables.items():
            d = fetch_dictionary(k, force_retrieve=force_retrieve)
            if not case_sensitive:
                d.index = d.index.str.lower()
            if v:
                d = d[v]
            d.columns = ['%s_%s' % (k, c) for c in d.columns]
            dicts.append(d)

        # Make sure none of the dictionaries have duplicate indices
        drop_dups = lambda d: d[~d.index.duplicated(keep='first')]
        dicts = [d if d.index.is_unique else drop_dups(d) for d in dicts]

        dictionary = pd.concat(dicts, axis=1, join='outer', sort=False)

        super(PredefinedDictionaryExtractor, self).__init__(
            dictionary, missing=missing)

class LengthExtractor(TextExtractor):

    ''' Extracts the length of the text in characters. '''

    VERSION = '1.0'

    def _extract(self, stim):
        return ExtractorResult(np.array([[len(stim.text.strip())]]), stim,
                               self, features=['text_length'])

class NumUniqueWordsExtractor(TextExtractor):

    ''' Extracts the number of unique words used in the text. '''

    _log_attributes = ('tokenizer',)
    VERSION = '1.0'

    def __init__(self, tokenizer=None):
        super(NumUniqueWordsExtractor, self).__init__()
        self.tokenizer = tokenizer

    @requires_nltk_corpus
    def _extract(self, stim):
        text = stim.text
        if self.tokenizer is None:
            if nltk is None:
                num_words = len(set(text.split()))
            else:
                num_words = len(set(nltk.word_tokenize(text)))
        else:
            num_words = len(set(self.tokenizer.tokenize(text)))

        return ExtractorResult(np.array([[num_words]]), stim, self,
                               features=['num_unique_words'])

class PartOfSpeechExtractor(BatchTransformerMixin, TextExtractor):

    ''' Tags parts of speech in text with nltk. '''

    _batch_size = sys.maxsize
    VERSION = '1.0'

    @requires_nltk_corpus
    def _extract(self, stims):
        words = [w.text for w in stims]
        pos = nltk.pos_tag(words)
        if len(words) != len(pos):
            raise PliersError(
                "The number of words does not match the number of tagged words"
                "returned by nltk's part-of-speech tagger.")

        results = []
        tagset = nltk.data.load('help/tagsets/upenn_tagset.pickle').keys()
        for i, s in enumerate(stims):
            pos_vector = dict.fromkeys(tagset, 0)
            pos_vector[pos[i][1]] = 1
            values = [list(pos_vector.values())]
            results.append(ExtractorResult(values, s, self,
                                           features=list(pos_vector.keys())))

        return results

class WordEmbeddingExtractor(TextExtractor):

    ''' An extractor that uses a word embedding file to look up embedding
    vectors for text.

    Args:
        embedding_file (str): Path to a word embedding file. Assumed to be in
            word2vec format compatible with gensim.
        binary (bool): Flag indicating whether embedding file is saved in a
            binary format.
        prefix (str): Prefix for feature names in the ExtractorResult.
        unk_vector (numpy array or str): Default vector to use for texts not
            found in the embedding file. If None is specified, uses a
            vector with all zeros. If 'random' is specified, uses a vector with
            random values between -1.0 and 1.0. Must have the same dimensions
            as the embeddings.
    '''

    _log_attributes = ('wvModel', 'prefix')

    def __init__(self, embedding_file, binary=False, prefix='embedding_dim',
                 unk_vector=None):
        verify_dependencies(['keyedvectors'])
        self.wvModel = keyedvectors.KeyedVectors.load_word2vec_format(embedding_file, binary=binary)
        self.prefix = prefix
        self.unk_vector = unk_vector
        super(WordEmbeddingExtractor, self).__init__()

    def _extract(self, stim):
        num_dims = self.wvModel.vector_size
        if stim.text in self.wvModel:
            embedding_vector = self.wvModel[stim.text]
        else:
            unk = self.unk_vector
            if hasattr(unk, 'shape') and unk.shape[0] == num_dims:
                embedding_vector = unk
            elif unk == 'random':
                embedding_vector = 2.0 * np.random.random(num_dims) - 1.0
            else:
                # By default, UNKs will have zeroed-out vectors
                embedding_vector = np.zeros(num_dims)

        features = ['%s%d' % (self.prefix, i) for i in range(num_dims)]
        return ExtractorResult([embedding_vector],
                               stim,
                               self,
                               features=features)

class TextVectorizerExtractor(BatchTransformerMixin, TextExtractor):

    ''' Uses a scikit-learn Vectorizer to extract bag-of-features
    from text.

    Args:
        vectorizer (sklearn Vectorizer or str): a scikit-learn Vectorizer
            (or the name in a string) to extract with. Will use the
            CountVectorizer by default. Uses supporting *args and **kwargs.
    '''

    _log_attributes = ('vectorizer',)
    _batch_size = sys.maxsize

    def __init__(self, vectorizer=None, *vectorizer_args, **vectorizer_kwargs):
        verify_dependencies(['sklearn_text'])
        if isinstance(vectorizer, sklearn_text.VectorizerMixin):
            self.vectorizer = vectorizer
        elif isinstance(vectorizer, str):
            vec = getattr(sklearn_text, vectorizer)
            self.vectorizer = vec(*vectorizer_args, **vectorizer_kwargs)
        else:
            self.vectorizer = sklearn_text.CountVectorizer(*vectorizer_args,
                                                           **vectorizer_kwargs)
        super(TextVectorizerExtractor, self).__init__()

    def _extract(self, stims):
        mat = self.vectorizer.fit_transform([s.text for s in stims]).toarray()
        results = []
        for i, row in enumerate(mat):
            results.append(
                ExtractorResult([row], stims[i], self,
                                features=self.vectorizer.get_feature_names()))
        return results

class VADERSentimentExtractor(TextExtractor):

    ''' Uses nltk's VADER lexicon to extract (0.0-1.0) values for the positve,
    neutral, and negative sentiment of a TextStim. Also returns a compound
    score ranging from -1 (very negative) to +1 (very positive). '''

    _log_attributes = ('analyzer',)
    VERSION = '1.0'

    def __init__(self):
        self.analyzer = SentimentIntensityAnalyzer()
        super(VADERSentimentExtractor, self).__init__()

    @requires_nltk_corpus
    def _extract(self, stim):
        scores = self.analyzer.polarity_scores(stim.text)
        features = ['sentiment_' + k for k in scores.keys()]
        return ExtractorResult([list(scores.values())], stim, self,
                               features=features)

class SpaCyExtractor(TextExtractor):

    ''' A generic class for Spacy Text extractors

    Uses SpaCy to extract features from text. Extracts features for every word
    (token) in a sentence.

    Args:
        extractor_type(str): The type of feature to extract. Must be one of
            'doc' (analyze an entire sentence/document) or 'token'
            (analyze each word).
        features(list): A list of strings giving the names of spaCy features to
            extract. See SpacY documentation for details. By default, returns
            all available features for the given extractor type.
        model (str): The name of the language model to use.
    '''

    def __init__(self, extractor_type='token', features=None,
                 model='en_core_web_sm'):

        verify_dependencies(['spacy'])

        try:
            self.model = spacy.load(model)
        except (ImportError, IOError, OSError) as e:
            logging.warning("Spacy Models ('{}') not found. Downloading and"
                            "installing".format(model))

            spacy.cli.download(model)
            self.model = spacy.load(model)

        logging.info('Loaded model: {}'.format(self.model))

        self.features = features
        self.extractor_type = extractor_type.lower()

        super(SpaCyExtractor, self).__init__()

    def _extract(self, stim):

        features_list = []
        elements = self.model(stim.text)
        order_list = []

        if self.extractor_type == 'token':
            if self.features is None:
                self.features = ['text', 'lemma_', 'pos_', 'tag_', 'dep_',
                                 'shape_', 'is_alpha', 'is_stop', 'is_punct',
                                 'sentiment', 'is_ascii', 'is_digit']

        elif self.extractor_type == 'doc':
             elements = [elem.as_doc() for elem in list(elements.sents)]
             if self.features is None:
                 self.features = ['text', 'is_tagged', 'is_parsed',
                                  'is_sentenced', 'sentiment']

        else:
            raise(ValueError("Invalid extractor_type; must be one of 'token'"
                             " or 'doc'."))

        features_list = []
        for elem in elements:
            arr = []
            for feat in self.features:
                arr.append(getattr(elem, feat))
            features_list.append(arr)

        order_list = list(range(1, len(elements) + 1))

        return ExtractorResult(features_list, stim, self,
                               features=self.features, orders=order_list)

class WordCounterExtractor(BatchTransformerMixin, TextExtractor):

    ''' Extracts number of times each unique word has occurred within text

    Args:
        lemmatize(bool): specifies if words are to be lemmatized before being counted.
        log_scale(bool): specifies if count values are to be returned in log scale (defaults to False)
        '''

    _log_attributes = ('lemmatize','log_scale',)
    _batch_size = sys.maxsize

    def __init__(self, lemmatize=None, log_scale=False):
        super(WordCounterExtractor, self).__init__()
        self.lemmatize = lemmatize
        self.log_scale = log_scale

    @requires_nltk_corpus
    def _extract(self, stims):

        tokens = [s.text for s in stims]

        if self.lemmatize:
            lemmatizer=nltk.WordNetLemmatizer()
            pos_map = dict(zip(['ADJ', 'ADJ_SAT', 'ADV', 'NOUN', 'VERB'],
                               ['a', 's', 'r', 'n', 'v'])) # map tags to wordnet tagset
            tokens = dict(nltk.pos_tag(tokens, tagset='universal'))
            tokens = {k: pos_map[v] if v in pos_map else 'n' for k, v in tokens.items()}
            tokens = [lemmatizer.lemmatize(k, pos=v) for k, v in tokens.items()]

        word_counter = pd.Series(tokens).groupby(tokens).cumcount() + 1

        if self.log_scale:
            word_counter = np.log(word_counter)

        results = []
        for i, count in enumerate(word_counter):
            results.append(ExtractorResult([count], stims[i], self,
                                features=['word_counter']))
        return results
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import matplotlib.pyplot as plt
import numpy as np
import matplotlib.lines as mlines

xlabel="Epoch"
title="Progress"
labels_override=["Test1",]

#Test Files to make plots from; Format: vgg_3/trained_vgg_3_autoencoder_test.txt
#Autoencoders:
test_files = ["vgg_3/trained_vgg_3_autoencoder_test.txt",
              #"vgg_3_reg-e9/trained_vgg_3_reg-e9_autoencoder_test.txt",
              #"vgg_3-eps4/trained_vgg_3-eps4_autoencoder_test.txt",
              #"vgg_3_dropout/trained_vgg_3_dropout_autoencoder_test.txt",
              #"vgg_3_max/trained_vgg_3_max_autoencoder_test.txt",
              #"vgg_3_stride/trained_vgg_3_stride_autoencoder_test.txt",
              #"vgg_3_stride_noRelu/trained_vgg_3_stride_noRelu_autoencoder_test.txt",
              "vgg_3-sgdlr01/trained_vgg_3-sgdlr01_autoencoder_test.txt",
              "vgg_3_eps/trained_vgg_3_eps_autoencoder_test.txt",]
              #"vgg_3_small/trained_vgg_3_small_autoencoder_test.txt",
              #"vgg_3_verysmall/trained_vgg_3_verysmall_autoencoder_test.txt",]

#Unfrozen
xtest_files = ["vgg_3/trained_vgg_3_supervised_up_down_test.txt",
              "vgg_3_dropout/trained_vgg_3_dropout_supervised_up_down_test.txt",
              "vgg_3_max/trained_vgg_3_max_supervised_up_down_test.txt",
              "vgg_3_stride/trained_vgg_3_stride_supervised_up_down_test.txt",
              "vgg_3_eps/trained_vgg_3_eps_supervised_up_down_test.txt",]

#Encoders Epoch 10
xtest_files = ["vgg_3/trained_vgg_3_autoencoder_epoch10_supervised_up_down_test.txt",
              "vgg_3_dropout/trained_vgg_3_dropout_autoencoder_epoch10_supervised_up_down_test.txt",
              "vgg_3_max/trained_vgg_3_max_autoencoder_epoch10_supervised_up_down_test.txt",
              "vgg_3_stride/trained_vgg_3_stride_autoencoder_epoch10_supervised_up_down_test.txt", ]

#sgdlr01 encoders
xtest_files = ["vgg_3-sgdlr01/trained_vgg_3-sgdlr01_autoencoder_epoch2_supervised_up_down_test.txt",
              "vgg_3-sgdlr01/trained_vgg_3-sgdlr01_autoencoder_epoch5_supervised_up_down_test.txt",
              "vgg_3-sgdlr01/trained_vgg_3-sgdlr01_autoencoder_epoch10_supervised_up_down_test.txt",]

#Enocders vgg_3_eps AE E10
xtest_files = ["vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch10_supervised_up_down_test.txt",
              "vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch10_supervised_up_down_triplebatchnorm_e1_test.txt",
              "vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch10_supervised_up_down_unfbatchnorm_test.txt",
              "vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch80_supervised_up_down_unfbatchnorm_test.txt",
              "vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch10_supervised_up_down_unfbatchnorm_no_addBN_test.txt",
              "vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch10_supervised_up_down_batchnorm_e1_test.txt",
              "vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch11_supervised_up_down_test.txt",]

#vgg3eps AE E10 Encoders: finaler test von vgg_3
xtest_files = [ "vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch10_supervised_up_down_zero_center_and_norm_test.txt",
               "vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch10_supervised_up_down_noNorm_BN_noBN_test.txt",
               "vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch10_supervised_up_down_norm_noBN_noDrop_BN_test.txt",
               "vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch10_supervised_up_downNorm_BN_noDrop_BN_test.txt",
               "vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch10_supervised_up_down_noNorm_BN_BN_test.txt",
               "vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch10_supervised_up_down_noNorm_noBN_BN_test.txt",
               "vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch10_supervised_up_down_zero_center_test.txt",]

#vgg4 autoencoders
xtest_files = ["vgg_3_eps/trained_vgg_3_eps_autoencoder_test.txt",
              "vgg_4_ConvAfterPool/trained_vgg_4_ConvAfterPool_autoencoder_test.txt",
              "vgg_4_6c/trained_vgg_4_6c_autoencoder_test.txt",
              "vgg_4_6c_scale/trained_vgg_4_6c_scale_autoencoder_test.txt",
              "vgg_4_8c/trained_vgg_4_8c_autoencoder_test.txt",
              "vgg_4_10c/trained_vgg_4_10c_autoencoder_test.txt",]

modelnames=[] #e.g. vgg_3_autoencoder
for modelident in test_files:
    modelnames.append(modelident.split("trained_")[1][:-9])

def make_dicts_from_files(test_files):
    dict_array=[]
    for test_file in test_files:
        with open("models/"+test_file, "r") as f:
            k = list(zip(*(line.strip().split('\t') for line in f)))
        data = {}
        for column in k:
            data[column[0]]=column[1:]
        dict_array.append(data)
    return dict_array

def make_loss_epoch(test_file, epoch): #"vgg_3/trained_vgg_3_autoencoder_test.txt"
    #Get losses or accuracy from logfile for one epoch, based on the name of the testfile
    #lin-spaced epoch data is added (slightly off)
    loss_epoch_file=test_file[:-8]+"epoch"+str(epoch)+"_log.txt"

    with open("models/"+loss_epoch_file, "r") as f:
        losses=[]
        use_column = 1 #0 is samples, 1 is loss, 2 is accuracy if supervised model
        for line in f:
            if "#" in line:
                if "Accuracy" in line:
                    use_column=2
                continue
            losses.append( float(line.strip().split('\t')[use_column]) )

    epoch_points = np.linspace(epoch-1,epoch,len(losses), endpoint=False)
    return [epoch_points, losses]

def make_test_train_plot(epochs, test, train, label, epochs_train, ylabel):
    plt.ylabel(ylabel)
    test_plot = plt.plot(epochs, test, marker="o", label=label)
    plt.plot(epochs_train, train, linestyle="-", color=test_plot[0].get_color(), alpha=0.5, lw=0.6, label="")
    return test_plot

def generate_data_for_plots(data_dict):
    #array that contains n arrays (n models to plot), each containing [epoch, test value, train value, train epoch, ylabel]
    max_epoch=0
    data_for_all_plots=[]
    for i,data_dict in enumerate(dict_array):
        current_max_epoch = max(tuple(map(int, data_dict["Epoch"])))
        data_array=[]
        if max_epoch<current_max_epoch:
            max_epoch=current_max_epoch

        if "Test acc" in data_dict:
            ylabel="Accuracy (%)"
            train_epoch_data=[]
            train_acc_data=[]
            for epoch in data_dict["Epoch"]:
                e,l = make_loss_epoch(test_files[i], int(epoch))
                train_epoch_data.extend(e)
                train_acc_data.extend(l)
            data_array=[data_dict["Epoch"], data_dict["Test acc"], train_acc_data, modelnames[i], train_epoch_data, ylabel]
        else:
            ylabel="Loss"
            train_epoch_data=[]
            train_loss_data=[]
            for epoch in data_dict["Epoch"]:
                e,l = make_loss_epoch(test_files[i], int(epoch))
                train_epoch_data.extend(e)
                train_loss_data.extend(l)
            data_array=[data_dict["Epoch"], data_dict["Test loss"], train_loss_data, modelnames[i], train_epoch_data, ylabel]

        data_for_all_plots.append(data_array)
    return np.array(data_for_all_plots), max_epoch

debug=True
if debug==False:
    #Generate data from files
    dict_array = make_dicts_from_files(test_files)
    data_for_plots, max_epoch=generate_data_for_plots(dict_array)
else:
    # Epoch, test, train, label, trainepoch, ylabel
    data_for_plots, max_epoch=[[np.linspace(0,9,10), np.linspace(0,9,10)*0.1, np.linspace(0,9,10)*0.1 + 1, "Test", np.linspace(0,9,10), "Loss" ], ], 10
    data_for_plots=np.array(data_for_plots*2)

fig, ax=plt.subplots()
plt.ylabel(data_for_plots[0][5])

label_array = data_for_plots[:,3]
if len(labels_override) == len(label_array):
    label_array=labels_override
else:
    print("Custom label array does not have the proper length (",len(label_array),"). Using default labels...")

handles1=[]
#plot the data in one plot
for i,data_of_model in enumerate(data_for_plots):
    test_plot = ax.plot(data_of_model[0], data_of_model[1], marker="o")
    ax.plot(data_of_model[4], data_of_model[2], linestyle="-", color=test_plot[0].get_color(), alpha=0.5, lw=0.6)
    handle_for_legend = mlines.Line2D([], [], color=test_plot[0].get_color(), lw=3, label=label_array[i])
    handles1.append(handle_for_legend)

#lhandles, llabels = ax.get_legend_handles_labels()
legend1 = plt.legend(handles=handles1, loc=1)

test_line = mlines.Line2D([], [], color='grey', marker="o", label='Test')
train_line = mlines.Line2D([], [], color='grey', linestyle="-", alpha=0.5, lw=0.6, label='Train')
legend2 = plt.legend(handles=[test_line,train_line], loc="lower right")

ax.add_artist(legend1)
ax.add_artist(legend2)

plt.xlim((0.2,max_epoch))
#plt.xticks( np.linspace(1,max_epoch,max_epoch) )

plt.xlabel(xlabel)
plt.title(title)
plt.grid(True)
plt.show()

<commit_msg>adjusted looks<commit_after># -*- coding: utf-8 -*-
import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import matplotlib.lines as mlines

matplotlib.rcParams.update({'font.size': 14})

xlabel="Epoch"
title="Progress of autoencoders with different optimization strategies"
labels_override=[r"Adam with $\epsilon = 10^{-1}$","SGD",r"Adam with $\epsilon = 10^{-8}$"]

#Test Files to make plots from; Format: vgg_3/trained_vgg_3_autoencoder_test.txt
#Autoencoders:
test_files = ["vgg_3/trained_vgg_3_autoencoder_test.txt",
              #"vgg_3_reg-e9/trained_vgg_3_reg-e9_autoencoder_test.txt",
              #"vgg_3-eps4/trained_vgg_3-eps4_autoencoder_test.txt",
              #"vgg_3_dropout/trained_vgg_3_dropout_autoencoder_test.txt",
              #"vgg_3_max/trained_vgg_3_max_autoencoder_test.txt",
              #"vgg_3_stride/trained_vgg_3_stride_autoencoder_test.txt",
              #"vgg_3_stride_noRelu/trained_vgg_3_stride_noRelu_autoencoder_test.txt",
              "vgg_3-sgdlr01/trained_vgg_3-sgdlr01_autoencoder_test.txt",
              "vgg_3_eps/trained_vgg_3_eps_autoencoder_test.txt",]
              #"vgg_3_small/trained_vgg_3_small_autoencoder_test.txt",
              #"vgg_3_verysmall/trained_vgg_3_verysmall_autoencoder_test.txt",]

#Unfrozen
xtest_files = ["vgg_3/trained_vgg_3_supervised_up_down_test.txt",
              "vgg_3_dropout/trained_vgg_3_dropout_supervised_up_down_test.txt",
              "vgg_3_max/trained_vgg_3_max_supervised_up_down_test.txt",
              "vgg_3_stride/trained_vgg_3_stride_supervised_up_down_test.txt",
              "vgg_3_eps/trained_vgg_3_eps_supervised_up_down_test.txt",]

#Encoders Epoch 10
xtest_files = ["vgg_3/trained_vgg_3_autoencoder_epoch10_supervised_up_down_test.txt",
              "vgg_3_dropout/trained_vgg_3_dropout_autoencoder_epoch10_supervised_up_down_test.txt",
              "vgg_3_max/trained_vgg_3_max_autoencoder_epoch10_supervised_up_down_test.txt",
              "vgg_3_stride/trained_vgg_3_stride_autoencoder_epoch10_supervised_up_down_test.txt", ]

#sgdlr01 encoders
xtest_files = ["vgg_3-sgdlr01/trained_vgg_3-sgdlr01_autoencoder_epoch2_supervised_up_down_test.txt",
              "vgg_3-sgdlr01/trained_vgg_3-sgdlr01_autoencoder_epoch5_supervised_up_down_test.txt",
              "vgg_3-sgdlr01/trained_vgg_3-sgdlr01_autoencoder_epoch10_supervised_up_down_test.txt",]

#Enocders vgg_3_eps AE E10
xtest_files = ["vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch10_supervised_up_down_test.txt",
              "vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch10_supervised_up_down_triplebatchnorm_e1_test.txt",
              "vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch10_supervised_up_down_unfbatchnorm_test.txt",
              "vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch80_supervised_up_down_unfbatchnorm_test.txt",
              "vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch10_supervised_up_down_unfbatchnorm_no_addBN_test.txt",
              "vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch10_supervised_up_down_batchnorm_e1_test.txt",
              "vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch11_supervised_up_down_test.txt",]

#vgg3eps AE E10 Encoders: finaler test von vgg_3
xtest_files = [ "vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch10_supervised_up_down_zero_center_and_norm_test.txt",
               "vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch10_supervised_up_down_noNorm_BN_noBN_test.txt",
               "vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch10_supervised_up_down_norm_noBN_noDrop_BN_test.txt",
               "vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch10_supervised_up_downNorm_BN_noDrop_BN_test.txt",
               "vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch10_supervised_up_down_noNorm_BN_BN_test.txt",
               "vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch10_supervised_up_down_noNorm_noBN_BN_test.txt",
               "vgg_3_eps/trained_vgg_3_eps_autoencoder_epoch10_supervised_up_down_zero_center_test.txt",]

#vgg4 autoencoders
xtest_files = ["vgg_3_eps/trained_vgg_3_eps_autoencoder_test.txt",
              "vgg_4_ConvAfterPool/trained_vgg_4_ConvAfterPool_autoencoder_test.txt",
              "vgg_4_6c/trained_vgg_4_6c_autoencoder_test.txt",
              "vgg_4_6c_scale/trained_vgg_4_6c_scale_autoencoder_test.txt",
              "vgg_4_8c/trained_vgg_4_8c_autoencoder_test.txt",
              "vgg_4_10c/trained_vgg_4_10c_autoencoder_test.txt",]

modelnames=[] #e.g. vgg_3_autoencoder
for modelident in test_files:
    modelnames.append(modelident.split("trained_")[1][:-9])

def make_dicts_from_files(test_files):
    dict_array=[]
    for test_file in test_files:
        with open("models/"+test_file, "r") as f:
            k = list(zip(*(line.strip().split('\t') for line in f)))
        data = {}
        for column in k:
            data[column[0]]=column[1:]
        dict_array.append(data)
    return dict_array

def make_loss_epoch(test_file, epoch): #"vgg_3/trained_vgg_3_autoencoder_test.txt"
    #Get losses or accuracy from logfile for one epoch, based on the name of the testfile
    #lin-spaced epoch data is added (slightly off)
    loss_epoch_file=test_file[:-8]+"epoch"+str(epoch)+"_log.txt"

    with open("models/"+loss_epoch_file, "r") as f:
        losses=[]
        use_column = 1 #0 is samples, 1 is loss, 2 is accuracy if supervised model
        for line in f:
            if "#" in line:
                if "Accuracy" in line:
                    use_column=2
                continue
            losses.append( float(line.strip().split('\t')[use_column]) )

    epoch_points = np.linspace(epoch-1,epoch,len(losses), endpoint=False)
    return [epoch_points, losses]

def make_test_train_plot(epochs, test, train, label, epochs_train, ylabel):
    plt.ylabel(ylabel)
    test_plot = plt.plot(epochs, test, marker="o", label=label)
    plt.plot(epochs_train, train, linestyle="-", color=test_plot[0].get_color(), alpha=0.5, lw=0.6, label="")
    return test_plot

def generate_data_for_plots(data_dict):
    #array that contains n arrays (n models to plot), each containing [epoch, test value, train value, train epoch, ylabel]
    max_epoch=0
    data_for_all_plots=[]
    for i,data_dict in enumerate(dict_array):
        current_max_epoch = max(tuple(map(int, data_dict["Epoch"])))
        data_array=[]
        if max_epoch<current_max_epoch:
            max_epoch=current_max_epoch

        if "Test acc" in data_dict:
            ylabel="Accuracy (%)"
            train_epoch_data=[]
            train_acc_data=[]
            for epoch in data_dict["Epoch"]:
                e,l = make_loss_epoch(test_files[i], int(epoch))
                train_epoch_data.extend(e)
                train_acc_data.extend(l)
            data_array=[data_dict["Epoch"], data_dict["Test acc"], train_acc_data, modelnames[i], train_epoch_data, ylabel]
        else:
            ylabel="Loss"
            train_epoch_data=[]
            train_loss_data=[]
            for epoch in data_dict["Epoch"]:
                e,l = make_loss_epoch(test_files[i], int(epoch))
                train_epoch_data.extend(e)
                train_loss_data.extend(l)
            data_array=[data_dict["Epoch"], data_dict["Test loss"], train_loss_data, modelnames[i], train_epoch_data, ylabel]

        data_for_all_plots.append(data_array)
    return np.array(data_for_all_plots), max_epoch

debug=False
if debug==False:
    #Generate data from files
    dict_array = make_dicts_from_files(test_files)
    data_for_plots, max_epoch=generate_data_for_plots(dict_array)
else:
    # Epoch, test, train, label, trainepoch, ylabel
    data_for_plots, max_epoch=[[np.linspace(0,9,10), np.linspace(0,9,10)*0.1, np.linspace(0,9,10)*0.1 + 1, "Test", np.linspace(0,9,10), "Loss" ], ], 10
    data_for_plots=np.array(data_for_plots*2)

fig, ax=plt.subplots()
plt.ylabel(data_for_plots[0][5])

label_array = data_for_plots[:,3]
if len(labels_override) == len(label_array):
    label_array=labels_override
else:
    print("Custom label array does not have the proper length (",len(label_array),"). Using default labels...")

handles1=[]
#plot the data in one plot
for i,data_of_model in enumerate(data_for_plots):
    test_plot = ax.plot(data_of_model[0], data_of_model[1], marker="o")
    ax.plot(data_of_model[4], data_of_model[2], linestyle="-", color=test_plot[0].get_color(), alpha=0.5, lw=0.6)
    handle_for_legend = mlines.Line2D([], [], color=test_plot[0].get_color(), lw=3, label=label_array[i])
    handles1.append(handle_for_legend)

#lhandles, llabels = ax.get_legend_handles_labels()
legend1 = plt.legend(handles=handles1, loc=1)

test_line = mlines.Line2D([], [], color='grey', marker="o", label='Test')
train_line = mlines.Line2D([], [], color='grey', linestyle="-", alpha=0.5, lw=2, label='Train')
legend2 = plt.legend(handles=[test_line,train_line], loc="lower right")

ax.add_artist(legend1)
ax.add_artist(legend2)

plt.xlim((0.2,max_epoch))
#plt.xticks( np.linspace(1,max_epoch,max_epoch) )
plt.xticks( np.arange(0,max_epoch,10) )

plt.xlabel(xlabel)
plt.title(title)
plt.grid(True)
plt.show()

<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright (C) 2006-2015  Music Technology Group - Universitat Pompeu Fabra
#
# This file is part of Essentia
#
# Essentia is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation (FSF), either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the Affero GNU General Public License
# version 3 along with this program. If not, see http://www.gnu.org/licenses/

from essentia_test import *
import math

import essentia
import essentia.streaming as es
import essentia.standard as std

def cutFrames(params, input = range(100)):

    if not 'validFrameThresholdRatio' in params:
      params['validFrameThresholdRatio'] = 0
    framegen = std.FrameGenerator(input,
                                frameSize = params['frameSize'],
                                hopSize = params['hopSize'],
                                validFrameThresholdRatio = params['validFrameThresholdRatio'],
                                startFromZero = params['startFromZero'])

    return [ frame for frame in framegen ]

def cleaningSineTracks(freqsTotal, minFrames):

  nFrames = freqsTotal.shape[0];
  begTrack = 0;
  freqsClean = freqsTotal.copy()

  if (nFrames > 0 ):

    f = 0;
    nTracks = freqsTotal.shape[1]# we assume all frames have a fix number of tracks

    for t in range (nTracks):

      f = 0;
      begTrack = f;

      while (f < nFrames-1):

        #// check if f is begin of track
        if (freqsClean[f][t] <= 0 and freqsClean[f+1][t] > 0 ):
          begTrack = f+1;

        # clean track if shorter than min duration
        if ((freqsClean[f][t] > 0 and freqsClean[f+1][t] <= 0 ) and ( (f - begTrack) < minFrames)):
          for i in range(begTrack, f+1):
            freqsClean[i][t] = 0;

        f+=1;

  return freqsClean

def analSineModelStreaming(params, signal):

    #out = numpy.array(0)
    pool = essentia.Pool()
    fcut = es.FrameCutter(frameSize = params['frameSize'], hopSize = params['hopSize'], startFromZero =  False);
    w = es.Windowing(type = "hann");
    fft = es.FFT(size = params['frameSize']);
    smanal = es.SineModelAnal(sampleRate = params['sampleRate'], maxnSines = params['maxnSines'], magnitudeThreshold = params['magnitudeThreshold'], freqDevOffset = params['freqDevOffset'], freqDevSlope = params['freqDevSlope'])

    # add half window of zeros to input signal to reach same ooutput length
    signal  = numpy.append(signal, zeros(params['frameSize']/2))
    insignal = VectorInput (signal)
    insignal.data >> fcut.signal
    fcut.frame >> w.frame
    w.frame >> fft.frame
    fft.fft >> smanal.fft
    smanal.magnitudes >> (pool, 'magnitudes')
    smanal.frequencies >> (pool, 'frequencies')
    smanal.phases >> (pool, 'phases')

    essentia.run(insignal)

    # remove first half window frames
    mags = pool['magnitudes']
    freqs = pool['frequencies']
    phases = pool['phases']

    # remove short tracks
    minFrames = int( params['minSineDur'] * params['sampleRate'] / params['hopSize']);
    freqsClean = cleaningSineTracks(freqs, minFrames)
    pool['frequencies'].data = freqsClean

    return mags, freqsClean, phases

def analsynthSineModelStreaming(params, signal):

    out = numpy.array(0)

    pool = essentia.Pool()
    fcut = es.FrameCutter(frameSize = params['frameSize'], hopSize = params['hopSize'], startFromZero =  False);
    w = es.Windowing(type = "blackmanharris92");
    fft = es.FFT(size = params['frameSize']);
    smanal = es.SineModelAnal(sampleRate = params['sampleRate'], maxnSines = params['maxnSines'], magnitudeThreshold = params['magnitudeThreshold'], freqDevOffset = params['freqDevOffset'], freqDevSlope = params['freqDevSlope'])
    smsyn = es.SineModelSynth(sampleRate = params['sampleRate'], fftSize = params['frameSize'], hopSize = params['hopSize'])
    ifft = es.IFFT(size = params['frameSize']);
    overl = es.OverlapAdd (frameSize = params['frameSize'], hopSize = params['hopSize']);

    # add half window of zeros to input signal to reach same ooutput length
    signal  = numpy.append(signal, zeros(params['frameSize']/2))
    insignal = VectorInput (signal)
    # analysis
    insignal.data >> fcut.signal
    fcut.frame >> w.frame
    w.frame >> fft.frame
    fft.fft >> smanal.fft
    smanal.magnitudes >> (pool, 'magnitudes')
    smanal.frequencies >> (pool, 'frequencies')
    smanal.phases >> (pool, 'phases')
    # synthesis
    smanal.magnitudes >> smsyn.magnitudes
    smanal.frequencies >> smsyn.frequencies
    smanal.phases >> smsyn.phases
    smsyn.fft >> ifft.fft
    #    fft.fft >> ifft.fft # debug
    ifft.frame >> overl.frame
    overl.signal >> (pool, 'audio')

    essentia.run(insignal)

    # remove short tracks
    freqs = pool['frequencies']
    minFrames = int( params['minSineDur'] * params['sampleRate'] / params['hopSize']);
    freqsClean = cleaningSineTracks(freqs, minFrames)
    pool['frequencies'].data = freqsClean

    # remove first half window frames
    outaudio = pool['audio']
    outaudio = outaudio [2*params['hopSize']:]

    return outaudio, pool

#-------------------------------------

class TestSineModel(TestCase):

    params = { 'frameSize': 2048, 'hopSize': 512, 'startFromZero': False, 'sampleRate': 44100,'maxnSines': 100,'magnitudeThreshold': -74,'minSineDur': 0.02,'freqDevOffset': 10, 'freqDevSlope': 0.001}

    # emprical value from a test signal x(t) = (0.5*sine(110*t)).
    # Assert diffference is computed as a relative value (abs(*y - *x)/abs(*y))
    # for audio signals another function measuring absolute value could be implemented instead.
    precision = 0.2

    def testZero(self):

        # generate test signal
        signalSize = 10 * self.params['frameSize']
        signal = zeros(signalSize)

        [mags, freqs, phases] = analSineModelStreaming(self.params, signal)

        # compare
        zerofreqs = numpy.zeros(freqs.shape)
        self.assertAlmostEqualMatrix(freqs, zerofreqs)

    def testWhiteNoise(self):
        from random import random
        # generate test signal
        signalSize = 10 * self.params['frameSize']
        signal = array([2*(random()-0.5)*i for i in ones(signalSize)])

        # for white noise test set sine minimum duration to 50ms, and min threshold of -24dB
        self.params['minSineDur'] = 0.05
        self.params['magnitudeThreshold']= -20

        [mags, freqs, phases]  = analSineModelStreaming(self.params, signal)

        # compare
        zerofreqs = numpy.zeros(freqs.shape)
        self.assertAlmostEqualMatrix(freqs, zerofreqs)

    def testRegression(self):

        # generate test signal: sine 110Hz @44100kHz
        signalSize = 10 * self.params['frameSize']
        signal = 0.5 * numpy.sin( (array(range(signalSize))/self.params['sampleRate']) * 110 * 2*math.pi)

        outsignal,pool = analsynthSineModelStreaming(self.params, signal)

        outsignal = outsignal[:signalSize] # cut to durations of input and output signal

        # compare without half-window bounds to avoid windowing effect
        halfwin = (self.params['frameSize']/2)

        numpy.savetxt('sine.txt',signal[halfwin:-halfwin])
        numpy.savetxt('sine_out.txt',outsignal[halfwin:-halfwin])

        # computing max difference between waveforms
        diffference = numpy.max(abs(outsignal[halfwin:-halfwin] - signal[halfwin:-halfwin]))
        ref_diff = 0.00616494

        self.assertAlmostEqual(diffference/ref_diff, 1, 1e-6)

suite = allTests(TestSineModel)

if __name__ == '__main__':
    TextTestRunner(verbosity=2).run(suite)

<commit_msg>unit test_sinemodel synthesis. Changing assert for regression test for fixed precision (40dB). Pneding double check. Test still fails.<commit_after>#!/usr/bin/env python

# Copyright (C) 2006-2015  Music Technology Group - Universitat Pompeu Fabra
#
# This file is part of Essentia
#
# Essentia is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation (FSF), either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the Affero GNU General Public License
# version 3 along with this program. If not, see http://www.gnu.org/licenses/

from essentia_test import *
import math

import essentia
import essentia.streaming as es
import essentia.standard as std

def cutFrames(params, input = range(100)):

    if not 'validFrameThresholdRatio' in params:
      params['validFrameThresholdRatio'] = 0
    framegen = std.FrameGenerator(input,
                                frameSize = params['frameSize'],
                                hopSize = params['hopSize'],
                                validFrameThresholdRatio = params['validFrameThresholdRatio'],
                                startFromZero = params['startFromZero'])

    return [ frame for frame in framegen ]

def cleaningSineTracks(freqsTotal, minFrames):

  nFrames = freqsTotal.shape[0];
  begTrack = 0;
  freqsClean = freqsTotal.copy()

  if (nFrames > 0 ):

    f = 0;
    nTracks = freqsTotal.shape[1]# we assume all frames have a fix number of tracks

    for t in range (nTracks):

      f = 0;
      begTrack = f;

      while (f < nFrames-1):

        #// check if f is begin of track
        if (freqsClean[f][t] <= 0 and freqsClean[f+1][t] > 0 ):
          begTrack = f+1;

        # clean track if shorter than min duration
        if ((freqsClean[f][t] > 0 and freqsClean[f+1][t] <= 0 ) and ( (f - begTrack) < minFrames)):
          for i in range(begTrack, f+1):
            freqsClean[i][t] = 0;

        f+=1;

  return freqsClean

def analSineModelStreaming(params, signal):

    #out = numpy.array(0)
    pool = essentia.Pool()
    fcut = es.FrameCutter(frameSize = params['frameSize'], hopSize = params['hopSize'], startFromZero =  False);
    w = es.Windowing(type = "hann");
    fft = es.FFT(size = params['frameSize']);
    smanal = es.SineModelAnal(sampleRate = params['sampleRate'], maxnSines = params['maxnSines'], magnitudeThreshold = params['magnitudeThreshold'], freqDevOffset = params['freqDevOffset'], freqDevSlope = params['freqDevSlope'])

    # add half window of zeros to input signal to reach same ooutput length
    signal  = numpy.append(signal, zeros(params['frameSize']/2))
    insignal = VectorInput (signal)
    insignal.data >> fcut.signal
    fcut.frame >> w.frame
    w.frame >> fft.frame
    fft.fft >> smanal.fft
    smanal.magnitudes >> (pool, 'magnitudes')
    smanal.frequencies >> (pool, 'frequencies')
    smanal.phases >> (pool, 'phases')

    essentia.run(insignal)

    # remove first half window frames
    mags = pool['magnitudes']
    freqs = pool['frequencies']
    phases = pool['phases']

    # remove short tracks
    minFrames = int( params['minSineDur'] * params['sampleRate'] / params['hopSize']);
    freqsClean = cleaningSineTracks(freqs, minFrames)
    pool['frequencies'].data = freqsClean

    return mags, freqsClean, phases

def analsynthSineModelStreaming(params, signal):

    out = numpy.array(0)

    pool = essentia.Pool()
    fcut = es.FrameCutter(frameSize = params['frameSize'], hopSize = params['hopSize'], startFromZero =  False);
    w = es.Windowing(type = "blackmanharris92");
    fft = es.FFT(size = params['frameSize']);
    smanal = es.SineModelAnal(sampleRate = params['sampleRate'], maxnSines = params['maxnSines'], magnitudeThreshold = params['magnitudeThreshold'], freqDevOffset = params['freqDevOffset'], freqDevSlope = params['freqDevSlope'])
    smsyn = es.SineModelSynth(sampleRate = params['sampleRate'], fftSize = params['frameSize'], hopSize = params['hopSize'])
    ifft = es.IFFT(size = params['frameSize']);
    overl = es.OverlapAdd (frameSize = params['frameSize'], hopSize = params['hopSize']);

    # add half window of zeros to input signal to reach same ooutput length
    signal  = numpy.append(signal, zeros(params['frameSize']/2))
    insignal = VectorInput (signal)
    # analysis
    insignal.data >> fcut.signal
    fcut.frame >> w.frame
    w.frame >> fft.frame
    fft.fft >> smanal.fft
    smanal.magnitudes >> (pool, 'magnitudes')
    smanal.frequencies >> (pool, 'frequencies')
    smanal.phases >> (pool, 'phases')
    # synthesis
    smanal.magnitudes >> smsyn.magnitudes
    smanal.frequencies >> smsyn.frequencies
    smanal.phases >> smsyn.phases
    smsyn.fft >> ifft.fft
    #    fft.fft >> ifft.fft # debug
    ifft.frame >> overl.frame
    overl.signal >> (pool, 'audio')

    essentia.run(insignal)

    # remove short tracks
    freqs = pool['frequencies']
    minFrames = int( params['minSineDur'] * params['sampleRate'] / params['hopSize']);
    freqsClean = cleaningSineTracks(freqs, minFrames)
    pool['frequencies'].data = freqsClean

    # remove first half window frames
    outaudio = pool['audio']
    outaudio = outaudio [2*params['hopSize']:]

    return outaudio, pool

#-------------------------------------

class TestSineModel(TestCase):

    params = { 'frameSize': 2048, 'hopSize': 512, 'startFromZero': False, 'sampleRate': 44100,'maxnSines': 100,'magnitudeThreshold': -74,'minSineDur': 0.02,'freqDevOffset': 10, 'freqDevSlope': 0.001}

    precisiondB = -40. # -60dB of allowed noise floor.
    precisionDigits = int(-numpy.round(precisiondB/20.))
    print precisionDigits

    def testZero(self):

        # generate test signal
        signalSize = 10 * self.params['frameSize']
        signal = zeros(signalSize)

        [mags, freqs, phases] = analSineModelStreaming(self.params, signal)

        # compare
        zerofreqs = numpy.zeros(freqs.shape)
        self.assertAlmostEqualMatrix(freqs, zerofreqs)

    def testWhiteNoise(self):
        from random import random
        # generate test signal
        signalSize = 10 * self.params['frameSize']
        signal = array([2*(random()-0.5)*i for i in ones(signalSize)])

        # for white noise test set sine minimum duration to 50ms, and min threshold of -20dB
        self.params['minSineDur'] = 0.05
        self.params['magnitudeThreshold']= -20

        [mags, freqs, phases]  = analSineModelStreaming(self.params, signal)

        # compare
        zerofreqs = numpy.zeros(freqs.shape)
        self.assertAlmostEqualMatrix(freqs, zerofreqs)

    def testRegression(self):

        # generate test signal: sine 110Hz @44100kHz
        signalSize = 10 * self.params['frameSize']
        signal = .5 * numpy.sin( (array(range(signalSize))/self.params['sampleRate']) * 110 * 2*math.pi)

        outsignal,pool = analsynthSineModelStreaming(self.params, signal)

        outsignal = outsignal[:signalSize] # cut to durations of input and output signal

        # compare without half-window bounds to avoid windowing effect
        halfwin = (self.params['frameSize']/2)

        numpy.savetxt('sine.txt',signal[halfwin:-halfwin])
        numpy.savetxt('sine_out.txt',outsignal[halfwin:-halfwin])

        # computing max difference between waveforms
#        diffference = numpy.max(abs(outsignal[halfwin:-halfwin] - signal[halfwin:-halfwin]))
#        print 'diffference', diffference

        self.assertAlmostEqualVectorFixedPrecision(outsignal[halfwin:-halfwin], signal[halfwin:-halfwin], self.precisionDigits)

suite = allTests(TestSineModel)

if __name__ == '__main__':
    TextTestRunner(verbosity=2).run(suite)

<|endoftext|>


<commit_before>from odoo import models, _
from odoo.exceptions import UserError
from odoo.addons.l10n_it_account.tools.account_tools import encode_for_export
from odoo.addons.l10n_it_fatturapa.bindings.fatturapa import (
    IdFiscaleType,
    AnagraficaType,
    IndirizzoType
)

class WizardExportFatturapa(models.TransientModel):
    _inherit = "wizard.export.fatturapa"

    def exportInvoiceXML(
        self, company, partner, invoice_ids, attach=False, context=None
    ):
        if context is None:
            context = {}
        invoices = self.env["account.invoice"].browse(invoice_ids)
        invoices_with_rc = False
        invoices_without_rc = False
        for invoice in invoices:
            if invoice.rc_purchase_invoice_id:
                invoices_with_rc = True
            else:
                invoices_without_rc = True
        if invoices_with_rc and invoices_without_rc:
            raise UserError(_(
                "Selected invoices are both with and without reverse charge. You "
                "should selected a smaller set of invoices"))
        rc_suppliers = invoices.mapped("rc_purchase_invoice_id.partner_id")
        if len(rc_suppliers) > 1:
            raise UserError(_(
                "Selected reverse charge invoices have different suppliers. Please "
                "select invoices with same supplier"))
        if rc_suppliers:
            context['rc_supplier'] = rc_suppliers[0]
        return super(WizardExportFatturapa, self).exportInvoiceXML(
            company, partner, invoice_ids, attach, context)

    def _setDatiAnagraficiCedente(self, CedentePrestatore, company):
        res = super(WizardExportFatturapa, self)._setDatiAnagraficiCedente(
            CedentePrestatore, company)
        if self.env.context.get("rc_supplier"):
            partner = self.env.context["rc_supplier"]
            CedentePrestatore.DatiAnagrafici.CodiceFiscale = None
            if partner.vat:
                CedentePrestatore.DatiAnagrafici.IdFiscaleIVA = IdFiscaleType(
                    IdPaese=partner.vat[0:2], IdCodice=partner.vat[2:])
            elif partner.country_id.code and partner.country_id.code != 'IT':
                CedentePrestatore.DatiAnagrafici.IdFiscaleIVA = IdFiscaleType(
                    IdPaese=partner.country_id.code, IdCodice='99999999999')
            else:
                raise UserError(
                    _("Impossible to set IdFiscaleIVA for %s") % partner.display_name)
            CedentePrestatore.DatiAnagrafici.Anagrafica = AnagraficaType(
                Denominazione=partner.name)
        return res

    def _setSedeCedente(self, CedentePrestatore, company):
        res = super(WizardExportFatturapa, self)._setSedeCedente(
            CedentePrestatore, company)
        if self.env.context.get("rc_supplier"):
            partner = self.env.context["rc_supplier"]
            if not partner.street:
                raise UserError(
                    _('Partner %s, Street is not set.') % partner.display_name)
            if not partner.city:
                raise UserError(
                    _('Partner %s, City is not set.') % partner.display_name)
            if not partner.country_id:
                raise UserError(
                    _('Partner %s, Country is not set.') % partner.display_name)
            if partner.codice_destinatario == 'XXXXXXX':
                CedentePrestatore.Sede = (
                    IndirizzoType(
                        Indirizzo=encode_for_export(partner.street, 60),
                        CAP='00000',
                        Comune=encode_for_export(partner.city, 60),
                        Provincia='EE',
                        Nazione=partner.country_id.code))
            else:
                if not partner.zip:
                    raise UserError(
                        _('Partner %s, ZIP is not set.') % partner.display_name)
                CedentePrestatore.Sede = IndirizzoType(
                    Indirizzo=encode_for_export(partner.street, 60),
                    CAP=partner.zip,
                    Comune=encode_for_export(partner.city, 60),
                    Nazione=partner.country_id.code)
                if partner.state_id:
                    CedentePrestatore.Sede.Provincia = partner.state_id.code
        return res

    def _setStabileOrganizzazione(self, CedentePrestatore, company):
        res = super(WizardExportFatturapa, self)._setStabileOrganizzazione(
            CedentePrestatore, company)
        if self.env.context.get("rc_supplier"):
            CedentePrestatore.StabileOrganizzazione = None
        return res

    def _setRea(self, CedentePrestatore, company):
        res = super(WizardExportFatturapa, self)._setRea(CedentePrestatore, company)
        if self.env.context.get("rc_supplier"):
            CedentePrestatore.IscrizioneREA = None
        return res

    def _setContatti(self, CedentePrestatore, company):
        res = super(WizardExportFatturapa, self)._setContatti(
            CedentePrestatore, company)
        if self.env.context.get("rc_supplier"):
            CedentePrestatore.Contatti = None
        return res

    def _setPubAdministrationRef(self, CedentePrestatore, company):
        res = super(WizardExportFatturapa, self)._setPubAdministrationRef(
            CedentePrestatore, company)
        if self.env.context.get("rc_supplier"):
            CedentePrestatore.RiferimentoAmministrazione = None
        return res

    def setDatiGeneraliDocumento(self, invoice, body):
        res = super(WizardExportFatturapa, self).setDatiGeneraliDocumento(
            invoice, body)
        if (
            invoice.rc_purchase_invoice_id and
            invoice.rc_purchase_invoice_id.fiscal_position_id and
            invoice.rc_purchase_invoice_id.fiscal_position_id.rc_type_id and
            invoice.rc_purchase_invoice_id.fiscal_position_id.rc_type_id.
                fiscal_document_type_id
        ):
            body.DatiGenerali.DatiGeneraliDocumento.TipoDocumento = (
                invoice.rc_purchase_invoice_id.fiscal_position_id.rc_type_id.
                fiscal_document_type_id.code
            )
        if invoice.type in ['out_refund', 'in_refund'] \
                and invoice.fiscal_document_type_id.code not in ['TD04', 'TD08']:
            body.DatiGenerali.DatiGeneraliDocumento.ImportoTotaleDocumento = \
                - body.DatiGenerali.DatiGeneraliDocumento.ImportoTotaleDocumento
        return res

    def setDettaglioLinea(
        self, line_no, line, body, price_precision, uom_precision
    ):
        DettaglioLinea = super(WizardExportFatturapa, self).setDettaglioLinea(
            line_no, line, body, price_precision, uom_precision)
        if line.invoice_id.type in ['out_refund', 'in_refund'] and \
                line.invoice_id.fiscal_document_type_id.code not in ['TD04', 'TD08']:
            DettaglioLinea.PrezzoUnitario = - DettaglioLinea.PrezzoUnitario
            DettaglioLinea.PrezzoTotale = - DettaglioLinea.PrezzoTotale
        return DettaglioLinea

    def setDatiRiepilogo(self, invoice, body):
        super(WizardExportFatturapa, self).setDatiRiepilogo(invoice, body)
        for DatiRiepilogo in body.DatiBeniServizi.DatiRiepilogo:
            if invoice.type in ['out_refund', 'in_refund'] \
                    and invoice.fiscal_document_type_id.code not in ['TD04', 'TD08']:
                DatiRiepilogo.ImponibileImporto = - DatiRiepilogo.ImponibileImporto
                DatiRiepilogo.Imposta = - DatiRiepilogo.Imposta
        return True

    def setDatiPagamento(self, invoice, body):
        super(WizardExportFatturapa, self).setDatiPagamento(invoice, body)
        for DatiPagamento in body.DatiPagamento:
            if invoice.type in ['out_refund', 'in_refund'] \
                    and invoice.fiscal_document_type_id.code not in ['TD04', 'TD08']:
                DatiPagamento.ImportoPagamento = - DatiPagamento.ImportoPagamento
        return True
<commit_msg>[FIX] check ImportoPagamento exists<commit_after>from odoo import models, _
from odoo.exceptions import UserError
from odoo.addons.l10n_it_account.tools.account_tools import encode_for_export
from odoo.addons.l10n_it_fatturapa.bindings.fatturapa import (
    IdFiscaleType,
    AnagraficaType,
    IndirizzoType
)

class WizardExportFatturapa(models.TransientModel):
    _inherit = "wizard.export.fatturapa"

    def exportInvoiceXML(
        self, company, partner, invoice_ids, attach=False, context=None
    ):
        if context is None:
            context = {}
        invoices = self.env["account.invoice"].browse(invoice_ids)
        invoices_with_rc = False
        invoices_without_rc = False
        for invoice in invoices:
            if invoice.rc_purchase_invoice_id:
                invoices_with_rc = True
            else:
                invoices_without_rc = True
        if invoices_with_rc and invoices_without_rc:
            raise UserError(_(
                "Selected invoices are both with and without reverse charge. You "
                "should selected a smaller set of invoices"))
        rc_suppliers = invoices.mapped("rc_purchase_invoice_id.partner_id")
        if len(rc_suppliers) > 1:
            raise UserError(_(
                "Selected reverse charge invoices have different suppliers. Please "
                "select invoices with same supplier"))
        if rc_suppliers:
            context['rc_supplier'] = rc_suppliers[0]
        return super(WizardExportFatturapa, self).exportInvoiceXML(
            company, partner, invoice_ids, attach, context)

    def _setDatiAnagraficiCedente(self, CedentePrestatore, company):
        res = super(WizardExportFatturapa, self)._setDatiAnagraficiCedente(
            CedentePrestatore, company)
        if self.env.context.get("rc_supplier"):
            partner = self.env.context["rc_supplier"]
            CedentePrestatore.DatiAnagrafici.CodiceFiscale = None
            if partner.vat:
                CedentePrestatore.DatiAnagrafici.IdFiscaleIVA = IdFiscaleType(
                    IdPaese=partner.vat[0:2], IdCodice=partner.vat[2:])
            elif partner.country_id.code and partner.country_id.code != 'IT':
                CedentePrestatore.DatiAnagrafici.IdFiscaleIVA = IdFiscaleType(
                    IdPaese=partner.country_id.code, IdCodice='99999999999')
            else:
                raise UserError(
                    _("Impossible to set IdFiscaleIVA for %s") % partner.display_name)
            CedentePrestatore.DatiAnagrafici.Anagrafica = AnagraficaType(
                Denominazione=partner.name)
        return res

    def _setSedeCedente(self, CedentePrestatore, company):
        res = super(WizardExportFatturapa, self)._setSedeCedente(
            CedentePrestatore, company)
        if self.env.context.get("rc_supplier"):
            partner = self.env.context["rc_supplier"]
            if not partner.street:
                raise UserError(
                    _('Partner %s, Street is not set.') % partner.display_name)
            if not partner.city:
                raise UserError(
                    _('Partner %s, City is not set.') % partner.display_name)
            if not partner.country_id:
                raise UserError(
                    _('Partner %s, Country is not set.') % partner.display_name)
            if partner.codice_destinatario == 'XXXXXXX':
                CedentePrestatore.Sede = (
                    IndirizzoType(
                        Indirizzo=encode_for_export(partner.street, 60),
                        CAP='00000',
                        Comune=encode_for_export(partner.city, 60),
                        Provincia='EE',
                        Nazione=partner.country_id.code))
            else:
                if not partner.zip:
                    raise UserError(
                        _('Partner %s, ZIP is not set.') % partner.display_name)
                CedentePrestatore.Sede = IndirizzoType(
                    Indirizzo=encode_for_export(partner.street, 60),
                    CAP=partner.zip,
                    Comune=encode_for_export(partner.city, 60),
                    Nazione=partner.country_id.code)
                if partner.state_id:
                    CedentePrestatore.Sede.Provincia = partner.state_id.code
        return res

    def _setStabileOrganizzazione(self, CedentePrestatore, company):
        res = super(WizardExportFatturapa, self)._setStabileOrganizzazione(
            CedentePrestatore, company)
        if self.env.context.get("rc_supplier"):
            CedentePrestatore.StabileOrganizzazione = None
        return res

    def _setRea(self, CedentePrestatore, company):
        res = super(WizardExportFatturapa, self)._setRea(CedentePrestatore, company)
        if self.env.context.get("rc_supplier"):
            CedentePrestatore.IscrizioneREA = None
        return res

    def _setContatti(self, CedentePrestatore, company):
        res = super(WizardExportFatturapa, self)._setContatti(
            CedentePrestatore, company)
        if self.env.context.get("rc_supplier"):
            CedentePrestatore.Contatti = None
        return res

    def _setPubAdministrationRef(self, CedentePrestatore, company):
        res = super(WizardExportFatturapa, self)._setPubAdministrationRef(
            CedentePrestatore, company)
        if self.env.context.get("rc_supplier"):
            CedentePrestatore.RiferimentoAmministrazione = None
        return res

    def setDatiGeneraliDocumento(self, invoice, body):
        res = super(WizardExportFatturapa, self).setDatiGeneraliDocumento(
            invoice, body)
        if (
            invoice.rc_purchase_invoice_id and
            invoice.rc_purchase_invoice_id.fiscal_position_id and
            invoice.rc_purchase_invoice_id.fiscal_position_id.rc_type_id and
            invoice.rc_purchase_invoice_id.fiscal_position_id.rc_type_id.
                fiscal_document_type_id
        ):
            body.DatiGenerali.DatiGeneraliDocumento.TipoDocumento = (
                invoice.rc_purchase_invoice_id.fiscal_position_id.rc_type_id.
                fiscal_document_type_id.code
            )
        if invoice.type in ['out_refund', 'in_refund'] \
                and invoice.fiscal_document_type_id.code not in ['TD04', 'TD08']:
            body.DatiGenerali.DatiGeneraliDocumento.ImportoTotaleDocumento = \
                - body.DatiGenerali.DatiGeneraliDocumento.ImportoTotaleDocumento
        return res

    def setDettaglioLinea(
        self, line_no, line, body, price_precision, uom_precision
    ):
        DettaglioLinea = super(WizardExportFatturapa, self).setDettaglioLinea(
            line_no, line, body, price_precision, uom_precision)
        if line.invoice_id.type in ['out_refund', 'in_refund'] and \
                line.invoice_id.fiscal_document_type_id.code not in ['TD04', 'TD08']:
            DettaglioLinea.PrezzoUnitario = - DettaglioLinea.PrezzoUnitario
            DettaglioLinea.PrezzoTotale = - DettaglioLinea.PrezzoTotale
        return DettaglioLinea

    def setDatiRiepilogo(self, invoice, body):
        super(WizardExportFatturapa, self).setDatiRiepilogo(invoice, body)
        for DatiRiepilogo in body.DatiBeniServizi.DatiRiepilogo:
            if invoice.type in ['out_refund', 'in_refund'] \
                    and invoice.fiscal_document_type_id.code not in ['TD04', 'TD08']:
                DatiRiepilogo.ImponibileImporto = - DatiRiepilogo.ImponibileImporto
                DatiRiepilogo.Imposta = - DatiRiepilogo.Imposta
        return True

    def setDatiPagamento(self, invoice, body):
        super(WizardExportFatturapa, self).setDatiPagamento(invoice, body)
        for DatiPagamento in body.DatiPagamento:
            if invoice.type in ['out_refund', 'in_refund'] \
                    and invoice.fiscal_document_type_id.code not in ['TD04', 'TD08']\
                    and DatiPagamento.ImportoPagamento:
                DatiPagamento.ImportoPagamento = - DatiPagamento.ImportoPagamento
        return True
<|endoftext|>


<commit_before>import pickle

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_blobs

from neupy import algorithms
from neupy.algorithms.competitive.growing_neural_gas import (
    NeuralGasGraph, NeuronNode)

from base import BaseTestCase

class GrowingNeuralGasTestCase(BaseTestCase):
    debug_plot = False

    def setUp(self):
        super(BaseTestCase, self).setUp()
        self.data, _ = make_blobs(
            n_samples=1000,
            n_features=2,
            centers=2,
            cluster_std=0.4,
            random_state=0
        )

    def make_plot(self, data, network):
        plt.figure(figsize=(5, 6))
        plt.scatter(*data.T, alpha=0.4)

        for node_1, node_2 in network.graph.edges:
            weights = np.concatenate([node_1.weight, node_2.weight])
            line, = plt.plot(*weights.T, color='black')
            plt.setp(line, color='black')

        plt.show()

    def test_neural_gas_exceptions(self):
        gng = algorithms.GrowingNeuralGas(n_inputs=2)

        with self.assertRaisesRegexp(ValueError, "more than 2 dimensions"):
            gng.train(np.random.random((10, 2, 1)), epochs=1)

        with self.assertRaisesRegexp(ValueError, "have 2 features, but got 1"):
            gng.train(np.random.random((10, 1)), epochs=1)

    def test_simple_neural_gas(self):
        gng = algorithms.GrowingNeuralGas(
            n_inputs=2,
            n_start_nodes=2,
            shuffle_data=True,

            step=0.2,
            neighbour_step=0.05,
            max_edge_age=10,
            n_iter_before_neuron_added=50,

            error_decay_rate=0.995,
            after_split_error_decay_rate=0.5,
            min_distance_for_update=0.1,
            max_nodes=100,
            verbose=False,
        )
        data = self.data
        gng.train(data, epochs=10)

        # Add one useless node to make sure that it
        # would be deleted at the end
        first_node = gng.graph.nodes[0]
        new_node = algorithms.NeuronNode(weight=np.array([[1000., 1000.]]))
        new_node.useless_node = True

        gng.graph.add_node(new_node)
        gng.graph.add_edge(new_node, first_node)

        # We run it twice in order to make sure that we won't
        # reset weights on the second run.
        gng.train(data, epochs=10)

        if self.debug_plot:
            self.make_plot(data, gng)

        self.assertEqual(len(gng.graph.nodes), gng.max_nodes)
        self.assertAlmostEqual(gng.training_errors[-1], 0.09, places=2)

        useless_node_present = any(
            getattr(node, 'useless_node', False) for node in gng.graph.nodes)

        self.assertFalse(useless_node_present)

        with self.assertRaises(NotImplementedError):
            gng.predict(data)

        # Check that we can stop training in case if we don't get any updates
        before_epochs = gng.last_epoch
        gng.min_distance_for_update = 10
        gng.train(data, epochs=10)
        self.assertEqual(before_epochs + 1, gng.last_epoch)

    def test_gng_storage(self):
        gng = algorithms.GrowingNeuralGas(
            n_inputs=2,
            step=0.2,
            verbose=False,
            n_iter_before_neuron_added=5 * len(self.data),
        )
        gng.train(self.data, epochs=10)
        self.assertEqual(gng.graph.n_nodes, 4)

        gng_recovered = pickle.loads(pickle.dumps(gng))
        self.assertEqual(gng_recovered.graph.n_nodes, 4)

        gng_recovered.train(self.data, epochs=10)
        self.assertEqual(gng_recovered.graph.n_nodes, 6)

    def test_gng_repr(self):
        gng = algorithms.GrowingNeuralGas(
            n_inputs=2,
            step=0.2,
            verbose=True,
        )
        gng.train(self.data, epochs=10)
        self.assertEqual(
            str(gng.graph),
            "<NeuralGasGraph n_nodes=12, n_edges=19>",
        )

class NeuralGasGraphTestCase(BaseTestCase):
    def test_simple_graph(self):
        graph = NeuralGasGraph()

        node_a = NeuronNode(1)
        node_b = NeuronNode(2)
        node_c = NeuronNode(3)

        graph.add_node(node_a)
        graph.add_node(node_b)
        graph.add_node(node_c)

        graph.add_edge(node_a, node_b)
        graph.add_edge(node_b, node_c)

        self.assertEqual(graph.n_nodes, 3)
        self.assertEqual(graph.n_edges, 2)

        self.assertEqual(graph.find_edge_id(node_a, node_b), (node_a, node_b))
        self.assertEqual(graph.find_edge_id(node_b, node_a), (node_a, node_b))

        error_message = "Edge between specified nodes doesn't exist"
        with self.assertRaisesRegexp(ValueError, error_message):
            graph.find_edge_id(node_a, node_c)

        graph.remove_edge(node_c, node_b)

        self.assertEqual(graph.n_nodes, 3)
        self.assertEqual(graph.n_edges, 1)

        graph.remove_node(node_c)

        self.assertEqual(graph.n_nodes, 2)
        self.assertEqual(graph.n_edges, 1)

        error_message = (
            "Cannot remove node, because it's "
            "connected to 1 other node\(s\)"
        )
        with self.assertRaisesRegexp(ValueError, error_message):
            graph.remove_node(node_a)

    def test_node_repr(self):
        node_a = NeuronNode(1)
        self.assertEqual(str(node_a), "<NeuronNode error=0.0>")

        node_a.error = 1
        self.assertEqual(str(node_a), "<NeuronNode error=1.0>")

        node_a.error = 3.141592654
        self.assertEqual(str(node_a), "<NeuronNode error=3.141593>")
<commit_msg>fixed neural gas and added more tests and checks<commit_after>import pickle

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_blobs

from neupy import algorithms
from neupy.algorithms.competitive.growing_neural_gas import (
    NeuralGasGraph, NeuronNode)

from base import BaseTestCase

class GrowingNeuralGasTestCase(BaseTestCase):
    debug_plot = False

    def setUp(self):
        super(BaseTestCase, self).setUp()
        self.data, _ = make_blobs(
            n_samples=1000,
            n_features=2,
            centers=2,
            cluster_std=0.4,
            random_state=0
        )

    def make_plot(self, data, network):
        plt.figure(figsize=(5, 6))
        plt.scatter(*data.T, alpha=0.4)

        for node_1, node_2 in network.graph.edges:
            weights = np.concatenate([node_1.weight, node_2.weight])
            line, = plt.plot(*weights.T, color='black')
            plt.setp(line, color='black')

        plt.show()

    def test_neural_gas_exceptions(self):
        gng = algorithms.GrowingNeuralGas(n_inputs=2)

        with self.assertRaisesRegexp(ValueError, "more than 2 dimensions"):
            gng.train(np.random.random((10, 2, 1)), epochs=1)

        with self.assertRaisesRegexp(ValueError, "have 2 features, but got 1"):
            gng.train(np.random.random((10, 1)), epochs=1)

    def test_simple_neural_gas(self):
        gng = algorithms.GrowingNeuralGas(
            n_inputs=2,
            n_start_nodes=2,
            shuffle_data=True,

            step=0.2,
            neighbour_step=0.05,
            max_edge_age=10,
            n_iter_before_neuron_added=50,

            error_decay_rate=0.995,
            after_split_error_decay_rate=0.5,
            min_distance_for_update=0.1,
            max_nodes=100,
            verbose=False,
        )
        data = self.data
        gng.train(data, epochs=10)

        # Add one useless node to make sure that it
        # would be deleted at the end
        first_node = gng.graph.nodes[0]
        new_node = algorithms.NeuronNode(weight=np.array([[1000., 1000.]]))
        new_node.useless_node = True

        gng.graph.add_node(new_node)
        gng.graph.add_edge(new_node, first_node)

        # We run it twice in order to make sure that we won't
        # reset weights on the second run.
        gng.train(data, epochs=10)

        if self.debug_plot:
            self.make_plot(data, gng)

        self.assertEqual(len(gng.graph.nodes), gng.max_nodes)
        self.assertAlmostEqual(gng.training_errors[-1], 0.09, places=2)

        useless_node_present = any(
            getattr(node, 'useless_node', False) for node in gng.graph.nodes)

        self.assertFalse(useless_node_present)

        with self.assertRaises(NotImplementedError):
            gng.predict(data)

        # Check that we can stop training in case if we don't get any updates
        before_epochs = gng.last_epoch
        gng.min_distance_for_update = 10
        gng.train(data, epochs=10)
        self.assertEqual(before_epochs + 1, gng.last_epoch)

    def test_gng_storage(self):
        gng = algorithms.GrowingNeuralGas(
            n_inputs=2,
            step=0.2,
            verbose=False,
            n_iter_before_neuron_added=5 * len(self.data),
        )
        gng.train(self.data, epochs=10)
        self.assertEqual(gng.graph.n_nodes, 4)

        gng_recovered = pickle.loads(pickle.dumps(gng))
        self.assertEqual(gng_recovered.graph.n_nodes, 4)

        gng_recovered.train(self.data, epochs=10)
        self.assertEqual(gng_recovered.graph.n_nodes, 6)

    def test_gng_repr(self):
        gng = algorithms.GrowingNeuralGas(
            n_inputs=2,
            step=0.2,
            verbose=False,
        )
        gng.train(self.data, epochs=10)
        self.assertEqual(
            str(gng.graph),
            "<NeuralGasGraph n_nodes=12, n_edges=19>",
        )

class NeuralGasGraphTestCase(BaseTestCase):
    def test_simple_graph(self):
        graph = NeuralGasGraph()

        node_a = NeuronNode(1)
        node_b = NeuronNode(2)
        node_c = NeuronNode(3)

        graph.add_node(node_a)
        graph.add_node(node_b)
        graph.add_node(node_c)

        graph.add_edge(node_a, node_b)
        graph.add_edge(node_b, node_c)

        self.assertEqual(graph.n_nodes, 3)
        self.assertEqual(graph.n_edges, 2)

        self.assertEqual(graph.find_edge_id(node_a, node_b), (node_a, node_b))
        self.assertEqual(graph.find_edge_id(node_b, node_a), (node_a, node_b))

        error_message = "Edge between specified nodes doesn't exist"
        with self.assertRaisesRegexp(ValueError, error_message):
            graph.find_edge_id(node_a, node_c)

        graph.remove_edge(node_c, node_b)

        self.assertEqual(graph.n_nodes, 3)
        self.assertEqual(graph.n_edges, 1)

        graph.remove_node(node_c)

        self.assertEqual(graph.n_nodes, 2)
        self.assertEqual(graph.n_edges, 1)

        error_message = (
            "Cannot remove node, because it's "
            "connected to 1 other node\(s\)"
        )
        with self.assertRaisesRegexp(ValueError, error_message):
            graph.remove_node(node_a)

    def test_node_repr(self):
        node_a = NeuronNode(1)
        self.assertEqual(str(node_a), "<NeuronNode error=0.0>")

        node_a.error = 1
        self.assertEqual(str(node_a), "<NeuronNode error=1.0>")

        node_a.error = 3.141592654
        self.assertEqual(str(node_a), "<NeuronNode error=3.141593>")
<|endoftext|>


<commit_before>import uuid
import pytest
from unittest.mock import patch

import werkzeug

from tests import service_json
from tests.conftest import api_user_active, platform_admin_user
from app.notify_client import NotifyAdminAPIClient

@pytest.mark.parametrize('method', [
    'put',
    'post',
    'delete'
])
@pytest.mark.parametrize('user', [
    api_user_active(str(uuid.uuid4())),
    platform_admin_user(str(uuid.uuid4()))
], ids=['api_user', 'platform_admin'])
@pytest.mark.parametrize('service', [
    service_json(active=True),
    None
], ids=['active_service', 'no_service'])
def test_active_service_can_be_modified(app_, method, user, service):
    api_client = NotifyAdminAPIClient('api_key', 'base_url', 'service_id')

    with app_.test_request_context() as request_context, app_.test_client() as client:
        client.login(user)
        request_context.service = service

        with patch.object(api_client, 'request') as request:
            ret = getattr(api_client, method)('url', 'data')

    assert request.called
    assert ret == request.return_value

@pytest.mark.parametrize('method', [
    'put',
    'post',
    'delete'
])
def test_inactive_service_cannot_be_modified_by_normal_user(app_, api_user_active, method):
    api_client = NotifyAdminAPIClient('api_key', 'base_url', 'service_id')

    with app_.test_request_context() as request_context, app_.test_client() as client:
        client.login(api_user_active)
        request_context.service = service_json(active=False)

        with patch.object(api_client, 'request') as request:
            with pytest.raises(werkzeug.exceptions.Forbidden):
                getattr(api_client, method)('url', 'data')

    assert not request.called

@pytest.mark.parametrize('method', [
    'put',
    'post',
    'delete'
])
def test_inactive_service_can_be_modified_by_platform_admin(app_, platform_admin_user, method):
    api_client = NotifyAdminAPIClient('api_key', 'base_url', 'service_id')

    with app_.test_request_context() as request_context, app_.test_client() as client:
        client.login(platform_admin_user)
        request_context.service = service_json(active=False)

        with patch.object(api_client, 'request') as request:
            ret = getattr(api_client, method)('url', 'data')

    assert request.called
    assert ret == request.return_value
<commit_msg>write tests for @minglis 's code :slightly_frowning_face:<commit_after>import uuid
import pytest
from unittest.mock import patch

import werkzeug

from tests import service_json
from tests.conftest import api_user_active, platform_admin_user
from app.notify_client import NotifyAdminAPIClient

@pytest.mark.parametrize('method', [
    'put',
    'post',
    'delete'
])
@pytest.mark.parametrize('user', [
    api_user_active(str(uuid.uuid4())),
    platform_admin_user(str(uuid.uuid4()))
], ids=['api_user', 'platform_admin'])
@pytest.mark.parametrize('service', [
    service_json(active=True),
    None
], ids=['active_service', 'no_service'])
def test_active_service_can_be_modified(app_, method, user, service):
    api_client = NotifyAdminAPIClient('api_key', 'base_url', 'service_id')

    with app_.test_request_context() as request_context, app_.test_client() as client:
        client.login(user)
        request_context.service = service

        with patch.object(api_client, 'request') as request:
            ret = getattr(api_client, method)('url', 'data')

    assert request.called
    assert ret == request.return_value

@pytest.mark.parametrize('method', [
    'put',
    'post',
    'delete'
])
def test_inactive_service_cannot_be_modified_by_normal_user(app_, api_user_active, method):
    api_client = NotifyAdminAPIClient('api_key', 'base_url', 'service_id')

    with app_.test_request_context() as request_context, app_.test_client() as client:
        client.login(api_user_active)
        request_context.service = service_json(active=False)

        with patch.object(api_client, 'request') as request:
            with pytest.raises(werkzeug.exceptions.Forbidden):
                getattr(api_client, method)('url', 'data')

    assert not request.called

@pytest.mark.parametrize('method', [
    'put',
    'post',
    'delete'
])
def test_inactive_service_can_be_modified_by_platform_admin(app_, platform_admin_user, method):
    api_client = NotifyAdminAPIClient('api_key', 'base_url', 'service_id')

    with app_.test_request_context() as request_context, app_.test_client() as client:
        client.login(platform_admin_user)
        request_context.service = service_json(active=False)

        with patch.object(api_client, 'request') as request:
            ret = getattr(api_client, method)('url', 'data')

    assert request.called
    assert ret == request.return_value

def test_generate_headers_sets_standard_headers():
    api_client = NotifyAdminAPIClient('api_key', 'base_url', 'service_id')

    # with patch('app.notify_client.has_request_context', return_value=False):
    headers = api_client.generate_headers('api_token')

    assert set(headers.keys()) == {'Authorization', 'Content-type', 'User-agent'}
    assert headers['Authorization'] == 'Bearer api_token'
    assert headers['Content-type'] == 'application/json'
    assert headers['User-agent'].startswith('NOTIFY-API-PYTHON-CLIENT')

def test_generate_headers_sets_request_id_if_in_request_context(app_):
    api_client = NotifyAdminAPIClient('api_key', 'base_url', 'service_id')

    with app_.test_request_context() as request_context:
        headers = api_client.generate_headers('api_token')

    assert set(headers.keys()) == {'Authorization', 'Content-type', 'User-agent', 'NotifyRequestID'}
    assert headers['NotifyRequestID'] == request_context.request.request_id
<|endoftext|>