<commit_before># -:- encoding: utf-8 -:-
# This file is part of the MapProxy project.
# Copyright (C) 2013 Omniscale <http://omniscale.de>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import shutil
import tempfile
from contextlib import contextmanager

import yaml

from mapproxy.script.conf.app import config_command

from nose.tools import eq_

@contextmanager
def capture():
    import sys
    from cStringIO import StringIO

    # setup the environment
    backup_stdout = sys.stdout
    backup_stderr = sys.stderr

    try:
        sys.stdout = StringIO()
        sys.stderr = StringIO()
        yield sys.stdout, sys.stderr
    except Exception, ex:
        backup_stdout.write(str(ex))
        backup_stdout.write(sys.stdout.getvalue())
        backup_stderr.write(sys.stderr.getvalue())
        raise
    finally:
        sys.stdout = backup_stdout
        sys.stderr = backup_stderr

def filename(name):
    return os.path.join(
        os.path.dirname(__file__),
        'fixture',
        name,
    )

class TestMapProxyConfCmd(object):
    def setup(self):
        self.dir = tempfile.mkdtemp()

    def teardown(self):
        if os.path.exists(self.dir):
            shutil.rmtree(self.dir)

    def tmp_filename(self, name):
        return os.path.join(
            self.dir,
            name,
        )

    def test_cmd_no_args(self):
        with capture() as (stdout, stderr):
            assert config_command(['mapproxy-conf']) == 2

        assert '--capabilities required' in stderr.getvalue()

    def test_stdout_output(self):
        with capture() as (stdout, stderr):
            assert config_command(['mapproxy-conf', '--capabilities', filename('util-conf-wms-111-cap.xml')]) == 0

        assert stdout.getvalue().startswith('# MapProxy configuration')

    def test_test_cap_output_no_base(self):
        with capture() as (stdout, stderr):
            assert config_command(['mapproxy-conf',
                '--capabilities', filename('util-conf-wms-111-cap.xml'),
                '--output', self.tmp_filename('mapproxy.yaml'),
                ]) == 0

        with open(self.tmp_filename('mapproxy.yaml')) as f:
            conf = yaml.load(f)

            assert 'grids' not in conf
            eq_(conf['sources'], {
                'osm_roads_wms': {
                    'supported_srs': ['EPSG:31467', 'EPSG:31466', 'EPSG:25832', 'EPSG:3857', 'EPSG:25831', 'EPSG:25833', 'EPSG:4326', 'EPSG:31468', 'EPSG:900913', 'CRS:84', 'EPSG:4258'],
                    'req': {'layers': 'osm_roads', 'url': 'http://osm.omniscale.net/proxy/service?', 'transparent': True},
                    'type': 'wms',
                    'coverage': {'srs': 'EPSG:4326', 'bbox': [-180.0, -85.0511287798, 180.0, 85.0511287798]}
                },
                'osm_wms': {
                    'supported_srs': ['EPSG:31467', 'EPSG:31466', 'EPSG:25832', 'EPSG:3857', 'EPSG:25831', 'EPSG:25833', 'EPSG:4326', 'EPSG:31468', 'EPSG:900913', 'CRS:84', 'EPSG:4258'],
                    'req': {'layers': 'osm', 'url': 'http://osm.omniscale.net/proxy/service?', 'transparent': True},
                    'type': 'wms',
                    'coverage': {
                        'srs': 'EPSG:4326',
                        'bbox': [-180.0, -85.0511287798, 180.0, 85.0511287798],
                    },
                },
            })

            eq_(conf['layers'], [{
                'title': 'Omniscale OpenStreetMap WMS',
                'layers': [
                    {
                        'name': 'osm',
                        'title': 'OpenStreetMap (complete map)',
                        'sources': ['osm_wms'],
                    },
                    {
                        'name': 'osm_roads',
                        'title': 'OpenStreetMap (streets only)',
                        'sources': ['osm_roads_wms'],
                     },
                ]
            }])
            eq_(len(conf['layers'][0]['layers']), 2)

    def test_test_cap_output(self):
        with capture() as (stdout, stderr):
            assert config_command(['mapproxy-conf',
                '--capabilities', filename('util-conf-wms-111-cap.xml'),
                '--output', self.tmp_filename('mapproxy.yaml'),
                '--base', filename('util-conf-base-grids.yaml'),
                ]) == 0

        with open(self.tmp_filename('mapproxy.yaml')) as f:
            conf = yaml.load(f)

            assert 'grids' not in conf
            eq_(len(conf['sources']), 2)

            eq_(conf['caches'], {
                'osm_cache': {
                    'grids': ['webmercator', 'geodetic'],
                    'sources': ['osm_wms']
                },
                'osm_roads_cache': {
                    'grids': ['webmercator', 'geodetic'],
                    'sources': ['osm_roads_wms']
                },
            })

            eq_(conf['layers'], [{
                'title': 'Omniscale OpenStreetMap WMS',
                'layers': [
                    {
                        'name': 'osm',
                        'title': 'OpenStreetMap (complete map)',
                        'sources': ['osm_cache'],
                    },
                    {
                        'name': 'osm_roads',
                        'title': 'OpenStreetMap (streets only)',
                        'sources': ['osm_roads_cache'],
                    },
                ]
            }])
            eq_(len(conf['layers'][0]['layers']), 2)

    def test_overwrites(self):
        with capture() as (stdout, stderr):
            assert config_command(['mapproxy-conf',
                '--capabilities', filename('util-conf-wms-111-cap.xml'),
                '--output', self.tmp_filename('mapproxy.yaml'),
                '--overwrite', filename('util-conf-overwrite.yaml'),
                '--base', filename('util-conf-base-grids.yaml'),
                ]) == 0

        with open(self.tmp_filename('mapproxy.yaml')) as f:
            conf = yaml.load(f)

            assert 'grids' not in conf
            eq_(len(conf['sources']), 2)

            eq_(conf['sources'], {
                'osm_roads_wms': {
                    'supported_srs': ['EPSG:3857'],
                    'req': {'layers': 'osm_roads', 'url': 'http://osm.omniscale.net/proxy/service?', 'transparent': True, 'param': 42},
                    'type': 'wms',
                    'coverage': {'srs': 'EPSG:4326', 'bbox': [0, 0, 90, 90]}
                },
                'osm_wms': {
                    'supported_srs': ['EPSG:31467', 'EPSG:31466', 'EPSG:25832', 'EPSG:3857', 'EPSG:25831', 'EPSG:25833', 'EPSG:4326', 'EPSG:31468', 'EPSG:900913', 'CRS:84', 'EPSG:4258'],
                    'req': {'layers': 'osm', 'url': 'http://osm.omniscale.net/proxy/service?', 'transparent': True, 'param': 42},
                    'type': 'wms',
                    'coverage': {
                        'srs': 'EPSG:4326',
                        'bbox': [-180.0, -85.0511287798, 180.0, 85.0511287798],
                    },
                },
            })

            eq_(conf['caches'], {
                'osm_cache': {
                    'grids': ['webmercator', 'geodetic'],
                    'sources': ['osm_wms'],
                    'cache': {
                        'type': 'sqlite'
                    },
                },
                'osm_roads_cache': {
                    'grids': ['webmercator'],
                    'sources': ['osm_roads_wms'],
                    'cache': {
                        'type': 'sqlite'
                    },
                },
            })

            eq_(conf['layers'], [{
                'title': 'Omniscale OpenStreetMap WMS',
                'layers': [
                    {
                        'name': 'osm',
                        'title': 'OpenStreetMap (complete map)',
                        'sources': ['osm_cache'],
                    },
                    {
                        'name': 'osm_roads',
                        'title': 'OpenStreetMap (streets only)',
                        'sources': ['osm_roads_cache'],
                     },
                ]
            }])
            eq_(len(conf['layers'][0]['layers']), 2)

<commit_msg>fixed python 2.5 compat<commit_after># -:- encoding: utf-8 -:-
# This file is part of the MapProxy project.
# Copyright (C) 2013 Omniscale <http://omniscale.de>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import with_statement

import os
import shutil
import tempfile
from contextlib import contextmanager

import yaml

from mapproxy.script.conf.app import config_command

from nose.tools import eq_

@contextmanager
def capture():
    import sys
    from cStringIO import StringIO

    # setup the environment
    backup_stdout = sys.stdout
    backup_stderr = sys.stderr

    try:
        sys.stdout = StringIO()
        sys.stderr = StringIO()
        yield sys.stdout, sys.stderr
    except Exception, ex:
        backup_stdout.write(str(ex))
        backup_stdout.write(sys.stdout.getvalue())
        backup_stderr.write(sys.stderr.getvalue())
        raise
    finally:
        sys.stdout = backup_stdout
        sys.stderr = backup_stderr

def filename(name):
    return os.path.join(
        os.path.dirname(__file__),
        'fixture',
        name,
    )

class TestMapProxyConfCmd(object):
    def setup(self):
        self.dir = tempfile.mkdtemp()

    def teardown(self):
        if os.path.exists(self.dir):
            shutil.rmtree(self.dir)

    def tmp_filename(self, name):
        return os.path.join(
            self.dir,
            name,
        )

    def test_cmd_no_args(self):
        with capture() as (stdout, stderr):
            assert config_command(['mapproxy-conf']) == 2

        assert '--capabilities required' in stderr.getvalue()

    def test_stdout_output(self):
        with capture() as (stdout, stderr):
            assert config_command(['mapproxy-conf', '--capabilities', filename('util-conf-wms-111-cap.xml')]) == 0

        assert stdout.getvalue().startswith('# MapProxy configuration')

    def test_test_cap_output_no_base(self):
        with capture() as (stdout, stderr):
            assert config_command(['mapproxy-conf',
                '--capabilities', filename('util-conf-wms-111-cap.xml'),
                '--output', self.tmp_filename('mapproxy.yaml'),
                ]) == 0

        with open(self.tmp_filename('mapproxy.yaml')) as f:
            conf = yaml.load(f)

            assert 'grids' not in conf
            eq_(conf['sources'], {
                'osm_roads_wms': {
                    'supported_srs': ['EPSG:31467', 'EPSG:31466', 'EPSG:25832', 'EPSG:3857', 'EPSG:25831', 'EPSG:25833', 'EPSG:4326', 'EPSG:31468', 'EPSG:900913', 'CRS:84', 'EPSG:4258'],
                    'req': {'layers': 'osm_roads', 'url': 'http://osm.omniscale.net/proxy/service?', 'transparent': True},
                    'type': 'wms',
                    'coverage': {'srs': 'EPSG:4326', 'bbox': [-180.0, -85.0511287798, 180.0, 85.0511287798]}
                },
                'osm_wms': {
                    'supported_srs': ['EPSG:31467', 'EPSG:31466', 'EPSG:25832', 'EPSG:3857', 'EPSG:25831', 'EPSG:25833', 'EPSG:4326', 'EPSG:31468', 'EPSG:900913', 'CRS:84', 'EPSG:4258'],
                    'req': {'layers': 'osm', 'url': 'http://osm.omniscale.net/proxy/service?', 'transparent': True},
                    'type': 'wms',
                    'coverage': {
                        'srs': 'EPSG:4326',
                        'bbox': [-180.0, -85.0511287798, 180.0, 85.0511287798],
                    },
                },
            })

            eq_(conf['layers'], [{
                'title': 'Omniscale OpenStreetMap WMS',
                'layers': [
                    {
                        'name': 'osm',
                        'title': 'OpenStreetMap (complete map)',
                        'sources': ['osm_wms'],
                    },
                    {
                        'name': 'osm_roads',
                        'title': 'OpenStreetMap (streets only)',
                        'sources': ['osm_roads_wms'],
                     },
                ]
            }])
            eq_(len(conf['layers'][0]['layers']), 2)

    def test_test_cap_output(self):
        with capture() as (stdout, stderr):
            assert config_command(['mapproxy-conf',
                '--capabilities', filename('util-conf-wms-111-cap.xml'),
                '--output', self.tmp_filename('mapproxy.yaml'),
                '--base', filename('util-conf-base-grids.yaml'),
                ]) == 0

        with open(self.tmp_filename('mapproxy.yaml')) as f:
            conf = yaml.load(f)

            assert 'grids' not in conf
            eq_(len(conf['sources']), 2)

            eq_(conf['caches'], {
                'osm_cache': {
                    'grids': ['webmercator', 'geodetic'],
                    'sources': ['osm_wms']
                },
                'osm_roads_cache': {
                    'grids': ['webmercator', 'geodetic'],
                    'sources': ['osm_roads_wms']
                },
            })

            eq_(conf['layers'], [{
                'title': 'Omniscale OpenStreetMap WMS',
                'layers': [
                    {
                        'name': 'osm',
                        'title': 'OpenStreetMap (complete map)',
                        'sources': ['osm_cache'],
                    },
                    {
                        'name': 'osm_roads',
                        'title': 'OpenStreetMap (streets only)',
                        'sources': ['osm_roads_cache'],
                    },
                ]
            }])
            eq_(len(conf['layers'][0]['layers']), 2)

    def test_overwrites(self):
        with capture() as (stdout, stderr):
            assert config_command(['mapproxy-conf',
                '--capabilities', filename('util-conf-wms-111-cap.xml'),
                '--output', self.tmp_filename('mapproxy.yaml'),
                '--overwrite', filename('util-conf-overwrite.yaml'),
                '--base', filename('util-conf-base-grids.yaml'),
                ]) == 0

        with open(self.tmp_filename('mapproxy.yaml')) as f:
            conf = yaml.load(f)

            assert 'grids' not in conf
            eq_(len(conf['sources']), 2)

            eq_(conf['sources'], {
                'osm_roads_wms': {
                    'supported_srs': ['EPSG:3857'],
                    'req': {'layers': 'osm_roads', 'url': 'http://osm.omniscale.net/proxy/service?', 'transparent': True, 'param': 42},
                    'type': 'wms',
                    'coverage': {'srs': 'EPSG:4326', 'bbox': [0, 0, 90, 90]}
                },
                'osm_wms': {
                    'supported_srs': ['EPSG:31467', 'EPSG:31466', 'EPSG:25832', 'EPSG:3857', 'EPSG:25831', 'EPSG:25833', 'EPSG:4326', 'EPSG:31468', 'EPSG:900913', 'CRS:84', 'EPSG:4258'],
                    'req': {'layers': 'osm', 'url': 'http://osm.omniscale.net/proxy/service?', 'transparent': True, 'param': 42},
                    'type': 'wms',
                    'coverage': {
                        'srs': 'EPSG:4326',
                        'bbox': [-180.0, -85.0511287798, 180.0, 85.0511287798],
                    },
                },
            })

            eq_(conf['caches'], {
                'osm_cache': {
                    'grids': ['webmercator', 'geodetic'],
                    'sources': ['osm_wms'],
                    'cache': {
                        'type': 'sqlite'
                    },
                },
                'osm_roads_cache': {
                    'grids': ['webmercator'],
                    'sources': ['osm_roads_wms'],
                    'cache': {
                        'type': 'sqlite'
                    },
                },
            })

            eq_(conf['layers'], [{
                'title': 'Omniscale OpenStreetMap WMS',
                'layers': [
                    {
                        'name': 'osm',
                        'title': 'OpenStreetMap (complete map)',
                        'sources': ['osm_cache'],
                    },
                    {
                        'name': 'osm_roads',
                        'title': 'OpenStreetMap (streets only)',
                        'sources': ['osm_roads_cache'],
                     },
                ]
            }])
            eq_(len(conf['layers'][0]['layers']), 2)

<|endoftext|>


<commit_before>"""Matchzoo toolkit for text pre-processing."""

import re
import abc
import nltk
import typing
import collections
import numpy as np

match_punc = re.compile('[^\w\s]')

class ProcessorUnit(metaclass=abc.ABCMeta):
    """Process unit do not persive state (i.e. do not need fit)."""

    @abc.abstractmethod
    def transform(self, input: typing.Any):
        """Abstract base method, need to be implemented in subclass."""

class StatefulProcessorUnit(ProcessorUnit, metaclass=abc.ABCMeta):
    """Process unit do persive state (i.e. need fit)."""

    def __init__(self):
        """Initialization."""
        self._state = {}

    @property
    def state(self):
        """Get current state."""
        return self._state

    @abc.abstractmethod
    def fit(self, input: typing.Any):
        """Abstract base method, need to be implemented in subclass."""

class TokenizeUnit(ProcessorUnit):
    """Process unit for text tokenization."""

    def transform(self, input: str) -> list:
        """
        Process input data from raw terms to list of tokens.

        :param input: raw textual input.

        :return tokens: tokenized tokens as a list.
        """
        return nltk.word_tokenize(input)

class LowercaseUnit(ProcessorUnit):
    """Process unit for text lower case."""

    def transform(self, tokens: list) -> list:
        """
        Convert list of tokens to lower case.

        :param tokens: list of tokens.

        :return tokens: lower-cased list of tokens.
        """
        return [token.lower() for token in tokens]

class PuncRemovalUnit(ProcessorUnit):
    """Process unit for remove punctuations."""

    def transform(self, tokens: list) -> list:
        """
        Remove punctuations from list of tokens.

        :param tokens: list of toekns.

        :return rv: tokens  without punctuation.
        """
        return [token for token in tokens if not match_punc.search(token)]

class DigitRemovalUnit(ProcessorUnit):
    """Process unit to remove digits."""

    def transform(self, tokens: list) -> list:
        """
        Remove digits from list of tokens.

        :param tokens: list of tokens to be filtered.

        :return tokens: tokens of tokens without digits.
        """
        return [token for token in tokens if not token.isdigit()]

class StopRemovalUnit(ProcessorUnit):
    """
    Process unit to remove stop words.

    Example:
        >>> unit = StopRemovalUnit()
        >>> unit.transform(['a', 'the', 'test'])
        ['test']
        >>> type(unit.stopwords)
        <class 'list'>
    """

    def __init__(self, lang: str='english'):
        """Initialization."""
        self._lang = lang
        self._stop = nltk.corpus.stopwords.words(self._lang)

    def transform(self, tokens: list) -> list:
        """
        Remove stopwords from list of tokenized tokens.

        :param tokens: list of tokenized tokens.
        :param lang: language code for stopwords.

        :return tokens: list of tokenized tokens without stopwords.
        """
        return [token
                for token
                in tokens
                if token not in self._stop]

    @property
    def stopwords(self) -> list:
        """
        Get stopwords based on language.

        :params lang: language code.
        :return: list of stop words.
        """
        return self._stop

class StemmingUnit(ProcessorUnit):
    """Process unit for token stemming."""

    def __init__(self, stemmer='porter'):
        """Initialization."""
        self.stemmer = stemmer

    def transform(self, tokens: list) -> list:
        """
        Reducing inflected words to their word stem, base or root form.

        :param tokens: list of string to be stemmed.
        :param stemmer: stemmer to use, `porter` or `lancaster`.

        :raise ValueError: stemmer type should be porter or lancaster.

        :return tokens: stemmed token.
        """
        if self.stemmer == 'porter':
            porter_stemmer = nltk.stem.PorterStemmer()
            return [porter_stemmer.stem(token) for token in tokens]
        elif self.stemmer == 'lancaster' or self.stemmer == 'krovetz':
            lancaster_stemmer = nltk.stem.lancaster.LancasterStemmer()
            return [lancaster_stemmer.stem(token) for token in tokens]
        else:
            raise ValueError(
                'Not supported supported stemmer type: {}'.format(
                    self.stemmer))

class LemmatizationUnit(ProcessorUnit):
    """Process unit for token lemmatization."""

    def transform(self, tokens: list) -> list:
        """
        Lemmatization a sequence of tokens.

        :param tokens: list of tokens to be lemmatized.

        :return tokens: list of lemmatizd tokens.
        """
        lemmatizer = nltk.WordNetLemmatizer()
        return [lemmatizer.lemmatize(token, pos='v') for token in tokens]

class NgramLetterUnit(ProcessorUnit):
    """
    Process unit for n-letter generation.

    Triletter is used in :class:`DSSMModel`.
    This processor is expected to execute before `Vocab`
    has been created.

    Examples:
        >>> triletter = NgramLetterUnit()
        >>> rv = triletter.transform(['word'])
        >>> len(rv)
        4
        >>> rv
        ['#wo', 'wor', 'ord', 'rd#']

    """

    def transform(self, tokens: list, ngram: int=3) -> list:
        """
        Transform token into tri-letter.

        For example, `word` should be represented as `#wo`,
        `wor`, `ord` and `rd#`.

        :param tokens: list of tokens to be transformed.
        :param ngram: By default use 3-gram (tri-letter).

        :return n_letters: generated n_letters.
        """
        n_letters = []
        for token in tokens:
            token = '#' + token + '#'
            while len(token) >= ngram:
                n_letters.append(token[:ngram])
                token = token[1:]
        return n_letters

class VocabularyUnit(StatefulProcessorUnit):
    """
    Vocabulary class.

    Examples:
        >>> vocab = VocabularyUnit()
        >>> vocab.fit(['A', 'B', 'C', 'D', 'E'])
        >>> term_index = vocab.state['term_index']
        >>> term_index  # doctest: +SKIP
        {'E': 1, 'C': 2, 'D': 3, 'A': 4, 'B': 5}
        >>> index_term = vocab.state['index_term']
        >>> index_term  # doctest: +SKIP
        {1: 'C', 2: 'A', 3: 'E', 4: 'B', 5: 'D'}

        >>> term_index['out-of-vocabulary-term']
        0
        >>> index_term[0]
        ''
        >>> index_term[42]
        Traceback (most recent call last):
            ...
        KeyError: 42

        >>> a_index = term_index['A']
        >>> c_index = term_index['C']
        >>> vocab.transform(['C', 'A', 'C']) == [c_index, a_index, c_index]
        True
        >>> vocab.transform(['C', 'A', 'OOV']) == [c_index, a_index, 0]
        True

        >>> indices = vocab.transform('ABCDDZZZ')
        >>> ''.join(vocab.state['index_term'][i] for i in indices)
        'ABCDD'

    """

    class IndexTerm(dict):
        """Map index to term."""

        def __missing__(self, key):
            """Map out-of-vocabulary indices to empty string."""
            if key == 0:
                return ''
            else:
                raise KeyError(key)

    class TermIndex(dict):
        """Map term to index."""

        def __missing__(self, key):
            """Map out-of-vocabulary terms to index 0."""
            return 0

    def fit(self, tokens: list):
        """Build a :class:`TermIndex` and a :class:`IndexTerm`."""
        self._state['term_index'] = self.TermIndex()
        self._state['index_term'] = self.IndexTerm()
        terms = set(tokens)
        for index, term in enumerate(terms):
            self._state['term_index'][term] = index + 1
            self._state['index_term'][index + 1] = term

    def transform(self, tokens: list) -> list:
        """Transform a list of tokens to corresponding indices."""
        return [self._state['term_index'][token] for token in tokens]

class WordHashingUnit(ProcessorUnit):
    """
    Create word-hashing layer for :class:`DSSMModel`.

    The input of :class:`WordHashingUnit` should be a list
    of `tri-letters` extracted from one document. The output
    of :class:`WordHashingUnit` is the word-hashing representation
    of this document.

    :class:`NgramLetterUnit` and :class:`VocabularyUnit` are two
    essential prerequisite of :class:`WordHashingUnit`.

    TODO Update :class:`WordHashingUnit` to generalize more `DSSM`
    like models such as `CDSSM` and `LSTM-DSSM`.

    Examples:
       >>> tri_letters = ['#te', 'tes','est', 'st#', 'oov']
       >>> word_hashing = WordHashingUnit(
       ...     term_index={'': 0,'st#': 1, '#te': 2, 'est': 3, 'tes': 4})
       >>> hashing = word_hashing.transform(tri_letters)
       >>> hashing[0]
       1.0
       >>> hashing[1]
       1.0
       >>> hashing.shape
       (6,)

    """

    def __init__(
        self,
        term_index: dict,
    ):
        """
        Class initialization.

        :param term_index: term-index mapping generated by
            :class:`VocabularyUnit`.
        :param dim_triletter: dimensionality of tri_leltters.
        """
        self._term_index = term_index

    def transform(
        self,
        tri_letters: list
    ) -> np.ndarray:
        """
        Transform list of :attr:`tri-letters` into word hashing layer.

        :param tri_letters: list of `tri_letters` generated by
            :class:`NgramLetterUnit`.
        :return: Word hashing representation of `tri-letters`.
        """
        hashing = np.zeros(len(self._term_index) + 1)
        counted_tri_letters = collections.Counter(tri_letters)
        for key, value in counted_tri_letters.items():
            idx = self._term_index.get(key, 0)
            hashing[idx] = value
        return hashing

class SlidingWindowUnit(ProcessorUnit):
    """
    SlidingWindowUnit class.

    Used to get information from a window sliding
    through the input data.

    Examples:
        >>> data = np.array([[0,0,0],[1,1,1],[2,2,2],[3,3,3]])
        >>> sliding = SlidingWindowUnit()
        >>> output = sliding.transform(data)
        >>> output.shape
        (2, 9)
        >>> output[0]
        array([0, 0, 0, 1, 1, 1, 2, 2, 2])

    """

    def __init__(self, sliding_window: int=3):
        """
        Class initialization.

        :param sliding_window: sliding window length.
        """
        self._sliding_window = sliding_window

    def transform(self, inputs: np.ndarray) -> np.ndarray:
        """
        Compute the result of a window sliding through input data.

        :param inputs: sequential input data.
        :return: window sliding result.
        """
        output = []
        while len(inputs) >= self._sliding_window:
            output.append(np.concatenate(
                inputs[:self._sliding_window], axis=-1))
            inputs = inputs[1:]
        return np.array(output)

class FixedLengthUnit(ProcessorUnit):
    """
    FixedLengthUnit Class.

    Process unit to get the fixed length text.

    Examples:
        >>> fixedlen = FixedLengthUnit(3)
        >>> fixedlen.transform(range(1, 6)) == [3, 4, 5]
        True
        >>> fixedlen = FixedLengthUnit(3)
        >>> fixedlen.transform(range(1, 3)) == [0, 1, 2]
        True

    """

    def __init__(self, text_length: int, pad_value: int=0,
                 pad_mode: str='pre', truncate_mode: str='pre'):
        """
        Class initialization.

        :param text_length: fixed length of the text.
        :param pad_value: if text length is smaller than :attr:`text_length`,
            filling text with :attr:`pad_value`.
        :param pad_mode: String, `pre` or `post`:
            pad either before or after each sequence.
        :param truncate_mode: String, `pre` or `post`:
            remove values from sequences larger than :attr:`text_length`,
            either at the beginning or at the end of the sequences.
        """
        self._text_length = text_length
        self._pad_value = pad_value
        self._pad_mode = pad_mode
        self._truncate_mode = truncate_mode

    def transform(self, tokens: list) -> list:
        """
        Transform list of tokenized tokens into the fixed length text.

        :param tokens: list of tokenized tokens.

        :return tokens: list of tokenized tokens in fixed length.
        """
        np_tokens = np.array(tokens)
        fixed_tokens = np.full([self._text_length], self._pad_value,
                               dtype=np_tokens.dtype)

        if self._truncate_mode == 'pre':
            trunc_tokens = tokens[-self._text_length:]
        elif self._truncate_mode == 'post':
            trunc_tokens = tokens[:self._text_length]
        else:
            raise ValueError('{} is not a vaild '
                             'truncate mode.'.format(self._truncate_mode))

        if self._pad_mode == 'post':
            fixed_tokens[:len(trunc_tokens)] = trunc_tokens
        elif self._pad_mode == 'pre':
            fixed_tokens[-len(trunc_tokens):] = trunc_tokens
        else:
            raise ValueError('{} is not a vaild '
                             'pad mode.'.format(self._pad_mode))

        return fixed_tokens.tolist()
<commit_msg>add comments about sliding window unit.<commit_after>"""Matchzoo toolkit for text pre-processing."""

import re
import abc
import nltk
import typing
import collections
import numpy as np

match_punc = re.compile('[^\w\s]')

class ProcessorUnit(metaclass=abc.ABCMeta):
    """Process unit do not persive state (i.e. do not need fit)."""

    @abc.abstractmethod
    def transform(self, input: typing.Any):
        """Abstract base method, need to be implemented in subclass."""

class StatefulProcessorUnit(ProcessorUnit, metaclass=abc.ABCMeta):
    """Process unit do persive state (i.e. need fit)."""

    def __init__(self):
        """Initialization."""
        self._state = {}

    @property
    def state(self):
        """Get current state."""
        return self._state

    @abc.abstractmethod
    def fit(self, input: typing.Any):
        """Abstract base method, need to be implemented in subclass."""

class TokenizeUnit(ProcessorUnit):
    """Process unit for text tokenization."""

    def transform(self, input: str) -> list:
        """
        Process input data from raw terms to list of tokens.

        :param input: raw textual input.

        :return tokens: tokenized tokens as a list.
        """
        return nltk.word_tokenize(input)

class LowercaseUnit(ProcessorUnit):
    """Process unit for text lower case."""

    def transform(self, tokens: list) -> list:
        """
        Convert list of tokens to lower case.

        :param tokens: list of tokens.

        :return tokens: lower-cased list of tokens.
        """
        return [token.lower() for token in tokens]

class PuncRemovalUnit(ProcessorUnit):
    """Process unit for remove punctuations."""

    def transform(self, tokens: list) -> list:
        """
        Remove punctuations from list of tokens.

        :param tokens: list of toekns.

        :return rv: tokens  without punctuation.
        """
        return [token for token in tokens if not match_punc.search(token)]

class DigitRemovalUnit(ProcessorUnit):
    """Process unit to remove digits."""

    def transform(self, tokens: list) -> list:
        """
        Remove digits from list of tokens.

        :param tokens: list of tokens to be filtered.

        :return tokens: tokens of tokens without digits.
        """
        return [token for token in tokens if not token.isdigit()]

class StopRemovalUnit(ProcessorUnit):
    """
    Process unit to remove stop words.

    Example:
        >>> unit = StopRemovalUnit()
        >>> unit.transform(['a', 'the', 'test'])
        ['test']
        >>> type(unit.stopwords)
        <class 'list'>
    """

    def __init__(self, lang: str='english'):
        """Initialization."""
        self._lang = lang
        self._stop = nltk.corpus.stopwords.words(self._lang)

    def transform(self, tokens: list) -> list:
        """
        Remove stopwords from list of tokenized tokens.

        :param tokens: list of tokenized tokens.
        :param lang: language code for stopwords.

        :return tokens: list of tokenized tokens without stopwords.
        """
        return [token
                for token
                in tokens
                if token not in self._stop]

    @property
    def stopwords(self) -> list:
        """
        Get stopwords based on language.

        :params lang: language code.
        :return: list of stop words.
        """
        return self._stop

class StemmingUnit(ProcessorUnit):
    """Process unit for token stemming."""

    def __init__(self, stemmer='porter'):
        """Initialization."""
        self.stemmer = stemmer

    def transform(self, tokens: list) -> list:
        """
        Reducing inflected words to their word stem, base or root form.

        :param tokens: list of string to be stemmed.
        :param stemmer: stemmer to use, `porter` or `lancaster`.

        :raise ValueError: stemmer type should be porter or lancaster.

        :return tokens: stemmed token.
        """
        if self.stemmer == 'porter':
            porter_stemmer = nltk.stem.PorterStemmer()
            return [porter_stemmer.stem(token) for token in tokens]
        elif self.stemmer == 'lancaster' or self.stemmer == 'krovetz':
            lancaster_stemmer = nltk.stem.lancaster.LancasterStemmer()
            return [lancaster_stemmer.stem(token) for token in tokens]
        else:
            raise ValueError(
                'Not supported supported stemmer type: {}'.format(
                    self.stemmer))

class LemmatizationUnit(ProcessorUnit):
    """Process unit for token lemmatization."""

    def transform(self, tokens: list) -> list:
        """
        Lemmatization a sequence of tokens.

        :param tokens: list of tokens to be lemmatized.

        :return tokens: list of lemmatizd tokens.
        """
        lemmatizer = nltk.WordNetLemmatizer()
        return [lemmatizer.lemmatize(token, pos='v') for token in tokens]

class NgramLetterUnit(ProcessorUnit):
    """
    Process unit for n-letter generation.

    Triletter is used in :class:`DSSMModel`.
    This processor is expected to execute before `Vocab`
    has been created.

    Examples:
        >>> triletter = NgramLetterUnit()
        >>> rv = triletter.transform(['word'])
        >>> len(rv)
        4
        >>> rv
        ['#wo', 'wor', 'ord', 'rd#']

    """

    def transform(self, tokens: list, ngram: int=3) -> list:
        """
        Transform token into tri-letter.

        For example, `word` should be represented as `#wo`,
        `wor`, `ord` and `rd#`.

        :param tokens: list of tokens to be transformed.
        :param ngram: By default use 3-gram (tri-letter).

        :return n_letters: generated n_letters.
        """
        n_letters = []
        for token in tokens:
            token = '#' + token + '#'
            while len(token) >= ngram:
                n_letters.append(token[:ngram])
                token = token[1:]
        return n_letters

class VocabularyUnit(StatefulProcessorUnit):
    """
    Vocabulary class.

    Examples:
        >>> vocab = VocabularyUnit()
        >>> vocab.fit(['A', 'B', 'C', 'D', 'E'])
        >>> term_index = vocab.state['term_index']
        >>> term_index  # doctest: +SKIP
        {'E': 1, 'C': 2, 'D': 3, 'A': 4, 'B': 5}
        >>> index_term = vocab.state['index_term']
        >>> index_term  # doctest: +SKIP
        {1: 'C', 2: 'A', 3: 'E', 4: 'B', 5: 'D'}

        >>> term_index['out-of-vocabulary-term']
        0
        >>> index_term[0]
        ''
        >>> index_term[42]
        Traceback (most recent call last):
            ...
        KeyError: 42

        >>> a_index = term_index['A']
        >>> c_index = term_index['C']
        >>> vocab.transform(['C', 'A', 'C']) == [c_index, a_index, c_index]
        True
        >>> vocab.transform(['C', 'A', 'OOV']) == [c_index, a_index, 0]
        True

        >>> indices = vocab.transform('ABCDDZZZ')
        >>> ''.join(vocab.state['index_term'][i] for i in indices)
        'ABCDD'

    """

    class IndexTerm(dict):
        """Map index to term."""

        def __missing__(self, key):
            """Map out-of-vocabulary indices to empty string."""
            if key == 0:
                return ''
            else:
                raise KeyError(key)

    class TermIndex(dict):
        """Map term to index."""

        def __missing__(self, key):
            """Map out-of-vocabulary terms to index 0."""
            return 0

    def fit(self, tokens: list):
        """Build a :class:`TermIndex` and a :class:`IndexTerm`."""
        self._state['term_index'] = self.TermIndex()
        self._state['index_term'] = self.IndexTerm()
        terms = set(tokens)
        for index, term in enumerate(terms):
            self._state['term_index'][term] = index + 1
            self._state['index_term'][index + 1] = term

    def transform(self, tokens: list) -> list:
        """Transform a list of tokens to corresponding indices."""
        return [self._state['term_index'][token] for token in tokens]

class WordHashingUnit(ProcessorUnit):
    """
    Create word-hashing layer for :class:`DSSMModel`.

    The input of :class:`WordHashingUnit` should be a list
    of `tri-letters` extracted from one document. The output
    of :class:`WordHashingUnit` is the word-hashing representation
    of this document.

    :class:`NgramLetterUnit` and :class:`VocabularyUnit` are two
    essential prerequisite of :class:`WordHashingUnit`.

    TODO Update :class:`WordHashingUnit` to generalize more `DSSM`
    like models such as `CDSSM` and `LSTM-DSSM`.

    Examples:
       >>> tri_letters = ['#te', 'tes','est', 'st#', 'oov']
       >>> word_hashing = WordHashingUnit(
       ...     term_index={'': 0,'st#': 1, '#te': 2, 'est': 3, 'tes': 4})
       >>> hashing = word_hashing.transform(tri_letters)
       >>> hashing[0]
       1.0
       >>> hashing[1]
       1.0
       >>> hashing.shape
       (6,)

    """

    def __init__(
        self,
        term_index: dict,
    ):
        """
        Class initialization.

        :param term_index: term-index mapping generated by
            :class:`VocabularyUnit`.
        :param dim_triletter: dimensionality of tri_leltters.
        """
        self._term_index = term_index

    def transform(
        self,
        tri_letters: list
    ) -> np.ndarray:
        """
        Transform list of :attr:`tri-letters` into word hashing layer.

        :param tri_letters: list of `tri_letters` generated by
            :class:`NgramLetterUnit`.
        :return: Word hashing representation of `tri-letters`.
        """
        hashing = np.zeros(len(self._term_index) + 1)
        counted_tri_letters = collections.Counter(tri_letters)
        for key, value in counted_tri_letters.items():
            idx = self._term_index.get(key, 0)
            hashing[idx] = value
        return hashing

class SlidingWindowUnit(ProcessorUnit):
    """
    SlidingWindowUnit class.

    Used to get information from a window sliding
    through the input data.

    Examples:
        >>> data = np.array([[0,0,0],[1,1,1],[2,2,2],[3,3,3]])
        >>> sliding = SlidingWindowUnit()
        >>> output = sliding.transform(data)
        >>> output.shape
        (2, 9)
        >>> output[0]
        array([0, 0, 0, 1, 1, 1, 2, 2, 2])

    """

    def __init__(self, sliding_window: int=3):
        """
        Class initialization.

        :param sliding_window: sliding window length.
        """
        self._sliding_window = sliding_window

    def transform(self, inputs: np.ndarray) -> np.ndarray:
        """
        Compute the result of a window sliding through input data.

        For short inputs that can not fill :attr:`sliding_window`, the
        unit return an empty array.

        :param inputs: sequential input data.
        :return: window sliding result.
        """
        output = []
        while len(inputs) >= self._sliding_window:
            output.append(np.concatenate(
                inputs[:self._sliding_window], axis=-1))
            inputs = inputs[1:]
        return np.array(output)

class FixedLengthUnit(ProcessorUnit):
    """
    FixedLengthUnit Class.

    Process unit to get the fixed length text.

    Examples:
        >>> fixedlen = FixedLengthUnit(3)
        >>> fixedlen.transform(range(1, 6)) == [3, 4, 5]
        True
        >>> fixedlen = FixedLengthUnit(3)
        >>> fixedlen.transform(range(1, 3)) == [0, 1, 2]
        True

    """

    def __init__(self, text_length: int, pad_value: int=0,
                 pad_mode: str='pre', truncate_mode: str='pre'):
        """
        Class initialization.

        :param text_length: fixed length of the text.
        :param pad_value: if text length is smaller than :attr:`text_length`,
            filling text with :attr:`pad_value`.
        :param pad_mode: String, `pre` or `post`:
            pad either before or after each sequence.
        :param truncate_mode: String, `pre` or `post`:
            remove values from sequences larger than :attr:`text_length`,
            either at the beginning or at the end of the sequences.
        """
        self._text_length = text_length
        self._pad_value = pad_value
        self._pad_mode = pad_mode
        self._truncate_mode = truncate_mode

    def transform(self, tokens: list) -> list:
        """
        Transform list of tokenized tokens into the fixed length text.

        :param tokens: list of tokenized tokens.

        :return tokens: list of tokenized tokens in fixed length.
        """
        np_tokens = np.array(tokens)
        fixed_tokens = np.full([self._text_length], self._pad_value,
                               dtype=np_tokens.dtype)

        if self._truncate_mode == 'pre':
            trunc_tokens = tokens[-self._text_length:]
        elif self._truncate_mode == 'post':
            trunc_tokens = tokens[:self._text_length]
        else:
            raise ValueError('{} is not a vaild '
                             'truncate mode.'.format(self._truncate_mode))

        if self._pad_mode == 'post':
            fixed_tokens[:len(trunc_tokens)] = trunc_tokens
        elif self._pad_mode == 'pre':
            fixed_tokens[-len(trunc_tokens):] = trunc_tokens
        else:
            raise ValueError('{} is not a vaild '
                             'pad mode.'.format(self._pad_mode))

        return fixed_tokens.tolist()
<|endoftext|>


<commit_before># vim: ts=4 sw=4 et ai:
# -*- coding: utf8 -*-
"""This module implements the TFTP Server functionality. Instantiate an
instance of the server, and then run the listen() method to listen for client
requests. Logging is performed via a standard logging object set in
TftpShared."""

import logging
import os
import select
import socket
import threading
import time
from errno import EINTR

from .TftpContexts import TftpContextServer
from .TftpPacketFactory import TftpPacketFactory
from .TftpPacketTypes import *
from .TftpShared import *

log = logging.getLogger('tftpy.TftpServer')

class TftpServer(TftpSession):
    """This class implements a tftp server object. Run the listen() method to
    listen for client requests.

    tftproot is the path to the tftproot directory to serve files from and/or
    write them to.

    dyn_file_func is a callable that takes a requested download
    path that is not present on the file system and must return either a
    file-like object to read from or None if the path should appear as not
    found. This permits the serving of dynamic content.

    upload_open is a callable that is triggered on every upload with the
    requested destination path and server context. It must either return a
    file-like object ready for writing or None if the path is invalid."""

    def __init__(self,
                 tftproot='/tftpboot',
                 dyn_file_func=None,
                 upload_open=None):
        self.listenip = None
        self.listenport = None
        self.sock = None
        # FIXME: What about multiple roots?
        self.root = os.path.abspath(tftproot)
        self.dyn_file_func = dyn_file_func
        self.upload_open = upload_open
        # A dict of sessions, where each session is keyed by a string like
        # ip:tid for the remote end.
        self.sessions = {}
        # A threading event to help threads synchronize with the server
        # is_running state.
        self.is_running = threading.Event()

        self.shutdown_gracefully = False
        self.shutdown_immediately = False

        for name in 'dyn_file_func', 'upload_open':
            attr = getattr(self, name)
            if attr and not callable(attr):
                raise TftpException("{} supplied, but it is not callable.".format(name))
        if os.path.exists(self.root):
            log.debug("tftproot %s does exist", self.root)
            if not os.path.isdir(self.root):
                raise TftpException("The tftproot must be a directory.")
            else:
                log.debug("tftproot %s is a directory" % self.root)
                if os.access(self.root, os.R_OK):
                    log.debug("tftproot %s is readable" % self.root)
                else:
                    raise TftpException("The tftproot must be readable")
                if os.access(self.root, os.W_OK):
                    log.debug("tftproot %s is writable" % self.root)
                else:
                    log.warning("The tftproot %s is not writable" % self.root)
        else:
            raise TftpException("The tftproot does not exist.")

    def listen(self, listenip="", listenport=DEF_TFTP_PORT,
               timeout=SOCK_TIMEOUT, retries=DEF_TIMEOUT_RETRIES):
        """Start a server listening on the supplied interface and port. This
        defaults to INADDR_ANY (all interfaces) and UDP port 69. You can also
        supply a different socket timeout value, if desired."""
        tftp_factory = TftpPacketFactory()

        # Don't use new 2.5 ternary operator yet
        # listenip = listenip if listenip else '0.0.0.0'
        if not listenip: listenip = '0.0.0.0'
        log.info("Server requested on ip %s, port %s" % (listenip, listenport))
        try:
            # FIXME - sockets should be non-blocking
            self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
            self.sock.bind((listenip, listenport))
            _, self.listenport = self.sock.getsockname()
        except socket.error as err:
            # Reraise it for now.
            raise err

        self.is_running.set()

        log.info("Starting receive loop...")
        while True:
            log.debug("shutdown_immediately is %s" % self.shutdown_immediately)
            log.debug("shutdown_gracefully is %s" % self.shutdown_gracefully)
            if self.shutdown_immediately:
                log.warning("Shutting down now. Session count: %d" % len(self.sessions))
                self.sock.close()
                for key in self.sessions:
                    self.sessions[key].end()
                self.sessions = []
                break

            elif self.shutdown_gracefully:
                if not self.sessions:
                    log.warning("In graceful shutdown mode and all sessions complete.")
                    self.sock.close()
                    break

            # Build the inputlist array of sockets to select() on.
            inputlist = [self.sock]
            for key in self.sessions:
                inputlist.append(self.sessions[key].sock)

            # Block until some socket has input on it.
            log.debug("Performing select on this inputlist: %s", inputlist)
            try:
                readyinput, readyoutput, readyspecial = select.select(inputlist, [], [], timeout)
            except select.error as err:
                if err[0] == EINTR:
                    # Interrupted system call
                    log.debug("Interrupted syscall, retrying")
                    continue
                else:
                    raise

            deletion_list = []

            # Handle the available data, if any. Maybe we timed-out.
            for readysock in readyinput:
                # Is the traffic on the main server socket? ie. new session?
                if readysock == self.sock:
                    log.debug("Data ready on our main socket")
                    buffer, (raddress, rport) = self.sock.recvfrom(MAX_BLKSIZE)

                    log.debug("Read %d bytes", len(buffer))

                    if self.shutdown_gracefully:
                        log.warning("Discarding data on main port, in graceful shutdown mode")
                        continue

                    # Forge a session key based on the client's IP and port,
                    # which should safely work through NAT.
                    key = "%s:%s" % (raddress, rport)

                    if not key in self.sessions:
                        log.debug("Creating new server context for session key = %s" % key)
                        self.sessions[key] = TftpContextServer(raddress,
                                                               rport,
                                                               timeout,
                                                               self.root,
                                                               self.dyn_file_func,
                                                               self.upload_open,
                                                               retries=retries)
                        try:
                            self.sessions[key].start(buffer)
                        except TftpException as err:
                            deletion_list.append(key)
                            log.error("Fatal exception thrown from session %s: %s" % (key, str(err)))
                    else:
                        log.warning("received traffic on main socket for existing session??")
                    log.info("Currently handling these sessions:")
                    for session_key, session in list(self.sessions.items()):
                        log.info("    %s" % session)

                else:
                    # Must find the owner of this traffic.
                    for key in self.sessions:
                        if readysock == self.sessions[key].sock:
                            log.debug("Matched input to session key %s" % key)
                            try:
                                self.sessions[key].cycle()
                                if self.sessions[key].state == None:
                                    log.info("Successful transfer.")
                                    deletion_list.append(key)
                            except TftpException as err:
                                deletion_list.append(key)
                                log.error("Fatal exception thrown from session %s: %s" % (key, str(err)))
                            # Break out of for loop since we found the correct
                            # session.
                            break
                    else:
                        log.error("Can't find the owner for this packet. "
                                  "Discarding.")

            log.debug("Looping on all sessions to check for timeouts")
            now = time.time()
            for key in self.sessions:
                try:
                    self.sessions[key].checkTimeout(now)
                except TftpTimeout as err:
                    log.error(str(err))
                    self.sessions[key].retry_count += 1
                    if self.sessions[key].retry_count >= self.sessions[key].retries:
                        log.debug("hit max retries on %s, giving up" % self.sessions[key])
                        deletion_list.append(key)
                    else:
                        log.debug("resending on session %s" % self.sessions[key])
                        self.sessions[key].state.resendLast()

            log.debug("Iterating deletion list.")
            for key in deletion_list:
                log.info('')
                log.info("Session %s complete" % key)
                if key in self.sessions:
                    log.debug("Gathering up metrics from session before deleting")
                    self.sessions[key].end()
                    metrics = self.sessions[key].metrics
                    if metrics.duration == 0:
                        log.info("Duration too short, rate undetermined")
                    else:
                        log.info("Transferred %d bytes in %.2f seconds" % (metrics.bytes, metrics.duration))
                        log.info("Average rate: %.2f kbps" % metrics.kbps)
                    log.info("%.2f bytes in resent data" % metrics.resent_bytes)
                    log.info("%d duplicate packets" % metrics.dupcount)
                    log.debug("Deleting session %s" % key)
                    del self.sessions[key]
                    log.debug("Session list is now %s" % self.sessions)
                else:
                    log.warning("Strange, session %s is not on the deletion list" % key)

        self.is_running.clear()

        log.debug("server returning from while loop")
        self.shutdown_gracefully = self.shutdown_immediately = False

    def stop(self, now=False):
        """Stop the server gracefully. Do not take any new transfers,
        but complete the existing ones. If force is True, drop everything
        and stop. Note, immediately will not interrupt the select loop, it
        will happen when the server returns on ready data, or a timeout.
        ie. SOCK_TIMEOUT"""
        if now:
            self.shutdown_immediately = True
        else:
            self.shutdown_gracefully = True
<commit_msg>No inline if<commit_after># vim: ts=4 sw=4 et ai:
# -*- coding: utf8 -*-
"""This module implements the TFTP Server functionality. Instantiate an
instance of the server, and then run the listen() method to listen for client
requests. Logging is performed via a standard logging object set in
TftpShared."""

import logging
import os
import select
import socket
import threading
import time
from errno import EINTR

from .TftpContexts import TftpContextServer
from .TftpPacketFactory import TftpPacketFactory
from .TftpPacketTypes import *
from .TftpShared import *

log = logging.getLogger('tftpy.TftpServer')

class TftpServer(TftpSession):
    """This class implements a tftp server object. Run the listen() method to
    listen for client requests.

    tftproot is the path to the tftproot directory to serve files from and/or
    write them to.

    dyn_file_func is a callable that takes a requested download
    path that is not present on the file system and must return either a
    file-like object to read from or None if the path should appear as not
    found. This permits the serving of dynamic content.

    upload_open is a callable that is triggered on every upload with the
    requested destination path and server context. It must either return a
    file-like object ready for writing or None if the path is invalid."""

    def __init__(self,
                 tftproot='/tftpboot',
                 dyn_file_func=None,
                 upload_open=None):
        self.listenip = None
        self.listenport = None
        self.sock = None
        # FIXME: What about multiple roots?
        self.root = os.path.abspath(tftproot)
        self.dyn_file_func = dyn_file_func
        self.upload_open = upload_open
        # A dict of sessions, where each session is keyed by a string like
        # ip:tid for the remote end.
        self.sessions = {}
        # A threading event to help threads synchronize with the server
        # is_running state.
        self.is_running = threading.Event()

        self.shutdown_gracefully = False
        self.shutdown_immediately = False

        for name in 'dyn_file_func', 'upload_open':
            attr = getattr(self, name)
            if attr and not callable(attr):
                raise TftpException("{} supplied, but it is not callable.".format(name))
        if os.path.exists(self.root):
            log.debug("tftproot %s does exist", self.root)
            if not os.path.isdir(self.root):
                raise TftpException("The tftproot must be a directory.")
            else:
                log.debug("tftproot %s is a directory" % self.root)
                if os.access(self.root, os.R_OK):
                    log.debug("tftproot %s is readable" % self.root)
                else:
                    raise TftpException("The tftproot must be readable")
                if os.access(self.root, os.W_OK):
                    log.debug("tftproot %s is writable" % self.root)
                else:
                    log.warning("The tftproot %s is not writable" % self.root)
        else:
            raise TftpException("The tftproot does not exist.")

    def listen(self, listenip="", listenport=DEF_TFTP_PORT,
               timeout=SOCK_TIMEOUT, retries=DEF_TIMEOUT_RETRIES):
        """Start a server listening on the supplied interface and port. This
        defaults to INADDR_ANY (all interfaces) and UDP port 69. You can also
        supply a different socket timeout value, if desired."""
        tftp_factory = TftpPacketFactory()

        # Don't use new 2.5 ternary operator yet
        # listenip = listenip if listenip else '0.0.0.0'
        if not listenip:
            listenip = '0.0.0.0'
        log.info("Server requested on ip %s, port %s" % (listenip, listenport))
        try:
            # FIXME - sockets should be non-blocking
            self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
            self.sock.bind((listenip, listenport))
            _, self.listenport = self.sock.getsockname()
        except socket.error as err:
            # Reraise it for now.
            raise err

        self.is_running.set()

        log.info("Starting receive loop...")
        while True:
            log.debug("shutdown_immediately is %s" % self.shutdown_immediately)
            log.debug("shutdown_gracefully is %s" % self.shutdown_gracefully)
            if self.shutdown_immediately:
                log.warning("Shutting down now. Session count: %d" % len(self.sessions))
                self.sock.close()
                for key in self.sessions:
                    self.sessions[key].end()
                self.sessions = []
                break

            elif self.shutdown_gracefully:
                if not self.sessions:
                    log.warning("In graceful shutdown mode and all sessions complete.")
                    self.sock.close()
                    break

            # Build the inputlist array of sockets to select() on.
            inputlist = [self.sock]
            for key in self.sessions:
                inputlist.append(self.sessions[key].sock)

            # Block until some socket has input on it.
            log.debug("Performing select on this inputlist: %s", inputlist)
            try:
                readyinput, readyoutput, readyspecial = select.select(inputlist, [], [], timeout)
            except select.error as err:
                if err[0] == EINTR:
                    # Interrupted system call
                    log.debug("Interrupted syscall, retrying")
                    continue
                else:
                    raise

            deletion_list = []

            # Handle the available data, if any. Maybe we timed-out.
            for readysock in readyinput:
                # Is the traffic on the main server socket? ie. new session?
                if readysock == self.sock:
                    log.debug("Data ready on our main socket")
                    buffer, (raddress, rport) = self.sock.recvfrom(MAX_BLKSIZE)

                    log.debug("Read %d bytes", len(buffer))

                    if self.shutdown_gracefully:
                        log.warning("Discarding data on main port, in graceful shutdown mode")
                        continue

                    # Forge a session key based on the client's IP and port,
                    # which should safely work through NAT.
                    key = "%s:%s" % (raddress, rport)

                    if not key in self.sessions:
                        log.debug("Creating new server context for session key = %s" % key)
                        self.sessions[key] = TftpContextServer(raddress,
                                                               rport,
                                                               timeout,
                                                               self.root,
                                                               self.dyn_file_func,
                                                               self.upload_open,
                                                               retries=retries)
                        try:
                            self.sessions[key].start(buffer)
                        except TftpException as err:
                            deletion_list.append(key)
                            log.error("Fatal exception thrown from session %s: %s" % (key, str(err)))
                    else:
                        log.warning("received traffic on main socket for existing session??")
                    log.info("Currently handling these sessions:")
                    for session_key, session in list(self.sessions.items()):
                        log.info("    %s" % session)

                else:
                    # Must find the owner of this traffic.
                    for key in self.sessions:
                        if readysock == self.sessions[key].sock:
                            log.debug("Matched input to session key %s" % key)
                            try:
                                self.sessions[key].cycle()
                                if self.sessions[key].state == None:
                                    log.info("Successful transfer.")
                                    deletion_list.append(key)
                            except TftpException as err:
                                deletion_list.append(key)
                                log.error("Fatal exception thrown from session %s: %s" % (key, str(err)))
                            # Break out of for loop since we found the correct
                            # session.
                            break
                    else:
                        log.error("Can't find the owner for this packet. "
                                  "Discarding.")

            log.debug("Looping on all sessions to check for timeouts")
            now = time.time()
            for key in self.sessions:
                try:
                    self.sessions[key].checkTimeout(now)
                except TftpTimeout as err:
                    log.error(str(err))
                    self.sessions[key].retry_count += 1
                    if self.sessions[key].retry_count >= self.sessions[key].retries:
                        log.debug("hit max retries on %s, giving up" % self.sessions[key])
                        deletion_list.append(key)
                    else:
                        log.debug("resending on session %s" % self.sessions[key])
                        self.sessions[key].state.resendLast()

            log.debug("Iterating deletion list.")
            for key in deletion_list:
                log.info('')
                log.info("Session %s complete" % key)
                if key in self.sessions:
                    log.debug("Gathering up metrics from session before deleting")
                    self.sessions[key].end()
                    metrics = self.sessions[key].metrics
                    if metrics.duration == 0:
                        log.info("Duration too short, rate undetermined")
                    else:
                        log.info("Transferred %d bytes in %.2f seconds" % (metrics.bytes, metrics.duration))
                        log.info("Average rate: %.2f kbps" % metrics.kbps)
                    log.info("%.2f bytes in resent data" % metrics.resent_bytes)
                    log.info("%d duplicate packets" % metrics.dupcount)
                    log.debug("Deleting session %s" % key)
                    del self.sessions[key]
                    log.debug("Session list is now %s" % self.sessions)
                else:
                    log.warning("Strange, session %s is not on the deletion list" % key)

        self.is_running.clear()

        log.debug("server returning from while loop")
        self.shutdown_gracefully = self.shutdown_immediately = False

    def stop(self, now=False):
        """Stop the server gracefully. Do not take any new transfers,
        but complete the existing ones. If force is True, drop everything
        and stop. Note, immediately will not interrupt the select loop, it
        will happen when the server returns on ready data, or a timeout.
        ie. SOCK_TIMEOUT"""
        if now:
            self.shutdown_immediately = True
        else:
            self.shutdown_gracefully = True
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2014 Compassion CH (http://www.compassion.ch)
#    Releasing children from poverty in Jesus' name
#    @author: Emanuel Cino, Cyril Sester
#
#    The licence is in the file __openerp__.py
#
##############################################################################

import requests
import json
from openerp.osv import orm, fields
from openerp.tools.translate import _
from openerp.tools.config import config
import pdb

class compassion_child(orm.Model):
    """ A sponsored child """
    _name = 'compassion.child'
    _rec_name = 'code'

    def get_portrait(self, cr, uid, ids, name, args, context=None):
        attachment_obj = self.pool.get('ir.attachment')
        ret = {}
        for child_id in ids:
            child = self.browse(cr, uid, child_id, context)
            case_study_id = -1
            if child.case_study_ids:
                case_study_id = child.case_study_ids[-1].id
            attachment_ids = attachment_obj.search(
                cr, uid, [('res_model', '=', 'compassion.child.property'),
                          ('res_id', '=', case_study_id),
                          ('datas_fname', '=', 'Headshot.jpeg')],
                limit=1, context=context)
            if not attachment_ids:
                ret[child_id] = None
                continue

            attachment = attachment_obj.browse(cr, uid, attachment_ids[0],
                                               context)
            ret[child_id] = attachment.datas

        return ret

    _columns = {
        'name': fields.char(_("Name"), size=128),
        'firstname': fields.char(_("Firstname"), size=128),
        'code': fields.char(_("Child code"), size=128, required=True),
        'unique_id': fields.char(_("Unique ID"), size=128),
        'birthdate': fields.date(_("Birthdate")),
        'type': fields.selection(
            [('CDSP', 'CDSP'),
             ('LDP', 'LDP')], _('Type of sponsorship program'), required=True),
        'date': fields.date(_("Allocation date"),
                            help=_("The date at which Compass allocated "
                                   "this child to Switzerland")),
        'gender': fields.selection(
            [('F', _('Female')),
             ('M', _('Male'))], _('Gender')),
        'completion_date': fields.date(_("Completion date")),
        'desc_en': fields.text(_('English description')),
        'desc_fr': fields.text(_('French description')),
        'desc_de': fields.text(_('German description')),
        'desc_it': fields.text(_('Italian description')),
        'start_date': fields.date(_("Start date")),
        'case_study_ids': fields.one2many(
            'compassion.child.property', 'child_id', string=_('Case studies'),
            readonly=False),  # FIXME readonly
        'portrait': fields.function(get_portrait, type='binary',
                                    string=_('Portrait')),
    }

    _defaults = {
        'type': 'CDSP'
    }

    def get_basic_informations(self, cr, uid, ids, context=None):
        if not isinstance(ids, list):
            ids = [ids]

        for child in self.browse(cr, uid, ids, context):
            case_study = child.case_study_ids[-1]
            if case_study:
                self.write(cr, uid, [child.id], {
                    'name': case_study.name,
                    'firstname': case_study.firstname,
                    'birthdate': case_study.birthdate,
                    'gender': case_study.gender
                    }, context=context)
        return True

    def get_last_case_study(self, cr, uid, ids, context=None):
        ''' Get the most recent case study and updates portrait picture '''
        if not isinstance(ids, list):
            ids = [ids]
        ret = {}
        for child in self.browse(cr, uid, ids, context):
            ret[child.id] = self._get_case_study(cr, uid, child, context)
            self._get_picture(cr, uid, child, 'Fullshot',
                              300, 1500, 1200, context=context)
            self._get_picture(cr, uid, child, context=context)
        return ret

    def generate_descriptions(self, cr, uid, child_id, context=None):
        child = self.browse(cr, uid, child_id, context)
        if not child:
            raise orm.except_orm('ObjectError', _('No valid child id given !'))
        child = child[0]
        if not child.case_study_ids:
            raise orm.except_orm('ValueError',
                                 _('Cannot generate a description '
                                   'for a child without a case study'))
        case_study = child.case_study_ids[-1]
        context['child_id'] = child_id
        context['property_id'] = case_study.id

        return {
            'name': _('Description generation'),
            'type': 'ir.actions.act_window',
            'view_type': 'form',
            'view_mode': 'form',
            'res_model': 'child.description.wizard',
            'context': context,
            'target': 'new',
        }

    ##################################################
    #            Case study retrieving               #
    ##################################################
    def _get_case_study(self, cr, uid, child, context=None):
        ''' Get case study from compassion webservices and parse
            the json response.
            Returns id of generated case_study or None if failed
        '''
        url = self._get_url(child.code, 'casestudy')
        r = requests.get(url)
        if not r.status_code/100 == 2:
            raise orm.except_orm('NetworkError',
                                 _('An error occured while fetching the last '
                                   'case study for child %s.') % child.code)

        json_data = json.loads(r.text)
        vals = {
            'child_id': child.id,
            'info_date': json_data['childCaseStudyDate'],
            'name': json_data['childName'],
            'firstname': json_data['childPersonalName'],
            'gender': json_data['gender'],
            'birthdate': json_data['birthDate'],
        }

        value_obj = self.pool.get('compassion.translated.value')
        values = []

        """ cs_sections_mapping holds the mapping of sections in case study
            to property.
            cs_sections_mapping is a dict of lists of the following form:
            {'property_name': ['CaseStudySectionName',
                               'OtherSectionAttribute']}
            For more information see documentation of compass at
            the following link (2 lines):
            http://developer.compassion.com/docs/read/private_cornerstone_test/
            REST_Get_Child_Case_Study
        """
        cs_sections_mapping = {
            'christian_activities': ['christianActivities',
                                     'otherChristianActivities'],
            'family_duties': ['familyDuties',
                              'otherFamilyDuties'],
            'hobbies': ['hobbiesAndSports', 'otherHobbies'],
            'health_conditions': ['healthConditions',
                                  'otherHealthConditions'],
            'guardians': ['guardians', False],
        }

        for prop_name, cs_section in cs_sections_mapping.iteritems():
            section_values = json_data[cs_section[0]]
            if cs_section[1]:
                other_values = json_data[cs_section[1]]
                if isinstance(other_values, list):
                    section_values += json_data[cs_section[1]]
                elif not other_values == 'None':
                    section_values.append(other_values)
            values.extend(value_obj.get_value_ids(cr, uid, section_values,
                                                  prop_name, context))

        """ Natural Parents and Employment Section.
            nps_sections_mapping is of the form:
            {'CaseStudySectionName':['property_name_male',
             'property_name_female', 'CaseStudyKey_male',
             'CaseStudyKey_female']}
        """
        npe_sections_mapping = {
            'naturalParents': ['father', 'mother', 'father', 'mother'],
            'employment': ['male_guardian', 'female_guardian',
                           'fatherOrMaleGuardian', 'motherOrFemaleGuardian'],
        }
        for section, prop_names in npe_sections_mapping.iteritems():
            for key, value in json_data[section].iteritems():
                property_name = ''
                if key.startswith('father'):
                    property_name = prop_names[0]
                elif key.startswith('mother'):
                    property_name = prop_names[1]
                else:
                    continue

                # otherEmployment is a string,
                # as well as motherIllnes and some others
                if value:
                    value = (key.replace(prop_names[2],
                             '').replace(prop_names[3], ''))
                else:
                    continue

                values.append(value_obj.get_value_ids(cr, uid, value,
                              property_name, context))
        # Other sections
        values.append(value_obj.get_value_ids(
            cr, uid, json_data['naturalParents']['maritalStatusOfParents'],
            'marital_status', context))
        vals['us_school_level'] = json_data['schooling']['uSSchoolEquivalent']
        values.append(value_obj.get_value_ids(cr, uid, json_data['schooling']
                                              ['schoolPerformance'],
                                              'school_performance', context))
        values.append(value_obj.get_value_ids(cr, uid, json_data['schooling']
                                              ['childsBestSubject'],
                                              'school_best_subject', context))
        vals['attending_school_flag'] = bool(json_data['schooling']
                                             ['childAttendingSchool'])
        vals['nb_children_family'] = int(json_data['familySize']
                                         ['totalFamilyFemalesUnder18'])
        vals['nb_sisters'] = int(json_data['familySize']
                                 ['totalFamilyFemalesUnder18'])
        vals['nb_children_family'] += int(json_data['familySize']
                                          ['totalFamilyMalesUnder18'])-1
        vals['nb_brothers'] = int(json_data['familySize']
                                  ['totalFamilyMalesUnder18'])
        if child.gender == 'M':
            vals['nb_brothers'] -= 1
        else:
            vals['nb_sisters'] -= 1
        pdb.set_trace()
        vals['hobbies_ids'] = [(6, 0, values)]
        child_prop_obj = self.pool.get('compassion.child.property')
        prop_id = child_prop_obj.create(cr, uid, vals, context)
        return prop_id

    def _get_picture(self, cr, uid, child, type='Headshot', dpi=72, width=400,
                     height=400, format='jpeg', context=None):
        ''' Gets a picture from Compassion webservice '''
        url = self._get_url(child.code, 'image')
        url += '&Height=%s&Width=%s&DPI=%s&ImageFormat=%s&ImageType=%s' \
            % (height, width, dpi, format, type)
        r = requests.get(url)
        if not r.status_code/100 == 2:
            raise orm.except_orm('NetworkError',
                                 _('An error occured while fetching the last '
                                   'picture for child %s.') % child.code)
        data = json.loads(r.text)['image']['imageData']
        attachment_obj = self.pool.get('ir.attachment')
        if not context:
            context = {}
        context['store_fname'] = type + '.' + format
        attachment_obj.create(cr, uid,
                              {'datas_fname': type + '.' + format,
                               'res_model': 'compassion.child.property',
                               'res_id': child.case_study_ids[-1].id,
                               'datas': data,
                               'name': type + '.' + format}, context)
        return False

    def _get_url(self, child_code, api_mess):
        url = config.get('compass_url')
        api_key = config.get('compass_api_key')
        if not url or not api_key:
            raise orm.except_orm('ConfigError',
                                 _('Missing compass_url or compass_api_key '
                                   'in conf file'))
        if url.endswith('/'):
            url = url[:-1]
        url += ('/ci/v1/children/' + child_code + '/' + api_mess + '?api_key='
                + api_key)
        return url
<commit_msg>Removed pdb breakpoint.<commit_after># -*- encoding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2014 Compassion CH (http://www.compassion.ch)
#    Releasing children from poverty in Jesus' name
#    @author: Emanuel Cino, Cyril Sester
#
#    The licence is in the file __openerp__.py
#
##############################################################################

import requests
import json
from openerp.osv import orm, fields
from openerp.tools.translate import _
from openerp.tools.config import config

class compassion_child(orm.Model):
    """ A sponsored child """
    _name = 'compassion.child'
    _rec_name = 'code'

    def get_portrait(self, cr, uid, ids, name, args, context=None):
        attachment_obj = self.pool.get('ir.attachment')
        ret = {}
        for child_id in ids:
            child = self.browse(cr, uid, child_id, context)
            case_study_id = -1
            if child.case_study_ids:
                case_study_id = child.case_study_ids[-1].id
            attachment_ids = attachment_obj.search(
                cr, uid, [('res_model', '=', 'compassion.child.property'),
                          ('res_id', '=', case_study_id),
                          ('datas_fname', '=', 'Headshot.jpeg')],
                limit=1, context=context)
            if not attachment_ids:
                ret[child_id] = None
                continue

            attachment = attachment_obj.browse(cr, uid, attachment_ids[0],
                                               context)
            ret[child_id] = attachment.datas

        return ret

    _columns = {
        'name': fields.char(_("Name"), size=128),
        'firstname': fields.char(_("Firstname"), size=128),
        'code': fields.char(_("Child code"), size=128, required=True),
        'unique_id': fields.char(_("Unique ID"), size=128),
        'birthdate': fields.date(_("Birthdate")),
        'type': fields.selection(
            [('CDSP', 'CDSP'),
             ('LDP', 'LDP')], _('Type of sponsorship program'), required=True),
        'date': fields.date(_("Allocation date"),
                            help=_("The date at which Compass allocated "
                                   "this child to Switzerland")),
        'gender': fields.selection(
            [('F', _('Female')),
             ('M', _('Male'))], _('Gender')),
        'completion_date': fields.date(_("Completion date")),
        'desc_en': fields.text(_('English description')),
        'desc_fr': fields.text(_('French description')),
        'desc_de': fields.text(_('German description')),
        'desc_it': fields.text(_('Italian description')),
        'start_date': fields.date(_("Start date")),
        'case_study_ids': fields.one2many(
            'compassion.child.property', 'child_id', string=_('Case studies'),
            readonly=False),  # FIXME readonly
        'portrait': fields.function(get_portrait, type='binary',
                                    string=_('Portrait')),
    }

    _defaults = {
        'type': 'CDSP'
    }

    def get_basic_informations(self, cr, uid, ids, context=None):
        if not isinstance(ids, list):
            ids = [ids]

        for child in self.browse(cr, uid, ids, context):
            case_study = child.case_study_ids[-1]
            if case_study:
                self.write(cr, uid, [child.id], {
                    'name': case_study.name,
                    'firstname': case_study.firstname,
                    'birthdate': case_study.birthdate,
                    'gender': case_study.gender
                    }, context=context)
        return True

    def get_last_case_study(self, cr, uid, ids, context=None):
        ''' Get the most recent case study and updates portrait picture '''
        if not isinstance(ids, list):
            ids = [ids]
        ret = {}
        for child in self.browse(cr, uid, ids, context):
            ret[child.id] = self._get_case_study(cr, uid, child, context)
            self._get_picture(cr, uid, child, 'Fullshot',
                              300, 1500, 1200, context=context)
            self._get_picture(cr, uid, child, context=context)
        return ret

    def generate_descriptions(self, cr, uid, child_id, context=None):
        child = self.browse(cr, uid, child_id, context)
        if not child:
            raise orm.except_orm('ObjectError', _('No valid child id given !'))
        child = child[0]
        if not child.case_study_ids:
            raise orm.except_orm('ValueError',
                                 _('Cannot generate a description '
                                   'for a child without a case study'))
        case_study = child.case_study_ids[-1]
        context['child_id'] = child_id
        context['property_id'] = case_study.id

        return {
            'name': _('Description generation'),
            'type': 'ir.actions.act_window',
            'view_type': 'form',
            'view_mode': 'form',
            'res_model': 'child.description.wizard',
            'context': context,
            'target': 'new',
        }

    ##################################################
    #            Case study retrieving               #
    ##################################################
    def _get_case_study(self, cr, uid, child, context=None):
        ''' Get case study from compassion webservices and parse
            the json response.
            Returns id of generated case_study or None if failed
        '''
        url = self._get_url(child.code, 'casestudy')
        r = requests.get(url)
        if not r.status_code/100 == 2:
            raise orm.except_orm('NetworkError',
                                 _('An error occured while fetching the last '
                                   'case study for child %s.') % child.code)

        json_data = json.loads(r.text)
        vals = {
            'child_id': child.id,
            'info_date': json_data['childCaseStudyDate'],
            'name': json_data['childName'],
            'firstname': json_data['childPersonalName'],
            'gender': json_data['gender'],
            'birthdate': json_data['birthDate'],
        }

        value_obj = self.pool.get('compassion.translated.value')
        values = []

        """ cs_sections_mapping holds the mapping of sections in case study
            to property.
            cs_sections_mapping is a dict of lists of the following form:
            {'property_name': ['CaseStudySectionName',
                               'OtherSectionAttribute']}
            For more information see documentation of compass at
            the following link (2 lines):
            http://developer.compassion.com/docs/read/private_cornerstone_test/
            REST_Get_Child_Case_Study
        """
        cs_sections_mapping = {
            'christian_activities': ['christianActivities',
                                     'otherChristianActivities'],
            'family_duties': ['familyDuties',
                              'otherFamilyDuties'],
            'hobbies': ['hobbiesAndSports', 'otherHobbies'],
            'health_conditions': ['healthConditions',
                                  'otherHealthConditions'],
            'guardians': ['guardians', False],
        }

        for prop_name, cs_section in cs_sections_mapping.iteritems():
            section_values = json_data[cs_section[0]]
            if cs_section[1]:
                other_values = json_data[cs_section[1]]
                if isinstance(other_values, list):
                    section_values += json_data[cs_section[1]]
                elif not other_values == 'None':
                    section_values.append(other_values)
            values.extend(value_obj.get_value_ids(cr, uid, section_values,
                                                  prop_name, context))

        """ Natural Parents and Employment Section.
            nps_sections_mapping is of the form:
            {'CaseStudySectionName':['property_name_male',
             'property_name_female', 'CaseStudyKey_male',
             'CaseStudyKey_female']}
        """
        npe_sections_mapping = {
            'naturalParents': ['father', 'mother', 'father', 'mother'],
            'employment': ['male_guardian', 'female_guardian',
                           'fatherOrMaleGuardian', 'motherOrFemaleGuardian'],
        }
        for section, prop_names in npe_sections_mapping.iteritems():
            for key, value in json_data[section].iteritems():
                property_name = ''
                if key.startswith('father'):
                    property_name = prop_names[0]
                elif key.startswith('mother'):
                    property_name = prop_names[1]
                else:
                    continue

                # otherEmployment is a string,
                # as well as motherIllnes and some others
                if value:
                    value = (key.replace(prop_names[2],
                             '').replace(prop_names[3], ''))
                else:
                    continue

                values.append(value_obj.get_value_ids(cr, uid, value,
                              property_name, context))
        # Other sections
        values.append(value_obj.get_value_ids(
            cr, uid, json_data['naturalParents']['maritalStatusOfParents'],
            'marital_status', context))
        vals['us_school_level'] = json_data['schooling']['uSSchoolEquivalent']
        values.append(value_obj.get_value_ids(cr, uid, json_data['schooling']
                                              ['schoolPerformance'],
                                              'school_performance', context))
        values.append(value_obj.get_value_ids(cr, uid, json_data['schooling']
                                              ['childsBestSubject'],
                                              'school_best_subject', context))
        vals['attending_school_flag'] = bool(json_data['schooling']
                                             ['childAttendingSchool'])
        vals['nb_children_family'] = int(json_data['familySize']
                                         ['totalFamilyFemalesUnder18'])
        vals['nb_sisters'] = int(json_data['familySize']
                                 ['totalFamilyFemalesUnder18'])
        vals['nb_children_family'] += int(json_data['familySize']
                                          ['totalFamilyMalesUnder18'])-1
        vals['nb_brothers'] = int(json_data['familySize']
                                  ['totalFamilyMalesUnder18'])
        if child.gender == 'M':
            vals['nb_brothers'] -= 1
        else:
            vals['nb_sisters'] -= 1
        vals['hobbies_ids'] = [(6, 0, values)]
        child_prop_obj = self.pool.get('compassion.child.property')
        prop_id = child_prop_obj.create(cr, uid, vals, context)
        return prop_id

    def _get_picture(self, cr, uid, child, type='Headshot', dpi=72, width=400,
                     height=400, format='jpeg', context=None):
        ''' Gets a picture from Compassion webservice '''
        url = self._get_url(child.code, 'image')
        url += '&Height=%s&Width=%s&DPI=%s&ImageFormat=%s&ImageType=%s' \
            % (height, width, dpi, format, type)
        r = requests.get(url)
        if not r.status_code/100 == 2:
            raise orm.except_orm('NetworkError',
                                 _('An error occured while fetching the last '
                                   'picture for child %s.') % child.code)
        data = json.loads(r.text)['image']['imageData']
        attachment_obj = self.pool.get('ir.attachment')
        if not context:
            context = {}
        context['store_fname'] = type + '.' + format
        attachment_obj.create(cr, uid,
                              {'datas_fname': type + '.' + format,
                               'res_model': 'compassion.child.property',
                               'res_id': child.case_study_ids[-1].id,
                               'datas': data,
                               'name': type + '.' + format}, context)
        return False

    def _get_url(self, child_code, api_mess):
        url = config.get('compass_url')
        api_key = config.get('compass_api_key')
        if not url or not api_key:
            raise orm.except_orm('ConfigError',
                                 _('Missing compass_url or compass_api_key '
                                   'in conf file'))
        if url.endswith('/'):
            url = url[:-1]
        url += ('/ci/v1/children/' + child_code + '/' + api_mess + '?api_key='
                + api_key)
        return url
<|endoftext|>