<commit_before>from . import fields, utils, exceptions
from abc import ABCMeta
import redis
import json

class MetaModel(ABCMeta):

    def __new__(mcs, model_name, bases, attrs):
        cls = super().__new__(mcs, model_name, bases, attrs)
        cls._pk = None
        cls._fields = {}
        cls._foreign = {}
        # Pre-generated DB keys (they're constant)
        cls._unique_keys = {}
        cls._unique_fields = {}
        cls._required_fields = {}
        cls._plain_fields = {}
        cls._standalone_proxy = {}
        cls._standalone_auto = {}
        cls._scripts = {}

        field_attrs = ((k, v) for k, v in attrs.items()
                         if isinstance(v, fields.Field))

        for name, field in field_attrs:
            field.name, field.model_name = name, model_name
            cls._fields[name] = field
            if isinstance(field, fields.PrimaryKey):
                cls._primary_key = name

            if isinstance(field, fields.Foreign):
                cls._foreign[name] = field

            if field.unique:
                cls._unique_fields[name] = field
                cls._unique_keys[name] = cls.qualified(name)
            elif field.standalone:
                if field.auto:
                    cls._standalone_auto[name] = field
                else:
                    cls._standalone_proxy[name] = field
            else:
                cls._plain_fields[name] = field

            if field.required:
                cls._required_fields[name] = field

        cls._standalone = dict(cls._standalone_proxy, **cls._standalone_auto)
        cls._plain = dict(cls._unique_fields, **cls._plain_fields)

        try:
            cls.__redis__ = cls.redis
        except AttributeError:
            # Base model class, ignore it
            pass
        else:
            # We can register those now and change the connection later
            if cls._unique_fields:
                cls._scripts['unique'] = cls.__redis__.register_script(
                                                utils.SCRIPTS['unique'])
            # Get some information from the connection instance
            # We need encoding and/or decode_responses to handle conversion
            _params = cls.redis.connection_pool.connection_kwargs
            cls._redis_encoding = _params['encoding']

        return cls

class Model(metaclass=MetaModel):

    _field_sep = ':'

    def __init__(self, data=None, **ka):
        self.__context_depth__ = 0
        self.data = {}
        # If the PK is present, we assume that the rest of fields
        # are there as well
        if data is None or self._primary_key not in data:
            if len(ka) > 1:
                raise ValueError('You can only search by 1 unique field')
            # Will only search by one pair
            field, value = ka.popitem()
            if field == self._primary_key:
                self.data.update(self._get_by_pk(value))
            elif field not in self._unique_fields:
                raise TypeError('Attempted to get by non-unique'
                                ' field {!r}'.format(field))
            else:
                self.data.update(self._get_unique(field, value))

        self.data.update(data or {})
        self._prepare()

    @property
    def primary_key(self):
        return self.data[self._primary_key]

    @classmethod
    def _get_by_pk(cls, pk):
        res, rv = cls.__redis__.hgetall(cls.qualified(pk=pk)), {}
        for key, value in res.items():
            decoded = fields.String.from_redis(key, cls._redis_encoding)
            ob = cls._plain[decoded]
            rv[decoded] = ob.from_redis(value, cls._redis_encoding)
        return rv

    @classmethod
    def _get_unique(cls, field, value):
        pk = cls.__redis__.hget(cls.qualified(field), value)
        decoded = fields.PrimaryKey.from_redis(pk, cls._redis_encoding)
        return cls._get_by_pk(decoded)

    def _update_plain(self, new_data):
        save = new_data.copy()
        for k, v in save.items():
            save[k] = self._plain[k].to_redis(v, self._redis_encoding)
        self.redis.hmset(self.qualified(pk=self.primary_key), save)
        self.data.update(new_data)

    def _update_unique(self, new_data):
        self._write_unique(new_data, self.primary_key, self.redis)
        self._update_plain(new_data)

    # TODO: Delete

    @classmethod
    def get_foreign(cls, name=None):
        if name is None:
            return list(cls._foreign)
        return [k for k, v in cls._foreign.items() if v.foreign == name]

    def _prepare(self):
        for field, ob in self._foreign.items():
            if field not in self.data:
                continue
            # TODO: There should be a better way
            gen = (t for t in Model.__subclasses__()
                      if t.__name__ == ob.foreign)
            ft, fv = next(gen), self.data[field]
            if not isinstance(fv, ft):
                ff = ft.get_foreign(type(self).__name__)
                self.data[field] = ft(data=dict.fromkeys(ff, self),
                                      **{ft._primary_key: fv})

    def __enter__(self):
        if not self.__context_depth__:
            pipe = self.__redis__.pipeline()
            self.redis = pipe.__enter__()
        self.__context_depth__ += 1
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        # Must be set at the beginning of this method
        self.__context_depth__ -= 1
        if not self.__context_depth__:
            self.redis.execute()
            self.redis.__exit__(exc_type, exc_value, traceback)
            self.redis = self.__redis__

    @classmethod
    def qualified(cls, *args, pk=None):
        parts = [cls.__name__]
        if pk is not None:
            parts.append(pk)
        parts.extend(args)
        return cls._field_sep.join(parts)

    @classmethod
    def _write_unique(cls, data, pk, connection=None):
        # Must have the same order
        keys, values, fields = [], [], []
        for k, v in data.items():
            keys.append(cls._unique_keys[k])
            fields.append(k)
            values.append(v)

        kwargs = {'args': [pk, json.dumps(values)], 'keys': keys}
        if connection is not None:
            kwargs['client'] = connection
        res = cls._scripts['unique'](**kwargs)
        # 0 for success
        # 1 ... len(fields) is an error
        #       (position of the first duplicate field from 'fields')
        if res:
            res -= 1
            raise exceptions.DuplicateEntry(fields[res], values[res])

    @classmethod
    def new(cls, **ka):
        if cls._required_fields.keys() - ka.keys():
            raise exceptions.MissingFields

        if cls._primary_key not in ka:
            raise exceptions.NoPrimaryKey

        pk = ka[cls._primary_key]

        if cls._unique_fields:
            data = {k: ka[k] for k in
                     cls._unique_keys.keys() & ka.keys()}
            cls._write_unique(data, pk)

        main_key = cls.qualified(pk=pk)

        # Set the rest of fields

        # All standalone fields
        with cls.__redis__.pipeline() as pipe:
            for field, ob in cls._standalone.items():
                try:
                    value = ka[field]
                except KeyError:
                    continue
                else:
                    ob.type.save(cls.qualified(field, pk=pk), pipe, value)
            pipe.execute()

        # Unique and plain fields
        save, data = {cls._primary_key: pk}, {cls._primary_key: pk}
        for field, ob in cls._plain.items():
            try:
                value = ka[field]
            except KeyError:
                continue
            else:
                save[field] = ob.to_redis(value, cls._redis_encoding)
                data[field] = value

        cls.__redis__.hmset(main_key, save)
        return cls(data)

    def __repr__(self):
        return ("<{0.__name__}/{0._primary_key}={1!r} instance"
                " at {2:#x}>").format(type(self), self.primary_key, id(self))

    # def __eq__(self, other):
    #     if not isinstance(other, type(self)):
    #         raise NotImplementedError('Can\'t compare instances of {!r} and '
    #                                   '{!r}'.format(type(self), type(other)))
    #     else:
    #         # FIXME:
    #         return self.data[self._pk] == other.data[other._pk]

    # def __hash__(self):
    #     return hash((type(self), self.data[self._pk]))

    __str__ = __repr__<commit_msg>Simplified MetaModel.__new__<commit_after>from . import fields, utils, exceptions
from abc import ABCMeta
import redis
import json

class MetaModel(ABCMeta):

    def __new__(mcs, model_name, bases, attrs):
        mappings = ('_fields', '_unique_keys', '_unique_fields',
                    '_required_fields', '_plain_fields', '_standalone_proxy',
                    '_standalone_auto', '_scripts', '_foreign')
        for m in mappings:
            attrs[m] = {}
        cls = super().__new__(mcs, model_name, bases, attrs)
        cls._pk = None

        field_attrs = ((k, v) for k, v in attrs.items()
                         if isinstance(v, fields.Field))

        for name, field in field_attrs:
            field.name, field.model_name = name, model_name
            cls._fields[name] = field
            if isinstance(field, fields.PrimaryKey):
                cls._primary_key = name

            if isinstance(field, fields.Foreign):
                cls._foreign[name] = field

            if field.unique:
                cls._unique_fields[name] = field
                cls._unique_keys[name] = cls.qualified(name)
            elif field.standalone:
                if field.auto:
                    cls._standalone_auto[name] = field
                else:
                    cls._standalone_proxy[name] = field
            else:
                cls._plain_fields[name] = field

            if field.required:
                cls._required_fields[name] = field

        cls._standalone = dict(cls._standalone_proxy, **cls._standalone_auto)
        cls._plain = dict(cls._unique_fields, **cls._plain_fields)

        try:
            cls.__redis__ = cls.redis
        except AttributeError:
            # Base model class, ignore it
            pass
        else:
            # We can register those now and change the connection later
            if cls._unique_fields:
                cls._scripts['unique'] = cls.__redis__.register_script(
                                                utils.SCRIPTS['unique'])
            # Get some information from the connection instance
            # We need encoding and/or decode_responses to handle conversion
            _params = cls.redis.connection_pool.connection_kwargs
            cls._redis_encoding = _params['encoding']

        return cls

class Model(metaclass=MetaModel):

    _field_sep = ':'

    def __init__(self, data=None, **ka):
        self.__context_depth__ = 0
        self.data = {}
        # If the PK is present, we assume that the rest of fields
        # are there as well
        if data is None or self._primary_key not in data:
            if len(ka) > 1:
                raise ValueError('You can only search by 1 unique field')
            # Will only search by one pair
            field, value = ka.popitem()
            if field == self._primary_key:
                self.data.update(self._get_by_pk(value))
            elif field not in self._unique_fields:
                raise TypeError('Attempted to get by non-unique'
                                ' field {!r}'.format(field))
            else:
                self.data.update(self._get_unique(field, value))

        self.data.update(data or {})
        self._prepare()

    @property
    def primary_key(self):
        return self.data[self._primary_key]

    @classmethod
    def _get_by_pk(cls, pk):
        res, rv = cls.__redis__.hgetall(cls.qualified(pk=pk)), {}
        for key, value in res.items():
            decoded = fields.String.from_redis(key, cls._redis_encoding)
            ob = cls._plain[decoded]
            rv[decoded] = ob.from_redis(value, cls._redis_encoding)
        return rv

    @classmethod
    def _get_unique(cls, field, value):
        pk = cls.__redis__.hget(cls.qualified(field), value)
        decoded = fields.PrimaryKey.from_redis(pk, cls._redis_encoding)
        return cls._get_by_pk(decoded)

    def _update_plain(self, new_data):
        save = new_data.copy()
        for k, v in save.items():
            save[k] = self._plain[k].to_redis(v, self._redis_encoding)
        self.redis.hmset(self.qualified(pk=self.primary_key), save)
        self.data.update(new_data)

    def _update_unique(self, new_data):
        self._write_unique(new_data, self.primary_key, self.redis)
        self._update_plain(new_data)

    # TODO: Delete

    @classmethod
    def get_foreign(cls, name=None):
        if name is None:
            return list(cls._foreign)
        return [k for k, v in cls._foreign.items() if v.foreign == name]

    def _prepare(self):
        for field, ob in self._foreign.items():
            if field not in self.data:
                continue
            # TODO: There should be a better way
            gen = (t for t in Model.__subclasses__()
                      if t.__name__ == ob.foreign)
            ft, fv = next(gen), self.data[field]
            if not isinstance(fv, ft):
                ff = ft.get_foreign(type(self).__name__)
                self.data[field] = ft(data=dict.fromkeys(ff, self),
                                      **{ft._primary_key: fv})

    def __enter__(self):
        if not self.__context_depth__:
            pipe = self.__redis__.pipeline()
            self.redis = pipe.__enter__()
        self.__context_depth__ += 1
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        # Must be set at the beginning of this method
        self.__context_depth__ -= 1
        if not self.__context_depth__:
            self.redis.execute()
            self.redis.__exit__(exc_type, exc_value, traceback)
            self.redis = self.__redis__

    @classmethod
    def qualified(cls, *args, pk=None):
        parts = [cls.__name__]
        if pk is not None:
            parts.append(pk)
        parts.extend(args)
        return cls._field_sep.join(parts)

    @classmethod
    def _write_unique(cls, data, pk, connection=None):
        # Must have the same order
        keys, values, fields = [], [], []
        for k, v in data.items():
            keys.append(cls._unique_keys[k])
            fields.append(k)
            values.append(v)

        kwargs = {'args': [pk, json.dumps(values)], 'keys': keys}
        if connection is not None:
            kwargs['client'] = connection
        res = cls._scripts['unique'](**kwargs)
        # 0 for success
        # 1 ... len(fields) is an error
        #       (position of the first duplicate field from 'fields')
        if res:
            res -= 1
            raise exceptions.DuplicateEntry(fields[res], values[res])

    @classmethod
    def new(cls, **ka):
        if cls._required_fields.keys() - ka.keys():
            raise exceptions.MissingFields

        if cls._primary_key not in ka:
            raise exceptions.NoPrimaryKey

        pk = ka[cls._primary_key]

        if cls._unique_fields:
            data = {k: ka[k] for k in
                     cls._unique_keys.keys() & ka.keys()}
            cls._write_unique(data, pk)

        main_key = cls.qualified(pk=pk)

        # Set the rest of fields

        # All standalone fields
        with cls.__redis__.pipeline() as pipe:
            for field, ob in cls._standalone.items():
                try:
                    value = ka[field]
                except KeyError:
                    continue
                else:
                    ob.type.save(cls.qualified(field, pk=pk), pipe, value)
            pipe.execute()

        # Unique and plain fields
        save, data = {cls._primary_key: pk}, {cls._primary_key: pk}
        for field, ob in cls._plain.items():
            try:
                value = ka[field]
            except KeyError:
                continue
            else:
                save[field] = ob.to_redis(value, cls._redis_encoding)
                data[field] = value

        cls.__redis__.hmset(main_key, save)
        return cls(data)

    def __repr__(self):
        return ("<{0.__name__}/{0._primary_key}={1!r} instance"
                " at {2:#x}>").format(type(self), self.primary_key, id(self))

    # def __eq__(self, other):
    #     if not isinstance(other, type(self)):
    #         raise NotImplementedError('Can\'t compare instances of {!r} and '
    #                                   '{!r}'.format(type(self), type(other)))
    #     else:
    #         # FIXME:
    #         return self.data[self._pk] == other.data[other._pk]

    # def __hash__(self):
    #     return hash((type(self), self.data[self._pk]))

    __str__ = __repr__<|endoftext|>


<commit_before>import time
import datetime

try:
    # py2.7
    from collections import OrderedDict
except ImportError:
    # py2.4, 2.5, 2.6
    from orderedict import OrderedDict

import simplejson
import simplejson.encoder

class FedMsgEncoder(simplejson.encoder.JSONEncoder):
    """ Encoder with support for __json__ methods. """

    def default(self, obj):
        if hasattr(obj, '__json__'):
            return obj.__json__()
        if isinstance(obj, datetime.datetime):
            return time.mktime(obj.timetuple())
        return super(FedMsgEncoder, self).default(obj)

encoder = FedMsgEncoder()

def dumps(d):
    """ Encode a dict as JSON.

    Ensure that the keys are ordered so that messages can be signed
    consistently.  See https://github.com/ralphbean/fedmsg/issues/42
    """
    return encoder.encode(OrderedDict(sorted(d.items())))

pretty_encoder = FedMsgEncoder(indent=2)
pretty_dumps = pretty_encoder.encode

loads = simplejson.loads

__all__ = [
    'pretty_dumps',
    'dumps',
    'loads',
]
<commit_msg>Added TODO note.<commit_after>import time
import datetime

try:
    # py2.7
    from collections import OrderedDict
except ImportError:
    # py2.4, 2.5, 2.6
    from orderedict import OrderedDict

import simplejson
import simplejson.encoder

class FedMsgEncoder(simplejson.encoder.JSONEncoder):
    """ Encoder with support for __json__ methods. """

    def default(self, obj):
        if hasattr(obj, '__json__'):
            return obj.__json__()
        if isinstance(obj, datetime.datetime):
            return time.mktime(obj.timetuple())
        return super(FedMsgEncoder, self).default(obj)

encoder = FedMsgEncoder()

def dumps(d):
    """ Encode a dict as JSON.

    Ensure that the keys are ordered so that messages can be signed
    consistently.  See https://github.com/ralphbean/fedmsg/issues/42
    """
    # TODO - Look into using the sort_keys argument to simplejson.  It should
    # introduce less overhead than all this potential dict-copying.
    return encoder.encode(OrderedDict(sorted(d.items())))

pretty_encoder = FedMsgEncoder(indent=2)
pretty_dumps = pretty_encoder.encode

loads = simplejson.loads

__all__ = [
    'pretty_dumps',
    'dumps',
    'loads',
]
<|endoftext|>


<commit_before>import unittest

import numpy

import chainer
from chainer.backends import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr
from chainer.testing import parameterize

@parameterize(
    {'axes': [1, 2], 'offsets': 0, 'sliced_shape': (4, 2, 1)},
    {'axes': [1, 2], 'offsets': [0, 1, 1], 'sliced_shape': (4, 2, 1)},
    {'axes': 1, 'offsets': 1, 'sliced_shape': (4, 2, 2)},
    {'axes': 1, 'offsets': [0, 1, 1], 'sliced_shape': (4, 2, 2)},
    {'axes': [], 'offsets': 0, 'new_axes': 0, 'sliced_shape': (1, 4, 3, 2)},
    {'axes': [], 'offsets': 0, 'new_axes': 2, 'sliced_shape': (4, 3, 1, 2)},
    {'axes': [], 'offsets': 0, 'new_axes': 3, 'sliced_shape': (4, 3, 2, 1)},
    {'slices': (1, -1, 0), 'sliced_shape': ()},
    {'slices': (1, -1), 'sliced_shape': (2,)},
    {'slices': (1, Ellipsis, -1), 'sliced_shape': (3,)},
    {'slices': (1, None, Ellipsis, None, -1), 'sliced_shape': (1, 3, 1)},
)
class TestGetItem(unittest.TestCase):

    def setUp(self):
        self.x_data = numpy.random.uniform(-1, 1, (4, 3, 2))
        self.shape = (4, 2, 1)
        self.gy_data = numpy.random.uniform(-1, 1, self.sliced_shape)
        self.ggx_data = numpy.random.uniform(-1, 1, (4, 3, 2))

        if not hasattr(self, 'slices'):
            # Convert axes, offsets and shape to slices
            if isinstance(self.offsets, int):
                self.offsets = tuple([self.offsets] * len(self.shape))
            if isinstance(self.axes, int):
                self.axes = tuple([self.axes])

            self.slices = [slice(None)] * len(self.shape)
            for axis in self.axes:
                self.slices[axis] = slice(
                    self.offsets[axis], self.offsets[axis] + self.shape[axis])

            if hasattr(self, 'new_axes'):
                self.slices.insert(self.new_axes, None)

            self.slices = tuple(self.slices)

    def check_forward(self, x_data):
        x = chainer.Variable(x_data)
        y = functions.get_item(x, self.slices)
        self.assertEqual(y.data.dtype, numpy.float)
        numpy.testing.assert_equal(cuda.to_cpu(x_data)[self.slices],
                                   cuda.to_cpu(y.data))

    def test_forward_cpu(self):
        self.check_forward(self.x_data)

    @attr.gpu
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x_data))

    def check_backward(self, x_data, y_grad):
        def f(x):
            return functions.get_item(x, self.slices)

        gradient_check.check_backward(
            f, (x_data,), y_grad, dtype='d')

    def test_backward_cpu(self):
        self.check_backward(self.x_data, self.gy_data)

    @attr.gpu
    def test_backward_gpu(self):
        self.check_backward(cuda.to_gpu(self.x_data),
                            cuda.to_gpu(self.gy_data))

    def check_double_backward(self, x_data, y_grad, ggx_data):
        def f(x):
            y = functions.get_item(x, self.slices)
            return y * y

        gradient_check.check_double_backward(
            f, (x_data,), y_grad, ggx_data, dtype='d')

    def test_double_backward_cpu(self):
        self.check_double_backward(self.x_data, self.gy_data, self.ggx_data)

    @attr.gpu
    def test_double_backward_gpu(self):
        self.check_double_backward(cuda.to_gpu(self.x_data),
                                   cuda.to_gpu(self.gy_data),
                                   cuda.to_gpu(self.ggx_data))

@testing.parameterize(
    {'slices': [], 'sliced_shape': (0, 3, 2)},
    {'slices': [[]], 'sliced_shape': (0, 3, 2)},
    {'slices': [[[]]], 'sliced_shape': (1, 0, 3, 2)},
    {'slices': ([[]],), 'sliced_shape': (1, 0, 3, 2)},
    {'slices': numpy.array([], dtype=numpy.bool),
        'sliced_shape': (0, 3, 2)},
    {'slices': [1, [1]], 'sliced_shape': (1, 2)},
    {'slices': [[1], slice(1, 2)], 'sliced_shape': (1, 1, 2)},
    {'slices': [1, 0], 'sliced_shape': (2, 3, 2)},
    {'slices': ([1, 0],), 'sliced_shape': (2, 3, 2)},
    {'slices': numpy.array([[1, 0], [2, 3]]),
        'sliced_shape': (2, 2, 3, 2)},
    {'slices': ([1, 0], [1, 1]), 'sliced_shape': (2, 2)},
    {'slices': ([1, 0], slice(None), [[1, 1], [1, 1]]),
        'sliced_shape': (2, 2, 3)},
    {'slices': ([1, 0], slice(1, 2), [0, 0]), 'sliced_shape': (2, 1)},
    {'slices': ([[1, 1], [1, 0]], slice(1, 2), 1),
        'sliced_shape': (2, 2, 1)},
    {'slices': numpy.array([True] * 18 + [False] * 6).reshape(4, 3, 2),
        'sliced_shape': (18,)},
    {'slices': numpy.array([True, False, False, True]),
        'sliced_shape': (2, 3, 2)},
    {'slices': (slice(None), numpy.array([True, False, True])),
        'sliced_shape': (4, 2, 2)},
    {'slices': numpy.array([False, False, False, False]),
        'sliced_shape': (0, 3, 2)},
    {'slices': (3, 2, Ellipsis, 1),
     'sliced_shape': ()},
)
class TestGetItemAdvanced(unittest.TestCase):

    def setUp(self):
        self.x_data = numpy.random.uniform(
            -1, 1, (4, 3, 2)).astype(numpy.float32)
        self.gy_data = numpy.random.uniform(
            -1, 1, self.sliced_shape).astype(numpy.float32)

    def check_forward(self, x_data):
        x = chainer.Variable(x_data)
        y = functions.get_item(x, self.slices)
        self.assertEqual(y.data.dtype, numpy.float32)
        numpy.testing.assert_equal(cuda.to_cpu(x_data)[self.slices],
                                   cuda.to_cpu(y.data))

    def test_forward_cpu(self):
        self.check_forward(self.x_data)

    @attr.gpu
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x_data))

    def check_backward(self, x_data, y_grad):
        def f(x):
            return functions.get_item(x, self.slices)

        gradient_check.check_backward(
            f, (x_data,), y_grad, dtype='d')

    def test_backward_cpu(self):
        self.check_backward(self.x_data, self.gy_data)

    @attr.gpu
    def test_backward_gpu(self):
        self.check_backward(cuda.to_gpu(self.x_data),
                            cuda.to_gpu(self.gy_data))

@parameterize(
    {'slices': ([1, 0], [1, 1]), 'sliced_shape': (2, 2)},
    {'slices': ([1, 0], slice(None), [[1, 1], [1, 1]]),
     'sliced_shape': (2, 2, 3)},
    {'slices': ([1, 0], [1, 1], [0, 0]), 'sliced_shape': (2,)},
    {'slices': (slice(None), numpy.array([True, False, True])),
     'sliced_shape': (4, 2, 2)},
)
class TestCupyIndicesGetItem(unittest.TestCase):

    def setUp(self):
        self.x_data = numpy.random.uniform(
            -1, 1, (4, 3, 2)).astype(numpy.float32)
        self.gy_data = numpy.random.uniform(
            -1, 1, self.sliced_shape).astype(numpy.float32)

    def check_forward(self, x_data):
        slices = []
        for i, s in enumerate(self.slices):
            if isinstance(s, numpy.ndarray):
                s = chainer.backends.cuda.cupy.array(s)
            if isinstance(s, list):
                s = chainer.backends.cuda.cupy.array(s, dtype=numpy.int32)
            slices.append(s)
        slices = tuple(slices)
        x = chainer.Variable(x_data)
        y = functions.get_item(x, slices)
        self.assertEqual(y.data.dtype, numpy.float32)
        numpy.testing.assert_equal(cuda.to_cpu(x_data)[self.slices],
                                   cuda.to_cpu(y.data))

    @attr.gpu
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x_data))

    def check_backward(self, x_data, y_grad):
        slices = []
        for i, s in enumerate(self.slices):
            if isinstance(s, numpy.ndarray):
                s = chainer.backends.cuda.cupy.array(s)
            if isinstance(s, list):
                s = chainer.backends.cuda.cupy.array(s, dtype=numpy.int32)
            slices.append(s)
        slices = tuple(slices)

        def f(x):
            return functions.get_item(x, slices)

        gradient_check.check_backward(
            f, (x_data,), y_grad, dtype='d')

    @attr.gpu
    def test_backward_gpu(self):
        self.check_backward(cuda.to_gpu(self.x_data),
                            cuda.to_gpu(self.gy_data))

class TestInvalidGetItem(unittest.TestCase):

    def setUp(self):
        self.default_debug = chainer.is_debug()
        chainer.set_debug(True)

        self.x_data = numpy.random.uniform(-1, 1, (4, 3, 2))

    def tearDown(self):
        chainer.set_debug(self.default_debug)

    def test_multiple_ellipsis(self):
        with self.assertRaises(ValueError):
            functions.get_item(self.x_data, (Ellipsis, Ellipsis))

testing.run_module(__name__, __file__)
<commit_msg>Add tests<commit_after>import unittest

import numpy

import chainer
from chainer.backends import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr
from chainer.testing import parameterize

@parameterize(
    {'axes': [1, 2], 'offsets': 0, 'sliced_shape': (4, 2, 1)},
    {'axes': [1, 2], 'offsets': [0, 1, 1], 'sliced_shape': (4, 2, 1)},
    {'axes': 1, 'offsets': 1, 'sliced_shape': (4, 2, 2)},
    {'axes': 1, 'offsets': [0, 1, 1], 'sliced_shape': (4, 2, 2)},
    {'axes': [], 'offsets': 0, 'new_axes': 0, 'sliced_shape': (1, 4, 3, 2)},
    {'axes': [], 'offsets': 0, 'new_axes': 2, 'sliced_shape': (4, 3, 1, 2)},
    {'axes': [], 'offsets': 0, 'new_axes': 3, 'sliced_shape': (4, 3, 2, 1)},
    {'slices': (1, -1, 0), 'sliced_shape': ()},
    {'slices': (1, -1), 'sliced_shape': (2,)},
    {'slices': (1, Ellipsis, -1), 'sliced_shape': (3,)},
    {'slices': (1, None, Ellipsis, None, -1), 'sliced_shape': (1, 3, 1)},
)
class TestGetItem(unittest.TestCase):

    def setUp(self):
        self.x_data = numpy.random.uniform(-1, 1, (4, 3, 2))
        self.shape = (4, 2, 1)
        self.gy_data = numpy.random.uniform(-1, 1, self.sliced_shape)
        self.ggx_data = numpy.random.uniform(-1, 1, (4, 3, 2))

        if not hasattr(self, 'slices'):
            # Convert axes, offsets and shape to slices
            if isinstance(self.offsets, int):
                self.offsets = tuple([self.offsets] * len(self.shape))
            if isinstance(self.axes, int):
                self.axes = tuple([self.axes])

            self.slices = [slice(None)] * len(self.shape)
            for axis in self.axes:
                self.slices[axis] = slice(
                    self.offsets[axis], self.offsets[axis] + self.shape[axis])

            if hasattr(self, 'new_axes'):
                self.slices.insert(self.new_axes, None)

            self.slices = tuple(self.slices)

    def check_forward(self, x_data):
        x = chainer.Variable(x_data)
        y = functions.get_item(x, self.slices)
        self.assertEqual(y.data.dtype, numpy.float)
        numpy.testing.assert_equal(cuda.to_cpu(x_data)[self.slices],
                                   cuda.to_cpu(y.data))

    def test_forward_cpu(self):
        self.check_forward(self.x_data)

    @attr.gpu
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x_data))

    def check_backward(self, x_data, y_grad):
        def f(x):
            return functions.get_item(x, self.slices)

        gradient_check.check_backward(
            f, (x_data,), y_grad, dtype='d')

    def test_backward_cpu(self):
        self.check_backward(self.x_data, self.gy_data)

    @attr.gpu
    def test_backward_gpu(self):
        self.check_backward(cuda.to_gpu(self.x_data),
                            cuda.to_gpu(self.gy_data))

    def check_double_backward(self, x_data, y_grad, ggx_data):
        def f(x):
            y = functions.get_item(x, self.slices)
            return y * y

        gradient_check.check_double_backward(
            f, (x_data,), y_grad, ggx_data, dtype='d')

    def test_double_backward_cpu(self):
        self.check_double_backward(self.x_data, self.gy_data, self.ggx_data)

    @attr.gpu
    def test_double_backward_gpu(self):
        self.check_double_backward(cuda.to_gpu(self.x_data),
                                   cuda.to_gpu(self.gy_data),
                                   cuda.to_gpu(self.ggx_data))

@testing.parameterize(
    {'slices': [], 'sliced_shape': (0, 3, 2)},
    {'slices': [[]], 'sliced_shape': (0, 3, 2)},
    {'slices': [[[]]], 'sliced_shape': (1, 0, 3, 2)},
    {'slices': ([[]],), 'sliced_shape': (1, 0, 3, 2)},
    {'slices': numpy.array([], dtype=numpy.bool),
        'sliced_shape': (0, 3, 2)},
    {'slices': [1, [1]], 'sliced_shape': (1, 2)},
    {'slices': [[1], slice(1, 2)], 'sliced_shape': (1, 1, 2)},
    {'slices': [1, 0], 'sliced_shape': (2, 3, 2)},
    {'slices': ([1, 0],), 'sliced_shape': (2, 3, 2)},
    {'slices': numpy.array([[1, 0], [2, 3]]),
        'sliced_shape': (2, 2, 3, 2)},
    {'slices': ([1, 0], [1, 1]), 'sliced_shape': (2, 2)},
    {'slices': ([1, 0], slice(None), [[1, 1], [1, 1]]),
        'sliced_shape': (2, 2, 3)},
    {'slices': ([1, 0], slice(1, 2), [0, 0]), 'sliced_shape': (2, 1)},
    {'slices': ([[1, 1], [1, 0]], slice(1, 2), 1),
        'sliced_shape': (2, 2, 1)},
    {'slices': numpy.array([True] * 18 + [False] * 6).reshape(4, 3, 2),
        'sliced_shape': (18,)},
    {'slices': numpy.array([True, False, False, True]),
        'sliced_shape': (2, 3, 2)},
    {'slices': (slice(None), numpy.array([True, False, True])),
        'sliced_shape': (4, 2, 2)},
    {'slices': numpy.array([False, False, False, False]),
        'sliced_shape': (0, 3, 2)},
    {'slices': (3, 2, Ellipsis, 1),
        'sliced_shape': ()},
    {'slices': (numpy.array(False)),
        'input_shape': (), 'sliced_shape': (0,)},
    {'slices': (numpy.array(True)),
        'input_shape': (), 'sliced_shape': (1,)},
)
class TestGetItemAdvanced(unittest.TestCase):

    input_shape = (4, 3, 2)

    def setUp(self):
        self.x_data = numpy.random.uniform(
            -1, 1, self.input_shape).astype(numpy.float32)
        self.gy_data = numpy.random.uniform(
            -1, 1, self.sliced_shape).astype(numpy.float32)

    def check_forward(self, x_data):
        x = chainer.Variable(x_data)
        y = functions.get_item(x, self.slices)
        self.assertEqual(y.data.dtype, numpy.float32)
        numpy.testing.assert_equal(cuda.to_cpu(x_data)[self.slices],
                                   cuda.to_cpu(y.data))

    def test_forward_cpu(self):
        self.check_forward(self.x_data)

    @attr.gpu
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x_data))

    def check_backward(self, x_data, y_grad):
        def f(x):
            return functions.get_item(x, self.slices)

        gradient_check.check_backward(
            f, (x_data,), y_grad, dtype='d')

    def test_backward_cpu(self):
        self.check_backward(self.x_data, self.gy_data)

    @attr.gpu
    def test_backward_gpu(self):
        self.check_backward(cuda.to_gpu(self.x_data),
                            cuda.to_gpu(self.gy_data))

@parameterize(
    {'slices': ([1, 0], [1, 1]), 'sliced_shape': (2, 2)},
    {'slices': ([1, 0], slice(None), [[1, 1], [1, 1]]),
     'sliced_shape': (2, 2, 3)},
    {'slices': ([1, 0], [1, 1], [0, 0]), 'sliced_shape': (2,)},
    {'slices': (slice(None), numpy.array([True, False, True])),
     'sliced_shape': (4, 2, 2)},
)
class TestCupyIndicesGetItem(unittest.TestCase):

    def setUp(self):
        self.x_data = numpy.random.uniform(
            -1, 1, (4, 3, 2)).astype(numpy.float32)
        self.gy_data = numpy.random.uniform(
            -1, 1, self.sliced_shape).astype(numpy.float32)

    def check_forward(self, x_data):
        slices = []
        for i, s in enumerate(self.slices):
            if isinstance(s, numpy.ndarray):
                s = chainer.backends.cuda.cupy.array(s)
            if isinstance(s, list):
                s = chainer.backends.cuda.cupy.array(s, dtype=numpy.int32)
            slices.append(s)
        slices = tuple(slices)
        x = chainer.Variable(x_data)
        y = functions.get_item(x, slices)
        self.assertEqual(y.data.dtype, numpy.float32)
        numpy.testing.assert_equal(cuda.to_cpu(x_data)[self.slices],
                                   cuda.to_cpu(y.data))

    @attr.gpu
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x_data))

    def check_backward(self, x_data, y_grad):
        slices = []
        for i, s in enumerate(self.slices):
            if isinstance(s, numpy.ndarray):
                s = chainer.backends.cuda.cupy.array(s)
            if isinstance(s, list):
                s = chainer.backends.cuda.cupy.array(s, dtype=numpy.int32)
            slices.append(s)
        slices = tuple(slices)

        def f(x):
            return functions.get_item(x, slices)

        gradient_check.check_backward(
            f, (x_data,), y_grad, dtype='d')

    @attr.gpu
    def test_backward_gpu(self):
        self.check_backward(cuda.to_gpu(self.x_data),
                            cuda.to_gpu(self.gy_data))

class TestInvalidGetItem(unittest.TestCase):

    def setUp(self):
        self.default_debug = chainer.is_debug()
        chainer.set_debug(True)

        self.x_data = numpy.random.uniform(-1, 1, (4, 3, 2))

    def tearDown(self):
        chainer.set_debug(self.default_debug)

    def test_multiple_ellipsis(self):
        with self.assertRaises(ValueError):
            functions.get_item(self.x_data, (Ellipsis, Ellipsis))

testing.run_module(__name__, __file__)
<|endoftext|>


<commit_before>''' Script to check the correctness of the analysis. The analysis is done on raw data and all results are compared to a recorded analysis.
'''
import os

import unittest

from testbeam_analysis import dut_alignment
from testbeam_analysis.tools import test_tools

tests_data_folder = r'tests/test_dut_alignment/'

class TestHitAnalysis(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        if os.name != 'nt':
            try:
                from xvfbwrapper import Xvfb  # virtual X server for plots under headless LINUX travis testing is needed
                cls.vdisplay = Xvfb()
                cls.vdisplay.start()
            except (ImportError, EnvironmentError):
                pass
        cls.data_files = [os.path.join(tests_data_folder + 'TestBeamData_FEI4_DUT0_small.h5'),
                          os.path.join(tests_data_folder + 'TestBeamData_FEI4_DUT1_small.h5'),
                          os.path.join(tests_data_folder + 'TestBeamData_FEI4_DUT2_small.h5'),
                          os.path.join(tests_data_folder + 'TestBeamData_FEI4_DUT3_small.h5')
                          ]
        cls.output_folder = tests_data_folder
        cls.pixel_size = ((250, 50), (250, 50), (250, 50), (250, 50))  # in um

    @classmethod
    def tearDownClass(cls):  # remove created files
        os.remove(os.path.join(cls.output_folder + 'Correlation.h5'))
        os.remove(os.path.join(cls.output_folder + 'Tracklets.h5'))
        os.remove(os.path.join(cls.output_folder + 'Tracklets_2.h5'))
#         os.remove(cls.output_folder + 'Alignment.h5')
#         os.remove(cls.output_folder + 'Alignment.pdf')

    def test_hit_correlation(self):  # check the hit correlation function
        dut_alignment.correlate_hits(input_hits_files=self.data_files,
                                     output_correlation_file=os.path.join(self.output_folder + 'Correlation.h5'),
                                     fraction=1,
                                     event_range=0)
        data_equal, error_msg = test_tools.compare_h5_files(os.path.join(tests_data_folder + 'Correlation_result.h5'), os.path.join(self.output_folder + 'Correlation.h5'), exact=False)
        self.assertTrue(data_equal, msg=error_msg)

    @unittest.SkipTest  # Unclear how to check interactive alignment automatically
    def test_coarse_alignment(self):  # Check the hit alignment function
        dut_alignment.coarse_alignment(input_correlation_file=os.path.join(tests_data_folder + 'Correlation_result.h5'),
                                       output_alignment_file=os.path.join(self.output_folder + 'Alignment.h5'),
                                       output_pdf=os.path.join(self.output_folder + 'Alignment.pdf'),
                                       pixel_size=self.pixel_size)
        data_equal, error_msg = test_tools.compare_h5_files(os.path.join(tests_data_folder + 'Alignment_result.h5'), os.path.join(self.output_folder + 'Alignment.h5'), exact=False)
        self.assertTrue(data_equal, msg=error_msg)

    def test_cluster_merging(self):
        cluster_files = [os.path.join(tests_data_folder + 'Cluster_DUT%d_cluster.h5') % i for i in range(4)]
        dut_alignment.merge_cluster_data(cluster_files,
                                         input_alignment_file=os.path.join(tests_data_folder + 'Alignment_result.h5'),
                                         output_tracklets_file=os.path.join(self.output_folder + 'Tracklets.h5'),
                                         pixel_size=self.pixel_size)
        data_equal, error_msg = test_tools.compare_h5_files(os.path.join(tests_data_folder + 'Tracklets_result.h5'), os.path.join(self.output_folder + 'Tracklets.h5'))
        self.assertTrue(data_equal, msg=error_msg)
        # Retest with tiny chunk size to force chunked merging
        dut_alignment.merge_cluster_data(cluster_files,
                                         input_alignment_file=os.path.join(tests_data_folder + 'Alignment_result.h5'),
                                         output_tracklets_file=os.path.join(self.output_folder + 'Tracklets_2.h5'),
                                         pixel_size=self.pixel_size,
                                         chunk_size=293)

        data_equal, error_msg = test_tools.compare_h5_files(tests_data_folder + 'Tracklets_result.h5', self.output_folder + 'Tracklets_2.h5')
        self.assertTrue(data_equal, msg=error_msg)

if __name__ == '__main__':
    tests_data_folder = r'test_dut_alignment/'
    suite = unittest.TestLoader().loadTestsFromTestCase(TestHitAnalysis)
    unittest.TextTestRunner(verbosity=2).run(suite)
<commit_msg>MAINT: activate coarse alignment test case<commit_after>''' Script to check the correctness of the analysis. The analysis is done on raw data and all results are compared to a recorded analysis.
'''
import os

import unittest

from testbeam_analysis import dut_alignment
from testbeam_analysis.tools import test_tools

tests_data_folder = r'tests/test_dut_alignment/'

class TestHitAnalysis(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        if os.name != 'nt':
            try:
                from xvfbwrapper import Xvfb  # virtual X server for plots under headless LINUX travis testing is needed
                cls.vdisplay = Xvfb()
                cls.vdisplay.start()
            except (ImportError, EnvironmentError):
                pass
        cls.data_files = [os.path.join(tests_data_folder + 'TestBeamData_FEI4_DUT0_small.h5'),
                          os.path.join(tests_data_folder + 'TestBeamData_FEI4_DUT1_small.h5'),
                          os.path.join(tests_data_folder + 'TestBeamData_FEI4_DUT2_small.h5'),
                          os.path.join(tests_data_folder + 'TestBeamData_FEI4_DUT3_small.h5')
                          ]
        cls.output_folder = tests_data_folder
        cls.n_pixels = ((80, 336), (80, 336), (80, 336), (80, 336))
        cls.pixel_size = ((250, 50), (250, 50), (250, 50), (250, 50))  # in um

    @classmethod
    def tearDownClass(cls):  # remove created files
         os.remove(os.path.join(cls.output_folder + 'Correlation.h5'))
         os.remove(os.path.join(cls.output_folder + 'Correlation.pdf'))
         os.remove(os.path.join(cls.output_folder + 'Tracklets.h5'))
         os.remove(os.path.join(cls.output_folder + 'Tracklets_2.h5'))
         os.remove(cls.output_folder + 'Alignment.h5')
         os.remove(cls.output_folder + 'Alignment.pdf')

    def test_hit_correlation(self):  # check the hit correlation function
        dut_alignment.correlate_hits(input_hits_files=self.data_files,
                                     output_correlation_file=os.path.join(self.output_folder + 'Correlation.h5'),
                                     n_pixels=self.n_pixels
                                     )
        data_equal, error_msg = test_tools.compare_h5_files(os.path.join(tests_data_folder + 'Correlation_result.h5'), os.path.join(self.output_folder + 'Correlation.h5'), exact=True)
        self.assertTrue(data_equal, msg=error_msg)
        # Retest with tiny chunk size to force chunked correlation
        dut_alignment.correlate_hits(input_hits_files=self.data_files,
                                     output_correlation_file=os.path.join(self.output_folder + 'Correlation_2.h5'),
                                     n_pixels=self.n_pixels,
                                     chunk_size=293
                                     )
        data_equal, error_msg = test_tools.compare_h5_files(os.path.join(tests_data_folder + 'Correlation_result.h5'), os.path.join(self.output_folder + 'Correlation_2.h5'), exact=True)
        self.assertTrue(data_equal, msg=error_msg)

    def test_coarse_alignment(self):  # Check the hit alignment function
        dut_alignment.coarse_alignment(input_correlation_file=os.path.join(tests_data_folder + 'Correlation_result.h5'),
                                       output_alignment_file=os.path.join(self.output_folder + 'Alignment.h5'),
                                       output_pdf_file=os.path.join(self.output_folder + 'Alignment.pdf'),
                                       pixel_size=self.pixel_size,
                                       non_interactive=True)
        data_equal, error_msg = test_tools.compare_h5_files(os.path.join(tests_data_folder + 'Alignment_result.h5'), os.path.join(self.output_folder + 'Alignment.h5'), exact=False)
        self.assertTrue(data_equal, msg=error_msg)

    def test_cluster_merging(self):
        cluster_files = [os.path.join(tests_data_folder + 'Cluster_DUT%d_cluster.h5') % i for i in range(4)]
        dut_alignment.merge_cluster_data(cluster_files,
                                         input_alignment_file=os.path.join(tests_data_folder + 'Alignment_result.h5'),
                                         output_tracklets_file=os.path.join(self.output_folder + 'Tracklets.h5'),
                                         pixel_size=self.pixel_size)
        data_equal, error_msg = test_tools.compare_h5_files(os.path.join(tests_data_folder + 'Tracklets_result.h5'), os.path.join(self.output_folder + 'Tracklets.h5'))
        self.assertTrue(data_equal, msg=error_msg)
        # Retest with tiny chunk size to force chunked merging
        dut_alignment.merge_cluster_data(cluster_files,
                                         input_alignment_file=os.path.join(tests_data_folder + 'Alignment_result.h5'),
                                         output_tracklets_file=os.path.join(self.output_folder + 'Tracklets_2.h5'),
                                         pixel_size=self.pixel_size,
                                         chunk_size=293)

        data_equal, error_msg = test_tools.compare_h5_files(tests_data_folder + 'Tracklets_result.h5', self.output_folder + 'Tracklets_2.h5')
        self.assertTrue(data_equal, msg=error_msg)

if __name__ == '__main__':
    tests_data_folder = r'test_dut_alignment/'
    suite = unittest.TestLoader().loadTestsFromTestCase(TestHitAnalysis)
    unittest.TextTestRunner(verbosity=2).run(suite)
<|endoftext|>


<commit_before>import logging
import functools
from datetime import date, timedelta

from celery import Celery

from scrapi import util
from scrapi import events
from scrapi import database
from scrapi import settings
from scrapi import registry
from scrapi import processing
from scrapi.util import timestamp

from scrapi.migrations import documents

app = Celery()
app.config_from_object(settings)

database.setup()
logger = logging.getLogger(__name__)

def task_autoretry(*args_task, **kwargs_task):
    def actual_decorator(func):
        @app.task(*args_task, **kwargs_task)
        @functools.wraps(func)
        def wrapper(*args, **kwargs):
            try:
                func(*args, **kwargs)
            except kwargs_task.get('autoretry_on', Exception) as exc:
                logger.info('Retrying with exception {}'.format(exc))
                wrapper.retry(exc=exc)
        return wrapper
    return actual_decorator

@app.task
@events.creates_task(events.HARVESTER_RUN)
def run_harvester(harvester_name, start_date=None, end_date=None):
    logger.info('Running harvester "{}"'.format(harvester_name))

    start_date = start_date or date.today() - timedelta(settings.DAYS_BACK)
    end_date = end_date or date.today()

    normalization = begin_normalization.s(harvester_name)
    start_harvest = harvest.si(harvester_name, timestamp(), start_date=start_date, end_date=end_date)

    # Form and start a celery chain
    (start_harvest | normalization).apply_async()

@app.task
@events.logged(events.HARVESTER_RUN)
def harvest(harvester_name, job_created, start_date=None, end_date=None):
    harvest_started = timestamp()
    harvester = registry[harvester_name]

    start_date = start_date or date.today() - timedelta(settings.DAYS_BACK)
    end_date = end_date or date.today()

    logger.info('Harvester "{}" has begun harvesting'.format(harvester_name))

    result = harvester.harvest(start_date=start_date, end_date=end_date)

    # result is a list of all of the RawDocuments harvested
    return result, {
        'harvestFinished': timestamp(),
        'harvestTaskCreated': job_created,
        'harvestStarted': harvest_started,
    }

@app.task
def begin_normalization((raw_docs, timestamps), harvester_name):
    '''harvest_ret is harvest return value:
        a tuple contaiing list of rawDocuments and
        a dictionary of timestamps
    '''
    logger.info('Normalizing {} documents for harvester "{}"'
                .format(len(raw_docs), harvester_name))
    # raw is a single raw document
    for raw in raw_docs:
        spawn_tasks(raw, timestamps, harvester_name)

@events.creates_task(events.PROCESSING)
@events.creates_task(events.NORMALIZATION)
def spawn_tasks(raw, timestamps, harvester_name):
    raw['timestamps'] = timestamps
    raw['timestamps']['normalizeTaskCreated'] = timestamp()
    chain = (normalize.si(raw, harvester_name) | process_normalized.s(raw))

    chain.apply_async()
    process_raw.delay(raw)

@app.task
@events.logged(events.PROCESSING, 'raw')
def process_raw(raw_doc, **kwargs):
    processing.process_raw(raw_doc, kwargs)

@app.task
@events.logged(events.NORMALIZATION)
def normalize(raw_doc, harvester_name):
    normalized_started = timestamp()
    harvester = registry[harvester_name]

    normalized = harvester.normalize(raw_doc)

    if not normalized:
        raise events.Skip('Did not normalize document with id {}'.format(raw_doc['docID']))

    normalized['timestamps'] = util.stamp_from_raw(raw_doc, normalizeStarted=normalized_started)

    return normalized  # returns a single normalized document

@app.task
@events.logged(events.PROCESSING, 'normalized')
def process_normalized(normalized_doc, raw_doc, **kwargs):
    try:
        if not normalized_doc:
            raise events.Skip('Not processing document with id {}'.format(raw_doc['docID']))
        processing.process_normalized(raw_doc, normalized_doc, kwargs)
    except Exception as e:
        logger.info('Retrying withiin PROCESS NORMALIZED with exception {}'.format(e))
        process_normalized.retry(exc=e)

@app.task
def migrate(migration, sources=tuple(), async=False, dry=True, **kwargs):
    count = 0
    for doc in documents(*sources):
        count += 1

        if async:
            migration.s(doc, dry=dry, **kwargs).apply_async()
        else:
            migration(doc, dry=dry, **kwargs)

    if dry:
        logger.info('Dry run complete')

    logger.info('{} documents processed for migration {}'.format(count, str(migration)))
<commit_msg>Change back import to fix circular thing<commit_after>import logging
import functools
from datetime import date, timedelta

from celery import Celery

from scrapi import util
from scrapi import events
from scrapi import database
from scrapi import settings
from scrapi import registry
from scrapi import processing
from scrapi.util import timestamp

from scripts.util import documents

app = Celery()
app.config_from_object(settings)

database.setup()
logger = logging.getLogger(__name__)

def task_autoretry(*args_task, **kwargs_task):
    def actual_decorator(func):
        @app.task(*args_task, **kwargs_task)
        @functools.wraps(func)
        def wrapper(*args, **kwargs):
            try:
                func(*args, **kwargs)
            except kwargs_task.get('autoretry_on', Exception) as exc:
                logger.info('Retrying with exception {}'.format(exc))
                wrapper.retry(exc=exc)
        return wrapper
    return actual_decorator

@app.task
@events.creates_task(events.HARVESTER_RUN)
def run_harvester(harvester_name, start_date=None, end_date=None):
    logger.info('Running harvester "{}"'.format(harvester_name))

    start_date = start_date or date.today() - timedelta(settings.DAYS_BACK)
    end_date = end_date or date.today()

    normalization = begin_normalization.s(harvester_name)
    start_harvest = harvest.si(harvester_name, timestamp(), start_date=start_date, end_date=end_date)

    # Form and start a celery chain
    (start_harvest | normalization).apply_async()

@app.task
@events.logged(events.HARVESTER_RUN)
def harvest(harvester_name, job_created, start_date=None, end_date=None):
    harvest_started = timestamp()
    harvester = registry[harvester_name]

    start_date = start_date or date.today() - timedelta(settings.DAYS_BACK)
    end_date = end_date or date.today()

    logger.info('Harvester "{}" has begun harvesting'.format(harvester_name))

    result = harvester.harvest(start_date=start_date, end_date=end_date)

    # result is a list of all of the RawDocuments harvested
    return result, {
        'harvestFinished': timestamp(),
        'harvestTaskCreated': job_created,
        'harvestStarted': harvest_started,
    }

@app.task
def begin_normalization((raw_docs, timestamps), harvester_name):
    '''harvest_ret is harvest return value:
        a tuple contaiing list of rawDocuments and
        a dictionary of timestamps
    '''
    logger.info('Normalizing {} documents for harvester "{}"'
                .format(len(raw_docs), harvester_name))
    # raw is a single raw document
    for raw in raw_docs:
        spawn_tasks(raw, timestamps, harvester_name)

@events.creates_task(events.PROCESSING)
@events.creates_task(events.NORMALIZATION)
def spawn_tasks(raw, timestamps, harvester_name):
    raw['timestamps'] = timestamps
    raw['timestamps']['normalizeTaskCreated'] = timestamp()
    chain = (normalize.si(raw, harvester_name) | process_normalized.s(raw))

    chain.apply_async()
    process_raw.delay(raw)

@app.task
@events.logged(events.PROCESSING, 'raw')
def process_raw(raw_doc, **kwargs):
    processing.process_raw(raw_doc, kwargs)

@app.task
@events.logged(events.NORMALIZATION)
def normalize(raw_doc, harvester_name):
    normalized_started = timestamp()
    harvester = registry[harvester_name]

    normalized = harvester.normalize(raw_doc)

    if not normalized:
        raise events.Skip('Did not normalize document with id {}'.format(raw_doc['docID']))

    normalized['timestamps'] = util.stamp_from_raw(raw_doc, normalizeStarted=normalized_started)

    return normalized  # returns a single normalized document

@app.task
@events.logged(events.PROCESSING, 'normalized')
def process_normalized(normalized_doc, raw_doc, **kwargs):
    try:
        if not normalized_doc:
            raise events.Skip('Not processing document with id {}'.format(raw_doc['docID']))
        processing.process_normalized(raw_doc, normalized_doc, kwargs)
    except Exception as e:
        logger.info('Retrying withiin PROCESS NORMALIZED with exception {}'.format(e))
        process_normalized.retry(exc=e)

@app.task
def migrate(migration, sources=tuple(), async=False, dry=True, **kwargs):
    count = 0
    for doc in documents(*sources):
        count += 1

        if async:
            migration.s(doc, dry=dry, **kwargs).apply_async()
        else:
            migration(doc, dry=dry, **kwargs)

    if dry:
        logger.info('Dry run complete')

    logger.info('{} documents processed for migration {}'.format(count, str(migration)))
<|endoftext|>


<commit_before>"""Simulates the car with telemetry."""
import math
import mock
import time

from command import Command
from telemetry import Telemetry

# pylint: disable=missing-docstring
# pylint: disable=no-self-use
# pylint: disable=protected-access
# pylint: disable=superfluous-parens
# pylint: disable=too-few-public-methods

class FakeLogger(object):
    def debug(self, message):
        pass
    def info(self, message):
        print(message)
    def warning(self, message):
        print(message)
    def error(self, message):
        print(message)

class FakeSocket(object):
    def send(self, message):
        pass

class TelemetrySimulator(object):
    def __init__(self, first_way_point):
        self._latitude, self._longitude = first_way_point
        self._latitude -= 0.001
        self._heading = 0.0
        self._last_command_time = time.time()
        self._throttle = 0.0
        self._turn = 0.0

        self.update_count = 1000
        self.command = None

    def get_raw_data(self):
        raise NotImplementedError

    def get_data(self):
        values = {
            'heading': self._heading,
            'latitude': self._latitude,
            'longitude': self._longitude,
        }
        return values

    def process_drive_command(self, throttle, turn):
        print(
            'Throttle: {throttle}, turn: {turn}'.format(
                throttle=throttle,
                turn=turn,
            )
        )
        self._throttle = throttle
        self._turn = turn

    def _update_position(self):
        self.update_count -= 1
        if self.update_count <= 0:
            assert self.command is not None
            self.command.kill()

        diff_time_s = 1.0

        if self._throttle > 0.0:
            self._heading += self._turn * 20.0
            self._heading = Telemetry.wrap_degrees(self._heading)

            point = (0, diff_time_s * self._throttle * 1.0 / Telemetry.m_per_d_latitude())
            radians = math.radians(self._heading)
            point = Telemetry.rotate_radians_clockwise(point, radians)
            self._latitude += point[1]
            self._longitude += point[0]

    def is_stopped(self):
        return False

def main():
    """Main function."""
    fake_logger = FakeLogger()
    box = [(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]
    waypoints = [(x * .001 + 10, y * .001 + 10) for x, y in box]
    telemetry_simulator = TelemetrySimulator(waypoints[0])
    fake_socket = FakeSocket()
    command = Command(
        telemetry_simulator,
        fake_socket,
        fake_logger,
        sleep_time_milliseconds=1,
        waypoints=waypoints,
    )
    telemetry_simulator.command = command
    command._wait = mock.Mock(side_effect=telemetry_simulator._update_position)
    command.run_course()
    command.run()

if __name__ == '__main__':
    main()
<commit_msg>Use the simulated values as if they were coming from the Android phone<commit_after>"""Simulates the car with telemetry."""
import math
import mock
import time

from command import Command
from telemetry import Telemetry

# pylint: disable=missing-docstring
# pylint: disable=no-self-use
# pylint: disable=protected-access
# pylint: disable=superfluous-parens
# pylint: disable=too-few-public-methods

class FakeLogger(object):
    def debug(self, message):
        pass
    def info(self, message):
        print(message)
    def warning(self, message):
        print(message)
    def error(self, message):
        print(message)

class FakeSocket(object):
    def send(self, message):
        pass

class TelemetrySimulator(object):
    MAX_SPEED_M_S = 4.7  # From observation

    def __init__(self, first_way_point):
        self._latitude, self._longitude = first_way_point
        self._latitude -= 0.001
        self._heading = 0.0
        self._last_command_time = time.time()
        self._throttle = 0.0
        self._turn = 0.0

        self.update_count = 1000
        self.command = None

    def get_raw_data(self):
        raise NotImplementedError

    def get_data(self):
        # The Android phone is rotated, so we need to mess with heading before
        # we give it to the real Telemetry module to simulate the phone
        android_heading = Telemetry.wrap_degrees(-self._heading - 90)
        values = {
            'heading': android_heading,
            'latitude': self._latitude,
            'longitude': self._longitude,
            'accelerometer': [],
            'speed': self._throttle * self.MAX_SPEED_M_S,
        }
        return values

    def process_drive_command(self, throttle, turn):
        self._throttle = throttle
        self._turn = turn

    def _update_position(self):
        self.update_count -= 1
        if self.update_count <= 0:
            assert self.command is not None
            self.command.kill()

        diff_time_s = 1.0

        if self._throttle > 0.0:
            self._heading += self._turn * 20.0
            self._heading = Telemetry.wrap_degrees(self._heading)

            step_m = diff_time_s * self._throttle * self.MAX_SPEED_M_S
            point = (0, step_m / Telemetry.m_per_d_latitude())
            radians = math.radians(self._heading)
            point = Telemetry.rotate_radians_clockwise(point, radians)
            self._latitude += point[1]
            self._longitude += point[0]

    def is_stopped(self):
        return False

def main():
    """Main function."""
    fake_logger = FakeLogger()
    box = [(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]
    waypoints = [(x * .001 + 10, y * .001 + 10) for x, y in box]

    fake_socket = FakeSocket()

    telemetry = Telemetry(fake_logger)
    telemetry_simulator = TelemetrySimulator(waypoints[0])
    original_telemetry_get_data = telemetry.get_data
    def call_and_return_original():
        telemetry.handle_message(telemetry_simulator.get_data())
        return original_telemetry_get_data()
    original_telemetry_process = telemetry.process_drive_command
    def call_both_processes(throttle, turn):
        original_telemetry_process(throttle, turn)
        telemetry_simulator.process_drive_command(throttle, turn)

    with mock.patch.object(
        telemetry,
        'get_data',
        new=call_and_return_original
    ):
        with mock.patch.object(
            telemetry,
            'process_drive_command',
            new=call_both_processes
        ):
            command = Command(
                telemetry,
                fake_socket,
                fake_logger,
                sleep_time_milliseconds=1,
                waypoints=waypoints,
            )
            telemetry_simulator.command = command
            command._wait = mock.Mock(
                side_effect=telemetry_simulator._update_position
            )
            command.run_course()
            command.run()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before><commit_msg>Updated from Brython Server: 3/9/2016 1:46:06 PM<commit_after><|endoftext|>


<commit_before>import re
import itertools
from mapit.postcodes.models import Postcode
from mapit.areas.models import Area, Generation
from django.shortcuts import get_object_or_404
from django.core import serializers
from django.http import HttpResponse

def postcode(request, postcode, format='html'):
    postcode = re.sub('\s+', '', postcode.upper())
    postcode = get_object_or_404(Postcode, postcode=postcode)
    current_generation = Generation.objects.current()
    areas = []
    for area in itertools.chain(
        Area.objects.filter(
            polygon__contains=postcode.location,
            #generation_low__lte=current_generation,
            #generation_high__gte=current_generation
        ),
        postcode.areas.filter(
            #generation_low__lte=current_generation,
            #generation_high__gte=current_generation
        )
    ):
        areas.append({
            'id': area.id,
            'parent_area': area.parent_area,
            'type': area.type,
            'country': area.country,
            'generation_low': area.generation_low,
            'generation_high': area.generation_high,
            'name': area.name,
            'codes': area.codes,
        })

    response = HttpResponse(content_type='application/javascript; charset=utf-8')

    json = serializers.get_serializer('json')()
    json.serialize(areas, ensure_ascii=False, stream=response)
    return response

<commit_msg>Not serializing any more.<commit_after>import re
import itertools
from mapit.postcodes.models import Postcode
from mapit.areas.models import Area, Generation
from django.shortcuts import get_object_or_404
from django.utils import simplejson
from django.http import HttpResponse

def postcode(request, postcode, format='html'):
    postcode = re.sub('\s+', '', postcode.upper())
    postcode = get_object_or_404(Postcode, postcode=postcode)
    current_generation = Generation.objects.current()
    areas = []
    for area in itertools.chain(
        Area.objects.filter(
            polygon__contains=postcode.location,
            #generation_low__lte=current_generation,
            #generation_high__gte=current_generation
        ),
        postcode.areas.filter(
            #generation_low__lte=current_generation,
            #generation_high__gte=current_generation
        )
    ):
        areas.append({
            'id': area.id,
            'parent_area': area.parent_area,
            'type': area.type,
            'country': area.country,
            'generation_low': area.generation_low,
            'generation_high': area.generation_high,
            'name': area.name,
            'codes': area.codes,
        })

    response = HttpResponse(content_type='application/javascript; charset=utf-8')
    simplejson.dump(areas, HttpResponse, ensure_ascii=False)
    return response

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function

__version__ = "0.0.3"

try:
    __FEEDFINDER2_SETUP__
except NameError:
    __FEEDFINDER2_SETUP__ = False

if not __FEEDFINDER2_SETUP__:
    __all__ = ["find_feeds"]

    import logging
    import requests
    from bs4 import BeautifulSoup
    from six.moves.urllib import parse as urlparse

def coerce_url(url):
    url = url.strip()
    if url.startswith("feed://"):
        return "http://{0}".format(url[7:])
    for proto in ["http://", "https://"]:
        if url.startswith(proto):
            return url
    return "http://{0}".format(url)

class FeedFinder(object):

    def __init__(self, user_agent=None):
        if user_agent is None:
            user_agent = "feedfinder2/{0}".format(__version__)
        self.user_agent = user_agent

    def get_feed(self, url):
        try:
            r = requests.get(url, headers={"User-Agent": self.user_agent})
        except Exception as e:
            logging.warn("Error while getting '{0}'".format(url))
            logging.warn("{0}".format(e))
            return None
        return r.text

    def is_feed_data(self, text):
        data = text.lower()
        if data.count("<html"):
            return False
        return data.count("<rss")+data.count("<rdf")+data.count("<feed")

    def is_feed(self, url):
        text = self.get_feed(url)
        if text is None:
            return False
        return self.is_feed_data(text)

    def is_feed_url(self, url):
        return any(map(url.lower().endswith,
                       [".rss", ".rdf", ".xml", ".atom"]))

    def is_feedlike_url(self, url):
        return any(map(url.lower().count,
                       ["rss", "rdf", "xml", "atom", "feed"]))

def find_feeds(url, check_all=False, user_agent=None):
    finder = FeedFinder(user_agent=user_agent)

    # Format the URL properly.
    url = coerce_url(url)

    # Download the requested URL.
    text = finder.get_feed(url)
    if text is None:
        return []

    # Check if it is already a feed.
    if finder.is_feed_data(text):
        return [url]

    # Look for <link> tags.
    logging.info("Looking for <link> tags.")
    tree = BeautifulSoup(text, "html.parser")
    links = []
    for link in tree.find_all("link"):
        if link.get("type") in ["application/rss+xml",
                                "text/xml",
                                "application/atom+xml",
                                "application/x.atom+xml",
                                "application/x-atom+xml"]:
            links.append(urlparse.urljoin(url, link.get("href", "")))

    # Check the detected links.
    urls = list(filter(finder.is_feed, links))
    logging.info("Found {0} feed <link> tags.".format(len(urls)))
    if len(urls) and not check_all:
        return sort_urls(urls)

    # Look for <a> tags.
    logging.info("Looking for <a> tags.")
    local, remote = [], []
    for a in tree.find_all("a"):
        href = a.get("href", None)
        if href is None:
            continue
        if "://" not in href and finder.is_feed_url(href):
            local.append(href)
        if finder.is_feedlike_url(href):
            remote.append(href)

    # Check the local URLs.
    local = [urlparse.urljoin(url, l) for l in local]
    urls += list(filter(finder.is_feed, local))
    logging.info("Found {0} local <a> links to feeds.".format(len(urls)))
    if len(urls) and not check_all:
        return sort_urls(urls)

    # Check the remote URLs.
    remote = [urlparse.urljoin(url, l) for l in remote]
    urls += list(filter(finder.is_feed, remote))
    logging.info("Found {0} remote <a> links to feeds.".format(len(urls)))
    if len(urls) and not check_all:
        return sort_urls(urls)

    # Guessing potential URLs.
    fns = ["atom.xml", "index.atom", "index.rdf", "rss.xml", "index.xml",
           "index.rss"]
    urls += list(filter(finder.is_feed, [urlparse.urljoin(url, f)
                                         for f in fns]))
    return sort_urls(urls)

def url_feed_prob(url):
    if "comments" in url:
        return -2
    if "georss" in url:
        return -1
    kw = ["atom", "rss", "rdf", ".xml", "feed"]
    for p, t in zip(range(len(kw), 0, -1), kw):
        if t in url:
            return p
    return 0

def sort_urls(feeds):
    return sorted(list(set(feeds)), key=url_feed_prob, reverse=True)

if __name__ == "__main__":
    print(find_feeds("www.preposterousuniverse.com/blog/"))
    print(find_feeds("http://xkcd.com"))
    print(find_feeds("dan.iel.fm/atom.xml"))
    print(find_feeds("dan.iel.fm", check_all=True))
    print(find_feeds("kapadia.github.io"))
    print(find_feeds("blog.jonathansick.ca"))
    print(find_feeds("asdasd"))
<commit_msg>Add optional read timeout for requests<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function

__version__ = "0.0.4"

try:
    __FEEDFINDER2_SETUP__
except NameError:
    __FEEDFINDER2_SETUP__ = False

if not __FEEDFINDER2_SETUP__:
    __all__ = ["find_feeds"]

    import logging
    import requests
    from bs4 import BeautifulSoup
    from six.moves.urllib import parse as urlparse

def coerce_url(url):
    url = url.strip()
    if url.startswith("feed://"):
        return "http://{0}".format(url[7:])
    for proto in ["http://", "https://"]:
        if url.startswith(proto):
            return url
    return "http://{0}".format(url)

class FeedFinder(object):

    def __init__(self, user_agent=None, timeout=None):
        if user_agent is None:
            user_agent = "feedfinder2/{0}".format(__version__)
        self.user_agent = user_agent
        self.timeout = timeout

    def get_feed(self, url):
        try:
            r = requests.get(url, headers={"User-Agent": self.user_agent}, timeout=self.timeout)
        except Exception as e:
            logging.warn("Error while getting '{0}'".format(url))
            logging.warn("{0}".format(e))
            return None
        return r.text

    def is_feed_data(self, text):
        data = text.lower()
        if data.count("<html"):
            return False
        return data.count("<rss")+data.count("<rdf")+data.count("<feed")

    def is_feed(self, url):
        text = self.get_feed(url)
        if text is None:
            return False
        return self.is_feed_data(text)

    def is_feed_url(self, url):
        return any(map(url.lower().endswith,
                       [".rss", ".rdf", ".xml", ".atom"]))

    def is_feedlike_url(self, url):
        return any(map(url.lower().count,
                       ["rss", "rdf", "xml", "atom", "feed"]))

def find_feeds(url, check_all=False, user_agent=None, timeout=None):
    finder = FeedFinder(user_agent=user_agent, timeout=timeout)

    # Format the URL properly.
    url = coerce_url(url)

    # Download the requested URL.
    text = finder.get_feed(url)
    if text is None:
        return []

    # Check if it is already a feed.
    if finder.is_feed_data(text):
        return [url]

    # Look for <link> tags.
    logging.info("Looking for <link> tags.")
    tree = BeautifulSoup(text, "html.parser")
    links = []
    for link in tree.find_all("link"):
        if link.get("type") in ["application/rss+xml",
                                "text/xml",
                                "application/atom+xml",
                                "application/x.atom+xml",
                                "application/x-atom+xml"]:
            links.append(urlparse.urljoin(url, link.get("href", "")))

    # Check the detected links.
    urls = list(filter(finder.is_feed, links))
    logging.info("Found {0} feed <link> tags.".format(len(urls)))
    if len(urls) and not check_all:
        return sort_urls(urls)

    # Look for <a> tags.
    logging.info("Looking for <a> tags.")
    local, remote = [], []
    for a in tree.find_all("a"):
        href = a.get("href", None)
        if href is None:
            continue
        if "://" not in href and finder.is_feed_url(href):
            local.append(href)
        if finder.is_feedlike_url(href):
            remote.append(href)

    # Check the local URLs.
    local = [urlparse.urljoin(url, l) for l in local]
    urls += list(filter(finder.is_feed, local))
    logging.info("Found {0} local <a> links to feeds.".format(len(urls)))
    if len(urls) and not check_all:
        return sort_urls(urls)

    # Check the remote URLs.
    remote = [urlparse.urljoin(url, l) for l in remote]
    urls += list(filter(finder.is_feed, remote))
    logging.info("Found {0} remote <a> links to feeds.".format(len(urls)))
    if len(urls) and not check_all:
        return sort_urls(urls)

    # Guessing potential URLs.
    fns = ["atom.xml", "index.atom", "index.rdf", "rss.xml", "index.xml",
           "index.rss"]
    urls += list(filter(finder.is_feed, [urlparse.urljoin(url, f)
                                         for f in fns]))
    return sort_urls(urls)

def url_feed_prob(url):
    if "comments" in url:
        return -2
    if "georss" in url:
        return -1
    kw = ["atom", "rss", "rdf", ".xml", "feed"]
    for p, t in zip(range(len(kw), 0, -1), kw):
        if t in url:
            return p
    return 0

def sort_urls(feeds):
    return sorted(list(set(feeds)), key=url_feed_prob, reverse=True)

if __name__ == "__main__":
    print(find_feeds("www.preposterousuniverse.com/blog/", timeout = 1))
    print(find_feeds("www.preposterousuniverse.com/blog/"))
    print(find_feeds("http://xkcd.com"))
    print(find_feeds("dan.iel.fm/atom.xml"))
    print(find_feeds("dan.iel.fm", check_all=True))
    print(find_feeds("kapadia.github.io"))
    print(find_feeds("blog.jonathansick.ca"))
    print(find_feeds("asdasd"))

<|endoftext|>


<commit_before>import unittest

import numpy

import chainer
from chainer import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr
from chainer.utils import type_check

@testing.parameterize(
    {'axis': 0, 'start': 2, 'out_shape': (3, 2, 4)},
    {'axis': 2, 'start': 0, 'out_shape': (4, 2, 3)},
    {'axis': 1, 'start': 1, 'out_shape': (2, 3, 4)},
    {'axis': -3, 'start': 2, 'out_shape': (3, 2, 4)},
    {'axis': -1, 'start': 0, 'out_shape': (4, 2, 3)},
    {'axis': -2, 'start': -2, 'out_shape': (2, 3, 4)},
    {'axis': 0, 'start': 3, 'out_shape': (3, 4, 2)},
    {'axis': 2, 'start': -3, 'out_shape': (4, 2, 3)},
)
class TestRollaxis(unittest.TestCase):

    dtype = numpy.float32

    def setUp(self):
        self.x = numpy.random.uniform(-1, 1, (2, 3, 4)).astype(self.dtype)
        self.g = numpy.random.uniform(-1, 1, self.out_shape).astype(self.dtype)

    def check_forward(self, x_data):
        x = chainer.Variable(x_data)
        y = functions.rollaxis(x, self.axis, self.start)

        expect = numpy.rollaxis(self.x, self.axis, self.start)
        testing.assert_allclose(y.data, expect)

    def test_forward_cpu(self):
        self.check_forward(self.x)

    @attr.gpu
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x))

    def check_backward(self, x_data, g_data):
        def f(x):
            return functions.rollaxis(x, self.axis, self.start)

        gradient_check.check_backward(
            f, x_data, g_data, dtype='d')

    def test_backward_cpu(self):
        self.check_backward(self.x, self.g)

    @attr.gpu
    def test_backward_gpu(self):
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.g))

@testing.parameterize(
    {'axis': 3, 'start': 0},
    {'axis': -4, 'start': 0},
    {'axis': 0, 'start': 4},
    {'axis': 0, 'start': -4},
)
class TestRollaxisInvalidType(unittest.TestCase):

    def setUp(self):
        self.x = numpy.random.uniform(-1, 1, (2, 3, 4)).astype('f')

    def check_type_error(self, x):
        with self.assertRaises(type_check.InvalidType):
            functions.rollaxis(x, self.axis, self.start)

    def test_type_error_cpu(self):
        self.check_type_error(self.x)

    @attr.gpu
    def test_type_error_gpu(self):
        self.check_type_error(cuda.to_gpu(self.x))

class TestRollaxisInvalidTypeError(unittest.TestCase):

    def test_invalid_axis(self):
        with self.assertRaises(TypeError):
            functions.Rollaxis('a', 0)

    def test_invalid_start(self):
        with self.assertRaises(TypeError):
            functions.Rollaxis(0, 'a')

testing.run_module(__name__, __file__)
<commit_msg>Add double-backward tests<commit_after>import unittest

import numpy

import chainer
from chainer import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr
from chainer.utils import type_check

@testing.parameterize(
    {'axis': 0, 'start': 2, 'out_shape': (3, 2, 4)},
    {'axis': 2, 'start': 0, 'out_shape': (4, 2, 3)},
    {'axis': 1, 'start': 1, 'out_shape': (2, 3, 4)},
    {'axis': -3, 'start': 2, 'out_shape': (3, 2, 4)},
    {'axis': -1, 'start': 0, 'out_shape': (4, 2, 3)},
    {'axis': -2, 'start': -2, 'out_shape': (2, 3, 4)},
    {'axis': 0, 'start': 3, 'out_shape': (3, 4, 2)},
    {'axis': 2, 'start': -3, 'out_shape': (4, 2, 3)},
)
class TestRollaxis(unittest.TestCase):

    dtype = numpy.float32

    def setUp(self):
        self.x = numpy.random.uniform(-1, 1, (2, 3, 4)).astype(self.dtype)
        self.g = numpy.random.uniform(-1, 1, self.out_shape).astype(self.dtype)
        self.gg = numpy.random.uniform(-1, 1, (2, 3, 4)).astype(self.dtype)

    def check_forward(self, x_data):
        x = chainer.Variable(x_data)
        y = functions.rollaxis(x, self.axis, self.start)

        expect = numpy.rollaxis(self.x, self.axis, self.start)
        testing.assert_allclose(y.data, expect)

    def test_forward_cpu(self):
        self.check_forward(self.x)

    @attr.gpu
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x))

    def check_backward(self, x_data, g_data):
        def f(x):
            return functions.rollaxis(x, self.axis, self.start)

        gradient_check.check_backward(
            f, x_data, g_data, dtype='d')

    def test_backward_cpu(self):
        self.check_backward(self.x, self.g)

    @attr.gpu
    def test_backward_gpu(self):
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.g))

    def check_double_backward(self, x_data, g_data, gg_data):
        def f(x):
            y = functions.rollaxis(x, self.axis, self.start)
            return y * y

        gradient_check.check_double_backward(f, x_data, g_data, gg_data)

    def test_double_backward_cpu(self):
        self.check_double_backward(self.x, self.g, self.gg)

    @attr.gpu
    def test_double_backward_gpu(self):
        self.check_double_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.g),
                                   cuda.to_gpu(self.gg))

@testing.parameterize(
    {'axis': 3, 'start': 0},
    {'axis': -4, 'start': 0},
    {'axis': 0, 'start': 4},
    {'axis': 0, 'start': -4},
)
class TestRollaxisInvalidType(unittest.TestCase):

    def setUp(self):
        self.x = numpy.random.uniform(-1, 1, (2, 3, 4)).astype('f')

    def check_type_error(self, x):
        with self.assertRaises(type_check.InvalidType):
            functions.rollaxis(x, self.axis, self.start)

    def test_type_error_cpu(self):
        self.check_type_error(self.x)

    @attr.gpu
    def test_type_error_gpu(self):
        self.check_type_error(cuda.to_gpu(self.x))

class TestRollaxisInvalidTypeError(unittest.TestCase):

    def test_invalid_axis(self):
        with self.assertRaises(TypeError):
            functions.Rollaxis('a', 0)

    def test_invalid_start(self):
        with self.assertRaises(TypeError):
            functions.Rollaxis(0, 'a')

testing.run_module(__name__, __file__)
<|endoftext|>