<commit_before># encoding=utf-8

import inspect
import itertools

from .types import BaseType
from .types.compound import ModelType
from .types.serializable import Serializable
from .exceptions import BaseError, ValidationError, ModelValidationError, ConversionError, ModelConversionError
from .serialize import atoms, serialize, flatten, expand
from .validate import validate
from .datastructures import OrderedDict as OrderedDictWithSort

class FieldDescriptor(object):

    def __init__(self, name):
        self.name = name

    def __get__(self, model, type=None):
        try:
            if model is None:
                return type.fields[self.name]
            return model._data[self.name]
        except KeyError:
            raise AttributeError(self.name)

    def __set__(self, model, value):
        field = model._fields[self.name]
        # TODO: read Options class for strict type checking flag
        #model._data[self.name] = field(value)
        if not isinstance(value, Model) and isinstance(field, ModelType):
            value = field.model_class(value)
        model._data[self.name] = value

    def __delete__(self, model):
        if self.name not in model._fields:
            raise AttributeError('%r has no attribute %r' %
                                 (type(model).__name__, self.name))
        del model._fields[self.name]

class ModelOptions(object):
    """
    This class is a container for all metaclass configuration options. Its
    primary purpose is to create an instance of a model's options for every
    instance of a model.

    It also creates errors in cases where unknown options parameters are found.

    :param roles:
        Allows to specify certain subsets of the model's fields for
        serialization.
    :param serialize_when_none:
        When ``False``, serialization skips fields that are None. Default: ``True``
    """
    def __init__(self, klass, namespace=None, roles=None, serialize_when_none=True):
        self.klass = klass
        self.namespace = namespace
        self.roles = roles or {}
        self.serialize_when_none = serialize_when_none

class ModelMeta(type):
    """Meta class for Models. Handles model inheritance and Options.
    """

    def __new__(cls, name, bases, attrs):
        fields = OrderedDictWithSort()
        serializables = {}
        validator_functions = {}  # Model level

        for base in reversed(bases):
            if hasattr(base, '_fields'):
                fields.update(base._fields)
            if hasattr(base, '_serializables'):
                serializables.update(base._serializables)
            if hasattr(base, '_validator_functions'):
                validator_functions.update(base._validator_functions)

        for key, value in attrs.iteritems():
            if key.startswith('validate_') and callable(value):
                validator_functions[key[9:]] = value
            if isinstance(value, BaseType):
                fields[key] = value
            if isinstance(value, Serializable):
                serializables[key] = value

        fields.sort(key=lambda i: i[1]._position_hint)

        for key, field in fields.iteritems():
            # For accessing internal data by field name attributes
            attrs[key] = FieldDescriptor(key)

        attrs['_options'] = cls._read_options(name, bases, attrs)

        attrs['_validator_functions'] = validator_functions
        attrs['_serializables'] = serializables
        attrs['_fields'] = fields

        klass = type.__new__(cls, name, bases, attrs)

        for field in fields.values():
            field.owner_model = klass

        return klass

    @classmethod
    def _read_options(cls, name, bases, attrs):
        options_members = {}

        for base in reversed(bases):
            if hasattr(base, "_options"):
                for k, v in inspect.getmembers(base._options):
                    if not k.startswith("_") and not k == "klass":
                        options_members[k] = v

        options_class = getattr(attrs, '__classoptions__', ModelOptions)
        if 'Options' in attrs:
            for k, v in inspect.getmembers(attrs['Options']):
                if not k.startswith("_"):
                    if k == "roles":
                        roles = options_members.get("roles", {}).copy()
                        roles.update(v)

                        options_members["roles"] = roles
                    else:
                        options_members[k] = v

        return options_class(cls, **options_members)

    @property
    def fields(cls):
        return cls._fields

    def __iter__(self):
        return itertools.chain(
            self._unbound_fields.iteritems(),
            self._unbound_serializables.iteritems()
        )

class Model(object):
    """Enclosure for fields and validation. Same pattern deployed by Django
    models, SQLAlchemy declarative extension and other developer friendly
    libraries.

    :param raw_data:
        Raw data to initialize the object with. Can raise ``ConversionError`` if
        it is not possible to convert the raw data into richer Python constructs.

    """

    __metaclass__ = ModelMeta
    __optionsclass__ = ModelOptions

    @classmethod
    def from_flat(cls, data):
        return cls(expand(data))

    def __init__(self, raw_data=None):
        if raw_data is None:
            raw_data = {}
        self._initial = raw_data
        self._data = self.convert(raw_data)

    def validate(self, partial=False, strict=False):
        """
        Validates the state of the model and adding additional untrusted data
        as well. If the models is invalid, raises ValidationError with error messages.

        :param partial:
            Allow partial data to validate; useful for PATCH requests.
            Essentially drops the ``required=True`` arguments from field
            definitions. Default: False
        :param strict:
            Complain about unrecognized keys. Default: False
        """
        try:
            data = validate(self, self._data, partial=partial, strict=strict)
            # Set internal data and touch the TypeDescriptors by setattr
            self._data.update(**data)
        except BaseError as e:
            raise ModelValidationError(e.messages)

    def serialize(self, role=None):
        """Return data as it would be validated. No filtering of output unless
        role is defined.

        :param role:
            Filter output by a specific role

        """
        return serialize(self, role)

    def flatten(self, role=None, prefix=""):
        """
        Return data as a pure key-value dictionary, where the values are
        primitive types (string, bool, int, long).

        :param role:
            Filter output by a specific role

        """
        return flatten(self, role, prefix=prefix)

    def convert(self, raw_data):
        """
        Converts the raw data into richer Python constructs according to the
        fields on the model
        """
        data = {}
        errors = {}

        is_class = isinstance(raw_data, self.__class__)
        is_dict = isinstance(raw_data, dict)

        if not is_class and not is_dict:
            error_msg = 'Model conversion requires a model or dict'
            raise ModelConversionError(error_msg)

        for field_name, field in self._fields.iteritems():
            serialized_field_name = field.serialized_name or field_name

            try:
                if serialized_field_name in raw_data:
                    raw_value = raw_data[serialized_field_name]
                else:
                    raw_value = raw_data[field_name]

                if raw_value is not None:
                    raw_value = field.convert(raw_value)
                data[field_name] = raw_value

            except KeyError:
                data[field_name] = field.default
            except ConversionError, e:
                errors[serialized_field_name] = e.messages

        if errors:
            raise ModelConversionError(errors)

        return data

    def __iter__(self):
        return self.iter()

    def iter(self):
        return iter(self._fields)

    def atoms(self, include_serializables=True):
        return atoms(self.__class__, self, include_serializables)

    def keys(self):
        return self._fields.keys()

    def items(self):
        return [(k, self.get(k)) for k in self._fields.iterkeys()]

    def values(self):
        return [self.get(k) for k in self._fields.iterkeys()]

    def __getitem__(self, name):
        try:
            return getattr(self, name)
        except AttributeError:
            pass
        raise KeyError(name)

    def __setitem__(self, name, value):
        # Ensure that the field exists before settings its value
        if name not in self._data:
            raise KeyError(name)
        return setattr(self, name, value)

    def __contains__(self, name):
        return name in self._data or name in self._serializables

    def __len__(self):
        return len(self._data)

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            for k in self._fields:
                if self.get(k) != other.get(k):
                    return False
            return True
        return False

    def __ne__(self, other):
        return not self == other

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

    def __repr__(self):
        try:
            u = unicode(self)
        except (UnicodeEncodeError, UnicodeDecodeError):
            u = '[Bad Unicode data]'
        return u"<%s: %s>" % (self.__class__.__name__, u)

    def __unicode__(self):
        return '%s object' % self.__class__.__name__
<commit_msg>Small refactor to group Mapping methods<commit_after># encoding=utf-8

import inspect
import itertools

from .types import BaseType
from .types.compound import ModelType
from .types.serializable import Serializable
from .exceptions import BaseError, ValidationError, ModelValidationError, ConversionError, ModelConversionError
from .serialize import atoms, serialize, flatten, expand
from .validate import validate
from .datastructures import OrderedDict as OrderedDictWithSort

class FieldDescriptor(object):

    def __init__(self, name):
        self.name = name

    def __get__(self, model, type=None):
        try:
            if model is None:
                return type.fields[self.name]
            return model._data[self.name]
        except KeyError:
            raise AttributeError(self.name)

    def __set__(self, model, value):
        field = model._fields[self.name]
        # TODO: read Options class for strict type checking flag
        #model._data[self.name] = field(value)
        if not isinstance(value, Model) and isinstance(field, ModelType):
            value = field.model_class(value)
        model._data[self.name] = value

    def __delete__(self, model):
        if self.name not in model._fields:
            raise AttributeError('%r has no attribute %r' %
                                 (type(model).__name__, self.name))
        del model._fields[self.name]

class ModelOptions(object):
    """
    This class is a container for all metaclass configuration options. Its
    primary purpose is to create an instance of a model's options for every
    instance of a model.

    It also creates errors in cases where unknown options parameters are found.

    :param roles:
        Allows to specify certain subsets of the model's fields for
        serialization.
    :param serialize_when_none:
        When ``False``, serialization skips fields that are None. Default: ``True``
    """
    def __init__(self, klass, namespace=None, roles=None, serialize_when_none=True):
        self.klass = klass
        self.namespace = namespace
        self.roles = roles or {}
        self.serialize_when_none = serialize_when_none

class ModelMeta(type):
    """Meta class for Models. Handles model inheritance and Options.
    """

    def __new__(cls, name, bases, attrs):
        fields = OrderedDictWithSort()
        serializables = {}
        validator_functions = {}  # Model level

        for base in reversed(bases):
            if hasattr(base, '_fields'):
                fields.update(base._fields)
            if hasattr(base, '_serializables'):
                serializables.update(base._serializables)
            if hasattr(base, '_validator_functions'):
                validator_functions.update(base._validator_functions)

        for key, value in attrs.iteritems():
            if key.startswith('validate_') and callable(value):
                validator_functions[key[9:]] = value
            if isinstance(value, BaseType):
                fields[key] = value
            if isinstance(value, Serializable):
                serializables[key] = value

        fields.sort(key=lambda i: i[1]._position_hint)

        for key, field in fields.iteritems():
            # For accessing internal data by field name attributes
            attrs[key] = FieldDescriptor(key)

        attrs['_options'] = cls._read_options(name, bases, attrs)

        attrs['_validator_functions'] = validator_functions
        attrs['_serializables'] = serializables
        attrs['_fields'] = fields

        klass = type.__new__(cls, name, bases, attrs)

        for field in fields.values():
            field.owner_model = klass

        return klass

    @classmethod
    def _read_options(cls, name, bases, attrs):
        options_members = {}

        for base in reversed(bases):
            if hasattr(base, "_options"):
                for k, v in inspect.getmembers(base._options):
                    if not k.startswith("_") and not k == "klass":
                        options_members[k] = v

        options_class = getattr(attrs, '__classoptions__', ModelOptions)
        if 'Options' in attrs:
            for k, v in inspect.getmembers(attrs['Options']):
                if not k.startswith("_"):
                    if k == "roles":
                        roles = options_members.get("roles", {}).copy()
                        roles.update(v)

                        options_members["roles"] = roles
                    else:
                        options_members[k] = v

        return options_class(cls, **options_members)

    @property
    def fields(cls):
        return cls._fields

    def __iter__(self):
        return itertools.chain(
            self._unbound_fields.iteritems(),
            self._unbound_serializables.iteritems()
        )

class Model(object):
    """Enclosure for fields and validation. Same pattern deployed by Django
    models, SQLAlchemy declarative extension and other developer friendly
    libraries.

    :param raw_data:
        Raw data to initialize the object with. Can raise ``ConversionError`` if
        it is not possible to convert the raw data into richer Python constructs.

    """

    __metaclass__ = ModelMeta
    __optionsclass__ = ModelOptions

    @classmethod
    def from_flat(cls, data):
        return cls(expand(data))

    def __init__(self, raw_data=None):
        if raw_data is None:
            raw_data = {}
        self._initial = raw_data
        self._data = self.convert(raw_data)

    def validate(self, partial=False, strict=False):
        """
        Validates the state of the model and adding additional untrusted data
        as well. If the models is invalid, raises ValidationError with error messages.

        :param partial:
            Allow partial data to validate; useful for PATCH requests.
            Essentially drops the ``required=True`` arguments from field
            definitions. Default: False
        :param strict:
            Complain about unrecognized keys. Default: False
        """
        try:
            data = validate(self, self._data, partial=partial, strict=strict)
            # Set internal data and touch the TypeDescriptors by setattr
            self._data.update(**data)
        except BaseError as e:
            raise ModelValidationError(e.messages)

    def serialize(self, role=None):
        """Return data as it would be validated. No filtering of output unless
        role is defined.

        :param role:
            Filter output by a specific role

        """
        return serialize(self, role)

    def flatten(self, role=None, prefix=""):
        """
        Return data as a pure key-value dictionary, where the values are
        primitive types (string, bool, int, long).

        :param role:
            Filter output by a specific role

        """
        return flatten(self, role, prefix=prefix)

    def convert(self, raw_data):
        """
        Converts the raw data into richer Python constructs according to the
        fields on the model
        """
        data = {}
        errors = {}

        is_class = isinstance(raw_data, self.__class__)
        is_dict = isinstance(raw_data, dict)

        if not is_class and not is_dict:
            error_msg = 'Model conversion requires a model or dict'
            raise ModelConversionError(error_msg)

        for field_name, field in self._fields.iteritems():
            serialized_field_name = field.serialized_name or field_name

            try:
                if serialized_field_name in raw_data:
                    raw_value = raw_data[serialized_field_name]
                else:
                    raw_value = raw_data[field_name]

                if raw_value is not None:
                    raw_value = field.convert(raw_value)
                data[field_name] = raw_value

            except KeyError:
                data[field_name] = field.default
            except ConversionError, e:
                errors[serialized_field_name] = e.messages

        if errors:
            raise ModelConversionError(errors)

        return data

    def __iter__(self):
        return self.iter()

    def iter(self):
        return iter(self._fields)

    def atoms(self, include_serializables=True):
        return atoms(self.__class__, self, include_serializables)

    def keys(self):
        return self._fields.keys()

    def items(self):
        return [(k, self.get(k)) for k in self._fields.iterkeys()]

    def values(self):
        return [self.get(k) for k in self._fields.iterkeys()]

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

    def __getitem__(self, name):
        try:
            return getattr(self, name)
        except AttributeError:
            pass
        raise KeyError(name)

    def __setitem__(self, name, value):
        # Ensure that the field exists before settings its value
        if name not in self._data:
            raise KeyError(name)
        return setattr(self, name, value)

    def __contains__(self, name):
        return name in self._data or name in self._serializables

    def __len__(self):
        return len(self._data)

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            for k in self._fields:
                if self.get(k) != other.get(k):
                    return False
            return True
        return False

    def __ne__(self, other):
        return not self == other

    def __repr__(self):
        try:
            u = unicode(self)
        except (UnicodeEncodeError, UnicodeDecodeError):
            u = '[Bad Unicode data]'
        return u"<%s: %s>" % (self.__class__.__name__, u)

    def __unicode__(self):
        return '%s object' % self.__class__.__name__
<|endoftext|>


<commit_before># encoding=utf-8

import inspect
import itertools

from .types import BaseType
from .types.compound import ModelType
from .types.serializable import Serializable
from .exceptions import (BaseError, ValidationError, ModelValidationError,
                         ConversionError, ModelConversionError)
from .transforms import allow_none, atoms, flatten, expand
from .transforms import to_primitive, to_native, convert
from .validate import validate
from .datastructures import OrderedDict as OrderedDictWithSort

class FieldDescriptor(object):
    """
    The FieldDescriptor serves as a wrapper for Types that converts them into
    fields.

    A field is then the merger of a Type and it's Model.
    """

    def __init__(self, name):
        """
        :param name:
            The field's name
        """
        self.name = name

    def __get__(self, instance, cls):
        """
        Checks the field name against the definition of the model and returns
        the corresponding data for valid fields or raises the appropriate error
        for fields missing from a class.
        """
        try:
            if instance is None:
                return cls._fields[self.name]
            return instance._data[self.name]
        except KeyError:
            raise AttributeError(self.name)

    def __set__(self, instance, value):
        """
        Checks the field name against a model and sets the value.
        """
        field = instance._fields[self.name]
        if not isinstance(value, Model) and isinstance(field, ModelType):
            value = field.model_class(value)
        instance._data[self.name] = value

    def __delete__(self, instance):
        """
        Checks the field name against a model and deletes the field.
        """
        if self.name not in instance._fields:
            raise AttributeError('%r has no attribute %r' %
                                 (type(instance).__name__, self.name))
        del instance._fields[self.name]

class ModelOptions(object):
    """
    This class is a container for all metaclass configuration options. Its
    primary purpose is to create an instance of a model's options for every
    instance of a model.
    """
    def __init__(self, klass, namespace=None, roles=None,
                 serialize_when_none=True):
        """
        :param klass:
            The class which this options instance belongs to.
        :param namespace:
            A namespace identifier that can be used with persistence layers.
        :param roles:
            Allows to specify certain subsets of the model's fields for
            serialization.
        :param serialize_when_none:
            When ``False``, serialization skips fields that are None.
            Default: ``True``
        """
        self.klass = klass
        self.namespace = namespace
        self.roles = roles or {}
        self.serialize_when_none = serialize_when_none

class ModelMeta(type):
    """
    Meta class for Models.
    """

    def __new__(cls, name, bases, attrs):
        """
        This metaclass adds four attributes to host classes: cls._fields,
        cls._serializables, cls._validator_functions, and cls._options.

        This function creates those attributes like this:

        ``cls._fields`` is list of fields that are schematics types
        ``cls._serializables`` is a list of functions that are used to generate
        values during serialization
        ``cls._validator_functions`` are class level validation functions
        ``cls._options`` is the end result of parsing the ``Options`` class
        """

        ### Structures used to accumulate meta info
        fields = OrderedDictWithSort()
        serializables = {}
        validator_functions = {}  # Model level

        ### Accumulate metas info from parent classes
        for base in reversed(bases):
            if hasattr(base, '_fields'):
                fields.update(base._fields)
            if hasattr(base, '_serializables'):
                serializables.update(base._serializables)
            if hasattr(base, '_validator_functions'):
                validator_functions.update(base._validator_functions)

        ### Parse this class's attributes into meta structures
        for key, value in attrs.iteritems():
            if key.startswith('validate_') and callable(value):
                validator_functions[key[9:]] = value
            if isinstance(value, BaseType):
                fields[key] = value
            if isinstance(value, Serializable):
                serializables[key] = value

        ### Parse meta options
        options = cls._read_options(name, bases, attrs)

        ### Convert list of types into fields for new klass
        fields.sort(key=lambda i: i[1]._position_hint)
        for key, field in fields.iteritems():
            attrs[key] = FieldDescriptor(key)

        ### Ready meta data to be klass attributes
        attrs['_fields'] = fields
        attrs['_serializables'] = serializables
        attrs['_validator_functions'] = validator_functions
        attrs['_options'] = options

        klass = type.__new__(cls, name, bases, attrs)

        ### Add reference to klass to each field instance
        for field in fields.values():
            field.owner_model = klass

        return klass

    @classmethod
    def _read_options(cls, name, bases, attrs):
        """
        Parses `ModelOptions` instance into the options value attached to
        `Model` instances.
        """
        options_members = {}

        for base in reversed(bases):
            if hasattr(base, "_options"):
                for k, v in inspect.getmembers(base._options):
                    if not k.startswith("_") and not k == "klass":
                        options_members[k] = v

        options_class = getattr(attrs, '__classoptions__', ModelOptions)
        if 'Options' in attrs:
            for k, v in inspect.getmembers(attrs['Options']):
                if not k.startswith("_"):
                    if k == "roles":
                        roles = options_members.get("roles", {}).copy()
                        roles.update(v)

                        options_members["roles"] = roles
                    else:
                        options_members[k] = v

        return options_class(cls, **options_members)

    @property
    def fields(cls):
        return cls._fields

    def __iter__(self):
        return itertools.chain(
            self._unbound_fields.iteritems(),
            self._unbound_serializables.iteritems()
        )

class Model(object):
    """
    Enclosure for fields and validation. Same pattern deployed by Django
    models, SQLAlchemy declarative extension and other developer friendly
    libraries.

    Initial field values can be passed in as keyword arguments to ``__init__``
    to initialize the object with. Can raise ``ConversionError`` if it is not
    possible to convert the raw data into richer Python constructs.
    """

    __metaclass__ = ModelMeta
    __optionsclass__ = ModelOptions

    def __init__(self, raw_data=None):  # TODO change back to keywords
        if raw_data is None:
            raw_data = {}
        self._initial = raw_data
        self._data = self.convert(raw_data, strict=True)

    def validate(self, partial=False, strict=False):
        """
        Validates the state of the model and adding additional untrusted data
        as well. If the models is invalid, raises ValidationError with error
        messages.

        :param partial:
            Allow partial data to validate; useful for PATCH requests.
            Essentially drops the ``required=True`` arguments from field
            definitions. Default: False
        :param strict:
            Complain about unrecognized keys. Default: False
        """
        try:
            data = validate(self.__class__, self._data, partial=partial,
                            strict=strict)
            self._data.update(**data)
        except BaseError as e:
            raise ModelValidationError(e.messages)

    def convert(self, raw_data, **kw):
        """
        Converts the raw data into richer Python constructs according to the
        fields on the model

        :param raw_data:
            The data to be converted
        """
        return convert(self.__class__, raw_data, **kw)

    def to_native(self, role=None):
        return to_native(self.__class__, self, role=role)

    def to_primitive(self, role=None):
        """Return data as it would be validated. No filtering of output unless
        role is defined.

        :param role:
            Filter output by a specific role

        """
        return to_primitive(self.__class__, self, role=role)

    def serialize(self, role=None):
        return self.to_primitive(role=role)

    def flatten(self, role=None, prefix=""):
        """
        Return data as a pure key-value dictionary, where the values are
        primitive types (string, bool, int, long).

        :param role:
            Filter output by a specific role
        :param prefix:
            A prefix to use for keynames during flattening.
        """
        return flatten(self.__class__, self, role=role, prefix=prefix)

    @classmethod
    def from_flat(cls, data):
        return cls(expand(data))

    def atoms(self):
        """
        Iterator for the atomic components of a model definition and relevant
        data that creates a threeple of the field's name, the instance of it's
        type, and it's value.
        """
        return atoms(self.__class__, self)

    @classmethod
    def allow_none(cls, field):
        """
        Inspects a field and class for ``serialize_when_none`` setting.

        The setting defaults to the value of the class.  A field can override
        the class setting with it's own ``serialize_when_none`` setting.
        """
        return allow_none(cls, field)

    def __iter__(self):
        return self.iter()

    def iter(self):
        return iter(self._fields)

    def __getitem__(self, name):
        try:
            return getattr(self, name)
        except AttributeError:
            pass
        raise KeyError(name)

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

    def __setitem__(self, name, value):
        if name not in self._data:
            raise KeyError(name)
        return setattr(self, name, value)

    def __contains__(self, name):
        return name in self._data or name in self._serializables

    def __len__(self):
        return len(self._data)

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            keys = self._fields

            for key in keys:
                if self[key] != other[key]:
                    return False
            return True
        return False

    def __ne__(self, other):
        return not self == other

    def __repr__(self):
        try:
            u = unicode(self)
        except (UnicodeEncodeError, UnicodeDecodeError):
            u = '[Bad Unicode data]'
        return u"<%s: %s>" % (self.__class__.__name__, u)

    def __unicode__(self):
        return '%s object' % self.__class__.__name__
<commit_msg>fix options parsing from __optionsclass__<commit_after># encoding=utf-8

import inspect
import itertools

from .types import BaseType
from .types.compound import ModelType
from .types.serializable import Serializable
from .exceptions import (BaseError, ValidationError, ModelValidationError,
                         ConversionError, ModelConversionError)
from .transforms import allow_none, atoms, flatten, expand
from .transforms import to_primitive, to_native, convert
from .validate import validate
from .datastructures import OrderedDict as OrderedDictWithSort

class FieldDescriptor(object):
    """
    The FieldDescriptor serves as a wrapper for Types that converts them into
    fields.

    A field is then the merger of a Type and it's Model.
    """

    def __init__(self, name):
        """
        :param name:
            The field's name
        """
        self.name = name

    def __get__(self, instance, cls):
        """
        Checks the field name against the definition of the model and returns
        the corresponding data for valid fields or raises the appropriate error
        for fields missing from a class.
        """
        try:
            if instance is None:
                return cls._fields[self.name]
            return instance._data[self.name]
        except KeyError:
            raise AttributeError(self.name)

    def __set__(self, instance, value):
        """
        Checks the field name against a model and sets the value.
        """
        field = instance._fields[self.name]
        if not isinstance(value, Model) and isinstance(field, ModelType):
            value = field.model_class(value)
        instance._data[self.name] = value

    def __delete__(self, instance):
        """
        Checks the field name against a model and deletes the field.
        """
        if self.name not in instance._fields:
            raise AttributeError('%r has no attribute %r' %
                                 (type(instance).__name__, self.name))
        del instance._fields[self.name]

class ModelOptions(object):
    """
    This class is a container for all metaclass configuration options. Its
    primary purpose is to create an instance of a model's options for every
    instance of a model.
    """
    def __init__(self, klass, namespace=None, roles=None,
                 serialize_when_none=True):
        """
        :param klass:
            The class which this options instance belongs to.
        :param namespace:
            A namespace identifier that can be used with persistence layers.
        :param roles:
            Allows to specify certain subsets of the model's fields for
            serialization.
        :param serialize_when_none:
            When ``False``, serialization skips fields that are None.
            Default: ``True``
        """
        self.klass = klass
        self.namespace = namespace
        self.roles = roles or {}
        self.serialize_when_none = serialize_when_none

class ModelMeta(type):
    """
    Meta class for Models.
    """

    def __new__(cls, name, bases, attrs):
        """
        This metaclass adds four attributes to host classes: cls._fields,
        cls._serializables, cls._validator_functions, and cls._options.

        This function creates those attributes like this:

        ``cls._fields`` is list of fields that are schematics types
        ``cls._serializables`` is a list of functions that are used to generate
        values during serialization
        ``cls._validator_functions`` are class level validation functions
        ``cls._options`` is the end result of parsing the ``Options`` class
        """

        ### Structures used to accumulate meta info
        fields = OrderedDictWithSort()
        serializables = {}
        validator_functions = {}  # Model level

        ### Accumulate metas info from parent classes
        for base in reversed(bases):
            if hasattr(base, '_fields'):
                fields.update(base._fields)
            if hasattr(base, '_serializables'):
                serializables.update(base._serializables)
            if hasattr(base, '_validator_functions'):
                validator_functions.update(base._validator_functions)

        ### Parse this class's attributes into meta structures
        for key, value in attrs.iteritems():
            if key.startswith('validate_') and callable(value):
                validator_functions[key[9:]] = value
            if isinstance(value, BaseType):
                fields[key] = value
            if isinstance(value, Serializable):
                serializables[key] = value

        ### Parse meta options
        options = cls._read_options(name, bases, attrs)

        ### Convert list of types into fields for new klass
        fields.sort(key=lambda i: i[1]._position_hint)
        for key, field in fields.iteritems():
            attrs[key] = FieldDescriptor(key)

        ### Ready meta data to be klass attributes
        attrs['_fields'] = fields
        attrs['_serializables'] = serializables
        attrs['_validator_functions'] = validator_functions
        attrs['_options'] = options

        klass = type.__new__(cls, name, bases, attrs)

        ### Add reference to klass to each field instance
        for field in fields.values():
            field.owner_model = klass

        return klass

    @classmethod
    def _read_options(cls, name, bases, attrs):
        """
        Parses `ModelOptions` instance into the options value attached to
        `Model` instances.
        """
        options_members = {}

        for base in reversed(bases):
            if hasattr(base, "_options"):
                for k, v in inspect.getmembers(base._options):
                    if not k.startswith("_") and not k == "klass":
                        options_members[k] = v

        options_class = attrs.get('__optionsclass__', ModelOptions)
        if 'Options' in attrs:
            for k, v in inspect.getmembers(attrs['Options']):
                if not k.startswith("_"):
                    if k == "roles":
                        roles = options_members.get("roles", {}).copy()
                        roles.update(v)

                        options_members["roles"] = roles
                    else:
                        options_members[k] = v

        return options_class(cls, **options_members)

    @property
    def fields(cls):
        return cls._fields

    def __iter__(self):
        return itertools.chain(
            self._unbound_fields.iteritems(),
            self._unbound_serializables.iteritems()
        )

class Model(object):
    """
    Enclosure for fields and validation. Same pattern deployed by Django
    models, SQLAlchemy declarative extension and other developer friendly
    libraries.

    Initial field values can be passed in as keyword arguments to ``__init__``
    to initialize the object with. Can raise ``ConversionError`` if it is not
    possible to convert the raw data into richer Python constructs.
    """

    __metaclass__ = ModelMeta
    __optionsclass__ = ModelOptions

    def __init__(self, raw_data=None):  # TODO change back to keywords
        if raw_data is None:
            raw_data = {}
        self._initial = raw_data
        self._data = self.convert(raw_data, strict=True)

    def validate(self, partial=False, strict=False):
        """
        Validates the state of the model and adding additional untrusted data
        as well. If the models is invalid, raises ValidationError with error
        messages.

        :param partial:
            Allow partial data to validate; useful for PATCH requests.
            Essentially drops the ``required=True`` arguments from field
            definitions. Default: False
        :param strict:
            Complain about unrecognized keys. Default: False
        """
        try:
            data = validate(self.__class__, self._data, partial=partial,
                            strict=strict)
            self._data.update(**data)
        except BaseError as e:
            raise ModelValidationError(e.messages)

    def convert(self, raw_data, **kw):
        """
        Converts the raw data into richer Python constructs according to the
        fields on the model

        :param raw_data:
            The data to be converted
        """
        return convert(self.__class__, raw_data, **kw)

    def to_native(self, role=None):
        return to_native(self.__class__, self, role=role)

    def to_primitive(self, role=None):
        """Return data as it would be validated. No filtering of output unless
        role is defined.

        :param role:
            Filter output by a specific role

        """
        return to_primitive(self.__class__, self, role=role)

    def serialize(self, role=None):
        return self.to_primitive(role=role)

    def flatten(self, role=None, prefix=""):
        """
        Return data as a pure key-value dictionary, where the values are
        primitive types (string, bool, int, long).

        :param role:
            Filter output by a specific role
        :param prefix:
            A prefix to use for keynames during flattening.
        """
        return flatten(self.__class__, self, role=role, prefix=prefix)

    @classmethod
    def from_flat(cls, data):
        return cls(expand(data))

    def atoms(self):
        """
        Iterator for the atomic components of a model definition and relevant
        data that creates a threeple of the field's name, the instance of it's
        type, and it's value.
        """
        return atoms(self.__class__, self)

    @classmethod
    def allow_none(cls, field):
        """
        Inspects a field and class for ``serialize_when_none`` setting.

        The setting defaults to the value of the class.  A field can override
        the class setting with it's own ``serialize_when_none`` setting.
        """
        return allow_none(cls, field)

    def __iter__(self):
        return self.iter()

    def iter(self):
        return iter(self._fields)

    def __getitem__(self, name):
        try:
            return getattr(self, name)
        except AttributeError:
            pass
        raise KeyError(name)

    def get(self, key, default=None):
        try:
            return self[key]
        except KeyError:
            return default

    def __setitem__(self, name, value):
        if name not in self._data:
            raise KeyError(name)
        return setattr(self, name, value)

    def __contains__(self, name):
        return name in self._data or name in self._serializables

    def __len__(self):
        return len(self._data)

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            keys = self._fields

            for key in keys:
                if self[key] != other[key]:
                    return False
            return True
        return False

    def __ne__(self, other):
        return not self == other

    def __repr__(self):
        try:
            u = unicode(self)
        except (UnicodeEncodeError, UnicodeDecodeError):
            u = '[Bad Unicode data]'
        return u"<%s: %s>" % (self.__class__.__name__, u)

    def __unicode__(self):
        return '%s object' % self.__class__.__name__
<|endoftext|>


<commit_before>"""Base class for sparse matrice with a .data attribute

    subclasses must provide a _with_data() method that
    creates a new matrix with the same sparsity pattern
    as self but with a different data array

"""

from __future__ import division, print_function, absolute_import

__all__ = []

import numpy as np

from .base import spmatrix, _ufuncs_with_fixed_point_at_zero
from .sputils import isscalarlike

# TODO implement all relevant operations
# use .data.__methods__() instead of /=, *=, etc.
class _data_matrix(spmatrix):
    def __init__(self):
        spmatrix.__init__(self)

    def _get_dtype(self):
        return self.data.dtype

    def _set_dtype(self,newtype):
        self.data.dtype = newtype
    dtype = property(fget=_get_dtype,fset=_set_dtype)

    def _deduped_data(self):
        if hasattr(self, 'sum_duplicates'):
            self.sum_duplicates()
        return self.data

    def __abs__(self):
        return self._with_data(abs(self._deduped_data()))

    def _real(self):
        return self._with_data(self.data.real)

    def _imag(self):
        return self._with_data(self.data.imag)

    def __neg__(self):
        return self._with_data(-self.data)

    def __imul__(self, other):  # self *= other
        if isscalarlike(other):
            self.data *= other
            return self
        else:
            return NotImplemented

    def __itruediv__(self, other):  # self /= other
        if isscalarlike(other):
            recip = 1.0 / other
            self.data *= recip
            return self
        else:
            return NotImplemented

    def astype(self, t):
        return self._with_data(self._deduped_data().astype(t))

    def conj(self):
        return self._with_data(self.data.conj())

    def copy(self):
        return self._with_data(self.data.copy(), copy=True)

    def count_nonzero(self):
        if self.has_canonical_format:
          return np.count_nonzero(self.data)
        return spmatrix.count_nonzero(self)

    def power(self, n, dtype=None):
        """
        This function performs element-wise power.

        Parameters
        ----------
        n : n is a scalar

        dtype : If dtype is not specified, the current dtype will be preserved.
        """
        if not isscalarlike(n):
            raise NotImplementedError("input is not scalar")

        data = self._deduped_data()
        if dtype is not None:
            data = data.astype(dtype)
        return self._with_data(data ** n)

    ###########################
    # Multiplication handlers #
    ###########################

    def _mul_scalar(self, other):
        return self._with_data(self.data * other)

# Add the numpy unary ufuncs for which func(0) = 0 to _data_matrix.
for npfunc in _ufuncs_with_fixed_point_at_zero:
    name = npfunc.__name__

    def _create_method(op):
        def method(self):
            result = op(self.data)
            x = self._with_data(result, copy=True)
            return x

        method.__doc__ = ("Element-wise %s.\n\n"
                          "See numpy.%s for more information." % (name, name))
        method.__name__ = name

        return method

    setattr(_data_matrix, name, _create_method(npfunc))

class _minmax_mixin(object):
    """Mixin for min and max methods.

    These are not implemented for dia_matrix, hence the separate class.
    """

    def _min_or_max_axis(self, axis, min_or_max):
        N = self.shape[axis]
        if N == 0:
            raise ValueError("zero-size array to reduction operation")
        M = self.shape[1 - axis]

        mat = self.tocsc() if axis == 0 else self.tocsr()
        mat.sum_duplicates()

        major_index, value = mat._minor_reduce(min_or_max)
        not_full = np.diff(mat.indptr)[major_index] < N
        value[not_full] = min_or_max(value[not_full], 0)

        mask = value != 0
        major_index = np.compress(mask, major_index)
        value = np.compress(mask, value)

        from . import coo_matrix
        if axis == 0:
            return coo_matrix((value, (np.zeros(len(value)), major_index)),
                              dtype=self.dtype, shape=(1, M))
        else:
            return coo_matrix((value, (major_index, np.zeros(len(value)))),
                              dtype=self.dtype, shape=(M, 1))

    def _min_or_max(self, axis, min_or_max):
        if axis is None:
            if 0 in self.shape:
                raise ValueError("zero-size array to reduction operation")

            zero = self.dtype.type(0)
            if self.nnz == 0:
                return zero
            m = min_or_max.reduce(self._deduped_data().ravel())
            if self.nnz != np.product(self.shape):
                m = min_or_max(zero, m)
            return m

        if axis < 0:
            axis += 2
        if (axis == 0) or (axis == 1):
            return self._min_or_max_axis(axis, min_or_max)
        else:
            raise ValueError("invalid axis, use 0 for rows, or 1 for columns")

    def max(self, axis=None):
        """Maximum of the elements of this matrix.

        This takes all elements into account, not just the non-zero ones.

        Returns
        -------
        amax : self.dtype
            Maximum element.
        """
        return self._min_or_max(axis, np.maximum)

    def min(self, axis=None):
        """Minimum of the elements of this matrix.

        This takes all elements into account, not just the non-zero ones.

        Returns
        -------
        amin : self.dtype
            Minimum element.
        """
        return self._min_or_max(axis, np.minimum)
<commit_msg>Fixing indentation<commit_after>"""Base class for sparse matrice with a .data attribute

    subclasses must provide a _with_data() method that
    creates a new matrix with the same sparsity pattern
    as self but with a different data array

"""

from __future__ import division, print_function, absolute_import

__all__ = []

import numpy as np

from .base import spmatrix, _ufuncs_with_fixed_point_at_zero
from .sputils import isscalarlike

# TODO implement all relevant operations
# use .data.__methods__() instead of /=, *=, etc.
class _data_matrix(spmatrix):
    def __init__(self):
        spmatrix.__init__(self)

    def _get_dtype(self):
        return self.data.dtype

    def _set_dtype(self,newtype):
        self.data.dtype = newtype
    dtype = property(fget=_get_dtype,fset=_set_dtype)

    def _deduped_data(self):
        if hasattr(self, 'sum_duplicates'):
            self.sum_duplicates()
        return self.data

    def __abs__(self):
        return self._with_data(abs(self._deduped_data()))

    def _real(self):
        return self._with_data(self.data.real)

    def _imag(self):
        return self._with_data(self.data.imag)

    def __neg__(self):
        return self._with_data(-self.data)

    def __imul__(self, other):  # self *= other
        if isscalarlike(other):
            self.data *= other
            return self
        else:
            return NotImplemented

    def __itruediv__(self, other):  # self /= other
        if isscalarlike(other):
            recip = 1.0 / other
            self.data *= recip
            return self
        else:
            return NotImplemented

    def astype(self, t):
        return self._with_data(self._deduped_data().astype(t))

    def conj(self):
        return self._with_data(self.data.conj())

    def copy(self):
        return self._with_data(self.data.copy(), copy=True)

    def count_nonzero(self):
        if self.has_canonical_format:
            return np.count_nonzero(self.data)
        return spmatrix.count_nonzero(self)

    def power(self, n, dtype=None):
        """
        This function performs element-wise power.

        Parameters
        ----------
        n : n is a scalar

        dtype : If dtype is not specified, the current dtype will be preserved.
        """
        if not isscalarlike(n):
            raise NotImplementedError("input is not scalar")

        data = self._deduped_data()
        if dtype is not None:
            data = data.astype(dtype)
        return self._with_data(data ** n)

    ###########################
    # Multiplication handlers #
    ###########################

    def _mul_scalar(self, other):
        return self._with_data(self.data * other)

# Add the numpy unary ufuncs for which func(0) = 0 to _data_matrix.
for npfunc in _ufuncs_with_fixed_point_at_zero:
    name = npfunc.__name__

    def _create_method(op):
        def method(self):
            result = op(self.data)
            x = self._with_data(result, copy=True)
            return x

        method.__doc__ = ("Element-wise %s.\n\n"
                          "See numpy.%s for more information." % (name, name))
        method.__name__ = name

        return method

    setattr(_data_matrix, name, _create_method(npfunc))

class _minmax_mixin(object):
    """Mixin for min and max methods.

    These are not implemented for dia_matrix, hence the separate class.
    """

    def _min_or_max_axis(self, axis, min_or_max):
        N = self.shape[axis]
        if N == 0:
            raise ValueError("zero-size array to reduction operation")
        M = self.shape[1 - axis]

        mat = self.tocsc() if axis == 0 else self.tocsr()
        mat.sum_duplicates()

        major_index, value = mat._minor_reduce(min_or_max)
        not_full = np.diff(mat.indptr)[major_index] < N
        value[not_full] = min_or_max(value[not_full], 0)

        mask = value != 0
        major_index = np.compress(mask, major_index)
        value = np.compress(mask, value)

        from . import coo_matrix
        if axis == 0:
            return coo_matrix((value, (np.zeros(len(value)), major_index)),
                              dtype=self.dtype, shape=(1, M))
        else:
            return coo_matrix((value, (major_index, np.zeros(len(value)))),
                              dtype=self.dtype, shape=(M, 1))

    def _min_or_max(self, axis, min_or_max):
        if axis is None:
            if 0 in self.shape:
                raise ValueError("zero-size array to reduction operation")

            zero = self.dtype.type(0)
            if self.nnz == 0:
                return zero
            m = min_or_max.reduce(self._deduped_data().ravel())
            if self.nnz != np.product(self.shape):
                m = min_or_max(zero, m)
            return m

        if axis < 0:
            axis += 2
        if (axis == 0) or (axis == 1):
            return self._min_or_max_axis(axis, min_or_max)
        else:
            raise ValueError("invalid axis, use 0 for rows, or 1 for columns")

    def max(self, axis=None):
        """Maximum of the elements of this matrix.

        This takes all elements into account, not just the non-zero ones.

        Returns
        -------
        amax : self.dtype
            Maximum element.
        """
        return self._min_or_max(axis, np.maximum)

    def min(self, axis=None):
        """Minimum of the elements of this matrix.

        This takes all elements into account, not just the non-zero ones.

        Returns
        -------
        amin : self.dtype
            Minimum element.
        """
        return self._min_or_max(axis, np.minimum)
<|endoftext|>


<commit_before>from seed_identity_store.settings import *  # flake8: noqa

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'TESTSEKRET'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

TEMPLATE_DEBUG = True

CELERY_EAGER_PROPAGATES_EXCEPTIONS = True
CELERY_ALWAYS_EAGER = True
BROKER_BACKEND = 'memory'
CELERY_RESULT_BACKEND = 'djcelery.backends.database:DatabaseBackend'

METRICS_URL = "http://metrics-url"
METRICS_AUTH_TOKEN = "REPLACEME"

# REST Framework conf defaults
REST_FRAMEWORK['PAGE_SIZE'] = 2
<commit_msg>Revert "Merge pull request #55 from praekelt/feature/SEED-1052-fix-test-settings"<commit_after>from seed_identity_store.settings import *  # flake8: noqa

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'TESTSEKRET'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

TEMPLATE_DEBUG = True

CELERY_EAGER_PROPAGATES_EXCEPTIONS = True
CELERY_ALWAYS_EAGER = True
BROKER_BACKEND = 'memory'
CELERY_RESULT_BACKEND = 'djcelery.backends.database:DatabaseBackend'

METRICS_URL = "http://metrics-url"
METRICS_AUTH_TOKEN = "REPLACEME"

# REST Framework conf defaults
REST_FRAMEWORK = {
    'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.CursorPagination',
    'PAGE_SIZE': 2,
    'DEFAULT_AUTHENTICATION_CLASSES': (
        'rest_framework.authentication.BasicAuthentication',
        'rest_framework.authentication.TokenAuthentication',
    ),
    'DEFAULT_PERMISSION_CLASSES': (
        'rest_framework.permissions.IsAuthenticated',
    ),
    'DEFAULT_FILTER_BACKENDS': ('rest_framework.filters.DjangoFilterBackend',)
}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import json

from django.core.files.uploadedfile import SimpleUploadedFile
from django.test import TestCase
from django.urls.base import reverse

from seqr.views.apis.family_api import update_family_pedigree_image, update_family_assigned_analyst, update_family_success_story_types
from seqr.views.utils.test_utils import _check_login

FAMILY_GUID = 'F000001_1'

class ProjectAPITest(TestCase):
    fixtures = ['users', '1kg_project']

    #  TODO test other family api methods

    def test_update_family_pedigree_image(self):
        url = reverse(update_family_pedigree_image, args=[FAMILY_GUID])
        _check_login(self, url)

        f = SimpleUploadedFile("new_ped_image_123.png", b"file_content")

        # send invalid request
        response = self.client.post(url, {'f1': f, 'f2': f})
        self.assertEqual(response.status_code, 400)
        self.assertEqual(response.reason_phrase, 'Received 2 files')

        # send valid add/update request
        response = self.client.post(url, {'f': f})
        self.assertEqual(response.status_code, 200)
        response_json = response.json()

        self.assertListEqual(response_json.keys(), ['F000001_1'])
        self.assertRegex(response_json['F000001_1']['pedigreeImage'], '/media/pedigree_images/new_ped_image_.+\.png')

        # send valid delete request
        response = self.client.post(url)
        self.assertEqual(response.status_code, 200)
        response_json = response.json()

        self.assertListEqual(response_json.keys(), ['F000001_1'])
        self.assertIsNone(response_json['F000001_1']['pedigreeImage'])

    def test_update_family_assigned_analyst(self):
        url = reverse(update_family_assigned_analyst, args=[FAMILY_GUID])
        _check_login(self, url)

        # send invalid request
        response = self.client.post(url, content_type='application/json', data=json.dumps({}))
        self.assertEqual(response.status_code, 400)
        self.assertEqual(response.reason_phrase, '\'assigned analyst\' is not specified')

        # send invalid username (without permission)
        response = self.client.post(url, content_type='application/json', data=json.dumps({'assigned_analyst_username': 'invalid_username'}))
        self.assertEqual(response.status_code, 400)
        self.assertEqual(response.reason_phrase, 'specified user does not exist')

        # send valid request
        response = self.client.post(url, content_type='application/json', data=json.dumps({'assigned_analyst_username': 'test_user'}))
        self.assertEqual(response.status_code, 200)
        response_json = response.json()

        self.assertListEqual(response_json.keys(), ['F000001_1'])
        self.assertEqual(response_json['F000001_1']['assignedAnalyst']['email'], 'test_user@test.com')
        self.assertEqual(response_json['F000001_1']['assignedAnalyst']['fullName'], 'Test User')

    def test_update_success_story_types(self):
        url = reverse(update_family_success_story_types, args=[FAMILY_GUID])
        _check_login(self, url)

        # clear all success story types
        response = self.client.post(url, content_type='application/json', data=json.dumps({'successStoryTypes': []}))
        self.assertEqual(response.status_code, 200)
        response_json = response.json()
        self.assertListEqual(response_json['F000001_1']['successStoryTypes'], [])

        # add multiple success story types
        response = self.client.post(url, content_type='application/json', data=json.dumps({'successStoryTypes': [
            {
                "color": "#019143",
                "name": "Novel Discovery",
            },
            {
                "color": "#833E7D",
                "name": "Collaboration",
            },
            {
                "color": "#FFAB57",
                "name": "Altered Clinical Outcome",
            },
            {
                "color": "#E76013",
                "name": "Technical Win",
            },
            {
                "color": "#6583EC",
                "name": "Data Sharing",
            },
            {
                "color": "#5D5D5F",
                "name": "Other",
            },
        ]}))
        self.assertEqual(response.status_code, 200)
        response_json = response.json()
        self.assertEqual(len(response_json['F000001_1']['successStoryTypes']), 6)
        self.assertEqual(response_json['F000001_1']['successStoryTypes'][0], {
                u"color": u"#019143",
                u"name": u"Novel Discovery",
            })
<commit_msg>add test for update success story type<commit_after># -*- coding: utf-8 -*-
import json

from django.core.files.uploadedfile import SimpleUploadedFile
from django.test import TestCase
from django.urls.base import reverse

from seqr.views.apis.family_api import update_family_pedigree_image, update_family_assigned_analyst, update_family_fields_handler
from seqr.views.utils.test_utils import _check_login

FAMILY_GUID = 'F000001_1'

class ProjectAPITest(TestCase):
    fixtures = ['users', '1kg_project']

    #  TODO test other family api methods

    def test_update_family_pedigree_image(self):
        url = reverse(update_family_pedigree_image, args=[FAMILY_GUID])
        _check_login(self, url)

        f = SimpleUploadedFile("new_ped_image_123.png", b"file_content")

        # send invalid request
        response = self.client.post(url, {'f1': f, 'f2': f})
        self.assertEqual(response.status_code, 400)
        self.assertEqual(response.reason_phrase, 'Received 2 files')

        # send valid add/update request
        response = self.client.post(url, {'f': f})
        self.assertEqual(response.status_code, 200)
        response_json = response.json()

        self.assertListEqual(response_json.keys(), ['F000001_1'])
        self.assertRegex(response_json['F000001_1']['pedigreeImage'], '/media/pedigree_images/new_ped_image_.+\.png')

        # send valid delete request
        response = self.client.post(url)
        self.assertEqual(response.status_code, 200)
        response_json = response.json()

        self.assertListEqual(response_json.keys(), ['F000001_1'])
        self.assertIsNone(response_json['F000001_1']['pedigreeImage'])

    def test_update_family_assigned_analyst(self):
        url = reverse(update_family_assigned_analyst, args=[FAMILY_GUID])
        _check_login(self, url)

        # send invalid request
        response = self.client.post(url, content_type='application/json', data=json.dumps({}))
        self.assertEqual(response.status_code, 400)
        self.assertEqual(response.reason_phrase, '\'assigned analyst\' is not specified')

        # send invalid username (without permission)
        response = self.client.post(url, content_type='application/json', data=json.dumps({'assigned_analyst_username': 'invalid_username'}))
        self.assertEqual(response.status_code, 400)
        self.assertEqual(response.reason_phrase, 'specified user does not exist')

        # send valid request
        response = self.client.post(url, content_type='application/json', data=json.dumps({'assigned_analyst_username': 'test_user'}))
        self.assertEqual(response.status_code, 200)
        response_json = response.json()

        self.assertListEqual(response_json.keys(), ['F000001_1'])
        self.assertEqual(response_json['F000001_1']['assignedAnalyst']['email'], 'test_user@test.com')
        self.assertEqual(response_json['F000001_1']['assignedAnalyst']['fullName'], 'Test User')

    def test_update_success_story_types(self):
        url = reverse(update_family_fields_handler, args=[FAMILY_GUID])
        _check_login(self, url)

        # send valid request
        response = self.client.post(url, content_type='application/json', data=json.dumps({'successStoryTypes': ['O', 'D']}))
        self.assertEqual(response.status_code, 200)
        response_json = response.json()
        self.assertListEqual(response_json['F000001_1']['successStoryTypes'], ['O', 'D'])
<|endoftext|>


<commit_before>from .common import BasePublishService, BasePublishResource, ITEM_KILL
from eve.utils import config
from superdesk.metadata.item import CONTENT_STATE, GUID_FIELD
from superdesk import get_resource_service
from superdesk.publish import SUBSCRIBER_TYPES
from superdesk.utc import utcnow
import logging
from copy import copy
from superdesk.emails import send_article_killed_email
from superdesk.errors import SuperdeskApiError
from apps.archive.common import is_item_in_package, ITEM_OPERATION

logger = logging.getLogger(__name__)

class KillPublishResource(BasePublishResource):
    def __init__(self, endpoint_name, app, service):
        super().__init__(endpoint_name, app, service, 'kill')

class KillPublishService(BasePublishService):
    publish_type = 'kill'
    published_state = 'killed'

    def __init__(self, datasource=None, backend=None):
        super().__init__(datasource=datasource, backend=backend)

    def on_update(self, updates, original):
        # check if we are trying to kill and item that is contained in normal non takes package
        if is_item_in_package(original):
            raise SuperdeskApiError.badRequestError(message='This item is in a package' +
                                                            ' it needs to be removed before the item can be killed')
        super().on_update(updates, original)
        updates[ITEM_OPERATION] = ITEM_KILL
        self.takes_package_service.process_killed_takes_package(original)

    def update(self, id, updates, original):
        """
        Kill will broadcast kill email notice to all subscriber in the system and then kill the item.
        If item is a take then all the takes are killed as well.
        """
        self._broadcast_kill_email(original)
        super().update(id, updates, original)
        self._publish_kill_for_takes(updates, original)

    def _broadcast_kill_email(self, original):
        """
        Sends the broadcast email to all subscribers (including in-active subscribers)
        :param original: Document to kill
        """
        # Get all subscribers
        subscribers = list(get_resource_service('subscribers').get(req=None, lookup=None))
        recipients = [s.get('email') for s in subscribers if s.get('email')]
        # send kill email.
        send_article_killed_email(original, recipients, utcnow())

    def _publish_kill_for_takes(self, updates, original):
        """
        Kill all the takes in a takes package.
        :param updates: Updates of the original document
        :param original: Document to kill
        """
        package = self.takes_package_service.get_take_package(original)
        last_updated = updates.get(config.LAST_UPDATED, utcnow())
        if package:
            for ref in[ref for group in package.get('groups', []) if group['id'] == 'main'
                       for ref in group.get('refs')]:
                if ref[GUID_FIELD] != original[config.ID_FIELD]:
                    original_data = super().find_one(req=None, _id=ref[GUID_FIELD])
                    updates_data = copy(updates)
                    queued = self.publish(doc=original_data,
                                          updates=updates_data,
                                          target_media_type=SUBSCRIBER_TYPES.WIRE)
                    # we need to update the archive item and not worry about queued as we could have
                    # a takes only going to digital client.
                    self._set_updates(original_data, updates_data, last_updated)
                    self._update_archive(original=original_data, updates=updates_data,
                                         should_insert_into_versions=True)
                    self.update_published_collection(published_item_id=original_data['_id'])

                    if not queued:
                        logger.exception("Could not publish the kill for take {} with headline {}".
                                         format(original_data.get(config.ID_FIELD), original_data.get('headline')))

    def get_subscribers(self, doc, target_media_type):
        """
        Get the subscribers for this document based on the target_media_type for kill.
        Kill is sent to all subscribers that have received the item previously (published or corrected)
        :param doc: Document to kill
        :param target_media_type: dictate if the doc being queued is a Takes Package or an Individual Article.
                Valid values are - Wire, Digital. If Digital then the doc being queued is a Takes Package and if Wire
                then the doc being queued is an Individual Article.
        :return: (list, list) List of filtered subscribers,
                List of subscribers that have not received item previously (empty list in this case).
        """

        subscribers, subscribers_yet_to_receive = [], []
        query = {'$and': [{'item_id': doc[config.ID_FIELD]},
                          {'publishing_action': {'$in': [CONTENT_STATE.PUBLISHED, CONTENT_STATE.CORRECTED]}}]}
        subscribers = self._get_subscribers_for_previously_sent_items(query)

        return subscribers, subscribers_yet_to_receive
<commit_msg>[SD-3513] Kill operation aborts as publish validators throw exception for pubstatus.<commit_after>from .common import BasePublishService, BasePublishResource, ITEM_KILL
from eve.utils import config
from superdesk.metadata.item import CONTENT_STATE, GUID_FIELD, PUB_STATUS
from superdesk import get_resource_service
from superdesk.publish import SUBSCRIBER_TYPES
from superdesk.utc import utcnow
import logging
from copy import copy
from superdesk.emails import send_article_killed_email
from superdesk.errors import SuperdeskApiError
from apps.archive.common import is_item_in_package, ITEM_OPERATION

logger = logging.getLogger(__name__)

class KillPublishResource(BasePublishResource):
    def __init__(self, endpoint_name, app, service):
        super().__init__(endpoint_name, app, service, 'kill')

class KillPublishService(BasePublishService):
    publish_type = 'kill'
    published_state = 'killed'

    def __init__(self, datasource=None, backend=None):
        super().__init__(datasource=datasource, backend=backend)

    def on_update(self, updates, original):
        # check if we are trying to kill and item that is contained in normal non takes package
        if is_item_in_package(original):
            raise SuperdeskApiError.badRequestError(message='This item is in a package' +
                                                            ' it needs to be removed before the item can be killed')
        updates['pubstatus'] = PUB_STATUS.CANCELED
        super().on_update(updates, original)
        updates[ITEM_OPERATION] = ITEM_KILL
        self.takes_package_service.process_killed_takes_package(original)

    def update(self, id, updates, original):
        """
        Kill will broadcast kill email notice to all subscriber in the system and then kill the item.
        If item is a take then all the takes are killed as well.
        """
        self._broadcast_kill_email(original)
        super().update(id, updates, original)
        self._publish_kill_for_takes(updates, original)

    def _broadcast_kill_email(self, original):
        """
        Sends the broadcast email to all subscribers (including in-active subscribers)
        :param original: Document to kill
        """
        # Get all subscribers
        subscribers = list(get_resource_service('subscribers').get(req=None, lookup=None))
        recipients = [s.get('email') for s in subscribers if s.get('email')]
        # send kill email.
        send_article_killed_email(original, recipients, utcnow())

    def _publish_kill_for_takes(self, updates, original):
        """
        Kill all the takes in a takes package.
        :param updates: Updates of the original document
        :param original: Document to kill
        """
        package = self.takes_package_service.get_take_package(original)
        last_updated = updates.get(config.LAST_UPDATED, utcnow())
        if package:
            for ref in[ref for group in package.get('groups', []) if group['id'] == 'main'
                       for ref in group.get('refs')]:
                if ref[GUID_FIELD] != original[config.ID_FIELD]:
                    original_data = super().find_one(req=None, _id=ref[GUID_FIELD])
                    updates_data = copy(updates)
                    queued = self.publish(doc=original_data,
                                          updates=updates_data,
                                          target_media_type=SUBSCRIBER_TYPES.WIRE)
                    # we need to update the archive item and not worry about queued as we could have
                    # a takes only going to digital client.
                    self._set_updates(original_data, updates_data, last_updated)
                    self._update_archive(original=original_data, updates=updates_data,
                                         should_insert_into_versions=True)
                    self.update_published_collection(published_item_id=original_data['_id'])

                    if not queued:
                        logger.exception("Could not publish the kill for take {} with headline {}".
                                         format(original_data.get(config.ID_FIELD), original_data.get('headline')))

    def get_subscribers(self, doc, target_media_type):
        """
        Get the subscribers for this document based on the target_media_type for kill.
        Kill is sent to all subscribers that have received the item previously (published or corrected)
        :param doc: Document to kill
        :param target_media_type: dictate if the doc being queued is a Takes Package or an Individual Article.
                Valid values are - Wire, Digital. If Digital then the doc being queued is a Takes Package and if Wire
                then the doc being queued is an Individual Article.
        :return: (list, list) List of filtered subscribers,
                List of subscribers that have not received item previously (empty list in this case).
        """

        subscribers, subscribers_yet_to_receive = [], []
        query = {'$and': [{'item_id': doc[config.ID_FIELD]},
                          {'publishing_action': {'$in': [CONTENT_STATE.PUBLISHED, CONTENT_STATE.CORRECTED]}}]}
        subscribers = self._get_subscribers_for_previously_sent_items(query)

        return subscribers, subscribers_yet_to_receive
<|endoftext|>


<commit_before>import dateparser
import logging
import datetime

from django.template.loader import render_to_string
from django.core.mail import send_mail
from django.conf import settings
from django.contrib.auth import get_user_model

from notifhain.event.models import Room, Slot, Artist, DancefloorEvent

logger = logging.getLogger('scrapy')

class DjangoPipeline(object):

    def item_to_model(self, item):
        model_class = getattr(item, 'django_model')
        if not model_class:
            raise TypeError("Item is not a `DjangoItem` or is misconfigured")

        return item.instance

    def get_or_create(self, model):
        model_class = type(model)
        created = False

        # Normally, we would use `get_or_create`. However, `get_or_create` would
        # match all properties of an object (i.e. create a new object
        # anytime it changed) rather than update an existing object.
        #
        # Instead, we do the two steps separately
        try:
            # We have no unique identifier at the moment; use the url for now.
            obj = model_class.objects.get(url=model.url)
        except model_class.DoesNotExist:
            created = True
            obj = model  # DjangoItem created a model for us.

        return (obj, created)

    def update_model(self, django_model, item, commit=True):
        pk = django_model.pk

        # item_dict = model_to_dict(item)
        item_dict = item.__dict__
        for (key, value) in item_dict.items():
            setattr(django_model, key, value)

        setattr(django_model, 'pk', pk)

        if commit:
            django_model.save()

        return django_model

    class Meta:
        abstract = True

class ProgramPipeline(DjangoPipeline):
    def process_item(self, item, spider):
        try:
            item_model = self.item_to_model(item)
            item_model.month = dateparser.parse(item["name"], "%Y-%m")
        except TypeError:
            return item

        model, created = self.get_or_create(item_model)
        self.update_model(model, item_model)
        logger.info("Program %s %s" % (item_model.name, created and 'created' or 'updated'))

        return item

class EventPipeline(DjangoPipeline):
    def process_item(self, item, spider):
        try:
            item_model = self.item_to_model(item)
            item_model.event_date = dateparser.parse(item["date_string"])
        except TypeError:
            return item

        model, created = self.get_or_create(item_model)
        # reset all state values
        # TODO should be handled differentyl, maybe a separte state model would be good
        item_model.is_completed = model.is_completed
        item_model.has_timetable = model.has_timetable
        item_model.is_notification_send = model.is_notification_send
        item_model.is_posted_to_rr = model.is_posted_to_rr
        item_model.timetable_updated = model.timetable_updated
        self.update_model(model, item_model)
        logger.info("Event %s %s" % (item_model.name, created and 'created' or 'updated'))

        return item

class EventDetailsPipeline(DjangoPipeline):
    def process_item(self, event_details_item, spider):
        try:
            item_model = self.item_to_model(event_details_item)
            item_model.event_date = dateparser.parse(
                "%s %s" % (event_details_item["date_string"],
                event_details_item["time_string"]),
                settings={'RETURN_AS_TIMEZONE_AWARE': True}
            )
        except TypeError:
            return event_details_item

        event_details, created = self.get_or_create(item_model)
        self.update_model(event_details, item_model)
        logger.info("EventDetail %s %s" % (item_model.name, created and 'created' or 'updated'))

        # delete all existing objects
        Slot.objects.filter(event_details=event_details).delete()
        for room_item in event_details_item["rooms"]:
            room, created = Room.objects.get_or_create(name=room_item["name"])
            for slot_item in room_item["slots"]:
                self.process_slot(slot_item, room, event_details, spider)
        event = event_details.event
        event.is_completed = event_details_item["is_completed"]
        event.has_timetable = event_details_item["has_timetable"]
        if not event.is_completed and datetime.datetime.now().date() > event.event_date:
            event.is_completed = True
        if event.is_completed and not event.timetable_updated:
            event.timetable_updated = datetime.datetime.now()
        event.save()

        return event_details_item

    def process_slot(self, slot_item, room, event_details, spider):
        try:
            item_model = self.item_to_model(slot_item)
        except TypeError:
            return slot_item
        item_model.room = room
        item_model.event_details = event_details
        # add artists to slot
        item_model.save()
        for artist in slot_item["artists"]:
            artist, created = Artist.objects.get_or_create(name=artist)
            item_model.artists.add(artist)
        return slot_item

    def close_spider(self, spider):
        today = datetime.date.today()
        next_monday = today + datetime.timedelta(days=-today.weekday(), weeks=1)
        logger.info("close_spider: " + next_monday)
        events = DancefloorEvent.objects.filter(has_timetable=True, is_notification_send=False, event_details__event_date__lte=next_monday)
        for event in events:
            logger.info("send_timetable_email: " + event)
            msg_plain = render_to_string('event-timetable-online.html', {'event': event})
            send_emails_to = get_user_model().objects.all().values_list('email', flat=True)
            send_mail(
                'Timetable online: %s %s' % (event.name, event.event_date),
                msg_plain,
                settings.DEFAULT_FROM_EMAIL,
                send_emails_to,
                fail_silently=False,
            )
            event.is_notification_send = True
            event.save()
<commit_msg>add proper logging<commit_after>import dateparser
import logging
import datetime

from django.template.loader import render_to_string
from django.core.mail import send_mail
from django.conf import settings
from django.contrib.auth import get_user_model

from notifhain.event.models import Room, Slot, Artist, DancefloorEvent

logger = logging.getLogger('scrapy')

class DjangoPipeline(object):

    def item_to_model(self, item):
        model_class = getattr(item, 'django_model')
        if not model_class:
            raise TypeError("Item is not a `DjangoItem` or is misconfigured")

        return item.instance

    def get_or_create(self, model):
        model_class = type(model)
        created = False

        # Normally, we would use `get_or_create`. However, `get_or_create` would
        # match all properties of an object (i.e. create a new object
        # anytime it changed) rather than update an existing object.
        #
        # Instead, we do the two steps separately
        try:
            # We have no unique identifier at the moment; use the url for now.
            obj = model_class.objects.get(url=model.url)
        except model_class.DoesNotExist:
            created = True
            obj = model  # DjangoItem created a model for us.

        return (obj, created)

    def update_model(self, django_model, item, commit=True):
        pk = django_model.pk

        # item_dict = model_to_dict(item)
        item_dict = item.__dict__
        for (key, value) in item_dict.items():
            setattr(django_model, key, value)

        setattr(django_model, 'pk', pk)

        if commit:
            django_model.save()

        return django_model

    class Meta:
        abstract = True

class ProgramPipeline(DjangoPipeline):
    def process_item(self, item, spider):
        try:
            item_model = self.item_to_model(item)
            item_model.month = dateparser.parse(item["name"], "%Y-%m")
        except TypeError:
            return item

        model, created = self.get_or_create(item_model)
        self.update_model(model, item_model)
        logger.info("Program %s %s" % (item_model.name, created and 'created' or 'updated'))

        return item

class EventPipeline(DjangoPipeline):
    def process_item(self, item, spider):
        try:
            item_model = self.item_to_model(item)
            item_model.event_date = dateparser.parse(item["date_string"])
        except TypeError:
            return item

        model, created = self.get_or_create(item_model)
        # reset all state values
        # TODO should be handled differentyl, maybe a separte state model would be good
        item_model.is_completed = model.is_completed
        item_model.has_timetable = model.has_timetable
        item_model.is_notification_send = model.is_notification_send
        item_model.is_posted_to_rr = model.is_posted_to_rr
        item_model.timetable_updated = model.timetable_updated
        self.update_model(model, item_model)
        logger.info("Event %s %s" % (item_model.name, created and 'created' or 'updated'))

        return item

class EventDetailsPipeline(DjangoPipeline):
    def process_item(self, event_details_item, spider):
        try:
            item_model = self.item_to_model(event_details_item)
            item_model.event_date = dateparser.parse(
                "%s %s" % (event_details_item["date_string"],
                event_details_item["time_string"]),
                settings={'RETURN_AS_TIMEZONE_AWARE': True}
            )
        except TypeError:
            return event_details_item

        event_details, created = self.get_or_create(item_model)
        self.update_model(event_details, item_model)
        logger.info("EventDetail %s %s" % (item_model.name, created and 'created' or 'updated'))

        # delete all existing objects
        Slot.objects.filter(event_details=event_details).delete()
        for room_item in event_details_item["rooms"]:
            room, created = Room.objects.get_or_create(name=room_item["name"])
            for slot_item in room_item["slots"]:
                self.process_slot(slot_item, room, event_details, spider)
        event = event_details.event
        event.is_completed = event_details_item["is_completed"]
        event.has_timetable = event_details_item["has_timetable"]
        if not event.is_completed and datetime.datetime.now().date() > event.event_date:
            event.is_completed = True
        if event.is_completed and not event.timetable_updated:
            event.timetable_updated = datetime.datetime.now()
        event.save()

        return event_details_item

    def process_slot(self, slot_item, room, event_details, spider):
        try:
            item_model = self.item_to_model(slot_item)
        except TypeError:
            return slot_item
        item_model.room = room
        item_model.event_details = event_details
        # add artists to slot
        item_model.save()
        for artist in slot_item["artists"]:
            artist, created = Artist.objects.get_or_create(name=artist)
            item_model.artists.add(artist)
        return slot_item

    def close_spider(self, spider):
        today = datetime.date.today()
        next_monday = today + datetime.timedelta(days=-today.weekday(), weeks=1)
        events = DancefloorEvent.objects.filter(has_timetable=True, is_notification_send=False, event_details__event_date__lte=next_monday)
        for event in events:
            logger.info("send_timetable_email: " + event.event_details.name)
            msg_plain = render_to_string('event-timetable-online.html', {'event': event})
            send_emails_to = get_user_model().objects.all().values_list('email', flat=True)
            send_mail(
                'Timetable online: %s %s' % (event.name, event.event_date),
                msg_plain,
                settings.DEFAULT_FROM_EMAIL,
                send_emails_to,
                fail_silently=False,
            )
            event.is_notification_send = True
            event.save()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: ascii -*-

import sys, os, re

RAW_IDENT = r'[a-zA-Z_$][a-zA-Z0-9_$]*'
DOT_IDENT = RAW_IDENT + r'(\s*\.\s*' + RAW_IDENT + ')*'
KEYWORDS = ('abstract|assert|boolean|break|byte|case|catch|char|class|const|'
    'continue|default|do|double|else|enum|extends|final|finally|float|for|'
    'goto|if|implements|import|instanceof|int|interface|long|native|new|'
    'package|private|protected|public|return|short|static|strictfp|super|'
    'switch|synchronized|this|throw|throws|transient|try|void|volatile|'
    'while')
REGEXES = {
    # Identifier. Not matching the "e" in float literals.
    # FIXME: Unicode support.
    'identifier': re.compile(r'(?!%s)(?<![a-zA-Z0-9_$])%s' %
                             (KEYWORDS, DOT_IDENT)),
    # Interesting statement.
    'import': re.compile(r'(?<![a-zA-Z0-9_$])import\s+(static\s+)?'
                         r'(?P<name>%s(\s*\.\s*\*));' % DOT_IDENT),
    # Character or string. (The code is supposed to be syntactically valid.)
    'charstring': re.compile(r'''(?s)'([^']|\.)+'|"([^"]|\.)*"'''),
    # Comments.
    'linecomment': re.compile(r'(?m)//.*?$'),
    'blockcomment': re.compile(r'(?s)/\*.*?\*/'),
    # Line separator.
    'newline': re.compile(r'\n')
    }

def regex_race(data, regexes):
    """
    Regex race algorithm, as seen in the Instant frontend
    """
    pos, hits = 0, dict((k, None) for k in regexes)
    while pos != len(data):
        # Update regex matches.
        for k, v in tuple(hits.items()):
            if v is None or v.start() < pos:
                m = REGEXES[k].search(data, pos)
                if m:
                    hits[k] = m
                else:
                    del hits[k]
        # Abort if no more matches.
        if not hits: break
        # Find first match.
        first = min(hits.values(), key=lambda m: m.start())
        # Emit preceding data and the match.
        if first.start() != pos: yield data[pos:first.start()]
        yield first
        # Prepare for next round.
        pos = first.end()
    # Emit remainder.
    if pos != len(data):
        yield data[pos:]

def main():
    """
    Main function
    """
    pass

if __name__ == '__main__': main()
<commit_msg>[Makefile] [Linter] Add specific tokenizer<commit_after>#!/usr/bin/env python3
# -*- coding: ascii -*-

import sys, os, re

RAW_IDENT = r'[a-zA-Z_$][a-zA-Z0-9_$]*'
DOT_IDENT = RAW_IDENT + r'(\s*\.\s*' + RAW_IDENT + ')*'
KEYWORDS = ('abstract|assert|boolean|break|byte|case|catch|char|class|const|'
    'continue|default|do|double|else|enum|extends|final|finally|float|for|'
    'goto|if|implements|import|instanceof|int|interface|long|native|new|'
    'package|private|protected|public|return|short|static|strictfp|super|'
    'switch|synchronized|this|throw|throws|transient|try|void|volatile|'
    'while|'
    # While not technically keywords, those are equivalent for our purposes.
    'true|false|null')
REGEXES = {
    # Identifier. Not matching the "e" in float literals.
    # FIXME: Unicode support.
    'identifier': re.compile(r'(?<![a-zA-Z0-9_$])(?!%s)(?P<name>%s)' %
                             (KEYWORDS, DOT_IDENT)),
    # Interesting statement.
    'import': re.compile(r'(?<![a-zA-Z0-9_$])import\s+(static\s+)?'
                         r'(?P<name>%s(\s*\.\s*\*)?);' % DOT_IDENT),
    # Character or string. (The code is supposed to be syntactically valid.)
    'charstring': re.compile(r'''(?s)'([^']|\.)+'|"([^"]|\.)*"'''),
    # Comments.
    'linecomment': re.compile(r'(?m)//.*?$'),
    'blockcomment': re.compile(r'(?s)/\*.*?\*/'),
    }

def match_tokens(data, regexes):
    """
    Regex race algorithm, as seen in the Instant frontend
    """
    pos, hits = 0, dict((k, None) for k in regexes)
    while pos != len(data):
        # Update regex matches.
        for k, v in tuple(hits.items()):
            if v is None or v.start() < pos:
                m = REGEXES[k].search(data, pos)
                if m:
                    hits[k] = m
                else:
                    del hits[k]
        # Abort if no more matches.
        if not hits: break
        # Find first match.
        firstname = min(hits, key=lambda k: hits[k].start())
        first = hits[firstname]
        # Emit preceding data and the match.
        if first.start() != pos: yield data[pos:first.start()]
        yield (firstname, first)
        # Prepare for next round.
        pos = first.end()
    # Emit remainder.
    if pos != len(data):
        yield data[pos:]

def tokenize(data):
    """
    Partition a Java source file into a list of interesting and
    non-interesting bits along with indexes of the formers
    """
    def normalize_tok_name(tok):
        return re.sub(r'\s+', '', tok.group('name'))
    lineno, ret, imports, idents = 1, [], [], []
    for ent in match_tokens(data, REGEXES):
        if isinstance(ent, str):
            ret.append(ent)
            # FIXME: Assuming no CR-only newlines...
            lineno += ent.count('\n')
            continue
        name, tok = ent
        if name == 'identifier':
            # Indirect addressing to ease sorting imports below.
            ret.append(('ident', len(idents), lineno))
            idents.append((tok.group(), normalize_tok_name(tok)))
        elif name == 'import':
            ret.append(('import', len(imports), lineno))
            imports.append((tok.group(), normalize_tok_name(tok)))
        else:
            ret.append(tok.group())
        lineno += ent.count('\n')
    return {None: ret, 'import': imports, 'ident': idents}

def main():
    """
    Main function
    """
    pass

if __name__ == '__main__': main()
<|endoftext|>


<commit_before><commit_msg>BUG: fixed issue #317<commit_after><|endoftext|>