<commit_before>import urllib2
from urllib import urlencode
import simplejson as json
import sys, re
import dateutil.parser

_API_URL = "https://www.comicvine.com/api/"

_cached_resources = {}

api_key = ""

class InvalidResourceError(Exception):
    pass

class InvalidAPIKeyError(Exception):
    pass

class ObjectNotFoundError(Exception):
    pass

class ErrorInURLFormatError(Exception):
    pass

class JSONError(Exception):
    pass

class FilterError(Exception):
    pass

class SubscriberOnlyError(Exception):
    pass

class UnknownStatusError(Exception):
    pass

class IllegalArquementException(Exception):
    pass

_EXCEPTIONS = {
        100: InvalidAPIKeyError,
        101: ObjectNotFoundError,
        102: ErrorInURLFormatError,
        103: JSONError,
        104: FilterError,
        105: SubscriberOnlyError,
    }

class _Resource(object):
    class _Response:
        def __init__(
                self,
                error,
                limit,
                offset,
                number_of_page_results,
                number_of_total_results,
                status_code,
                version,
                results
            ):
            self.error = error
            self.limit = limit
            self.offset = offset
            self.number_of_page_results = number_of_page_results
            self.number_of_total_results = number_of_total_results
            self.status_code = status_code
            self.results = results

    def __init__(self, *args, **kwargs):
        raise NotImplemented()

    def _request_object(self, baseurl, **params):
        raise NotImplemented()

    @classmethod
    def _ensure_resource_url(type):
        if not '_resource_url' in type.__dict__:
            resource_type = Types.snakify_type_name(type)
            type._resource_url = _API_URL + resource_type + "/"

    @classmethod
    def _request(type, baseurl, **params):
        if 'api_key' not in params:
            if len(api_key) == 0:
                raise InvalidAPIKeyError(
                        "Invalid API Key"
                    )
            params['api_key'] = api_key
        if 'field_list' in params and params['field_list'] != None:
            if not isinstance(params['field_list'], basestring):
                field_list = ""
                try:
                    for field_name in params['field_list']:
                        field_list += str(field_name)+","
                except TypeError, e:
                    raise IllegalArquementException(
                            "'field_list' must be iterable"
                        )
                params['field_list'] = field_list
            if re.search(
                    "([a-z_]*,)*id(,[a-z_]*)*", params['field_list']
                ) == None:
                params['field_list'] = "id," + params['field_list']
        params['format'] = 'json'
        params = urlencode(params)
        response = type._Response(**json.loads(urllib2.urlopen(
                baseurl+"?"+params
            ).read()))
        if response.status_code != 1:
            raise _EXCEPTIONS.get(response.status_code,UnknownStatusError)(
                    response.error
                )
        if 'aliases' in response.results:
            response.results['aliases'] = response.results[
                    'aliases'
                ].split('\n')
        return response

class _SingularResource(_Resource):
    def __new__(type, id, **kwargs):
        resource_type = kwargs.get(
                'resource_type',
                type
            )
        try:
            type_id = Types()[resource_type]['id']
        except KeyError:
            return InvalidResourceError(resource_type)
        type._ensure_resource_url()
        key = "{0:d}-{1:d}".format(type_id, id)
        obj = _cached_resources.get(key)
        if obj == None:
            obj = object.__new__(type, id, **kwargs)
            _cached_resources[key] = obj
        return obj

    def __init__(self, id, **kwargs):
        if '_ready' not in self.__dict__:
            self._ready = True
            try:
                type_id = Types()[type(self)]['id']
            except KeyError:
                raise InvalidResourceError(
                        "Resource type '{0!s}' does not exist.".format(
                                resource_type
                            )
                    )
            if 'api_detail_url' in kwargs:
                self._detail_url = kwargs['api_detail_url']
            else:
                self._detail_url = type(self)._resource_url + \
                        "{0:d}-{1:d}/".format(type_id, id)
            self._fields = {'id': id}
            if 'field_list' in kwargs:
                self._fields.update(self._request_object(
                        kwargs['field_list']
                    ).results)
                del kwargs['field_list']
            self._fields.update(kwargs)
        elif 'field_list' in kwargs:
            self._fields.update(self._request_object(
                    kwargs['field_list']
                ).results)

    def _request_object(self, field_list = None):
        return type(self)._request(
                self._detail_url,
                field_list=field_list
            )

    def __getattribute__(self, name):
        def _object_attribute(name):
            return object.__getattribute__(self, name)
        try:
            if name not in ['__dict__', '_request_object'] and \
                    name not in self.__dict__:
                if name in _object_attribute('_fields'):
                    return _object_attribute('_fields')[name]
                else:
                    self._fields.update(
                            _object_attribute('_request_object')(
                                    [name]
                                ).results
                        )
                    return _object_attribute('_fields')[name]
        except KeyError:
            pass
        return _object_attribute(name)

    def __str__(self):
        if 'name' in self._fields:
            return str(self.name)+" ["+str(self.id)+"]"
        else:
            return "["+str(self.id)+"]"

    def __repr__(self):
        return "<"+type(self).__name__+": '"+str(self)+"'>"

class _ListResource(_Resource):
    def _request_object(self, **params):
        type(self)._ensure_resource_url()
        return type(self)._request(type(self)._resource_url, **params)

    def __init__(self, init_list = None, **kwargs):
        if init_list != None:
            if len(kwargs) > 0:
                raise TypeError(
                        "If 'init_list' is given it is the only "+
                        "allowed argument"
                    )
            self._results = init_list
            self._total = len(init_list)
            self._limit = len(init_list)
        else:
            response = self._request_object(**kwargs)
            self._results = response.offset*[None] + response.results
            self._total = response.number_of_total_results
            self._limit = response.limit
            if 'limit' in kwargs:
                del kwargs['limit']
            if 'offset' in kwargs:
                del kwargs['offset']
            self._args = kwargs

    def __len__(self):
        return self._total

    def __getitem__(self, index):
        if type(index) == slice:
            start = index.start or 0
            stop = index.stop or self._total
            step = index.step or 1
        else:
            start = index
            stop = index+1
            step = 1
        if start < 0 or start >= self._total or \
                stop > self._total:
            raise IndexError('Index out of range')
        if len(self._results) < stop or \
                None in self._results[start:stop:step]:
            for i in range(start, stop, 100):
                response = self._request_object(
                        limit=self._limit,
                        offset=i,
                        **self._args
                    )
                self._results[i:self._limit+i] = response.results
        if type(self._results[index]) == list:
            for i in range(start, stop, step):
                if type(self._results[i]) == dict:
                    self._parse_result(i)
        elif type(self._results[index]) == dict:
            self._parse_result(index)
        return self._results[index]

    def __iter__(self):
        for index in xrange(self._total):
            yield self[index]

    def __str__(self):
        if len(self) == 0:
            return "[]"
        string = "["
        for element in self:
            string += str(element)+", "
        string = string[:-2]+"]"
        return string

    def __repr__(self):
        if len(self) == 0:
            return type(self).__name__+"[]"
        string = type(self).__name__+"["
        for element in self:
            string += repr(element)+","
        string = string[:-1]+"]"
        return string

    def _parse_result(self, index):
        if type(self) != Types:
            type_dict = Types()
            if type(self) == Search:
                self._results[index] = type_dict[
                        self._results[index]['resource_type']
                    ]['singular_resource_class'](**self._results[index])
            else:
                self._results[index] = type_dict[type(self)][
                        'singular_resource_class'
                    ](**self._results[index])

class _SortableListResource(_ListResource):
    def __init__(self, init_list = None, sort = None, **kwargs):
        if sort != None:
            if isinstance(sort, basestring):
                if ':' not in sort:
                    sort += ":asc"
            else:
                try:
                    sort = str(sort[0])+":"+str(sort[1])
                except KeyError:
                    if 'field' not in sort:
                        raise IllegalArquementException(
                                "Argument 'sort' must contain item 'field'"
                            )
                    if 'direction' in sort:
                        sort = sort['field']+":"+str(sort['direction'])
                    else:
                        sort = sort['field']+":asc"
            kwargs['sort'] = sort
        super(_SortableListResource, self).__init__(init_list, **kwargs)

class Character(_SingularResource):
    def __getattribute__(self, name):
        value = super(Character, self).__getattribute__(name)
        if name == 'birth':
            if isinstance(value, basestring):
                try:
                    self._fields[name] = dateutil.parser.parse(value)
                except ValueError:
                    pass
            return self._fields[name]
        elif name == 'character_enemies':
            if isinstance(value, list):
                self._fields[name] = Characters(value)
            return self._fields[name]
        elif name == 'character_friends':
            if isinstance(value, list):
                self._fields[name] = Characters(value)
            return self._fields[name]
        elif name == 'creators':
            if isinstance(value, list):
                self._fields[name] = People(value)
            return self._fields[name]
        elif name == 'date_added':
            if isinstance(value, basestring):
                try:
                    self._fields[name] = dateutil.parser.parse(value)
                except ValueError:
                    pass
            return self._fields[name]
        elif name == 'date_last_updated':
            if isinstance(value, basestring):
                try:
                    self._fields[name] = dateutil.parser.parse(value)
                except ValueError:
                    pass
            return self._fields[name]
        elif name == 'first_appeared_in_issue':
            if isinstance(value, dict):
                self._fields[name] = Issue(**value)
            return self._fields[name]
        elif name == 'gender':
            if isinstance(value, int):
                self._fields[name] = \
                        u'\u2642' if value == 1 else \
                        u'\u2640' if value == 2 else u'\u26a7'
            return self._fields[name]
        elif name == 'issue_credits':
            if isinstance(value, list):
                self._fields[name] = Issues(value)
            return self._fields[name]
        elif name == 'issues_died_in':
            if isinstance(value, list):
                    self._fields[name] = Issues(value)
            return self._fields[name]
        elif name == 'movies':
            if isinstance(value, list):
                    self._fields[name] = Movies(value)
            return self._fields[name]
        elif name == 'origin':
            if isinstance(value, dict):
                self._fields[name] = Origin(**value)
            elif isinstance(value, list):
                self._fields[name] = Origins(value)
            return self._fields[name]
        elif name == 'powers':
            if isinstance(value, list):
                    self._fields[name] = Powers(value)
            return self._fields[name]
        elif name == 'publisher':
            if isinstance(value, dict):
                self._fields[name] = Publisher(**value)
            elif isinstance(value, list):
                self._fields[name] = Publishers(value)
            return self._fields[name]
        elif name == 'story_arc_credits':
            if isinstance(value, list):
                self._fields[name] = StoryArcs(value)
            return self._fields[name]
        elif name == 'team_enemies':
            if isinstance(value, list):
                self._fields[name] = Teams(value)
            return self._fields[name]
        elif name == 'team_friends':
            if isinstance(value, list):
                self._fields[name] = Teams(value)
            return self._fields[name]
        elif name == 'teams':
            if isinstance(value, list):
                self._fields[name] = Teams(value)
            return self._fields[name]
        elif name == 'volume_credits':
            if isinstance(value, list):
                self._fields[name] = Volumes(value)
            return self._fields[name]
        else:
            return value

class Characters(_SortableListResource):
    pass

class Chat(_SingularResource):
    pass

class Chats(_SortableListResource):
    pass

class Concept(_SingularResource):
    pass

class Concepts(_SortableListResource):
    pass

class Issue(_SingularResource):
    pass

class Issues(_SortableListResource):
    pass

class Location(_SingularResource):
    pass

class Locations(_SortableListResource):
    pass

class Movie(_SingularResource):
    pass

class Movies(_SortableListResource):
    pass

class Object(_SingularResource):
    pass

class Objects(_SortableListResource):
    pass

class Origin(_SingularResource):
    pass

class Origins(_SortableListResource):
    pass

class Person(_SingularResource):
    pass

class People(_SortableListResource):
    pass

class Power(_SingularResource):
    pass

class Powers(_SortableListResource):
    pass

class Promo(_SingularResource):
    pass

class Promos(_SortableListResource):
    pass

class Publisher(_SingularResource):
    pass

class Publishers(_SortableListResource):
    pass

class Search(_ListResource):
    def __init__(self, resources, query, **kwargs):
        super(Search, self).__init__(
                resources=resources,
                query=query,
                **kwargs
            )

class StoryArc(_SingularResource):
    pass

class StoryArcs(_SortableListResource):
    pass

class Team(_SingularResource):
    pass

class Teams(_SortableListResource):
    pass

class Types(_ListResource):
    def __new__(type):
        if not '_instance' in type.__dict__:
            type._instance = object.__new__(type)
        return type._instance

    def __init__(self):
        if not '_ready' in dir(self):
            super(Types, self).__init__()
            self._mapping = {}
            for type in self:
                type['singular_resource_class'] = getattr(
                        sys.modules[__name__],
                        Types._camilify_type_name(
                                type['detail_resource_name']
                            )
                    )
                self._mapping[type['detail_resource_name']] = type
                self._mapping[type['list_resource_name']] = type
            self._ready = True

    def __getitem__(self, key):
        if isinstance(key, (int, long, slice)):
            return super(Types, self).__getitem__(key)
        if isinstance(key, type):
            return self._mapping[self.snakify_type_name(key)]
        return self._mapping[key]

    @staticmethod
    def snakify_type_name(type):
        return re.sub(r'([A-Z]+)',r"_\1", type.__name__)[1:].lower()

    @staticmethod
    def _camilify_type_name(string):
        string = string[0].upper() + string[1:].lower()
        camel_string = ""
        next_upper = False
        for c in string:
            if c == "_":
                next_upper = True
                continue
            if next_upper:
                camel_string += c.upper()
                next_upper = False
            else:
                camel_string += c
        return camel_string

class Video(_SingularResource):
    pass

class Videos(_SortableListResource):
    pass

class VideoType(_SingularResource):
    pass

class VideoTypes(_SortableListResource):
    pass

class Volume(_SingularResource):
    pass

class Volumes(_SortableListResource):
    pass
<commit_msg>Catch fact that version sometimes is omited<commit_after>import urllib2
from urllib import urlencode
import simplejson as json
import sys, re
import dateutil.parser

_API_URL = "https://www.comicvine.com/api/"

_cached_resources = {}

api_key = ""

class InvalidResourceError(Exception):
    pass

class InvalidAPIKeyError(Exception):
    pass

class ObjectNotFoundError(Exception):
    pass

class ErrorInURLFormatError(Exception):
    pass

class JSONError(Exception):
    pass

class FilterError(Exception):
    pass

class SubscriberOnlyError(Exception):
    pass

class UnknownStatusError(Exception):
    pass

class IllegalArquementException(Exception):
    pass

_EXCEPTIONS = {
        100: InvalidAPIKeyError,
        101: ObjectNotFoundError,
        102: ErrorInURLFormatError,
        103: JSONError,
        104: FilterError,
        105: SubscriberOnlyError,
    }

class _Resource(object):
    class _Response:
        def __init__(
                self,
                error,
                limit,
                offset,
                number_of_page_results,
                number_of_total_results,
                status_code,
                results,
                version = None
            ):
            self.error = error
            self.limit = limit
            self.offset = offset
            self.number_of_page_results = number_of_page_results
            self.number_of_total_results = number_of_total_results
            self.status_code = status_code
            self.results = results

    def __init__(self, *args, **kwargs):
        raise NotImplemented()

    def _request_object(self, baseurl, **params):
        raise NotImplemented()

    @classmethod
    def _ensure_resource_url(type):
        if not '_resource_url' in type.__dict__:
            resource_type = Types.snakify_type_name(type)
            type._resource_url = _API_URL + resource_type + "/"

    @classmethod
    def _request(type, baseurl, **params):
        if 'api_key' not in params:
            if len(api_key) == 0:
                raise InvalidAPIKeyError(
                        "Invalid API Key"
                    )
            params['api_key'] = api_key
        if 'field_list' in params and params['field_list'] != None:
            if not isinstance(params['field_list'], basestring):
                field_list = ""
                try:
                    for field_name in params['field_list']:
                        field_list += str(field_name)+","
                except TypeError, e:
                    raise IllegalArquementException(
                            "'field_list' must be iterable"
                        )
                params['field_list'] = field_list
            if re.search(
                    "([a-z_]*,)*id(,[a-z_]*)*", params['field_list']
                ) == None:
                params['field_list'] = "id," + params['field_list']
        params['format'] = 'json'
        params = urlencode(params)
        response = type._Response(**json.loads(urllib2.urlopen(
                baseurl+"?"+params
            ).read()))
        if response.status_code != 1:
            raise _EXCEPTIONS.get(response.status_code,UnknownStatusError)(
                    response.error
                )
        if 'aliases' in response.results:
            response.results['aliases'] = response.results[
                    'aliases'
                ].split('\n')
        return response

class _SingularResource(_Resource):
    def __new__(type, id, **kwargs):
        resource_type = kwargs.get(
                'resource_type',
                type
            )
        try:
            type_id = Types()[resource_type]['id']
        except KeyError:
            return InvalidResourceError(resource_type)
        type._ensure_resource_url()
        key = "{0:d}-{1:d}".format(type_id, id)
        obj = _cached_resources.get(key)
        if obj == None:
            obj = object.__new__(type, id, **kwargs)
            _cached_resources[key] = obj
        return obj

    def __init__(self, id, **kwargs):
        if '_ready' not in self.__dict__:
            self._ready = True
            try:
                type_id = Types()[type(self)]['id']
            except KeyError:
                raise InvalidResourceError(
                        "Resource type '{0!s}' does not exist.".format(
                                resource_type
                            )
                    )
            if 'api_detail_url' in kwargs:
                self._detail_url = kwargs['api_detail_url']
            else:
                self._detail_url = type(self)._resource_url + \
                        "{0:d}-{1:d}/".format(type_id, id)
            self._fields = {'id': id}
            if 'field_list' in kwargs:
                self._fields.update(self._request_object(
                        kwargs['field_list']
                    ).results)
                del kwargs['field_list']
            self._fields.update(kwargs)
        elif 'field_list' in kwargs:
            self._fields.update(self._request_object(
                    kwargs['field_list']
                ).results)

    def _request_object(self, field_list = None):
        return type(self)._request(
                self._detail_url,
                field_list=field_list
            )

    def __getattribute__(self, name):
        def _object_attribute(name):
            return object.__getattribute__(self, name)
        try:
            if name not in ['__dict__', '_request_object'] and \
                    name not in self.__dict__:
                if name in _object_attribute('_fields'):
                    return _object_attribute('_fields')[name]
                else:
                    self._fields.update(
                            _object_attribute('_request_object')(
                                    [name]
                                ).results
                        )
                    return _object_attribute('_fields')[name]
        except KeyError:
            pass
        return _object_attribute(name)

    def __str__(self):
        if 'name' in self._fields:
            return str(self.name)+" ["+str(self.id)+"]"
        else:
            return "["+str(self.id)+"]"

    def __repr__(self):
        return "<"+type(self).__name__+": '"+str(self)+"'>"

class _ListResource(_Resource):
    def _request_object(self, **params):
        type(self)._ensure_resource_url()
        return type(self)._request(type(self)._resource_url, **params)

    def __init__(self, init_list = None, **kwargs):
        if init_list != None:
            if len(kwargs) > 0:
                raise TypeError(
                        "If 'init_list' is given it is the only "+
                        "allowed argument"
                    )
            self._results = init_list
            self._total = len(init_list)
            self._limit = len(init_list)
        else:
            response = self._request_object(**kwargs)
            self._results = response.offset*[None] + response.results
            self._total = response.number_of_total_results
            self._limit = response.limit
            if 'limit' in kwargs:
                del kwargs['limit']
            if 'offset' in kwargs:
                del kwargs['offset']
            self._args = kwargs

    def __len__(self):
        return self._total

    def __getitem__(self, index):
        if type(index) == slice:
            start = index.start or 0
            stop = index.stop or self._total
            step = index.step or 1
        else:
            start = index
            stop = index+1
            step = 1
        if start < 0 or start >= self._total or \
                stop > self._total:
            raise IndexError('Index out of range')
        if len(self._results) < stop or \
                None in self._results[start:stop:step]:
            for i in range(start, stop, 100):
                response = self._request_object(
                        limit=self._limit,
                        offset=i,
                        **self._args
                    )
                self._results[i:self._limit+i] = response.results
        if type(self._results[index]) == list:
            for i in range(start, stop, step):
                if type(self._results[i]) == dict:
                    self._parse_result(i)
        elif type(self._results[index]) == dict:
            self._parse_result(index)
        return self._results[index]

    def __iter__(self):
        for index in xrange(self._total):
            yield self[index]

    def __str__(self):
        if len(self) == 0:
            return "[]"
        string = "["
        for element in self:
            string += str(element)+", "
        string = string[:-2]+"]"
        return string

    def __repr__(self):
        if len(self) == 0:
            return type(self).__name__+"[]"
        string = type(self).__name__+"["
        for element in self:
            string += repr(element)+","
        string = string[:-1]+"]"
        return string

    def _parse_result(self, index):
        if type(self) != Types:
            type_dict = Types()
            if type(self) == Search:
                self._results[index] = type_dict[
                        self._results[index]['resource_type']
                    ]['singular_resource_class'](**self._results[index])
            else:
                self._results[index] = type_dict[type(self)][
                        'singular_resource_class'
                    ](**self._results[index])

class _SortableListResource(_ListResource):
    def __init__(self, init_list = None, sort = None, **kwargs):
        if sort != None:
            if isinstance(sort, basestring):
                if ':' not in sort:
                    sort += ":asc"
            else:
                try:
                    sort = str(sort[0])+":"+str(sort[1])
                except KeyError:
                    if 'field' not in sort:
                        raise IllegalArquementException(
                                "Argument 'sort' must contain item 'field'"
                            )
                    if 'direction' in sort:
                        sort = sort['field']+":"+str(sort['direction'])
                    else:
                        sort = sort['field']+":asc"
            kwargs['sort'] = sort
        super(_SortableListResource, self).__init__(init_list, **kwargs)

class Character(_SingularResource):
    def __getattribute__(self, name):
        value = super(Character, self).__getattribute__(name)
        if name == 'birth':
            if isinstance(value, basestring):
                try:
                    self._fields[name] = dateutil.parser.parse(value)
                except ValueError:
                    pass
            return self._fields[name]
        elif name == 'character_enemies':
            if isinstance(value, list):
                self._fields[name] = Characters(value)
            return self._fields[name]
        elif name == 'character_friends':
            if isinstance(value, list):
                self._fields[name] = Characters(value)
            return self._fields[name]
        elif name == 'creators':
            if isinstance(value, list):
                self._fields[name] = People(value)
            return self._fields[name]
        elif name == 'date_added':
            if isinstance(value, basestring):
                try:
                    self._fields[name] = dateutil.parser.parse(value)
                except ValueError:
                    pass
            return self._fields[name]
        elif name == 'date_last_updated':
            if isinstance(value, basestring):
                try:
                    self._fields[name] = dateutil.parser.parse(value)
                except ValueError:
                    pass
            return self._fields[name]
        elif name == 'first_appeared_in_issue':
            if isinstance(value, dict):
                self._fields[name] = Issue(**value)
            return self._fields[name]
        elif name == 'gender':
            if isinstance(value, int):
                self._fields[name] = \
                        u'\u2642' if value == 1 else \
                        u'\u2640' if value == 2 else u'\u26a7'
            return self._fields[name]
        elif name == 'issue_credits':
            if isinstance(value, list):
                self._fields[name] = Issues(value)
            return self._fields[name]
        elif name == 'issues_died_in':
            if isinstance(value, list):
                    self._fields[name] = Issues(value)
            return self._fields[name]
        elif name == 'movies':
            if isinstance(value, list):
                    self._fields[name] = Movies(value)
            return self._fields[name]
        elif name == 'origin':
            if isinstance(value, dict):
                self._fields[name] = Origin(**value)
            elif isinstance(value, list):
                self._fields[name] = Origins(value)
            return self._fields[name]
        elif name == 'powers':
            if isinstance(value, list):
                    self._fields[name] = Powers(value)
            return self._fields[name]
        elif name == 'publisher':
            if isinstance(value, dict):
                self._fields[name] = Publisher(**value)
            elif isinstance(value, list):
                self._fields[name] = Publishers(value)
            return self._fields[name]
        elif name == 'story_arc_credits':
            if isinstance(value, list):
                self._fields[name] = StoryArcs(value)
            return self._fields[name]
        elif name == 'team_enemies':
            if isinstance(value, list):
                self._fields[name] = Teams(value)
            return self._fields[name]
        elif name == 'team_friends':
            if isinstance(value, list):
                self._fields[name] = Teams(value)
            return self._fields[name]
        elif name == 'teams':
            if isinstance(value, list):
                self._fields[name] = Teams(value)
            return self._fields[name]
        elif name == 'volume_credits':
            if isinstance(value, list):
                self._fields[name] = Volumes(value)
            return self._fields[name]
        else:
            return value

class Characters(_SortableListResource):
    pass

class Chat(_SingularResource):
    pass

class Chats(_SortableListResource):
    pass

class Concept(_SingularResource):
    pass

class Concepts(_SortableListResource):
    pass

class Issue(_SingularResource):
    pass

class Issues(_SortableListResource):
    pass

class Location(_SingularResource):
    pass

class Locations(_SortableListResource):
    pass

class Movie(_SingularResource):
    pass

class Movies(_SortableListResource):
    pass

class Object(_SingularResource):
    pass

class Objects(_SortableListResource):
    pass

class Origin(_SingularResource):
    pass

class Origins(_SortableListResource):
    pass

class Person(_SingularResource):
    pass

class People(_SortableListResource):
    pass

class Power(_SingularResource):
    pass

class Powers(_SortableListResource):
    pass

class Promo(_SingularResource):
    pass

class Promos(_SortableListResource):
    pass

class Publisher(_SingularResource):
    pass

class Publishers(_SortableListResource):
    pass

class Search(_ListResource):
    def __init__(self, resources, query, **kwargs):
        super(Search, self).__init__(
                resources=resources,
                query=query,
                **kwargs
            )

class StoryArc(_SingularResource):
    pass

class StoryArcs(_SortableListResource):
    pass

class Team(_SingularResource):
    pass

class Teams(_SortableListResource):
    pass

class Types(_ListResource):
    def __new__(type):
        if not '_instance' in type.__dict__:
            type._instance = object.__new__(type)
        return type._instance

    def __init__(self):
        if not '_ready' in dir(self):
            super(Types, self).__init__()
            self._mapping = {}
            for type in self:
                type['singular_resource_class'] = getattr(
                        sys.modules[__name__],
                        Types._camilify_type_name(
                                type['detail_resource_name']
                            )
                    )
                self._mapping[type['detail_resource_name']] = type
                self._mapping[type['list_resource_name']] = type
            self._ready = True

    def __getitem__(self, key):
        if isinstance(key, (int, long, slice)):
            return super(Types, self).__getitem__(key)
        if isinstance(key, type):
            return self._mapping[self.snakify_type_name(key)]
        return self._mapping[key]

    @staticmethod
    def snakify_type_name(type):
        return re.sub(r'([A-Z]+)',r"_\1", type.__name__)[1:].lower()

    @staticmethod
    def _camilify_type_name(string):
        string = string[0].upper() + string[1:].lower()
        camel_string = ""
        next_upper = False
        for c in string:
            if c == "_":
                next_upper = True
                continue
            if next_upper:
                camel_string += c.upper()
                next_upper = False
            else:
                camel_string += c
        return camel_string

class Video(_SingularResource):
    pass

class Videos(_SortableListResource):
    pass

class VideoType(_SingularResource):
    pass

class VideoTypes(_SortableListResource):
    pass

class Volume(_SingularResource):
    pass

class Volumes(_SortableListResource):
    pass
<|endoftext|>


<commit_before>''' This module provides functions for embedding Bokeh plots in various
different ways.

'''

import uuid

from .protocol import serialize_json
from .templates import (
    AUTOLOAD, AUTOLOAD_SERVER, AUTOLOAD_STATIC, FILE,
    NOTEBOOK_DIV, PLOT_DIV, PLOT_JS, PLOT_SCRIPT, RESOURCES
)

def components(plot_object, resources):
    ''' Return HTML components to embed a Bokeh plot.

    The data for the plot is stored directly in the returned HTML.

    .. note:: The returned components assume that BokehJS resources
              are **already loaded**.

    Args:
        plot_object (PlotObject) : Bokeh object to render
            typically a Plot or PlotContext

    Returns:
        (script, div)

    '''
    ref = plot_object.get_ref()
    elementid = str(uuid.uuid4())

    js = PLOT_JS.render(
        elementid = elementid,
        modelid = ref["id"],
        modeltype = ref["type"],
        all_models = serialize_json(plot_object.dump()),
    )
    script = PLOT_SCRIPT.render(
        plot_js = resources.js_wrapper(js),
    )
    div = PLOT_DIV.render(elementid=elementid)

    return script, div

def notebook_div(plot_object):
    ''' Return HTML for a div that will display a Bokeh plot in an
    IPython Notebook

    The data for the plot is stored directly in the returned HTML.

    Args:
        plot_object (PlotObject) : Bokeh object to render
            typically a Plot or PlotContext

    Returns:
        div : UTF-8 encoded HTML text

    .. note:: Assumes ``bokeh.load_notebook()`` or the equivalent has
              already been executed.

    '''
    ref = plot_object.get_ref()
    elementid = str(uuid.uuid4())

    js = PLOT_JS.render(
        elementid = elementid,
        modelid = ref["id"],
        modeltype = ref["type"],
        all_models = serialize_json(plot_object.dump()),
    )
    script = PLOT_SCRIPT.render(
        plot_js = js,
    )
    div = PLOT_DIV.render(elementid=elementid)
    html = NOTEBOOK_DIV.render(
        plot_script = script,
        plot_div = div,
    )
    return html.encode("utf-8")

def file_html(plot_object, resources, title, template=FILE):
    ''' Return an HTML document that embeds a Bokeh plot.

    The data for the plot is stored directly in the returned HTML.

    Args:
        plot_object (PlotObject) : Bokeh object to render
            typically a Plot or PlotContext
        resources (Resources) : a resource configuration for BokehJS assets
        title (str) : a title for the HTML document ``<title>`` tags
        template (Template, optional) : HTML document template (default: FILE)
            A Jinja2 Template, see bokeh.templates.FILE for the required
            template parameters

    Returns:
        html : standalone HTML document with embedded plot

    '''
    plot_resources = RESOURCES.render(
        js_raw = resources.js_raw,
        css_raw = resources.css_raw,
        js_files = resources.js_files,
        css_files = resources.css_files,
    )
    script, div = components(plot_object, resources)
    html = template.render(
        title = title,
        plot_resources = plot_resources,
        plot_script = script,
        plot_div = div,
    )
    return html.encode("utf-8")

def autoload_static(plot_object, resources, script_path):
    ''' Return JavaScript code and a script tag that can be used to embed
    Bokeh Plots.

    The data for the plot is stored directly in the returned JavaScript code.

    Args:
        plot_object (PlotObject) :
        resources (Resources) :
        script_path (str) :

    Returns:
        (js, tag) :
            JavaScript code to be saved at ``script_path`` and a ``<script>``
            tag to load it

    Raises:
        ValueError

    '''
    if resources.mode != 'inline':
        raise ValueError("autoload_static() requires non-inline resources")

    if resources.dev:
        raise ValueError("autoload_static() only works with non-dev resources")

    elementid = str(uuid.uuid4())

    js = AUTOLOAD.render(
        js_url = resources.js_files[0],
        css_files = resources.css_files,
        elementid = elementid,
    )

    tag = AUTOLOAD_STATIC.render(
        src_path = script_path,
        elementid = elementid,
        modelid = plot_object._id,
        modeltype = plot_object.__view_model__,
    )

    return js, tag

def autoload_server(plot_object, session):
    ''' Return a script tag that can be used to embed Bokeh Plots from
    a Bokeh Server.

    The data for the plot is stored on the Bokeh Server.

    Args:
        plot_object (PlotObject) :
        session (session) :

    Returns:
        tag :
            a ``<script>`` tag that will execute an autoload script
            loaded from the Bokeh Server

    '''
    elementid = str(uuid.uuid4())

    tag = AUTOLOAD_SERVER.render(
        src_path = session.root_url + "/bokeh/autoload.js/%s" % elementid,
        elementid = elementid,
        modelid = plot_object._id,
        modeltype = plot_object.__view_model__,
        root_url = session.root_url,
        docid =  session.docid,
        docapikey = session.api_key,
        conn_string = sess.conn_string,
    )

    return tag

<commit_msg>doc fix<commit_after>''' This module provides functions for embedding Bokeh plots in various
different ways.

'''

import uuid

from .protocol import serialize_json
from .templates import (
    AUTOLOAD, AUTOLOAD_SERVER, AUTOLOAD_STATIC, FILE,
    NOTEBOOK_DIV, PLOT_DIV, PLOT_JS, PLOT_SCRIPT, RESOURCES
)

def components(plot_object, resources):
    ''' Return HTML components to embed a Bokeh plot.

    The data for the plot is stored directly in the returned HTML.

    .. note:: The returned components assume that BokehJS resources
              are **already loaded**.

    Args:
        plot_object (PlotObject) : Bokeh object to render
            typically a Plot or PlotContext
        resources (Resources, optional) : BokehJS resources config

    Returns:
        (script, div)

    '''
    ref = plot_object.get_ref()
    elementid = str(uuid.uuid4())

    js = PLOT_JS.render(
        elementid = elementid,
        modelid = ref["id"],
        modeltype = ref["type"],
        all_models = serialize_json(plot_object.dump()),
    )
    script = PLOT_SCRIPT.render(
        plot_js = resources.js_wrapper(js),
    )
    div = PLOT_DIV.render(elementid=elementid)

    return script, div

def notebook_div(plot_object):
    ''' Return HTML for a div that will display a Bokeh plot in an
    IPython Notebook

    The data for the plot is stored directly in the returned HTML.

    Args:
        plot_object (PlotObject) : Bokeh object to render
            typically a Plot or PlotContext

    Returns:
        div : UTF-8 encoded HTML text

    .. note:: Assumes ``bokeh.load_notebook()`` or the equivalent has
              already been executed.

    '''
    ref = plot_object.get_ref()
    elementid = str(uuid.uuid4())

    js = PLOT_JS.render(
        elementid = elementid,
        modelid = ref["id"],
        modeltype = ref["type"],
        all_models = serialize_json(plot_object.dump()),
    )
    script = PLOT_SCRIPT.render(
        plot_js = js,
    )
    div = PLOT_DIV.render(elementid=elementid)
    html = NOTEBOOK_DIV.render(
        plot_script = script,
        plot_div = div,
    )
    return html.encode("utf-8")

def file_html(plot_object, resources, title, template=FILE):
    ''' Return an HTML document that embeds a Bokeh plot.

    The data for the plot is stored directly in the returned HTML.

    Args:
        plot_object (PlotObject) : Bokeh object to render
            typically a Plot or PlotContext
        resources (Resources) : a resource configuration for BokehJS assets
        title (str) : a title for the HTML document ``<title>`` tags
        template (Template, optional) : HTML document template (default: FILE)
            A Jinja2 Template, see bokeh.templates.FILE for the required
            template parameters

    Returns:
        html : standalone HTML document with embedded plot

    '''
    plot_resources = RESOURCES.render(
        js_raw = resources.js_raw,
        css_raw = resources.css_raw,
        js_files = resources.js_files,
        css_files = resources.css_files,
    )
    script, div = components(plot_object, resources)
    html = template.render(
        title = title,
        plot_resources = plot_resources,
        plot_script = script,
        plot_div = div,
    )
    return html.encode("utf-8")

def autoload_static(plot_object, resources, script_path):
    ''' Return JavaScript code and a script tag that can be used to embed
    Bokeh Plots.

    The data for the plot is stored directly in the returned JavaScript code.

    Args:
        plot_object (PlotObject) :
        resources (Resources) :
        script_path (str) :

    Returns:
        (js, tag) :
            JavaScript code to be saved at ``script_path`` and a ``<script>``
            tag to load it

    Raises:
        ValueError

    '''
    if resources.mode != 'inline':
        raise ValueError("autoload_static() requires non-inline resources")

    if resources.dev:
        raise ValueError("autoload_static() only works with non-dev resources")

    elementid = str(uuid.uuid4())

    js = AUTOLOAD.render(
        js_url = resources.js_files[0],
        css_files = resources.css_files,
        elementid = elementid,
    )

    tag = AUTOLOAD_STATIC.render(
        src_path = script_path,
        elementid = elementid,
        modelid = plot_object._id,
        modeltype = plot_object.__view_model__,
    )

    return js, tag

def autoload_server(plot_object, session):
    ''' Return a script tag that can be used to embed Bokeh Plots from
    a Bokeh Server.

    The data for the plot is stored on the Bokeh Server.

    Args:
        plot_object (PlotObject) :
        session (session) :

    Returns:
        tag :
            a ``<script>`` tag that will execute an autoload script
            loaded from the Bokeh Server

    '''
    elementid = str(uuid.uuid4())

    tag = AUTOLOAD_SERVER.render(
        src_path = session.root_url + "/bokeh/autoload.js/%s" % elementid,
        elementid = elementid,
        modelid = plot_object._id,
        modeltype = plot_object.__view_model__,
        root_url = session.root_url,
        docid =  session.docid,
        docapikey = session.api_key,
        conn_string = sess.conn_string,
    )

    return tag

<|endoftext|>


<commit_before># Common settings for all environments
from os import path
from glob import glob

# Django settings for harvardcards project.

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'America/Chicago'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

### SET PATH INFORMATION ###############################################
# Example: "/home/ubuntu/harvardcards"
ROOT_DIR = reduce(lambda l,r: path.dirname(l), range(3), path.realpath(__file__))

# Example: "/home/ubuntu/harvardcards/harvardcards"
PROJECT_ROOT = path.join(ROOT_DIR, 'harvardcards')

# Example: "/home/ubuntu/harvardcards/harvardcards/apps"
APPS_ROOT = path.join(PROJECT_ROOT, 'apps')

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/media/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
MEDIA_URL = ''

INSTALLED_APPS = (
    # Other apps here
    # ...
    'django.contrib.staticfiles',
)

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = 'static'

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = [
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
	path.join(PROJECT_ROOT, 'static'),
]

STATICFILES_DIRS.extend([
	f for f in glob(path.join(APPS_ROOT, '*', 'static')) if path.isdir(f)
])

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = '#5g0vp545jp644!hha1)fb7v1hd!*t#b@fv&1(mrnt5)$q%w0g'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'harvardcards.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'harvardcards.wsgi.application'

TEMPLATE_DIRS = [
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
	path.join(PROJECT_ROOT, 'templates'),
]

TEMPLATE_DIRS.extend([
	f for f in glob(path.join(APPS_ROOT, '*', 'templates')) if path.isdir(f)
])

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    #'django_openid_auth',
    # Uncomment the next line to enable the admin:
    'django.contrib.admin',
    # Uncomment the next line to enable admin documentation:
    # 'django.contrib.admindocs',
	'harvardcards.apps.flash',
	#'harvardcards.apps.jasmine',
)

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

#AUTHENTICATION_BACKENDS = (
#	'harvardcards.libs.auth.GoogleBackend',
#    'django_openid_auth.auth.OpenIDBackend',
#	'django.contrib.auth.backends.ModelBackend',
#)
#OPENID_CREATE_USERS = True
#OPENID_UPDATE_DETAILS_FROM_SREG = True
#LOGIN_URL = '/login/'
#LOGIN_REDIRECT_URL = '/'
#LOGOUT_URL = '/logout/'
#OPENID_SSO_SERVER_URL = 'https://www.google.com/accounts/o8/id'
<commit_msg>enabling all openid settings<commit_after># Common settings for all environments
from os import path
from glob import glob

# Django settings for harvardcards project.

DEBUG = True
TEMPLATE_DEBUG = DEBUG

ADMINS = (
    # ('Your Name', 'your_email@example.com'),
)

MANAGERS = ADMINS

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'America/Chicago'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

### SET PATH INFORMATION ###############################################
# Example: "/home/ubuntu/harvardcards"
ROOT_DIR = reduce(lambda l,r: path.dirname(l), range(3), path.realpath(__file__))

# Example: "/home/ubuntu/harvardcards/harvardcards"
PROJECT_ROOT = path.join(ROOT_DIR, 'harvardcards')

# Example: "/home/ubuntu/harvardcards/harvardcards/apps"
APPS_ROOT = path.join(PROJECT_ROOT, 'apps')

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/home/media/media.lawrence.com/media/"
MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://media.lawrence.com/media/", "http://example.com/media/"
MEDIA_URL = ''

INSTALLED_APPS = (
    # Other apps here
    # ...
    'django.contrib.staticfiles',
)

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/home/media/media.lawrence.com/static/"
STATIC_ROOT = 'static'

# URL prefix for static files.
# Example: "http://media.lawrence.com/static/"
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = [
    # Put strings here, like "/home/html/static" or "C:/www/django/static".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
	path.join(PROJECT_ROOT, 'static'),
]

STATICFILES_DIRS.extend([
	f for f in glob(path.join(APPS_ROOT, '*', 'static')) if path.isdir(f)
])

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
#    'django.contrib.staticfiles.finders.DefaultStorageFinder',
)

# Make this unique, and don't share it with anybody.
SECRET_KEY = '#5g0vp545jp644!hha1)fb7v1hd!*t#b@fv&1(mrnt5)$q%w0g'

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
#     'django.template.loaders.eggs.Loader',
)

MIDDLEWARE_CLASSES = (
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'harvardcards.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'harvardcards.wsgi.application'

TEMPLATE_DIRS = [
    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
	path.join(PROJECT_ROOT, 'templates'),
]

TEMPLATE_DIRS.extend([
	f for f in glob(path.join(APPS_ROOT, '*', 'templates')) if path.isdir(f)
])

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django_openid_auth',
    # Uncomment the next line to enable the admin:
    'django.contrib.admin',
    # Uncomment the next line to enable admin documentation:
    # 'django.contrib.admindocs',
	'harvardcards.apps.flash',
	#'harvardcards.apps.jasmine',
)

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}

AUTHENTICATION_BACKENDS = (
	'harvardcards.libs.auth.GoogleBackend',
    'django_openid_auth.auth.OpenIDBackend',
	'django.contrib.auth.backends.ModelBackend',
)
OPENID_CREATE_USERS = True
OPENID_UPDATE_DETAILS_FROM_SREG = True
LOGIN_URL = '/login/'
LOGIN_REDIRECT_URL = '/'
LOGOUT_URL = '/logout/'
OPENID_SSO_SERVER_URL = 'https://www.google.com/accounts/o8/id'
<|endoftext|>


<commit_before>"""Get the status of an integration from the identity vault and auth0."""
import json
import logging
import uuid
from botocore.exceptions import ClientError
from cis_aws import connect
from cis_change_service import common
from cis_identity_vault.models import user
from cis_profile.profile import User
from cis_change_service.exceptions import IntegrationError
from cis_change_service.exceptions import VerificationError
from cis_profile.exceptions import PublisherVerificationFailure
from cis_profile.exceptions import SignatureVerificationFailure
from traceback import format_exc

logger = logging.getLogger(__name__)

class Vault(object):
    """Handles flushing profiles to Dynamo when running local or in stream bypass mode."""

    def __init__(self, sequence_number=None):
        self.connection_object = connect.AWS()
        self.identity_vault_client = None
        self.config = common.get_config()

        if sequence_number is not None:
            self.sequence_number = str(sequence_number)
        else:
            self.sequence_number = str(uuid.uuid4().int)

    def _connect(self):
        self.connection_object.session()
        self.identity_vault_client = self.connection_object.identity_vault_client()
        return self.identity_vault_client

    def _verify(self, profile_json):
        cis_profile = User(user_structure_json=profile_json)
        user_id = cis_profile.user_id.value

        try:
            if self.config("verify_publishers", namespace="cis") == "true":
                cis_profile.verify_all_publishers(User())

            if self.config("verify_signatures", namespace="cis") == "true":
                cis_profile.verify_all_signatures()
        except SignatureVerificationFailure as e:
            logger.error(
                "The profile vailed to pass signature verification for user: {}".format(user_id),
                extra={"user_id": user_id, "profile": cis_profile.as_dict(), "reason": e, "trace": format_exc()},
            )

            raise VerificationError({"code": "invalid_signature", "description": "{}".format(e)}, 403)
        except PublisherVerificationFailure as e:
            logger.error(
                "The profile vailed to pass publisher verification for user: {}".format(user_id),
                extra={"user_id": user_id, "profile": cis_profile.as_dict(), "reason": e, "trace": format_exc()},
            )
            raise VerificationError({"code": "invalid_publisher", "description": "{}".format(e)}, 403)
        except Exception as e:
            logger.error(
                "The profile produced an unknown error and is not trusted for user: {}".format(user_id),
                extra={"user_id": user_id, "profile": cis_profile.as_dict(), "reason": e, "trace": format_exc()},
            )
            raise VerificationError({"code": "unknown_error", "description": "{}".format(e)}, 500)
        return True

    def _update_attr_owned_by_cis(self, profile_json):
        """Updates the attributes owned by cisv2.  Takes profiles profile_json
        and returns a profile json with updated values and sigs."""

        # New up a a cis_profile object
        user = User(user_structure_json=profile_json)
        user.update_timestamp("last_modified")
        user.last_modified.value = user._get_current_utc_time()
        user.sign_attribute("last_modified", "cis")

    def put_profile(self, profile_json):
        """Write profile to the identity vault."""
        try:
            self._connect()

            if isinstance(profile_json, str):
                profile_json = json.loads(profile_json)

            # Run some code that updates attrs and metadata for attributes cis is trusted to assert
            self._update_attr_owned_by_cis(profile_json)
            verified = self._verify(profile_json)

            if verified:
                if self.config("dynamodb_transactions", namespace="cis") == "true":
                    vault = user.Profile(
                        self.identity_vault_client.get("table"),
                        self.identity_vault_client.get("client"),
                        transactions=True,
                    )
                else:
                    vault = user.Profile(
                        self.identity_vault_client.get("table"),
                        self.identity_vault_client.get("client"),
                        transactions=False,
                    )

                user_profile = dict(
                    id=profile_json["user_id"]["value"],
                    primary_email=profile_json["primary_email"]["value"],
                    uuid=profile_json["uuid"]["value"],
                    primary_username=profile_json["primary_username"]["value"],
                    sequence_number=self.sequence_number,
                    profile=json.dumps(profile_json),
                )

                res = vault.find_or_create(user_profile)
                logger.debug(
                    "The result of writing the profile to the identity vault was: {}".format(res),
                    extra={"user_id": profile_json["user_id"]["value"], "profile": profile_json, "result": res},
                )
                return res
        except ClientError as e:
            logger.error(
                "An error occured writing this profile to dynamodb",
                extra={"profile": profile_json, "error": e, "trace": format_exc()},
            )
            raise IntegrationError({"code": "integration_exception", "description": "{}".format(e)}, 500)

    def put_profiles(self, profile_list):
        """Write profile to the identity vault."""
        try:
            self._connect()

            if self.config("dynamodb_transactions", namespace="cis") == "true":
                logger.debug("Attempting to put batch of profiles using transactions.")
                vault = user.Profile(
                    self.identity_vault_client.get("table"), self.identity_vault_client.get("client"), transactions=True
                )
            else:
                logger.info("Attempting to put batch of profiles without transactions.")
                vault = user.Profile(
                    self.identity_vault_client.get("table"),
                    self.identity_vault_client.get("client"),
                    transactions=False,
                )

            user_profiles = []

            for profile_json in profile_list:
                if isinstance(profile_json, str):
                    profile_json = json.loads(profile_json)

                # Run some code that updates attrs and metadata for attributes cis is trusted to assert
                self._update_attr_owned_by_cis(profile_json)
                verified = self._verify(profile_json)
                if verified:
                    logger.debug("Profiles have been verified. Constructing dictionary for storage.")
                    user_profile = dict(
                        id=profile_json["user_id"]["value"],
                        primary_email=profile_json["primary_email"]["value"],
                        uuid=profile_json["uuid"]["value"],
                        primary_username=profile_json["primary_username"]["value"],
                        sequence_number=self.sequence_number,
                        profile=json.dumps(profile_json),
                    )
                    user_profiles.append(user_profile)

            logger.debug(
                "Attempting to send a list numbering: {} profiles as a transaction.".format(len(user_profiles))
            )
            result = vault.find_or_create_batch(user_profiles)
        except ClientError as e:
            logger.error(
                "An error occured writing these profiles to dynamodb",
                extra={"profiles": profile_list, "error": e, "trace": format_exc()},
            )
            raise IntegrationError({"code": "integration_exception", "description": "{}".format(e)}, 500)
        return {"creates": result[0], "updates": result[1]}

    def delete_profile(self, profile_json):
        try:
            user_profile = dict(
                id=profile_json["user_id"]["value"],
                primary_email=profile_json["primary_email"]["value"],
                uuid=profile_json["uuid"]["value"],
                primary_username=profile_json["primary_username"]["value"],
                sequence_number=self.sequence_number,
                profile=json.dumps(profile_json),
            )

            if self.config("dynamodb_transactions", namespace="cis") == "true":
                logger.debug("Attempting to put batch of profiles using transactions.")
                vault = user.Profile(
                    self.identity_vault_client.get("table"), self.identity_vault_client.get("client"), transactions=True
                )
            else:
                logger.info("Attempting to put batch of profiles without transactions.")
                vault = user.Profile(
                    self.identity_vault_client.get("table"),
                    self.identity_vault_client.get("client"),
                    transactions=False,
                )

            vault.delete(user_profile)
        except ClientError as e:
            logger.error(
                "An error occured removing this profile from dynamodb",
                extra={"profile": profile_json, "error": e, "trace": format_exc()},
            )
            raise IntegrationError({"code": "integration_exception", "description": "{}".format(e)}, 500)
        return {"status": 200, "message": "user profile deleted for user: {}".format(profile_json["user_id"]["value"])}

    def _get_id(self, profile_json):
        if isinstance(profile_json, str):
            profile_json = json.loads(profile_json)
        return profile_json.get("user_id").get("value").lower()

    def _get_primary_email(self, profile_json):
        if isinstance(profile_json, str):
            profile_json = json.loads(profile_json)
        return profile_json.get("primary_email").get("value").lower()

class Status(object):
    """Does the right thing to query if the event was integrated and return the results."""

    def __init__(self, sequence_number):
        self.connection_object = connect.AWS()
        self.identity_vault_client = None
        self.sequence_number = sequence_number

    def _connect(self):
        self.connection_object.session()
        self.identity_vault_client = self.connection_object.identity_vault_client()
        return self.identity_vault_client

    def query(self):
        """Query the identity vault using the named Global Secondary Index for the sequence number."""
        # Vault returns a dictionary of dictionaries for the statuses of each check.
        self._connect()

        client = self.identity_vault_client.get("client")

        result = client.query(
            TableName=self.identity_vault_client.get("arn").split("/")[1],
            IndexName="{}-sequence_number".format(self.identity_vault_client.get("arn").split("/")[1]),
            Select="ALL_ATTRIBUTES",
            KeyConditionExpression="sequence_number = :sequence_number",
            ExpressionAttributeValues={":sequence_number": {"S": self.sequence_number}},
        )

        return result

    @property
    def all(self):
        """Run all checks and return the results for the given sequence number as a dict."""
        return {"identity_vault": self.check_identity_vault()}

    def check_identity_vault(self):
        """Check the sequence number of the last record put to the identity vault."""
        if self.identity_vault_client is None:
            self._connect()

        query_result = self.query()
        if len(query_result.get("Items")) == 1:
            return True
        else:
            return False
<commit_msg>handle user update condition for publisher verification<commit_after>"""Get the status of an integration from the identity vault and auth0."""
import json
import logging
import uuid
from botocore.exceptions import ClientError
from cis_aws import connect
from cis_change_service import common
from cis_identity_vault.models import user
from cis_profile.profile import User
from cis_change_service.exceptions import IntegrationError
from cis_change_service.exceptions import VerificationError
from cis_profile.exceptions import PublisherVerificationFailure
from cis_profile.exceptions import SignatureVerificationFailure
from traceback import format_exc

logger = logging.getLogger(__name__)

class Vault(object):
    """Handles flushing profiles to Dynamo when running local or in stream bypass mode."""

    def __init__(self, sequence_number=None):
        self.connection_object = connect.AWS()
        self.identity_vault_client = None
        self.config = common.get_config()

        if sequence_number is not None:
            self.sequence_number = str(sequence_number)
        else:
            self.sequence_number = str(uuid.uuid4().int)

    def _connect(self):
        self.connection_object.session()
        self.identity_vault_client = self.connection_object.identity_vault_client()
        return self.identity_vault_client

    def _verify(self, profile_json, old_profile=None):
        cis_profile = User(user_structure_json=profile_json)
        user_id = cis_profile.user_id.value

        try:
            if self.config("verify_publishers", namespace="cis") == "true":
                cis_profile.verify_all_publishers(self._search(user_id))

            if self.config("verify_signatures", namespace="cis") == "true":
                cis_profile.verify_all_signatures()
        except SignatureVerificationFailure as e:
            logger.error(
                "The profile vailed to pass signature verification for user: {}".format(user_id),
                extra={"user_id": user_id, "profile": cis_profile.as_dict(), "reason": e, "trace": format_exc()},
            )

            raise VerificationError({"code": "invalid_signature", "description": "{}".format(e)}, 403)
        except PublisherVerificationFailure as e:
            logger.error(
                "The profile vailed to pass publisher verification for user: {}".format(user_id),
                extra={"user_id": user_id, "profile": cis_profile.as_dict(), "reason": e, "trace": format_exc()},
            )
            raise VerificationError({"code": "invalid_publisher", "description": "{}".format(e)}, 403)
        except Exception as e:
            logger.error(
                "The profile produced an unknown error and is not trusted for user: {}".format(user_id),
                extra={"user_id": user_id, "profile": cis_profile.as_dict(), "reason": e, "trace": format_exc()},
            )
            raise VerificationError({"code": "unknown_error", "description": "{}".format(e)}, 500)
        return True

    def _update_attr_owned_by_cis(self, profile_json):
        """Updates the attributes owned by cisv2.  Takes profiles profile_json
        and returns a profile json with updated values and sigs."""

        # New up a a cis_profile object
        user = User(user_structure_json=profile_json)
        user.update_timestamp("last_modified")
        user.last_modified.value = user._get_current_utc_time()
        user.sign_attribute("last_modified", "cis")

    def _search(self, user_id):
        self._connect()

        vault = user.Profile(
            self.identity_vault_client.get("table"),
            self.identity_vault_client.get("client"),
        )

        res = vault.find_by_id(user_id)

        if len(res['Items']) > 0:
            logger.info(
                'A record already exists in the identity vault for user: {}.'.format(
                    user_id
                ),
                extra={'user_id': user_id}
            )
            return User(user_structure_json=json.loads(res['Items'][0]['profile']))
        else:
            logger.info(
                'A record does not exist in the identity vault for user: {}.'.format(
                    user_id
                ),
                extra={'user_id': user_id}
            )
            return User(user_structure_json=None)

    def put_profile(self, profile_json):
        """Write profile to the identity vault."""
        try:
            self._connect()

            if isinstance(profile_json, str):
                profile_json = json.loads(profile_json)

            # Run some code that updates attrs and metadata for attributes cis is trusted to assert
            self._update_attr_owned_by_cis(profile_json)
            verified = self._verify(profile_json)

            if verified:
                if self.config("dynamodb_transactions", namespace="cis") == "true":
                    vault = user.Profile(
                        self.identity_vault_client.get("table"),
                        self.identity_vault_client.get("client"),
                        transactions=True,
                    )
                else:
                    vault = user.Profile(
                        self.identity_vault_client.get("table"),
                        self.identity_vault_client.get("client"),
                        transactions=False,
                    )

                user_profile = dict(
                    id=profile_json["user_id"]["value"],
                    primary_email=profile_json["primary_email"]["value"],
                    uuid=profile_json["uuid"]["value"],
                    primary_username=profile_json["primary_username"]["value"],
                    sequence_number=self.sequence_number,
                    profile=json.dumps(profile_json),
                )

                res = vault.find_or_create(user_profile)
                logger.debug(
                    "The result of writing the profile to the identity vault was: {}".format(res),
                    extra={"user_id": profile_json["user_id"]["value"], "profile": profile_json, "result": res},
                )
                return res
        except ClientError as e:
            logger.error(
                "An error occured writing this profile to dynamodb",
                extra={"profile": profile_json, "error": e, "trace": format_exc()},
            )
            raise IntegrationError({"code": "integration_exception", "description": "{}".format(e)}, 500)

    def put_profiles(self, profile_list):
        """Write profile to the identity vault."""
        try:
            self._connect()

            if self.config("dynamodb_transactions", namespace="cis") == "true":
                logger.debug("Attempting to put batch of profiles using transactions.")
                vault = user.Profile(
                    self.identity_vault_client.get("table"), self.identity_vault_client.get("client"), transactions=True
                )
            else:
                logger.info("Attempting to put batch of profiles without transactions.")
                vault = user.Profile(
                    self.identity_vault_client.get("table"),
                    self.identity_vault_client.get("client"),
                    transactions=False,
                )

            user_profiles = []

            for profile_json in profile_list:
                if isinstance(profile_json, str):
                    profile_json = json.loads(profile_json)

                # Run some code that updates attrs and metadata for attributes cis is trusted to assert
                self._update_attr_owned_by_cis(profile_json)
                verified = self._verify(profile_json)
                if verified:
                    logger.debug("Profiles have been verified. Constructing dictionary for storage.")
                    user_profile = dict(
                        id=profile_json["user_id"]["value"],
                        primary_email=profile_json["primary_email"]["value"],
                        uuid=profile_json["uuid"]["value"],
                        primary_username=profile_json["primary_username"]["value"],
                        sequence_number=self.sequence_number,
                        profile=json.dumps(profile_json),
                    )
                    user_profiles.append(user_profile)

            logger.debug(
                "Attempting to send a list numbering: {} profiles as a transaction.".format(len(user_profiles))
            )
            result = vault.find_or_create_batch(user_profiles)
        except ClientError as e:
            logger.error(
                "An error occured writing these profiles to dynamodb",
                extra={"profiles": profile_list, "error": e, "trace": format_exc()},
            )
            raise IntegrationError({"code": "integration_exception", "description": "{}".format(e)}, 500)
        return {"creates": result[0], "updates": result[1]}

    def delete_profile(self, profile_json):
        try:
            user_profile = dict(
                id=profile_json["user_id"]["value"],
                primary_email=profile_json["primary_email"]["value"],
                uuid=profile_json["uuid"]["value"],
                primary_username=profile_json["primary_username"]["value"],
                sequence_number=self.sequence_number,
                profile=json.dumps(profile_json),
            )

            if self.config("dynamodb_transactions", namespace="cis") == "true":
                logger.debug("Attempting to put batch of profiles using transactions.")
                vault = user.Profile(
                    self.identity_vault_client.get("table"), self.identity_vault_client.get("client"), transactions=True
                )
            else:
                logger.info("Attempting to put batch of profiles without transactions.")
                vault = user.Profile(
                    self.identity_vault_client.get("table"),
                    self.identity_vault_client.get("client"),
                    transactions=False,
                )

            vault.delete(user_profile)
        except ClientError as e:
            logger.error(
                "An error occured removing this profile from dynamodb",
                extra={"profile": profile_json, "error": e, "trace": format_exc()},
            )
            raise IntegrationError({"code": "integration_exception", "description": "{}".format(e)}, 500)
        return {"status": 200, "message": "user profile deleted for user: {}".format(profile_json["user_id"]["value"])}

    def _get_id(self, profile_json):
        if isinstance(profile_json, str):
            profile_json = json.loads(profile_json)
        return profile_json.get("user_id").get("value").lower()

    def _get_primary_email(self, profile_json):
        if isinstance(profile_json, str):
            profile_json = json.loads(profile_json)
        return profile_json.get("primary_email").get("value").lower()

class Status(object):
    """Does the right thing to query if the event was integrated and return the results."""

    def __init__(self, sequence_number):
        self.connection_object = connect.AWS()
        self.identity_vault_client = None
        self.sequence_number = sequence_number

    def _connect(self):
        self.connection_object.session()
        self.identity_vault_client = self.connection_object.identity_vault_client()
        return self.identity_vault_client

    def query(self):
        """Query the identity vault using the named Global Secondary Index for the sequence number."""
        # Vault returns a dictionary of dictionaries for the statuses of each check.
        self._connect()

        client = self.identity_vault_client.get("client")

        result = client.query(
            TableName=self.identity_vault_client.get("arn").split("/")[1],
            IndexName="{}-sequence_number".format(self.identity_vault_client.get("arn").split("/")[1]),
            Select="ALL_ATTRIBUTES",
            KeyConditionExpression="sequence_number = :sequence_number",
            ExpressionAttributeValues={":sequence_number": {"S": self.sequence_number}},
        )

        return result

    @property
    def all(self):
        """Run all checks and return the results for the given sequence number as a dict."""
        return {"identity_vault": self.check_identity_vault()}

    def check_identity_vault(self):
        """Check the sequence number of the last record put to the identity vault."""
        if self.identity_vault_client is None:
            self._connect()

        query_result = self.query()
        if len(query_result.get("Items")) == 1:
            return True
        else:
            return False
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
from builtins import range, next, zip, map
from future.utils import viewvalues, viewitems
import sys
if sys.version < '3':
    text_type = unicode
    binary_type = str
else:
    text_type = str
    binary_type = bytes
    unicode = str

import itertools
import time
import tempfile
import os
import operator
import random

import numpy

import dedupe.backport as backport

class ChildProcessError(Exception) :
    pass

def randomPairs(n_records, sample_size):
    """
    Return random combinations of indices for a square matrix of size
    n records
    """
    n = int(n_records * (n_records - 1) / 2)

    if sample_size >= n :
        random_pairs = numpy.arange(n, dtype='f')
    else:
        random_pairs = numpy.array(random.sample(range(n), sample_size), dtype='f')

    b = 1 - 2 * n_records

    i = numpy.floor((-b - numpy.sqrt(b ** 2 - 8 * random_pairs)) / 2).astype('uint')
    j = numpy.rint(random_pairs + i * (b + i + 2) / 2 + 1).astype('uint')

    return zip(i, j)

def randomPairsMatch(n_records_A, n_records_B, sample_size):
    """
    Return random combinations of indices for record list A and B
    """
    n = int(n_records_A * n_records_B)

    random_pairs = numpy.array(random.sample(range(n), sample_size), dtype='f')

    i = numpy.floor(random_pairs/n_records_B).astype('uint')
    j = (random_pairs - n_records_B * i).astype('uint')

    return zip(i, j)

class ScoreRecords(object) :
    def __init__(self, data_model, classifier, threshold) :
        self.data_model = data_model
        self.classifier = classifier
        self.threshold = threshold
        self.score_queue = None

    def __call__(self, records_queue, score_queue) :
        self.score_queue = score_queue
        while True :
            record_pairs = records_queue.get()
            if record_pairs is None :
                break

            try :
                filtered_pairs = self.fieldDistance(record_pairs)
                if filtered_pairs is not None :
                    score_queue.put(filtered_pairs)
            except Exception as e :
                score_queue.put(e)
                raise

        score_queue.put(None)

    def fieldDistance(self, record_pairs) :
        ids = []
        records = []

        for record_pair in record_pairs :
            ((id_1, record_1, smaller_ids_1),
             (id_2, record_2, smaller_ids_2)) = record_pair

            if smaller_ids_1.isdisjoint(smaller_ids_2) :

                ids.append((id_1, id_2))
                records.append((record_1, record_2))

        if records :
            ids = numpy.array(ids)

            distances = self.data_model.distances(records)
            scores = self.classifier.predict_proba(distances)[:,-1]

            mask = scores > self.threshold

            scored_pairs = numpy.empty(numpy.count_nonzero(mask),
                                       dtype=[('pairs', 'object', 2),
                                              ('score', 'f4', 1)])
            scored_pairs['pairs'] = ids[mask]
            scored_pairs['score'] = scores[mask]

            return scored_pairs

def mergeScores(score_queue, result_queue, stop_signals) :
    scored_pairs = numpy.empty(0, dtype= [('pairs', 'object', 2),
                                          ('score', 'f4', 1)])

    seen_signals = 0
    while seen_signals < stop_signals  :
        score_chunk = score_queue.get()
        if isinstance(score_chunk, Exception) :
            result_queue.put(score_chunk)
            return

        if score_chunk is not None :
            scored_pairs = numpy.concatenate((scored_pairs, score_chunk))
        else :
            seen_signals += 1

    if len(scored_pairs) :
        python_type = type(scored_pairs['pairs'][0][0])
        if python_type is binary_type or python_type is text_type :
            max_length = len(max(numpy.ravel(scored_pairs['pairs']), key=len))
            python_type = (unicode, max_length)

        write_dtype = [('pairs', python_type, 2),
                       ('score', 'f4', 1)]

        scored_pairs = scored_pairs.astype(write_dtype)

        scored_pairs_file, file_path = tempfile.mkstemp()

        os.close(scored_pairs_file)

        fp = numpy.memmap(file_path,
                          dtype=scored_pairs.dtype,
                          shape=scored_pairs.shape)
        fp[:] = scored_pairs[:]

        result_queue.put((file_path, scored_pairs.dtype))

    else :
        result_queue.put(scored_pairs)

def scoreDuplicates(records, data_model, classifier, num_cores=1, threshold=0) :
    if num_cores < 2 :
        from multiprocessing.dummy import Process, Pool, Queue
        SimpleQueue = Queue
    else :
        from .backport import Process, Pool, SimpleQueue

    record_pairs_queue = SimpleQueue()
    score_queue =  SimpleQueue()
    result_queue = SimpleQueue()

    n_map_processes = max(num_cores-1, 1)
    score_records = ScoreRecords(data_model, classifier, threshold)
    map_processes = [Process(target=score_records,
                             args=(record_pairs_queue,
                                   score_queue))
                     for _ in range(n_map_processes)]
    [process.start() for process in map_processes]

    reduce_process = Process(target=mergeScores,
                             args=(score_queue,
                                   result_queue,
                                   n_map_processes))
    reduce_process.start()

    fillQueue(record_pairs_queue, records, n_map_processes)

    result = result_queue.get()
    if isinstance(result, Exception) :
        raise ChildProcessError

    if result :
        scored_pairs_file, dtype = result
        scored_pairs = numpy.memmap(scored_pairs_file,
                                    dtype=dtype)
    else :
        scored_pairs = result

    reduce_process.join()
    [process.join() for process in map_processes]

    return scored_pairs

def fillQueue(queue, iterable, stop_signals) :
    iterable = iter(iterable)
    chunk_size = 100000
    multiplier = 1.1

    # initial values
    i = 0
    n_records = 0
    t0 = time.clock()
    last_rate = 10000

    while True :
        chunk = list(itertools.islice(iterable, int(chunk_size)))
        if chunk :
            queue.put(chunk)

            n_records += chunk_size
            i += 1

            if i % 10 :
                time_delta = max(time.clock() - t0, 0.0001)

                current_rate = n_records/time_delta

                # chunk_size is always either growing or shrinking, if
                # the shrinking led to a faster rate, keep
                # shrinking. Same with growing. If the rate decreased,
                # reverse directions
                if current_rate < last_rate :
                    multiplier = 1/multiplier

                chunk_size = max(chunk_size * multiplier, 1)

                last_rate = current_rate
                n_records = 0
                t0 = time.clock()

        else :
            # put poison pills in queue to tell scorers that they are
            # done
            [queue.put(None) for _ in range(stop_signals)]
            break

def peek(records) :
    try :
        record = next(records)
    except TypeError as e:
        if "not an iterator" not in str(e) :
            raise
        try :
            records = iter(records)
            record = next(records)
        except StopIteration :
            return None, records
    except StopIteration :
        return None, records

    return record, itertools.chain([record], records)

def isIndexed(data, offset) :
    return all(i in data for i in range(offset, offset + len(data)))

def index(data, offset=0) :
    if isIndexed(data, offset):
        return data
    else :
        data = dict(zip(itertools.count(offset),
                        viewvalues(data)))
        return data

def iunzip(iterable, internal_length): # pragma: no cover
    """Iunzip is the same as zip(*iter) but returns iterators, instead of
    expand the iterator. Mostly used for large sequence"""

    _tmp, iterable = itertools.tee(iterable, 2)
    iters = itertools.tee(iterable, internal_length)
    return (map(operator.itemgetter(i), it) for i, it in enumerate(iters))

<commit_msg>handle case where sample is larger than population in randomPairsMatch<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
from builtins import range, next, zip, map
from future.utils import viewvalues, viewitems
import sys
if sys.version < '3':
    text_type = unicode
    binary_type = str
else:
    text_type = str
    binary_type = bytes
    unicode = str

import itertools
import time
import tempfile
import os
import operator
import random

import numpy

import dedupe.backport as backport

class ChildProcessError(Exception) :
    pass

def randomPairs(n_records, sample_size):
    """
    Return random combinations of indices for a square matrix of size
    n records
    """
    n = int(n_records * (n_records - 1) / 2)

    if sample_size >= n :
        random_pairs = numpy.arange(n, dtype='f')
    else:
        random_pairs = numpy.array(random.sample(range(n), sample_size), dtype='f')

    b = 1 - 2 * n_records

    i = numpy.floor((-b - numpy.sqrt(b ** 2 - 8 * random_pairs)) / 2).astype('uint')
    j = numpy.rint(random_pairs + i * (b + i + 2) / 2 + 1).astype('uint')

    return zip(i, j)

def randomPairsMatch(n_records_A, n_records_B, sample_size):
    """
    Return random combinations of indices for record list A and B
    """
    n = int(n_records_A * n_records_B)

    if sample_size >= n:
        random_pairs = numpy.arange(n)
    else:
        random_pairs = numpy.array(random.sample(range(n), sample_size), dtype='f')

    i = numpy.floor(random_pairs/n_records_B).astype('uint')
    j = (random_pairs - n_records_B * i).astype('uint')

    return zip(i, j)

class ScoreRecords(object) :
    def __init__(self, data_model, classifier, threshold) :
        self.data_model = data_model
        self.classifier = classifier
        self.threshold = threshold
        self.score_queue = None

    def __call__(self, records_queue, score_queue) :
        self.score_queue = score_queue
        while True :
            record_pairs = records_queue.get()
            if record_pairs is None :
                break

            try :
                filtered_pairs = self.fieldDistance(record_pairs)
                if filtered_pairs is not None :
                    score_queue.put(filtered_pairs)
            except Exception as e :
                score_queue.put(e)
                raise

        score_queue.put(None)

    def fieldDistance(self, record_pairs) :
        ids = []
        records = []

        for record_pair in record_pairs :
            ((id_1, record_1, smaller_ids_1),
             (id_2, record_2, smaller_ids_2)) = record_pair

            if smaller_ids_1.isdisjoint(smaller_ids_2) :

                ids.append((id_1, id_2))
                records.append((record_1, record_2))

        if records :
            ids = numpy.array(ids)

            distances = self.data_model.distances(records)
            scores = self.classifier.predict_proba(distances)[:,-1]

            mask = scores > self.threshold

            scored_pairs = numpy.empty(numpy.count_nonzero(mask),
                                       dtype=[('pairs', 'object', 2),
                                              ('score', 'f4', 1)])
            scored_pairs['pairs'] = ids[mask]
            scored_pairs['score'] = scores[mask]

            return scored_pairs

def mergeScores(score_queue, result_queue, stop_signals) :
    scored_pairs = numpy.empty(0, dtype= [('pairs', 'object', 2),
                                          ('score', 'f4', 1)])

    seen_signals = 0
    while seen_signals < stop_signals  :
        score_chunk = score_queue.get()
        if isinstance(score_chunk, Exception) :
            result_queue.put(score_chunk)
            return

        if score_chunk is not None :
            scored_pairs = numpy.concatenate((scored_pairs, score_chunk))
        else :
            seen_signals += 1

    if len(scored_pairs) :
        python_type = type(scored_pairs['pairs'][0][0])
        if python_type is binary_type or python_type is text_type :
            max_length = len(max(numpy.ravel(scored_pairs['pairs']), key=len))
            python_type = (unicode, max_length)

        write_dtype = [('pairs', python_type, 2),
                       ('score', 'f4', 1)]

        scored_pairs = scored_pairs.astype(write_dtype)

        scored_pairs_file, file_path = tempfile.mkstemp()

        os.close(scored_pairs_file)

        fp = numpy.memmap(file_path,
                          dtype=scored_pairs.dtype,
                          shape=scored_pairs.shape)
        fp[:] = scored_pairs[:]

        result_queue.put((file_path, scored_pairs.dtype))

    else :
        result_queue.put(scored_pairs)

def scoreDuplicates(records, data_model, classifier, num_cores=1, threshold=0) :
    if num_cores < 2 :
        from multiprocessing.dummy import Process, Pool, Queue
        SimpleQueue = Queue
    else :
        from .backport import Process, Pool, SimpleQueue

    record_pairs_queue = SimpleQueue()
    score_queue =  SimpleQueue()
    result_queue = SimpleQueue()

    n_map_processes = max(num_cores-1, 1)
    score_records = ScoreRecords(data_model, classifier, threshold)
    map_processes = [Process(target=score_records,
                             args=(record_pairs_queue,
                                   score_queue))
                     for _ in range(n_map_processes)]
    [process.start() for process in map_processes]

    reduce_process = Process(target=mergeScores,
                             args=(score_queue,
                                   result_queue,
                                   n_map_processes))
    reduce_process.start()

    fillQueue(record_pairs_queue, records, n_map_processes)

    result = result_queue.get()
    if isinstance(result, Exception) :
        raise ChildProcessError

    if result :
        scored_pairs_file, dtype = result
        scored_pairs = numpy.memmap(scored_pairs_file,
                                    dtype=dtype)
    else :
        scored_pairs = result

    reduce_process.join()
    [process.join() for process in map_processes]

    return scored_pairs

def fillQueue(queue, iterable, stop_signals) :
    iterable = iter(iterable)
    chunk_size = 100000
    multiplier = 1.1

    # initial values
    i = 0
    n_records = 0
    t0 = time.clock()
    last_rate = 10000

    while True :
        chunk = list(itertools.islice(iterable, int(chunk_size)))
        if chunk :
            queue.put(chunk)

            n_records += chunk_size
            i += 1

            if i % 10 :
                time_delta = max(time.clock() - t0, 0.0001)

                current_rate = n_records/time_delta

                # chunk_size is always either growing or shrinking, if
                # the shrinking led to a faster rate, keep
                # shrinking. Same with growing. If the rate decreased,
                # reverse directions
                if current_rate < last_rate :
                    multiplier = 1/multiplier

                chunk_size = max(chunk_size * multiplier, 1)

                last_rate = current_rate
                n_records = 0
                t0 = time.clock()

        else :
            # put poison pills in queue to tell scorers that they are
            # done
            [queue.put(None) for _ in range(stop_signals)]
            break

def peek(records) :
    try :
        record = next(records)
    except TypeError as e:
        if "not an iterator" not in str(e) :
            raise
        try :
            records = iter(records)
            record = next(records)
        except StopIteration :
            return None, records
    except StopIteration :
        return None, records

    return record, itertools.chain([record], records)

def isIndexed(data, offset) :
    return all(i in data for i in range(offset, offset + len(data)))

def index(data, offset=0) :
    if isIndexed(data, offset):
        return data
    else :
        data = dict(zip(itertools.count(offset),
                        viewvalues(data)))
        return data

def iunzip(iterable, internal_length): # pragma: no cover
    """Iunzip is the same as zip(*iter) but returns iterators, instead of
    expand the iterator. Mostly used for large sequence"""

    _tmp, iterable = itertools.tee(iterable, 2)
    iters = itertools.tee(iterable, internal_length)
    return (map(operator.itemgetter(i), it) for i, it in enumerate(iters))

<|endoftext|>