<commit_before>import urllib

from django.conf import settings
from django.core.urlresolvers import reverse
from django.http import HttpResponse, Http404
from django.shortcuts import render, redirect
from django.utils import simplejson as json
from django.views.decorators.cache import cache_page

from ui import voyager, apis
from ui.sort import libsort, availsort, elecsort, splitsort

def home(request):
    return render(request, 'home.html', {
        'title': 'launchpad home',
        'google_analytics_ua': settings.GOOGLE_ANALYTICS_UA,
        })

def _openurl_dict(params):
    """Split openurl params into a useful structure"""
    p = {}
    for k, v in dict(params).items():
        p[k] = ','.join(v)
    d = {'params':  p}
    d['query_string'] = '&'.join(['%s=%s' % (k, v) for k, v
        in params.items()])
    d['query_string_encoded'] = urllib.urlencode(params)
    return d

@cache_page(settings.ITEM_PAGE_CACHE_SECONDS)
def item(request, bibid):
    try:
        bib = voyager.get_bib_data(bibid)
        if not bib:
            return render(request, '404.html', {'num': bibid,
                'num_type': 'BIB ID'}, status=404)
        bib['openurl'] = _openurl_dict(request.GET)
        # Ensure bib data is ours if possible
        if not bib['LIBRARY_NAME'] == settings.PREF_LIB:
            for alt_bib in bib['BIB_ID_LIST']:
                if alt_bib['LIBRARY_NAME'] == settings.PREF_LIB:
                    return item(request, alt_bib['BIB_ID'])
        holdings = voyager.get_holdings(bib)
        ours, theirs, shared = splitsort(holdings)
        holdings = availsort(elecsort(ours)) + availsort(elecsort(shared)) \
            + libsort(elecsort(availsort(theirs), rev=True))
        return render(request, 'item.html', {
            'bibid': bibid,
            'bib': bib,
            'debug': settings.DEBUG,
            'title_chars': settings.TITLE_CHARS,
            'holdings': holdings,
            'link': bib.get('LINK', '')[9:],
            'google_analytics_ua': settings.GOOGLE_ANALYTICS_UA,
            'link_resolver': settings.LINK_RESOLVER,
            'enable_humans': settings.ENABLE_HUMANS,
            })
    except DatabaseError, e:
        from datetime import datetime
        if datetime.now().hour == 3:
            return render(request, '503.html', status=503)
        else:
            raise

@cache_page(settings.ITEM_PAGE_CACHE_SECONDS)
def item_bib(request, bibid):
    bib = voyager.get_bib_data(bibid, expand_ids=False)
    if not bib:
        return render(request, '404.html', {'num': bibid,
            'num_type': 'BIB ID'}, status=404)
    # Don't bother expanding bibids; we don't need correct holdings
    return render(request, 'item.html', {
        'bibid': bibid,
        'bib': bib,
        'debug': settings.DEBUG,
        'title_chars': settings.TITLE_CHARS,
        'link': bib.get('LINK', '')[9:],
        'google_analytics_ua': settings.GOOGLE_ANALYTICS_UA,
        'link_resolver': settings.LINK_RESOLVER,
        'enable_humans': settings.ENABLE_HUMANS,
        })

def _date_handler(obj):
    return obj.isoformat() if hasattr(obj, 'isoformat') else obj

@cache_page(settings.ITEM_PAGE_CACHE_SECONDS)
def item_json(request, bibid):
    bib_data = voyager.get_bib_data(bibid)
    bib_data['holdings'] = voyager.get_holdings(bib_data)
    bib_data['openurl'] = _openurl_dict(request.GET)
    return HttpResponse(json.dumps(bib_data, default=_date_handler,
        indent=2), content_type='application/json')

def non_wrlc_item(request, num, num_type):
    bib = apis.get_bib_data(num=num, num_type=num_type)
    if not bib:
        return render(request, '404.html', {'num': num,
            'num_type': num_type.upper()}, status=404)
    bib['ILLIAD_LINK'] = voyager.get_illiad_link(bib)
    return render(request, 'item.html', {
       'bibid': '',
       'bib': bib,
       'debug': settings.DEBUG,
       'title_chars': settings.TITLE_CHARS,
       'holdings': [],
       'link': '',
       'google_analytics_ua': settings.GOOGLE_ANALYTICS_UA,
       'link_resolver': settings.LINK_RESOLVER,
       })

def gtitem(request, gtbibid):
    bibid = voyager.get_wrlcbib_from_gtbib(gtbibid)
    if bibid:
        return redirect('item', bibid=bibid)
    return render(request, '404.html', {'num': gtbibid,
        'num_type': 'Georgetown BIB ID'}, status=404)

def gtitem_json(request, gtbibid):
    bibid = voyager.get_wrlcbib_from_gtbib(gtbibid)
    if bibid:
        return redirect('item_json', bibid=bibid)
    raise Http404

def gmitem(request, gmbibid):
    bibid = voyager.get_wrlcbib_from_gmbib(gmbibid)
    if bibid:
        return redirect('item', bibid=bibid)
    return render(request, '404.html', {'num': gmbibid,
        'num_type': 'George Mason BIB ID'}, status=404)

def gmitem_json(request, gmbibid):
    bibid = voyager.get_wrlcbib_from_gmbib(gmbibid)
    if bibid:
        return redirect('item_json', bibid=bibid)
    raise Http404

def isbn(request, isbn):
    bibid = voyager.get_primary_bibid(num=isbn, num_type='isbn')
    openurl = _openurl_dict(request.GET)
    if bibid:
        url = '%s?%s' % (reverse('item', args=[bibid]),
            openurl['query_string_encoded'])
        return redirect(url)
    return non_wrlc_item(request, num=isbn, num_type='isbn')

def issn(request, issn):
    bibid = voyager.get_primary_bibid(num=issn, num_type='issn')
    openurl = _openurl_dict(request.GET)
    if bibid:
        url = '%s?%s' % (reverse('item', args=[bibid]),
            openurl['query_string_encoded'])
        return redirect(url)
    return non_wrlc_item(request, num=issn, num_type='issn')

def oclc(request, oclc):
    bibid = voyager.get_primary_bibid(num=oclc, num_type='oclc')
    openurl = _openurl_dict(request.GET)
    if bibid:
        url = '%s?%s' % (reverse('item', args=[bibid]),
            openurl['query_string_encoded'])
        return redirect(url)
    return render(request, '404.html', {'num': oclc,
        'num_type': 'OCLC number'}, status=404)

def error500(request):
    return render(request, '500.html', {
        'title': 'error',
        'google_analytics_ua': settings.GOOGLE_ANALYTICS_UA,
        }, status=500)

def robots(request):
    return render(request, 'robots.txt', {
        'enable_sitemaps': settings.ENABLE_SITEMAPS,
        'sitemaps_base_url': settings.SITEMAPS_BASE_URL,
        }, content_type='text/plain')

def humans(request):
    return render(request, 'humans.txt', {}, content_type='text/plain')
<commit_msg>Added import statement to views.py for DatabaseError<commit_after>import urllib

from django.conf import settings
from django.core.urlresolvers import reverse
from django.db.utils import DatabaseError
from django.http import HttpResponse, Http404
from django.shortcuts import render, redirect
from django.utils import simplejson as json
from django.views.decorators.cache import cache_page

from ui import voyager, apis
from ui.sort import libsort, availsort, elecsort, splitsort

def home(request):
    return render(request, 'home.html', {
        'title': 'launchpad home',
        'google_analytics_ua': settings.GOOGLE_ANALYTICS_UA,
        })

def _openurl_dict(params):
    """Split openurl params into a useful structure"""
    p = {}
    for k, v in dict(params).items():
        p[k] = ','.join(v)
    d = {'params':  p}
    d['query_string'] = '&'.join(['%s=%s' % (k, v) for k, v
        in params.items()])
    d['query_string_encoded'] = urllib.urlencode(params)
    return d

@cache_page(settings.ITEM_PAGE_CACHE_SECONDS)
def item(request, bibid):
    try:
        bib = voyager.get_bib_data(bibid)
        if not bib:
            return render(request, '404.html', {'num': bibid,
                'num_type': 'BIB ID'}, status=404)
        bib['openurl'] = _openurl_dict(request.GET)
        # Ensure bib data is ours if possible
        if not bib['LIBRARY_NAME'] == settings.PREF_LIB:
            for alt_bib in bib['BIB_ID_LIST']:
                if alt_bib['LIBRARY_NAME'] == settings.PREF_LIB:
                    return item(request, alt_bib['BIB_ID'])
        holdings = voyager.get_holdings(bib)
        ours, theirs, shared = splitsort(holdings)
        holdings = availsort(elecsort(ours)) + availsort(elecsort(shared)) \
            + libsort(elecsort(availsort(theirs), rev=True))
        return render(request, 'item.html', {
            'bibid': bibid,
            'bib': bib,
            'debug': settings.DEBUG,
            'title_chars': settings.TITLE_CHARS,
            'holdings': holdings,
            'link': bib.get('LINK', '')[9:],
            'google_analytics_ua': settings.GOOGLE_ANALYTICS_UA,
            'link_resolver': settings.LINK_RESOLVER,
            'enable_humans': settings.ENABLE_HUMANS,
            })
    except DatabaseError, e:
        from datetime import datetime
        if datetime.now().hour == 3:
            return render(request, '503.html', status=503)
        else:
            raise

@cache_page(settings.ITEM_PAGE_CACHE_SECONDS)
def item_bib(request, bibid):
    bib = voyager.get_bib_data(bibid, expand_ids=False)
    if not bib:
        return render(request, '404.html', {'num': bibid,
            'num_type': 'BIB ID'}, status=404)
    # Don't bother expanding bibids; we don't need correct holdings
    return render(request, 'item.html', {
        'bibid': bibid,
        'bib': bib,
        'debug': settings.DEBUG,
        'title_chars': settings.TITLE_CHARS,
        'link': bib.get('LINK', '')[9:],
        'google_analytics_ua': settings.GOOGLE_ANALYTICS_UA,
        'link_resolver': settings.LINK_RESOLVER,
        'enable_humans': settings.ENABLE_HUMANS,
        })

def _date_handler(obj):
    return obj.isoformat() if hasattr(obj, 'isoformat') else obj

@cache_page(settings.ITEM_PAGE_CACHE_SECONDS)
def item_json(request, bibid):
    bib_data = voyager.get_bib_data(bibid)
    bib_data['holdings'] = voyager.get_holdings(bib_data)
    bib_data['openurl'] = _openurl_dict(request.GET)
    return HttpResponse(json.dumps(bib_data, default=_date_handler,
        indent=2), content_type='application/json')

def non_wrlc_item(request, num, num_type):
    bib = apis.get_bib_data(num=num, num_type=num_type)
    if not bib:
        return render(request, '404.html', {'num': num,
            'num_type': num_type.upper()}, status=404)
    bib['ILLIAD_LINK'] = voyager.get_illiad_link(bib)
    return render(request, 'item.html', {
       'bibid': '',
       'bib': bib,
       'debug': settings.DEBUG,
       'title_chars': settings.TITLE_CHARS,
       'holdings': [],
       'link': '',
       'google_analytics_ua': settings.GOOGLE_ANALYTICS_UA,
       'link_resolver': settings.LINK_RESOLVER,
       })

def gtitem(request, gtbibid):
    bibid = voyager.get_wrlcbib_from_gtbib(gtbibid)
    if bibid:
        return redirect('item', bibid=bibid)
    return render(request, '404.html', {'num': gtbibid,
        'num_type': 'Georgetown BIB ID'}, status=404)

def gtitem_json(request, gtbibid):
    bibid = voyager.get_wrlcbib_from_gtbib(gtbibid)
    if bibid:
        return redirect('item_json', bibid=bibid)
    raise Http404

def gmitem(request, gmbibid):
    bibid = voyager.get_wrlcbib_from_gmbib(gmbibid)
    if bibid:
        return redirect('item', bibid=bibid)
    return render(request, '404.html', {'num': gmbibid,
        'num_type': 'George Mason BIB ID'}, status=404)

def gmitem_json(request, gmbibid):
    bibid = voyager.get_wrlcbib_from_gmbib(gmbibid)
    if bibid:
        return redirect('item_json', bibid=bibid)
    raise Http404

def isbn(request, isbn):
    bibid = voyager.get_primary_bibid(num=isbn, num_type='isbn')
    openurl = _openurl_dict(request.GET)
    if bibid:
        url = '%s?%s' % (reverse('item', args=[bibid]),
            openurl['query_string_encoded'])
        return redirect(url)
    return non_wrlc_item(request, num=isbn, num_type='isbn')

def issn(request, issn):
    bibid = voyager.get_primary_bibid(num=issn, num_type='issn')
    openurl = _openurl_dict(request.GET)
    if bibid:
        url = '%s?%s' % (reverse('item', args=[bibid]),
            openurl['query_string_encoded'])
        return redirect(url)
    return non_wrlc_item(request, num=issn, num_type='issn')

def oclc(request, oclc):
    bibid = voyager.get_primary_bibid(num=oclc, num_type='oclc')
    openurl = _openurl_dict(request.GET)
    if bibid:
        url = '%s?%s' % (reverse('item', args=[bibid]),
            openurl['query_string_encoded'])
        return redirect(url)
    return render(request, '404.html', {'num': oclc,
        'num_type': 'OCLC number'}, status=404)

def error500(request):
    return render(request, '500.html', {
        'title': 'error',
        'google_analytics_ua': settings.GOOGLE_ANALYTICS_UA,
        }, status=500)

def robots(request):
    return render(request, 'robots.txt', {
        'enable_sitemaps': settings.ENABLE_SITEMAPS,
        'sitemaps_base_url': settings.SITEMAPS_BASE_URL,
        }, content_type='text/plain')

def humans(request):
    return render(request, 'humans.txt', {}, content_type='text/plain')
<|endoftext|>


<commit_before>import urllib2
from urllib import urlencode
import simplejson as json
import sys, re

_API_URL = "https://www.comicvine.com/api/"

_cached_resources = {}

api_key = ""

class InvalidResourceError(Exception):
    pass

class InvalidAPIKeyError(Exception):
    pass

class ObjectNotFoundError(Exception):
    pass

class ErrorInURLFormatError(Exception):
    pass

class JSONError(Exception):
    pass

class FilterError(Exception):
    pass

class SubscriberOnlyError(Exception):
    pass

class UnknownStatusError(Exception):
    pass

class IllegalArquementException(Exception):
    pass

_EXCEPTIONS = {
        100: InvalidAPIKeyError,
        101: ObjectNotFoundError,
        102: ErrorInURLFormatError,
        103: JSONError,
        104: FilterError,
        105: SubscriberOnlyError,
    }

class _Resource(object):
    class _Response:
        def __init__(
                self,
                error,
                limit,
                offset,
                number_of_page_results,
                number_of_total_results,
                status_code,
                version,
                results
            ):
            self.error = error
            self.limit = limit
            self.offset = offset
            self.number_of_page_results = number_of_page_results
            self.number_of_total_results = number_of_total_results
            self.status_code = status_code
            self.results = results

    def __init__(self, *args, **kwargs):
        raise NotImplemented()

    def _request_object(self, baseurl, **params):
        raise NotImplemented()

    @classmethod
    def _ensure_resource_url(type):
        if not '_resource_url' in type.__dict__:
            resource_type = type.__name__.lower()
            type._resource_url = _API_URL + resource_type + "/"

    @classmethod
    def _request(type, baseurl, **params):
        if 'api_key' not in params:
            if len(api_key) == 0:
                raise InvalidAPIKeyError(
                        "Invalid API Key"
                    )
            params['api_key'] = api_key
        if 'field_list' in params and params['field_list'] != None:
            if not isinstance(params['field_list'], (str, unicode)):
                field_list = ""
                try:
                    for field_name in params['field_list']:
                        field_list += str(field_name)+","
                except TypeError, e:
                    raise IllegalArquementException(
                            "'field_list' must be iterable"
                        )
                params['field_list'] = field_list
            if re.search(
                    "([a-z_]*,)*id(,[a-z_]*)*", params['field_list']
                ) == None:
                params['field_list'] = "id," + params['field_list']
        params['format'] = 'json'
        params = urlencode(params)
        response = type._Response(**json.loads(urllib2.urlopen(
                baseurl+"?"+params
            ).read()))
        if response.status_code != 1:
            raise _EXCEPTIONS.get(response.status_code,UnknownStatusError)(
                    response.error
                )
        if 'aliases' in response.results:
            response.results['aliases'] = response.results[
                    'aliases'
                ].split('\n')
        return response

class _SingularResource(_Resource):
    def __new__(type, id, **kwargs):
        resource_type = kwargs.get(
                'resource_type',
                type.__name__.lower()
            )
        try:
            type_id = Types()[resource_type]['id']
        except KeyError:
            return InvalidResourceError(resource_type)
        type._ensure_resource_url()
        key = "{0:d}-{1:d}".format(type_id, id)
        obj = _cached_resources.get(key)
        if obj == None:
            obj = object.__new__(type, id, **kwargs)
            _cached_resources[key] = obj
        return obj

    def __init__(self, id, **kwargs):
        if '_ready' not in self.__dict__:
            self._ready = True
            resource_type = type(self).__name__.lower()
            try:
                type_id = Types()[resource_type]['id']
            except KeyError:
                raise InvalidResourceError(
                        "Resource type '{0!s}' does not exist.".format(
                                resource_type
                            )
                    )
            if 'api_detail_url' in kwargs:
                self._detail_url = kwargs['api_detail_url']
            else:
                self._detail_url = type(self)._resource_url + \
                        "{0:d}-{1:d}/".format(type_id, id)
            self._fields = {'id': id}
            if 'field_list' in kwargs:
                self._fields.update(self._request_object(
                        kwargs['field_list']
                    ).results)
                del kwargs['field_list']
            self._fields.update(kwargs)
        elif 'field_list' in kwargs:
            self._fields.update(self._request_object(
                    kwargs['field_list']
                ).results)

    def _request_object(self, field_list = None):
        return type(self)._request(
                self._detail_url,
                field_list=field_list
            )

    def __getattribute__(self, name):
        def _object_attribute(name):
            return object.__getattribute__(self, name)
        try:
            if name not in ['__dict__', '_request_object'] and \
                    name not in self.__dict__:
                if name in _object_attribute('_fields'):
                    return _object_attribute('_fields')[name]
                else:
                    self._fields.update(
                            _object_attribute('_request_object')(
                                    [name]
                                ).results
                        )
                    return _object_attribute('_fields')[name]
        except KeyError:
            pass
        return _object_attribute(name)

class _ListResource(_Resource):
    def _request_object(self, **params):
        type(self)._ensure_resource_url()
        return type(self)._request(type(self)._resource_url, **params)

    def __init__(self, init_list = None, **kwargs):
        if init_list != None:
            if len(kwargs) > 0:
                raise TypeError(
                        "If 'init_list' is given it is the only "+
                        "allowed argument"
                    )
            self._results = init_list
            self._total = len(init_list)
            self._limit = len(init_list)
        else:
            response = self._request_object(**kwargs)
            self._results = response.offset*[None] + response.results
            self._total = response.number_of_total_results
            self._limit = response.limit
            if 'limit' in kwargs:
                del kwargs['limit']
            if 'offset' in kwargs:
                del kwargs['offset']
            self._args = kwargs

    def __len__(self):
        return self._total

    def __getitem__(self, index):
        if type(index) == slice:
            start = index.start or 0
            stop = index.stop or self._total
            step = index.step or 1
        else:
            start = index
            stop = index+1
            step = 1
        if start < 0 or start >= self._total or \
                stop > self._total:
            raise IndexError('Index out of range')
        if len(self._results) < stop or \
                None in self._results[start:stop:step]:
            for i in range(start, stop, 100):
                response = self._request_object(
                        limit=self._limit,
                        offset=i,
                        **self._args
                    )
                self._results[i:self._limit+i] = response.results
        if type(self._results[index]) == list:
            for i in range(start, stop, step):
                if type(self._results[i]) == dict:
                    self._parse_result(i)
        elif type(self._results[index]) == dict:
            self._parse_result(index)
        return self._results[index]

    def __iter__(self):
        for index in xrange(self._total):
            yield self[index]

    def __str__(self):
        string = "["
        for element in self:
            string += str(element)+","
        string = string[:-1]+"]"
        return string

    def _parse_result(self, index):
        if type(self) != Types:
            type_dict = Types()
            if type(self) == Search:
                self._results[index] = type_dict.singular_resource_class(
                        self._results[index]['resource_type']
                    )(**self._results[index])
            else:
                resource_type = type(self).__name__.lower()
                self._results[index] = type_dict.singular_resource_class(
                        resource_type
                    )(**self._results[index])

class _SortableListResource(_ListResource):
    def __init__(self, init_list = [], sort = None, **kwargs):
        if sort != None:
            if isinstance(sort, (str, unicode)):
                if ':' not in sort:
                    sort += ":asc"
            else:
                try:
                    sort = str(sort[0])+":"+str(sort[1])
                except KeyError:
                    if 'field' not in sort:
                        raise IllegalArquementException(
                                "Argument 'sort' must contain item 'field'"
                            )
                    if 'direction' in sort:
                        sort = sort['field']+":"+str(sort['direction'])
                    else:
                        sort = sort['field']+":asc"
            kwargs['sort'] = sort
        super(_SortableListResource, self).__init__(init_list, **kwargs)

class Search(_ListResource):
    def __init__(self, resources, query, **kwargs):
        super(Search, self).__init__(
                resources=resources,
                query=query,
                **kwargs
            )

class Types(_ListResource):
    def __new__(type):
        if not '_instance' in type.__dict__:
            type._instance = object.__new__(type)
        return type._instance

    def __init__(self):
        if not '_ready' in dir(self):
            super(Types, self).__init__()
            self._mapping = {}
            for type in self:
                self._mapping[type['detail_resource_name']] = type
                self._mapping[type['list_resource_name']] = type
            self._ready = True

    def __getitem__(self, key):
        if type(key) in [int, long, slice]:
            return super(Types, self).__getitem__(key)
        return self._mapping[key]

    def singular_resource_class(self, key):
        key = key.lower()
        if key not in self._mapping:
            raise InvalidResourceError(key)
        classname = self._mapping[key]['detail_resource_name']
        classname = classname[0].upper() + classname[1:].lower()
        return getattr(sys.modules[__name__], classname)
<commit_msg>Add __str__ and __repr__ for all resource types<commit_after>import urllib2
from urllib import urlencode
import simplejson as json
import sys, re

_API_URL = "https://www.comicvine.com/api/"

_cached_resources = {}

api_key = ""

class InvalidResourceError(Exception):
    pass

class InvalidAPIKeyError(Exception):
    pass

class ObjectNotFoundError(Exception):
    pass

class ErrorInURLFormatError(Exception):
    pass

class JSONError(Exception):
    pass

class FilterError(Exception):
    pass

class SubscriberOnlyError(Exception):
    pass

class UnknownStatusError(Exception):
    pass

class IllegalArquementException(Exception):
    pass

_EXCEPTIONS = {
        100: InvalidAPIKeyError,
        101: ObjectNotFoundError,
        102: ErrorInURLFormatError,
        103: JSONError,
        104: FilterError,
        105: SubscriberOnlyError,
    }

class _Resource(object):
    class _Response:
        def __init__(
                self,
                error,
                limit,
                offset,
                number_of_page_results,
                number_of_total_results,
                status_code,
                version,
                results
            ):
            self.error = error
            self.limit = limit
            self.offset = offset
            self.number_of_page_results = number_of_page_results
            self.number_of_total_results = number_of_total_results
            self.status_code = status_code
            self.results = results

    def __init__(self, *args, **kwargs):
        raise NotImplemented()

    def _request_object(self, baseurl, **params):
        raise NotImplemented()

    @classmethod
    def _ensure_resource_url(type):
        if not '_resource_url' in type.__dict__:
            resource_type = type.__name__.lower()
            type._resource_url = _API_URL + resource_type + "/"

    @classmethod
    def _request(type, baseurl, **params):
        if 'api_key' not in params:
            if len(api_key) == 0:
                raise InvalidAPIKeyError(
                        "Invalid API Key"
                    )
            params['api_key'] = api_key
        if 'field_list' in params and params['field_list'] != None:
            if not isinstance(params['field_list'], (str, unicode)):
                field_list = ""
                try:
                    for field_name in params['field_list']:
                        field_list += str(field_name)+","
                except TypeError, e:
                    raise IllegalArquementException(
                            "'field_list' must be iterable"
                        )
                params['field_list'] = field_list
            if re.search(
                    "([a-z_]*,)*id(,[a-z_]*)*", params['field_list']
                ) == None:
                params['field_list'] = "id," + params['field_list']
        params['format'] = 'json'
        params = urlencode(params)
        response = type._Response(**json.loads(urllib2.urlopen(
                baseurl+"?"+params
            ).read()))
        if response.status_code != 1:
            raise _EXCEPTIONS.get(response.status_code,UnknownStatusError)(
                    response.error
                )
        if 'aliases' in response.results:
            response.results['aliases'] = response.results[
                    'aliases'
                ].split('\n')
        return response

class _SingularResource(_Resource):
    def __new__(type, id, **kwargs):
        resource_type = kwargs.get(
                'resource_type',
                type.__name__.lower()
            )
        try:
            type_id = Types()[resource_type]['id']
        except KeyError:
            return InvalidResourceError(resource_type)
        type._ensure_resource_url()
        key = "{0:d}-{1:d}".format(type_id, id)
        obj = _cached_resources.get(key)
        if obj == None:
            obj = object.__new__(type, id, **kwargs)
            _cached_resources[key] = obj
        return obj

    def __init__(self, id, **kwargs):
        if '_ready' not in self.__dict__:
            self._ready = True
            resource_type = type(self).__name__.lower()
            try:
                type_id = Types()[resource_type]['id']
            except KeyError:
                raise InvalidResourceError(
                        "Resource type '{0!s}' does not exist.".format(
                                resource_type
                            )
                    )
            if 'api_detail_url' in kwargs:
                self._detail_url = kwargs['api_detail_url']
            else:
                self._detail_url = type(self)._resource_url + \
                        "{0:d}-{1:d}/".format(type_id, id)
            self._fields = {'id': id}
            if 'field_list' in kwargs:
                self._fields.update(self._request_object(
                        kwargs['field_list']
                    ).results)
                del kwargs['field_list']
            self._fields.update(kwargs)
        elif 'field_list' in kwargs:
            self._fields.update(self._request_object(
                    kwargs['field_list']
                ).results)

    def _request_object(self, field_list = None):
        return type(self)._request(
                self._detail_url,
                field_list=field_list
            )

    def __getattribute__(self, name):
        def _object_attribute(name):
            return object.__getattribute__(self, name)
        try:
            if name not in ['__dict__', '_request_object'] and \
                    name not in self.__dict__:
                if name in _object_attribute('_fields'):
                    return _object_attribute('_fields')[name]
                else:
                    self._fields.update(
                            _object_attribute('_request_object')(
                                    [name]
                                ).results
                        )
                    return _object_attribute('_fields')[name]
        except KeyError:
            pass
        return _object_attribute(name)

    def __str__(self):
        if 'name' in self._fields:
            return str(self.name)+" ["+str(self.id)+"]"
        else:
            return "["+str(self.id)+"]"

    def __repr__(self):
        return "<"+type(self).__name__+": '"+str(self)+"'>"

class _ListResource(_Resource):
    def _request_object(self, **params):
        type(self)._ensure_resource_url()
        return type(self)._request(type(self)._resource_url, **params)

    def __init__(self, init_list = None, **kwargs):
        if init_list != None:
            if len(kwargs) > 0:
                raise TypeError(
                        "If 'init_list' is given it is the only "+
                        "allowed argument"
                    )
            self._results = init_list
            self._total = len(init_list)
            self._limit = len(init_list)
        else:
            response = self._request_object(**kwargs)
            self._results = response.offset*[None] + response.results
            self._total = response.number_of_total_results
            self._limit = response.limit
            if 'limit' in kwargs:
                del kwargs['limit']
            if 'offset' in kwargs:
                del kwargs['offset']
            self._args = kwargs

    def __len__(self):
        return self._total

    def __getitem__(self, index):
        if type(index) == slice:
            start = index.start or 0
            stop = index.stop or self._total
            step = index.step or 1
        else:
            start = index
            stop = index+1
            step = 1
        if start < 0 or start >= self._total or \
                stop > self._total:
            raise IndexError('Index out of range')
        if len(self._results) < stop or \
                None in self._results[start:stop:step]:
            for i in range(start, stop, 100):
                response = self._request_object(
                        limit=self._limit,
                        offset=i,
                        **self._args
                    )
                self._results[i:self._limit+i] = response.results
        if type(self._results[index]) == list:
            for i in range(start, stop, step):
                if type(self._results[i]) == dict:
                    self._parse_result(i)
        elif type(self._results[index]) == dict:
            self._parse_result(index)
        return self._results[index]

    def __iter__(self):
        for index in xrange(self._total):
            yield self[index]

    def __str__(self):
        if len(self) == 0:
            return "[]"
        string = "["
        for element in self:
            string += str(element)+", "
        string = string[:-2]+"]"
        return string

    def __repr__(self):
        if len(self) == 0:
            return type(self).__name__+"[]"
        string = type(self).__name__+"["
        for element in self:
            string += repr(element)+","
        string = string[:-1]+"]"
        return string

    def _parse_result(self, index):
        if type(self) != Types:
            type_dict = Types()
            if type(self) == Search:
                self._results[index] = type_dict.singular_resource_class(
                        self._results[index]['resource_type']
                    )(**self._results[index])
            else:
                resource_type = type(self).__name__.lower()
                self._results[index] = type_dict.singular_resource_class(
                        resource_type
                    )(**self._results[index])

class _SortableListResource(_ListResource):
    def __init__(self, init_list = [], sort = None, **kwargs):
        if sort != None:
            if isinstance(sort, (str, unicode)):
                if ':' not in sort:
                    sort += ":asc"
            else:
                try:
                    sort = str(sort[0])+":"+str(sort[1])
                except KeyError:
                    if 'field' not in sort:
                        raise IllegalArquementException(
                                "Argument 'sort' must contain item 'field'"
                            )
                    if 'direction' in sort:
                        sort = sort['field']+":"+str(sort['direction'])
                    else:
                        sort = sort['field']+":asc"
            kwargs['sort'] = sort
        super(_SortableListResource, self).__init__(init_list, **kwargs)

class Search(_ListResource):
    def __init__(self, resources, query, **kwargs):
        super(Search, self).__init__(
                resources=resources,
                query=query,
                **kwargs
            )

class Types(_ListResource):
    def __new__(type):
        if not '_instance' in type.__dict__:
            type._instance = object.__new__(type)
        return type._instance

    def __init__(self):
        if not '_ready' in dir(self):
            super(Types, self).__init__()
            self._mapping = {}
            for type in self:
                self._mapping[type['detail_resource_name']] = type
                self._mapping[type['list_resource_name']] = type
            self._ready = True

    def __getitem__(self, key):
        if type(key) in [int, long, slice]:
            return super(Types, self).__getitem__(key)
        return self._mapping[key]

    def singular_resource_class(self, key):
        key = key.lower()
        if key not in self._mapping:
            raise InvalidResourceError(key)
        classname = self._mapping[key]['detail_resource_name']
        classname = classname[0].upper() + classname[1:].lower()
        return getattr(sys.modules[__name__], classname)
<|endoftext|>


<commit_before>import abc
import time
import multiprocessing

from sklearn import grid_search, cross_validation, decomposition, svm

from manifold.infrastructure import Displayer
from manifold.learning.algorithms import Isomap

class Example(metaclass=abc.ABCMeta):
    title = None

    _displayer = None

    @property
    def displayer(self):
        self._displayer = self._displayer or Displayer(t=self.title)
        return self._displayer

    def run(self):
        raise NotImplementedError

    def dispose(self):
        pass

    def start(self):
        print(self.title)

        self.run()
        self.dispose()

class LearningExample(Example, metaclass=abc.ABCMeta):
    learner = svm.SVC
    data = target = None

    learning_parameters = [
            {'C': (1, 10, 100, 1000), 'kernel': ('linear',)},
            {'C': (1, 10, 100, 1000), 'gamma': (.001, .01, .1, 1, 10), 'kernel': ('linear', 'rbf', 'sigmoid')}
        ]

    def learn(self):
        start = time.time()
        print('GridSearch started at %s...' % start)

        X_train, X_test, y_train, y_test = cross_validation.train_test_split(self.data, self.target, test_size=.2)

        grid = grid_search.GridSearchCV(self.learner(), self.learning_parameters, n_jobs=multiprocessing.cpu_count())
        grid.fit(X_train, y_train)

        print('\tAccuracy: %.2f%%\n'
              '\tTime elapsed: %.0fs\n'
              '\tBest parameters: %s'
              % (grid.best_score_, time.time() - start, grid.best_params_))

        y_predicted = grid.predict(X_test)

        self.displayer.confusion_matrix_for(y_test, y_predicted)

class ReductionExample(Example, metaclass=abc.ABCMeta):
    data = reduced_data = target = None

    method = 'isomap'
    params = {
        'k': 4,
        'n_components': 3
    }

    def reduce(self):
        to_dimension = self.params['to_dimension'] if 'to_dimension' in self.params else \
            self.params['n_components'] if 'n_components' in self.params else \
                3

        print('Dimensionality reduction process has started')
        print('\tR^%i -to-> R^%i' % (self.data.shape[1], to_dimension))
        start = time.time()

        if self.method == 'pca':
            self.reduced_data = decomposition.PCA(**self.params).fit_transform(self.data)
        else:
            self.reduced_data = Isomap(self.data, **self.params).run()

        elapsed = time.time() - start
        print('Done (%f).' % elapsed)

        self.displayer.load(self.reduced_data, self.target, title='Data set reduced with %s' % self.method)
<commit_msg> Update Example interface  Update ReductionExample printed info.<commit_after>import abc
import time
import multiprocessing

from sklearn import grid_search, cross_validation, decomposition, svm, manifold

from manifold.infrastructure import Displayer
from manifold.learning.algorithms import Isomap

class Example(metaclass=abc.ABCMeta):
    title = None

    _displayer = None

    @property
    def displayer(self):
        self._displayer = self._displayer or Displayer()
        return self._displayer

    def _run(self):
        raise NotImplementedError

    def _dispose(self):
        pass

    def start(self):
        print(self.title)

        self._run()
        self._dispose()

class LearningExample(Example, metaclass=abc.ABCMeta):
    learner = svm.SVC
    data = target = None

    learning_parameters = [
        {'C': (1, 10, 100, 1000), 'kernel': ('linear',)},
        {'C': (1, 10, 100, 1000), 'gamma': (.001, .01, .1, 1, 10), 'kernel': ('rbf', 'sigmoid')}
    ]

    def learn(self):
        start = time.time()
        print('GridSearch started at %s...' % start)

        X_train, X_test, y_train, y_test = cross_validation.train_test_split(self.data, self.target, test_size=.2)

        grid = grid_search.GridSearchCV(self.learner(), self.learning_parameters, n_jobs=multiprocessing.cpu_count())
        grid.fit(X_train, y_train)

        print('\tAccuracy: %.2f%%\n'
              '\tTime elapsed: %.0fs\n'
              '\tBest parameters: %s'
              % (grid.best_score_, time.time() - start, grid.best_params_))

        y_predicted = grid.predict(X_test)

        self.displayer.confusion_matrix_for(y_test, y_predicted)

class ReductionExample(Example, metaclass=abc.ABCMeta):
    data = reduced_data = target = None

    method = 'isomap'
    params = {
        'k': 4,
        'n_components': 3
    }

    def reduce(self):
        to_dimension = self.params['to_dimension'] if 'to_dimension' in self.params else \
            self.params['n_components'] if 'n_components' in self.params else \
                3

        print('Dimensionality reduction process has started')
        print('\tMethod: %s' % self.method)
        print('\tR^%i -to-> R^%i' % (self.data.shape[1], to_dimension))

        start = time.time()

        if self.method == 'pca':
            self.reduced_data = decomposition.PCA(**self.params).fit_transform(self.data)
        elif self.method == 'skisomap':
            self.reduced_data = manifold.Isomap(**self.params).fit_transform(self.data)
        else:
            self.reduced_data = Isomap(self.data, **self.params).run()

        elapsed = time.time() - start
        print('\tNew data set\'s size: %i' % self.reduced_data.nbytes)
        print('Done (%f).' % elapsed)

        self.displayer.load(self.reduced_data, self.target)
<|endoftext|>


<commit_before>"""Definitions of common enumerations to be used together with ``Enum`` property. """

from six import string_types

class Enumeration(object):
    pass

def enumeration(*values):
    if not values or any(not isinstance(value, string_types) for value in values):
        raise ValueError("expected a non-empty sequence of strings, got %s" % values)

    attrs = dict([ (value, value) for value in values ])
    attrs.update({
        "__slots__": [],
        "_values": list(values),
        "_default": values[0],
    })

    return type("Enumeration", (Enumeration,), attrs)()

LineJoin = enumeration("miter", "round", "bevel")
LineDash = enumeration("solid", "dashed", "dotted", "dotdash", "dashdot")
LineCap = enumeration("butt", "round", "square")
FontStyle = enumeration("normal", "italic", "bold")
TextAlign = enumeration("left", "right", "center")
TextBaseline = enumeration("top", "middle", "bottom", "alphabetic", "hanging")
Direction = enumeration("clock", "anticlock")
Units = enumeration("screen", "data")
AngleUnits = enumeration("deg", "rad")
Dimension = enumeration("width", "height")
Location = enumeration("top", "bottom", "left", "right", "min")
Orientation = enumeration("top_right", "top_left", "bottom_left", "bottom_right")
NamedColor = enumeration(
    "indigo", "gold", "firebrick", "indianred", "yellow",
    "darkolivegreen", "darkseagreen", "darkslategrey", "mediumvioletred",
    "mediumorchid", "chartreuse", "mediumblue", "black", "springgreen",
    "orange", "lightsalmon", "brown", "turquoise", "olivedrab", "cyan",
    "silver", "skyblue", "gray", "darkturquoise", "goldenrod", "darkgreen",
    "darkviolet", "darkgray", "lightpink", "teal", "darkmagenta",
    "lightgoldenrodyellow", "lavender", "yellowgreen", "thistle", "violet",
    "navy", "dimgrey", "orchid", "blue", "ghostwhite", "honeydew",
    "cornflowerblue", "purple", "darkkhaki", "mediumpurple", "cornsilk", "red",
    "bisque", "slategray", "darkcyan", "khaki", "wheat", "deepskyblue",
    "darkred", "steelblue", "aliceblue", "lightslategrey", "gainsboro",
    "mediumturquoise", "floralwhite", "coral", "aqua", "burlywood",
    "darksalmon", "beige", "azure", "lightsteelblue", "oldlace", "greenyellow",
    "royalblue", "lightseagreen", "mistyrose", "sienna", "lightcoral",
    "orangered", "navajowhite", "lime", "palegreen", "lightcyan", "seashell",
    "mediumspringgreen", "fuchsia", "papayawhip", "blanchedalmond", "peru",
    "aquamarine", "white", "darkslategray", "ivory", "darkgoldenrod",
    "lawngreen", "lightgreen", "crimson", "forestgreen", "maroon", "olive",
    "mintcream", "antiquewhite", "dimgray", "hotpink", "moccasin", "limegreen",
    "saddlebrown", "grey", "darkslateblue", "lightskyblue", "deeppink",
    "plum", "lightgrey", "dodgerblue", "slateblue", "sandybrown", "magenta",
    "tan", "rosybrown", "pink", "lightblue", "palevioletred", "mediumseagreen",
    "linen", "darkorange", "powderblue", "seagreen", "snow", "mediumslateblue",
    "midnightblue", "paleturquoise", "palegoldenrod", "whitesmoke",
    "darkorchid", "salmon", "lightslategray", "lemonchiffon", "chocolate",
    "tomato", "cadetblue", "lightyellow", "lavenderblush", "darkblue",
    "mediumaquamarine", "green", "blueviolet", "peachpuff", "darkgrey")
<commit_msg>Add "x" and "y" to Dimension enumeration<commit_after>"""Definitions of common enumerations to be used together with ``Enum`` property. """

from six import string_types

class Enumeration(object):
    pass

def enumeration(*values):
    if not values or any(not isinstance(value, string_types) for value in values):
        raise ValueError("expected a non-empty sequence of strings, got %s" % values)

    attrs = dict([ (value, value) for value in values ])
    attrs.update({
        "__slots__": [],
        "_values": list(values),
        "_default": values[0],
    })

    return type("Enumeration", (Enumeration,), attrs)()

LineJoin = enumeration("miter", "round", "bevel")
LineDash = enumeration("solid", "dashed", "dotted", "dotdash", "dashdot")
LineCap = enumeration("butt", "round", "square")
FontStyle = enumeration("normal", "italic", "bold")
TextAlign = enumeration("left", "right", "center")
TextBaseline = enumeration("top", "middle", "bottom", "alphabetic", "hanging")
Direction = enumeration("clock", "anticlock")
Units = enumeration("screen", "data")
AngleUnits = enumeration("deg", "rad")
Dimension = enumeration("width", "height", "x", "y")
Location = enumeration("top", "bottom", "left", "right", "min")
Orientation = enumeration("top_right", "top_left", "bottom_left", "bottom_right")
NamedColor = enumeration(
    "indigo", "gold", "firebrick", "indianred", "yellow",
    "darkolivegreen", "darkseagreen", "darkslategrey", "mediumvioletred",
    "mediumorchid", "chartreuse", "mediumblue", "black", "springgreen",
    "orange", "lightsalmon", "brown", "turquoise", "olivedrab", "cyan",
    "silver", "skyblue", "gray", "darkturquoise", "goldenrod", "darkgreen",
    "darkviolet", "darkgray", "lightpink", "teal", "darkmagenta",
    "lightgoldenrodyellow", "lavender", "yellowgreen", "thistle", "violet",
    "navy", "dimgrey", "orchid", "blue", "ghostwhite", "honeydew",
    "cornflowerblue", "purple", "darkkhaki", "mediumpurple", "cornsilk", "red",
    "bisque", "slategray", "darkcyan", "khaki", "wheat", "deepskyblue",
    "darkred", "steelblue", "aliceblue", "lightslategrey", "gainsboro",
    "mediumturquoise", "floralwhite", "coral", "aqua", "burlywood",
    "darksalmon", "beige", "azure", "lightsteelblue", "oldlace", "greenyellow",
    "royalblue", "lightseagreen", "mistyrose", "sienna", "lightcoral",
    "orangered", "navajowhite", "lime", "palegreen", "lightcyan", "seashell",
    "mediumspringgreen", "fuchsia", "papayawhip", "blanchedalmond", "peru",
    "aquamarine", "white", "darkslategray", "ivory", "darkgoldenrod",
    "lawngreen", "lightgreen", "crimson", "forestgreen", "maroon", "olive",
    "mintcream", "antiquewhite", "dimgray", "hotpink", "moccasin", "limegreen",
    "saddlebrown", "grey", "darkslateblue", "lightskyblue", "deeppink",
    "plum", "lightgrey", "dodgerblue", "slateblue", "sandybrown", "magenta",
    "tan", "rosybrown", "pink", "lightblue", "palevioletred", "mediumseagreen",
    "linen", "darkorange", "powderblue", "seagreen", "snow", "mediumslateblue",
    "midnightblue", "paleturquoise", "palegoldenrod", "whitesmoke",
    "darkorchid", "salmon", "lightslategray", "lemonchiffon", "chocolate",
    "tomato", "cadetblue", "lightyellow", "lavenderblush", "darkblue",
    "mediumaquamarine", "green", "blueviolet", "peachpuff", "darkgrey")
<|endoftext|>


<commit_before># Copyright (c) 2015 Nicolas JOUANIN
#
# See the file license.txt for copying permission.
import logging
import asyncio
from asyncio import futures
from hbmqtt.mqtt.packet import MQTTFixedHeader
from hbmqtt.mqtt import packet_class
from hbmqtt.errors import NoDataException
from hbmqtt.mqtt.packet import PacketType
from hbmqtt.mqtt.connack import ConnackPacket
from hbmqtt.mqtt.connect import ConnectPacket
from hbmqtt.mqtt.pingresp import PingRespPacket
from hbmqtt.mqtt.pingreq import PingReqPacket
from hbmqtt.mqtt.publish import PublishPacket
from hbmqtt.mqtt.pubrel import PubrelPacket
from hbmqtt.mqtt.puback import PubackPacket
from hbmqtt.mqtt.pubrec import PubrecPacket
from hbmqtt.mqtt.pubcomp import PubcompPacket
from hbmqtt.mqtt.suback import SubackPacket
from hbmqtt.mqtt.subscribe import SubscribePacket
from hbmqtt.mqtt.unsubscribe import UnsubscribePacket
from hbmqtt.mqtt.unsuback import UnsubackPacket
from hbmqtt.mqtt.disconnect import DisconnectPacket
from hbmqtt.session import Session
from transitions import Machine

class InFlightMessage:
    states = ['new', 'published', 'acknowledged', 'received', 'released', 'completed']

    def __init__(self, packet, qos):
        self.packet = packet
        self.qos = qos
        self.puback = None
        self.pubrec = None
        self.pubcomp = None
        self.pubrel = None
        self._init_states()

    def _init_states(self):
        self.machine = Machine(model=self, states=InFlightMessage.states, initial='new')
        self.machine.add_transition(trigger='publish', source='new', dest='published')
        if self.qos == 0x01:
            self.machine.add_transition(trigger='acknowledge', source='published', dest='acknowledged')
        if self.qos == 0x02:
            self.machine.add_transition(trigger='receive', source='published', dest='received')
            self.machine.add_transition(trigger='release', source='received', dest='released')
            self.machine.add_transition(trigger='complete', source='released', dest='completed')

class ProtocolHandler:
    """
    Class implementing the MQTT communication protocol using asyncio features
    """

    def __init__(self, loop=None):
        self.logger = logging.getLogger(__name__)
        self.session = None
        if loop is None:
            self._loop = asyncio.get_event_loop()
        else:
            self._loop = loop
        self._reader_task = None
        self._writer_task = None
        self._reader_ready = asyncio.Event(loop=self._loop)
        self._writer_ready = asyncio.Event(loop=self._loop)

        self._running = False

        self.incoming_queues = dict()
        self.application_messages = asyncio.Queue()
        for p in PacketType:
            self.incoming_queues[p] = asyncio.Queue()
        self.outgoing_queue = asyncio.Queue()
        self._puback_waiters = dict()
        self._pubrec_waiters = dict()
        self._pubrel_waiters = dict()
        self._pubcomp_waiters = dict()
        self.delivered_message = asyncio.Queue()

    def attach_to_session(self, session: Session):
        self.session = session
        self.session.handler = self
        extra_info = self.session.writer.get_extra_info('sockname')
        self.session.local_address = extra_info[0]
        self.session.local_port = extra_info[1]

    def detach_from_session(self):
        self.session.handler = None
        self.session = None

    @asyncio.coroutine
    def start(self):
        self._running = True
        self._reader_task = asyncio.Task(self._reader_coro(), loop=self._loop)
        self._writer_task = asyncio.Task(self._writer_coro(), loop=self._loop)
        yield from asyncio.wait(
            [self._reader_ready.wait(), self._writer_ready.wait()], loop=self._loop)
        self.logger.debug("Handler tasks started")

    @asyncio.coroutine
    def mqtt_publish(self, topic, message, packet_id, dup, qos, retain):
        if packet_id in self.session.inflight_out:
            self.logger.warn("A message with the same packet ID is already in flight")
        packet = PublishPacket.build(topic, message, packet_id, dup, qos, retain)
        yield from self.outgoing_queue.put(packet)
        inflight_message = InFlightMessage(packet, qos)
        self.session.inflight_out[packet.variable_header.packet_id] = inflight_message

        inflight_message.publish()
        if qos == 0x01:
            waiter = futures.Future(loop=self._loop)
            self._puback_waiters[packet_id] = waiter
            yield from waiter
            inflight_message.puback = waiter.result()
            inflight_message.acknowledge()
            del self._puback_waiters[packet_id]
        if qos == 0x02:
            # Wait for PUBREC
            waiter = futures.Future(loop=self._loop)
            self._pubrec_waiters[packet_id] = waiter
            yield from waiter
            inflight_message.pubrec = waiter.result()
            del self._pubrec_waiters[packet_id]
            inflight_message.receive()

            # Send pubrel
            pubrel = PubrelPacket.build(packet_id)
            yield from self.outgoing_queue.put(pubrel)
            inflight_message.pubrel = pubrel
            inflight_message.release()

            # Wait for pubcomp
            waiter = futures.Future(loop=self._loop)
            self._pubcomp_waiters[packet_id] = waiter
            yield from waiter
            inflight_message.pubcomp = waiter.result()
            del self._pubcomp_waiters[packet_id]
            inflight_message.complete()

        del self.session.inflight_out[packet_id]
        return inflight_message

    @asyncio.coroutine
    def stop(self):
        self._running = False
        self.session.reader.feed_eof()
        yield from self.outgoing_queue.put("STOP")
        yield from asyncio.wait([self._writer_task, self._reader_task], loop=self._loop)

    @asyncio.coroutine
    def _reader_coro(self):
        self.logger.debug("Starting reader coro")
        while self._running:
            try:
                self._reader_ready.set()
                keepalive_timeout = self.session.keep_alive
                if keepalive_timeout <= 0:
                    keepalive_timeout = None
                fixed_header = yield from asyncio.wait_for(MQTTFixedHeader.from_stream(self.session.reader), keepalive_timeout)
                if fixed_header:
                    cls = packet_class(fixed_header)
                    packet = yield from cls.from_stream(self.session.reader, fixed_header=fixed_header)
                    self.logger.debug(" <-in-- " + repr(packet))

                    if packet.fixed_header.packet_type == PacketType.CONNACK:
                        asyncio.Task(self.handle_connack(packet))
                    elif packet.fixed_header.packet_type == PacketType.SUBSCRIBE:
                        asyncio.Task(self.handle_subscribe(packet))
                    elif packet.fixed_header.packet_type == PacketType.UNSUBSCRIBE:
                        asyncio.Task(self.handle_unsubscribe(packet))
                    elif packet.fixed_header.packet_type == PacketType.SUBACK:
                        asyncio.Task(self.handle_suback(packet))
                    elif packet.fixed_header.packet_type == PacketType.UNSUBACK:
                        asyncio.Task(self.handle_unsuback(packet))
                    elif packet.fixed_header.packet_type == PacketType.PUBACK:
                        asyncio.Task(self.handle_puback(packet))
                    elif packet.fixed_header.packet_type == PacketType.PUBREC:
                        asyncio.Task(self.handle_pubrec(packet))
                    elif packet.fixed_header.packet_type == PacketType.PUBREL:
                        asyncio.Task(self.handle_pubrel(packet))
                    elif packet.fixed_header.packet_type == PacketType.PUBCOMP:
                        asyncio.Task(self.handle_pubcomp(packet))
                    elif packet.fixed_header.packet_type == PacketType.PINGREQ:
                        asyncio.Task(self.handle_pingreq(packet))
                    elif packet.fixed_header.packet_type == PacketType.PINGRESP:
                        asyncio.Task(self.handle_pingresp(packet))
                    elif packet.fixed_header.packet_type == PacketType.PUBLISH:
                        asyncio.Task(self.handle_publish(packet))
                    elif packet.fixed_header.packet_type == PacketType.DISCONNECT:
                        asyncio.Task(self.handle_disconnect(packet))
                    elif packet.fixed_header.packet_type == PacketType.CONNECT:
                        asyncio.Task(self.handle_connect(packet))
                    else:
                        self.logger.warn("Unhandled packet type: %s" % packet.fixed_header.packet_type)
                else:
                    self.logger.debug("No more data, stopping reader coro")
                    break
            except asyncio.TimeoutError:
                self.logger.debug("Input stream read timeout")
                self.handle_read_timeout()
            except NoDataException as nde:
                self.logger.debug("No data available")
            except Exception as e:
                self.logger.warn("Unhandled exception in reader coro: %s" % e)
                break
        self.logger.debug("Reader coro stopped")

    @asyncio.coroutine
    def _writer_coro(self):
        self.logger.debug("Starting writer coro")
        while self._running:
            try:
                self._writer_ready.set()
                keepalive_timeout = self.session.keep_alive
                if keepalive_timeout <= 0:
                    keepalive_timeout = None
                packet = yield from asyncio.wait_for(self.outgoing_queue.get(), keepalive_timeout)
                if packet == "STOP":
                    self.logger.debug("Writer interruption")
                    break
                yield from packet.to_stream(self.session.writer)
                self.logger.debug(" -out-> " + repr(packet))
                yield from self.session.writer.drain()
                #self.outgoing_queue.task_done() # to be used with Python 3.5
            except asyncio.TimeoutError as ce:
                self.logger.debug("Output queue get timeout")
                if self._running:
                    self.handle_write_timeout()
            except Exception as e:
                self.logger.warn("Unhandled exception in writer coro: %s" % e)
                break
        self.logger.debug("Writer coro stopping")
        # Flush queue before stopping
        if not self.outgoing_queue.empty():
            while True:
                try:
                    packet = self.outgoing_queue.get_nowait()
                    yield from packet.to_stream(self.session.writer)
                    self.logger.debug(" -out-> " + repr(packet))
                except asyncio.QueueEmpty:
                    break
                except Exception as e:
                    self.logger.warn("Unhandled exception in writer coro: %s" % e)
        self.logger.debug("Writer coro stopped")

    @asyncio.coroutine
    def mqtt_deliver_next_message(self):
        inflight_message = yield from self.delivered_message.get()
        return inflight_message

    def handle_write_timeout(self):
        self.logger.warn('write timeout unhandled')

    def handle_read_timeout(self):
        self.logger.warn('read timeout unhandled')

    @asyncio.coroutine
    def handle_connack(self, connack: ConnackPacket):
        self.logger.warn('CONNACK unhandled')

    @asyncio.coroutine
    def handle_connect(self, connect: ConnectPacket):
        self.logger.warn('CONNECT unhandled')

    @asyncio.coroutine
    def handle_subscribe(self, subscribe: SubscribePacket):
        self.logger.warn('SUBSCRIBE unhandled')

    @asyncio.coroutine
    def handle_unsubscribe(self, subscribe: UnsubscribePacket):
        self.logger.warn('UNSUBSCRIBE unhandled')

    @asyncio.coroutine
    def handle_suback(self, suback: SubackPacket):
        self.logger.warn('SUBACK unhandled')

    @asyncio.coroutine
    def handle_unsuback(self, unsuback: UnsubackPacket):
        self.logger.warn('UNSUBACK unhandled')

    @asyncio.coroutine
    def handle_pingresp(self, pingresp: PingRespPacket):
        self.logger.warn('PINGRESP unhandled')

    @asyncio.coroutine
    def handle_pingreq(self, pingreq: PingReqPacket):
        self.logger.warn('PINGREQ unhandled')

    @asyncio.coroutine
    def handle_disconnect(self, disconnect: DisconnectPacket):
        self.logger.warn('DISCONNECT unhandled')

    @asyncio.coroutine
    def handle_puback(self, puback: PubackPacket):
        packet_id = puback.variable_header.packet_id
        try:
            waiter = self._puback_waiters[packet_id]
            waiter.set_result(puback)
        except KeyError as ke:
            self.logger.warn("Received PUBACK for unknown pending subscription with Id: %s" % packet_id)

    @asyncio.coroutine
    def handle_pubrec(self, pubrec: PubrecPacket):
        packet_id = pubrec.variable_header.packet_id
        try:
            waiter = self._pubrec_waiters[packet_id]
            waiter.set_result(pubrec)
        except KeyError as ke:
            self.logger.warn("Received PUBREC for unknown pending subscription with Id: %s" % packet_id)

    @asyncio.coroutine
    def handle_pubcomp(self, pubcomp: PubcompPacket):
        packet_id = pubcomp.variable_header.packet_id
        try:
            waiter = self._pubcomp_waiters[packet_id]
            waiter.set_result(pubcomp)
        except KeyError as ke:
            self.logger.warn("Received PUBCOMP for unknown pending subscription with Id: %s" % packet_id)

    @asyncio.coroutine
    def handle_pubrel(self, pubrel: PubrecPacket):
        packet_id = pubrel.variable_header.packet_id
        try:
            waiter = self._pubrel_waiters[packet_id]
            waiter.set_result(pubrel)
        except KeyError as ke:
            self.logger.warn("Received PUBREL for unknown pending subscription with Id: %s" % packet_id)

    @asyncio.coroutine
    def handle_publish(self, publish : PublishPacket):
        inflight_message = None
        packet_id = publish.variable_header.packet_id
        qos = (publish.fixed_header.flags >> 1) & 0x03

        if qos == 0:
            inflight_message = InFlightMessage(publish, qos)
            yield from self.delivered_message.put(inflight_message)
        else:
            if packet_id in self.session.inflight_in:
                inflight_message = self.session.inflight_in[packet_id]
            else:
                inflight_message = InFlightMessage(publish, qos)
                self.session.inflight_in[packet_id] = inflight_message
                inflight_message.publish()

            if qos == 1:
                puback = PubackPacket.build(packet_id)
                yield from self.outgoing_queue.put(puback)
                inflight_message.acknowledge()
            if qos == 2:
                pubrec = PubrecPacket.build(packet_id)
                yield from self.outgoing_queue.put(pubrec)
                inflight_message.receive()
                waiter = futures.Future(loop=self._loop)
                self._pubrel_waiters[packet_id] = waiter
                yield from waiter
                inflight_message.pubrel = waiter.result()
                del self._pubrel_waiters[packet_id]
                inflight_message.release()
                pubcomp = PubcompPacket.build(packet_id)
                yield from self.outgoing_queue.put(pubcomp)
                inflight_message.complete()
            yield from self.delivered_message.put(inflight_message)
            del self.session.inflight_in[packet_id]
<commit_msg>Clean coro stopping<commit_after># Copyright (c) 2015 Nicolas JOUANIN
#
# See the file license.txt for copying permission.
import logging
import asyncio
from asyncio import futures
from hbmqtt.mqtt.packet import MQTTFixedHeader, MQTTPacket
from hbmqtt.mqtt import packet_class
from hbmqtt.errors import NoDataException
from hbmqtt.mqtt.packet import PacketType
from hbmqtt.mqtt.connack import ConnackPacket
from hbmqtt.mqtt.connect import ConnectPacket
from hbmqtt.mqtt.pingresp import PingRespPacket
from hbmqtt.mqtt.pingreq import PingReqPacket
from hbmqtt.mqtt.publish import PublishPacket
from hbmqtt.mqtt.pubrel import PubrelPacket
from hbmqtt.mqtt.puback import PubackPacket
from hbmqtt.mqtt.pubrec import PubrecPacket
from hbmqtt.mqtt.pubcomp import PubcompPacket
from hbmqtt.mqtt.suback import SubackPacket
from hbmqtt.mqtt.subscribe import SubscribePacket
from hbmqtt.mqtt.unsubscribe import UnsubscribePacket
from hbmqtt.mqtt.unsuback import UnsubackPacket
from hbmqtt.mqtt.disconnect import DisconnectPacket
from hbmqtt.session import Session
from transitions import Machine

class InFlightMessage:
    states = ['new', 'published', 'acknowledged', 'received', 'released', 'completed']

    def __init__(self, packet, qos):
        self.packet = packet
        self.qos = qos
        self.puback = None
        self.pubrec = None
        self.pubcomp = None
        self.pubrel = None
        self._init_states()

    def _init_states(self):
        self.machine = Machine(model=self, states=InFlightMessage.states, initial='new')
        self.machine.add_transition(trigger='publish', source='new', dest='published')
        if self.qos == 0x01:
            self.machine.add_transition(trigger='acknowledge', source='published', dest='acknowledged')
        if self.qos == 0x02:
            self.machine.add_transition(trigger='receive', source='published', dest='received')
            self.machine.add_transition(trigger='release', source='received', dest='released')
            self.machine.add_transition(trigger='complete', source='released', dest='completed')

class ProtocolHandler:
    """
    Class implementing the MQTT communication protocol using asyncio features
    """

    def __init__(self, loop=None):
        self.logger = logging.getLogger(__name__)
        self.session = None
        if loop is None:
            self._loop = asyncio.get_event_loop()
        else:
            self._loop = loop
        self._reader_task = None
        self._writer_task = None
        self._reader_ready = asyncio.Event(loop=self._loop)
        self._writer_ready = asyncio.Event(loop=self._loop)

        self._running = False

        self.incoming_queues = dict()
        self.application_messages = asyncio.Queue()
        for p in PacketType:
            self.incoming_queues[p] = asyncio.Queue()
        self.outgoing_queue = asyncio.Queue()
        self._puback_waiters = dict()
        self._pubrec_waiters = dict()
        self._pubrel_waiters = dict()
        self._pubcomp_waiters = dict()
        self.delivered_message = asyncio.Queue()

    def attach_to_session(self, session: Session):
        self.session = session
        self.session.handler = self
        extra_info = self.session.writer.get_extra_info('sockname')
        self.session.local_address = extra_info[0]
        self.session.local_port = extra_info[1]

    def detach_from_session(self):
        self.session.handler = None
        self.session = None

    @asyncio.coroutine
    def start(self):
        self._running = True
        self._reader_task = asyncio.Task(self._reader_coro(), loop=self._loop)
        self._writer_task = asyncio.Task(self._writer_coro(), loop=self._loop)
        yield from asyncio.wait(
            [self._reader_ready.wait(), self._writer_ready.wait()], loop=self._loop)
        self.logger.debug("Handler tasks started")

    @asyncio.coroutine
    def mqtt_publish(self, topic, message, packet_id, dup, qos, retain):
        if packet_id in self.session.inflight_out:
            self.logger.warn("A message with the same packet ID is already in flight")
        packet = PublishPacket.build(topic, message, packet_id, dup, qos, retain)
        yield from self.outgoing_queue.put(packet)
        inflight_message = InFlightMessage(packet, qos)
        self.session.inflight_out[packet.variable_header.packet_id] = inflight_message

        inflight_message.publish()
        if qos == 0x01:
            waiter = futures.Future(loop=self._loop)
            self._puback_waiters[packet_id] = waiter
            yield from waiter
            inflight_message.puback = waiter.result()
            inflight_message.acknowledge()
            del self._puback_waiters[packet_id]
        if qos == 0x02:
            # Wait for PUBREC
            waiter = futures.Future(loop=self._loop)
            self._pubrec_waiters[packet_id] = waiter
            yield from waiter
            inflight_message.pubrec = waiter.result()
            del self._pubrec_waiters[packet_id]
            inflight_message.receive()

            # Send pubrel
            pubrel = PubrelPacket.build(packet_id)
            yield from self.outgoing_queue.put(pubrel)
            inflight_message.pubrel = pubrel
            inflight_message.release()

            # Wait for pubcomp
            waiter = futures.Future(loop=self._loop)
            self._pubcomp_waiters[packet_id] = waiter
            yield from waiter
            inflight_message.pubcomp = waiter.result()
            del self._pubcomp_waiters[packet_id]
            inflight_message.complete()

        del self.session.inflight_out[packet_id]
        return inflight_message

    @asyncio.coroutine
    def stop(self):
        self._running = False
        self.session.reader.feed_eof()
        yield from self.outgoing_queue.put("STOP")
        yield from asyncio.wait([self._writer_task, self._reader_task], loop=self._loop)

    @asyncio.coroutine
    def _reader_coro(self):
        self.logger.debug("Starting reader coro")
        while self._running:
            try:
                self._reader_ready.set()
                keepalive_timeout = self.session.keep_alive
                if keepalive_timeout <= 0:
                    keepalive_timeout = None
                fixed_header = yield from asyncio.wait_for(MQTTFixedHeader.from_stream(self.session.reader), keepalive_timeout)
                if fixed_header:
                    cls = packet_class(fixed_header)
                    packet = yield from cls.from_stream(self.session.reader, fixed_header=fixed_header)
                    self.logger.debug(" <-in-- " + repr(packet))

                    if packet.fixed_header.packet_type == PacketType.CONNACK:
                        asyncio.Task(self.handle_connack(packet))
                    elif packet.fixed_header.packet_type == PacketType.SUBSCRIBE:
                        asyncio.Task(self.handle_subscribe(packet))
                    elif packet.fixed_header.packet_type == PacketType.UNSUBSCRIBE:
                        asyncio.Task(self.handle_unsubscribe(packet))
                    elif packet.fixed_header.packet_type == PacketType.SUBACK:
                        asyncio.Task(self.handle_suback(packet))
                    elif packet.fixed_header.packet_type == PacketType.UNSUBACK:
                        asyncio.Task(self.handle_unsuback(packet))
                    elif packet.fixed_header.packet_type == PacketType.PUBACK:
                        asyncio.Task(self.handle_puback(packet))
                    elif packet.fixed_header.packet_type == PacketType.PUBREC:
                        asyncio.Task(self.handle_pubrec(packet))
                    elif packet.fixed_header.packet_type == PacketType.PUBREL:
                        asyncio.Task(self.handle_pubrel(packet))
                    elif packet.fixed_header.packet_type == PacketType.PUBCOMP:
                        asyncio.Task(self.handle_pubcomp(packet))
                    elif packet.fixed_header.packet_type == PacketType.PINGREQ:
                        asyncio.Task(self.handle_pingreq(packet))
                    elif packet.fixed_header.packet_type == PacketType.PINGRESP:
                        asyncio.Task(self.handle_pingresp(packet))
                    elif packet.fixed_header.packet_type == PacketType.PUBLISH:
                        asyncio.Task(self.handle_publish(packet))
                    elif packet.fixed_header.packet_type == PacketType.DISCONNECT:
                        asyncio.Task(self.handle_disconnect(packet))
                    elif packet.fixed_header.packet_type == PacketType.CONNECT:
                        asyncio.Task(self.handle_connect(packet))
                    else:
                        self.logger.warn("Unhandled packet type: %s" % packet.fixed_header.packet_type)
                else:
                    self.logger.debug("No more data, stopping reader coro")
                    break
            except asyncio.TimeoutError:
                self.logger.debug("Input stream read timeout")
                self.handle_read_timeout()
            except NoDataException as nde:
                self.logger.debug("No data available")
            except Exception as e:
                self.logger.warn("Unhandled exception in reader coro: %s" % e)
                break
        self.logger.debug("Reader coro stopped")

    @asyncio.coroutine
    def _writer_coro(self):
        self.logger.debug("Starting writer coro")
        while self._running:
            try:
                self._writer_ready.set()
                keepalive_timeout = self.session.keep_alive
                if keepalive_timeout <= 0:
                    keepalive_timeout = None
                packet = yield from asyncio.wait_for(self.outgoing_queue.get(), keepalive_timeout)
                if not isinstance(packet, MQTTPacket):
                    self.logger.debug("Writer interruption")
                    break
                yield from packet.to_stream(self.session.writer)
                self.logger.debug(" -out-> " + repr(packet))
                yield from self.session.writer.drain()
                #self.outgoing_queue.task_done() # to be used with Python 3.5
            except asyncio.TimeoutError as ce:
                self.logger.debug("Output queue get timeout")
                if self._running:
                    self.handle_write_timeout()
            except Exception as e:
                self.logger.warn("Unhandled exception in writer coro: %s" % e)
                break
        self.logger.debug("Writer coro stopping")
        # Flush queue before stopping
        if not self.outgoing_queue.empty():
            while True:
                try:
                    packet = self.outgoing_queue.get_nowait()
                    if not isinstance(packet, MQTTPacket):
                        break
                    yield from packet.to_stream(self.session.writer)
                    self.logger.debug(" -out-> " + repr(packet))
                except asyncio.QueueEmpty:
                    break
                except Exception as e:
                    self.logger.warn("Unhandled exception in writer coro: %s" % e)
        self.logger.debug("Writer coro stopped")

    @asyncio.coroutine
    def mqtt_deliver_next_message(self):
        inflight_message = yield from self.delivered_message.get()
        return inflight_message

    def handle_write_timeout(self):
        self.logger.warn('write timeout unhandled')

    def handle_read_timeout(self):
        self.logger.warn('read timeout unhandled')

    @asyncio.coroutine
    def handle_connack(self, connack: ConnackPacket):
        self.logger.warn('CONNACK unhandled')

    @asyncio.coroutine
    def handle_connect(self, connect: ConnectPacket):
        self.logger.warn('CONNECT unhandled')

    @asyncio.coroutine
    def handle_subscribe(self, subscribe: SubscribePacket):
        self.logger.warn('SUBSCRIBE unhandled')

    @asyncio.coroutine
    def handle_unsubscribe(self, subscribe: UnsubscribePacket):
        self.logger.warn('UNSUBSCRIBE unhandled')

    @asyncio.coroutine
    def handle_suback(self, suback: SubackPacket):
        self.logger.warn('SUBACK unhandled')

    @asyncio.coroutine
    def handle_unsuback(self, unsuback: UnsubackPacket):
        self.logger.warn('UNSUBACK unhandled')

    @asyncio.coroutine
    def handle_pingresp(self, pingresp: PingRespPacket):
        self.logger.warn('PINGRESP unhandled')

    @asyncio.coroutine
    def handle_pingreq(self, pingreq: PingReqPacket):
        self.logger.warn('PINGREQ unhandled')

    @asyncio.coroutine
    def handle_disconnect(self, disconnect: DisconnectPacket):
        self.logger.warn('DISCONNECT unhandled')

    @asyncio.coroutine
    def handle_puback(self, puback: PubackPacket):
        packet_id = puback.variable_header.packet_id
        try:
            waiter = self._puback_waiters[packet_id]
            waiter.set_result(puback)
        except KeyError as ke:
            self.logger.warn("Received PUBACK for unknown pending subscription with Id: %s" % packet_id)

    @asyncio.coroutine
    def handle_pubrec(self, pubrec: PubrecPacket):
        packet_id = pubrec.variable_header.packet_id
        try:
            waiter = self._pubrec_waiters[packet_id]
            waiter.set_result(pubrec)
        except KeyError as ke:
            self.logger.warn("Received PUBREC for unknown pending subscription with Id: %s" % packet_id)

    @asyncio.coroutine
    def handle_pubcomp(self, pubcomp: PubcompPacket):
        packet_id = pubcomp.variable_header.packet_id
        try:
            waiter = self._pubcomp_waiters[packet_id]
            waiter.set_result(pubcomp)
        except KeyError as ke:
            self.logger.warn("Received PUBCOMP for unknown pending subscription with Id: %s" % packet_id)

    @asyncio.coroutine
    def handle_pubrel(self, pubrel: PubrecPacket):
        packet_id = pubrel.variable_header.packet_id
        try:
            waiter = self._pubrel_waiters[packet_id]
            waiter.set_result(pubrel)
        except KeyError as ke:
            self.logger.warn("Received PUBREL for unknown pending subscription with Id: %s" % packet_id)

    @asyncio.coroutine
    def handle_publish(self, publish : PublishPacket):
        inflight_message = None
        packet_id = publish.variable_header.packet_id
        qos = (publish.fixed_header.flags >> 1) & 0x03

        if qos == 0:
            inflight_message = InFlightMessage(publish, qos)
            yield from self.delivered_message.put(inflight_message)
        else:
            if packet_id in self.session.inflight_in:
                inflight_message = self.session.inflight_in[packet_id]
            else:
                inflight_message = InFlightMessage(publish, qos)
                self.session.inflight_in[packet_id] = inflight_message
                inflight_message.publish()

            if qos == 1:
                puback = PubackPacket.build(packet_id)
                yield from self.outgoing_queue.put(puback)
                inflight_message.acknowledge()
            if qos == 2:
                pubrec = PubrecPacket.build(packet_id)
                yield from self.outgoing_queue.put(pubrec)
                inflight_message.receive()
                waiter = futures.Future(loop=self._loop)
                self._pubrel_waiters[packet_id] = waiter
                yield from waiter
                inflight_message.pubrel = waiter.result()
                del self._pubrel_waiters[packet_id]
                inflight_message.release()
                pubcomp = PubcompPacket.build(packet_id)
                yield from self.outgoing_queue.put(pubcomp)
                inflight_message.complete()
            yield from self.delivered_message.put(inflight_message)
            del self.session.inflight_in[packet_id]
<|endoftext|>