<commit_before>import urllib
import urllib2
import uuid
import json
import re

from datetime import date
from functools import wraps

from pymongo import Connection

from tornado import (
    httpserver,
    httpclient,
    ioloop,
    options,
    web,
    gen
    )

from tornado.options import (
    define,
    options
    )

import tornado
import asyncmongo

define("host", default='localhost', help="run on the given host")
define("port", default=80, help="run on the given port", type=int)
define("mongodb_port", default=27017, help="run MongoDB on the given port", type=int)
define("mongodb_host", default='localhost', help="run MongoDB on the given hostname")
define("mongodb_database", default='analytics', help="Record accesses on the given database")
define("mongodb_max_connections", default=2000, help="run MongoDB with the given max connections", type=int)
define("mongodb_max_cached", default=0, help="run MongoDB with the given max cached", type=int)
define("mongodb_max_usage", default=0, help="run MongoDB with the given max cached", type=int)
define("mongodb_min_cached", default=1000, help="run MongoDB with the given min cached", type=int)
define("allowed_hosts", default=None, help="indicates a txt file with hostnames allowed to post data.", type=str)
define("manager_token", default=str(uuid.uuid4()), help="indicates a token that must be used to manage allowed_tokens.", type=str)
define("broadcast_timeout", default=1, help="indicates the max timeout in seconds that the broadcast must finish.", type=str)

LIMIT = 20
REGEX_ISSN = re.compile("^[0-9]{4}-[0-9]{3}[0-9xX]$")
REGEX_ISSUE = re.compile("^[0-9]{4}-[0-9]{3}[0-9xX][0-2][0-9]{3}[0-9]{4}$")
REGEX_ARTICLE = re.compile("^S[0-9]{4}-[0-9]{3}[0-9xX][0-2][0-9]{3}[0-9]{4}[0-9]{5}$")
REGEX_FBPE = re.compile("^S[0-9]{4}-[0-9]{3}[0-9xX]\([0-9]{2}\)[0-9]{8}$")

def get_next(endpoint, total, limit, offset, fltr=None):

    offset += limit
    if offset >= total:
        return None

    fltrs = ''
    if fltr:
        fltrs += '&%s' % '&'.join(set(['='.join([key, value]) for key, value in fltr.items()]))

    return '/api/v1/%s?offset=%s%s' % (endpoint, offset, fltrs)

def get_previous(endpoint, total, limit, offset, fltr=None):
    offset -= limit
    if offset < 0:
        return None

    fltrs = ''
    if fltr:
        fltrs += '&%s' % '&'.join(set(['='.join([key, value]) for key, value in fltr.items()]))

    return '/api/v1/%s?offset=%s%s' % (endpoint, offset, fltrs)

def authenticated(func):
    @wraps(func)
    def wrapper(self, *args, **kwargs):

        api_token = self.get_argument('api_token', None)

        if not api_token:
            raise tornado.web.HTTPError(401)

        if api_token != self.application.manager_token:
            raise tornado.web.HTTPError(401)
        else:
            r = func(self, *args, **kwargs)
            return r

    return wrapper

class Application(tornado.web.Application):
    def __init__(self):
        handlers = [
            (r"^/?", RootHandler),
            (r"^/api/v1/?", EndpointsHandler),
            (r"^/api/v1/general/?", GeneralHandler),
            (r"^/api/v1/general/bulk/?", BulkGeneralHandler),
            (r"^/api/v1/journals/?", JournalHandler),
            (r"^/api/v1/journals/(?P<code>.*?)/?", JournalHandler),
            (r"^/api/v1/issues/?", IssueHandler),
            (r"^/api/v1/issues/(?P<code>.*?)/?", IssueHandler),
            (r"^/api/v1/articles/?", ArticleHandler),
            (r"^/api/v1/articles/(?P<code>.*?)/?", ArticleHandler),
        ]

        # Creating Indexes without asyncmongo.
        coll = Connection(options.mongodb_host, options.mongodb_port)[options.mongodb_database]['accesses']
        coll.ensure_index('code')
        coll.ensure_index('page')
        coll.ensure_index('type')

        self.db = asyncmongo.Client(
            pool_id='accesses',
            host=options.mongodb_host,
            port=options.mongodb_port,
            maxcached=options.mongodb_max_cached,
            maxconnections=options.mongodb_max_connections,
            dbname=options.mongodb_database
        )

        # Loading Alpha-3 Country codes for regions definition
        self.alpha3 = {}
        with open('iso_alpha3.txt') as f:
            for line in f:
                line = line.split(';')
                self.alpha3[line[0].lower()] = line[1]

        # Local is the default the default way that ratchet works.
        self.broadcast_timeout = options.broadcast_timeout
        self.manager_token = options.manager_token
        self.api_style = 'local'

        tornado.web.Application.__init__(self, handlers)

class BaseHandler(tornado.web.RequestHandler):
    def set_default_headers(self):
        self.set_header("Access-Control-Allow-Methods", "GET, POST")
        self.set_header("Access-Control-Allow-Origin", "*")

class RootHandler(tornado.web.RequestHandler):
    def get(self):

        self.write("Ratchet API")
        self.finish()

class GeneralHandler(tornado.web.RequestHandler):

    rdata = {}
    total = None

    def _on_count_get_response(self, response, error):
        if error:
            raise tornado.web.HTTPError(500)

        self.total = response['n']

    def _on_get_response(self, response, error):
        if error:
            raise tornado.web.HTTPError(500)

        meta = self.rdata.setdefault('meta', {})
        meta['limit'] = LIMIT
        meta['offset'] = self.get_argument('offset', 0)
        self.rdata['objects'] = response

        while not self.total:
            pass

        meta['total'] = self.total
        meta['next'] = get_next('general', meta['total'], meta['limit'], int(meta['offset']), self.query)
        meta['previous'] = get_previous('general', meta['total'], meta['limit'], int(meta['offset']), self.query)

        self.write(json.dumps(self.rdata))
        self.finish()

    @property
    def db(self):
        self._db = self.application.db
        return self._db

    @authenticated
    @tornado.web.addslash
    def post(self):
        code = self.get_argument('code')
        page = self.get_argument('page', None)
        type_doc = self.get_argument('type_doc', None)
        access_date = self.get_argument('access_date', None)

        if access_date:
            day = access_date[8:10]
            month = access_date[5:7]
            year = access_date[0:4]
        else:
            day = '%02d' % date.today().day
            month = '%02d' % date.today().month
            year = '%02d' % date.today().year

        lday = 'y{0}.m{1}.d{2}'.format(year, month, day)
        lmonth = 'y{0}.m{1}.total'.format(year, month)
        lyear = 'y{0}.total'.format(year)

        inc = {
            lday: 1,
            lmonth: 1,
            lyear: 1,
            'total': 1
        }

        dat = {}
        if page:
            inc[page + '.' + lday] = 1
            inc[page + '.' + lmonth] = 1
            inc[page + '.' + lyear] = 1
            inc[page + '.total'] = 1
            dat['$inc'] = inc

        if type_doc:
            dat['$set'] = {'type': type_doc}

        self.db.accesses.update(
            {'code': code}, dat,
            safe=False,
            upsert=True)

    @tornado.web.asynchronous
    @tornado.gen.engine
    @tornado.web.addslash
    def get(self):
        code = self.get_argument('code', None)
        type_doc = self.get_argument('type', None)

        self.query = {}

        if type_doc:
            self.query = {"type": type_doc}

        if code:
            self.query = {"code": code}

        command = {
            'count': 'accesses',
            'query': self.query
        }

        self.db.command(command, callback=self._on_count_get_response)

        self.db.accesses.find(self.query, {"_id": 0}, limit=LIMIT, callback=self._on_get_response)

class BulkGeneralHandler(tornado.web.RequestHandler):

    @property
    def db(self):
        self._db = self.application.db
        return self._db

    @authenticated
    def post(self):
        data = self.get_argument('data', 'No data received')

        data = json.loads(data)

        code = data['code']

        if 'journal' in data:
            journal = data['journal']
            del data['journal']

        if 'issue' in data:
            issue = data['issue']
            del data['issue']

        include_set = {
                        'journal': journal,
                        'issue': issue,
                      }

        del data['code']

        self.db.accesses.update(
            {'code': code}, {
                '$set': include_set,
                '$inc': data
                },
            safe=False,
            upsert=True)

class EndpointsHandler(tornado.web.RequestHandler):

    rdata = {}

    @tornado.web.asynchronous
    @tornado.gen.engine
    @tornado.web.addslash
    def get(self, code=None):

        endpoints = ['general', 'journals', 'issues', 'articles']

        available_endpoints = {}
        for endpoint in endpoints:
            available_endpoints[endpoint] = {'list_endpoint': '/api/v1/%s/' % endpoint}

        self.write(json.dumps(available_endpoints))
        self.finish()

class JournalHandler(tornado.web.RequestHandler):

    rdata = {}
    total = None

    def _on_count_get_response(self, response, error):
        if error:
            raise tornado.web.HTTPError(500)

        self.total = response['n']

    def _on_get_response(self, response, error):
        if error:
            raise tornado.web.HTTPError(500)

        meta = self.rdata.setdefault('meta', {})
        meta['limit'] = LIMIT
        meta['offset'] = self.get_argument('offset', 0)
        self.rdata['objects'] = response

        while not self.total:
            pass

        meta['total'] = self.total
        meta['next'] = get_next('journals', meta['total'], meta['limit'], int(meta['offset']))
        meta['previous'] = get_previous('journals', meta['total'], meta['limit'], int(meta['offset']))

        self.write(json.dumps(self.rdata))
        self.finish()

    @property
    def db(self):
        self._db = self.application.db
        return self._db

    @tornado.web.asynchronous
    @tornado.gen.engine
    @tornado.web.addslash
    def get(self, code=None):
        offset = int(self.get_argument('offset', 0))

        query = {'type': 'journal'}

        if code:
            if REGEX_ISSN.search(code):
                query['code'] = code
            else:
                raise tornado.web.HTTPError(400)

        command = {
            'count': 'accesses',
            'query': query
        }

        self.db.command(command, callback=self._on_count_get_response)

        self.db.accesses.find(
            query,
            {"_id": 0},
            limit=LIMIT,
            skip=offset,
            sort=[('total', -1)],
            callback=self._on_get_response
        )

class IssueHandler(tornado.web.RequestHandler):

    rdata = {}
    total = None

    def _on_count_get_response(self, response, error):
        if error:
            raise tornado.web.HTTPError(500)

        self.total = response['n']

    def _on_get_response(self, response, error):
        if error:
            raise tornado.web.HTTPError(500)

        meta = self.rdata.setdefault('meta', {})
        meta['limit'] = LIMIT
        meta['offset'] = self.get_argument('offset', 0)
        self.rdata['objects'] = response

        while not self.total:
            pass

        meta['total'] = self.total
        meta['next'] = get_next('journals', meta['total'], meta['limit'], int(meta['offset']))
        meta['previous'] = get_previous('journals', meta['total'], meta['limit'], int(meta['offset']))

        self.write(json.dumps(self.rdata))

        self.finish()

    @property
    def db(self):
        self._db = self.application.db
        return self._db

    @tornado.web.asynchronous
    @tornado.gen.engine
    @tornado.web.addslash
    def get(self, code=None):
        offset = int(self.get_argument('offset', 0))

        query = {'type': 'toc'}

        if code:
            if REGEX_ISSUE.search(code):
                query['code'] = code
            else:
                raise tornado.web.HTTPError(400)

        command = {
            'count': 'accesses',
            'query': query
        }

        self.db.command(command, callback=self._on_count_get_response)

        self.db.accesses.find(
            query,
            {"_id": 0},
            limit=LIMIT,
            skip=offset,
            sort=[('total', -1)],
            callback=self._on_get_response
        )

class ArticleHandler(tornado.web.RequestHandler):

    rdata = {}
    total = None

    def _on_count_get_response(self, response, error):
        if error:
            raise tornado.web.HTTPError(500)

        self.total = response['n']

    def _on_get_response(self, response, error):
        if error:
            raise tornado.web.HTTPError(500)

        meta = self.rdata.setdefault('meta', {})
        meta['limit'] = LIMIT
        meta['offset'] = self.get_argument('offset', 0)
        self.rdata['objects'] = response

        while not self.total:
            pass

        meta['total'] = self.total
        meta['next'] = get_next('journals', meta['total'], meta['limit'], int(meta['offset']))
        meta['previous'] = get_previous('journals', meta['total'], meta['limit'], int(meta['offset']))

        self.write(json.dumps(self.rdata))

        self.finish()

    @property
    def db(self):
        self._db = self.application.db
        return self._db

    @tornado.web.asynchronous
    @tornado.gen.engine
    @tornado.web.addslash
    def get(self, code=None):
        offset = int(self.get_argument('offset', 0))

        query = {'type': 'article'}

        if code:
            if REGEX_ARTICLE.search(code) or REGEX_FBPE.search(code):
                query['code'] = code
            else:
                raise tornado.web.HTTPError(400)

        command = {
            'count': 'accesses',
            'query': query
        }

        self.db.command(command, callback=self._on_count_get_response)

        self.db.accesses.find(
            query,
            {"_id": 0},
            limit=LIMIT,
            skip=offset,
            sort=[('total', -1)],
            callback=self._on_get_response
        )

if __name__ == '__main__':
    tornado.options.parse_command_line()
    http_server = tornado.httpserver.HTTPServer(
        Application(),
        no_keep_alive=True
    )
    http_server.listen(options.port, address=options.host)
    tornado.ioloop.IOLoop.instance().start()
<commit_msg>Ajustes em parametrização de endpoint general<commit_after>import urllib
import urllib2
import uuid
import json
import re

from datetime import date
from functools import wraps

from pymongo import Connection

from tornado import (
    httpserver,
    httpclient,
    ioloop,
    options,
    web,
    gen
    )

from tornado.options import (
    define,
    options
    )

import tornado
import asyncmongo

define("host", default='localhost', help="run on the given host")
define("port", default=80, help="run on the given port", type=int)
define("mongodb_port", default=27017, help="run MongoDB on the given port", type=int)
define("mongodb_host", default='localhost', help="run MongoDB on the given hostname")
define("mongodb_database", default='analytics', help="Record accesses on the given database")
define("mongodb_max_connections", default=2000, help="run MongoDB with the given max connections", type=int)
define("mongodb_max_cached", default=0, help="run MongoDB with the given max cached", type=int)
define("mongodb_max_usage", default=0, help="run MongoDB with the given max cached", type=int)
define("mongodb_min_cached", default=1000, help="run MongoDB with the given min cached", type=int)
define("allowed_hosts", default=None, help="indicates a txt file with hostnames allowed to post data.", type=str)
define("manager_token", default=str(uuid.uuid4()), help="indicates a token that must be used to manage allowed_tokens.", type=str)
define("broadcast_timeout", default=1, help="indicates the max timeout in seconds that the broadcast must finish.", type=str)

LIMIT = 20
REGEX_ISSN = re.compile("^[0-9]{4}-[0-9]{3}[0-9xX]$")
REGEX_ISSUE = re.compile("^[0-9]{4}-[0-9]{3}[0-9xX][0-2][0-9]{3}[0-9]{4}$")
REGEX_ARTICLE = re.compile("^S[0-9]{4}-[0-9]{3}[0-9xX][0-2][0-9]{3}[0-9]{4}[0-9]{5}$")
REGEX_FBPE = re.compile("^S[0-9]{4}-[0-9]{3}[0-9xX]\([0-9]{2}\)[0-9]{8}$")

def get_next(endpoint, total, limit, offset, fltr=None):

    offset += limit
    if offset >= total:
        return None

    fltrs = ''
    if fltr:
        fltrs += '&%s' % '&'.join(set(['='.join([key, value]) for key, value in fltr.items()]))

    return '/api/v1/%s/?offset=%s%s' % (endpoint, offset, fltrs)

def get_previous(endpoint, total, limit, offset, fltr=None):
    offset -= limit
    if offset < 0:
        return None

    fltrs = ''
    if fltr:
        fltrs += '&%s' % '&'.join(set(['='.join([key, value]) for key, value in fltr.items()]))

    return '/api/v1/%s/?offset=%s%s' % (endpoint, offset, fltrs)

def authenticated(func):
    @wraps(func)
    def wrapper(self, *args, **kwargs):

        api_token = self.get_argument('api_token', None)

        if not api_token:
            raise tornado.web.HTTPError(401)

        if api_token != self.application.manager_token:
            raise tornado.web.HTTPError(401)
        else:
            r = func(self, *args, **kwargs)
            return r

    return wrapper

class Application(tornado.web.Application):
    def __init__(self):
        handlers = [
            (r"^/?", RootHandler),
            (r"^/api/v1/?", EndpointsHandler),
            (r"^/api/v1/general/?", GeneralHandler),
            (r"^/api/v1/general/bulk/?", BulkGeneralHandler),
            (r"^/api/v1/journals/?", JournalHandler),
            (r"^/api/v1/journals/(?P<code>.*?)/?", JournalHandler),
            (r"^/api/v1/issues/?", IssueHandler),
            (r"^/api/v1/issues/(?P<code>.*?)/?", IssueHandler),
            (r"^/api/v1/articles/?", ArticleHandler),
            (r"^/api/v1/articles/(?P<code>.*?)/?", ArticleHandler),
        ]

        # Creating Indexes without asyncmongo.
        coll = Connection(options.mongodb_host, options.mongodb_port)[options.mongodb_database]['accesses']
        coll.ensure_index('code')
        coll.ensure_index('page')
        coll.ensure_index('type')

        self.db = asyncmongo.Client(
            pool_id='accesses',
            host=options.mongodb_host,
            port=options.mongodb_port,
            maxcached=options.mongodb_max_cached,
            maxconnections=options.mongodb_max_connections,
            dbname=options.mongodb_database
        )

        # Loading Alpha-3 Country codes for regions definition
        self.alpha3 = {}
        with open('iso_alpha3.txt') as f:
            for line in f:
                line = line.split(';')
                self.alpha3[line[0].lower()] = line[1]

        # Local is the default the default way that ratchet works.
        self.broadcast_timeout = options.broadcast_timeout
        self.manager_token = options.manager_token
        self.api_style = 'local'

        tornado.web.Application.__init__(self, handlers)

class BaseHandler(tornado.web.RequestHandler):
    def set_default_headers(self):
        self.set_header("Access-Control-Allow-Methods", "GET, POST")
        self.set_header("Access-Control-Allow-Origin", "*")

class RootHandler(tornado.web.RequestHandler):
    def get(self):

        self.write("Ratchet API")
        self.finish()

class GeneralHandler(tornado.web.RequestHandler):

    rdata = {}
    total = None

    def _on_count_get_response(self, response, error):
        if error:
            raise tornado.web.HTTPError(500)

        self.total = response['n']

    def _on_get_response(self, response, error):
        if error:
            raise tornado.web.HTTPError(500)

        meta = self.rdata.setdefault('meta', {})
        meta['limit'] = LIMIT
        meta['offset'] = self.get_argument('offset', 0)
        self.rdata['objects'] = response

        while not self.total:
            pass

        meta['total'] = self.total
        meta['next'] = get_next('general', meta['total'], meta['limit'], int(meta['offset']), self.query)
        meta['previous'] = get_previous('general', meta['total'], meta['limit'], int(meta['offset']), self.query)

        self.write(json.dumps(self.rdata))
        self.finish()

    @property
    def db(self):
        self._db = self.application.db
        return self._db

    @authenticated
    @tornado.web.addslash
    def post(self):
        code = self.get_argument('code')
        page = self.get_argument('page', None)
        type_doc = self.get_argument('type_doc', None)
        access_date = self.get_argument('access_date', None)

        if access_date:
            day = access_date[8:10]
            month = access_date[5:7]
            year = access_date[0:4]
        else:
            day = '%02d' % date.today().day
            month = '%02d' % date.today().month
            year = '%02d' % date.today().year

        lday = 'y{0}.m{1}.d{2}'.format(year, month, day)
        lmonth = 'y{0}.m{1}.total'.format(year, month)
        lyear = 'y{0}.total'.format(year)

        inc = {
            lday: 1,
            lmonth: 1,
            lyear: 1,
            'total': 1
        }

        dat = {}
        if page:
            inc[page + '.' + lday] = 1
            inc[page + '.' + lmonth] = 1
            inc[page + '.' + lyear] = 1
            inc[page + '.total'] = 1
            dat['$inc'] = inc

        if type_doc:
            dat['$set'] = {'type': type_doc}

        self.db.accesses.update(
            {'code': code}, dat,
            safe=False,
            upsert=True)

    @tornado.web.asynchronous
    @tornado.gen.engine
    @tornado.web.addslash
    def get(self):
        code = self.get_argument('code', None)
        type_doc = self.get_argument('type', None)

        self.query = {}

        if type_doc:
            self.query = {"type": type_doc}

        if code:
            self.query = {"code": code}

        command = {
            'count': 'accesses',
            'query': self.query
        }

        self.db.command(command, callback=self._on_count_get_response)

        self.db.accesses.find(self.query, {"_id": 0}, limit=LIMIT, callback=self._on_get_response)

class BulkGeneralHandler(tornado.web.RequestHandler):

    @property
    def db(self):
        self._db = self.application.db
        return self._db

    @authenticated
    def post(self):
        data = self.get_argument('data', 'No data received')

        data = json.loads(data)

        code = data['code']

        if 'journal' in data:
            journal = data['journal']
            del data['journal']

        if 'issue' in data:
            issue = data['issue']
            del data['issue']

        include_set = {
                        'journal': journal,
                        'issue': issue,
                      }

        del data['code']

        self.db.accesses.update(
            {'code': code}, {
                '$set': include_set,
                '$inc': data
                },
            safe=False,
            upsert=True)

class EndpointsHandler(tornado.web.RequestHandler):

    rdata = {}

    @tornado.web.asynchronous
    @tornado.gen.engine
    @tornado.web.addslash
    def get(self, code=None):

        endpoints = ['general', 'journals', 'issues', 'articles']

        available_endpoints = {}
        for endpoint in endpoints:
            available_endpoints[endpoint] = {'list_endpoint': '/api/v1/%s/' % endpoint}

        self.write(json.dumps(available_endpoints))
        self.finish()

class JournalHandler(tornado.web.RequestHandler):

    rdata = {}
    total = None

    def _on_count_get_response(self, response, error):
        if error:
            raise tornado.web.HTTPError(500)

        self.total = response['n']

    def _on_get_response(self, response, error):
        if error:
            raise tornado.web.HTTPError(500)

        meta = self.rdata.setdefault('meta', {})
        meta['limit'] = LIMIT
        meta['offset'] = self.get_argument('offset', 0)
        self.rdata['objects'] = response

        while not self.total:
            pass

        meta['total'] = self.total
        meta['next'] = get_next('journals', meta['total'], meta['limit'], int(meta['offset']))
        meta['previous'] = get_previous('journals', meta['total'], meta['limit'], int(meta['offset']))

        self.write(json.dumps(self.rdata))
        self.finish()

    @property
    def db(self):
        self._db = self.application.db
        return self._db

    @tornado.web.asynchronous
    @tornado.gen.engine
    @tornado.web.addslash
    def get(self, code=None):
        offset = int(self.get_argument('offset', 0))

        query = {'type': 'journal'}

        if code:
            if REGEX_ISSN.search(code):
                query['code'] = code
            else:
                raise tornado.web.HTTPError(400)

        command = {
            'count': 'accesses',
            'query': query
        }

        self.db.command(command, callback=self._on_count_get_response)

        self.db.accesses.find(
            query,
            {"_id": 0},
            limit=LIMIT,
            skip=offset,
            sort=[('total', -1)],
            callback=self._on_get_response
        )

class IssueHandler(tornado.web.RequestHandler):

    rdata = {}
    total = None

    def _on_count_get_response(self, response, error):
        if error:
            raise tornado.web.HTTPError(500)

        self.total = response['n']

    def _on_get_response(self, response, error):
        if error:
            raise tornado.web.HTTPError(500)

        meta = self.rdata.setdefault('meta', {})
        meta['limit'] = LIMIT
        meta['offset'] = self.get_argument('offset', 0)
        self.rdata['objects'] = response

        while not self.total:
            pass

        meta['total'] = self.total
        meta['next'] = get_next('journals', meta['total'], meta['limit'], int(meta['offset']))
        meta['previous'] = get_previous('journals', meta['total'], meta['limit'], int(meta['offset']))

        self.write(json.dumps(self.rdata))

        self.finish()

    @property
    def db(self):
        self._db = self.application.db
        return self._db

    @tornado.web.asynchronous
    @tornado.gen.engine
    @tornado.web.addslash
    def get(self, code=None):
        offset = int(self.get_argument('offset', 0))

        query = {'type': 'toc'}

        if code:
            if REGEX_ISSUE.search(code):
                query['code'] = code
            else:
                raise tornado.web.HTTPError(400)

        command = {
            'count': 'accesses',
            'query': query
        }

        self.db.command(command, callback=self._on_count_get_response)

        self.db.accesses.find(
            query,
            {"_id": 0},
            limit=LIMIT,
            skip=offset,
            sort=[('total', -1)],
            callback=self._on_get_response
        )

class ArticleHandler(tornado.web.RequestHandler):

    rdata = {}
    total = None

    def _on_count_get_response(self, response, error):
        if error:
            raise tornado.web.HTTPError(500)

        self.total = response['n']

    def _on_get_response(self, response, error):
        if error:
            raise tornado.web.HTTPError(500)

        meta = self.rdata.setdefault('meta', {})
        meta['limit'] = LIMIT
        meta['offset'] = self.get_argument('offset', 0)
        self.rdata['objects'] = response

        while not self.total:
            pass

        meta['total'] = self.total
        meta['next'] = get_next('journals', meta['total'], meta['limit'], int(meta['offset']))
        meta['previous'] = get_previous('journals', meta['total'], meta['limit'], int(meta['offset']))

        self.write(json.dumps(self.rdata))

        self.finish()

    @property
    def db(self):
        self._db = self.application.db
        return self._db

    @tornado.web.asynchronous
    @tornado.gen.engine
    @tornado.web.addslash
    def get(self, code=None):
        offset = int(self.get_argument('offset', 0))

        query = {'type': 'article'}

        if code:
            if REGEX_ARTICLE.search(code) or REGEX_FBPE.search(code):
                query['code'] = code
            else:
                raise tornado.web.HTTPError(400)

        command = {
            'count': 'accesses',
            'query': query
        }

        self.db.command(command, callback=self._on_count_get_response)

        self.db.accesses.find(
            query,
            {"_id": 0},
            limit=LIMIT,
            skip=offset,
            sort=[('total', -1)],
            callback=self._on_get_response
        )

if __name__ == '__main__':
    tornado.options.parse_command_line()
    http_server = tornado.httpserver.HTTPServer(
        Application(),
        no_keep_alive=True
    )
    http_server.listen(options.port, address=options.host)
    tornado.ioloop.IOLoop.instance().start()
<|endoftext|>


<commit_before>from twisted.internet.protocol import DatagramProtocol
import petlib.pack
from petlib.ec import EcGroup, EcPt
from petlib.bn import Bn
from twisted.internet import reactor, defer
import format3
from mixnode import MixNode
import base64
import random
import threading
import codecs
import time
import databaseConnect as dc
import sys
import sqlite3
import io

from twisted.logger import jsonFileLogObserver, Logger
log = Logger(observer=jsonFileLogObserver(io.open("log.json", "a")))

class Provider(MixNode):

    def __init__(self, name, port, host, setup, privk=None, pubk=None):
        """ A class representing a provider. """
        MixNode.__init__(self, name, port, host, setup, privk, pubk)

        self.clientList = []
        self.usersPubs = []
        self.storage = {}
        self.replyBlocks = {}
        self.MAX_RETRIEVE = 10000

    def startProtocol(self):
        print "[%s] > Start protocol." % self.name
        log.info("[%s] > Sstart protocol." % self.name)

        self.transport.write("INFO", ("127.0.0.1", 9998))
        print "[%s] > Request for network info sent." % self.name
        log.info("[%s] > Request for network info sent." % self.name)

        self.run()
        self.d.addCallback(self.turnOnHeartbeats)
        self.d.addErrback(self.errbackHeartbeats)
        self.turnOnReliableUDP()
        self.saveInDB('example.db')

    def stopProtocol(self):
        print "[%s] > Stop protocol" % self.name
        log.info("[%s] > Stop protocol" % self.name)

    def datagramReceived(self, data, (host, port)):
        if data == "PULL_MSG":
            print "[%s] > Provider received pull messages request from (%s, %d)" % (self.name, host, port)
            log.info("[%s] > Provider received pull messages request from (%s, %d)" % (self.name, host, port))
            self.do_PULL((host, port))

        if data[:4] == "RINF":
            print "[%s] > Provider received mixnode metadata informations from bulletin board %s, %d" % (self.name, host, port)
            log.info("[%s] > Provider received mixnode metadata informations from bulletin board %s, %d" % (self.name, host, port))
            dataList = petlib.pack.decode(data[4:])
            for element in dataList:
                self.mixList.append(
                    format3.Mix(
                        element[0],
                        element[1],
                        element[2],
                        element[3]))
        if data[:4] == "UPUB":
            dataList = petlib.pack.decode(data[4:])
            for element in dataList:
                self.usersPubs.append(format3.User(element[0], element[1], element[2], element[3], element[4]))
            print "[%s] > Provider received public information of users registered in the system" % self.name
            log.info("[%s] > Provider received public information of users registered in the system" % self.name)
        if data[:4] == "INFO":
            self.sendInfoMixnet(host, port)
            print "[%s] > Provider received request for information from %s, %d " % (self.name, host, port)
            log.info("[%s] > Provider received request for information from %s, %d " % (self.name, host, port))
        if data[:4] == "ROUT":
            try:
                self.bytesReceived += sys.getsizeof(data[4:])
                idt, msgData = petlib.pack.decode(data[4:])
                self.sendMessage("ACKN"+idt, (host, port))
                self.do_ROUT(msgData, (host, port))
            except Exception, e:
                print "[%s] > ERROR: " % self.name, str(e)
                log.error("[%s] > Error during ROUT received: %s" % (self.name, str(e)))
        if data[:4] == "ACKN":
            if data in self.expectedACK:
                self.expectedACK.remove(data)
        if data[:4] == "PING":
            log.info("[%s] > provider received assign message from client (%s, %d)" % (self.name, host, port))
            self.clientList.append((host, port))

    def do_PULL(self, (host, port)):
        """ Function which responds the pull message request from the client. First, the function checks if the requesting
        client belogs to this provider, next takes all the messages destinated to this client and sends a fixed number of
        messages to the client.

                Args:
                host (str): host of the requesting client,
                port (int): port of the requesting client.
        """
        if port in self.storage.keys():
            if self.storage[port]:
                for _ in range(self.MAX_RETRIEVE):
                    if self.storage[port]:
                        message = self.storage[port].pop(0)
                        self.transport.write("PMSG" + message, (host, port))
                        print "[%s] > Message fetched for user (%s, %d)." % (self.name, host, port)
                        log.info("[%s] > Message fetched for user (%s, %d)." % (self.name, host, port))
            else:
                self.transport.write("NOMSG", (host, port))

        else:
            self.transport.write("NOMSG", (host, port))

    def do_ROUT(self, data, (host, port), tag=None):
        """ Function operates on the received route packet. First, the function decrypts one layer on the packet. Next, if
        it is a forward message adds it to the queue. If it is a message destinated to one of the providers clients, the message
        is saved in the storage, where it awaits for the client to fetch it.

                Args:
                data (str): traversing packet string,
                host (str): host of the sender of the packet
                port (int): port of the sender of the packet
        """
        print "[%s] > Received ROUT message from %s, %d " % (self.name, host, port)
        log.info("[%s] > Received ROUT message from %s, %d " % (self.name, host, port))
        try:
            peeledData = self.mix_operate(self.setup, data)
        except Exception, e:
            log.info("[%s] > error during packet processing." % self.name)
        else:
            if peeledData:
                (xtoPort, xtoHost), msg_forw, idt, delay = peeledData
                if (xtoHost, int(xtoPort)) in self.clientList:
                    self.saveInStorage(xtoPort, msg_forw)
                else:
                    self.addToQueue(
                        ("ROUT" + petlib.pack.encode((idt ,msg_forw)), (xtoHost, xtoPort), idt), delay)

    def saveInStorage(self, key, value):
        """ Function saves a message in the local storage, where it awaits till the client will fetch it.

                Args:
                key (int): clients key,
                value (str): message (encrypted) stored for the client.
        """
        if key not in self.storage.keys():
            self.storage[key] = [petlib.pack.encode((value, time.time()))]
        else:
            self.storage[key].append(petlib.pack.encode((value, time.time())))
        print "[%s] > Saved message for User %s in storage" % (self.name, key)
        log.info("[%s] > Saved message for User %s in storage" % (self.name, key))

    def sendInfoMixnet(self, host, port):
        """ Function forwards the public information about the mixnodes and users in the system to the requesting address.

                Args:
                host (str): requesting host
                port (port): requesting port
        """
        if not self.mixList:
            self.transport.write("EMPT", (host, port))
        else:
            print "> Sending Mixnet Public Info."
            self.transport.write("RINF" + petlib.pack.encode(self.mixList), (host, port))

    def sendInfoUsers(self, host, port):
        print "Sending users info."
        if not self.usersPubs:
            self.transport.write("EMPT", (host, port))
        else:
            print "> Sending Users Public Info."
            self.transport.write("UINF" + petlib.pack.encode(self.usersPubs), (host, port))

    def saveInDB(self, databaseName):
        data = [self.name, self.port, self.host, self.pubk]
        db = sqlite3.connect(databaseName)
        c = db.cursor()
        c.execute('''CREATE TABLE IF NOT EXISTS %s (id INTEGER PRIMARY KEY, name text, port integer, host text, pubk blob)'''%"Providers")
        insertQuery = "INSERT INTO Providers VALUES(?, ?, ?, ?, ?)"
        c.execute(insertQuery, [None, self.name, str(self.port), self.host, sqlite3.Binary(petlib.pack.encode(self.pubk))])
        db.commit()
        db.close()
        print "[%s] > Provider public information saved in database." % self.name
        log.info("[%s] > Provider public information saved in database." % self.name)

<commit_msg>Stopped provided from making db<commit_after>from twisted.internet.protocol import DatagramProtocol
import petlib.pack
from petlib.ec import EcGroup, EcPt
from petlib.bn import Bn
from twisted.internet import reactor, defer
import format3
from mixnode import MixNode
import base64
import random
import threading
import codecs
import time
import databaseConnect as dc
import sys
import sqlite3
import io

from twisted.logger import jsonFileLogObserver, Logger
log = Logger(observer=jsonFileLogObserver(io.open("log.json", "a")))

class Provider(MixNode):

    def __init__(self, name, port, host, setup, privk=None, pubk=None):
        """ A class representing a provider. """
        MixNode.__init__(self, name, port, host, setup, privk, pubk)

        self.clientList = []
        self.usersPubs = []
        self.storage = {}
        self.replyBlocks = {}
        self.MAX_RETRIEVE = 10000

    def startProtocol(self):
        print "[%s] > Start protocol." % self.name
        log.info("[%s] > Sstart protocol." % self.name)

        self.transport.write("INFO", ("127.0.0.1", 9998))
        print "[%s] > Request for network info sent." % self.name
        log.info("[%s] > Request for network info sent." % self.name)

        self.run()
        self.d.addCallback(self.turnOnHeartbeats)
        self.d.addErrback(self.errbackHeartbeats)
        self.turnOnReliableUDP()
        # self.saveInDB('example.db')

    def stopProtocol(self):
        print "[%s] > Stop protocol" % self.name
        log.info("[%s] > Stop protocol" % self.name)

    def datagramReceived(self, data, (host, port)):
        if data == "PULL_MSG":
            print "[%s] > Provider received pull messages request from (%s, %d)" % (self.name, host, port)
            log.info("[%s] > Provider received pull messages request from (%s, %d)" % (self.name, host, port))
            self.do_PULL((host, port))

        if data[:4] == "RINF":
            print "[%s] > Provider received mixnode metadata informations from bulletin board %s, %d" % (self.name, host, port)
            log.info("[%s] > Provider received mixnode metadata informations from bulletin board %s, %d" % (self.name, host, port))
            dataList = petlib.pack.decode(data[4:])
            for element in dataList:
                self.mixList.append(
                    format3.Mix(
                        element[0],
                        element[1],
                        element[2],
                        element[3]))
        if data[:4] == "UPUB":
            dataList = petlib.pack.decode(data[4:])
            for element in dataList:
                self.usersPubs.append(format3.User(element[0], element[1], element[2], element[3], element[4]))
            print "[%s] > Provider received public information of users registered in the system" % self.name
            log.info("[%s] > Provider received public information of users registered in the system" % self.name)
        if data[:4] == "INFO":
            self.sendInfoMixnet(host, port)
            print "[%s] > Provider received request for information from %s, %d " % (self.name, host, port)
            log.info("[%s] > Provider received request for information from %s, %d " % (self.name, host, port))
        if data[:4] == "ROUT":
            try:
                self.bytesReceived += sys.getsizeof(data[4:])
                idt, msgData = petlib.pack.decode(data[4:])
                self.sendMessage("ACKN"+idt, (host, port))
                self.do_ROUT(msgData, (host, port))
            except Exception, e:
                print "[%s] > ERROR: " % self.name, str(e)
                log.error("[%s] > Error during ROUT received: %s" % (self.name, str(e)))
        if data[:4] == "ACKN":
            if data in self.expectedACK:
                self.expectedACK.remove(data)
        if data[:4] == "PING":
            log.info("[%s] > provider received assign message from client (%s, %d)" % (self.name, host, port))
            self.clientList.append((host, port))

    def do_PULL(self, (host, port)):
        """ Function which responds the pull message request from the client. First, the function checks if the requesting
        client belogs to this provider, next takes all the messages destinated to this client and sends a fixed number of
        messages to the client.

                Args:
                host (str): host of the requesting client,
                port (int): port of the requesting client.
        """
        if port in self.storage.keys():
            if self.storage[port]:
                for _ in range(self.MAX_RETRIEVE):
                    if self.storage[port]:
                        message = self.storage[port].pop(0)
                        self.transport.write("PMSG" + message, (host, port))
                        print "[%s] > Message fetched for user (%s, %d)." % (self.name, host, port)
                        log.info("[%s] > Message fetched for user (%s, %d)." % (self.name, host, port))
            else:
                self.transport.write("NOMSG", (host, port))

        else:
            self.transport.write("NOMSG", (host, port))

    def do_ROUT(self, data, (host, port), tag=None):
        """ Function operates on the received route packet. First, the function decrypts one layer on the packet. Next, if
        it is a forward message adds it to the queue. If it is a message destinated to one of the providers clients, the message
        is saved in the storage, where it awaits for the client to fetch it.

                Args:
                data (str): traversing packet string,
                host (str): host of the sender of the packet
                port (int): port of the sender of the packet
        """
        print "[%s] > Received ROUT message from %s, %d " % (self.name, host, port)
        log.info("[%s] > Received ROUT message from %s, %d " % (self.name, host, port))
        try:
            peeledData = self.mix_operate(self.setup, data)
        except Exception, e:
            log.info("[%s] > error during packet processing." % self.name)
        else:
            if peeledData:
                (xtoPort, xtoHost), msg_forw, idt, delay = peeledData
                if (xtoHost, int(xtoPort)) in self.clientList:
                    self.saveInStorage(xtoPort, msg_forw)
                else:
                    self.addToQueue(
                        ("ROUT" + petlib.pack.encode((idt ,msg_forw)), (xtoHost, xtoPort), idt), delay)

    def saveInStorage(self, key, value):
        """ Function saves a message in the local storage, where it awaits till the client will fetch it.

                Args:
                key (int): clients key,
                value (str): message (encrypted) stored for the client.
        """
        if key not in self.storage.keys():
            self.storage[key] = [petlib.pack.encode((value, time.time()))]
        else:
            self.storage[key].append(petlib.pack.encode((value, time.time())))
        print "[%s] > Saved message for User %s in storage" % (self.name, key)
        log.info("[%s] > Saved message for User %s in storage" % (self.name, key))

    def sendInfoMixnet(self, host, port):
        """ Function forwards the public information about the mixnodes and users in the system to the requesting address.

                Args:
                host (str): requesting host
                port (port): requesting port
        """
        if not self.mixList:
            self.transport.write("EMPT", (host, port))
        else:
            print "> Sending Mixnet Public Info."
            self.transport.write("RINF" + petlib.pack.encode(self.mixList), (host, port))

    def sendInfoUsers(self, host, port):
        print "Sending users info."
        if not self.usersPubs:
            self.transport.write("EMPT", (host, port))
        else:
            print "> Sending Users Public Info."
            self.transport.write("UINF" + petlib.pack.encode(self.usersPubs), (host, port))

    def saveInDB(self, databaseName):
        data = [self.name, self.port, self.host, self.pubk]
        db = sqlite3.connect(databaseName)
        c = db.cursor()
        c.execute('''CREATE TABLE IF NOT EXISTS %s (id INTEGER PRIMARY KEY, name text, port integer, host text, pubk blob)'''%"Providers")
        insertQuery = "INSERT INTO Providers VALUES(?, ?, ?, ?, ?)"
        c.execute(insertQuery, [None, self.name, str(self.port), self.host, sqlite3.Binary(petlib.pack.encode(self.pubk))])
        db.commit()
        db.close()
        print "[%s] > Provider public information saved in database." % self.name
        log.info("[%s] > Provider public information saved in database." % self.name)

<|endoftext|>


<commit_before>"""A pure Python package providing the core RDF constructs.

The packages is intended to provide the core RDF types and interfaces
for working with RDF. The package defines a plugin interface for
parsers, stores, and serializers that other packages can use to
implement parsers, stores, and serializers that will plug into the
rdflib package.

The primary interface `rdflib` exposes to work with RDF is
`rdflib.graph.Graph`.

A tiny example:

    >>> from rdflib import Graph, URIRef, Literal

    >>> g = Graph()
    >>> result = g.parse("http://www.w3.org/2000/10/swap/test/meet/blue.rdf")

    >>> print("graph has %s statements." % len(g))
    graph has 4 statements.
    >>>
    >>> for s, p, o in g:
    ...     if (s, p, o) not in g:
    ...         raise Exception("It better be!")

    >>> s = g.serialize(format='nt')
    >>>
    >>> sorted(g) == [
    ...  (URIRef(u'http://meetings.example.com/cal#m1'),
    ...   URIRef(u'http://www.example.org/meeting_organization#homePage'),
    ...   URIRef(u'http://meetings.example.com/m1/hp')),
    ...  (URIRef(u'http://www.example.org/people#fred'),
    ...   URIRef(u'http://www.example.org/meeting_organization#attending'),
    ...   URIRef(u'http://meetings.example.com/cal#m1')),
    ...  (URIRef(u'http://www.example.org/people#fred'),
    ...   URIRef(u'http://www.example.org/personal_details#GivenName'),
    ...   Literal(u'Fred')),
    ...  (URIRef(u'http://www.example.org/people#fred'),
    ...   URIRef(u'http://www.example.org/personal_details#hasEmail'),
    ...   URIRef(u'mailto:fred@example.com'))
    ... ]
    True

"""
__docformat__ = "restructuredtext en"

# The format of the __version__ line is matched by a regex in setup.py
__version__ = "6.0.1a0"
__date__ = "2021-07-21"

__all__ = [
    "URIRef",
    "BNode",
    "Literal",
    "Variable",
    "Namespace",
    "Dataset",
    "Graph",
    "ConjunctiveGraph",
    "CSVW",
    "DC",
    "DCAT",
    "DCTERMS",
    "DOAP",
    "FOAF",
    "ODRL2",
    "ORG",
    "OWL",
    "PROF",
    "PROV",
    "QB",
    "RDF",
    "RDFS",
    "SDO",
    "SH",
    "SKOS",
    "SOSA",
    "SSN",
    "TIME",
    "VOID",
    "XMLNS",
    "XSD",
    "util",
]

import sys
import logging

logger = logging.getLogger(__name__)
_interactive_mode = False
try:
    import __main__

    if (
        not hasattr(__main__, "__file__")
        and sys.stdout is not None
        and sys.stderr.isatty()
    ):
        # show log messages in interactive mode
        _interactive_mode = True
        logger.setLevel(logging.INFO)
        logger.addHandler(logging.StreamHandler())
    del __main__
except ImportError:
    # Main already imported from elsewhere
    import warnings

    warnings.warn("__main__ already imported", ImportWarning)
    del warnings

if _interactive_mode:
    logger.info("RDFLib Version: %s" % __version__)
else:
    logger.debug("RDFLib Version: %s" % __version__)
del _interactive_mode
del sys

NORMALIZE_LITERALS = True
"""
If True - Literals lexical forms are normalized when created.
I.e. the lexical forms is parsed according to data-type, then the
stored lexical form is the re-serialized value that was parsed.

Illegal values for a datatype are simply kept.  The normalized keyword
for Literal.__new__ can override this.

For example:

>>> from rdflib import Literal,XSD
>>> Literal("01", datatype=XSD.int)
rdflib.term.Literal(u'1', datatype=rdflib.term.URIRef(u'http://www.w3.org/2001/XMLSchema#integer'))

This flag may be changed at any time, but will only affect literals
created after that time, previously created literals will remain
(un)normalized.

"""

DAWG_LITERAL_COLLATION = False
"""
DAWG_LITERAL_COLLATION determines how literals are ordered or compared
to each other.

In SPARQL, applying the >,<,>=,<= operators to literals of
incompatible data-types is an error, i.e:

Literal(2)>Literal('cake') is neither true nor false, but an error.

This is a problem in PY3, where lists of Literals of incompatible
types can no longer be sorted.

Setting this flag to True gives you strict DAWG/SPARQL compliance,
setting it to False will order Literals with incompatible datatypes by
datatype URI

In particular, this determines how the rich comparison operators for
Literal work, eq, __neq__, __lt__, etc.
"""

from rdflib.term import URIRef, BNode, Literal, Variable

from rdflib.graph import Dataset, Graph, ConjunctiveGraph

from rdflib import plugin
from rdflib import query

from rdflib.namespace import (
    CSVW,
    DC,
    DCAT,
    DCTERMS,
    DOAP,
    FOAF,
    ODRL2,
    ORG,
    OWL,
    PROF,
    PROV,
    QB,
    RDF,
    RDFS,
    SDO,
    SH,
    SKOS,
    SOSA,
    SSN,
    TIME,
    VOID,
    XMLNS,
    XSD,
    Namespace,
)

# tedious sop to flake8
assert plugin
assert query

from rdflib import util

from rdflib.container import *
<commit_msg>fix: remove log message on import<commit_after>"""A pure Python package providing the core RDF constructs.

The packages is intended to provide the core RDF types and interfaces
for working with RDF. The package defines a plugin interface for
parsers, stores, and serializers that other packages can use to
implement parsers, stores, and serializers that will plug into the
rdflib package.

The primary interface `rdflib` exposes to work with RDF is
`rdflib.graph.Graph`.

A tiny example:

    >>> from rdflib import Graph, URIRef, Literal

    >>> g = Graph()
    >>> result = g.parse("http://www.w3.org/2000/10/swap/test/meet/blue.rdf")

    >>> print("graph has %s statements." % len(g))
    graph has 4 statements.
    >>>
    >>> for s, p, o in g:
    ...     if (s, p, o) not in g:
    ...         raise Exception("It better be!")

    >>> s = g.serialize(format='nt')
    >>>
    >>> sorted(g) == [
    ...  (URIRef(u'http://meetings.example.com/cal#m1'),
    ...   URIRef(u'http://www.example.org/meeting_organization#homePage'),
    ...   URIRef(u'http://meetings.example.com/m1/hp')),
    ...  (URIRef(u'http://www.example.org/people#fred'),
    ...   URIRef(u'http://www.example.org/meeting_organization#attending'),
    ...   URIRef(u'http://meetings.example.com/cal#m1')),
    ...  (URIRef(u'http://www.example.org/people#fred'),
    ...   URIRef(u'http://www.example.org/personal_details#GivenName'),
    ...   Literal(u'Fred')),
    ...  (URIRef(u'http://www.example.org/people#fred'),
    ...   URIRef(u'http://www.example.org/personal_details#hasEmail'),
    ...   URIRef(u'mailto:fred@example.com'))
    ... ]
    True

"""
__docformat__ = "restructuredtext en"

# The format of the __version__ line is matched by a regex in setup.py
__version__ = "6.0.1a0"
__date__ = "2021-07-21"

__all__ = [
    "URIRef",
    "BNode",
    "Literal",
    "Variable",
    "Namespace",
    "Dataset",
    "Graph",
    "ConjunctiveGraph",
    "CSVW",
    "DC",
    "DCAT",
    "DCTERMS",
    "DOAP",
    "FOAF",
    "ODRL2",
    "ORG",
    "OWL",
    "PROF",
    "PROV",
    "QB",
    "RDF",
    "RDFS",
    "SDO",
    "SH",
    "SKOS",
    "SOSA",
    "SSN",
    "TIME",
    "VOID",
    "XMLNS",
    "XSD",
    "util",
]

import sys
import logging

logger = logging.getLogger(__name__)
_interactive_mode = False
try:
    import __main__

    if (
        not hasattr(__main__, "__file__")
        and sys.stdout is not None
        and sys.stderr.isatty()
    ):
        # show log messages in interactive mode
        _interactive_mode = True
        logger.setLevel(logging.INFO)
        logger.addHandler(logging.StreamHandler())
    del __main__
except ImportError:
    # Main already imported from elsewhere
    import warnings

    warnings.warn("__main__ already imported", ImportWarning)
    del warnings

del _interactive_mode
del sys

NORMALIZE_LITERALS = True
"""
If True - Literals lexical forms are normalized when created.
I.e. the lexical forms is parsed according to data-type, then the
stored lexical form is the re-serialized value that was parsed.

Illegal values for a datatype are simply kept.  The normalized keyword
for Literal.__new__ can override this.

For example:

>>> from rdflib import Literal,XSD
>>> Literal("01", datatype=XSD.int)
rdflib.term.Literal(u'1', datatype=rdflib.term.URIRef(u'http://www.w3.org/2001/XMLSchema#integer'))

This flag may be changed at any time, but will only affect literals
created after that time, previously created literals will remain
(un)normalized.

"""

DAWG_LITERAL_COLLATION = False
"""
DAWG_LITERAL_COLLATION determines how literals are ordered or compared
to each other.

In SPARQL, applying the >,<,>=,<= operators to literals of
incompatible data-types is an error, i.e:

Literal(2)>Literal('cake') is neither true nor false, but an error.

This is a problem in PY3, where lists of Literals of incompatible
types can no longer be sorted.

Setting this flag to True gives you strict DAWG/SPARQL compliance,
setting it to False will order Literals with incompatible datatypes by
datatype URI

In particular, this determines how the rich comparison operators for
Literal work, eq, __neq__, __lt__, etc.
"""

from rdflib.term import URIRef, BNode, Literal, Variable

from rdflib.graph import Dataset, Graph, ConjunctiveGraph

from rdflib import plugin
from rdflib import query

from rdflib.namespace import (
    CSVW,
    DC,
    DCAT,
    DCTERMS,
    DOAP,
    FOAF,
    ODRL2,
    ORG,
    OWL,
    PROF,
    PROV,
    QB,
    RDF,
    RDFS,
    SDO,
    SH,
    SKOS,
    SOSA,
    SSN,
    TIME,
    VOID,
    XMLNS,
    XSD,
    Namespace,
)

# tedious sop to flake8
assert plugin
assert query

from rdflib import util

from rdflib.container import *
<|endoftext|>


<commit_before>from trees.models import Stand, ForestProperty, IdbSummary
from django.contrib.gis.gdal import DataSource
from django.contrib.gis.gdal.error import OGRIndexError
from django.conf import settings
from madrona.common.utils import get_logger
from django.db.models import Min, Max, Avg
from django.contrib.gis.geos import GEOSGeometry
from shapely.ops import cascaded_union
import numpy as np
from scipy.spatial import KDTree
import math

logger = get_logger()

class StandImporter:

    def __init__(self, user):
        self.user = user
        self.required_fields = ['name']
        self.optional_fields = ['domspp','rx'] # model must provide defaults!

    def _validate_field_mapping(self, layer, field_mapping):
        fields = layer.fields
        if not field_mapping:
            field_mapping = {}

        for fname in self.required_fields:
            if fname not in field_mapping.keys():
                # if not mapped, try the attribute name directly
                field_mapping[fname] = fname

            if field_mapping[fname] not in fields:
                raise Exception("Dataset does not have a required field called '%s'" % field_mapping[fname])

        for fname in self.optional_fields:
            if fname not in field_mapping.keys():
                # if not mapped, try the attribute name directly
                field_mapping[fname] = fname

        return field_mapping

    def import_ogr(self, shp_path, field_mapping=None, layer_num=0,
            forest_property=None, new_property_name=None, pre_impute=False):
        ds = DataSource(shp_path)
        layer = ds[0]
        num_features = len(layer)
        field_mapping = self._validate_field_mapping(layer, field_mapping)

        if not forest_property and not new_property_name:
            raise Exception("Must provide either existing forest_property OR new_property_name")

        if new_property_name:
            # Calculating property outline from stands
            stands = []
            for feature in layer:
                stands.append(wkt.loads(feature.geom.wkt))
            casc_poly = cascaded_union(stands)

            if casc_poly.type == 'MultiPolygon':
                polys = []
                for c in casc_poly:
                    # Identify small 'slivers' or areas of empty space b/t polygons that are unintentional
                    # If they're smaller than the threshold, remove them
                    interiors = [x for x in c.interiors if Polygon(x).area > settings.SLIVER_THRESHOLD]
                    polys.append(Polygon(shell=c.exterior, holes=interiors))
            elif casc_poly.type == 'Polygon':
                # Identify small 'slivers' or areas of empty space b/t polygons that are unintentional
                # If they're smaller than the threshold, remove them
                interiors = [x for x in casc_poly.interiors if Polygon(x).area > settings.SLIVER_THRESHOLD]
                polys = [Polygon(shell=casc_poly.exterior, holes=interiors)]

            casc = MultiPolygon(polys)

            # Creating Property
            self.forest_property = ForestProperty.objects.create(user=self.user, name=new_property_name, geometry_final=casc.wkt)
        else:
            self.forest_property = forest_property

        stands = []
        for feature in layer:
            stand = Stand(user=self.user,
                    name=feature.get(field_mapping['name']),
                    geometry_orig=feature.geom.geos)
                    #geometry_final=feature.geom.geos)

            for fname in self.optional_fields:
                if fname in field_mapping.keys():
                    try:
                        stand.__dict__[fname] = feature.get(field_mapping[fname])
                    except OGRIndexError:
                        pass

            stand.full_clean()
            stands.append(stand)
            del stand

        for stand in stands:
            stand.save()
            self.forest_property.add(stand)
            if pre_impute:
                tmp = stand.geojson()

def calculate_adjacency(qs, threshold):
    """
    Determines a matrix of adjacent polygons from a Feature queryset.
    Since the data are not topological, adjacency is determined
    by a minimum distance threshold.
    """
    features = list(qs)
    adj = {}

    for feat in features:
        fid = feat.pk
        adj[fid] = []

        geom_orig = feat.geometry_final
        geom_buf = feat.geometry_final.buffer(threshold)

        filterqs = qs.filter(geometry_final__bboverlaps = geom_buf, geometry_final__intersects = geom_buf)
        for feat2 in filterqs:
            fid2 = feat2.pk
            if fid == fid2:
                continue
            adj[fid].append(fid2)

        if len(adj[fid]) == 0:
            adj[fid] = None

        del filterqs

    return adj

class NoPlotMatchError(Exception):
    pass

def process_categories(categories, search_params):
    keys = search_params.keys()
    for sp in keys:
        categories[sp+"__isnull"] = False

    if 'for_type_name' not in categories:
        raise NoPlotMatchError("Must provide a primary Forest Type")

    # If secondary forest type not provided, assume NO secondary forest type
    if 'for_type_secdry_name' not in categories:
        categories['for_type_secdry_name__isnull'] = True

    return categories

def nearest_plots(categories, input_params, weight_dict, k=10):
    search_params = input_params.copy()
    origkeys = search_params.keys()
    keys = [x for x in origkeys if x not in ['calc_aspect',]] # special case
    categories = process_categories(categories, search_params)

    def plot_attrs(ps, keys):
        vals = []
        for attr in keys:
            vals.append(ps.__dict__[attr])

        # Aspect is a special case
        if 'calc_aspect' in origkeys:
            angle = angular_diff(ps.calc_aspect, input_params['calc_aspect'])
        else:
            angle = 0
        vals.append(angle)
        search_params['_aspect'] = 0 # anglular difference to self is 0

        return vals

    plotsum_qs = IdbSummary.objects.filter(**categories)
    plotsummaries = list(plotsum_qs)
    ps_attr_list= [plot_attrs(ps, keys) for ps in plotsummaries]

    # include our special case
    keys.append('_aspect')

    num_candidates = len(plotsummaries)
    if num_candidates == 0:
        raise NoPlotMatchError("There are no candidate plots matching the categorical variables: %s" % categories)

    weights = np.ones(len(keys))
    for i in range(len(keys)):
        key = keys[i]
        if key in weight_dict:
            weights[i] = weight_dict[key]

    querypoint = np.array([float(search_params[attr]) for attr in keys])

    rawpoints = np.array(ps_attr_list)

    # Normalize to max of 100; linear scale
    high = 100.0
    low = 0.0
    mins = np.min(rawpoints, axis=0)
    maxs = np.max(rawpoints, axis=0)
    rng = maxs - mins
    scaled_points = high - (((high - low) * (maxs - rawpoints)) / rng)
    scaled_querypoint = high - (((high - low) * (maxs - querypoint)) / rng)

    # Apply weights
    scaled_points *= weights
    scaled_querypoint *= weights

    # Create tree and query it for nearest plot
    tree = KDTree(scaled_points)
    querypoints = np.array([scaled_querypoint])
    result = tree.query(querypoints, k=k)
    distances = result[0][0]
    plots = result[1][0]

    top = zip(plots, distances)
    ps = []

    xs = [100 * x for x in weights if x > 0]
    squares = [x * x for x in xs]
    max_dist = math.sqrt(sum(squares)) # the real max
    for t in top:
        try:
            p = plotsummaries[t[0]]
            p.__dict__['_kdtree_distance'] = t[1]
            # certainty of 0 -> distance is furthest possible
            # certainty of 1 -> the point matches exactly
            # sqrt of ratio taken to exagerate small diffs
            p.__dict__['_certainty'] = 1.0 - ((t[1] / max_dist) ** 0.5)
        except:
            pass
        ps.append(p)
    return ps, num_candidates

def classify_aspect(angle):
    '''
    inspired by http://xkcd.com/cyborg.py
    '''
    try:
        angle = float(angle)
    except:
        return ""

    while angle > 360.0:
        angle = angle - 360.0
    angle += 22.5
    angle = int(angle/45.0)
    words=["North", "North-East", "East", "South-East", "South", "South-West", "West", "North-West", "North"]
    return words[angle]

def angular_diff(x,y):
    '''
    input: two angles in degrees
    output: absolute value of the angular difference in degrees
    '''
    import math
    x = math.radians(x)
    y = math.radians(y)
    return math.fabs(math.degrees(min(y-x, y-x+2*math.pi, y-x-2*math.pi, key=abs)))

def potential_minmax(categories, weight_dict, search_params):
    categories = process_categories(categories, search_params)
    ps = IdbSummary.objects.filter(**categories)
    keys = [k for k in weight_dict.keys() if not k.startswith("_")]
    args = [Min(k) for k in keys] + [Max(k) for k in keys] + [Avg(k) for k in keys]
    return ps.aggregate(*args)
<commit_msg>don't search for aspect if not specified<commit_after>from trees.models import Stand, ForestProperty, IdbSummary
from django.contrib.gis.gdal import DataSource
from django.contrib.gis.gdal.error import OGRIndexError
from django.conf import settings
from madrona.common.utils import get_logger
from django.db.models import Min, Max, Avg
from django.contrib.gis.geos import GEOSGeometry
from shapely.ops import cascaded_union
import numpy as np
from scipy.spatial import KDTree
import math

logger = get_logger()

class StandImporter:

    def __init__(self, user):
        self.user = user
        self.required_fields = ['name']
        self.optional_fields = ['domspp','rx'] # model must provide defaults!

    def _validate_field_mapping(self, layer, field_mapping):
        fields = layer.fields
        if not field_mapping:
            field_mapping = {}

        for fname in self.required_fields:
            if fname not in field_mapping.keys():
                # if not mapped, try the attribute name directly
                field_mapping[fname] = fname

            if field_mapping[fname] not in fields:
                raise Exception("Dataset does not have a required field called '%s'" % field_mapping[fname])

        for fname in self.optional_fields:
            if fname not in field_mapping.keys():
                # if not mapped, try the attribute name directly
                field_mapping[fname] = fname

        return field_mapping

    def import_ogr(self, shp_path, field_mapping=None, layer_num=0,
            forest_property=None, new_property_name=None, pre_impute=False):
        ds = DataSource(shp_path)
        layer = ds[0]
        num_features = len(layer)
        field_mapping = self._validate_field_mapping(layer, field_mapping)

        if not forest_property and not new_property_name:
            raise Exception("Must provide either existing forest_property OR new_property_name")

        if new_property_name:
            # Calculating property outline from stands
            stands = []
            for feature in layer:
                stands.append(wkt.loads(feature.geom.wkt))
            casc_poly = cascaded_union(stands)

            if casc_poly.type == 'MultiPolygon':
                polys = []
                for c in casc_poly:
                    # Identify small 'slivers' or areas of empty space b/t polygons that are unintentional
                    # If they're smaller than the threshold, remove them
                    interiors = [x for x in c.interiors if Polygon(x).area > settings.SLIVER_THRESHOLD]
                    polys.append(Polygon(shell=c.exterior, holes=interiors))
            elif casc_poly.type == 'Polygon':
                # Identify small 'slivers' or areas of empty space b/t polygons that are unintentional
                # If they're smaller than the threshold, remove them
                interiors = [x for x in casc_poly.interiors if Polygon(x).area > settings.SLIVER_THRESHOLD]
                polys = [Polygon(shell=casc_poly.exterior, holes=interiors)]

            casc = MultiPolygon(polys)

            # Creating Property
            self.forest_property = ForestProperty.objects.create(user=self.user, name=new_property_name, geometry_final=casc.wkt)
        else:
            self.forest_property = forest_property

        stands = []
        for feature in layer:
            stand = Stand(user=self.user,
                    name=feature.get(field_mapping['name']),
                    geometry_orig=feature.geom.geos)
                    #geometry_final=feature.geom.geos)

            for fname in self.optional_fields:
                if fname in field_mapping.keys():
                    try:
                        stand.__dict__[fname] = feature.get(field_mapping[fname])
                    except OGRIndexError:
                        pass

            stand.full_clean()
            stands.append(stand)
            del stand

        for stand in stands:
            stand.save()
            self.forest_property.add(stand)
            if pre_impute:
                tmp = stand.geojson()

def calculate_adjacency(qs, threshold):
    """
    Determines a matrix of adjacent polygons from a Feature queryset.
    Since the data are not topological, adjacency is determined
    by a minimum distance threshold.
    """
    features = list(qs)
    adj = {}

    for feat in features:
        fid = feat.pk
        adj[fid] = []

        geom_orig = feat.geometry_final
        geom_buf = feat.geometry_final.buffer(threshold)

        filterqs = qs.filter(geometry_final__bboverlaps = geom_buf, geometry_final__intersects = geom_buf)
        for feat2 in filterqs:
            fid2 = feat2.pk
            if fid == fid2:
                continue
            adj[fid].append(fid2)

        if len(adj[fid]) == 0:
            adj[fid] = None

        del filterqs

    return adj

class NoPlotMatchError(Exception):
    pass

def process_categories(categories, search_params):
    keys = search_params.keys()
    for sp in keys:
        categories[sp+"__isnull"] = False

    if 'for_type_name' not in categories:
        raise NoPlotMatchError("Must provide a primary Forest Type")

    # If secondary forest type not provided, assume NO secondary forest type
    if 'for_type_secdry_name' not in categories:
        categories['for_type_secdry_name__isnull'] = True

    return categories

def nearest_plots(categories, input_params, weight_dict, k=10):
    search_params = input_params.copy()
    origkeys = search_params.keys()
    keys = [x for x in origkeys if x not in ['calc_aspect',]] # special case
    categories = process_categories(categories, search_params)

    def plot_attrs(ps, keys):
        vals = []
        for attr in keys:
            vals.append(ps.__dict__[attr])

        # Aspect is a special case
        if 'calc_aspect' in origkeys:
            angle = angular_diff(ps.calc_aspect, input_params['calc_aspect'])
            vals.append(angle)
            search_params['_aspect'] = 0 # anglular difference to self is 0

        return vals

    plotsum_qs = IdbSummary.objects.filter(**categories)
    plotsummaries = list(plotsum_qs)
    ps_attr_list= [plot_attrs(ps, keys) for ps in plotsummaries]

    # include our special case
    if 'calc_aspect' in origkeys:
        keys.append('_aspect')

    num_candidates = len(plotsummaries)
    if num_candidates == 0:
        raise NoPlotMatchError("There are no candidate plots matching the categorical variables: %s" % categories)

    weights = np.ones(len(keys))
    for i in range(len(keys)):
        key = keys[i]
        if key in weight_dict:
            weights[i] = weight_dict[key]

    querypoint = np.array([float(search_params[attr]) for attr in keys])

    rawpoints = np.array(ps_attr_list)

    # Normalize to max of 100; linear scale
    high = 100.0
    low = 0.0
    mins = np.min(rawpoints, axis=0)
    maxs = np.max(rawpoints, axis=0)
    rng = maxs - mins
    scaled_points = high - (((high - low) * (maxs - rawpoints)) / rng)
    scaled_querypoint = high - (((high - low) * (maxs - querypoint)) / rng)

    # Apply weights
    scaled_points *= weights
    scaled_querypoint *= weights

    # Create tree and query it for nearest plot
    tree = KDTree(scaled_points)
    querypoints = np.array([scaled_querypoint])
    result = tree.query(querypoints, k=k)
    distances = result[0][0]
    plots = result[1][0]

    top = zip(plots, distances)
    ps = []

    xs = [100 * x for x in weights if x > 0]
    squares = [x * x for x in xs]
    max_dist = math.sqrt(sum(squares)) # the real max
    for t in top:
        try:
            p = plotsummaries[t[0]]
            p.__dict__['_kdtree_distance'] = t[1]
            # certainty of 0 -> distance is furthest possible
            # certainty of 1 -> the point matches exactly
            # sqrt of ratio taken to exagerate small diffs
            p.__dict__['_certainty'] = 1.0 - ((t[1] / max_dist) ** 0.5)
        except:
            pass
        ps.append(p)
    return ps, num_candidates

def classify_aspect(angle):
    '''
    inspired by http://xkcd.com/cyborg.py
    '''
    try:
        angle = float(angle)
    except:
        return ""

    while angle > 360.0:
        angle = angle - 360.0
    angle += 22.5
    angle = int(angle/45.0)
    words=["North", "North-East", "East", "South-East", "South", "South-West", "West", "North-West", "North"]
    return words[angle]

def angular_diff(x,y):
    '''
    input: two angles in degrees
    output: absolute value of the angular difference in degrees
    '''
    import math
    x = math.radians(x)
    y = math.radians(y)
    return math.fabs(math.degrees(min(y-x, y-x+2*math.pi, y-x-2*math.pi, key=abs)))

def potential_minmax(categories, weight_dict, search_params):
    categories = process_categories(categories, search_params)
    ps = IdbSummary.objects.filter(**categories)
    keys = [k for k in weight_dict.keys() if not k.startswith("_")]
    args = [Min(k) for k in keys] + [Max(k) for k in keys] + [Avg(k) for k in keys]
    return ps.aggregate(*args)
<|endoftext|>