<commit_before>from __future__ import absolute_import, division, unicode_literals

import os
import sys
import traceback
import warnings
import re

warnings.simplefilter("error")

from .support import get_data_files
from .support import TestData, convert, convertExpected, treeTypes
from html5lib import html5parser, constants

# Run the parse error checks
checkParseErrors = False

# XXX - There should just be one function here but for some reason the testcase
# format differs from the treedump format by a single space character

def convertTreeDump(data):
    return "\n".join(convert(3)(data).split("\n")[1:])

namespaceExpected = re.compile(r"^(\s*)<(\S+)>", re.M).sub

def runParserTest(innerHTML, input, expected, errors, treeClass,
                  namespaceHTMLElements):
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter("always")
        p = html5parser.HTMLParser(tree=treeClass,
                                   namespaceHTMLElements=namespaceHTMLElements)

        try:
            if innerHTML:
                document = p.parseFragment(input, innerHTML)
            else:
                document = p.parse(input)
        except:
            errorMsg = "\n".join(["\n\nInput:", input, "\nExpected:", expected,
                                  "\nTraceback:", traceback.format_exc()])
            assert False, errorMsg

    otherW = [x for x in w if not issubclass(x.category, constants.DataLossWarning)]
    assert len(otherW) == 0, [(x.category, x.message) for x in otherW]
    if len(w):
        return

    output = convertTreeDump(p.tree.testSerializer(document))

    expected = convertExpected(expected)
    if namespaceHTMLElements:
        expected = namespaceExpected(r"\1<html \2>", expected)

    errorMsg = "\n".join(["\n\nInput:", input, "\nExpected:", expected,
                          "\nReceived:", output])
    assert expected == output, errorMsg

    errStr = []
    for (line, col), errorcode, datavars in p.errors:
        assert isinstance(datavars, dict), "%s, %s" % (errorcode, repr(datavars))
        errStr.append("Line: %i Col: %i %s" % (line, col,
                                               constants.E[errorcode] % datavars))

    errorMsg2 = "\n".join(["\n\nInput:", input,
                           "\nExpected errors (" + str(len(errors)) + "):\n" + "\n".join(errors),
                           "\nActual errors (" + str(len(p.errors)) + "):\n" + "\n".join(errStr)])
    if checkParseErrors:
            assert len(p.errors) == len(errors), errorMsg2

def test_parser():
    sys.stderr.write('Testing tree builders ' + " ".join(list(treeTypes.keys())) + "\n")
    files = get_data_files('tree-construction')

    for filename in files:
        testName = os.path.basename(filename).replace(".dat", "")
        if testName in ("template",):
            continue

        tests = TestData(filename, "data")

        for index, test in enumerate(tests):
            input, errors, innerHTML, expected = [test[key] for key in
                                                  ('data', 'errors',
                                                   'document-fragment',
                                                   'document')]
            if errors:
                errors = errors.split("\n")

            for treeName, treeCls in treeTypes.items():
                for namespaceHTMLElements in (True, False):
                    yield (runParserTest, innerHTML, input, expected, errors, treeCls,
                           namespaceHTMLElements)
<commit_msg>Clarify variable names in test_parser.py.<commit_after>from __future__ import absolute_import, division, unicode_literals

import os
import sys
import traceback
import warnings
import re

warnings.simplefilter("error")

from .support import get_data_files
from .support import TestData, convert, convertExpected, treeTypes
from html5lib import html5parser, constants

# Run the parse error checks
checkParseErrors = False

# XXX - There should just be one function here but for some reason the testcase
# format differs from the treedump format by a single space character

def convertTreeDump(data):
    return "\n".join(convert(3)(data).split("\n")[1:])

namespaceExpected = re.compile(r"^(\s*)<(\S+)>", re.M).sub

def runParserTest(innerHTML, input, expected, errors, treeClass,
                  namespaceHTMLElements):
    with warnings.catch_warnings(record=True) as caughtWarnings:
        warnings.simplefilter("always")
        p = html5parser.HTMLParser(tree=treeClass,
                                   namespaceHTMLElements=namespaceHTMLElements)

        try:
            if innerHTML:
                document = p.parseFragment(input, innerHTML)
            else:
                document = p.parse(input)
        except:
            errorMsg = "\n".join(["\n\nInput:", input, "\nExpected:", expected,
                                  "\nTraceback:", traceback.format_exc()])
            assert False, errorMsg

    otherWarnings = [x for x in caughtWarnings
                     if not issubclass(x.category, constants.DataLossWarning)]
    assert len(otherWarnings) == 0, [(x.category, x.message) for x in otherWarnings]
    if len(caughtWarnings):
        return

    output = convertTreeDump(p.tree.testSerializer(document))

    expected = convertExpected(expected)
    if namespaceHTMLElements:
        expected = namespaceExpected(r"\1<html \2>", expected)

    errorMsg = "\n".join(["\n\nInput:", input, "\nExpected:", expected,
                          "\nReceived:", output])
    assert expected == output, errorMsg

    errStr = []
    for (line, col), errorcode, datavars in p.errors:
        assert isinstance(datavars, dict), "%s, %s" % (errorcode, repr(datavars))
        errStr.append("Line: %i Col: %i %s" % (line, col,
                                               constants.E[errorcode] % datavars))

    errorMsg2 = "\n".join(["\n\nInput:", input,
                           "\nExpected errors (" + str(len(errors)) + "):\n" + "\n".join(errors),
                           "\nActual errors (" + str(len(p.errors)) + "):\n" + "\n".join(errStr)])
    if checkParseErrors:
            assert len(p.errors) == len(errors), errorMsg2

def test_parser():
    sys.stderr.write('Testing tree builders ' + " ".join(list(treeTypes.keys())) + "\n")
    files = get_data_files('tree-construction')

    for filename in files:
        testName = os.path.basename(filename).replace(".dat", "")
        if testName in ("template",):
            continue

        tests = TestData(filename, "data")

        for index, test in enumerate(tests):
            input, errors, innerHTML, expected = [test[key] for key in
                                                  ('data', 'errors',
                                                   'document-fragment',
                                                   'document')]
            if errors:
                errors = errors.split("\n")

            for treeName, treeCls in treeTypes.items():
                for namespaceHTMLElements in (True, False):
                    yield (runParserTest, innerHTML, input, expected, errors, treeCls,
                           namespaceHTMLElements)
<|endoftext|>


<commit_before>import urllib

from django.conf import settings
from django.core.exceptions import PermissionDenied
from django.core.urlresolvers import reverse
from django_conneg.views import ContentNegotiatedView, HTMLView, JSONView

from .core import StoreView, QueryView
from humfrey.linkeddata.resource import ResourceRegistry
from humfrey.linkeddata.views import MappingView
from humfrey.desc import views as desc_views
from humfrey.misc import views as misc_views
from humfrey.results.views.standard import RDFView
from humfrey.sparql.views import StoreView, CannedQueryView
from humfrey.sparql.models import Store
from humfrey.utils.namespaces import NS

class IndexView(HTMLView, JSONView):
    def get(self, request):
        stores = Store.objects.all().order_by('name')
        if not request.user.has_perm('sparql.query_store'):
            stores = [s for s in stores if request.user.has_perm('sparql.query_store', s)]
        context = {'stores': stores,
                   'with_elasticsearch': 'humfrey.elasticsearch' in settings.INSTALLED_APPS}
        return self.render(request, context, 'sparql/index')

class StoreChooseMixin(object):
    @property
    def doc_view(self):
        return ('admin', 'sparql-admin:view',
                (), {},
                (), {'store': self.store_name})
    desc_view = doc_view

    id_mapping = ()
    resource_registry = ResourceRegistry()

    def dispatch(self, request, *args, **kwargs):
        self.store_name = kwargs.pop('store')
        if not self.store.can_query(request.user):
            raise PermissionDenied
        return super(StoreChooseMixin, self).dispatch(request, *args, **kwargs)

class DocView(StoreChooseMixin, desc_views.DocView):
    pass

class QueryView(StoreChooseMixin, QueryView):
    pass

class GraphDataView(StoreChooseMixin, StoreView, misc_views.PassThroughView):
    def get_target_url(self, request):
        if request.method != 'GET' and not self.store.can_update(request.user):
            raise PermissionDenied
        return "{0}?{1}".format(self.store.graph_store_endpoint,
                                urllib.urlencode({'graph': request.GET['graph']}))

class GraphListView(HTMLView, RDFView, StoreChooseMixin, CannedQueryView, MappingView):
    query = """
        CONSTRUCT {
         ?g a sd:Graph ;
           dcterms:publisher ?publisher ;
           dcterms:license ?license ;
           dcterms:created ?created ;
           dcterms:modified ?modified
        } WHERE {
          GRAPH ?g { ?s ?p ?o } .
          OPTIONAL { ?g rdfs:label ?label } .
          OPTIONAL { ?g dcterms:publisher ?publisher } .
          OPTIONAL { ?g dcterms:license ?license } .
          OPTIONAL { ?g dcterms:created ?created } .
          OPTIONAL { ?g dcterms:modified ?modified } .
        }
    """

    template_name = 'sparql/graph-list'
    def get_subjects(self, request, graph):
        return sorted(graph.subjects(NS.rdf.type, NS.sd.Graph))
    def get_additional_context(self, request):
        return {'store': self.store}
    def process_response(self, request, response):
        response['X-URI-Lookup'] = reverse('sparql-admin:view', args=[self.store_name]) + '?uri='
        response['X-SPARQL-Endpoint'] = reverse('sparql-admin:query', args=[self.store_name])
        return response

class GraphView(ContentNegotiatedView):
    graph_data_view = staticmethod(GraphDataView.as_view())
    graph_list_view = staticmethod(GraphListView.as_view())
    def get(self, request, store):
        if 'graph' in request.GET:
            return self.graph_data_view(request, store=store)
        else:
            return self.graph_list_view(request, store=store)

if 'humfrey.elasticsearch' in settings.INSTALLED_APPS:
    from humfrey.elasticsearch import views as elasticsearch_views

    class SearchView(StoreChooseMixin, elasticsearch_views.SearchView):
        pass

    class ElasticSearchPassThroughView(StoreChooseMixin, StoreView, misc_views.PassThroughView):
        def get_target_url(self, request, index=None):
            params = {'host': settings.ELASTICSEARCH_SERVER['host'],
                      'port': settings.ELASTICSEARCH_SERVER['port'],
                      'store': self.store_name,
                      'index': index}
            if index:
                url = 'http://{host}:{port}/{store}/{index}/_search'.format(**params)
            else:
                url = 'http://{host}:{port}/{store}/_search'.format(**params)
            if request.META.get('QUERY_STRING'):
                url += '?' + request.META['QUERY_STRING']
            return url
        def process_response(self, request, response, index=None):
            response['X-URI-Lookup'] = reverse('sparql-admin:view', args=[self.store_name]) + '?uri='
            response['X-SPARQL-Endpoint'] = reverse('sparql-admin:query', args=[self.store_name])
            return response

else:
    SearchView = None
    ElasticSearchPassThroughView = None
<commit_msg>GraphView now handles PUT, POST and DELETE methods properly.<commit_after>import urllib

from django.conf import settings
from django.core.exceptions import PermissionDenied
from django.core.urlresolvers import reverse
from django_conneg.views import ContentNegotiatedView, HTMLView, JSONView

from .core import StoreView, QueryView
from humfrey.linkeddata.resource import ResourceRegistry
from humfrey.linkeddata.views import MappingView
from humfrey.desc import views as desc_views
from humfrey.misc import views as misc_views
from humfrey.results.views.standard import RDFView
from humfrey.sparql.views import StoreView, CannedQueryView
from humfrey.sparql.models import Store
from humfrey.utils.namespaces import NS

class IndexView(HTMLView, JSONView):
    def get(self, request):
        stores = Store.objects.all().order_by('name')
        if not request.user.has_perm('sparql.query_store'):
            stores = [s for s in stores if request.user.has_perm('sparql.query_store', s)]
        context = {'stores': stores,
                   'with_elasticsearch': 'humfrey.elasticsearch' in settings.INSTALLED_APPS}
        return self.render(request, context, 'sparql/index')

class StoreChooseMixin(object):
    @property
    def doc_view(self):
        return ('admin', 'sparql-admin:view',
                (), {},
                (), {'store': self.store_name})
    desc_view = doc_view

    id_mapping = ()
    resource_registry = ResourceRegistry()

    def dispatch(self, request, *args, **kwargs):
        self.store_name = kwargs.pop('store')
        if not self.store.can_query(request.user):
            raise PermissionDenied
        return super(StoreChooseMixin, self).dispatch(request, *args, **kwargs)

class DocView(StoreChooseMixin, desc_views.DocView):
    pass

class QueryView(StoreChooseMixin, QueryView):
    pass

class GraphDataView(StoreChooseMixin, StoreView, misc_views.PassThroughView):
    def get_target_url(self, request):
        if request.method != 'GET' and not self.store.can_update(request.user):
            raise PermissionDenied
        return "{0}?{1}".format(self.store.graph_store_endpoint,
                                urllib.urlencode({'graph': request.GET['graph']}))

class GraphListView(HTMLView, RDFView, StoreChooseMixin, CannedQueryView, MappingView):
    query = """
        CONSTRUCT {
         ?g a sd:Graph ;
           dcterms:publisher ?publisher ;
           dcterms:license ?license ;
           dcterms:created ?created ;
           dcterms:modified ?modified
        } WHERE {
          GRAPH ?g { ?s ?p ?o } .
          OPTIONAL { ?g rdfs:label ?label } .
          OPTIONAL { ?g dcterms:publisher ?publisher } .
          OPTIONAL { ?g dcterms:license ?license } .
          OPTIONAL { ?g dcterms:created ?created } .
          OPTIONAL { ?g dcterms:modified ?modified } .
        }
    """

    template_name = 'sparql/graph-list'
    def get_subjects(self, request, graph):
        return sorted(graph.subjects(NS.rdf.type, NS.sd.Graph))
    def get_additional_context(self, request):
        return {'store': self.store}
    def process_response(self, request, response):
        response['X-URI-Lookup'] = reverse('sparql-admin:view', args=[self.store_name]) + '?uri='
        response['X-SPARQL-Endpoint'] = reverse('sparql-admin:query', args=[self.store_name])
        return response

class GraphView(ContentNegotiatedView):
    graph_data_view = staticmethod(GraphDataView.as_view())
    graph_list_view = staticmethod(GraphListView.as_view())
    def get(self, request, store):
        if 'graph' in request.GET:
            return self.graph_data_view(request, store=store)
        else:
            return self.graph_list_view(request, store=store)
    post = delete = put = get

if 'humfrey.elasticsearch' in settings.INSTALLED_APPS:
    from humfrey.elasticsearch import views as elasticsearch_views

    class SearchView(StoreChooseMixin, elasticsearch_views.SearchView):
        pass

    class ElasticSearchPassThroughView(StoreChooseMixin, StoreView, misc_views.PassThroughView):
        def get_target_url(self, request, index=None):
            params = {'host': settings.ELASTICSEARCH_SERVER['host'],
                      'port': settings.ELASTICSEARCH_SERVER['port'],
                      'store': self.store_name,
                      'index': index}
            if index:
                url = 'http://{host}:{port}/{store}/{index}/_search'.format(**params)
            else:
                url = 'http://{host}:{port}/{store}/_search'.format(**params)
            if request.META.get('QUERY_STRING'):
                url += '?' + request.META['QUERY_STRING']
            return url
        def process_response(self, request, response, index=None):
            response['X-URI-Lookup'] = reverse('sparql-admin:view', args=[self.store_name]) + '?uri='
            response['X-SPARQL-Endpoint'] = reverse('sparql-admin:query', args=[self.store_name])
            return response

else:
    SearchView = None
    ElasticSearchPassThroughView = None
<|endoftext|>


<commit_before>#from direnaj_api.config.config import *
import utils.drnj_time as drnj_time

from direnaj_api.utils.direnaj_collection_templates import *

from direnaj_api.utils.direnaj_auth import direnaj_simple_auth

import tornado.ioloop
import tornado.web

from tornado import gen

from tornado.web import HTTPError
from tornado.web import MissingArgumentError

from tornado.escape import json_decode,json_encode

from jinja2 import Environment, FileSystemLoader

import json
import time, datetime

import bson.json_util

import pymongo

from direnaj_api.utils.direnajneo4jmanager import update_task_state_in_watchlist, init_user_to_graph

from direnaj_api.celery_app.server_endpoint import app_object

import logging

logger = logging.getLogger("direnaj_api")

def extract_arguments(tweets):
    campaign_id = None
    user_objects = []
    if tweets:
        campaign_id = tweets[0]['campaign_id']

        for tweet in tweets:
            user_objects.append(tweet['tweet']['user'])
    return [campaign_id, user_objects]

class StatusesHandler(tornado.web.RequestHandler):

    ## def datetime_hook(self, dct):
    ##     # TODO: this only checks for the first level 'created_at'
    ##     # We should think about whether making this recursive.
    ##     if 'created_at' in dct:
    ##         time_struct = time.strptime(dct['created_at'], "%a %b %d %H:%M:%S +0000 %Y") #Tue Apr 26 08:57:55 +0000 2011
    ##         dct['created_at'] = datetime.datetime.fromtimestamp(time.mktime(time_struct))
    ##         return bson.json_util.object_hook(dct)
    ##     return bson.json_util.object_hook(dct)

    def get(self, *args):
        self.post(*args)
        #self.write("not implemented yet")

    #@direnaj_simple_auth
    @tornado.web.asynchronous
    @gen.coroutine
    def post(self, *args, **keywords):
        """
        `view`
            - params:
                - user_id: user_id of the twitter user that we want to
                retrieve all tweets.
        `store`
            - params:
                - tweet_data: array of tweets in json format.
        `retweets`
            - params:
                - tweet_id: `id` of the tweet that we want to
                retrieve all retweets.
        `filter`
            - not implemented yet.
        """

        if len(args) > 1:
            (action) = args
        else:
            action = args[0]

        print action

        verbose_response = self.get_argument('verbose', '')

        if (action == 'view'):
            try:
                # TODO: we need to get here again.
                user_id = self.get_argument('user_id')
                # if no user_id is supplied.
                if user_id == '':
                    tmp = []
                else:

                    tweets_coll = self.application.db.motor_column.tweets
                    # running the query
                    cursor = tweets_coll.find({
                        'tweet.user.id_str': str(user_id),
                    })

                    tmp = [x for x in (yield cursor.to_list(length=100))]

                self.write(bson.json_util.dumps({'results': tmp}))
                self.add_header('Content-Type', 'application/json')
            except MissingArgumentError as e:
                # TODO: implement logging.
                raise HTTPError(500, 'You didn''t supply %s as an argument' % e.arg_name)
        elif (action == 'store'):
            try:
                tweet_data = self.get_argument('tweet_data')
                campaign_id = self.get_argument('campaign_id', 'default')
                watchlist_related = self.get_argument('watchlist_related', '')
                if tweet_data:
                    #tweet_array = bson.json_util.loads(tweet_data, object_hook=self.datetime_hook)
                    tweet_array = bson.json_util.loads(tweet_data)
                    tmp_tweets = []
                    # TODO: Sanity check the data!
                    # For example, treat 'entities', 'user' specially.
                    DB_TEST_VERSION = 0.2
                    for tweet_obj in tweet_array:
                        tweet_obj['user']['history'] = False
                        tmp_tweets.append(validate_document(new_tweet_template(), {
                            "tweet": tweet_obj,
                            # TODO: Replace this DB_TEST_VERSION with source code
                            # version later
                            "direnaj_service_version": DB_TEST_VERSION,
                            # TODO: "retrieved_by": keywords['drnjID'],
                            "retrieved_by": "direnaj",
                            "campaign_id": campaign_id,
                            "record_retrieved_at": drnj_time.now_in_drnj_time(),
                        }, fail=False))
                        ### TODO: add these users later.
                        ### tmp_users.append(validate_document(new_user_template(), tweet_obj['user']))
                    if tmp_tweets:
                        self.application.db.insert_tweet(tmp_tweets)
                        ##init_user_to_graph(tmp_tweets)
                        campaign_id, user_objects = extract_arguments(tmp_tweets)
                        res = app_object.send_task('init_user_to_graph_offline',
                                           [[ campaign_id, bson.json_util.dumps(user_objects) ]],
                                           queue="offline_jobs")
                        if watchlist_related:
                            print watchlist_related
                            watchlist_related = bson.json_util.loads(watchlist_related)
                            print watchlist_related
                            #self.application.db.update_watchlist(**watchlist_related)
                            update_task_state_in_watchlist(**watchlist_related)

                    else:
                        raise HTTPError(500, 'You tried to insert no tweets?!')
#                    tweets_coll.insert(tmp_tweets)
                else:
                    tmp = []

                if verbose_response:
                    self.write(bson.json_util.dumps({'results': tmp}))
                else:
                    self.write(bson.json_util.dumps({'results': 'ok'}))
                self.add_header('Content-Type', 'application/json')
                self.finish()
            except MissingArgumentError as e:
                # TODO: implement logging.
                raise HTTPError(500, 'You didn''t supply %s as an argument' % e.arg_name)
        elif (action == 'retweets'):
            try:
                tweet_id = self.get_argument('tweet_id')
                self.write('not implemented yet')
            except MissingArgumentError as e:
                # TODO: implement logging.
                raise HTTPError(500, 'You didn''t supply %s as an argument' % e.arg_name)
        elif (action == 'filter'):
            try:
                campaign_id = self.get_argument('campaign_id')
                skip = self.get_argument('skip', 0)
                limit = self.get_argument('limit', 100)
                res_format = self.get_argument('format', 'json')
                since_datetime = self.get_argument('since_datetime', -1)
                until_datetime = self.get_argument('until_datetime', -1)
                sort_by_datetime = self.get_argument('sort_by_datetime', 0)

                tweets_coll = self.application.db.motor_column.tweets

                query_string = {'campaign_id' : '%s' % campaign_id}
                if since_datetime != -1:
                    if 'tweet.created_at' not in query_string:
                        query_string['tweet.created_at'] = {}
                    query_string['tweet.created_at']['$gte'] = float(since_datetime)
                if until_datetime != -1:
                    if 'tweet.created_at' not in query_string:
                        query_string['tweet.created_at'] = {}
                    query_string['tweet.created_at']['$lt'] = float(until_datetime)
                sort_string = []
                if sort_by_datetime != 0:
                    sort_string = [('tweet.created_at', pymongo.ASCENDING)] # ascending
                print "STARTED " + str(query_string)
                cursor = tweets_coll.find(query_string)
                if sort_string:
                    cursor = cursor.sort(sort_string)
                cursor = cursor.skip(int(skip)).limit(int(limit))
                           # TODO: removing because of complaint:
                           # TypeError: if no direction is specified, key_or_list must be an instance of list
                           # .sort({"$natural" : 1})\
                tmp = [x for x in (yield cursor.to_list(length=100))]
                print "ENDED " + str(query_string)
                DB_TEST_VERSION = 0.2
                if res_format == 'json':
                    self.write(bson.json_util.dumps(
                            {'results': tmp,
                            # TODO: Replace this DB_TEST_VERSION with source code
                            # version later
                            "direnaj_service_version": DB_TEST_VERSION,
                            # TODO: "requested_by": keywords['drnjID'],
                            "requested_by": "direnaj",
                            "campaign_id": campaign_id,
                            "served_at": drnj_time.now_in_drnj_time(),
                             'skip': int(skip),
                             'limit': int(limit)}))
                    self.add_header('Content-Type', 'application/json')
                elif res_format == 'html':
                    env = Environment(loader=FileSystemLoader('templates'))

                    template = env.get_template('statuses/filter.html')
                    result = template.render(statuses=[x['tweet'] for x in tmp])
                    self.write(result)

                self.finish()

            except MissingArgumentError as e:
                # TODO: implement logging.
                raise HTTPError(500, 'You didn''t supply %s as an argument' % e.arg_name)
            # db.tweets.find({'campaign_id' : 'syria'}).sort({$natural : 1}).skip(10).limit(14)
        else:
            # Tornado will make sure that this point will not be reached, if
            # it's regexp based router works correctly
            raise HTTPError(404, 'No such method')
<commit_msg>Adding logger: Moved user_upserts to offline_jobs<commit_after>#from direnaj_api.config.config import *
import utils.drnj_time as drnj_time

from direnaj_api.utils.direnaj_collection_templates import *

from direnaj_api.utils.direnaj_auth import direnaj_simple_auth

import tornado.ioloop
import tornado.web

from tornado import gen

from tornado.web import HTTPError
from tornado.web import MissingArgumentError

from tornado.escape import json_decode,json_encode

from jinja2 import Environment, FileSystemLoader

import json
import time, datetime

import bson.json_util

import pymongo

from direnaj_api.utils.direnajneo4jmanager import update_task_state_in_watchlist, init_user_to_graph

from direnaj_api.celery_app.server_endpoint import app_object

import logging

logger = logging.getLogger("direnaj_api")

def extract_arguments(tweets):
    campaign_id = None
    user_objects = []
    if tweets:
        campaign_id = tweets[0]['campaign_id']

        for tweet in tweets:
            user_objects.append(tweet['tweet']['user'])
    return [campaign_id, user_objects]

class StatusesHandler(tornado.web.RequestHandler):

    ## def datetime_hook(self, dct):
    ##     # TODO: this only checks for the first level 'created_at'
    ##     # We should think about whether making this recursive.
    ##     if 'created_at' in dct:
    ##         time_struct = time.strptime(dct['created_at'], "%a %b %d %H:%M:%S +0000 %Y") #Tue Apr 26 08:57:55 +0000 2011
    ##         dct['created_at'] = datetime.datetime.fromtimestamp(time.mktime(time_struct))
    ##         return bson.json_util.object_hook(dct)
    ##     return bson.json_util.object_hook(dct)

    def get(self, *args):
        self.post(*args)
        #self.write("not implemented yet")

    #@direnaj_simple_auth
    @tornado.web.asynchronous
    @gen.coroutine
    def post(self, *args, **keywords):
        """
        `view`
            - params:
                - user_id: user_id of the twitter user that we want to
                retrieve all tweets.
        `store`
            - params:
                - tweet_data: array of tweets in json format.
        `retweets`
            - params:
                - tweet_id: `id` of the tweet that we want to
                retrieve all retweets.
        `filter`
            - not implemented yet.
        """

        if len(args) > 1:
            (action) = args
        else:
            action = args[0]

        print action

        verbose_response = self.get_argument('verbose', '')

        if (action == 'view'):
            try:
                # TODO: we need to get here again.
                user_id = self.get_argument('user_id')
                # if no user_id is supplied.
                if user_id == '':
                    tmp = []
                else:

                    tweets_coll = self.application.db.motor_column.tweets
                    # running the query
                    cursor = tweets_coll.find({
                        'tweet.user.id_str': str(user_id),
                    })

                    tmp = [x for x in (yield cursor.to_list(length=100))]

                self.write(bson.json_util.dumps({'results': tmp}))
                self.add_header('Content-Type', 'application/json')
            except MissingArgumentError as e:
                # TODO: implement logging.
                raise HTTPError(500, 'You didn''t supply %s as an argument' % e.arg_name)
        elif (action == 'store'):
            try:
                tweet_data = self.get_argument('tweet_data')
                campaign_id = self.get_argument('campaign_id', 'default')
                watchlist_related = self.get_argument('watchlist_related', '')
                if tweet_data:
                    #tweet_array = bson.json_util.loads(tweet_data, object_hook=self.datetime_hook)
                    tweet_array = bson.json_util.loads(tweet_data)
                    tmp_tweets = []
                    # TODO: Sanity check the data!
                    # For example, treat 'entities', 'user' specially.
                    DB_TEST_VERSION = 0.2
                    for tweet_obj in tweet_array:
                        tweet_obj['user']['history'] = False
                        tmp_tweets.append(validate_document(new_tweet_template(), {
                            "tweet": tweet_obj,
                            # TODO: Replace this DB_TEST_VERSION with source code
                            # version later
                            "direnaj_service_version": DB_TEST_VERSION,
                            # TODO: "retrieved_by": keywords['drnjID'],
                            "retrieved_by": "direnaj",
                            "campaign_id": campaign_id,
                            "record_retrieved_at": drnj_time.now_in_drnj_time(),
                        }, fail=False))
                        ### TODO: add these users later.
                        ### tmp_users.append(validate_document(new_user_template(), tweet_obj['user']))
                    if tmp_tweets:
                        self.application.db.insert_tweet(tmp_tweets)
                        ##init_user_to_graph(tmp_tweets)
                        campaign_id, user_objects = extract_arguments(tmp_tweets)
                        res = app_object.send_task('init_user_to_graph_offline',
                                           [[ campaign_id, bson.json_util.dumps(user_objects) ]],
                                           queue="offline_jobs")
                        logger.info(str(res))
                        if watchlist_related:
                            print watchlist_related
                            watchlist_related = bson.json_util.loads(watchlist_related)
                            print watchlist_related
                            #self.application.db.update_watchlist(**watchlist_related)
                            update_task_state_in_watchlist(**watchlist_related)

                    else:
                        raise HTTPError(500, 'You tried to insert no tweets?!')
#                    tweets_coll.insert(tmp_tweets)
                else:
                    tmp = []

                if verbose_response:
                    self.write(bson.json_util.dumps({'results': tmp}))
                else:
                    self.write(bson.json_util.dumps({'results': 'ok'}))
                self.add_header('Content-Type', 'application/json')
                self.finish()
            except MissingArgumentError as e:
                # TODO: implement logging.
                raise HTTPError(500, 'You didn''t supply %s as an argument' % e.arg_name)
        elif (action == 'retweets'):
            try:
                tweet_id = self.get_argument('tweet_id')
                self.write('not implemented yet')
            except MissingArgumentError as e:
                # TODO: implement logging.
                raise HTTPError(500, 'You didn''t supply %s as an argument' % e.arg_name)
        elif (action == 'filter'):
            try:
                campaign_id = self.get_argument('campaign_id')
                skip = self.get_argument('skip', 0)
                limit = self.get_argument('limit', 100)
                res_format = self.get_argument('format', 'json')
                since_datetime = self.get_argument('since_datetime', -1)
                until_datetime = self.get_argument('until_datetime', -1)
                sort_by_datetime = self.get_argument('sort_by_datetime', 0)

                tweets_coll = self.application.db.motor_column.tweets

                query_string = {'campaign_id' : '%s' % campaign_id}
                if since_datetime != -1:
                    if 'tweet.created_at' not in query_string:
                        query_string['tweet.created_at'] = {}
                    query_string['tweet.created_at']['$gte'] = float(since_datetime)
                if until_datetime != -1:
                    if 'tweet.created_at' not in query_string:
                        query_string['tweet.created_at'] = {}
                    query_string['tweet.created_at']['$lt'] = float(until_datetime)
                sort_string = []
                if sort_by_datetime != 0:
                    sort_string = [('tweet.created_at', pymongo.ASCENDING)] # ascending
                print "STARTED " + str(query_string)
                cursor = tweets_coll.find(query_string)
                if sort_string:
                    cursor = cursor.sort(sort_string)
                cursor = cursor.skip(int(skip)).limit(int(limit))
                           # TODO: removing because of complaint:
                           # TypeError: if no direction is specified, key_or_list must be an instance of list
                           # .sort({"$natural" : 1})\
                tmp = [x for x in (yield cursor.to_list(length=100))]
                print "ENDED " + str(query_string)
                DB_TEST_VERSION = 0.2
                if res_format == 'json':
                    self.write(bson.json_util.dumps(
                            {'results': tmp,
                            # TODO: Replace this DB_TEST_VERSION with source code
                            # version later
                            "direnaj_service_version": DB_TEST_VERSION,
                            # TODO: "requested_by": keywords['drnjID'],
                            "requested_by": "direnaj",
                            "campaign_id": campaign_id,
                            "served_at": drnj_time.now_in_drnj_time(),
                             'skip': int(skip),
                             'limit': int(limit)}))
                    self.add_header('Content-Type', 'application/json')
                elif res_format == 'html':
                    env = Environment(loader=FileSystemLoader('templates'))

                    template = env.get_template('statuses/filter.html')
                    result = template.render(statuses=[x['tweet'] for x in tmp])
                    self.write(result)

                self.finish()

            except MissingArgumentError as e:
                # TODO: implement logging.
                raise HTTPError(500, 'You didn''t supply %s as an argument' % e.arg_name)
            # db.tweets.find({'campaign_id' : 'syria'}).sort({$natural : 1}).skip(10).limit(14)
        else:
            # Tornado will make sure that this point will not be reached, if
            # it's regexp based router works correctly
            raise HTTPError(404, 'No such method')
<|endoftext|>


<commit_before>#!/usr/bin/python

import sys
from klampt import *
from klampt import vis
from klampt.io import resource
from klampt.model import coordinates
from klampt.math import vectorops,so3,se3
from klampt.model import trajectory
import random
import time
import math

#set to True to use vis in multithreaded mode
MULTITHREADED = False

def random_rotation():
    """Returns a uniformly distributed rotation matrix."""
    q = [random.gauss(0,1),random.gauss(0,1),random.gauss(0,1),random.gauss(0,1)]
    q = vectorops.unit(q)
    theta = math.acos(q[3])*2.0
    if abs(theta) < 1e-8:
        m = [0,0,0]
    else:
        m = vectorops.mul(vectorops.unit(q[0:3]),theta)
    return so3.from_moment(m)

if __name__ == "__main__":
    print "trajectorytest.py: This example demonstrates several types of trajectory"
    if len(sys.argv)<=1:
        print "USAGE: trajectorytest.py [robot or world file]"
        print "With no arguments, shows some 3D trajectories"

        #add a point to the visualizer and animate it
        point = coordinates.addPoint("point")
        vis.add("point",point)
        traj = trajectory.Trajectory()
        for i in range(10):
            traj.times.append(i)
            traj.milestones.append([random.uniform(-1,1),random.uniform(-1,1),random.uniform(-1,1)])

        traj2 = trajectory.HermiteTrajectory()
        traj2.makeSpline(traj)
        vis.animate("point",traj2)

        #add a transform to the visualizer and animate it
        xform = vis.add("xform",se3.identity())
        traj3 = trajectory.SE3Trajectory()
        for i in range(10):
            traj3.times.append(i)
            rrot = random_rotation()
            rpoint = [random.uniform(-1,1),random.uniform(-1,1),random.uniform(-1,1)]
            traj3.milestones.append(rrot+rpoint)

        vis.animate("xform",traj3)

        #testing adding the trajectory to the visualization... only works with piecewise linear Trajectories at the moment
        #vis.add("point_milestones",traj)
        #vis.add("point_milestones_smooth",traj2)    #not implemented yet
        vis.add("xform_milestones",traj3)
        vis.setAttribute("xform_milestones","width",1.0)
        vis.setColor("xform_milestones",1,1,0)
    else:
        #creates a world and loads all the items on the command line
        world = WorldModel()
        for fn in sys.argv[1:]:
            res = world.readFile(fn)
            if not res:
                raise RuntimeError("Unable to load model "+fn)

        #add the world to the visualizer
        robot = world.robot(0)
        vis.add("robot",robot)
        traj = trajectory.RobotTrajectory(robot)
        qmin,qmax = robot.getJointLimits()
        q0 = robot.getConfig()
        for i in range(robot.numLinks()):
            if math.isinf(qmin[i]) or math.isinf(qmax[i]):
                #don't animate floating base
                continue
            traj.times.append(len(traj.times)*0.5)
            q = q0[:]
            q[i] = qmin[i]
            traj.milestones.append(q)

            traj.times.append(len(traj.times)*0.5)
            q[i] = qmax[i]
            traj.milestones.append(q)

        save,traj.milestones = resource.edit("trajectory",traj.milestones,world=world)
        vis.animate("robot",traj)

    vis.show()

    #pop up the window to show the trajectory
    vis.spin(float('inf'))

    if len(sys.argv)>1:
        vis.animate("robot",None)
        sim = Simulator(world)
        sim.simulate(0)
        trajectory.execute_path(traj.milestones,sim.controller(0))
        if MULTITHREADED:
            #for some tricky Qt reason, need to sleep before showing a window again
            #Perhaps the event loop must complete some extra cycles?
            time.sleep(0.01)
            vis.show()
            t0 = time.time()
            while vis.shown():
                #print "Time",sim.getTime()
                sim.simulate(0.01)
                if sim.controller(0).remainingTime() <= 0:
                    print "Executing timed trajectory"
                    trajectory.execute_trajectory(traj,sim.controller(0),smoothing='pause')
                vis.setItemConfig("config",sim.controller(0).getCommandedConfig())
                t1 = time.time()
                time.sleep(max(0.01-(t1-t0),0.0))
                t0 = t1
        else:
            data = {'next_sim_time':time.time()}
            def callback():
                if time.time() >= data['next_sim_time']:
                    sim.simulate(0.01)
                    if sim.controller(0).remainingTime() <= 0:
                        print "Executing timed trajectory"
                        trajectory.execute_trajectory(traj,sim.controller(0),smoothing='pause')
                    vis.setItemConfig("config",sim.controller(0).getCommandedConfig())
                    data['next_sim_time'] += 0.01
            vis.loop(callback=callback,setup=vis.show)
    print "Ending vis."
    vis.kill()
<commit_msg>Fixed for dynamic multithreaded determination<commit_after>#!/usr/bin/python

import sys
from klampt import *
from klampt import vis
from klampt.io import resource
from klampt.model import coordinates
from klampt.math import vectorops,so3,se3
from klampt.model import trajectory
import random
import time
import math

def random_rotation():
    """Returns a uniformly distributed rotation matrix."""
    q = [random.gauss(0,1),random.gauss(0,1),random.gauss(0,1),random.gauss(0,1)]
    q = vectorops.unit(q)
    theta = math.acos(q[3])*2.0
    if abs(theta) < 1e-8:
        m = [0,0,0]
    else:
        m = vectorops.mul(vectorops.unit(q[0:3]),theta)
    return so3.from_moment(m)

if __name__ == "__main__":
    print "trajectorytest.py: This example demonstrates several types of trajectory"
    if len(sys.argv)<=1:
        print "USAGE: trajectorytest.py [robot or world file]"
        print "With no arguments, shows some 3D trajectories"

        #add a point to the visualizer and animate it
        point = coordinates.addPoint("point")
        vis.add("point",point)
        traj = trajectory.Trajectory()
        for i in range(10):
            traj.times.append(i)
            traj.milestones.append([random.uniform(-1,1),random.uniform(-1,1),random.uniform(-1,1)])

        traj2 = trajectory.HermiteTrajectory()
        traj2.makeSpline(traj)
        vis.animate("point",traj2)

        #add a transform to the visualizer and animate it
        xform = vis.add("xform",se3.identity())
        traj3 = trajectory.SE3Trajectory()
        for i in range(10):
            traj3.times.append(i)
            rrot = random_rotation()
            rpoint = [random.uniform(-1,1),random.uniform(-1,1),random.uniform(-1,1)]
            traj3.milestones.append(rrot+rpoint)

        vis.animate("xform",traj3)

        #testing adding the trajectory to the visualization... only works with piecewise linear Trajectories at the moment
        #vis.add("point_milestones",traj)
        #vis.add("point_milestones_smooth",traj2)    #not implemented yet
        vis.add("xform_milestones",traj3)
        vis.setAttribute("xform_milestones","width",1.0)
        vis.setColor("xform_milestones",1,1,0)
    else:
        #creates a world and loads all the items on the command line
        world = WorldModel()
        for fn in sys.argv[1:]:
            res = world.readFile(fn)
            if not res:
                raise RuntimeError("Unable to load model "+fn)

        #add the world to the visualizer
        robot = world.robot(0)
        vis.add("robot",robot)
        traj = trajectory.RobotTrajectory(robot)
        qmin,qmax = robot.getJointLimits()
        q0 = robot.getConfig()
        for i in range(robot.numLinks()):
            if math.isinf(qmin[i]) or math.isinf(qmax[i]):
                #don't animate floating base
                continue
            traj.times.append(len(traj.times)*0.5)
            q = q0[:]
            q[i] = qmin[i]
            traj.milestones.append(q)

            traj.times.append(len(traj.times)*0.5)
            q[i] = qmax[i]
            traj.milestones.append(q)

        save,traj.milestones = resource.edit("trajectory",traj.milestones,world=world)
        vis.animate("robot",traj)

    #pop up the window to show the trajectory
    vis.spin(float('inf'))

    if len(sys.argv)>1:
        vis.animate("robot",None)
        sim = Simulator(world)
        sim.simulate(0)
        trajectory.execute_path(traj.milestones,sim.controller(0))
        vis.setWindowTitle("Simulating path using trajectory.execute_trajectory")
        if vis.multithreaded():
            #for some tricky Qt reason, need to sleep before showing a window again
            #Perhaps the event loop must complete some extra cycles?
            time.sleep(0.01)
            vis.show()
            t0 = time.time()
            while vis.shown():
                #print "Time",sim.getTime()
                sim.simulate(0.01)
                if sim.controller(0).remainingTime() <= 0:
                    print "Executing timed trajectory"
                    trajectory.execute_trajectory(traj,sim.controller(0),smoothing='pause')
                vis.setItemConfig("robot",sim.controller(0).getCommandedConfig())
                t1 = time.time()
                time.sleep(max(0.01-(t1-t0),0.0))
                t0 = t1
        else:
            data = {'next_sim_time':time.time()}
            def callback():
                if time.time() >= data['next_sim_time']:
                    sim.simulate(0.01)
                    if sim.controller(0).remainingTime() <= 0:
                        print "Executing timed trajectory"
                        trajectory.execute_trajectory(traj,sim.controller(0),smoothing='pause')
                    vis.setItemConfig("robot",sim.controller(0).getCommandedConfig())
                    data['next_sim_time'] += 0.01
            vis.loop(callback=callback,setup=vis.show)
    print "Ending vis."
    vis.kill()
<|endoftext|>


<commit_before>import os
import sys
from six import iteritems

from . import build
from . import log
from . import path
from .arguments import parser as argparse
from .backends import list_backends
from .environment import Environment, EnvVersionError
from .platforms import platform_info
from .app_version import version

logger = log.getLogger(__name__)

description = """
bfg9000 ("build file generator") is a cross-platform build configuration system
with an emphasis on making it easy to define how to build your software. It
converts a Python-based build script into the appropriate files for your
underlying build system of choice.
"""

configure_desc = """
Generate the necessary build files to perform actual builds. If DIRECTORY is a
source directory (i.e. it contains a build.bfg file), the build files will be
created in the current directory. Otherwise, DIRECTORY is treated as the build
directory, and bfg9000 will look for a build.bfg file in the current directory.
"""

configureinto_desc = """
Generate the necessary build files to perform actual builds from a build.bfg
file in SRCDIR, and place them in BUILDDIR.
"""

refresh_desc = """
Regenerate an existing set of build files needed to perform actual builds. This
is run automatically if bfg9000 determines that the build files are out of
date.
"""

env_desc = """
Print the environment variables stored by this build configuration.
"""

def environment_from_args(args, extra_args=None):
    # Get the bin directory holding bfg's executables.
    bfgdir = path.abspath(sys.argv[0]).parent()

    backend = list_backends()[args.backend]
    env = Environment(
        bfgdir=bfgdir,
        backend=args.backend,
        backend_version=backend.version(),
        srcdir=args.srcdir,
        builddir=args.builddir,
        install_dirs={i: getattr(args, i.name) for i in path.InstallRoot},
        library_mode=(args.shared, args.static),
        extra_args=extra_args,
    )

    return env, backend

class Directory(object):
    def __init__(self, must_exist=False):
        self.must_exist = must_exist

    def __call__(self, string):
        if os.path.exists(string):
            if not os.path.isdir(string):
                raise ValueError("'{}' is not a directory".format(string))
        elif self.must_exist:
            raise ValueError("'{}' does not exist".format(string))

        return path.abspath(string)

def directory_pair(srcname, buildname):
    class DirectoryPair(argparse.Action):
        def __call__(self, parser, namespace, values, option_string=None):
            cwd = path.abspath('.')

            if build.is_srcdir(values):
                srcdir, builddir = values, cwd
            else:
                srcdir, builddir = cwd, values

            setattr(namespace, srcname, srcdir)
            setattr(namespace, buildname, builddir)

    return DirectoryPair

class ConfigureHelp(argparse.Action):
    def __init__(self, option_strings, dest=argparse.SUPPRESS,
                 default=argparse.SUPPRESS, help=None):
        argparse.Action.__init__(
            self, option_strings=option_strings, dest=dest, default=default,
            nargs=0, help=help
        )

    def __call__(self, parser, namespace, values, option_string=None):
        cwd = path.abspath('.')
        if not hasattr(namespace, 'srcdir') and build.is_srcdir(cwd):
            namespace.srcdir = cwd
            namespace.builddir = None

        if getattr(namespace, 'srcdir', None):
            env, backend = environment_from_args(namespace)
            build.print_user_help(env, parser)
        else:
            parser.print_help()
        parser.exit()

def add_generic_args(parser):
    parser.add_argument('--version', action='version',
                        version='%(prog)s ' + version)
    parser.add_argument('--debug', action='store_true', help=argparse.SUPPRESS)
    parser.add_argument('--color', metavar='WHEN',
                        choices=['always', 'never', 'auto'], default='auto',
                        help=('show colored output (one of: %(choices)s; ' +
                              'default: %(default)s)'))
    parser.add_argument('-c', action='store_const', const='always',
                        dest='color',
                        help=('show colored output (equivalent to ' +
                              '`--color=always`)'))
    parser.add_argument('--warn-once', action='store_true',
                        help='only emit a given warning once')

def add_configure_args(parser):
    backends = list_backends()

    parser.add_argument('-h', '--help', action=ConfigureHelp,
                        help='show this help message and exit')

    build = parser.add_argument_group('build arguments')
    build.add_argument('--backend', metavar='BACKEND',
                       choices=list(backends.keys()),
                       default=list(backends.keys())[0],
                       help=('build backend (one of %(choices)s; default: ' +
                             '%(default)s)'))
    build.add_argument('--shared', action='enable', default=True,
                       help='build shared libraries (default: enabled)')
    build.add_argument('--static', action='enable', default=False,
                       help='build static libraries (default: disabled)')

    install_dirs = platform_info().install_dirs
    common_path_help = 'installation path for {} (default: %(default)r)'
    path_help = {
        'prefix': 'installation prefix (default: %(default)r)',
        'exec_prefix': ('installation prefix for architecture-dependent ' +
                        'files (default: %(default)r)'),
        'bindir': common_path_help.format('executables'),
        'libdir': common_path_help.format('libraries'),
        'includedir': common_path_help.format('headers'),
    }

    install = parser.add_argument_group('installation arguments')
    for root in path.InstallRoot:
        name = '--' + root.name.replace('_', '-')
        install.add_argument(name, type=Directory(), metavar='PATH',
                             default=install_dirs[root],
                             help=path_help[root.name])

def configure(parser, args, extra):
    if not build.is_srcdir(args.srcdir):
        parser.error('source directory must contain a {} file'
                     .format(build.bfgfile))
    if build.is_srcdir(args.builddir):
        parser.error('build directory must not contain a {} file'
                     .format(build.bfgfile))

    if path.exists(args.builddir):
        if path.samefile(args.srcdir, args.builddir):
            parser.error('source and build directories must be different')
    else:
        os.mkdir(args.builddir.string())

    env, backend = environment_from_args(args, extra)
    env.save(args.builddir.string())
    try:
        argv = build.parse_user_args(env)
        build_inputs = build.execute_script(env, argv)
        backend.write(env, build_inputs)
    except Exception as e:
        logger.exception(e)
        return 1

def refresh(parser, args, extra):
    if extra:
        parser.error('unrecongized arguments: {}'.format(' '.join(extra)))

    if build.is_srcdir(args.builddir):
        parser.error('build directory must not contain a {} file'
                     .format(build.bfgfile))

    try:
        env = Environment.load(args.builddir.string())

        backend = list_backends()[env.backend]
        argv = build.parse_user_args(env)
        build_inputs = build.execute_script(env, argv)
        backend.write(env, build_inputs)
    except Exception as e:
        msg = 'Unable to reload environment'
        if str(e):
            msg += ': {}'.format(str(e))
        if isinstance(e, EnvVersionError):
            msg += '\n  Please re-run bfg9000 manually'
        logger.error(msg, exc_info=True)
        return 1

def env(parser, args, extra):
    if extra:
        parser.error('unrecongized arguments: {}'.format(' '.join(extra)))

    if build.is_srcdir(args.builddir):
        parser.error('build directory must not contain a {} file'
                     .format(build.bfgfile))

    try:
        env = Environment.load(args.builddir.string())

        for k, v in sorted(iteritems(env.variables)):
            if not args.unique or os.getenv(k) != v:
                print('{}={}'.format(k, v))
    except Exception as e:
        msg = 'Unable to reload environment'
        if str(e):
            msg += ': {}'.format(str(e))
        logger.error(msg, exc_info=True)
        return 1

def help(parser, args, extra):
    parser.parse_args(extra + ['--help'])

def main():
    parser = argparse.ArgumentParser(prog='bfg9000', description=description)
    subparsers = parser.add_subparsers()

    add_generic_args(parser)

    conf_p = subparsers.add_parser(
        'configure', description=configure_desc, add_help=False,
        help='create build files'
    )
    conf_p.set_defaults(func=configure)
    conf_p.add_argument(argparse.SUPPRESS,
                        action=directory_pair('srcdir', 'builddir'),
                        type=Directory(), metavar='DIRECTORY',
                        help='source or build directory')
    add_configure_args(conf_p)

    confinto_p = subparsers.add_parser(
        'configure-into', description=configureinto_desc, add_help=False,
        help='create build files in a chosen directory'
    )
    confinto_p.add_argument('srcdir', type=Directory(must_exist=True),
                            metavar='SRCDIR', help='source directory')
    confinto_p.add_argument('builddir', type=Directory(), metavar='BUILDDIR',
                            help='build directory')
    confinto_p.set_defaults(func=configure)
    add_configure_args(confinto_p)

    refresh_p = subparsers.add_parser(
        'refresh', description=refresh_desc, help='regenerate build files'
    )
    refresh_p.set_defaults(func=refresh)
    refresh_p.add_argument('builddir', type=Directory(must_exist=True),
                           metavar='BUILDDIR', nargs='?', default='.',
                           help='build directory')

    env_p = subparsers.add_parser(
        'env', description=env_desc, help='print environment'
    )
    env_p.set_defaults(func=env)
    env_p.add_argument('-u', '--unique', action='store_true',
                       help='only show variables that differ from the ' +
                       'current environment')
    env_p.add_argument('builddir', type=Directory(must_exist=True),
                       metavar='BUILDDIR', nargs='?', default='.',
                       help='build directory')

    help_p = subparsers.add_parser(
        'help', help='show this help message and exit', add_help=False
    )
    help_p.set_defaults(func=help)

    args, extra = parser.parse_known_args()
    log.init(args.color, debug=args.debug, warn_once=args.warn_once)

    return args.func(parser, args, extra)

def simple_main():
    parser = argparse.ArgumentParser(prog='9k', description=configure_desc,
                                     add_help=False)
    parser.add_argument(argparse.SUPPRESS,
                        action=directory_pair('srcdir', 'builddir'),
                        type=Directory(), metavar='DIRECTORY',
                        help='source or build directory')
    add_generic_args(parser)
    add_configure_args(parser)

    args, extra = parser.parse_known_args()
    log.init(args.color, debug=args.debug, warn_once=args.warn_once)

    return configure(parser, args, extra)
<commit_msg>Factor out env-reload exception handling into a common function<commit_after>import os
import sys
from six import iteritems

from . import build
from . import log
from . import path
from .arguments import parser as argparse
from .backends import list_backends
from .environment import Environment, EnvVersionError
from .platforms import platform_info
from .app_version import version

logger = log.getLogger(__name__)

description = """
bfg9000 ("build file generator") is a cross-platform build configuration system
with an emphasis on making it easy to define how to build your software. It
converts a Python-based build script into the appropriate files for your
underlying build system of choice.
"""

configure_desc = """
Generate the necessary build files to perform actual builds. If DIRECTORY is a
source directory (i.e. it contains a build.bfg file), the build files will be
created in the current directory. Otherwise, DIRECTORY is treated as the build
directory, and bfg9000 will look for a build.bfg file in the current directory.
"""

configureinto_desc = """
Generate the necessary build files to perform actual builds from a build.bfg
file in SRCDIR, and place them in BUILDDIR.
"""

refresh_desc = """
Regenerate an existing set of build files needed to perform actual builds. This
is run automatically if bfg9000 determines that the build files are out of
date.
"""

env_desc = """
Print the environment variables stored by this build configuration.
"""

def handle_reload_exception(e, suggest_rerun=False):
    msg = 'Unable to reload environment'
    if str(e):
        msg += ': {}'.format(str(e))
    if suggest_rerun and isinstance(e, EnvVersionError):
        msg += '\n  Please re-run bfg9000 manually'
    logger.error(msg, exc_info=True)
    return 1

def environment_from_args(args, extra_args=None):
    # Get the bin directory holding bfg's executables.
    bfgdir = path.abspath(sys.argv[0]).parent()

    backend = list_backends()[args.backend]
    env = Environment(
        bfgdir=bfgdir,
        backend=args.backend,
        backend_version=backend.version(),
        srcdir=args.srcdir,
        builddir=args.builddir,
        install_dirs={i: getattr(args, i.name) for i in path.InstallRoot},
        library_mode=(args.shared, args.static),
        extra_args=extra_args,
    )

    return env, backend

class Directory(object):
    def __init__(self, must_exist=False):
        self.must_exist = must_exist

    def __call__(self, string):
        if os.path.exists(string):
            if not os.path.isdir(string):
                raise ValueError("'{}' is not a directory".format(string))
        elif self.must_exist:
            raise ValueError("'{}' does not exist".format(string))

        return path.abspath(string)

def directory_pair(srcname, buildname):
    class DirectoryPair(argparse.Action):
        def __call__(self, parser, namespace, values, option_string=None):
            cwd = path.abspath('.')

            if build.is_srcdir(values):
                srcdir, builddir = values, cwd
            else:
                srcdir, builddir = cwd, values

            setattr(namespace, srcname, srcdir)
            setattr(namespace, buildname, builddir)

    return DirectoryPair

class ConfigureHelp(argparse.Action):
    def __init__(self, option_strings, dest=argparse.SUPPRESS,
                 default=argparse.SUPPRESS, help=None):
        argparse.Action.__init__(
            self, option_strings=option_strings, dest=dest, default=default,
            nargs=0, help=help
        )

    def __call__(self, parser, namespace, values, option_string=None):
        cwd = path.abspath('.')
        if not hasattr(namespace, 'srcdir') and build.is_srcdir(cwd):
            namespace.srcdir = cwd
            namespace.builddir = None

        if getattr(namespace, 'srcdir', None):
            env, backend = environment_from_args(namespace)
            build.print_user_help(env, parser)
        else:
            parser.print_help()
        parser.exit()

def add_generic_args(parser):
    parser.add_argument('--version', action='version',
                        version='%(prog)s ' + version)
    parser.add_argument('--debug', action='store_true', help=argparse.SUPPRESS)
    parser.add_argument('--color', metavar='WHEN',
                        choices=['always', 'never', 'auto'], default='auto',
                        help=('show colored output (one of: %(choices)s; ' +
                              'default: %(default)s)'))
    parser.add_argument('-c', action='store_const', const='always',
                        dest='color',
                        help=('show colored output (equivalent to ' +
                              '`--color=always`)'))
    parser.add_argument('--warn-once', action='store_true',
                        help='only emit a given warning once')

def add_configure_args(parser):
    backends = list_backends()

    parser.add_argument('-h', '--help', action=ConfigureHelp,
                        help='show this help message and exit')

    build = parser.add_argument_group('build arguments')
    build.add_argument('--backend', metavar='BACKEND',
                       choices=list(backends.keys()),
                       default=list(backends.keys())[0],
                       help=('build backend (one of %(choices)s; default: ' +
                             '%(default)s)'))
    build.add_argument('--shared', action='enable', default=True,
                       help='build shared libraries (default: enabled)')
    build.add_argument('--static', action='enable', default=False,
                       help='build static libraries (default: disabled)')

    install_dirs = platform_info().install_dirs
    common_path_help = 'installation path for {} (default: %(default)r)'
    path_help = {
        'prefix': 'installation prefix (default: %(default)r)',
        'exec_prefix': ('installation prefix for architecture-dependent ' +
                        'files (default: %(default)r)'),
        'bindir': common_path_help.format('executables'),
        'libdir': common_path_help.format('libraries'),
        'includedir': common_path_help.format('headers'),
    }

    install = parser.add_argument_group('installation arguments')
    for root in path.InstallRoot:
        name = '--' + root.name.replace('_', '-')
        install.add_argument(name, type=Directory(), metavar='PATH',
                             default=install_dirs[root],
                             help=path_help[root.name])

def configure(parser, args, extra):
    if not build.is_srcdir(args.srcdir):
        parser.error('source directory must contain a {} file'
                     .format(build.bfgfile))
    if build.is_srcdir(args.builddir):
        parser.error('build directory must not contain a {} file'
                     .format(build.bfgfile))

    if path.exists(args.builddir):
        if path.samefile(args.srcdir, args.builddir):
            parser.error('source and build directories must be different')
    else:
        os.mkdir(args.builddir.string())

    env, backend = environment_from_args(args, extra)
    env.save(args.builddir.string())
    try:
        argv = build.parse_user_args(env)
        build_inputs = build.execute_script(env, argv)
        backend.write(env, build_inputs)
    except Exception as e:
        logger.exception(e)
        return 1

def refresh(parser, args, extra):
    if extra:
        parser.error('unrecongized arguments: {}'.format(' '.join(extra)))

    if build.is_srcdir(args.builddir):
        parser.error('build directory must not contain a {} file'
                     .format(build.bfgfile))

    try:
        env = Environment.load(args.builddir.string())

        backend = list_backends()[env.backend]
        argv = build.parse_user_args(env)
        build_inputs = build.execute_script(env, argv)
        backend.write(env, build_inputs)
    except Exception as e:
        return handle_reload_exception(e, suggest_rerun=True)

def env(parser, args, extra):
    if extra:
        parser.error('unrecongized arguments: {}'.format(' '.join(extra)))

    if build.is_srcdir(args.builddir):
        parser.error('build directory must not contain a {} file'
                     .format(build.bfgfile))

    try:
        env = Environment.load(args.builddir.string())

        for k, v in sorted(iteritems(env.variables)):
            if not args.unique or os.getenv(k) != v:
                print('{}={}'.format(k, v))
    except Exception as e:
        return handle_reload_exception(e)

def help(parser, args, extra):
    parser.parse_args(extra + ['--help'])

def main():
    parser = argparse.ArgumentParser(prog='bfg9000', description=description)
    subparsers = parser.add_subparsers()

    add_generic_args(parser)

    conf_p = subparsers.add_parser(
        'configure', description=configure_desc, add_help=False,
        help='create build files'
    )
    conf_p.set_defaults(func=configure)
    conf_p.add_argument(argparse.SUPPRESS,
                        action=directory_pair('srcdir', 'builddir'),
                        type=Directory(), metavar='DIRECTORY',
                        help='source or build directory')
    add_configure_args(conf_p)

    confinto_p = subparsers.add_parser(
        'configure-into', description=configureinto_desc, add_help=False,
        help='create build files in a chosen directory'
    )
    confinto_p.add_argument('srcdir', type=Directory(must_exist=True),
                            metavar='SRCDIR', help='source directory')
    confinto_p.add_argument('builddir', type=Directory(), metavar='BUILDDIR',
                            help='build directory')
    confinto_p.set_defaults(func=configure)
    add_configure_args(confinto_p)

    refresh_p = subparsers.add_parser(
        'refresh', description=refresh_desc, help='regenerate build files'
    )
    refresh_p.set_defaults(func=refresh)
    refresh_p.add_argument('builddir', type=Directory(must_exist=True),
                           metavar='BUILDDIR', nargs='?', default='.',
                           help='build directory')

    env_p = subparsers.add_parser(
        'env', description=env_desc, help='print environment'
    )
    env_p.set_defaults(func=env)
    env_p.add_argument('-u', '--unique', action='store_true',
                       help='only show variables that differ from the ' +
                       'current environment')
    env_p.add_argument('builddir', type=Directory(must_exist=True),
                       metavar='BUILDDIR', nargs='?', default='.',
                       help='build directory')

    help_p = subparsers.add_parser(
        'help', help='show this help message and exit', add_help=False
    )
    help_p.set_defaults(func=help)

    args, extra = parser.parse_known_args()
    log.init(args.color, debug=args.debug, warn_once=args.warn_once)

    return args.func(parser, args, extra)

def simple_main():
    parser = argparse.ArgumentParser(prog='9k', description=configure_desc,
                                     add_help=False)
    parser.add_argument(argparse.SUPPRESS,
                        action=directory_pair('srcdir', 'builddir'),
                        type=Directory(), metavar='DIRECTORY',
                        help='source or build directory')
    add_generic_args(parser)
    add_configure_args(parser)

    args, extra = parser.parse_known_args()
    log.init(args.color, debug=args.debug, warn_once=args.warn_once)

    return configure(parser, args, extra)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2017 Shoichi Sakane <sakane@tanu.org>, All rights reserved.
# See the file LICENSE in the top level directory for more details.
#

import sys
import json
import logging
from functools import wraps
from bottle import Bottle, GeventServer, request, abort
from gevent import monkey; monkey.patch_all()
from importlib import import_module
from argparse import ArgumentParser

PROG_NAME = "lrwssas"
KEY_TOPOBJ = "DevEUI_uplink"
KEY_TIME = "Time"
KEY_PAYLOAD_HEX = "payload_hex"
KEY_DEVEUI = "DevEUI"
KEY_APP_DATA = "__app_data"

CONF_MODULE_PATH = "module_path"
CONF_SENSORS = "sensors"
CONF_SENSOR_DESC = "desc"
CONF_SENSOR_HANDLER = "handler"
CONF_HANDLERS = "handlers"
CONF_PARSER = "parser"
CONF_DB = "db"
CONF_DB_CONN = "connector"
CONF_DB_ARGS = "args"
CONF_SERVER_ADDR = "server_addr"
CONF_SERVER_PORT = "server_port"
CONF_SERVER_CERT = "server_cert"
CONF_DEBUG_LEVEL = "debug_level"
CONF_OPT_DEBUG = "opt_debug"
CONF_LOG_STDOUT = "log_stdout"
CONF_TZ = "tz"

LOG_FMT = "%(asctime)s.%(msecs)d %(lineno)d %(message)s"
LOG_DATE_FMT = "%Y-%m-%dT%H:%M:%S"

app = Bottle()
logger = logging.getLogger(PROG_NAME)
handler_map = {}

@app.route("/up", method="POST")
def app_up():
    log_common(request)
    if config[CONF_DEBUG_LEVEL] > 1:
        payload = request.body.read()
        logger.debug("---BEGIN OF REQUESTED HEADER---")
        for k,v in request.headers.items():
            logger.debug("{}: {}".format(k,v))
        logger.debug("---END OF REQUESTED HEADER---")
        logger.debug("---BEGIN OF REQUESTED DATA---")
        logger.debug(payload)
        logger.debug("---END OF REQUESTED DATA---")
        request.body.seek(0)
    # convert the payload into JSON.
    try:
        kv_data = json.load(request.body)
    except json.decoder.JSONDecodeError as e:
        emsg = "The payload format was not likely JSON."
        logger.error("{}: {}".format(emsg, e))
        abort(404, emsg)
    #
    kv_root = kv_data.get(KEY_TOPOBJ)
    if not kv_root:
        emsg = "Key {} does not exist in the request.".format(KEY_TOPOBJ)
        logger.error("{}".format(emsg))
        abort(404, emsg)
    #
    ret = proc_tp_uplink(kv_root)
    if ret:
        abort(ret)
    return "OK"

def proc_tp_uplink(kv_data):
    """
    kv_data is a Python object, which is seriaized of the JSON-like string
    of KEY_TOPOBJ sent by a TP NS.
    assuming that all required keys exist in the config.
    """
    # get deveui
    deveui = kv_data.get(KEY_DEVEUI)
    if not deveui:
        logger.error("Key {} does not exist in the request.".format(KEY_DEVEUI))
        return 400  # bad request.
    # get payload_hex
    payload_hex = kv_data.get(KEY_PAYLOAD_HEX)
    if not payload_hex:
        logger.error("Key {} does not exist in the request."
                     .format(KEY_PAYLOAD_HEX))
        return 400  # bad request.
    # get time, for processing later.
    kv_time = kv_data.get(KEY_TIME)
    if not kv_time:
        logger.error("key {} does not exist in the request.".format(KEY_TIME))
        return 400  # bad request.
    #
    logger.info("Received data from {}".format(deveui))
    #
    # get the parser.
    sensor_def = config[CONF_SENSORS].get(deveui)
    if not sensor_def:
        logger.error("DevEUI {} is not defined in the config.".format(deveui))
        return 403  # not found.
    """
    call parser.parse() to try converting the payload_hex.
    if succeeded, it is set into KEY_APP_DATA.
    """
    handler_name = sensor_def.get(CONF_SENSOR_HANDLER)
    handler = handler_map[handler_name]
    logger.debug("Applied Handler: {}".format(handler_name))
    if handler.parser:
        kv_payload = handler.parser.parse_hex(payload_hex)
    else:
        kv_payload = payload_hex
    """
    return value of parser.paser():
        None: ignore parsing.
        False: something error.
        Others: a Python dict should be put in KEY_APP_DATA.
    """
    if kv_payload is None:
        logger.debug("The parser returned None.")
        pass
    elif kv_payload is False:
        # if kv_payload is False variants except of None, then error.
        logger.error("Parsing payload for DevEUI {} failed.".format(deveui))
        return 400  # bad request.
    else:
        kv_data[KEY_APP_DATA] = kv_payload
        if handler.db_conn:
            logger.debug("db_submit() has been called.")
            handler.db_conn.db_submit(kv_data)
    return 0

def log_common(request):
    logger.info("Access from {} {} {}".format(request.remote_addr,
                                              request.method, request.url))

def set_logger(config):
    #
    # set logger.
    #   "syslog", not yet
    #   "stdout"
    #   filename
    #
    log_file = config.get("log_file", "stdout")
    #
    if config[CONF_LOG_STDOUT] or log_file == "stdout":
        ch = logging.StreamHandler()
    else:
        ch = logging.FileHandler(log_file)
    #
    ch.setFormatter(logging.Formatter(fmt=LOG_FMT, datefmt=LOG_DATE_FMT))
    ch.setLevel(logging.DEBUG)
    logger.addHandler(ch)
    if config[CONF_DEBUG_LEVEL] > 0:
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)

def check_config(config):
    # set PYTHONPATH
    sys.path.extend(["./parsers", "./db_connectors"])
    for i in config.get(CONF_MODULE_PATH, []):
        sys.path.append(i)
    # check if the sensors key exists.
    sensors = config.get(CONF_SENSORS)
    if not sensors:
        print("ERROR: Key {} does not exist in the config."
              .format(CONF_SENSORS))
        return False
    # check if the handlers key exists.
    handlers = config.get(CONF_HANDLERS)
    if not handlers:
        print("ERROR: Key {} does not exist in the config."
              .format(CONF_HANDLERS))
        return False
    #
    config.setdefault(CONF_SERVER_ADDR, "::")
    config.setdefault(CONF_SERVER_PORT, "18886")
    config.setdefault(CONF_SERVER_CERT, None)
    config.setdefault(CONF_DEBUG_LEVEL, 0)
    config.setdefault(CONF_TZ, "Asia/Tokyo")
    # overwrite the debug level if opt.debug is True.
    if config[CONF_OPT_DEBUG] == True and config[CONF_DEBUG_LEVEL] == 0:
        config[CONF_DEBUG_LEVEL] = 99
    #
    # NOTE: below check should be placed at the end of this method.
    # check if the handlers are properly defined.
    for deveui,defs in sensors.items():
        handler_name = defs.get(CONF_SENSOR_HANDLER)
        if handler_name is None:
            print("ERROR: sensor {} doesn't have handler.".format(deveui))
            return False
        handler = handlers.get(handler_name)
        if not handler:
            print("ERROR: handler for {} is not defined in the config."
                  .format(deveui))
            return False
        #
        parser_handler = None
        parser_name = handler.get(CONF_PARSER)
        if parser_name:
            try:
                mod = import_module("parser_{}".format(parser_name))
            except Exception as e:
                print("ERROR: importing parser failed for {}, {}"
                      .format(parser_name, e))
                return False
            parser_handler = mod.parser(
                    logger=logger,
                    debug_level=config[CONF_DEBUG_LEVEL])
        #
        db_handler = None
        if handler.get(CONF_DB):
            db_conn_name = handler.get(CONF_DB).get(CONF_DB_CONN)
            if not db_conn_name:
                print("ERROR: {} for {} is not defined."
                      .format(CONF_DB_CONN, handler_name))
                return False
            try:
                mod = import_module("db_connector_{}".format(db_conn_name))
            except Exception as e:
                print("ERROR: importing db_connector failed for {}, {}"
                        .format(db_conn_name, e))
                return False
            db_handler = mod.db_connector(
                    logger=logger,
                    tz=config[CONF_TZ],
                    debug_level=config[CONF_DEBUG_LEVEL],
                    **handler[CONF_DB][CONF_DB_ARGS])
        #
        handler_map[handler_name] = type("handler", (object,), {})
        handler_map[handler_name].parser = parser_handler
        handler_map[handler_name].db_conn = db_handler
    return True

"""
main
"""
ap = ArgumentParser()
ap.add_argument("config_file", metavar="CONFIG_FILE",
                help="specify the config file.")
ap.add_argument("-d", action="store_true", dest="debug",
               help="enable debug mode.")
ap.add_argument("-D", action="store_true", dest="log_stdout",
               help="enable to show messages onto stdout.")
opt = ap.parse_args()
# get config.
try:
    config = json.load(open(opt.config_file))
except Exception as e:
    print("ERROR: {} read error. {}".format(opt.config_file, e))
    exit(1)
# update config.
config[CONF_OPT_DEBUG] = opt.debug
config[CONF_LOG_STDOUT] = opt.log_stdout
if not check_config(config):
    print("ERROR: error in {}.".format(opt.config_file))
    exit(1)
#
set_logger(config)
#
server_addr = config.get("server_addr")
server_port = int(config.get("server_port"))
server_cert = config.get("server_cert")
server_scheme = "https" if server_cert else "http"
logger.info("Starting {} listening on {}://{}:{}/"
            .format(PROG_NAME, server_scheme, server_addr, server_port))
#
if server_cert:
    app.run(host=server_addr, port=server_port,
            certfile=server_cert,
            server=GeventServer, quiet=True, debug=False)
else:
    app.run(host=server_addr, port=server_port,
            server=GeventServer, quiet=True, debug=False)
<commit_msg>fixed to enter the debug mode.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2017 Shoichi Sakane <sakane@tanu.org>, All rights reserved.
# See the file LICENSE in the top level directory for more details.
#

import sys
import json
import logging
from functools import wraps
from bottle import Bottle, GeventServer, request, abort
from gevent import monkey; monkey.patch_all()
from importlib import import_module
from argparse import ArgumentParser

PROG_NAME = "lrwssas"
KEY_TOPOBJ = "DevEUI_uplink"
KEY_TIME = "Time"
KEY_PAYLOAD_HEX = "payload_hex"
KEY_DEVEUI = "DevEUI"
KEY_APP_DATA = "__app_data"

CONF_MODULE_PATH = "module_path"
CONF_SENSORS = "sensors"
CONF_SENSOR_DESC = "desc"
CONF_SENSOR_HANDLER = "handler"
CONF_HANDLERS = "handlers"
CONF_PARSER = "parser"
CONF_DB = "db"
CONF_DB_CONN = "connector"
CONF_DB_ARGS = "args"
CONF_SERVER_ADDR = "server_addr"
CONF_SERVER_PORT = "server_port"
CONF_SERVER_CERT = "server_cert"
CONF_DEBUG_LEVEL = "debug_level"
CONF_OPT_DEBUG = "opt_debug"
CONF_LOG_STDOUT = "log_stdout"
CONF_TZ = "tz"

LOG_FMT = "%(asctime)s.%(msecs)d %(lineno)d %(message)s"
LOG_DATE_FMT = "%Y-%m-%dT%H:%M:%S"

app = Bottle()
logger = logging.getLogger(PROG_NAME)
handler_map = {}

@app.route("/up", method="POST")
def app_up():
    log_common(request)
    if config[CONF_DEBUG_LEVEL] > 1:
        payload = request.body.read()
        logger.debug("---BEGIN OF REQUESTED HEADER---")
        for k,v in request.headers.items():
            logger.debug("{}: {}".format(k,v))
        logger.debug("---END OF REQUESTED HEADER---")
        logger.debug("---BEGIN OF REQUESTED DATA---")
        logger.debug(payload)
        logger.debug("---END OF REQUESTED DATA---")
        request.body.seek(0)
    # convert the payload into JSON.
    try:
        kv_data = json.load(request.body)
    except json.decoder.JSONDecodeError as e:
        emsg = "The payload format was not likely JSON."
        logger.error("{}: {}".format(emsg, e))
        abort(404, emsg)
    #
    kv_root = kv_data.get(KEY_TOPOBJ)
    if not kv_root:
        emsg = "Key {} does not exist in the request.".format(KEY_TOPOBJ)
        logger.error("{}".format(emsg))
        abort(404, emsg)
    #
    ret = proc_tp_uplink(kv_root)
    if ret:
        abort(ret)
    return "OK"

def proc_tp_uplink(kv_data):
    """
    kv_data is a Python object, which is seriaized of the JSON-like string
    of KEY_TOPOBJ sent by a TP NS.
    assuming that all required keys exist in the config.
    """
    # get deveui
    deveui = kv_data.get(KEY_DEVEUI)
    if not deveui:
        logger.error("Key {} does not exist in the request.".format(KEY_DEVEUI))
        return 400  # bad request.
    # get payload_hex
    payload_hex = kv_data.get(KEY_PAYLOAD_HEX)
    if not payload_hex:
        logger.error("Key {} does not exist in the request."
                     .format(KEY_PAYLOAD_HEX))
        return 400  # bad request.
    # get time, for processing later.
    kv_time = kv_data.get(KEY_TIME)
    if not kv_time:
        logger.error("key {} does not exist in the request.".format(KEY_TIME))
        return 400  # bad request.
    #
    logger.info("Received data from {}".format(deveui))
    #
    # get the parser.
    sensor_def = config[CONF_SENSORS].get(deveui)
    if not sensor_def:
        logger.error("DevEUI {} is not defined in the config.".format(deveui))
        return 403  # not found.
    """
    call parser.parse() to try converting the payload_hex.
    if succeeded, it is set into KEY_APP_DATA.
    """
    handler_name = sensor_def.get(CONF_SENSOR_HANDLER)
    handler = handler_map[handler_name]
    logger.debug("Applied Handler: {}".format(handler_name))
    if handler.parser:
        kv_payload = handler.parser.parse_hex(payload_hex)
    else:
        kv_payload = payload_hex
    """
    return value of parser.paser():
        None: ignore parsing.
        False: something error.
        Others: a Python dict should be put in KEY_APP_DATA.
    """
    if kv_payload is None:
        logger.debug("The parser returned None.")
        pass
    elif kv_payload is False:
        # if kv_payload is False variants except of None, then error.
        logger.error("Parsing payload for DevEUI {} failed.".format(deveui))
        return 400  # bad request.
    else:
        kv_data[KEY_APP_DATA] = kv_payload
        if handler.db_conn:
            logger.debug("db_submit() has been called.")
            handler.db_conn.db_submit(kv_data)
    return 0

def log_common(request):
    logger.info("Access from {} {} {}".format(request.remote_addr,
                                              request.method, request.url))

def set_logger(config):
    #
    # set logger.
    #   "syslog", not yet
    #   "stdout"
    #   filename
    #
    log_file = config.get("log_file", "stdout")
    #
    if config[CONF_LOG_STDOUT] or log_file == "stdout":
        ch = logging.StreamHandler()
    else:
        ch = logging.FileHandler(log_file)
    #
    ch.setFormatter(logging.Formatter(fmt=LOG_FMT, datefmt=LOG_DATE_FMT))
    ch.setLevel(logging.DEBUG)
    logger.addHandler(ch)
    if config[CONF_DEBUG_LEVEL] > 0:
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)

def check_config(config):
    # set PYTHONPATH
    sys.path.extend(["./parsers", "./db_connectors"])
    for i in config.get(CONF_MODULE_PATH, []):
        sys.path.append(i)
    # check if the sensors key exists.
    sensors = config.get(CONF_SENSORS)
    if not sensors:
        print("ERROR: Key {} does not exist in the config."
              .format(CONF_SENSORS))
        return False
    # check if the handlers key exists.
    handlers = config.get(CONF_HANDLERS)
    if not handlers:
        print("ERROR: Key {} does not exist in the config."
              .format(CONF_HANDLERS))
        return False
    #
    config.setdefault(CONF_SERVER_ADDR, "::")
    config.setdefault(CONF_SERVER_PORT, "18886")
    config.setdefault(CONF_SERVER_CERT, None)
    config.setdefault(CONF_DEBUG_LEVEL, 0)
    config.setdefault(CONF_TZ, "Asia/Tokyo")
    # overwrite the debug level if opt.debug is True.
    if config[CONF_OPT_DEBUG] == True:
        config[CONF_DEBUG_LEVEL] = 99
    #
    # NOTE: below check should be placed at the end of this method.
    # check if the handlers are properly defined.
    for deveui,defs in sensors.items():
        handler_name = defs.get(CONF_SENSOR_HANDLER)
        if handler_name is None:
            print("ERROR: sensor {} doesn't have handler.".format(deveui))
            return False
        handler = handlers.get(handler_name)
        if not handler:
            print("ERROR: handler for {} is not defined in the config."
                  .format(deveui))
            return False
        #
        parser_handler = None
        parser_name = handler.get(CONF_PARSER)
        if parser_name:
            try:
                mod = import_module("parser_{}".format(parser_name))
            except Exception as e:
                print("ERROR: importing parser failed for {}, {}"
                      .format(parser_name, e))
                return False
            parser_handler = mod.parser(
                    logger=logger,
                    debug_level=config[CONF_DEBUG_LEVEL])
        #
        db_handler = None
        if handler.get(CONF_DB):
            db_conn_name = handler.get(CONF_DB).get(CONF_DB_CONN)
            if not db_conn_name:
                print("ERROR: {} for {} is not defined."
                      .format(CONF_DB_CONN, handler_name))
                return False
            try:
                mod = import_module("db_connector_{}".format(db_conn_name))
            except Exception as e:
                print("ERROR: importing db_connector failed for {}, {}"
                        .format(db_conn_name, e))
                return False
            db_handler = mod.db_connector(
                    logger=logger,
                    tz=config[CONF_TZ],
                    debug_level=config[CONF_DEBUG_LEVEL],
                    **handler[CONF_DB][CONF_DB_ARGS])
        #
        handler_map[handler_name] = type("handler", (object,), {})
        handler_map[handler_name].parser = parser_handler
        handler_map[handler_name].db_conn = db_handler
    return True

"""
main
"""
ap = ArgumentParser()
ap.add_argument("config_file", metavar="CONFIG_FILE",
                help="specify the config file.")
ap.add_argument("-d", action="store_true", dest="debug",
               help="enable debug mode.")
ap.add_argument("-D", action="store_true", dest="log_stdout",
               help="enable to show messages onto stdout.")
opt = ap.parse_args()
# get config.
try:
    config = json.load(open(opt.config_file))
except Exception as e:
    print("ERROR: {} read error. {}".format(opt.config_file, e))
    exit(1)
# update config.
config[CONF_OPT_DEBUG] = opt.debug
config[CONF_LOG_STDOUT] = opt.log_stdout
if not check_config(config):
    print("ERROR: error in {}.".format(opt.config_file))
    exit(1)
#
set_logger(config)
#
server_addr = config.get("server_addr")
server_port = int(config.get("server_port"))
server_cert = config.get("server_cert")
server_scheme = "https" if server_cert else "http"
logger.info("Starting {} listening on {}://{}:{}/"
            .format(PROG_NAME, server_scheme, server_addr, server_port))
#
if server_cert:
    app.run(host=server_addr, port=server_port,
            certfile=server_cert,
            server=GeventServer, quiet=True, debug=False)
else:
    app.run(host=server_addr, port=server_port,
            server=GeventServer, quiet=True, debug=False)
<|endoftext|>