<commit_before>#  Copyright 2008-2012 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

from __future__ import with_statement

from robot import utils
from robot.errors import DataError

from outputcapture import OutputCapturer
from runkwregister import RUN_KW_REGISTER
from keywords import Keywords, Keyword
from arguments import (PythonKeywordArguments, JavaKeywordArguments,
                       DynamicKeywordArguments, RunKeywordArguments,
                       PythonInitArguments, JavaInitArguments)
from signalhandler import STOP_SIGNAL_MONITOR

if utils.is_jython:
    from org.python.core import PyReflectedFunction, PyReflectedConstructor

    def _is_java_init(init):
        return isinstance(init, PyReflectedConstructor)
    def _is_java_method(method):
        return hasattr(method, 'im_func') \
               and isinstance(method.im_func, PyReflectedFunction)
else:
    _is_java_init = _is_java_method = lambda item: False

def Handler(library, name, method):
    if RUN_KW_REGISTER.is_run_keyword(library.orig_name, name):
        return _RunKeywordHandler(library, name, method)
    if _is_java_method(method):
        return _JavaHandler(library, name, method)
    else:
        return _PythonHandler(library, name, method)

def DynamicHandler(library, name, method, doc, argspec):
    if RUN_KW_REGISTER.is_run_keyword(library.orig_name, name):
        return _DynamicRunKeywordHandler(library, name, method, doc, argspec)
    return _DynamicHandler(library, name, method, doc, argspec)

def InitHandler(library, method, docgetter=None):
    Init = _PythonInitHandler if not _is_java_init(method) else _JavaInitHandler
    return Init(library, '__init__', method, docgetter)

class _BaseHandler(object):
    type = 'library'
    _doc = ''

    def __init__(self, library, handler_name, handler_method):
        self.library = library
        self.name = utils.printable_name(handler_name, code_style=True)
        self.arguments = self._parse_arguments(handler_method)

    def _parse_arguments(self, handler_method):
        raise NotImplementedError(self.__class__.__name__)

    @property
    def doc(self):
        return self._doc

    @property
    def longname(self):
        return '%s.%s' % (self.library.name, self.name)

    @property
    def shortdoc(self):
        return self.doc.splitlines()[0] if self.doc else ''

    @property
    def libname(self):
        return self.library.name

class _RunnableHandler(_BaseHandler):

    def __init__(self, library, handler_name, handler_method):
        _BaseHandler.__init__(self, library, handler_name, handler_method)
        self._handler_name = handler_name
        self._method = self._get_initial_handler(library, handler_name,
                                                 handler_method)

    def _get_initial_handler(self, library, name, method):
        if library.scope == 'GLOBAL':
            return self._get_global_handler(method, name)
        return None

    def init_keyword(self, varz):
        pass

    def run(self, context, args):
        if context.dry_run:
            return self._dry_run(context, args)
        return self._run(context, args)

    def _dry_run(self, context, args):
        if self.longname == 'BuiltIn.Import Library':
            return self._run(context, args)
        self.arguments.check_arg_limits_for_dry_run(args)
        return None

    def _run(self, context, args):
        output = context.output
        positional, named = \
            self.arguments.resolve(args, context.get_current_vars(), output)
        runner = self._runner_for(self._current_handler(), output, positional,
                                  named, self._get_timeout(context.namespace))
        return self._run_with_output_captured_and_signal_monitor(runner, context)

    def _runner_for(self, handler, output, positional, named, timeout):
        if timeout and timeout.active:
            output.debug(timeout.get_message)
            return lambda: timeout.run(handler, args=positional, kwargs=named)
        return lambda: handler(*positional, **named)

    def _run_with_output_captured_and_signal_monitor(self, runner, context):
        with OutputCapturer():
            return self._run_with_signal_monitoring(runner, context)

    def _run_with_signal_monitoring(self, runner, context):
        try:
            STOP_SIGNAL_MONITOR.start_running_keyword(context.teardown)
            return runner()
        finally:
            STOP_SIGNAL_MONITOR.stop_running_keyword()

    def _current_handler(self):
        if self._method:
            return self._method
        return self._get_handler(self.library.get_instance(),
                                 self._handler_name)

    def _get_global_handler(self, method, name):
        return method

    def _get_handler(self, lib_instance, handler_name):
        return getattr(lib_instance, handler_name)

    def _get_timeout(self, namespace):
        timeoutable = self._get_timeoutable_items(namespace)
        if timeoutable:
            return min(item.timeout for item in timeoutable)
        return None

    def _get_timeoutable_items(self, namespace):
        items = namespace.uk_handlers[:]
        if self._test_running_and_not_in_teardown(namespace.test):
            items.append(namespace.test)
        return items

    def _test_running_and_not_in_teardown(self, test):
        return test and test.status == 'RUNNING'

class _PythonHandler(_RunnableHandler):

    def __init__(self, library, handler_name, handler_method):
        _RunnableHandler.__init__(self, library, handler_name, handler_method)
        self._doc = utils.getdoc(handler_method)

    def _parse_arguments(self, handler_method):
        return PythonKeywordArguments(handler_method, self.longname)

class _JavaHandler(_RunnableHandler):

    def _parse_arguments(self, handler_method):
        return JavaKeywordArguments(handler_method, self.longname)

class _DynamicHandler(_RunnableHandler):

    def __init__(self, library, handler_name, handler_method, doc='',
                 argspec=None):
        self._argspec = argspec
        _RunnableHandler.__init__(self, library, handler_name, handler_method)
        self._run_keyword_method_name = handler_method.__name__
        self._doc = doc is not None and utils.unic(doc) or ''

    def _parse_arguments(self, handler_method):
        return DynamicKeywordArguments(self._argspec, self.longname)

    def _get_handler(self, lib_instance, handler_name):
        runner = getattr(lib_instance, self._run_keyword_method_name)
        return self._get_dynamic_handler(runner, handler_name)

    def _get_global_handler(self, method, name):
        return self._get_dynamic_handler(method, name)

    def _get_dynamic_handler(self, runner, name):
        def handler(*args):
            return runner(name, list(args))
        return handler

class _RunKeywordHandler(_PythonHandler):

    def __init__(self, library, handler_name, handler_method):
        _PythonHandler.__init__(self, library, handler_name, handler_method)
        self._handler_method = handler_method

    def _run_with_signal_monitoring(self, runner, context):
        # With run keyword variants, only the keyword to be run can fail
        # and therefore monitoring should not raise exception yet.
        return runner()

    def _parse_arguments(self, handler_method):
        arg_index = RUN_KW_REGISTER.get_args_to_process(self.library.orig_name,
                                                        self.name)
        return RunKeywordArguments(handler_method, self.longname, arg_index)

    def _get_timeout(self, namespace):
        return None

    def _dry_run(self, context, args):
        _RunnableHandler._dry_run(self, context, args)
        keywords = self._get_runnable_keywords(context, args)
        keywords.run(context)

    def _get_runnable_keywords(self, context, args):
        keywords = Keywords([])
        for keyword in self._get_keywords(args):
            if self._variable_syntax_in(keyword.name, context):
                continue
            keywords.add_keyword(keyword)
        return keywords

    def _get_keywords(self, args):
        if self._handler_name == 'run_keyword_if':
            return list(self._get_run_kw_if_keywords(args))
        arg_names = self.arguments.names
        if 'name' in arg_names:
            name_index = arg_names.index('name')
            return [ Keyword(args[name_index], args[name_index+1:]) ]
        elif self.arguments.varargs == 'names':
            return [ Keyword(name, []) for name in args[len(arg_names):] ]
        return []

    def _get_run_kw_if_keywords(self, given_args):
        while 'ELSE IF' in given_args:
            kw_call, given_args = self._split_run_kw_if_args(given_args, 'ELSE IF', 2)
            yield Keyword(kw_call[1], kw_call[2:])
        if 'ELSE' in given_args:
            kw_call, given_args = self._split_run_kw_if_args(given_args, 'ELSE', 1)
            yield Keyword(kw_call[1], kw_call[2:])
            yield Keyword(given_args[0], given_args[1:])
        else:
            yield Keyword(given_args[1], given_args[2:])

    def _split_run_kw_if_args(self, given_args, control_word, required_after):
        index = given_args.index(control_word)
        kw_call = given_args[:index]
        given_args = given_args[index+1:]
        if len(kw_call) < 2 or len(given_args) < required_after:
            raise DataError('Invalid ELSE IF/ELSE usage.')
        return kw_call, given_args

    def _variable_syntax_in(self, kw_name, context):
        try:
            resolved = context.namespace.variables.replace_string(kw_name)
            #Variable can contain value, but it might be wrong,
            #therefore it cannot be returned
            return resolved != kw_name
        except DataError:
            return True

class _XTimesHandler(_RunKeywordHandler):

    def __init__(self, handler, name):
        _RunKeywordHandler.__init__(self, handler.library, handler.name,
                                    handler._handler_method)
        self.name = name
        self._doc = "*DEPRECATED* Replace X times syntax with 'Repeat Keyword'."

    def run(self, context, args):
        resolved_times = context.namespace.variables.replace_string(self.name)
        _RunnableHandler.run(self, context, [resolved_times] + args)

    @property
    def longname(self):
        return self.name

class _DynamicRunKeywordHandler(_DynamicHandler, _RunKeywordHandler):
    _parse_arguments = _RunKeywordHandler._parse_arguments
    _get_timeout = _RunKeywordHandler._get_timeout

class _PythonInitHandler(_PythonHandler):

    def __init__(self, library, handler_name, handler_method, docgetter):
        _PythonHandler.__init__(self, library, handler_name, handler_method)
        self._docgetter = docgetter

    @property
    def doc(self):
        if self._docgetter:
            self._doc = self._docgetter() or self._doc
            self._docgetter = None
        return self._doc

    def _parse_arguments(self, handler_method):
        return PythonInitArguments(handler_method, self.library.name)

class _JavaInitHandler(_BaseHandler):

    def __init__(self, library, handler_name, handler_method, docgetter):
        _BaseHandler.__init__(self, library, handler_name, handler_method)
        self._docgetter = docgetter

    @property
    def doc(self):
        if self._docgetter:
            self._doc = self._docgetter() or self._doc
            self._docgetter = None
        return self._doc

    def _parse_arguments(self, handler_method):
        return JavaInitArguments(handler_method, self.library.name)
<commit_msg>refactor handling run kw if variants with dry-run<commit_after>#  Copyright 2008-2012 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

from __future__ import with_statement

from robot import utils
from robot.errors import DataError

from outputcapture import OutputCapturer
from runkwregister import RUN_KW_REGISTER
from keywords import Keywords, Keyword
from arguments import (PythonKeywordArguments, JavaKeywordArguments,
                       DynamicKeywordArguments, RunKeywordArguments,
                       PythonInitArguments, JavaInitArguments)
from signalhandler import STOP_SIGNAL_MONITOR

if utils.is_jython:
    from org.python.core import PyReflectedFunction, PyReflectedConstructor

    def _is_java_init(init):
        return isinstance(init, PyReflectedConstructor)
    def _is_java_method(method):
        return hasattr(method, 'im_func') \
               and isinstance(method.im_func, PyReflectedFunction)
else:
    _is_java_init = _is_java_method = lambda item: False

def Handler(library, name, method):
    if RUN_KW_REGISTER.is_run_keyword(library.orig_name, name):
        return _RunKeywordHandler(library, name, method)
    if _is_java_method(method):
        return _JavaHandler(library, name, method)
    else:
        return _PythonHandler(library, name, method)

def DynamicHandler(library, name, method, doc, argspec):
    if RUN_KW_REGISTER.is_run_keyword(library.orig_name, name):
        return _DynamicRunKeywordHandler(library, name, method, doc, argspec)
    return _DynamicHandler(library, name, method, doc, argspec)

def InitHandler(library, method, docgetter=None):
    Init = _PythonInitHandler if not _is_java_init(method) else _JavaInitHandler
    return Init(library, '__init__', method, docgetter)

class _BaseHandler(object):
    type = 'library'
    _doc = ''

    def __init__(self, library, handler_name, handler_method):
        self.library = library
        self.name = utils.printable_name(handler_name, code_style=True)
        self.arguments = self._parse_arguments(handler_method)

    def _parse_arguments(self, handler_method):
        raise NotImplementedError(self.__class__.__name__)

    @property
    def doc(self):
        return self._doc

    @property
    def longname(self):
        return '%s.%s' % (self.library.name, self.name)

    @property
    def shortdoc(self):
        return self.doc.splitlines()[0] if self.doc else ''

    @property
    def libname(self):
        return self.library.name

class _RunnableHandler(_BaseHandler):

    def __init__(self, library, handler_name, handler_method):
        _BaseHandler.__init__(self, library, handler_name, handler_method)
        self._handler_name = handler_name
        self._method = self._get_initial_handler(library, handler_name,
                                                 handler_method)

    def _get_initial_handler(self, library, name, method):
        if library.scope == 'GLOBAL':
            return self._get_global_handler(method, name)
        return None

    def init_keyword(self, varz):
        pass

    def run(self, context, args):
        if context.dry_run:
            return self._dry_run(context, args)
        return self._run(context, args)

    def _dry_run(self, context, args):
        if self.longname == 'BuiltIn.Import Library':
            return self._run(context, args)
        self.arguments.check_arg_limits_for_dry_run(args)
        return None

    def _run(self, context, args):
        output = context.output
        positional, named = \
            self.arguments.resolve(args, context.get_current_vars(), output)
        runner = self._runner_for(self._current_handler(), output, positional,
                                  named, self._get_timeout(context.namespace))
        return self._run_with_output_captured_and_signal_monitor(runner, context)

    def _runner_for(self, handler, output, positional, named, timeout):
        if timeout and timeout.active:
            output.debug(timeout.get_message)
            return lambda: timeout.run(handler, args=positional, kwargs=named)
        return lambda: handler(*positional, **named)

    def _run_with_output_captured_and_signal_monitor(self, runner, context):
        with OutputCapturer():
            return self._run_with_signal_monitoring(runner, context)

    def _run_with_signal_monitoring(self, runner, context):
        try:
            STOP_SIGNAL_MONITOR.start_running_keyword(context.teardown)
            return runner()
        finally:
            STOP_SIGNAL_MONITOR.stop_running_keyword()

    def _current_handler(self):
        if self._method:
            return self._method
        return self._get_handler(self.library.get_instance(),
                                 self._handler_name)

    def _get_global_handler(self, method, name):
        return method

    def _get_handler(self, lib_instance, handler_name):
        return getattr(lib_instance, handler_name)

    def _get_timeout(self, namespace):
        timeoutable = self._get_timeoutable_items(namespace)
        if timeoutable:
            return min(item.timeout for item in timeoutable)
        return None

    def _get_timeoutable_items(self, namespace):
        items = namespace.uk_handlers[:]
        if self._test_running_and_not_in_teardown(namespace.test):
            items.append(namespace.test)
        return items

    def _test_running_and_not_in_teardown(self, test):
        return test and test.status == 'RUNNING'

class _PythonHandler(_RunnableHandler):

    def __init__(self, library, handler_name, handler_method):
        _RunnableHandler.__init__(self, library, handler_name, handler_method)
        self._doc = utils.getdoc(handler_method)

    def _parse_arguments(self, handler_method):
        return PythonKeywordArguments(handler_method, self.longname)

class _JavaHandler(_RunnableHandler):

    def _parse_arguments(self, handler_method):
        return JavaKeywordArguments(handler_method, self.longname)

class _DynamicHandler(_RunnableHandler):

    def __init__(self, library, handler_name, handler_method, doc='',
                 argspec=None):
        self._argspec = argspec
        _RunnableHandler.__init__(self, library, handler_name, handler_method)
        self._run_keyword_method_name = handler_method.__name__
        self._doc = doc is not None and utils.unic(doc) or ''

    def _parse_arguments(self, handler_method):
        return DynamicKeywordArguments(self._argspec, self.longname)

    def _get_handler(self, lib_instance, handler_name):
        runner = getattr(lib_instance, self._run_keyword_method_name)
        return self._get_dynamic_handler(runner, handler_name)

    def _get_global_handler(self, method, name):
        return self._get_dynamic_handler(method, name)

    def _get_dynamic_handler(self, runner, name):
        def handler(*args):
            return runner(name, list(args))
        return handler

class _RunKeywordHandler(_PythonHandler):

    def __init__(self, library, handler_name, handler_method):
        _PythonHandler.__init__(self, library, handler_name, handler_method)
        self._handler_method = handler_method

    def _run_with_signal_monitoring(self, runner, context):
        # With run keyword variants, only the keyword to be run can fail
        # and therefore monitoring should not raise exception yet.
        return runner()

    def _parse_arguments(self, handler_method):
        arg_index = RUN_KW_REGISTER.get_args_to_process(self.library.orig_name,
                                                        self.name)
        return RunKeywordArguments(handler_method, self.longname, arg_index)

    def _get_timeout(self, namespace):
        return None

    def _dry_run(self, context, args):
        _RunnableHandler._dry_run(self, context, args)
        keywords = self._get_runnable_keywords(context, args)
        keywords.run(context)

    def _get_runnable_keywords(self, context, args):
        keywords = Keywords([])
        for keyword in self._get_keywords(args):
            if self._variable_syntax_in(keyword.name, context):
                continue
            keywords.add_keyword(keyword)
        return keywords

    def _variable_syntax_in(self, kw_name, context):
        try:
            resolved = context.namespace.variables.replace_string(kw_name)
            #Variable can contain value, but it might be wrong,
            #therefore it cannot be returned
            return resolved != kw_name
        except DataError:
            return True

    def _get_keywords(self, args):
        if self._handler_name == 'run_keyword_if':
            return list(self._get_run_kw_if_keywords(args))
        if self._handler_name == 'run_keywords':
            return self._get_run_kws_keywords(args)
        if 'name' in self.arguments.names:
            return self._get_default_run_kw_keywords(args)
        return []

    def _get_run_kw_if_keywords(self, given_args):
        while 'ELSE IF' in given_args:
            kw_call, given_args = self._split_run_kw_if_args(given_args, 'ELSE IF', 2)
            yield Keyword(kw_call[1], kw_call[2:])
        if 'ELSE' in given_args:
            kw_call, given_args = self._split_run_kw_if_args(given_args, 'ELSE', 1)
            yield Keyword(kw_call[1], kw_call[2:])
            yield Keyword(given_args[0], given_args[1:])
        else:
            yield Keyword(given_args[1], given_args[2:])

    def _split_run_kw_if_args(self, given_args, control_word, required_after):
        index = given_args.index(control_word)
        kw_call = given_args[:index]
        given_args = given_args[index+1:]
        if len(kw_call) < 2 or len(given_args) < required_after:
            raise DataError('Invalid ELSE IF/ELSE usage.')
        return kw_call, given_args

    def _get_run_kws_keywords(self, given_args):
        return [Keyword(name, []) for name in given_args]

    def _get_default_run_kw_keywords(self, given_args):
        index = self.arguments.names.index('name')
        return [Keyword(given_args[index], given_args[index+1:])]

class _XTimesHandler(_RunKeywordHandler):

    def __init__(self, handler, name):
        _RunKeywordHandler.__init__(self, handler.library, handler.name,
                                    handler._handler_method)
        self.name = name
        self._doc = "*DEPRECATED* Replace X times syntax with 'Repeat Keyword'."

    def run(self, context, args):
        resolved_times = context.namespace.variables.replace_string(self.name)
        _RunnableHandler.run(self, context, [resolved_times] + args)

    @property
    def longname(self):
        return self.name

class _DynamicRunKeywordHandler(_DynamicHandler, _RunKeywordHandler):
    _parse_arguments = _RunKeywordHandler._parse_arguments
    _get_timeout = _RunKeywordHandler._get_timeout

class _PythonInitHandler(_PythonHandler):

    def __init__(self, library, handler_name, handler_method, docgetter):
        _PythonHandler.__init__(self, library, handler_name, handler_method)
        self._docgetter = docgetter

    @property
    def doc(self):
        if self._docgetter:
            self._doc = self._docgetter() or self._doc
            self._docgetter = None
        return self._doc

    def _parse_arguments(self, handler_method):
        return PythonInitArguments(handler_method, self.library.name)

class _JavaInitHandler(_BaseHandler):

    def __init__(self, library, handler_name, handler_method, docgetter):
        _BaseHandler.__init__(self, library, handler_name, handler_method)
        self._docgetter = docgetter

    @property
    def doc(self):
        if self._docgetter:
            self._doc = self._docgetter() or self._doc
            self._docgetter = None
        return self._doc

    def _parse_arguments(self, handler_method):
        return JavaInitArguments(handler_method, self.library.name)
<|endoftext|>


<commit_before>import os
import copy
import json
import urllib2

from include.constants import CONTEXTBROKER, SUBSCRIPTION_LENGTH

# PubSub Handlers
from include.pubsub.contextbroker.cbPublisher import CbPublisher

contexts = {}
Publisher = CbPublisher()

def registerContext(entity_id, data_type, robot,  isPattern=False):
    print "REGISTER CONTEXT"
    url = "http://{}:{}/NGSI9/registerContext".format(CONTEXTBROKER["ADDRESS"], CONTEXTBROKER["PORT"])
    attributes = topics2NGSI9(robot)
    current_path = os.path.dirname(os.path.abspath(__file__))
    json_path = current_path.replace("scripts/include/pubsub/contextbroker", "config/robotdescriptions.json")
    description_data = json.load(open(json_path))
    if entity_id in description_data:
        attributes.append({
            "name": "descriptions",
            "type":"publisher:DescriptionData",
            "isDomain": "false"
        })
    data = {
        "contextRegistrations": [
            {
                "entities": [
                    {
                        "type": data_type,
                        "isPattern": "false",
                        "id": entity_id
                    }
                ],
                "attributes": attributes,
                "providingApplication": "http://{}:{}/{}".format(CONTEXTBROKER["ADDRESS"], CONTEXTBROKER["PORT"], CONTEXTBROKER["PROTOCOL"])
            }
        ],
        "duration": SUBSCRIPTION_LENGTH
    }
    response = _sendRequest(url, json.dumps(data))
    if "registrationId" in response:
        contexts[entity_id] = {
            "data": data,
            "registrationId": response["registrationId"]
        }

        if entity_id in description_data:
            _descs = ""
            for link in description_data[entity_id]["descriptions"]:
                _descs =  _descs + "," + link
            Publisher.publish(entity_id, data_type, [{
                "name" : "descriptions",
                "type" : "DescriptionData",
                "value" : _descs[1:]
            }])

def deleteAllContexts():
    for key in contexts:
        deleteContext(key)

def deleteContext(entity_id, delete=False):
    url = "http://{}:{}/NGSI9/registerContext".format(CONTEXTBROKER["ADDRESS"], CONTEXTBROKER["PORT"])
    data = {
        "contextRegistrations": [
            {
                "providingApplication": "http://{}:{}/{}".format(CONTEXTBROKER["ADDRESS"], CONTEXTBROKER["PORT"], CONTEXTBROKER["PROTOCOL"])
            }
        ],
        "duration": "P0D",
        "registrationId": contexts[entity_id]["registrationId"]
    }
    response = _sendRequest(url, json.dumps(data))
    if delete:
        del contexts[entity_id]

def refreshAllContexts():
    for key in contexts:
        refreshContext(key)

def refreshContext(entity_id):
    url = "http://{}:{}/NGSI9/registerContext".format(CONTEXTBROKER["ADDRESS"], CONTEXTBROKER["PORT"])
    data = copy.deepcopy(contexts[entity_id]["data"])
    data["registrationId"] = contexts[entity_id]["registrationId"]
    response = _sendRequest(url, json.dumps(data))

def topics2NGSI9(robot):
    return iterateTopics(robot["publisher"], "publisher") + iterateTopics(robot["subscriber"], "subscriber")

def iterateTopics(topics, topic_type):
    elems = []
    for key in topics:
        elems.append( {
            "name": key,
            "type": topic_type + ":" + topics[key]["class"]._type.replace("/", ".msg."),
            "isDomain": "false"
        })
    return elems

def _sendRequest(url, data, method=None):
    request = urllib2.Request(url, data, {'Content-Type': 'application/json', 'Accept': 'application/json'})
    if method is not None:
        request.get_method = lambda: method
    response = urllib2.urlopen(request)
    data = response.read()
    response_body = json.loads(data)
    response.close()
    return response_body

<commit_msg>Changed descriptor separator<commit_after>import os
import copy
import json
import urllib2

from include.constants import CONTEXTBROKER, SUBSCRIPTION_LENGTH

# PubSub Handlers
from include.pubsub.contextbroker.cbPublisher import CbPublisher

contexts = {}
Publisher = CbPublisher()

def registerContext(entity_id, data_type, robot,  isPattern=False):
    print "REGISTER CONTEXT"
    url = "http://{}:{}/NGSI9/registerContext".format(CONTEXTBROKER["ADDRESS"], CONTEXTBROKER["PORT"])
    attributes = topics2NGSI9(robot)
    current_path = os.path.dirname(os.path.abspath(__file__))
    json_path = current_path.replace("scripts/include/pubsub/contextbroker", "config/robotdescriptions.json")
    description_data = json.load(open(json_path))
    if entity_id in description_data:
        attributes.append({
            "name": "descriptions",
            "type":"publisher:DescriptionData",
            "isDomain": "false"
        })
    data = {
        "contextRegistrations": [
            {
                "entities": [
                    {
                        "type": data_type,
                        "isPattern": "false",
                        "id": entity_id
                    }
                ],
                "attributes": attributes,
                "providingApplication": "http://{}:{}/{}".format(CONTEXTBROKER["ADDRESS"], CONTEXTBROKER["PORT"], CONTEXTBROKER["PROTOCOL"])
            }
        ],
        "duration": SUBSCRIPTION_LENGTH
    }
    response = _sendRequest(url, json.dumps(data))
    if "registrationId" in response:
        contexts[entity_id] = {
            "data": data,
            "registrationId": response["registrationId"]
        }

        if entity_id in description_data:
            _descs = ""
            for link in description_data[entity_id]["descriptions"]:
                _descs =  _descs + "||" + link
            Publisher.publish(entity_id, data_type, [{
                "name" : "descriptions",
                "type" : "DescriptionData",
                "value" : _descs[2:]
            }])

def deleteAllContexts():
    for key in contexts:
        deleteContext(key)

def deleteContext(entity_id, delete=False):
    url = "http://{}:{}/NGSI9/registerContext".format(CONTEXTBROKER["ADDRESS"], CONTEXTBROKER["PORT"])
    data = {
        "contextRegistrations": [
            {
                "providingApplication": "http://{}:{}/{}".format(CONTEXTBROKER["ADDRESS"], CONTEXTBROKER["PORT"], CONTEXTBROKER["PROTOCOL"])
            }
        ],
        "duration": "P0D",
        "registrationId": contexts[entity_id]["registrationId"]
    }
    response = _sendRequest(url, json.dumps(data))
    if delete:
        del contexts[entity_id]

def refreshAllContexts():
    for key in contexts:
        refreshContext(key)

def refreshContext(entity_id):
    url = "http://{}:{}/NGSI9/registerContext".format(CONTEXTBROKER["ADDRESS"], CONTEXTBROKER["PORT"])
    data = copy.deepcopy(contexts[entity_id]["data"])
    data["registrationId"] = contexts[entity_id]["registrationId"]
    response = _sendRequest(url, json.dumps(data))

def topics2NGSI9(robot):
    return iterateTopics(robot["publisher"], "publisher") + iterateTopics(robot["subscriber"], "subscriber")

def iterateTopics(topics, topic_type):
    elems = []
    for key in topics:
        elems.append( {
            "name": key,
            "type": topic_type + ":" + topics[key]["class"]._type.replace("/", ".msg."),
            "isDomain": "false"
        })
    return elems

def _sendRequest(url, data, method=None):
    request = urllib2.Request(url, data, {'Content-Type': 'application/json', 'Accept': 'application/json'})
    if method is not None:
        request.get_method = lambda: method
    response = urllib2.urlopen(request)
    data = response.read()
    response_body = json.loads(data)
    response.close()
    return response_body

<|endoftext|>


<commit_before>from unittest import TestCase
import numpy as np
from pumha.env import Landscape
from pumha.pop import (PumaPopulation,
                       HarePopulation)

land_arr = np.array([[0, 0, 0, 0],
                     [0, 1, 1, 0],
                     [0, 1, 0, 0],
                     [0, 0, 1, 0],
                     [0, 0, 0, 0]])

dry_squares = np.array([[0, 1, 1, 0],
                        [1, 2, 1, 1],
                        [1, 1, 3, 0],
                        [0, 2, 0, 0],
                        [0, 0, 1, 0]])

P_density = np.array([[0, 0, 0, 0],
                      [0, 1.1, 1.9, 0],
                      [0, 0.1, 0, 0],
                      [0, 0, 2.6, 0],
                      [0, 0, 0, 0]])

H_density = np.array([[0, 0, 0, 0],
                      [0, 3.1, 2.4, 0],
                      [0, 3.1, 0, 0],
                      [0, 0, 0.6, 0],
                      [0, 0, 0, 0]])

# create new landscape
env = Landscape('pumha/test/data/test_land.dat')
# add arrays so we know what to expect
env.landscape = np.copy(land_arr)
env.dry_squares = np.copy(dry_squares)
env.land_indices = [(1, 1), (1, 2), (2, 1), (3, 2)]
# create puma and hare populations
# and add density arrays for testing
puma = PumaPopulation(env)
puma.density = np.copy(P_density)
hare = HarePopulation(env)
hare.density = np.copy(H_density)
pop_list = [hare, puma]

class TestPopulation(TestCase):
    def test_find_density_arr(self):
        arr = puma.find_density_arr(PumaPopulation, pop_list)
        self.assertTrue(isinstance(arr, np.ndarray))
        self.assertTrue(np.array_equal(arr, puma.density))
        # repeat with reversed list
        pop_list.reverse()
        arr = puma.find_density_arr(PumaPopulation, pop_list)
        self.assertTrue(isinstance(arr, np.ndarray))
        self.assertTrue(np.array_equal(arr, puma.density))

    def test_population_attributes(self):
        for pop in pop_list:
            self.assertIs(type(pop.min_ro), float)
            self.assertIs(type(pop.max_ro), float)
            self.assertIs(type(pop.birth), float)
            self.assertIs(type(pop.death), float)
            self.assertIs(type(pop.diffusion), float)
            self.assertIs(type(pop.dt), float)
            self.assertIs(type(pop.density), np.ndarray)
            self.assertIs(type(pop._N), np.ndarray)
            self.assertIs(type(pop._landscape), np.ndarray)
            self.assertTrue(pop.min_ro < pop.max_ro)
            self.assertTrue(pop.min_ro >= 0 and pop.max_ro >= 0)
            self.assertTrue(pop.birth >= 0 and pop.death >= 0)
            self.assertTrue(pop.diffusion >= 0)
            self.assertTrue(pop.dt >= 0)
            self.assertTrue(pop._N.shape == pop._landscape.shape)
            self.assertTrue(pop._N.shape == pop.density.shape)

    def test_load_config(self):
        birth = 0.2
        death = 0.3
        diffusion = 0.4
        dt = 0.5
        hare.load_config(birth, death, diffusion, dt)
        self.test_population_attributes()
        self.assertTrue(hare.birth == birth)
        self.assertTrue(hare.death == death)
        self.assertTrue(hare.diffusion == diffusion)
        self.assertTrue(hare.dt == dt)

    # check is halo of zeros in the hare density array not updated
    def test_hare_update_density_boundary(self):
        pop_list_new = np.copy(pop_list)
        hare.update_density(pop_list, pop_list_new)
        self.assertTrue(zero_surrounded(hare.density))
        # reset to original so no issues for other tests
        hare.density = np.copy(H_density)

    # check is halo of zeros in the puma density array not updated
    def test_puma_update_density_boundary(self):
        pop_list_new = np.copy(pop_list)
        puma.update_density(pop_list, pop_list_new)
        self.assertTrue(zero_surrounded(puma.density))
        # reset to original so no issues for other tests
        puma.density = np.copy(P_density)

    def test_puma_population_update_density_ij(self):
        b = .02
        m = .06
        l = .02
        dt = .4
        # test 1,2: 2nd row, 3rd column
        P = puma.density
        H = hare.density
        P_new_ij = puma.update_density_ij(1, 2, P, H)
        P_test = 1.9+dt*(b*1.9*2.4-m*1.9+l*((1.1)-1*1.9))
        self.assertAlmostEqual(P_new_ij, P_test)
        # test 1,1 : 2nd row, 2nd column
        P_new_ij = puma.update_density_ij(1, 1, P, H)
        P_test = 1.1+dt*(b*1.1*3.1-m*1.1+l*((1.9+0.1)-2*1.1))
        self.assertAlmostEqual(P_new_ij, P_test)

    def test_hare_population_update_density_ij(self):
        r = .08
        a = .04
        k = .02
        dt = .4
        # test 1,2: 2nd row, 3rd column
        P = puma.density
        H = hare.density
        H_new_ij = hare.update_density_ij(1, 2, P, H)
        H_test = 2.4+dt*(r*2.4-a*2.4*1.9+k*((3.1)-1*2.4))
        self.assertAlmostEqual(H_new_ij, H_test)
        # test 1,1 : 2nd row, 2nd column
        H_new_ij = hare.update_density_ij(1, 1, P, H)
        H_test = 3.1+dt*(r*3.1-a*3.1*1.1+k*((2.4+3.1)-2*3.1))
        self.assertAlmostEqual(H_new_ij, H_test)

    #Elen's comment
    def test_random_density(self):
        for pop in pop_list:
            # check that grids are numpy arrays
            self.assertTrue(isinstance(pop.random_density(env), np.ndarray))
            # check that the output array is the same shape as the landscape
            self.assertTrue(
                pop.random_density(env).shape == env.landscape.shape)
            # check that no nonzero density was assigned to the water squares
            self.assertAlmostEqual(env.landscape[env.landscape == 0].all(),
                                   pop.random_density(env)[
                                       pop.random_density(env) == 0].all())
            # check that all the random densities are between min_ro and max_ro
            self.assertTrue(pop.min_ro <= pop.random_density(env)[
                pop.random_density(env) != 0].all() <= pop.max_ro)

        # return True if all matrix perimeter (boundary) elements are zeroes
def zero_surrounded(array):
    return not (array[0, :].any() or array[-1, :].any() or array[:, 0].any()
                or array[:, -1].any())
<commit_msg>Corrected diffusion rate values<commit_after>from unittest import TestCase
import numpy as np
from pumha.env import Landscape
from pumha.pop import (PumaPopulation,
                       HarePopulation)

land_arr = np.array([[0, 0, 0, 0],
                     [0, 1, 1, 0],
                     [0, 1, 0, 0],
                     [0, 0, 1, 0],
                     [0, 0, 0, 0]])

dry_squares = np.array([[0, 1, 1, 0],
                        [1, 2, 1, 1],
                        [1, 1, 3, 0],
                        [0, 2, 0, 0],
                        [0, 0, 1, 0]])

P_density = np.array([[0, 0, 0, 0],
                      [0, 1.1, 1.9, 0],
                      [0, 0.1, 0, 0],
                      [0, 0, 2.6, 0],
                      [0, 0, 0, 0]])

H_density = np.array([[0, 0, 0, 0],
                      [0, 3.1, 2.4, 0],
                      [0, 3.1, 0, 0],
                      [0, 0, 0.6, 0],
                      [0, 0, 0, 0]])

# create new landscape
env = Landscape('pumha/test/data/test_land.dat')
# add arrays so we know what to expect
env.landscape = np.copy(land_arr)
env.dry_squares = np.copy(dry_squares)
env.land_indices = [(1, 1), (1, 2), (2, 1), (3, 2)]
# create puma and hare populations
# and add density arrays for testing
puma = PumaPopulation(env)
puma.density = np.copy(P_density)
hare = HarePopulation(env)
hare.density = np.copy(H_density)
pop_list = [hare, puma]

class TestPopulation(TestCase):
    def test_find_density_arr(self):
        arr = puma.find_density_arr(PumaPopulation, pop_list)
        self.assertTrue(isinstance(arr, np.ndarray))
        self.assertTrue(np.array_equal(arr, puma.density))
        # repeat with reversed list
        pop_list.reverse()
        arr = puma.find_density_arr(PumaPopulation, pop_list)
        self.assertTrue(isinstance(arr, np.ndarray))
        self.assertTrue(np.array_equal(arr, puma.density))

    def test_population_attributes(self):
        for pop in pop_list:
            self.assertIs(type(pop.min_ro), float)
            self.assertIs(type(pop.max_ro), float)
            self.assertIs(type(pop.birth), float)
            self.assertIs(type(pop.death), float)
            self.assertIs(type(pop.diffusion), float)
            self.assertIs(type(pop.dt), float)
            self.assertIs(type(pop.density), np.ndarray)
            self.assertIs(type(pop._N), np.ndarray)
            self.assertIs(type(pop._landscape), np.ndarray)
            self.assertTrue(pop.min_ro < pop.max_ro)
            self.assertTrue(pop.min_ro >= 0 and pop.max_ro >= 0)
            self.assertTrue(pop.birth >= 0 and pop.death >= 0)
            self.assertTrue(pop.diffusion >= 0)
            self.assertTrue(pop.dt >= 0)
            self.assertTrue(pop._N.shape == pop._landscape.shape)
            self.assertTrue(pop._N.shape == pop.density.shape)

    def test_load_config(self):
        birth = 0.2
        death = 0.3
        diffusion = 0.4
        dt = 0.5
        hare.load_config(birth, death, diffusion, dt)
        self.test_population_attributes()
        self.assertTrue(hare.birth == birth)
        self.assertTrue(hare.death == death)
        self.assertTrue(hare.diffusion == diffusion)
        self.assertTrue(hare.dt == dt)

    # check is halo of zeros in the hare density array not updated
    def test_hare_update_density_boundary(self):
        pop_list_new = np.copy(pop_list)
        hare.update_density(pop_list, pop_list_new)
        self.assertTrue(zero_surrounded(hare.density))
        # reset to original so no issues for other tests
        hare.density = np.copy(H_density)

    # check is halo of zeros in the puma density array not updated
    def test_puma_update_density_boundary(self):
        pop_list_new = np.copy(pop_list)
        puma.update_density(pop_list, pop_list_new)
        self.assertTrue(zero_surrounded(puma.density))
        # reset to original so no issues for other tests
        puma.density = np.copy(P_density)

    def test_puma_population_update_density_ij(self):
        b = .02
        m = .06
        l = .2
        dt = .4
        # test 1,2: 2nd row, 3rd column
        P = puma.density
        H = hare.density
        P_new_ij = puma.update_density_ij(1, 2, P, H)
        P_test = 1.9+dt*(b*1.9*2.4-m*1.9+l*((1.1)-1*1.9))
        self.assertAlmostEqual(P_new_ij, P_test)
        # test 1,1 : 2nd row, 2nd column
        P_new_ij = puma.update_density_ij(1, 1, P, H)
        P_test = 1.1+dt*(b*1.1*3.1-m*1.1+l*((1.9+0.1)-2*1.1))
        self.assertAlmostEqual(P_new_ij, P_test)

    def test_hare_population_update_density_ij(self):
        r = .08
        a = .04
        k = .2
        dt = .4
        # test 1,2: 2nd row, 3rd column
        P = puma.density
        H = hare.density
        H_new_ij = hare.update_density_ij(1, 2, P, H)
        H_test = 2.4+dt*(r*2.4-a*2.4*1.9+k*((3.1)-1*2.4))
        self.assertAlmostEqual(H_new_ij, H_test)
        # test 1,1 : 2nd row, 2nd column
        H_new_ij = hare.update_density_ij(1, 1, P, H)
        H_test = 3.1+dt*(r*3.1-a*3.1*1.1+k*((2.4+3.1)-2*3.1))
        self.assertAlmostEqual(H_new_ij, H_test)

    #Elen's comment
    def test_random_density(self):
        for pop in pop_list:
            # check that grids are numpy arrays
            self.assertTrue(isinstance(pop.random_density(env), np.ndarray))
            # check that the output array is the same shape as the landscape
            self.assertTrue(
                pop.random_density(env).shape == env.landscape.shape)
            # check that no nonzero density was assigned to the water squares
            self.assertAlmostEqual(env.landscape[env.landscape == 0].all(),
                                   pop.random_density(env)[
                                       pop.random_density(env) == 0].all())
            # check that all the random densities are between min_ro and max_ro
            self.assertTrue(pop.min_ro <= pop.random_density(env)[
                pop.random_density(env) != 0].all() <= pop.max_ro)

        # return True if all matrix perimeter (boundary) elements are zeroes
def zero_surrounded(array):
    return not (array[0, :].any() or array[-1, :].any() or array[:, 0].any()
                or array[:, -1].any())
<|endoftext|>


<commit_before>import os
import copy
import glob
import json
import uuid
import logging
from pupa.utils import get_psuedo_id
from pupa.utils.topsort import Network

def omnihash(obj):
    """ recursively hash unhashable objects """
    if isinstance(obj, set):
        return hash(frozenset(omnihash(e) for e in obj))
    elif isinstance(obj, (tuple, list)):
        return hash(tuple(omnihash(e) for e in obj))
    elif isinstance(obj, dict):
        return hash(frozenset((k, omnihash(v)) for k, v in obj.items()))
    else:
        return hash(obj)

def items_differ(jsonitems, dbitems, subfield_dict):
    """ check whether or not jsonitems and dbitems differ """

    # short circuit common cases
    if len(jsonitems) == len(dbitems) == 0:
        # both are empty
        return False
    elif len(jsonitems) != len(dbitems):
        # if lengths differ, they're definitely different
        return True

    jsonitems = copy.deepcopy(jsonitems)
    keys = jsonitems[0].keys()

    # go over dbitems looking for matches
    for dbitem in dbitems:
        match = None
        for i, jsonitem in enumerate(jsonitems):
            # check if all keys (excluding subfields) match
            for k in keys:
                if k not in subfield_dict and getattr(dbitem, k) != jsonitem[k]:
                    break
            else:
                # all fields match so far, possibly equal, just check subfields now
                for k in subfield_dict:
                    jsonsubitems = jsonitem[k]
                    dbsubitems = list(getattr(dbitem, k).all())
                    if items_differ(jsonsubitems, dbsubitems, subfield_dict[k]):
                        break
                else:
                    # these items are equal, so let's mark it for removal
                    match = i
                    break

        if match is not None:
            # item exists in both, remove from jsonitems
            jsonitems.pop(match)
        else:
            # exists in db but not json
            return True

    # if we get here, jsonitems has to be empty because we asserted that the length was
    # the same and we found a match for each thing in dbitems, here's a safety check just in case
    if jsonitems:       # pragma: no cover
        return True

    return False

class BaseImporter(object):
    """ BaseImporter

    Override:
        get_object(data)
        limit_spec(spec)                [optional, required if psuedo_ids are used]
        prepare_for_db(data)            [optional]
        postimport()                    [optional]
    """
    _type = None
    model_class = None
    related_models = {}
    preserve_order = set()

    def __init__(self, jurisdiction_id):
        self.jurisdiction_id = jurisdiction_id
        self.json_to_db_id = {}
        self.duplicates = {}
        self.logger = logging.getLogger("pupa")
        self.info = self.logger.info
        self.debug = self.logger.debug
        self.warning = self.logger.warning
        self.error = self.logger.error
        self.critical = self.logger.critical

    # no-ops to be overriden
    def prepare_for_db(self, data):
        return data

    def postimport(self):
        pass

    def resolve_json_id(self, json_id):
        """
            Given an id found in scraped JSON, return a DB id for the object.

            params:
                json_id:    id from json

            returns:
                database id

            raises:
                ValueError if id couldn't be resolved
        """
        if not json_id:
            return None

        if json_id.startswith('~'):
            spec = get_psuedo_id(json_id)
            spec = self.limit_spec(spec)
            try:
                return self.model_class.objects.get(**spec).id
            except self.model_class.DoesNotExist:
                raise ValueError('cannot resolve psuedo-id to {}: {}'.format(
                    self.model_class.__name__, json_id))

        # get the id that the duplicate points to, or use self
        json_id = self.duplicates.get(json_id, json_id)

        try:
            return self.json_to_db_id[json_id]
        except KeyError:
            raise ValueError('cannot resolve id: {}'.format(json_id))

    def import_directory(self, datadir):
        """ import a JSON directory into the database """

        def json_stream():
            # load all json, mapped by json_id
            for fname in glob.glob(os.path.join(datadir, self._type + '_*.json')):
                with open(fname) as f:
                    yield json.load(f)

        return self.import_data(json_stream())

    def _prepare_imports(self, dicts):
        """ filters the import stream to remove duplicates

        also serves as a good place to override if anything special has to be done to the
        order of the import stream (see OrganizationImporter)
        """
        # hash(json): id
        seen_hashes = {}

        for data in dicts:
            json_id = data.pop('_id')

            # map duplicates (using omnihash to tell if json dicts are identical-ish)
            objhash = omnihash(data)
            if objhash not in seen_hashes:
                seen_hashes[objhash] = json_id
                yield json_id, data
            else:
                self.duplicates[json_id] = seen_hashes[objhash]

    def import_data(self, data_items):
        """ import a bunch of dicts together """
        # keep counts of all actions
        results = {'insert': 0, 'update': 0, 'noop': 0}

        for json_id, data in self._prepare_imports(data_items):
            obj, what = self.import_item(data)
            self.json_to_db_id[json_id] = obj.id
            results[what] += 1

        # all objects are loaded, a perfect time to do inter-object resolution and other tasks
        self.postimport()

        return {self._type: results}

    def import_item(self, data):
        """ function used by import_data """
        what = 'noop'

        # remove the JSON _id (may still be there if called directly)
        data.pop('_id', None)

        # add fields/etc.
        data = self.prepare_for_db(data)

        try:
            obj = self.get_object(data)
        except self.model_class.DoesNotExist:
            obj = None

        # pull related fields off
        related = {}
        for field in self.related_models:
            related[field] = data.pop(field)

        # obj existed, check if we need to do an update
        if obj:
            if obj.id in self.json_to_db_id.values():
                raise Exception('attempt to import data that would conflict with data already in '
                                'the import: {} (already imported as {})'''.format(data, obj))
            # check base object for changes
            for key, value in data.items():
                if getattr(obj, key) != value:
                    setattr(obj, key, value)
                    what = 'update'
            if what == 'update':
                obj.save()

            updated = self._update_related(obj, related, self.related_models)
            if updated:
                what = 'update'

        # need to create the data
        else:
            what = 'insert'
            obj = self.model_class.objects.create(**data)
            self._create_related(obj, related, self.related_models)

        return obj, what

    def _update_related(self, obj, related, subfield_dict):
        """
        update DB objects related to a base object
            obj:            a base object to create related
            related:        dict mapping field names to lists of related objects
            subfield_list:  where to get the next layer of subfields
        """
        # keep track of whether or not anything was updated
        updated = False

        # for each related field - check if there are differences
        for field, items in related.items():
            # get items from database
            dbitems = getattr(obj, field).all()
            dbitems_count = dbitems.count()

            # default to doing nothing
            do_delete = do_update = False

            if items and dbitems_count:         # we have items, so does db, check for conflict
                do_delete = do_update = items_differ(items, dbitems, subfield_dict[field])
            elif items and not dbitems_count:   # we have items, db doesn't, just update
                do_update = True
            elif not items and dbitems_count:   # db has items, we don't, just delete
                do_delete = True
            # otherwise: no items or dbitems, so nothing is done

            if do_delete:
                updated = True
                getattr(obj, field).all().delete()
            if do_update:
                updated = True
                self._create_related(obj, {field: items}, subfield_dict)

        return updated

    def _create_related(self, obj, related, subfield_dict):
        """
        create DB objects related to a base object
            obj:            a base object to create related
            related:        dict mapping field names to lists of related objects
            subfield_list:  where to get the next layer of subfields
        """
        for field, items in related.items():
            for order, item in enumerate(items):
                # pull off 'subrelated' (things that are related to this obj)
                subrelated = {}
                for subfield in subfield_dict[field]:
                    subrelated[subfield] = item.pop(subfield)

                if field in self.preserve_order:
                    item['order'] = order

                try:
                    subobj = getattr(obj, field).create(**item)
                except Exception as e:
                    # TODO: revisit exception type
                    raise TypeError(str(e) + ' while importing ' + str(item))

                self._create_related(subobj, subrelated, subfield_dict[field])
<commit_msg>cache psuedo-id lookups (#108)<commit_after>import os
import copy
import glob
import json
import uuid
import logging
from pupa.utils import get_psuedo_id
from pupa.utils.topsort import Network

def omnihash(obj):
    """ recursively hash unhashable objects """
    if isinstance(obj, set):
        return hash(frozenset(omnihash(e) for e in obj))
    elif isinstance(obj, (tuple, list)):
        return hash(tuple(omnihash(e) for e in obj))
    elif isinstance(obj, dict):
        return hash(frozenset((k, omnihash(v)) for k, v in obj.items()))
    else:
        return hash(obj)

def items_differ(jsonitems, dbitems, subfield_dict):
    """ check whether or not jsonitems and dbitems differ """

    # short circuit common cases
    if len(jsonitems) == len(dbitems) == 0:
        # both are empty
        return False
    elif len(jsonitems) != len(dbitems):
        # if lengths differ, they're definitely different
        return True

    jsonitems = copy.deepcopy(jsonitems)
    keys = jsonitems[0].keys()

    # go over dbitems looking for matches
    for dbitem in dbitems:
        match = None
        for i, jsonitem in enumerate(jsonitems):
            # check if all keys (excluding subfields) match
            for k in keys:
                if k not in subfield_dict and getattr(dbitem, k) != jsonitem[k]:
                    break
            else:
                # all fields match so far, possibly equal, just check subfields now
                for k in subfield_dict:
                    jsonsubitems = jsonitem[k]
                    dbsubitems = list(getattr(dbitem, k).all())
                    if items_differ(jsonsubitems, dbsubitems, subfield_dict[k]):
                        break
                else:
                    # these items are equal, so let's mark it for removal
                    match = i
                    break

        if match is not None:
            # item exists in both, remove from jsonitems
            jsonitems.pop(match)
        else:
            # exists in db but not json
            return True

    # if we get here, jsonitems has to be empty because we asserted that the length was
    # the same and we found a match for each thing in dbitems, here's a safety check just in case
    if jsonitems:       # pragma: no cover
        return True

    return False

class BaseImporter(object):
    """ BaseImporter

    Override:
        get_object(data)
        limit_spec(spec)                [optional, required if psuedo_ids are used]
        prepare_for_db(data)            [optional]
        postimport()                    [optional]
    """
    _type = None
    model_class = None
    related_models = {}
    preserve_order = set()

    def __init__(self, jurisdiction_id):
        self.jurisdiction_id = jurisdiction_id
        self.json_to_db_id = {}
        self.duplicates = {}
        self.psuedo_id_cache = {}
        self.logger = logging.getLogger("pupa")
        self.info = self.logger.info
        self.debug = self.logger.debug
        self.warning = self.logger.warning
        self.error = self.logger.error
        self.critical = self.logger.critical

    # no-ops to be overriden
    def prepare_for_db(self, data):
        return data

    def postimport(self):
        pass

    def resolve_json_id(self, json_id):
        """
            Given an id found in scraped JSON, return a DB id for the object.

            params:
                json_id:    id from json

            returns:
                database id

            raises:
                ValueError if id couldn't be resolved
        """
        if not json_id:
            return None

        if json_id.startswith('~'):
            # keep caches of all the psuedo-ids to avoid doing 1000s of lookups during import
            if json_id not in self.psuedo_id_cache:
                spec = get_psuedo_id(json_id)
                spec = self.limit_spec(spec)
                try:
                    self.psuedo_id_cache[json_id] = self.model_class.objects.get(**spec).id
                except self.model_class.DoesNotExist:
                    raise ValueError('cannot resolve psuedo-id to {}: {}'.format(
                        self.model_class.__name__, json_id))

            # return the cached object
            return self.psuedo_id_cache[json_id]

        # get the id that the duplicate points to, or use self
        json_id = self.duplicates.get(json_id, json_id)

        try:
            return self.json_to_db_id[json_id]
        except KeyError:
            raise ValueError('cannot resolve id: {}'.format(json_id))

    def import_directory(self, datadir):
        """ import a JSON directory into the database """

        def json_stream():
            # load all json, mapped by json_id
            for fname in glob.glob(os.path.join(datadir, self._type + '_*.json')):
                with open(fname) as f:
                    yield json.load(f)

        return self.import_data(json_stream())

    def _prepare_imports(self, dicts):
        """ filters the import stream to remove duplicates

        also serves as a good place to override if anything special has to be done to the
        order of the import stream (see OrganizationImporter)
        """
        # hash(json): id
        seen_hashes = {}

        for data in dicts:
            json_id = data.pop('_id')

            # map duplicates (using omnihash to tell if json dicts are identical-ish)
            objhash = omnihash(data)
            if objhash not in seen_hashes:
                seen_hashes[objhash] = json_id
                yield json_id, data
            else:
                self.duplicates[json_id] = seen_hashes[objhash]

    def import_data(self, data_items):
        """ import a bunch of dicts together """
        # keep counts of all actions
        results = {'insert': 0, 'update': 0, 'noop': 0}

        for json_id, data in self._prepare_imports(data_items):
            obj, what = self.import_item(data)
            self.json_to_db_id[json_id] = obj.id
            results[what] += 1

        # all objects are loaded, a perfect time to do inter-object resolution and other tasks
        self.postimport()

        return {self._type: results}

    def import_item(self, data):
        """ function used by import_data """
        what = 'noop'

        # remove the JSON _id (may still be there if called directly)
        data.pop('_id', None)

        # add fields/etc.
        data = self.prepare_for_db(data)

        try:
            obj = self.get_object(data)
        except self.model_class.DoesNotExist:
            obj = None

        # pull related fields off
        related = {}
        for field in self.related_models:
            related[field] = data.pop(field)

        # obj existed, check if we need to do an update
        if obj:
            if obj.id in self.json_to_db_id.values():
                raise Exception('attempt to import data that would conflict with data already in '
                                'the import: {} (already imported as {})'''.format(data, obj))
            # check base object for changes
            for key, value in data.items():
                if getattr(obj, key) != value:
                    setattr(obj, key, value)
                    what = 'update'
            if what == 'update':
                obj.save()

            updated = self._update_related(obj, related, self.related_models)
            if updated:
                what = 'update'

        # need to create the data
        else:
            what = 'insert'
            obj = self.model_class.objects.create(**data)
            self._create_related(obj, related, self.related_models)

        return obj, what

    def _update_related(self, obj, related, subfield_dict):
        """
        update DB objects related to a base object
            obj:            a base object to create related
            related:        dict mapping field names to lists of related objects
            subfield_list:  where to get the next layer of subfields
        """
        # keep track of whether or not anything was updated
        updated = False

        # for each related field - check if there are differences
        for field, items in related.items():
            # get items from database
            dbitems = getattr(obj, field).all()
            dbitems_count = dbitems.count()

            # default to doing nothing
            do_delete = do_update = False

            if items and dbitems_count:         # we have items, so does db, check for conflict
                do_delete = do_update = items_differ(items, dbitems, subfield_dict[field])
            elif items and not dbitems_count:   # we have items, db doesn't, just update
                do_update = True
            elif not items and dbitems_count:   # db has items, we don't, just delete
                do_delete = True
            # otherwise: no items or dbitems, so nothing is done

            if do_delete:
                updated = True
                getattr(obj, field).all().delete()
            if do_update:
                updated = True
                self._create_related(obj, {field: items}, subfield_dict)

        return updated

    def _create_related(self, obj, related, subfield_dict):
        """
        create DB objects related to a base object
            obj:            a base object to create related
            related:        dict mapping field names to lists of related objects
            subfield_list:  where to get the next layer of subfields
        """
        for field, items in related.items():
            for order, item in enumerate(items):
                # pull off 'subrelated' (things that are related to this obj)
                subrelated = {}
                for subfield in subfield_dict[field]:
                    subrelated[subfield] = item.pop(subfield)

                if field in self.preserve_order:
                    item['order'] = order

                try:
                    subobj = getattr(obj, field).create(**item)
                except Exception as e:
                    # TODO: revisit exception type
                    raise TypeError(str(e) + ' while importing ' + str(item))

                self._create_related(subobj, subrelated, subfield_dict[field])
<|endoftext|>


<commit_before>__author__ = 'Igor Maculan <n3wtron@gmail.com>'

import logging
import time
import json
from threading import Thread

import requests
import websocket

log = logging.getLogger('pushbullet.Listener')

WEBSOCKET_URL = 'wss://stream.pushbullet.com/websocket/'
PUSHES_URL = 'https://api.pushbullet.com/v2/pushes'

class PushTypeNotProvided(Exception):
    def __init__(self, message):
        self.message = message

    def __str__(self):
        return self.message

class Listener(Thread, websocket.WebSocketApp):
    def __init__(self, api_key,
                 on_push=None,
                 http_proxy_host=None,
                 http_proxy_port=None):
        """
        :param api_key: pushbullet Key
        :param on_push: function that get's called on all pushes
        :param http_proxy_host: host proxy (ie localhost)
        :param http_proxy_port: host port (ie 3128)
        """
        Thread.__init__(self)
        websocket.WebSocketApp.__init__(self, WEBSOCKET_URL + api_key,
                                        on_open=self.on_open,
                                        on_message=self.on_message,
                                        on_close=self.on_close)
        self.api_key = api_key
        self.connected = False
        self.last_update = time.time()

        # History
        self.history = None
        self.clean_history()

        # proxy configuration
        self.http_proxy_host = http_proxy_host
        self.http_proxy_port = http_proxy_port
        self.proxies = None
        if http_proxy_port is not None and http_proxy_port is not None:
            self.proxies = {
                "http": "http://" + http_proxy_host + ":" + str(http_proxy_port),
                "https": "http://" + http_proxy_host + ":" + str(http_proxy_port),
            }

    def clean_history(self):
        self.history = []

    def on_open(self, ws):
        self.connected = True
        self.last_update = time.time()

    def on_close(self, ws):
        log.debug('Listener closed')
        self.connected = False

    def on_message(self, ws, message):
        log.debug('Message received:' + message)
        try:
            json_message = json.loads(message)
            if json_message['type'] == 'tickle':
                pushes_url = PUSHES_URL + '?modified_after=' + str(self.last_update)
                logging.debug('calling:' + pushes_url)
                res = requests.get(pushes_url, auth=(self.api_key, ""), proxies=self.proxies)
                self.last_update = time.time()
                pushes = res.json()['pushes']
                for push in pushes:
                    log.debug(str(push))
                    # add the push to history
                    self.history.append(push)
                    self.on_push(push)
                log.debug(str(self.history))
        except Exception as e:
            logging.exception(e)

    def run_forever(self, sockopt=None, sslopt=None, ping_interval=0, ping_timeout=None):
        websocket.WebSocketApp.run_forever(self, sockopt=sockopt, sslopt=sslopt, ping_interval=ping_interval,
                                           ping_timeout=ping_timeout,
                                           http_proxy_host=self.http_proxy_host,
                                           http_proxy_port=self.http_proxy_port)

    def run(self):
        self.run_forever()

<commit_msg>Remove duplicate functionalty<commit_after>__author__ = 'Igor Maculan <n3wtron@gmail.com>'

import logging
import time
import json
from threading import Thread

import requests
import websocket

log = logging.getLogger('pushbullet.Listener')

WEBSOCKET_URL = 'wss://stream.pushbullet.com/websocket/'
PUSHES_URL = 'https://api.pushbullet.com/v2/pushes'

class PushTypeNotProvided(Exception):
    def __init__(self, message):
        self.message = message

    def __str__(self):
        return self.message

class Listener(Thread, websocket.WebSocketApp):
    def __init__(self, api_key,
                 on_push=None,
                 http_proxy_host=None,
                 http_proxy_port=None):
        """
        :param api_key: pushbullet Key
        :param on_push: function that get's called on all pushes
        :param http_proxy_host: host proxy (ie localhost)
        :param http_proxy_port: host port (ie 3128)
        """
        Thread.__init__(self)
        websocket.WebSocketApp.__init__(self, WEBSOCKET_URL + api_key,
                                        on_open=self.on_open,
                                        on_message=self.on_message,
                                        on_close=self.on_close)
        self.api_key = api_key
        self.connected = False
        self.last_update = time.time()

        # History
        self.history = None
        self.clean_history()

        # proxy configuration
        self.http_proxy_host = http_proxy_host
        self.http_proxy_port = http_proxy_port
        self.proxies = None
        if http_proxy_port is not None and http_proxy_port is not None:
            self.proxies = {
                "http": "http://" + http_proxy_host + ":" + str(http_proxy_port),
                "https": "http://" + http_proxy_host + ":" + str(http_proxy_port),
            }

    def clean_history(self):
        self.history = []

    def on_open(self, ws):
        self.connected = True
        self.last_update = time.time()

    def on_close(self, ws):
        log.debug('Listener closed')
        self.connected = False

    def on_message(self, ws, message):
        log.debug('Message received:' + message)
        try:
            json_message = json.loads(message)
            if json_message['type'] == 'tickle':
                self.on_push()
        except Exception as e:
            logging.exception(e)

    def run_forever(self, sockopt=None, sslopt=None, ping_interval=0, ping_timeout=None):
        websocket.WebSocketApp.run_forever(self, sockopt=sockopt, sslopt=sslopt, ping_interval=ping_interval,
                                           ping_timeout=ping_timeout,
                                           http_proxy_host=self.http_proxy_host,
                                           http_proxy_port=self.http_proxy_port)

    def run(self):
        self.run_forever()

<|endoftext|>


<commit_before>import sys

if sys.version_info.major >= 3:
    # For mypy PEP-484 static typing validation
    from . import px_process   # NOQA
    from six import text_type  # NOQA
    from typing import List    # NOQA
    from typing import Tuple   # NOQA
    from typing import Dict    # NOQA
    from typing import Optional  # NOQA

class Launchcounter(object):
    def __init__(self):
        self._hierarchies = {}  # type: Dict[Tuple[text_type, ...], int]

        # Most recent process snapshot
        self._last_processlist = None  # type: Optional[List[px_process.PxProcess]]

    def _callchain(self, process):
        # type: (px_process.PxProcess) -> Tuple[text_type, ...]

        reverse_callchain = []  # type: List[text_type]

        current = process  # type: Optional[px_process.PxProcess]
        while current is not None:
            reverse_callchain.append(current.command)
            current = current.parent

        return tuple(reversed(reverse_callchain))

    def _register_launches(self, new_processes):
        # type: (List[px_process.PxProcess]) -> None

        for new_process in new_processes:
            callchain = self._callchain(new_process)
            if callchain in self._hierarchies:
                self._hierarchies[callchain] += 1
            else:
                self._hierarchies[callchain] = 1

    def _list_new_launches(
        self,
        before,  # type: List[px_process.PxProcess]
        after    # type: List[px_process.PxProcess]
    ):
        # type: (...) -> List[px_process.PxProcess]
        pid2oldProc = {}  # type: Dict[int,px_process.PxProcess]
        for old_proc in before:
            pid2oldProc[old_proc.pid] = old_proc

        new_procs = []  # List[px_process.PxProcess]
        for new_proc in after:
            if new_proc.pid not in pid2oldProc:
                # This is a new process
                new_procs.append(new_proc)
                continue

            old_proc = pid2oldProc[new_proc.pid]
            if old_proc.start_time != new_proc.start_time:
                # This is a new process, PID has been reused
                new_procs.append(new_proc)
                continue

        return new_procs

    def update(self, procs_snapshot):
        # type: (List[px_process.PxProcess]) -> None

        if self._last_processlist is None:
            self._last_processlist = procs_snapshot
            return

        new_processes = self._list_new_launches(self._last_processlist, procs_snapshot)
        self._register_launches(new_processes)

        self._last_processlist = procs_snapshot

    def get_screen_lines(self, rows, columns):
        # type: (int, int) -> List[text_type]

        # FIXME: Render this as a tree, not a list
        # FIXME: Should we sort the tree somehow?
        # FIXME: Should we print counts somewhere?
        # FIXME: How to handle rows?
        lines = []  # type: List[text_type]
        for row in sorted(self._hierarchies.keys()):
            count = self._hierarchies[row]
            lines.append('{}: {}'.format(str(row), count))

        return lines
<commit_msg>Drop counts from launches viz<commit_after>import sys

if sys.version_info.major >= 3:
    # For mypy PEP-484 static typing validation
    from . import px_process   # NOQA
    from six import text_type  # NOQA
    from typing import List    # NOQA
    from typing import Tuple   # NOQA
    from typing import Dict    # NOQA
    from typing import Optional  # NOQA

class Launchcounter(object):
    def __init__(self):
        self._hierarchies = {}  # type: Dict[Tuple[text_type, ...], int]

        # Most recent process snapshot
        self._last_processlist = None  # type: Optional[List[px_process.PxProcess]]

    def _callchain(self, process):
        # type: (px_process.PxProcess) -> Tuple[text_type, ...]

        reverse_callchain = []  # type: List[text_type]

        current = process  # type: Optional[px_process.PxProcess]
        while current is not None:
            reverse_callchain.append(current.command)
            current = current.parent

        return tuple(reversed(reverse_callchain))

    def _register_launches(self, new_processes):
        # type: (List[px_process.PxProcess]) -> None

        for new_process in new_processes:
            callchain = self._callchain(new_process)
            if callchain in self._hierarchies:
                self._hierarchies[callchain] += 1
            else:
                self._hierarchies[callchain] = 1

    def _list_new_launches(
        self,
        before,  # type: List[px_process.PxProcess]
        after    # type: List[px_process.PxProcess]
    ):
        # type: (...) -> List[px_process.PxProcess]
        pid2oldProc = {}  # type: Dict[int,px_process.PxProcess]
        for old_proc in before:
            pid2oldProc[old_proc.pid] = old_proc

        new_procs = []  # List[px_process.PxProcess]
        for new_proc in after:
            if new_proc.pid not in pid2oldProc:
                # This is a new process
                new_procs.append(new_proc)
                continue

            old_proc = pid2oldProc[new_proc.pid]
            if old_proc.start_time != new_proc.start_time:
                # This is a new process, PID has been reused
                new_procs.append(new_proc)
                continue

        return new_procs

    def update(self, procs_snapshot):
        # type: (List[px_process.PxProcess]) -> None

        if self._last_processlist is None:
            self._last_processlist = procs_snapshot
            return

        new_processes = self._list_new_launches(self._last_processlist, procs_snapshot)
        self._register_launches(new_processes)

        self._last_processlist = procs_snapshot

    def get_screen_lines(self, rows, columns):
        # type: (int, int) -> List[text_type]

        # FIXME: Render this as a tree, not a list
        # FIXME: Should we sort the tree somehow?
        # FIXME: Should we print counts somewhere?
        # FIXME: How to handle rows?
        lines = []  # type: List[text_type]
        for row in sorted(self._hierarchies.keys()):
            lines.append(str(row))

        return lines
<|endoftext|>


<commit_before>__version__ = '0.14.0'
<commit_msg>bump dev version post 0.14.0 tag<commit_after>__version__ = '0.14.0.dev684'
<|endoftext|>


<commit_before>import os
import tempfile
import random
import jinja2
import json

import mpld3
from mpld3.utils import get_id
from mpld3.mpld3renderer import MPLD3Renderer
from mpld3.mplexporter import Exporter
from mpld3._server import serve_and_open

MAP_HTML = jinja2.Template("""

<link rel="stylesheet" href={{leaflet_css_url}} type="text/css"/>
<link rel="stylesheet" href={{pyLeaflet_css_url}} type="text/css"/>

""")

def plotWithMap(fig,tile_layer = "http://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png",mapWidth=1040,mapHeight=800, **kwargs):
  figid = 'fig_' + get_id(fig) + str(int(random.random() * 1E10))

  renderer = MPLD3Renderer()
  Exporter(renderer, close_mpl=True).run(fig)

  fig, figure_json, extra_css, extra_js = renderer.finished_figures[0]

  extra_css = ""
  extra_js = ""

  # print json.dumps(figure_json)

  leaflet_init_js = """
    var width = 1040,
    height = 640;

    var mouseLat = 37;
    var mouseLng = -90;

    /*******************************************************************************
    *
    * Initialize map layout
    *
    *******************************************************************************/
    var map = L.map('map').setView([mouseLat, mouseLng],4);
    //map.scrollWheelZoom.disable();

    L.tileLayer('%s', {
        attribution: '',
        maxZoom: 18
    }).addTo(map);

    // Use Leaflet to implement a D3 geometric transformation.
    function projectPoint(x, y) {
      var point = map.latLngToLayerPoint(new L.LatLng(x, y));
      this.stream.point(point.x, point.y);
    }

    var transform = d3.geo.transform({point: projectPoint});
    var ppath = d3.geo.path().projection(transform);

    var background = d3.select(map.getPanes().overlayPane)
        .insert("svg")
        .attr('width',width+500+'px')
        .attr('height',height+500+'px')
    background.append('rect')
      .attr('width',width+500+'px')
      .attr('height',height+500+'px')
      .style('fill','white')
      .style('fill-opacity',.4)

    var svg = d3.select(map.getPanes().overlayPane).append('svg').attr('width',5000).attr('height',3000);
    var g2   = svg.append('g').attr('class', 'leaflet-zoom-hide');

    """%(tile_layer)
  # tile_layer = "http://{s}.www.toolserver.org/tiles/bw-mapnik/{z}/{x}/{y}.png"
  # tile_layer = "http://{s}.tile.stamen.com/terrain/{z}/{x}/{y}.jpg"
  # tile_layer = "http://{s}.tile.stamen.com/toner/{z}/{x}/{y}.jpg"

  datafile = tempfile.NamedTemporaryFile(delete=False)
  datafile.write('mdata = '+json.dumps(figure_json))
  datafile.close()

  kwargs['mpld3_url'] = '/mpld3.js'
  kwargs['d3_url'] = '/d3.js'
  files = {'/mpld3.js': ["text/javascript",
                      open(mpld3.urls.MPLD3MIN_LOCAL, 'r').read()],
            '/d3.js': ["text/javascript",
                      open(mpld3.urls.D3_LOCAL, 'r').read()],
            '/draw.js': ["text/javascript",
                      open(os.path.join(os.path.dirname(__file__), 'js/draw.js'),'r').read()],
            '/pyLeaflet.css': ["text/css",
                      open(os.path.join(os.path.dirname(__file__), 'css/pyLeaflet.css'),'r').read()],
            '/leaflet.js': ["text/javascript",
                      open(os.path.join(os.path.dirname(__file__), 'js/leaflet.js'),'r').read()],
            '/leaflet.css': ["text/css",
                      open(os.path.join(os.path.dirname(__file__), 'css/leaflet.css'),'r').read()],
            '/images/layers-2x.png': ["text/css",
                      open(os.path.join(os.path.dirname(__file__), 'images/layers-2x.png'),'r').read()],
            '/images/layers.png': ["text/css",
                      open(os.path.join(os.path.dirname(__file__), 'images/layers.png'),'r').read()],
            '/images/marker-icon-2x.png': ["text/css",
                      open(os.path.join(os.path.dirname(__file__), 'images/marker-icon-2x.png'),'r').read()],
            '/images/marker-icon.png': ["text/css",
                      open(os.path.join(os.path.dirname(__file__), 'images/marker-icon.png'),'r').read()],
            '/images/marker-shadow.png': ["text/css",
                      open(os.path.join(os.path.dirname(__file__), 'images/marker-shadow.png'),'r').read()],
            '/data.js': ["text/javascript",
                      open(datafile.name,'r').read()]}

  html = MAP_HTML.render(figid=json.dumps(figid),
                        mapWidth = mapWidth,
                        mapHeight=mapHeight,
                         d3_url=kwargs['d3_url'],
                         mpld3_url=kwargs['mpld3_url'],
                         draw_js_url='draw.js',
                         data_url='data.js',
                         leaflet_js_url='leaflet.js',
                         leaflet_css_url='leaflet.css',
                         extra_css=extra_css,
                         extra_js=extra_js,
                         pyLeaflet_css_url='pyLeaflet.css',
                         leaflet_init_js=leaflet_init_js,
                         tile_layer=tile_layer)

  serve_and_open(html, ip='localhost', port=8888, n_retries=50, files=files)
  os.unlink(datafile.name)
  return html<commit_msg>modified bug for firefox in zoomend function<commit_after>import os
import tempfile
import random
import jinja2
import json

import mpld3
from mpld3.utils import get_id
from mpld3.mpld3renderer import MPLD3Renderer
from mpld3.mplexporter import Exporter
from mpld3._server import serve_and_open

MAP_HTML = jinja2.Template("""

<link rel="stylesheet" href={{leaflet_css_url}} type="text/css"/>
<link rel="stylesheet" href={{pyLeaflet_css_url}} type="text/css"/>

""")

def plotWithMap(fig,tile_layer = "http://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png",mapWidth=1040,mapHeight=800, **kwargs):
  figid = 'fig_' + get_id(fig) + str(int(random.random() * 1E10))

  renderer = MPLD3Renderer()
  Exporter(renderer, close_mpl=True).run(fig)

  fig, figure_json, extra_css, extra_js = renderer.finished_figures[0]

  extra_css = ""
  extra_js = ""

  # print json.dumps(figure_json)

  leaflet_init_js = """
    var width = 1040,
    height = 640;

    var mouseLat = 37;
    var mouseLng = -90;

    /*******************************************************************************
    *
    * Initialize map layout
    *
    *******************************************************************************/
    var map = L.map('map').setView([mouseLat, mouseLng],4);
    //map.scrollWheelZoom.disable();

    L.tileLayer('%s', {
        attribution: '',
        maxZoom: 18
    }).addTo(map);

    // Use Leaflet to implement a D3 geometric transformation.
    function projectPoint(x, y) {
      var point = map.latLngToLayerPoint(new L.LatLng(x, y));
      this.stream.point(point.x, point.y);
    }

    var transform = d3.geo.transform({point: projectPoint});
    var ppath = d3.geo.path().projection(transform);

    var background = d3.select(map.getPanes().overlayPane)
        .insert("svg")
        .attr('width',width+500+'px')
        .attr('height',height+500+'px')
    background.append('rect')
      .attr('width',width+500+'px')
      .attr('height',height+500+'px')
      .style('fill','white')
      .style('fill-opacity',.4)

    var svg = d3.select(map.getPanes().overlayPane).append('svg').attr('width',5000).attr('height',3000);
    var g2   = svg.append('g').attr('class', 'leaflet-zoom-hide');

    """%(tile_layer)
  # tile_layer = "http://{s}.www.toolserver.org/tiles/bw-mapnik/{z}/{x}/{y}.png"
  # tile_layer = "http://{s}.tile.stamen.com/terrain/{z}/{x}/{y}.jpg"
  # tile_layer = "http://{s}.tile.stamen.com/toner/{z}/{x}/{y}.jpg"

  datafile = tempfile.NamedTemporaryFile(delete=False)
  datafile.write('mdata = '+json.dumps(figure_json))
  datafile.close()

  kwargs['mpld3_url'] = '/mpld3.js'
  kwargs['d3_url'] = '/d3.js'
  files = {'/mpld3.js': ["text/javascript",
                      open(mpld3.urls.MPLD3MIN_LOCAL, 'r').read()],
            '/d3.js': ["text/javascript",
                      open(mpld3.urls.D3_LOCAL, 'r').read()],
            '/draw.js': ["text/javascript",
                      open(os.path.join(os.path.dirname(__file__), 'js/draw.js'),'r').read()],
            '/pyLeaflet.css': ["text/css",
                      open(os.path.join(os.path.dirname(__file__), 'css/pyLeaflet.css'),'r').read()],
            '/leaflet.js': ["text/javascript",
                      open(os.path.join(os.path.dirname(__file__), 'js/leaflet.js'),'r').read()],
            '/leaflet.css': ["text/css",
                      open(os.path.join(os.path.dirname(__file__), 'css/leaflet.css'),'r').read()],
            '/images/layers-2x.png': ["text/css",
                      open(os.path.join(os.path.dirname(__file__), 'images/layers-2x.png'),'r').read()],
            '/images/layers.png': ["text/css",
                      open(os.path.join(os.path.dirname(__file__), 'images/layers.png'),'r').read()],
            '/images/marker-icon-2x.png': ["text/css",
                      open(os.path.join(os.path.dirname(__file__), 'images/marker-icon-2x.png'),'r').read()],
            '/images/marker-icon.png': ["text/css",
                      open(os.path.join(os.path.dirname(__file__), 'images/marker-icon.png'),'r').read()],
            '/images/marker-shadow.png': ["text/css",
                      open(os.path.join(os.path.dirname(__file__), 'images/marker-shadow.png'),'r').read()],
            '/data.js': ["text/javascript",
                      open(datafile.name,'r').read()]}

  html = MAP_HTML.render(figid=json.dumps(figid),
                        mapWidth = mapWidth,
                        mapHeight=mapHeight,
                         d3_url=kwargs['d3_url'],
                         mpld3_url=kwargs['mpld3_url'],
                         draw_js_url='draw.js',
                         data_url='data.js',
                         leaflet_js_url='leaflet.js',
                         leaflet_css_url='leaflet.css',
                         extra_css=extra_css,
                         extra_js=extra_js,
                         pyLeaflet_css_url='pyLeaflet.css',
                         leaflet_init_js=leaflet_init_js,
                         tile_layer=tile_layer)

  serve_and_open(html, ip='localhost', port=8888, n_retries=50, files=files)
  os.unlink(datafile.name)
  return html<|endoftext|>


<commit_before>import asyncio
import logging
import websockets
from gabriel_protocol import gabriel_pb2
from collections import namedtuple

URI_FORMAT = 'ws://{host}:{port}'

logger = logging.getLogger(__name__)
websockets_logger = logging.getLogger(websockets.__name__)

# The entire payload will be printed if this is allowed to be DEBUG
websockets_logger.setLevel(logging.INFO)

ProducerWrapper = namedtuple('ProducerWrapper', ['producer', 'source_name'])

class WebsocketClient:
    def __init__(self, host, port, producer_wrappers, consumer):
        '''
        producer should take no arguments and return an instance of
        gabriel_pb2.InputFrame.
        consumer should take one gabriel_pb2.ResultWrapper argument and return
        None.
        '''

        self._welcome_event = asyncio.Event()
        self._sources = {}
        self._running = True
        self._uri = URI_FORMAT.format(host=host, port=port)
        self.producer_wrappers = producer_wrappers
        self.consumer = consumer

    def launch(self):
        event_loop = asyncio.get_event_loop()

        # TODO remove this line once we stop supporting Python 3.5
        asyncio.set_event_loop(event_loop)

        try:
            self._websocket = event_loop.run_until_complete(
                websockets.connect(self._uri))
        except ConnectionRefusedError:
            logger.error('Could not connect to server')
            return

        consumer_task = asyncio.ensure_future(self._consumer_handler())
        tasks = [
            asyncio.ensure_future(self._producer_handler(
                producer_wrapper.producer, producer_wrapper.source_name))
            for producer_wrapper in self.producer_wrappers
        ]
        tasks.append(consumer_task)

        _, pending = event_loop.run_until_complete(asyncio.wait(
                tasks, return_when=asyncio.FIRST_COMPLETED))
        for task in pending:
            task.cancel()
        logger.info('Disconnected From Server')

    def get_source_names(self):
        return self._sources.keys()

    def stop(self):
        self._running = False
        logger.info('stopping server')

    async def _consumer_handler(self):
        while self._running:
            try:
                raw_input = await self._websocket.recv()
            except websockets.exceptions.ConnectionClosed:
                return  # stop the handler
            logger.debug('Recieved input from server')

            to_client = gabriel_pb2.ToClient()
            to_client.ParseFromString(raw_input)

            if to_client.HasField('welcome'):
                self._process_welcome(to_client.welcome)
            elif to_client.HasField('response'):
                self._process_response(to_client.resopnse)
            else:
                raise Exception('Empty to_client message')

    def _process_welcome(self, welcome):
        for source_name in welcome.sources_consumed:
            self._sources[source_name] = _Source(welcome.num_tokens_per_source)
        self._welcome_event.set()

    def _process_response(self, response):
        result_wrapper = response.result_wrapper
        if (result_wrapper.status == gabriel_pb2.ResultWrapper.SUCCESS):
            self.consumer(result_wrapper)
        elif (result_wrapper.status ==
              gabriel_pb2.ResultWrapper.NO_ENGINE_FOR_SOURCE):
            raise Exception('No engine for source')
        else:
            status = result_wrapper.Status.Name(result_wrapper.status)
            logger.error('Output status was: %s', status)

        if response.return_token:
            self._sources[response.source_name].return_token()

    async def _producer_handler(self, producer, source_name):
        '''
        Loop waiting until there is a token available. Then calls supplier to
        get the partially built FromClient to send.
        '''

        await self._welcome_event.wait()
        while self._running:
            assert source_name in self._sources, (
                'No engines consume source: {}'.format(source_name))
            source = self._sources[source_name]
            await source.get_token()

            input_frame = await producer()
            if input_frame is None:
                source.return_token()
                logger.info('Received None from producer')
                continue

            from_client = gabriel_pb2.FromClient()
            from_client.frame_id = source.get_frame_id()
            from_client.source_name = source_name
            from_client.input_frame.CopyFrom(input_frame)
            await self._send_from_client(from_client)
            logger.debug('num_tokens for %s is now %d', source_name,
                         source.get_num_tokens())
            source.next_frame()

    async def _send_from_client(self, from_client):
        try:
            await self._websocket.send(from_client.SerializeToString())
        except websockets.exceptions.ConnectionClosed:
            return  # stop the handler

class _Source:
    def __init__(self, num_tokens):
        self._num_tokens = num_tokens
        self._event = asyncio.Event()
        self._frame_id = 0

    def return_token(self):
        self._num_tokens += 1
        self._event.set()

    async def get_token(self):
        while self._num_tokens < 1:
            logger.debug('Waiting for token')
            self._event.clear()  # Clear because we definitely want wait
            await self._event.wait()

        self._num_tokens -= 1

    def get_num_tokens(self):
        return self._num_tokens

    def get_frame_id(self):
        return self._frame_id

    def next_frame(self):
        self._frame_id += 1
<commit_msg>bug fix<commit_after>import asyncio
import logging
import websockets
from gabriel_protocol import gabriel_pb2
from collections import namedtuple

URI_FORMAT = 'ws://{host}:{port}'

logger = logging.getLogger(__name__)
websockets_logger = logging.getLogger(websockets.__name__)

# The entire payload will be printed if this is allowed to be DEBUG
websockets_logger.setLevel(logging.INFO)

ProducerWrapper = namedtuple('ProducerWrapper', ['producer', 'source_name'])

class WebsocketClient:
    def __init__(self, host, port, producer_wrappers, consumer):
        '''
        producer should take no arguments and return an instance of
        gabriel_pb2.InputFrame.
        consumer should take one gabriel_pb2.ResultWrapper and does not need to
        return anything.
        '''

        self._welcome_event = asyncio.Event()
        self._sources = {}
        self._running = True
        self._uri = URI_FORMAT.format(host=host, port=port)
        self.producer_wrappers = producer_wrappers
        self.consumer = consumer

    def launch(self):
        event_loop = asyncio.get_event_loop()

        # TODO remove this line once we stop supporting Python 3.5
        asyncio.set_event_loop(event_loop)

        try:
            self._websocket = event_loop.run_until_complete(
                websockets.connect(self._uri))
        except ConnectionRefusedError:
            logger.error('Could not connect to server')
            return

        consumer_task = asyncio.ensure_future(self._consumer_handler())
        tasks = [
            asyncio.ensure_future(self._producer_handler(
                producer_wrapper.producer, producer_wrapper.source_name))
            for producer_wrapper in self.producer_wrappers
        ]
        tasks.append(consumer_task)

        _, pending = event_loop.run_until_complete(asyncio.wait(
                tasks, return_when=asyncio.FIRST_COMPLETED))
        for task in pending:
            task.cancel()
        logger.info('Disconnected From Server')

    def get_source_names(self):
        return self._sources.keys()

    def stop(self):
        self._running = False
        logger.info('stopping server')

    async def _consumer_handler(self):
        while self._running:
            try:
                raw_input = await self._websocket.recv()
            except websockets.exceptions.ConnectionClosed:
                return  # stop the handler
            logger.debug('Recieved input from server')

            to_client = gabriel_pb2.ToClient()
            to_client.ParseFromString(raw_input)

            if to_client.HasField('welcome'):
                self._process_welcome(to_client.welcome)
            elif to_client.HasField('response'):
                self._process_response(to_client.response)
            else:
                raise Exception('Empty to_client message')

    def _process_welcome(self, welcome):
        for source_name in welcome.sources_consumed:
            self._sources[source_name] = _Source(welcome.num_tokens_per_source)
        self._welcome_event.set()

    def _process_response(self, response):
        result_wrapper = response.result_wrapper
        if (result_wrapper.status == gabriel_pb2.ResultWrapper.SUCCESS):
            self.consumer(result_wrapper)
        elif (result_wrapper.status ==
              gabriel_pb2.ResultWrapper.NO_ENGINE_FOR_SOURCE):
            raise Exception('No engine for source')
        else:
            status = result_wrapper.Status.Name(result_wrapper.status)
            logger.error('Output status was: %s', status)

        if response.return_token:
            self._sources[response.source_name].return_token()

    async def _producer_handler(self, producer, source_name):
        '''
        Loop waiting until there is a token available. Then calls supplier to
        get the partially built FromClient to send.
        '''

        await self._welcome_event.wait()
        while self._running:
            assert source_name in self._sources, (
                'No engines consume source: {}'.format(source_name))
            source = self._sources[source_name]
            await source.get_token()

            input_frame = await producer()
            if input_frame is None:
                source.return_token()
                logger.info('Received None from producer')
                continue

            from_client = gabriel_pb2.FromClient()
            from_client.frame_id = source.get_frame_id()
            from_client.source_name = source_name
            from_client.input_frame.CopyFrom(input_frame)
            await self._send_from_client(from_client)
            logger.debug('num_tokens for %s is now %d', source_name,
                         source.get_num_tokens())
            source.next_frame()

    async def _send_from_client(self, from_client):
        try:
            await self._websocket.send(from_client.SerializeToString())
        except websockets.exceptions.ConnectionClosed:
            return  # stop the handler

class _Source:
    def __init__(self, num_tokens):
        self._num_tokens = num_tokens
        self._event = asyncio.Event()
        self._frame_id = 0

    def return_token(self):
        self._num_tokens += 1
        self._event.set()

    async def get_token(self):
        while self._num_tokens < 1:
            logger.debug('Waiting for token')
            self._event.clear()  # Clear because we definitely want wait
            await self._event.wait()

        self._num_tokens -= 1

    def get_num_tokens(self):
        return self._num_tokens

    def get_frame_id(self):
        return self._frame_id

    def next_frame(self):
        self._frame_id += 1
<|endoftext|>


<commit_before>import logging

__version__ = '0.1.1'

log = logging.getLogger(__name__)
# log=logging

log.debug('version=' + __version__)

from avrgcc import *
from avrsize import *
from arduino import *
<commit_msg>version++<commit_after>import logging

__version__ = '0.1.2'

log = logging.getLogger(__name__)
# log=logging

log.debug('version=' + __version__)

from avrgcc import *
from avrsize import *
from arduino import *
<|endoftext|>