<commit_before># -*- coding: utf-8 -*-

"""
    zeit.recommend.morelikethis
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~

    This module has no description.

    Copyright: (c) 2013 by Nicolas Drebenstedt.
    License: BSD, see LICENSE for more details.
"""

from storm import Bolt
from storm import log
from storm import emit
import urllib2
import time
import json

class MorelikethisBolt(Bolt):

    connections = {}

    def initialize(self, conf, context):
        host = conf.get('zeit.recommend.zonapi.host', 'localhost')
        port = conf.get('zeit.recommend.zonapi.port', 8983)
        self.url = 'http://%s:%s/solr/' % (host, port)

    def recommend(self, paths):
        q = 'q=' + '%20'.join(['*' + p for p in paths])
        raw = urllib2.urlopen(self.url + 'select?fl=body&wt=json&df=href&' + q)
        data = raw.read()
        response = json.loads(data)['response']
        body = ' '.join([d['body'] for d in response['docs']]).encode('utf-8')

        header = {'Content-Type': 'text/plain'}
        req = urllib2.Request(self.url + 'mlt?fl=href', body, header)
        raw = urllib2.urlopen(req)
        data = raw.read()
        response = json.loads(data)['response']
        return list(i['href'][18:] for i in response['docs'])

    def process(self, tup):
        if tup.stream == 'control':
            action, user = tup.values
            if action == 'connect':
                self.connections[user] = int(time.time())
            elif action == 'disconnect':
                del self.connections[user]

        elif tup.stream == 'default':
            user, paths = tup.values
            if user in self.connections:
                log('[MorelikethisBolt] Incoming: %s' % user)

                recommendations = self.recommend(paths)
                paths = list(set(paths))[:10]

                emit([user, paths, recommendations])

if __name__ == '__main__':
    MorelikethisBolt().run()
<commit_msg>Fix encoding<commit_after># -*- coding: utf-8 -*-

"""
    zeit.recommend.morelikethis
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~

    This module has no description.

    Copyright: (c) 2013 by Nicolas Drebenstedt.
    License: BSD, see LICENSE for more details.
"""

from storm import Bolt
from storm import log
from storm import emit
import urllib2
import time
import json

class MorelikethisBolt(Bolt):

    connections = {}

    def initialize(self, conf, context):
        host = conf.get('zeit.recommend.zonapi.host', 'localhost')
        port = conf.get('zeit.recommend.zonapi.port', 8983)
        self.url = 'http://%s:%s/solr/' % (host, port)

    def recommend(self, paths):
        q = 'q=' + '%20'.join(['*' + p for p in paths])
        raw = urllib2.urlopen(self.url + 'select?fl=body&wt=json&df=href&' + q)
        data = raw.read()
        response = json.loads(data)['response']
        bodies = [d['body'] for d in response['docs']]
        body = ' '.join(bodies).encode('ascii', 'ignore')
        header = {'Content-Type': 'text/plain; charset=utf-8'}
        req = urllib2.Request(self.url + 'mlt?fl=href', body, header)
        raw = urllib2.urlopen(req)
        data = raw.read()
        response = json.loads(data)['response']
        return list(i['href'][18:] for i in response['docs'])

    def process(self, tup):
        if tup.stream == 'control':
            action, user = tup.values
            if action == 'connect':
                self.connections[user] = int(time.time())
            elif action == 'disconnect':
                del self.connections[user]

        elif tup.stream == 'default':
            user, paths = tup.values
            if user in self.connections:
                log('[MorelikethisBolt] Incoming: %s' % user)

                recommendations = self.recommend(paths)
                paths = list(set(paths))[:10]

                emit([user, paths, recommendations])

if __name__ == '__main__':
    MorelikethisBolt().run()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import collections

import numpy as np

from . import periods, holders
from .tools import empty_clone, stringify_array

class Simulation(object):
    compact_legislation_by_instant_cache = None
    debug = False
    debug_all = False  # When False, log only formula calls with non-default parameters.
    period = None
    reference_compact_legislation_by_instant_cache = None
    stack_trace = None
    steps_count = 1
    tax_benefit_system = None
    trace = False
    traceback = None

    def __init__(self, debug = False, debug_all = False, period = None, tax_benefit_system = None,
    trace = False, opt_out_cache = False):
        assert isinstance(period, periods.Period)
        self.period = period
        self.holder_by_name = {}

        # To keep track of the values (formulas and periods) being calculated to detect circular definitions.
        # See use in formulas.py.
        # The data structure of requested_periods_by_variable_name is: {variable_name: [period1, period2]}
        self.requested_periods_by_variable_name = {}
        self.max_nb_cycles = None

        if debug:
            self.debug = True
        if debug_all:
            assert debug
            self.debug_all = True
        assert tax_benefit_system is not None
        self.tax_benefit_system = tax_benefit_system
        if trace:
            self.trace = True
        self.opt_out_cache = opt_out_cache
        if debug or trace:
            self.stack_trace = collections.deque()
            self.traceback = collections.OrderedDict()

        # Note: Since simulations are short-lived and must be fast, don't use weakrefs for cache.
        self.compact_legislation_by_instant_cache = {}
        self.reference_compact_legislation_by_instant_cache = {}

        self.instantiate_entities()

    def instantiate_entities(self):
        self.persons = self.tax_benefit_system.person_entity(self)
        setattr(self, self.persons.key, self.persons)
        self.entities = {self.persons.key: self.persons}

        for entity_definition in self.tax_benefit_system.group_entities:
            entity = entity_definition(self)
            self.entities[entity_definition.key] = entity
            setattr(self, entity.key, entity)

    def calculate(self, column_name, period = None, **parameters):
        if period is None:
            period = self.period
        return self.compute(column_name, period = period, **parameters).array

    def calculate_add(self, column_name, period = None, **parameters):
        if period is None:
            period = self.period
        return self.compute_add(column_name, period = period, **parameters).array

    def calculate_add_divide(self, column_name, period = None, **parameters):
        if period is None:
            period = self.period
        return self.compute_add_divide(column_name, period = period, **parameters).array

    def calculate_divide(self, column_name, period = None, **parameters):
        if period is None:
            period = self.period
        return self.compute_divide(column_name, period = period, **parameters).array

    def calculate_output(self, column_name, period = None):
        """Calculate the value using calculate_output hooks in formula classes."""
        if period is None:
            period = self.period
        elif not isinstance(period, periods.Period):
            period = periods.period(period)
        holder = self.get_or_new_holder(column_name)
        return holder.calculate_output(period)

    def clone(self, debug = False, debug_all = False, trace = False):
        """Copy the simulation just enough to be able to run the copy without modifying the original simulation."""
        new = empty_clone(self)
        new_dict = new.__dict__

        for key, value in self.__dict__.iteritems():
            if key not in ('debug', 'debug_all', 'trace'):
                new_dict[key] = value

        if debug:
            new_dict['debug'] = True
        if debug_all:
            new_dict['debug_all'] = True
        if trace:
            new_dict['trace'] = True
        if debug or trace:
            new_dict['stack_trace'] = collections.deque()
            new_dict['traceback'] = collections.OrderedDict()

        new_dict['holder_by_name'] = {
            name: holder.clone()
            for name, holder in self.holder_by_name.iteritems()
            }
        return new

    def compute(self, column_name, period = None, **parameters):
        if period is None:
            period = self.period
        elif not isinstance(period, periods.Period):
            period = periods.period(period)
        print_trace = parameters.pop('print_trace', False)
        if print_trace:
            print_trace_kwargs = {}
            max_depth = parameters.pop('max_depth', None)
            if max_depth is not None:
                print_trace_kwargs['max_depth'] = max_depth
            show_default_values = parameters.pop('show_default_values', None)
            if show_default_values is not None:
                print_trace_kwargs['show_default_values'] = show_default_values
        if (self.debug or self.trace) and self.stack_trace:
            variable_infos = (column_name, period)
            calling_frame = self.stack_trace[-1]
            caller_input_variables_infos = calling_frame['input_variables_infos']
            if variable_infos not in caller_input_variables_infos:
                caller_input_variables_infos.append(variable_infos)
        holder = self.get_or_new_holder(column_name)
        result = holder.compute(period = period, **parameters)
        if print_trace:
            self.print_trace(variable_name=column_name, period=period, **print_trace_kwargs)
        return result

    def compute_add(self, column_name, period = None, **parameters):
        if period is None:
            period = self.period
        elif not isinstance(period, periods.Period):
            period = periods.period(period)
        if (self.debug or self.trace) and self.stack_trace:
            variable_infos = (column_name, period)
            calling_frame = self.stack_trace[-1]
            caller_input_variables_infos = calling_frame['input_variables_infos']
            if variable_infos not in caller_input_variables_infos:
                caller_input_variables_infos.append(variable_infos)
        holder = self.get_or_new_holder(column_name)
        return holder.compute_add(period = period, **parameters)

    def compute_add_divide(self, column_name, period = None, **parameters):
        if period is None:
            period = self.period
        elif not isinstance(period, periods.Period):
            period = periods.period(period)
        if (self.debug or self.trace) and self.stack_trace:
            variable_infos = (column_name, period)
            calling_frame = self.stack_trace[-1]
            caller_input_variables_infos = calling_frame['input_variables_infos']
            if variable_infos not in caller_input_variables_infos:
                caller_input_variables_infos.append(variable_infos)
        holder = self.get_or_new_holder(column_name)
        return holder.compute_add_divide(period = period, **parameters)

    def compute_divide(self, column_name, period = None, **parameters):
        if period is None:
            period = self.period
        elif not isinstance(period, periods.Period):
            period = periods.period(period)
        if (self.debug or self.trace) and self.stack_trace:
            variable_infos = (column_name, period)
            calling_frame = self.stack_trace[-1]
            caller_input_variables_infos = calling_frame['input_variables_infos']
            if variable_infos not in caller_input_variables_infos:
                caller_input_variables_infos.append(variable_infos)
        holder = self.get_or_new_holder(column_name)
        return holder.compute_divide(period = period, **parameters)

    def get_array(self, column_name, period = None):
        if period is None:
            period = self.period
        elif not isinstance(period, periods.Period):
            period = periods.period(period)
        if (self.debug or self.trace) and self.stack_trace:
            variable_infos = (column_name, period)
            calling_frame = self.stack_trace[-1]
            caller_input_variables_infos = calling_frame['input_variables_infos']
            if variable_infos not in caller_input_variables_infos:
                caller_input_variables_infos.append(variable_infos)
        return self.get_or_new_holder(column_name).get_array(period)

    def get_compact_legislation(self, instant):
        compact_legislation = self.compact_legislation_by_instant_cache.get(instant)
        if compact_legislation is None:
            compact_legislation = self.tax_benefit_system.get_compact_legislation(
                instant = instant,
                traced_simulation = self if self.trace else None,
                )
            self.compact_legislation_by_instant_cache[instant] = compact_legislation
        return compact_legislation

    def get_holder(self, column_name, default = UnboundLocalError):
        if default is UnboundLocalError:
            return self.holder_by_name[column_name]
        return self.holder_by_name.get(column_name, default)

    def get_or_new_holder(self, column_name):
        holder = self.holder_by_name.get(column_name)
        if holder is None:
            column = self.tax_benefit_system.get_column(column_name, check_existence = True)
            self.holder_by_name[column_name] = holder = holders.Holder(
                self,
                column = column,
                )
            if column.formula_class is not None:
                holder.formula = column.formula_class(holder = holder)
        return holder

    def get_reference_compact_legislation(self, instant):
        reference_compact_legislation = self.reference_compact_legislation_by_instant_cache.get(instant)
        if reference_compact_legislation is None:
            reference_compact_legislation = self.tax_benefit_system.get_reference_compact_legislation(
                instant = instant,
                traced_simulation = self if self.trace else None,
                )
            self.reference_compact_legislation_by_instant_cache[instant] = reference_compact_legislation
        return reference_compact_legislation

    def graph(self, column_name, edges, get_input_variables_and_parameters, nodes, visited):
        self.get_or_new_holder(column_name).graph(edges, get_input_variables_and_parameters, nodes, visited)

    def legislation_at(self, instant, reference = False):
        if isinstance(instant, periods.Period):
            instant = instant.start
        assert isinstance(instant, periods.Instant), "Expected an instant. Got: {}".format(instant)
        if reference:
            return self.get_reference_compact_legislation(instant)
        return self.get_compact_legislation(instant)

    def find_traceback_step(self, variable_name, period):
        assert isinstance(period, periods.Period), period
        column = self.tax_benefit_system.get_column(variable_name, check_existence=True)
        step = self.traceback.get((variable_name, period))
        if step is None and column.is_period_size_independent:
            period = None
        step = self.traceback.get((variable_name, period))
        return step

    def print_trace(self, variable_name, period, max_depth=3, show_default_values=True):
        """
        Print the dependencies of all the variables computed since the creation of the simulation.

        The `max_depth` parameter tells how much levels of the printed tree to show. Use -1 to disable limit.

        The simulation must have been initialized with `trace=True` argument.
        """
        def traverse(current_variable_name, current_period, depth):
            step = self.find_traceback_step(current_variable_name, current_period)
            assert step is not None
            holder = self.get_holder(current_variable_name)
            has_default_value = np.all(holder.get_array(current_period) == holder.column.default)
            if depth == 0 or (show_default_values or not has_default_value):
                indent = u'|     ' * (depth - 1) + u'|---> ' \
                    if depth > 0 \
                    else ''
                print(indent + self.stringify_variable_for_period_with_array(current_variable_name, current_period))
            input_variables_infos = step.get('input_variables_infos')
            if (max_depth == -1 or depth < max_depth) and input_variables_infos is not None:
                    for index, (child_variable_name, child_period) in enumerate(input_variables_infos):
                        traverse(child_variable_name, child_period, depth + 1)
        if not isinstance(max_depth, int) or max_depth < -1:
            raise ValueError(u'`max_depth` argument must be >= -1')
        if not self.trace:
            raise ValueError(u'This simulation has not been initialized with `trace=True` so the computation '
                'did not collect any trace information.')
        assert self.traceback is not None
        if not isinstance(period, periods.Period):
            period = periods.period(period)
        step = self.find_traceback_step(variable_name, period)
        if step is None:
            raise ValueError(u'The given `variable_name` "{0}" was not calculated for the given `period` "{1}". '
                u'It is therefore not possible to display the trace. '
                u'You should do `simulation.calculate({0!r}, {1}, print_trace=True)`.'.format(
                    variable_name, period))
        traverse(variable_name, period, depth=0)

    def stringify_variable_for_period_with_array(self, variable_name, period):
        holder = self.get_holder(variable_name)
        return u'{}@{}<{}>{}'.format(
            variable_name,
            holder.entity.key,
            str(period),
            stringify_array(holder.get_array(period)),
            )

    def stringify_input_variables_infos(self, input_variables_infos):
        return u', '.join(
            self.stringify_variable_for_period_with_array(
                variable_name=input_variable_name,
                period=input_variable_period,
                )
            for input_variable_name, input_variable_period in input_variables_infos
            )

    # Fixme: to rewrite
    def to_input_variables_json(self):
        return None

    def get_variable_entity(self, variable_name):
        column = self.tax_benefit_system.get_column(variable_name, check_existence = True)
        return self.get_entity(column.entity)

    def get_entity(self, entity_type):
        return self.entities[entity_type.key]
<commit_msg>Adapt simulation.clone<commit_after># -*- coding: utf-8 -*-

import collections

import numpy as np

from . import periods, holders
from .tools import empty_clone, stringify_array

class Simulation(object):
    compact_legislation_by_instant_cache = None
    debug = False
    debug_all = False  # When False, log only formula calls with non-default parameters.
    period = None
    reference_compact_legislation_by_instant_cache = None
    stack_trace = None
    steps_count = 1
    tax_benefit_system = None
    trace = False
    traceback = None

    def __init__(self, debug = False, debug_all = False, period = None, tax_benefit_system = None,
    trace = False, opt_out_cache = False):
        assert isinstance(period, periods.Period)
        self.period = period
        self.holder_by_name = {}

        # To keep track of the values (formulas and periods) being calculated to detect circular definitions.
        # See use in formulas.py.
        # The data structure of requested_periods_by_variable_name is: {variable_name: [period1, period2]}
        self.requested_periods_by_variable_name = {}
        self.max_nb_cycles = None

        if debug:
            self.debug = True
        if debug_all:
            assert debug
            self.debug_all = True
        assert tax_benefit_system is not None
        self.tax_benefit_system = tax_benefit_system
        if trace:
            self.trace = True
        self.opt_out_cache = opt_out_cache
        if debug or trace:
            self.stack_trace = collections.deque()
            self.traceback = collections.OrderedDict()

        # Note: Since simulations are short-lived and must be fast, don't use weakrefs for cache.
        self.compact_legislation_by_instant_cache = {}
        self.reference_compact_legislation_by_instant_cache = {}

        self.instantiate_entities()

    def instantiate_entities(self):
        self.persons = self.tax_benefit_system.person_entity(self)
        setattr(self, self.persons.key, self.persons)
        self.entities = {self.persons.key: self.persons}

        for entity_definition in self.tax_benefit_system.group_entities:
            entity = entity_definition(self)
            self.entities[entity_definition.key] = entity
            setattr(self, entity.key, entity)

    def calculate(self, column_name, period = None, **parameters):
        if period is None:
            period = self.period
        return self.compute(column_name, period = period, **parameters).array

    def calculate_add(self, column_name, period = None, **parameters):
        if period is None:
            period = self.period
        return self.compute_add(column_name, period = period, **parameters).array

    def calculate_add_divide(self, column_name, period = None, **parameters):
        if period is None:
            period = self.period
        return self.compute_add_divide(column_name, period = period, **parameters).array

    def calculate_divide(self, column_name, period = None, **parameters):
        if period is None:
            period = self.period
        return self.compute_divide(column_name, period = period, **parameters).array

    def calculate_output(self, column_name, period = None):
        """Calculate the value using calculate_output hooks in formula classes."""
        if period is None:
            period = self.period
        elif not isinstance(period, periods.Period):
            period = periods.period(period)
        holder = self.get_or_new_holder(column_name)
        return holder.calculate_output(period)

    def clone(self, debug = False, debug_all = False, trace = False):
        """Copy the simulation just enough to be able to run the copy without modifying the original simulation."""
        new = empty_clone(self)
        new_dict = new.__dict__

        for key, value in self.__dict__.iteritems():
            if key not in ('debug', 'debug_all', 'trace'):
                new_dict[key] = value

        for entity in new.entities.itervalues():
            entity.simulation = new

        if debug:
            new_dict['debug'] = True
        if debug_all:
            new_dict['debug_all'] = True
        if trace:
            new_dict['trace'] = True
        if debug or trace:
            new_dict['stack_trace'] = collections.deque()
            new_dict['traceback'] = collections.OrderedDict()

        new_dict['holder_by_name'] = {
            name: holder.clone()
            for name, holder in self.holder_by_name.iteritems()
            }
        return new

    def compute(self, column_name, period = None, **parameters):
        if period is None:
            period = self.period
        elif not isinstance(period, periods.Period):
            period = periods.period(period)
        print_trace = parameters.pop('print_trace', False)
        if print_trace:
            print_trace_kwargs = {}
            max_depth = parameters.pop('max_depth', None)
            if max_depth is not None:
                print_trace_kwargs['max_depth'] = max_depth
            show_default_values = parameters.pop('show_default_values', None)
            if show_default_values is not None:
                print_trace_kwargs['show_default_values'] = show_default_values
        if (self.debug or self.trace) and self.stack_trace:
            variable_infos = (column_name, period)
            calling_frame = self.stack_trace[-1]
            caller_input_variables_infos = calling_frame['input_variables_infos']
            if variable_infos not in caller_input_variables_infos:
                caller_input_variables_infos.append(variable_infos)
        holder = self.get_or_new_holder(column_name)
        result = holder.compute(period = period, **parameters)
        if print_trace:
            self.print_trace(variable_name=column_name, period=period, **print_trace_kwargs)
        return result

    def compute_add(self, column_name, period = None, **parameters):
        if period is None:
            period = self.period
        elif not isinstance(period, periods.Period):
            period = periods.period(period)
        if (self.debug or self.trace) and self.stack_trace:
            variable_infos = (column_name, period)
            calling_frame = self.stack_trace[-1]
            caller_input_variables_infos = calling_frame['input_variables_infos']
            if variable_infos not in caller_input_variables_infos:
                caller_input_variables_infos.append(variable_infos)
        holder = self.get_or_new_holder(column_name)
        return holder.compute_add(period = period, **parameters)

    def compute_add_divide(self, column_name, period = None, **parameters):
        if period is None:
            period = self.period
        elif not isinstance(period, periods.Period):
            period = periods.period(period)
        if (self.debug or self.trace) and self.stack_trace:
            variable_infos = (column_name, period)
            calling_frame = self.stack_trace[-1]
            caller_input_variables_infos = calling_frame['input_variables_infos']
            if variable_infos not in caller_input_variables_infos:
                caller_input_variables_infos.append(variable_infos)
        holder = self.get_or_new_holder(column_name)
        return holder.compute_add_divide(period = period, **parameters)

    def compute_divide(self, column_name, period = None, **parameters):
        if period is None:
            period = self.period
        elif not isinstance(period, periods.Period):
            period = periods.period(period)
        if (self.debug or self.trace) and self.stack_trace:
            variable_infos = (column_name, period)
            calling_frame = self.stack_trace[-1]
            caller_input_variables_infos = calling_frame['input_variables_infos']
            if variable_infos not in caller_input_variables_infos:
                caller_input_variables_infos.append(variable_infos)
        holder = self.get_or_new_holder(column_name)
        return holder.compute_divide(period = period, **parameters)

    def get_array(self, column_name, period = None):
        if period is None:
            period = self.period
        elif not isinstance(period, periods.Period):
            period = periods.period(period)
        if (self.debug or self.trace) and self.stack_trace:
            variable_infos = (column_name, period)
            calling_frame = self.stack_trace[-1]
            caller_input_variables_infos = calling_frame['input_variables_infos']
            if variable_infos not in caller_input_variables_infos:
                caller_input_variables_infos.append(variable_infos)
        return self.get_or_new_holder(column_name).get_array(period)

    def get_compact_legislation(self, instant):
        compact_legislation = self.compact_legislation_by_instant_cache.get(instant)
        if compact_legislation is None:
            compact_legislation = self.tax_benefit_system.get_compact_legislation(
                instant = instant,
                traced_simulation = self if self.trace else None,
                )
            self.compact_legislation_by_instant_cache[instant] = compact_legislation
        return compact_legislation

    def get_holder(self, column_name, default = UnboundLocalError):
        if default is UnboundLocalError:
            return self.holder_by_name[column_name]
        return self.holder_by_name.get(column_name, default)

    def get_or_new_holder(self, column_name):
        holder = self.holder_by_name.get(column_name)
        if holder is None:
            column = self.tax_benefit_system.get_column(column_name, check_existence = True)
            self.holder_by_name[column_name] = holder = holders.Holder(
                self,
                column = column,
                )
            if column.formula_class is not None:
                holder.formula = column.formula_class(holder = holder)
        return holder

    def get_reference_compact_legislation(self, instant):
        reference_compact_legislation = self.reference_compact_legislation_by_instant_cache.get(instant)
        if reference_compact_legislation is None:
            reference_compact_legislation = self.tax_benefit_system.get_reference_compact_legislation(
                instant = instant,
                traced_simulation = self if self.trace else None,
                )
            self.reference_compact_legislation_by_instant_cache[instant] = reference_compact_legislation
        return reference_compact_legislation

    def graph(self, column_name, edges, get_input_variables_and_parameters, nodes, visited):
        self.get_or_new_holder(column_name).graph(edges, get_input_variables_and_parameters, nodes, visited)

    def legislation_at(self, instant, reference = False):
        if isinstance(instant, periods.Period):
            instant = instant.start
        assert isinstance(instant, periods.Instant), "Expected an instant. Got: {}".format(instant)
        if reference:
            return self.get_reference_compact_legislation(instant)
        return self.get_compact_legislation(instant)

    def find_traceback_step(self, variable_name, period):
        assert isinstance(period, periods.Period), period
        column = self.tax_benefit_system.get_column(variable_name, check_existence=True)
        step = self.traceback.get((variable_name, period))
        if step is None and column.is_period_size_independent:
            period = None
        step = self.traceback.get((variable_name, period))
        return step

    def print_trace(self, variable_name, period, max_depth=3, show_default_values=True):
        """
        Print the dependencies of all the variables computed since the creation of the simulation.

        The `max_depth` parameter tells how much levels of the printed tree to show. Use -1 to disable limit.

        The simulation must have been initialized with `trace=True` argument.
        """
        def traverse(current_variable_name, current_period, depth):
            step = self.find_traceback_step(current_variable_name, current_period)
            assert step is not None
            holder = self.get_holder(current_variable_name)
            has_default_value = np.all(holder.get_array(current_period) == holder.column.default)
            if depth == 0 or (show_default_values or not has_default_value):
                indent = u'|     ' * (depth - 1) + u'|---> ' \
                    if depth > 0 \
                    else ''
                print(indent + self.stringify_variable_for_period_with_array(current_variable_name, current_period))
            input_variables_infos = step.get('input_variables_infos')
            if (max_depth == -1 or depth < max_depth) and input_variables_infos is not None:
                    for index, (child_variable_name, child_period) in enumerate(input_variables_infos):
                        traverse(child_variable_name, child_period, depth + 1)
        if not isinstance(max_depth, int) or max_depth < -1:
            raise ValueError(u'`max_depth` argument must be >= -1')
        if not self.trace:
            raise ValueError(u'This simulation has not been initialized with `trace=True` so the computation '
                'did not collect any trace information.')
        assert self.traceback is not None
        if not isinstance(period, periods.Period):
            period = periods.period(period)
        step = self.find_traceback_step(variable_name, period)
        if step is None:
            raise ValueError(u'The given `variable_name` "{0}" was not calculated for the given `period` "{1}". '
                u'It is therefore not possible to display the trace. '
                u'You should do `simulation.calculate({0!r}, {1}, print_trace=True)`.'.format(
                    variable_name, period))
        traverse(variable_name, period, depth=0)

    def stringify_variable_for_period_with_array(self, variable_name, period):
        holder = self.get_holder(variable_name)
        return u'{}@{}<{}>{}'.format(
            variable_name,
            holder.entity.key,
            str(period),
            stringify_array(holder.get_array(period)),
            )

    def stringify_input_variables_infos(self, input_variables_infos):
        return u', '.join(
            self.stringify_variable_for_period_with_array(
                variable_name=input_variable_name,
                period=input_variable_period,
                )
            for input_variable_name, input_variable_period in input_variables_infos
            )

    # Fixme: to rewrite
    def to_input_variables_json(self):
        return None

    def get_variable_entity(self, variable_name):
        column = self.tax_benefit_system.get_column(variable_name, check_existence = True)
        return self.get_entity(column.entity)

    def get_entity(self, entity_type):
        return self.entities[entity_type.key]
<|endoftext|>


<commit_before>from django import template
from django.utils.html import format_html, escape

register = template.Library()

@register.filter(name='parse_tweet_text')
def parse_tweet_text(tweet_object, autoescape=False):
    parsed_tweet_dict = tweet_object.parse_all()

    # return a dictionary that looks like
    # {'hashtag0': #hash</span", 'mention0': @mike}
    encoded = encode_tweet(parsed_tweet_dict)

    # merge
    parsed_tweet_dict[u'parsed_text'] = escape(
        parsed_tweet_dict[u'parsed_text'])
    statement = parsed_tweet_dict.get(u'parsed_text').format(**encoded)
    return statement

def encode_tweet(parsed_tweet_dict):
    result = {}

    # adding html component to hashtags
    for index, value in enumerate(parsed_tweet_dict.get(u'hashtags')):
        result['hashtag'+str(index)] = \
            u'#{hashtag}'\
                .format(hashtag=escape(value), link="/hashtag/{0}"\
                    .format(escape(value[1:])))

    # adding html component to mentions
    for index, value in enumerate(parsed_tweet_dict.get(u'mentions')):
        result['mention'+str(index)] = \
            u'@{mention}'\
                .format(mention=escape(value), link="/user/{0}"\
                    .format(escape(value[1:])))

    return result
<commit_msg>fix mention filter tag<commit_after>from django import template
from django.utils.html import format_html, escape

register = template.Library()

@register.filter(name='parse_tweet_text')
def parse_tweet_text(tweet_object, autoescape=False):
    parsed_tweet_dict = tweet_object.parse_all()

    # return a dictionary that looks like
    # {'hashtag0': #hash</span", 'mention0': @mike}
    encoded = encode_tweet(parsed_tweet_dict)

    # merge
    parsed_tweet_dict[u'parsed_text'] = escape(
        parsed_tweet_dict[u'parsed_text'])
    statement = parsed_tweet_dict.get(u'parsed_text').format(**encoded)
    return statement

def encode_tweet(parsed_tweet_dict):
    result = {}

    # adding html component to hashtags
    for index, value in enumerate(parsed_tweet_dict.get(u'hashtags')):
        result['hashtag'+str(index)] = \
            u'#{hashtag}'\
                .format(hashtag=escape(value), link="/hashtag/{0}"\
                .format(escape(value)))

    # adding html component to mentions
    for index, value in enumerate(parsed_tweet_dict.get(u'mentions')):
        result['mention'+str(index)] = \
            u'@{mention}'\
                .format(mention=escape(value), link="/user/{0}"\
                .format(escape(value)))

    return result
<|endoftext|>


<commit_before><commit_msg>working through commit args<commit_after><|endoftext|>


<commit_before>from datetime import datetime
from notebook.utils import url_path_join as ujoin
from rpaths import Path
import subprocess
import sys
from tornado.process import Subprocess
from tornado.web import RequestHandler, asynchronous

try:
    unicode_ = unicode
except NameError:
    unicode_ = str

class TraceHandler(RequestHandler):
    def initialize(self, nbapp=None):
        self.nbapp = nbapp
        self._tempdir = None

    @asynchronous
    def post(self):
        self._notebook_file = Path(self.get_body_argument('file'))
        name = self._notebook_file.unicodename
        if name.endswith('.ipynb'):
            name = name[:-6]
        name = u'%s.%s.rpz' % (name, datetime.now().strftime('%Y%m%d%H%M%S'))
        self._pack_file = self._notebook_file.parent / name
        self.nbapp.log.info("reprozip: tracing request from client: file=%r",
                            self._notebook_file)
        self._tempdir = Path.tempdir()
        self.nbapp.log.info("reprozip: created temp directory %r",
                            self._tempdir)
        proc = Subprocess(
            [sys.executable, '-c',
             'from reprozip_jupyter.main import main; main()',
             'trace',
             '--dont-save-notebook',
             '-d', self._tempdir.path,
             self._notebook_file.path],
            stdin=subprocess.PIPE)
        proc.stdin.close()
        proc.set_exit_callback(self._trace_done)
        self.nbapp.log.info("reprozip: started tracing...")

    def _trace_done(self, returncode):
        self.nbapp.log.info("reprozip: tracing done, returned %d", returncode)
        if returncode == 0:
            # Pack
            if self._pack_file.exists():
                self._pack_file.remove()
            proc = Subprocess(
                ['reprozip', 'pack', '-d',
                 self._tempdir.path,
                 self._pack_file.path],
                stdin=subprocess.PIPE)
            proc.stdin.close()
            proc.set_exit_callback(self._packing_done)
            self.nbapp.log.info("reprozip: started packing...")
        else:
            self._tempdir.rmtree()
            self.send_error(500)

    def _packing_done(self, returncode):
        self.nbapp.log.info("reprozip: packing done, returned %d", returncode)
        if returncode == 0:
            # Send the response
            self.set_header('Content-Type', 'application/json')
            self.finish({'bundle': unicode_(self._pack_file)})
            self.nbapp.log.info("reprozip: response sent!")
        else:
            self.send_error(500)
        self._tempdir.rmtree()

def load_jupyter_server_extension(nbapp):
    nbapp.log.info('reprozip: notebook extension loaded')

    webapp = nbapp.web_app
    base_url = webapp.settings['base_url']
    webapp.add_handlers(".*$", [
        (ujoin(base_url, r"/reprozip/trace"), TraceHandler, {'nbapp': nbapp}),
    ])
<commit_msg>Change package name format<commit_after>from datetime import datetime
from notebook.utils import url_path_join as ujoin
from rpaths import Path
import subprocess
import sys
from tornado.process import Subprocess
from tornado.web import RequestHandler, asynchronous

try:
    unicode_ = unicode
except NameError:
    unicode_ = str

class TraceHandler(RequestHandler):
    def initialize(self, nbapp=None):
        self.nbapp = nbapp
        self._tempdir = None

    @asynchronous
    def post(self):
        self._notebook_file = Path(self.get_body_argument('file'))
        name = self._notebook_file.unicodename
        if name.endswith('.ipynb'):
            name = name[:-6]
        name = u'%s_%s.rpz' % (name, datetime.now().strftime('%Y%m%d-%H%M%S'))
        self._pack_file = self._notebook_file.parent / name
        self.nbapp.log.info("reprozip: tracing request from client: file=%r",
                            self._notebook_file)
        self._tempdir = Path.tempdir()
        self.nbapp.log.info("reprozip: created temp directory %r",
                            self._tempdir)
        proc = Subprocess(
            [sys.executable, '-c',
             'from reprozip_jupyter.main import main; main()',
             'trace',
             '--dont-save-notebook',
             '-d', self._tempdir.path,
             self._notebook_file.path],
            stdin=subprocess.PIPE)
        proc.stdin.close()
        proc.set_exit_callback(self._trace_done)
        self.nbapp.log.info("reprozip: started tracing...")

    def _trace_done(self, returncode):
        self.nbapp.log.info("reprozip: tracing done, returned %d", returncode)
        if returncode == 0:
            # Pack
            if self._pack_file.exists():
                self._pack_file.remove()
            proc = Subprocess(
                ['reprozip', 'pack', '-d',
                 self._tempdir.path,
                 self._pack_file.path],
                stdin=subprocess.PIPE)
            proc.stdin.close()
            proc.set_exit_callback(self._packing_done)
            self.nbapp.log.info("reprozip: started packing...")
        else:
            self._tempdir.rmtree()
            self.send_error(500)

    def _packing_done(self, returncode):
        self.nbapp.log.info("reprozip: packing done, returned %d", returncode)
        if returncode == 0:
            # Send the response
            self.set_header('Content-Type', 'application/json')
            self.finish({'bundle': unicode_(self._pack_file)})
            self.nbapp.log.info("reprozip: response sent!")
        else:
            self.send_error(500)
        self._tempdir.rmtree()

def load_jupyter_server_extension(nbapp):
    nbapp.log.info('reprozip: notebook extension loaded')

    webapp = nbapp.web_app
    base_url = webapp.settings['base_url']
    webapp.add_handlers(".*$", [
        (ujoin(base_url, r"/reprozip/trace"), TraceHandler, {'nbapp': nbapp}),
    ])
<|endoftext|>


<commit_before>import numpy as np
from scipy.ndimage.filters import maximum_filter, minimum_filter

from ..transform import integral_image
from ..feature.corner import _compute_auto_correlation
from ..util import img_as_float

from .censure_cy import _censure_dob_loop, _slanted_integral_image, _censure_octagon_loop

def _get_filtered_image(image, no_of_scales, mode):
    # TODO : Implement the STAR mode
    if mode == 'DoB':
        scales = np.zeros((image.shape[0], image.shape[1], no_of_scales))
        for i in range(no_of_scales):
            n = i + 1
            inner_wt = (1.0 / (2 * n + 1)**2)
            outer_wt = (1.0 / (12 * n**2 + 4 * n))
            integral_img = integral_image(image)
            filtered_image = np.zeros(image.shape)
            _censure_dob_loop(image, n, integral_img, filtered_image, inner_wt, outer_wt)
            scales[:, :, i] = filtered_image
        return scales
    elif mode == 'Octagon':
        # TODO : Decide the shapes of Octagon filters for scales > 7
        outer_shape = [(5, 2), (5, 3), (7, 3), (9, 4), (9, 7), (13, 7), (15, 10)]
        inner_shape = [(3, 0), (3, 1), (3, 2), (5, 2), (5, 3), (5, 4), (5, 5)]
        scales = np.zeros((image.shape[0], image.shape[1], no_of_scales))
        integral_img = integral_image(image)
        integral_img1 = _slanted_integral_image_modes(image, 1)
        integral_img2 = _slanted_integral_image_modes(image, 2)
        integral_img2 = np.ascontiguousarray(integral_img2)
        integral_img3 = _slanted_integral_image_modes(image, 3)
        integral_img3 = np.ascontiguousarray(integral_img3)
        integral_img4 = _slanted_integral_image_modes(image, 4)
        integral_img4 = np.ascontiguousarray(integral_img4)
        for k in range(no_of_scales):
            n = k + 1
            filtered_image = np.zeros(image.shape)
            mo = outer_shape[n - 1][0]
            no = outer_shape[n - 1][1]
            mi = inner_shape[n - 1][0]
            ni = inner_shape[n - 1][1]
            outer_pixels = (mo + 2 * no)**2 - 2 * no * (no + 1)
            inner_pixels = (mi + 2 * ni)**2 - 2 * ni * (ni + 1)
            outer_wt = 1.0 / (outer_pixels - inner_pixels)
            inner_wt = 1.0 / inner_pixels

            _censure_octagon_loop(image, integral_img, integral_img1, integral_img2, integral_img3, integral_img4, filtered_image, outer_wt, inner_wt, mo, no, mi, ni)

            scales[:, :, k] = filtered_image
        return scales

def _slanted_integral_image_modes(img, mode=1):
    if mode == 1:
        """
        The following figures describe area that is summed up to calculate
        the value at point @ in slanted integral image.
         _________________
        |********/        |
        |*******/         |
        |******/          |
        |-----@           |
        |                 |
        |                 |
        |_________________|
        """
        image = np.copy(img, order='C')
        mode1 = np.zeros((image.shape[0] + 1, image.shape[1]), order='C')
        _slanted_integral_image(image, mode1)
        return mode1[1:, :]

    elif mode == 2:
        """
         _________________
        |                 |
        |                 |
        |                 |
        |           @_____|
        |          /******|
        |         /*******|
        |________/________|
        """
        image = np.copy(img, order='C')
        image = np.fliplr(image)
        image = np.flipud(image)
        image = np.ascontiguousarray(image)
        mode2 = np.zeros((image.shape[0] + 1, image.shape[1]), order='C')
        _slanted_integral_image(image, mode2)
        mode2 = mode2[1:, :]
        mode2 = np.fliplr(mode2)
        mode2 = np.flipud(mode2)
        return mode2

    elif mode == 3:
        """
         _________________
        |                 |
        |\\               |
        |*\\              |
        |**\\             |
        |***@             |
        |***|             |
        |___|_____________|
        """
        image = np.copy(img, order='C')
        image = np.flipud(image)
        image = image.T
        image = np.ascontiguousarray(image)
        mode3 = np.zeros((image.shape[0] + 1, image.shape[1]), order='C')
        _slanted_integral_image(image, mode3)
        mode3 = mode3[1:, :]
        mode3 = np.flipud(mode3.T)
        return mode3

    else:
        """
         ________________
        |           |****|
        |           |****|
        |           @****|
        |            \\**|
        |             \\*|
        |              \\|
        |________________|
        """
        image = np.copy(img, order='C')
        image = np.fliplr(image)
        image = image.T
        image = np.ascontiguousarray(image)
        mode4 = np.zeros((image.shape[0] + 1, image.shape[1]), order='C')
        _slanted_integral_image(image, mode4)
        mode4 = mode4[1:, :]
        mode4 = np.fliplr(mode4.T)
        return mode4

def _suppress_line(response, sigma, rpc_threshold):
    Axx, Axy, Ayy = _compute_auto_correlation(response, sigma)
    detA = Axx * Ayy - Axy**2
    traceA = Axx + Ayy
    # ratio of principal curvatures
    rpc = traceA**2 / (detA + 0.001)
    response[rpc > rpc_threshold] = 0
    return response

def censure_keypoints(image, no_of_scales=7, mode='DoB', threshold=0.03, rpc_threshold=10):
    """
    Extracts Censure keypoints along with the corresponding scale using
    either Difference of Boxes, Octagon or STAR bilevel filter.

    Parameters
    ----------
    image : 2D ndarray
        Input image.

    no_of_scales : positive integer
        Number of scales to extract keypoints from. The keypoints will be
        extracted from all the scales except the first and the last.

    mode : {'DoB', 'Octagon', 'STAR'}
        Type of bilevel filter used to get the scales of input image. Possible
        values are 'DoB', 'Octagon' and 'STAR'.

    threshold : float
        Threshold value used to suppress maximas and minimas with a weak
        magnitude response obtained after Non-Maximal Suppression.

    rpc_threshold : float
        Threshold for rejecting interest points which have ratio of principal
        curvatures greater than this value.

    Returns
    -------
    keypoints : (N, 3) array
        Location of extracted keypoints along with the corresponding scale.

    References
    ----------
    .. [1] Motilal Agrawal, Kurt Konolige and Morten Rufus Blas
           "CenSurE: Center Surround Extremas for Realtime Feature
           Detection and Matching",
           http://link.springer.com/content/pdf/10.1007%2F978-3-540-88693-8_8.pdf

    .. [2] Adam Schmidt, Marek Kraft, Michal Fularz and Zuzanna Domagala
           "Comparative Assessment of Point Feature Detectors and
           Descriptors in the Context of Robot Navigation"
           http://www.jamris.org/01_2013/saveas.php?QUEST=JAMRIS_No01_2013_P_11-20.pdf

    """

    image = np.squeeze(image)
    if image.ndim != 2:
        raise ValueError("Only 2-D gray-scale images supported.")
    image = img_as_float(image)

    image = np.ascontiguousarray(image)

    # Generating all the scales
    scales = np.zeros((image.shape[0], image.shape[1], no_of_scales))
    scales = _get_filtered_image(image, no_of_scales, mode)

    # Suppressing points that are neither minima or maxima in their 3 x 3 x 3
    # neighbourhood to zero
    minimas = (minimum_filter(scales, (3, 3, 3)) == scales).astype(int) * scales
    maximas = (maximum_filter(scales, (3, 3, 3)) == scales).astype(int) * scales

    # Suppressing minimas and maximas weaker than threshold
    minimas[np.abs(minimas) < threshold] = 0
    maximas[np.abs(maximas) < threshold] = 0
    response = maximas + minimas

    for i in range(1, no_of_scales - 1):
        response[:, :, i] = _suppress_line(response[:, :, i], (1 + i / 3.0), rpc_threshold)

    # Returning keypoints with its scale
    keypoints = np.transpose(np.nonzero(response[:, :, 1:no_of_scales - 1])) + [0, 0, 2]
    return keypoints
<commit_msg>Commenting the sigma passed to _suppress_line<commit_after>import numpy as np
from scipy.ndimage.filters import maximum_filter, minimum_filter

from ..transform import integral_image
from ..feature.corner import _compute_auto_correlation
from ..util import img_as_float

from .censure_cy import _censure_dob_loop, _slanted_integral_image, _censure_octagon_loop

def _get_filtered_image(image, no_of_scales, mode):
    # TODO : Implement the STAR mode
    if mode == 'DoB':
        scales = np.zeros((image.shape[0], image.shape[1], no_of_scales))
        for i in range(no_of_scales):
            n = i + 1
            inner_wt = (1.0 / (2 * n + 1)**2)
            outer_wt = (1.0 / (12 * n**2 + 4 * n))
            integral_img = integral_image(image)
            filtered_image = np.zeros(image.shape)
            _censure_dob_loop(image, n, integral_img, filtered_image, inner_wt, outer_wt)
            scales[:, :, i] = filtered_image
        return scales
    elif mode == 'Octagon':
        # TODO : Decide the shapes of Octagon filters for scales > 7
        outer_shape = [(5, 2), (5, 3), (7, 3), (9, 4), (9, 7), (13, 7), (15, 10)]
        inner_shape = [(3, 0), (3, 1), (3, 2), (5, 2), (5, 3), (5, 4), (5, 5)]
        scales = np.zeros((image.shape[0], image.shape[1], no_of_scales))
        integral_img = integral_image(image)
        integral_img1 = _slanted_integral_image_modes(image, 1)
        integral_img2 = _slanted_integral_image_modes(image, 2)
        integral_img2 = np.ascontiguousarray(integral_img2)
        integral_img3 = _slanted_integral_image_modes(image, 3)
        integral_img3 = np.ascontiguousarray(integral_img3)
        integral_img4 = _slanted_integral_image_modes(image, 4)
        integral_img4 = np.ascontiguousarray(integral_img4)
        for k in range(no_of_scales):
            n = k + 1
            filtered_image = np.zeros(image.shape)
            mo = outer_shape[n - 1][0]
            no = outer_shape[n - 1][1]
            mi = inner_shape[n - 1][0]
            ni = inner_shape[n - 1][1]
            outer_pixels = (mo + 2 * no)**2 - 2 * no * (no + 1)
            inner_pixels = (mi + 2 * ni)**2 - 2 * ni * (ni + 1)
            outer_wt = 1.0 / (outer_pixels - inner_pixels)
            inner_wt = 1.0 / inner_pixels

            _censure_octagon_loop(image, integral_img, integral_img1, integral_img2, integral_img3, integral_img4, filtered_image, outer_wt, inner_wt, mo, no, mi, ni)

            scales[:, :, k] = filtered_image
        return scales

def _slanted_integral_image_modes(img, mode=1):
    if mode == 1:
        """
        The following figures describe area that is summed up to calculate
        the value at point @ in slanted integral image.
         _________________
        |********/        |
        |*******/         |
        |******/          |
        |-----@           |
        |                 |
        |                 |
        |_________________|
        """
        image = np.copy(img, order='C')
        mode1 = np.zeros((image.shape[0] + 1, image.shape[1]), order='C')
        _slanted_integral_image(image, mode1)
        return mode1[1:, :]

    elif mode == 2:
        """
         _________________
        |                 |
        |                 |
        |                 |
        |           @_____|
        |          /******|
        |         /*******|
        |________/________|
        """
        image = np.copy(img, order='C')
        image = np.fliplr(image)
        image = np.flipud(image)
        image = np.ascontiguousarray(image)
        mode2 = np.zeros((image.shape[0] + 1, image.shape[1]), order='C')
        _slanted_integral_image(image, mode2)
        mode2 = mode2[1:, :]
        mode2 = np.fliplr(mode2)
        mode2 = np.flipud(mode2)
        return mode2

    elif mode == 3:
        """
         _________________
        |                 |
        |\\               |
        |*\\              |
        |**\\             |
        |***@             |
        |***|             |
        |___|_____________|
        """
        image = np.copy(img, order='C')
        image = np.flipud(image)
        image = image.T
        image = np.ascontiguousarray(image)
        mode3 = np.zeros((image.shape[0] + 1, image.shape[1]), order='C')
        _slanted_integral_image(image, mode3)
        mode3 = mode3[1:, :]
        mode3 = np.flipud(mode3.T)
        return mode3

    else:
        """
         ________________
        |           |****|
        |           |****|
        |           @****|
        |            \\**|
        |             \\*|
        |              \\|
        |________________|
        """
        image = np.copy(img, order='C')
        image = np.fliplr(image)
        image = image.T
        image = np.ascontiguousarray(image)
        mode4 = np.zeros((image.shape[0] + 1, image.shape[1]), order='C')
        _slanted_integral_image(image, mode4)
        mode4 = mode4[1:, :]
        mode4 = np.fliplr(mode4.T)
        return mode4

def _suppress_line(response, sigma, rpc_threshold):
    Axx, Axy, Ayy = _compute_auto_correlation(response, sigma)
    detA = Axx * Ayy - Axy**2
    traceA = Axx + Ayy
    # ratio of principal curvatures
    rpc = traceA**2 / (detA + 0.001)
    response[rpc > rpc_threshold] = 0
    return response

def censure_keypoints(image, no_of_scales=7, mode='DoB', threshold=0.03, rpc_threshold=10):
    """
    Extracts Censure keypoints along with the corresponding scale using
    either Difference of Boxes, Octagon or STAR bilevel filter.

    Parameters
    ----------
    image : 2D ndarray
        Input image.

    no_of_scales : positive integer
        Number of scales to extract keypoints from. The keypoints will be
        extracted from all the scales except the first and the last.

    mode : {'DoB', 'Octagon', 'STAR'}
        Type of bilevel filter used to get the scales of input image. Possible
        values are 'DoB', 'Octagon' and 'STAR'.

    threshold : float
        Threshold value used to suppress maximas and minimas with a weak
        magnitude response obtained after Non-Maximal Suppression.

    rpc_threshold : float
        Threshold for rejecting interest points which have ratio of principal
        curvatures greater than this value.

    Returns
    -------
    keypoints : (N, 3) array
        Location of extracted keypoints along with the corresponding scale.

    References
    ----------
    .. [1] Motilal Agrawal, Kurt Konolige and Morten Rufus Blas
           "CenSurE: Center Surround Extremas for Realtime Feature
           Detection and Matching",
           http://link.springer.com/content/pdf/10.1007%2F978-3-540-88693-8_8.pdf

    .. [2] Adam Schmidt, Marek Kraft, Michal Fularz and Zuzanna Domagala
           "Comparative Assessment of Point Feature Detectors and
           Descriptors in the Context of Robot Navigation"
           http://www.jamris.org/01_2013/saveas.php?QUEST=JAMRIS_No01_2013_P_11-20.pdf

    """

    image = np.squeeze(image)
    if image.ndim != 2:
        raise ValueError("Only 2-D gray-scale images supported.")
    image = img_as_float(image)

    image = np.ascontiguousarray(image)

    # Generating all the scales
    scales = np.zeros((image.shape[0], image.shape[1], no_of_scales))
    scales = _get_filtered_image(image, no_of_scales, mode)

    # Suppressing points that are neither minima or maxima in their 3 x 3 x 3
    # neighbourhood to zero
    minimas = (minimum_filter(scales, (3, 3, 3)) == scales).astype(int) * scales
    maximas = (maximum_filter(scales, (3, 3, 3)) == scales).astype(int) * scales

    # Suppressing minimas and maximas weaker than threshold
    minimas[np.abs(minimas) < threshold] = 0
    maximas[np.abs(maximas) < threshold] = 0
    response = maximas + minimas

    for i in range(1, no_of_scales - 1):
        # sigma = (window_size - 1) / 6.0
        # window_size = 7 + 2 * i
        # Hence sigma = 1 + i / 3.0
        response[:, :, i] = _suppress_line(response[:, :, i], (1 + i / 3.0), rpc_threshold)

    # Returning keypoints with its scale
    keypoints = np.transpose(np.nonzero(response[:, :, 1:no_of_scales - 1])) + [0, 0, 2]
    return keypoints
<|endoftext|>


<commit_before>import abc
import uuid
import struct
import weakref

from bson import json_util, EPOCH_NAIVE
# Monkeypatch to not include tz_info in decoded JSON.
# Kind of a ridiculous solution, but works.
json_util.EPOCH_AWARE = EPOCH_NAIVE

from sqlalchemy import String, Text, event
from sqlalchemy.types import TypeDecorator, BINARY
from sqlalchemy.interfaces import PoolListener
from sqlalchemy.engine import Engine
from sqlalchemy.ext import baked
from sqlalchemy.ext.mutable import Mutable
from sqlalchemy.ext.declarative import DeclarativeMeta

from inbox.util.encoding import base36encode, base36decode

from nylas.logging import get_logger
log = get_logger()

MAX_SANE_QUERIES_PER_SESSION = 100
MAX_TEXT_LENGTH = 65535

# Automatically bake queries for lazily-loaded relationships.
# Lazy loads are often suboptimal, but this makes them less so!
baked.bake_lazy_loaders()

bakery = baked.bakery(size=1024)

query_counts = weakref.WeakKeyDictionary()

@event.listens_for(Engine, "before_cursor_execute")
def before_cursor_execute(conn, cursor, statement,
                          parameters, context, executemany):
    if conn not in query_counts:
        query_counts[conn] = 1
    else:
        query_counts[conn] += 1

@event.listens_for(Engine, 'commit')
def before_commit(conn):
    if query_counts.get(conn, 0) > MAX_SANE_QUERIES_PER_SESSION:
        log.warning('Dubiously many queries per session!',
                    query_count=query_counts.get(conn))

class SQLAlchemyCompatibleAbstractMetaClass(DeclarativeMeta, abc.ABCMeta):
    """Declarative model classes that *also* inherit from an abstract base
    class need a metaclass like this one, in order to prevent metaclass
    conflict errors."""
    pass

class ABCMixin(object):
    """Use this if you want a mixin class which is actually an abstract base
    class, for example in order to enforce that concrete subclasses define
    particular methods or properties."""
    __metaclass__ = SQLAlchemyCompatibleAbstractMetaClass
    __abstract__ = True

# Column Types

# http://docs.sqlalchemy.org/en/rel_0_9/core/types.html#marshal-json-strings
class JSON(TypeDecorator):
    impl = Text

    def process_bind_param(self, value, dialect):
        if value is None:
            return None

        return json_util.dumps(value)

    def process_result_value(self, value, dialect):
        if not value:
            return None

        # Unfortunately loads() is strict about invalid utf-8 whereas dumps()
        # is not. This can result in ValueErrors during decoding - we simply
        # log and return None for now.
        # http://bugs.python.org/issue11489
        try:
            return json_util.loads(value)
        except ValueError:
            log.error('ValueError on decoding JSON', value=value)

def json_field_too_long(value):
    return len(json_util.dumps(value)) > MAX_TEXT_LENGTH

class LittleJSON(JSON):
    impl = String(255)

class BigJSON(JSON):
    # if all characters were 4-byte, this would fit in mysql's MEDIUMTEXT
    impl = Text(4194304)

class Base36UID(TypeDecorator):
    impl = BINARY(16)  # 128 bit unsigned integer

    def process_bind_param(self, value, dialect):
        if not value:
            return None
        return b36_to_bin(value)

    def process_result_value(self, value, dialect):
        return int128_to_b36(value)

# http://bit.ly/1LbMnqu
# Can simply use this as is because though we use bson.json_util, loads()
# dumps() return standard Python dicts like the json.* equivalents
# (because these are simply called under the hood)
class MutableDict(Mutable, dict):

    @classmethod
    def coerce(cls, key, value):
        """ Convert plain dictionaries to MutableDict. """
        if not isinstance(value, MutableDict):
            if isinstance(value, dict):
                return MutableDict(value)

            # this call will raise ValueError
            return Mutable.coerce(key, value)
        else:
            return value

    def __setitem__(self, key, value):
        """ Detect dictionary set events and emit change events. """
        dict.__setitem__(self, key, value)
        self.changed()

    def __delitem__(self, key):
        """ Detect dictionary del events and emit change events. """
        dict.__delitem__(self, key)
        self.changed()

    def update(self, *args, **kwargs):
        for k, v in dict(*args, **kwargs).iteritems():
            self[k] = v

    # To support pickling:
    def __getstate__(self):
        return dict(self)

    def __setstate__(self, state):
        self.update(state)

class MutableList(Mutable, list):

    @classmethod
    def coerce(cls, key, value):
        """Convert plain list to MutableList"""
        if not isinstance(value, MutableList):
            if isinstance(value, list):
                return MutableList(value)

            # this call will raise ValueError
            return Mutable.coerce(key, value)
        else:
            return value

    def __setitem__(self, idx, value):
        list.__setitem__(self, idx, value)
        self.changed()

    def __setslice__(self, start, stop, values):
        list.__setslice__(self, start, stop, values)
        self.changed()

    def __delitem__(self, idx):
        list.__delitem__(self, idx)
        self.changed()

    def __delslice__(self, start, stop):
        list.__delslice__(self, start, stop)
        self.changed()

    def append(self, value):
        list.append(self, value)
        self.changed()

    def insert(self, idx, value):
        list.insert(self, idx, value)
        self.changed()

    def extend(self, values):
        list.extend(self, values)
        self.changed()

    def pop(self, *args, **kw):
        value = list.pop(self, *args, **kw)
        self.changed()
        return value

    def remove(self, value):
        list.remove(self, value)
        self.changed()

def int128_to_b36(int128):
    """ int128: a 128 bit unsigned integer
        returns a base-36 string representation
    """
    if not int128:
        return None
    assert len(int128) == 16, "should be 16 bytes (128 bits)"
    a, b = struct.unpack('>QQ', int128)  # uuid() is big-endian
    pub_id = (a << 64) | b
    return base36encode(pub_id).lower()

def b36_to_bin(b36_string):
    """ b36_string: a base-36 encoded string
        returns binary 128 bit unsigned integer
    """
    int128 = base36decode(b36_string)
    MAX_INT64 = 0xFFFFFFFFFFFFFFFF
    return struct.pack(
        '>QQ',
        (int128 >> 64) & MAX_INT64,
        int128 & MAX_INT64)

def generate_public_id():
    """ Returns a base-36 string UUID """
    u = uuid.uuid4().bytes
    return int128_to_b36(u)

# Other utilities

# My good old friend Enrico to the rescue:
# http://www.enricozini.org/2012/tips/sa-sqlmode-traditional/
#
# We set sql-mode=traditional on the server side as well, but enforce at the
# application level to be extra safe.
#
# Without this, MySQL will silently insert invalid values in the database if
# not running with sql-mode=traditional.
class ForceStrictMode(PoolListener):

    def connect(self, dbapi_con, connection_record):
        cur = dbapi_con.cursor()
        cur.execute("SET SESSION sql_mode='TRADITIONAL'")
        cur = None

def maybe_refine_query(query, subquery):
    if subquery is None:
        return query
    return query.join(subquery.subquery())

def safer_yield_per(query, id_field, start_id, count):
    """Incautious execution of 'for result in query.yield_per(N):' may cause
    slowness or OOMing over large tables. This is a less general but less
    dangerous alternative.

    Parameters
    ----------
    query: sqlalchemy.Query
        The query to yield windowed results from.
    id_field: A SQLAlchemy attribute to use for windowing. E.g.,
        `Transaction.id`
    start_id: The value of id_field at which to start iterating.
    count: int
        The number of results to fetch at a time.
    """
    cur_id = start_id
    while True:
        results = query.filter(id_field >= cur_id).order_by(id_field).\
            limit(count).all()
        if not results:
            return
        for result in results:
            yield result
        cur_id = results[-1].id + 1
<commit_msg>Revert "Call SQLAlchemy's bake_lazy_loaders() for performance."<commit_after>import abc
import uuid
import struct
import weakref

from bson import json_util, EPOCH_NAIVE
# Monkeypatch to not include tz_info in decoded JSON.
# Kind of a ridiculous solution, but works.
json_util.EPOCH_AWARE = EPOCH_NAIVE

from sqlalchemy import String, Text, event
from sqlalchemy.types import TypeDecorator, BINARY
from sqlalchemy.interfaces import PoolListener
from sqlalchemy.engine import Engine
from sqlalchemy.ext import baked
from sqlalchemy.ext.mutable import Mutable
from sqlalchemy.ext.declarative import DeclarativeMeta

from inbox.util.encoding import base36encode, base36decode

from nylas.logging import get_logger
log = get_logger()

MAX_SANE_QUERIES_PER_SESSION = 100
MAX_TEXT_LENGTH = 65535

bakery = baked.bakery()

query_counts = weakref.WeakKeyDictionary()

@event.listens_for(Engine, "before_cursor_execute")
def before_cursor_execute(conn, cursor, statement,
                          parameters, context, executemany):
    if conn not in query_counts:
        query_counts[conn] = 1
    else:
        query_counts[conn] += 1

@event.listens_for(Engine, 'commit')
def before_commit(conn):
    if query_counts.get(conn, 0) > MAX_SANE_QUERIES_PER_SESSION:
        log.warning('Dubiously many queries per session!',
                    query_count=query_counts.get(conn))

class SQLAlchemyCompatibleAbstractMetaClass(DeclarativeMeta, abc.ABCMeta):
    """Declarative model classes that *also* inherit from an abstract base
    class need a metaclass like this one, in order to prevent metaclass
    conflict errors."""
    pass

class ABCMixin(object):
    """Use this if you want a mixin class which is actually an abstract base
    class, for example in order to enforce that concrete subclasses define
    particular methods or properties."""
    __metaclass__ = SQLAlchemyCompatibleAbstractMetaClass
    __abstract__ = True

# Column Types

# http://docs.sqlalchemy.org/en/rel_0_9/core/types.html#marshal-json-strings
class JSON(TypeDecorator):
    impl = Text

    def process_bind_param(self, value, dialect):
        if value is None:
            return None

        return json_util.dumps(value)

    def process_result_value(self, value, dialect):
        if not value:
            return None

        # Unfortunately loads() is strict about invalid utf-8 whereas dumps()
        # is not. This can result in ValueErrors during decoding - we simply
        # log and return None for now.
        # http://bugs.python.org/issue11489
        try:
            return json_util.loads(value)
        except ValueError:
            log.error('ValueError on decoding JSON', value=value)

def json_field_too_long(value):
    return len(json_util.dumps(value)) > MAX_TEXT_LENGTH

class LittleJSON(JSON):
    impl = String(255)

class BigJSON(JSON):
    # if all characters were 4-byte, this would fit in mysql's MEDIUMTEXT
    impl = Text(4194304)

class Base36UID(TypeDecorator):
    impl = BINARY(16)  # 128 bit unsigned integer

    def process_bind_param(self, value, dialect):
        if not value:
            return None
        return b36_to_bin(value)

    def process_result_value(self, value, dialect):
        return int128_to_b36(value)

# http://bit.ly/1LbMnqu
# Can simply use this as is because though we use bson.json_util, loads()
# dumps() return standard Python dicts like the json.* equivalents
# (because these are simply called under the hood)
class MutableDict(Mutable, dict):

    @classmethod
    def coerce(cls, key, value):
        """ Convert plain dictionaries to MutableDict. """
        if not isinstance(value, MutableDict):
            if isinstance(value, dict):
                return MutableDict(value)

            # this call will raise ValueError
            return Mutable.coerce(key, value)
        else:
            return value

    def __setitem__(self, key, value):
        """ Detect dictionary set events and emit change events. """
        dict.__setitem__(self, key, value)
        self.changed()

    def __delitem__(self, key):
        """ Detect dictionary del events and emit change events. """
        dict.__delitem__(self, key)
        self.changed()

    def update(self, *args, **kwargs):
        for k, v in dict(*args, **kwargs).iteritems():
            self[k] = v

    # To support pickling:
    def __getstate__(self):
        return dict(self)

    def __setstate__(self, state):
        self.update(state)

class MutableList(Mutable, list):

    @classmethod
    def coerce(cls, key, value):
        """Convert plain list to MutableList"""
        if not isinstance(value, MutableList):
            if isinstance(value, list):
                return MutableList(value)

            # this call will raise ValueError
            return Mutable.coerce(key, value)
        else:
            return value

    def __setitem__(self, idx, value):
        list.__setitem__(self, idx, value)
        self.changed()

    def __setslice__(self, start, stop, values):
        list.__setslice__(self, start, stop, values)
        self.changed()

    def __delitem__(self, idx):
        list.__delitem__(self, idx)
        self.changed()

    def __delslice__(self, start, stop):
        list.__delslice__(self, start, stop)
        self.changed()

    def append(self, value):
        list.append(self, value)
        self.changed()

    def insert(self, idx, value):
        list.insert(self, idx, value)
        self.changed()

    def extend(self, values):
        list.extend(self, values)
        self.changed()

    def pop(self, *args, **kw):
        value = list.pop(self, *args, **kw)
        self.changed()
        return value

    def remove(self, value):
        list.remove(self, value)
        self.changed()

def int128_to_b36(int128):
    """ int128: a 128 bit unsigned integer
        returns a base-36 string representation
    """
    if not int128:
        return None
    assert len(int128) == 16, "should be 16 bytes (128 bits)"
    a, b = struct.unpack('>QQ', int128)  # uuid() is big-endian
    pub_id = (a << 64) | b
    return base36encode(pub_id).lower()

def b36_to_bin(b36_string):
    """ b36_string: a base-36 encoded string
        returns binary 128 bit unsigned integer
    """
    int128 = base36decode(b36_string)
    MAX_INT64 = 0xFFFFFFFFFFFFFFFF
    return struct.pack(
        '>QQ',
        (int128 >> 64) & MAX_INT64,
        int128 & MAX_INT64)

def generate_public_id():
    """ Returns a base-36 string UUID """
    u = uuid.uuid4().bytes
    return int128_to_b36(u)

# Other utilities

# My good old friend Enrico to the rescue:
# http://www.enricozini.org/2012/tips/sa-sqlmode-traditional/
#
# We set sql-mode=traditional on the server side as well, but enforce at the
# application level to be extra safe.
#
# Without this, MySQL will silently insert invalid values in the database if
# not running with sql-mode=traditional.
class ForceStrictMode(PoolListener):

    def connect(self, dbapi_con, connection_record):
        cur = dbapi_con.cursor()
        cur.execute("SET SESSION sql_mode='TRADITIONAL'")
        cur = None

def maybe_refine_query(query, subquery):
    if subquery is None:
        return query
    return query.join(subquery.subquery())

def safer_yield_per(query, id_field, start_id, count):
    """Incautious execution of 'for result in query.yield_per(N):' may cause
    slowness or OOMing over large tables. This is a less general but less
    dangerous alternative.

    Parameters
    ----------
    query: sqlalchemy.Query
        The query to yield windowed results from.
    id_field: A SQLAlchemy attribute to use for windowing. E.g.,
        `Transaction.id`
    start_id: The value of id_field at which to start iterating.
    count: int
        The number of results to fetch at a time.
    """
    cur_id = start_id
    while True:
        results = query.filter(id_field >= cur_id).order_by(id_field).\
            limit(count).all()
        if not results:
            return
        for result in results:
            yield result
        cur_id = results[-1].id + 1
<|endoftext|>


<commit_before>import time
import neovim
import pytest

from . util import joinPath

NVIM_LISTEN_ADDRESS = "/tmp/nvim-LanguageClient-IntegrationTest"
PROJECT_ROOT_PATH = joinPath("tests/sample-rs")
MAINRS_PATH = joinPath("tests/sample-rs/src/main.rs")
LIBRS_PATH = joinPath("tests/sample-rs/src/lib.rs")

@pytest.fixture(scope="module")
def nvim() -> neovim.Nvim:
    nvim = neovim.attach('socket', path=NVIM_LISTEN_ADDRESS)
    time.sleep(0.5)
    nvim.command("edit! {}".format(MAINRS_PATH))
    time.sleep(0.5)
    nvim.command("LanguageClientStart")
    time.sleep(15)
    return nvim

def test_fixture(nvim):
    pass

def test_textDocument_hover(nvim):
    nvim.command("normal! 3G23|")
    nvim.command("redir => g:echo")
    nvim.call("LanguageClient_textDocument_hover")
    time.sleep(2)
    nvim.command("redir END")
    assert nvim.eval("g:echo").strip() == "fn () -> i32"

def test_textDocument_definition(nvim):
    nvim.command("normal! 3G23|")
    nvim.call("LanguageClient_textDocument_definition")
    time.sleep(2)
    _, line, character, _ = nvim.eval("getpos('.')")
    assert [line, character] == [8, 4]

def test_textDocument_rename(nvim):
    bufferContent = str.join("\n", nvim.eval("getline(1, '$')"))
    nvim.command("normal! 3G23|")
    nvim.call("LanguageClient_textDocument_rename", {"newName": "hello"})
    time.sleep(2)
    updatedBufferContent = str.join("\n", nvim.eval("getline(1, '$')"))
    assert updatedBufferContent == bufferContent.replace("greet", "hello")
    nvim.command("edit! {}".format(MAINRS_PATH))

def test_textDocument_rename_multiple_files(nvim):
    bufferContent = str.join("\n", nvim.eval("getline(1, '$')"))
    nvim.command("normal! 17G6|")
    nvim.call("LanguageClient_textDocument_rename", {"newName": "hello"})
    time.sleep(2)
    updatedBufferContent = str.join("\n", nvim.eval("getline(1, '$')"))
    assert updatedBufferContent == bufferContent.replace("yo", "hello")
    nvim.command("bd!")
    nvim.command("bd!")
    nvim.command("edit! {}".format(MAINRS_PATH))

def test_textDocument_documentSymbol(nvim):
    nvim.call("LanguageClient_textDocument_documentSymbol")

def test_textDocument_didChange(nvim):
    nvim.call("setline", 12, "fn greet_again() -> i64 { 7 }")
    nvim.call("setline", 4, "    println!(\"{}\", greet_again());")
    time.sleep(10)
    nvim.command("normal! 4G23|")
    nvim.call("LanguageClient_textDocument_definition")
    time.sleep(2)
    _, line, character, _ = nvim.call("getpos", ".")
    assert (line, character) == (12, 4)
    nvim.command("edit! {}".format(MAINRS_PATH))

def test_textDocument_didClose(nvim):
    nvim.call("LanguageClient_textDocument_didClose")

def test_exit(nvim):
    nvim.call("LanguageClient_exit")
<commit_msg>Test fixture.<commit_after>import time
import neovim
import pytest

from . util import joinPath

NVIM_LISTEN_ADDRESS = "/tmp/nvim-LanguageClient-IntegrationTest"
PROJECT_ROOT_PATH = joinPath("tests/sample-rs")
MAINRS_PATH = joinPath("tests/sample-rs/src/main.rs")
LIBRS_PATH = joinPath("tests/sample-rs/src/lib.rs")

@pytest.fixture(scope="module")
def nvim() -> neovim.Nvim:
    nvim = neovim.attach('socket', path=NVIM_LISTEN_ADDRESS)
    time.sleep(0.5)
    nvim.command("edit! {}".format(MAINRS_PATH))
    time.sleep(0.5)
    nvim.command("LanguageClientStart")
    time.sleep(15)
    assert nvim.call("LanguageClient_isAlive") == True
    return nvim

def test_fixture(nvim):
    pass

def test_textDocument_hover(nvim):
    nvim.command("normal! 3G23|")
    nvim.command("redir => g:echo")
    nvim.call("LanguageClient_textDocument_hover")
    time.sleep(2)
    nvim.command("redir END")
    assert nvim.eval("g:echo").strip() == "fn () -> i32"

def test_textDocument_definition(nvim):
    nvim.command("normal! 3G23|")
    nvim.call("LanguageClient_textDocument_definition")
    time.sleep(2)
    _, line, character, _ = nvim.eval("getpos('.')")
    assert [line, character] == [8, 4]

def test_textDocument_rename(nvim):
    bufferContent = str.join("\n", nvim.eval("getline(1, '$')"))
    nvim.command("normal! 3G23|")
    nvim.call("LanguageClient_textDocument_rename", {"newName": "hello"})
    time.sleep(2)
    updatedBufferContent = str.join("\n", nvim.eval("getline(1, '$')"))
    assert updatedBufferContent == bufferContent.replace("greet", "hello")
    nvim.command("edit! {}".format(MAINRS_PATH))

def test_textDocument_rename_multiple_files(nvim):
    bufferContent = str.join("\n", nvim.eval("getline(1, '$')"))
    nvim.command("normal! 17G6|")
    nvim.call("LanguageClient_textDocument_rename", {"newName": "hello"})
    time.sleep(2)
    updatedBufferContent = str.join("\n", nvim.eval("getline(1, '$')"))
    assert updatedBufferContent == bufferContent.replace("yo", "hello")
    nvim.command("bd!")
    nvim.command("bd!")
    nvim.command("edit! {}".format(MAINRS_PATH))

def test_textDocument_documentSymbol(nvim):
    nvim.call("LanguageClient_textDocument_documentSymbol")

def test_textDocument_didChange(nvim):
    nvim.call("setline", 12, "fn greet_again() -> i64 { 7 }")
    nvim.call("setline", 4, "    println!(\"{}\", greet_again());")
    time.sleep(10)
    nvim.command("normal! 4G23|")
    nvim.call("LanguageClient_textDocument_definition")
    time.sleep(2)
    _, line, character, _ = nvim.call("getpos", ".")
    assert (line, character) == (12, 4)
    nvim.command("edit! {}".format(MAINRS_PATH))

def test_textDocument_didClose(nvim):
    nvim.call("LanguageClient_textDocument_didClose")

def test_exit(nvim):
    nvim.call("LanguageClient_exit")
<|endoftext|>


<commit_before>#!/usr/bin/env python

import yaml
import math
import numpy as np

import cfsim.cffirmware as firm

# main class of simulation.
# crazyflies keep reference to this object to ask what time it is.
# also does the plotting.
#
class TimeHelper:
    def __init__(self, vis, dt, writecsv):
        if vis == "mpl":
            import visualizer.visMatplotlib
            self.visualizer = visualizer.visMatplotlib.VisMatplotlib()
        elif vis == "vispy":
            import visualizer.visVispy
            self.visualizer = visualizer.visVispy.VisVispy()
        else:
            raise Exception("Unknown visualization backend: {}".format(vis))
        self.t = 0.0
        self.dt = dt
        self.crazyflies = []
        if writecsv:
            import output
            self.output = output.Output()
        else:
            self.output = None

    def time(self):
        return self.t

    def step(self, duration):
        self.t += duration

    # should be called "animate" or something
    # but called "sleep" for source-compatibility with real-robot scripts
    def sleep(self, duration):
        for t in np.arange(self.t, self.t + duration, self.dt):
            self.visualizer.update(t, self.crazyflies)
            if self.output:
                self.output.update(t, self.crazyflies)
            self.step(self.dt)

    def nextPhase(self):
        if self.output:
            self.output.nextPhase()

    def addObserver(self, observer):
        self.observers.append(observer)

# helper func to convert python/numpy arrays to firmware 3d vector type.
def arr2vec(a):
    return firm.mkvec(a[0], a[1], a[2])

class Crazyflie:

    def __init__(self, id, initialPosition, timeHelper):
        self.id = id
        self.initialPosition = np.array(initialPosition)
        self.time = lambda: timeHelper.time()

        self.planner = firm.planner()
        self.cmdHighLevel = True
        firm.plan_init(self.planner)
        self.planner.lastKnownPosition = arr2vec(initialPosition)
        self.groupMask = 0
        self.trajectories = dict()

        # for visualization - default to blueish-grey
        self.ledRGB = (0.5, 0.5, 1)

    def setGroupMask(self, groupMask):
        self.groupMask = groupMask

    def takeoff(self, targetHeight, duration, groupMask = 0):
        if self._isGroup(groupMask):
            firm.plan_takeoff(self.planner,
                self._vposition(), self.yaw(), targetHeight, duration, self.time())

    def land(self, targetHeight, duration, groupMask = 0):
        if self._isGroup(groupMask):
            firm.plan_land(self.planner,
                self._vposition(), self.yaw(), targetHeight, duration, self.time())

    def stop(self, groupMask = 0):
        if self._isGroup(groupMask):
            firm.plan_stop(self.planner)

    def goTo(self, goal, yaw, duration, relative = False, groupMask = 0):
        if self._isGroup(groupMask):
            firm.plan_go_to(self.planner, relative, arr2vec(goal), yaw, duration, self.time())

    def uploadTrajectory(self, trajectoryId, pieceOffset, trajectory):
        traj = firm.piecewise_traj()
        traj.t_begin = 0
        traj.timescale = 1.0
        traj.shift = firm.mkvec(0, 0, 0)
        traj.n_pieces = len(trajectory.polynomials)
        traj.pieces = firm.malloc_poly4d(len(trajectory.polynomials))
        for i, poly in enumerate(trajectory.polynomials):
            piece = firm.pp_get_piece(traj, i)
            piece.duration = poly.duration
            for coef in range(0, 8):
                firm.poly4d_set(piece, 0, coef, poly.px.p[coef])
                firm.poly4d_set(piece, 1, coef, poly.py.p[coef])
                firm.poly4d_set(piece, 2, coef, poly.pz.p[coef])
                firm.poly4d_set(piece, 3, coef, poly.pyaw.p[coef])
        self.trajectories[trajectoryId] = traj

    def startTrajectory(self, trajectoryId, timescale = 1.0, reverse = False, relative = True, groupMask = 0):
        if self._isGroup(groupMask):
            traj = self.trajectories[trajectoryId]
            # if traj is also current traj, must take pos before changing t_begin.
            pos = self._vposition()
            traj.t_begin = self.time()
            traj.timescale = timescale
            if relative:
                traj.shift = firm.vzero()
                if reverse:
                    traj_init = firm.piecewise_eval_reversed(traj, traj.t_begin)
                else:
                    traj_init = firm.piecewise_eval(traj, traj.t_begin)
                traj.shift = firm.vsub(pos, traj_init.pos)
            else:
                traj.shift = firm.vzero()
            firm.plan_start_trajectory(self.planner, traj, reverse)

    def position(self):
        pos = self._vposition()
        return np.array([pos.x, pos.y, pos.z])

    def getParam(self, name):
        print("WARNING: getParam not implemented in simulation!")

    def setParam(self, name, value):
        print("WARNING: setParam not implemented in simulation!")

    def setParams(self, params):
        print("WARNING: setParams not implemented in simulation!")

    # - this is a part of the param system on the real crazyflie,
    #   but we implement it in simulation too for debugging
    # - is a blocking command on real CFs, so may cause stability problems
    def setLEDColor(self, r, g, b):
        self.ledRGB = (r, g, b)

    # simulation only functions
    def yaw(self):
        ev = firm.plan_current_goal(self.planner, self.time())
        if not firm.is_traj_eval_valid(ev):
            return 0.0
        return ev.yaw

    def acceleration(self):
        if self.planner.state == firm.TRAJECTORY_STATE_IDLE:
            return np.array([0, 0, 0])
        else:
            ev = firm.plan_current_goal(self.planner, self.time())
            return np.array([ev.acc.x, ev.acc.y, ev.acc.z])

    def rpy(self):
        acc = self.acceleration()
        yaw = self.yaw()
        norm = np.linalg.norm(acc)
        # print(acc)
        if norm < 1e-6:
            return (0.0, 0.0, yaw)
        else:
            thrust = acc + np.array([0, 0, 9.81])
            z_body = thrust / np.linalg.norm(thrust)
            x_world = np.array([math.cos(yaw), math.sin(yaw), 0])
            y_body = np.cross(z_body, x_world)
            x_body = np.cross(y_body, z_body)
            pitch = math.asin(-x_body[2])
            roll = math.atan2(y_body[2], z_body[2])
            return (roll, pitch, yaw)

    def cmdFullState(self, pos, vel, acc, yaw, omega):
        self.planner.lastKnownPosition = pos
        self.cmdHighLevel = False
        # TODO store other state variables

    # "private" methods
    def _isGroup(self, groupMask):
        return groupMask == 0 or (self.groupMask & groupMask) > 0

    def _vposition(self):
        # TODO this should be implemented in C
        # print(self.id, self.planner, self.planner.state)
        if (not self.cmdHighLevel) or self.planner.state == firm.TRAJECTORY_STATE_IDLE:
            return self.planner.lastKnownPosition
        else:
            ev = firm.plan_current_goal(self.planner, self.time())
            self.planner.lastKnownPosition = firm.mkvec(ev.pos.x, ev.pos.y, ev.pos.z)
            # print(self.id, ev.pos.z)
            # not totally sure why, but if we don't do this, we don't actually return by value
            return firm.mkvec(ev.pos.x, ev.pos.y, ev.pos.z)

class CrazyflieServer:
    def __init__(self, timeHelper):
        with open("../launch/crazyflies.yaml", 'r') as ymlfile:
            cfg = yaml.load(ymlfile)

        self.crazyflies = []
        self.crazyfliesById = dict()
        for crazyflie in cfg["crazyflies"]:
            id = int(crazyflie["id"])
            initialPosition = crazyflie["initialPosition"]
            cf = Crazyflie(id, initialPosition, timeHelper)
            self.crazyflies.append(cf)
            self.crazyfliesById[id] = cf

        self.timeHelper = timeHelper
        self.timeHelper.crazyflies = self.crazyflies

    def emergency(self):
        print("WARNING: setParams not implemented in simulation!")

    def takeoff(self, targetHeight, duration, groupMask = 0):
        for crazyflie in self.crazyflies:
            crazyflie.takeoff(targetHeight, duration, groupMask)

    def land(self, targetHeight, duration, groupMask = 0):
        for crazyflie in self.crazyflies:
            crazyflie.land(targetHeight, duration, groupMask)

    def stop(self, groupMask = 0):
        for crazyflie in self.crazyflies:
            crazyflie.stop(groupMask)

    def goTo(self, goal, yaw, duration, groupMask = 0):
        for crazyflie in self.crazyflies:
            crazyflie.goTo(goal, yaw, duration, False, groupMask)

    def startTrajectory(self, trajectoryId, timescale = 1.0, reverse = False, relative = True, groupMask = 0):
        for crazyflie in self.crazyflies:
            crazyflie.startTrajectory(trajectoryId, timescale, reverse, relative, groupMask)
<commit_msg>accept not-ndarray _lastpos in cfsim<commit_after>#!/usr/bin/env python

import yaml
import math
import numpy as np

import cfsim.cffirmware as firm

# main class of simulation.
# crazyflies keep reference to this object to ask what time it is.
# also does the plotting.
#
class TimeHelper:
    def __init__(self, vis, dt, writecsv):
        if vis == "mpl":
            import visualizer.visMatplotlib
            self.visualizer = visualizer.visMatplotlib.VisMatplotlib()
        elif vis == "vispy":
            import visualizer.visVispy
            self.visualizer = visualizer.visVispy.VisVispy()
        else:
            raise Exception("Unknown visualization backend: {}".format(vis))
        self.t = 0.0
        self.dt = dt
        self.crazyflies = []
        if writecsv:
            import output
            self.output = output.Output()
        else:
            self.output = None

    def time(self):
        return self.t

    def step(self, duration):
        self.t += duration

    # should be called "animate" or something
    # but called "sleep" for source-compatibility with real-robot scripts
    def sleep(self, duration):
        for t in np.arange(self.t, self.t + duration, self.dt):
            self.visualizer.update(t, self.crazyflies)
            if self.output:
                self.output.update(t, self.crazyflies)
            self.step(self.dt)

    def nextPhase(self):
        if self.output:
            self.output.nextPhase()

    def addObserver(self, observer):
        self.observers.append(observer)

# helper func to convert python/numpy arrays to firmware 3d vector type.
def arr2vec(a):
    return firm.mkvec(a[0], a[1], a[2])

class Crazyflie:

    def __init__(self, id, initialPosition, timeHelper):
        self.id = id
        self.initialPosition = np.array(initialPosition)
        self.time = lambda: timeHelper.time()

        self.planner = firm.planner()
        self.cmdHighLevel = True
        firm.plan_init(self.planner)
        self.planner.lastKnownPosition = arr2vec(initialPosition)
        self.groupMask = 0
        self.trajectories = dict()

        # for visualization - default to blueish-grey
        self.ledRGB = (0.5, 0.5, 1)

    def setGroupMask(self, groupMask):
        self.groupMask = groupMask

    def takeoff(self, targetHeight, duration, groupMask = 0):
        if self._isGroup(groupMask):
            firm.plan_takeoff(self.planner,
                self._vposition(), self.yaw(), targetHeight, duration, self.time())

    def land(self, targetHeight, duration, groupMask = 0):
        if self._isGroup(groupMask):
            firm.plan_land(self.planner,
                self._vposition(), self.yaw(), targetHeight, duration, self.time())

    def stop(self, groupMask = 0):
        if self._isGroup(groupMask):
            firm.plan_stop(self.planner)

    def goTo(self, goal, yaw, duration, relative = False, groupMask = 0):
        if self._isGroup(groupMask):
            firm.plan_go_to(self.planner, relative, arr2vec(goal), yaw, duration, self.time())

    def uploadTrajectory(self, trajectoryId, pieceOffset, trajectory):
        traj = firm.piecewise_traj()
        traj.t_begin = 0
        traj.timescale = 1.0
        traj.shift = firm.mkvec(0, 0, 0)
        traj.n_pieces = len(trajectory.polynomials)
        traj.pieces = firm.malloc_poly4d(len(trajectory.polynomials))
        for i, poly in enumerate(trajectory.polynomials):
            piece = firm.pp_get_piece(traj, i)
            piece.duration = poly.duration
            for coef in range(0, 8):
                firm.poly4d_set(piece, 0, coef, poly.px.p[coef])
                firm.poly4d_set(piece, 1, coef, poly.py.p[coef])
                firm.poly4d_set(piece, 2, coef, poly.pz.p[coef])
                firm.poly4d_set(piece, 3, coef, poly.pyaw.p[coef])
        self.trajectories[trajectoryId] = traj

    def startTrajectory(self, trajectoryId, timescale = 1.0, reverse = False, relative = True, groupMask = 0):
        if self._isGroup(groupMask):
            traj = self.trajectories[trajectoryId]
            # if traj is also current traj, must take pos before changing t_begin.
            pos = self._vposition()
            traj.t_begin = self.time()
            traj.timescale = timescale
            if relative:
                traj.shift = firm.vzero()
                if reverse:
                    traj_init = firm.piecewise_eval_reversed(traj, traj.t_begin)
                else:
                    traj_init = firm.piecewise_eval(traj, traj.t_begin)
                traj.shift = firm.vsub(pos, traj_init.pos)
            else:
                traj.shift = firm.vzero()
            firm.plan_start_trajectory(self.planner, traj, reverse)

    def position(self):
        pos = self._vposition()
        if not type(pos) is np.ndarray:
            return np.array([pos.x, pos.y, pos.z])
        return pos

    def getParam(self, name):
        print("WARNING: getParam not implemented in simulation!")

    def setParam(self, name, value):
        print("WARNING: setParam not implemented in simulation!")

    def setParams(self, params):
        print("WARNING: setParams not implemented in simulation!")

    # - this is a part of the param system on the real crazyflie,
    #   but we implement it in simulation too for debugging
    # - is a blocking command on real CFs, so may cause stability problems
    def setLEDColor(self, r, g, b):
        self.ledRGB = (r, g, b)

    # simulation only functions
    def yaw(self):
        ev = firm.plan_current_goal(self.planner, self.time())
        if not firm.is_traj_eval_valid(ev):
            return 0.0
        return ev.yaw

    def acceleration(self):
        if self.planner.state == firm.TRAJECTORY_STATE_IDLE:
            return np.array([0, 0, 0])
        else:
            ev = firm.plan_current_goal(self.planner, self.time())
            return np.array([ev.acc.x, ev.acc.y, ev.acc.z])

    def rpy(self):
        acc = self.acceleration()
        yaw = self.yaw()
        norm = np.linalg.norm(acc)
        # print(acc)
        if norm < 1e-6:
            return (0.0, 0.0, yaw)
        else:
            thrust = acc + np.array([0, 0, 9.81])
            z_body = thrust / np.linalg.norm(thrust)
            x_world = np.array([math.cos(yaw), math.sin(yaw), 0])
            y_body = np.cross(z_body, x_world)
            x_body = np.cross(y_body, z_body)
            pitch = math.asin(-x_body[2])
            roll = math.atan2(y_body[2], z_body[2])
            return (roll, pitch, yaw)

    def cmdFullState(self, pos, vel, acc, yaw, omega):
        self.planner.lastKnownPosition = pos
        self.cmdHighLevel = False
        # TODO store other state variables

    # "private" methods
    def _isGroup(self, groupMask):
        return groupMask == 0 or (self.groupMask & groupMask) > 0

    def _vposition(self):
        # TODO this should be implemented in C
        # print(self.id, self.planner, self.planner.state)
        if (not self.cmdHighLevel) or self.planner.state == firm.TRAJECTORY_STATE_IDLE:
            return self.planner.lastKnownPosition
        else:
            ev = firm.plan_current_goal(self.planner, self.time())
            self.planner.lastKnownPosition = firm.mkvec(ev.pos.x, ev.pos.y, ev.pos.z)
            # print(self.id, ev.pos.z)
            # not totally sure why, but if we don't do this, we don't actually return by value
            return firm.mkvec(ev.pos.x, ev.pos.y, ev.pos.z)

class CrazyflieServer:
    def __init__(self, timeHelper):
        with open("../launch/crazyflies.yaml", 'r') as ymlfile:
            cfg = yaml.load(ymlfile)

        self.crazyflies = []
        self.crazyfliesById = dict()
        for crazyflie in cfg["crazyflies"]:
            id = int(crazyflie["id"])
            initialPosition = crazyflie["initialPosition"]
            cf = Crazyflie(id, initialPosition, timeHelper)
            self.crazyflies.append(cf)
            self.crazyfliesById[id] = cf

        self.timeHelper = timeHelper
        self.timeHelper.crazyflies = self.crazyflies

    def emergency(self):
        print("WARNING: setParams not implemented in simulation!")

    def takeoff(self, targetHeight, duration, groupMask = 0):
        for crazyflie in self.crazyflies:
            crazyflie.takeoff(targetHeight, duration, groupMask)

    def land(self, targetHeight, duration, groupMask = 0):
        for crazyflie in self.crazyflies:
            crazyflie.land(targetHeight, duration, groupMask)

    def stop(self, groupMask = 0):
        for crazyflie in self.crazyflies:
            crazyflie.stop(groupMask)

    def goTo(self, goal, yaw, duration, groupMask = 0):
        for crazyflie in self.crazyflies:
            crazyflie.goTo(goal, yaw, duration, False, groupMask)

    def startTrajectory(self, trajectoryId, timescale = 1.0, reverse = False, relative = True, groupMask = 0):
        for crazyflie in self.crazyflies:
            crazyflie.startTrajectory(trajectoryId, timescale, reverse, relative, groupMask)
<|endoftext|>