<commit_before>#! /usr/bin/env python

import random
import networkx as nx
import matplotlib.pyplot as plt
import re
import logging

import mann.agent
import mann.agent_binary
import mann.agent_lens_recurrent

class NetworkAgent(object):
    def __init__(self):
        pass

    def __eq__(self, x, y):
        return x.agent_id == y.agent_id

    def create_multidigraph_of_agents_from_edge_list(
            self, number_of_agents, edge_list, fig_path,
            agent_type=tuple(['None']), logger=None, **kwargs):
        """Create multi directed networkx graph of agents from an edge list

        :param num_of_agents: number of agents in the network
        :type num_of_agents: int

        :param edge_list: edge list of network
        :type edge_list: iterable

        :param fig_path: figure path of output network image
        :type fig_path: str

        :param agent_type: type of agent, binary or lens
        :type agent_type: tuple

        :param logger: logger object, not used
        :type logger: None

        :param **kwargs: kwargs used for lens agent creation
        :type **kwargs: dict
        """
        logging.info('In mann.network_agent.'
                     'create_multidigraph_of_agents_from_edge_list()')

        # create the graph
        self.G = nx.MultiDiGraph()

        # dictonary container for agents, key values will be the agent.get_key
        all_agents = {}

        for i in range(number_of_agents):
            logging.info("creating agent # {}".format(i))

            # createing the different types of agents for the network
            if agent_type[0] == 'binary':
                new_agent = mann.agent_binary.BinaryAgent(agent_type[1],
                                                          agent_type[2])
            elif agent_type[0] == 'lens':
                if agent_type[2] == 'feed_forward_global_cascade':
                    new_agent = mann.agent.LensAgent(agent_type[1])
                    new_agent.create_weight_file(
                        kwargs.get('weight_in_file'),
                        kwargs.get('weight_dir'),
                        kwargs.get('base_example'),
                        kwargs.get('num_train_examples'),
                        kwargs.get('prototype_mutation_prob'),
                        kwargs.get('training_criterion'))
                elif agent_type[2] == 'recurrent_attitude':
                    # nothing really happens after the agent gets created
                    # this is more of a place holder for later training
                    # procedures
                    new_agent = mann.agent_lens_recurrent.LensAgentRecurrent(
                        agent_type[1])
                else:
                    raise ValueError('Unknown Lens Agent Type')
            else:
                raise mann.agent.UnknownAgentTypeError(
                    'Unknown agent specified as nodes for network')

            logging.info("agent {} created: type: {}".
                         format(new_agent.get_key(), type(new_agent)))
            logging.info("agent {} state: {}".
                         format(new_agent.get_key(), str(new_agent.state)))

            try:
                logging.info("agent {} threshold: {}".
                             format(new_agent.get_key(),
                                    str(new_agent.threshold)))
            except AttributeError:  # threshold is not a parameter for agent
                pass

            all_agents[new_agent.agent_id] = new_agent

        logging.info('total number of agents created: {}'.
                     format(new_agent.agent_count))

        self.G.add_nodes_from(all_agents.values())
        logging.info('number of nodes created: {}'.format(len(self.G)))

        logging.info('Creating edges')
        for edge in edge_list:
            u, v = edge
            self.G.add_edge(all_agents[u], all_agents[v])

        logging.info('Saving plot of mann copied graph')
        nx.draw_circular(self.G)
        # plt.show()
        plt.savefig(fig_path)

        return self.G

    def export_edge_list(self, export_file_dir, **kwargs):
        nx.write_edgelist(self.G, export_file_dir, **kwargs)

    def set_predecessors_for_each_node(self):
        # iterate through all nodes in network
        for node_agent in self.G.nodes_iter():
            # look up the predessors for each node
            predecessors = self.G.predecessors(node_agent)
            # since the nodes are an Agent class we can
            # assign the predecessors agent instance variable to the iter
            node_agent.set_predecessors(predecessors)
            logging.info('Agent {} predecessors assigned'.
                         format(node_agent.agent_id))
            print(node_agent.predecessors)

    def sample_network(self, number_of_agents_to_sample):
        '''
        From the random.sample documentation:
        Return a k length list of unique elements
        chosen from the population sequence or set.
        Used for random sampling without replacement.
        '''
        agents_picked = random.sample(self.G.nodes(),
                                      number_of_agents_to_sample)
        return agents_picked

    def str_list_with_out_brackets(self, list_to_str):
        # reg ex str replace multiple
        # http://stackoverflow.com/questions/6116978/python-replace-multiple-strings
        # dict.iteritems() is a python 2 syntax
        # python 3 has dict.itemd()
        rep = {"[": "", "]": "", "(": "", ")": ""}
        rep = dict((re.escape(k), v) for k, v in rep.items())
        pattern = re.compile("|".join(rep.keys()))
        text = pattern.sub(lambda m: rep[re.escape(m.group(0))],
                           str(list_to_str))
        return text

    def update_simultaneous(self, num_agents_update, update_algorithm,
                            update_type='simultaneous'):
        assert isinstance(num_agents_update, int)
        agents_for_update = self.sample_network(num_agents_update)

        logging.info('Num agents for update: {}'.
                     format(len(agents_for_update)))

        # assign new temp value
        for selected_agent in agents_for_update:
            logging.info('Updating: {}'.
                         format(self.G.nodes()[selected_agent.agent_id]))
            assert selected_agent.temp_new_state is None
            selected_agent.update_agent_state(update_type,
                                              update_algorithm)

        # assign new temp value as final state simultaneous update
        logging.info('Performing simultaneous update')
        for selected_agent in agents_for_update:
            if selected_agent.temp_new_state == 1:
                logging.info('Update agent {}: setting state to {}, was {}'.
                             format(selected_agent.agent_id,
                                    selected_agent.temp_new_state,
                                    selected_agent.state))
                assert selected_agent.temp_new_state is not None
                selected_agent.state = selected_agent.temp_new_state
                selected_agent.num_update += 1
            else:
                logging.info('Agent {}: No state change to {}, was {}'.
                             format(selected_agent.agent_id,
                                    selected_agent.temp_new_state,
                                    selected_agent.state))
            selected_agent.temp_new_state = None

    def update_sequential(self, num_agents_update, update_algorithm,
                          update_type='sequential'):
        assert isinstance(num_agents_update, int)
        agents_for_update = self.sample_network(num_agents_update)

        logging.info('Num agents for update: {}'.
                     format(len(agents_for_update)))

        # assign new temp value
        for selected_agent in agents_for_update:
            logging.info('Updating: {}'.
                         format(self.G.nodes()[selected_agent.agent_id]))
            selected_agent.update_agent_state(update_type, update_algorithm)

    def write_network_agent_step_info(self, time_step, file_to_write,
                                      file_mode, agent_type, **kwargs):
        """Write agent info for each time step

        Writes the following information respectively for binary agent
        - time step
        - agent id
        - total number of updates
        - update state for this time step
        - agent state at end of time

        Writes the following information respectively for lens agent
        - time step
        - agent id
        - total number of updates
        - update state for this time step
        - infl agent ID
        - agent state at end of time
        - input agent state
        - lens target
        - prototype
        """
        with open(file_to_write, mode=file_mode, encoding='utf-8') as f:
            for node in self.G.__iter__():
                if agent_type == 'binary':
                    f.write(",".join([
                        str(time_step),  # time step
                        str(node.agent_id),  # agent ID
                        str(node.num_update),  # total num updates
                        str(node.step_update_status),  # update state
                        # str(node.step_input_agent_id),  # infl ID
                        # agent state
                        self.str_list_with_out_brackets(node.state)
                        # input state
                        # self.str_list_with_out_brackets(
                        #     node.step_input_state_values),
                    ]) + "\n")

                elif agent_type == 'lens':
                    f.write(",".join([
                        str(time_step),  # time step
                        str(node.agent_id),  # agent ID
                        str(node.num_update),  # total num updates
                        str(node.step_update_status),  # update state
                        str(node.step_input_agent_id),  # infl ID
                        # agent state
                        self.str_list_with_out_brackets(node.state),
                        # input state
                        self.str_list_with_out_brackets(
                            node.step_input_state_values),
                        # lens target
                        self.str_list_with_out_brackets(node.step_lens_target),
                        # prototype
                        self.str_list_with_out_brackets(node.prototype)
                    ]) + "\n")
                node.reset_step_variables()
<commit_msg>write_network_agent_step_info lens_agent_type kwarg to write different lines<commit_after>#! /usr/bin/env python

import random
import networkx as nx
import matplotlib.pyplot as plt
import re
import logging

import mann.agent
import mann.agent_binary
import mann.agent_lens_recurrent

class NetworkAgent(object):
    def __init__(self):
        pass

    def __eq__(self, x, y):
        return x.agent_id == y.agent_id

    def create_multidigraph_of_agents_from_edge_list(
            self, number_of_agents, edge_list, fig_path,
            agent_type=tuple(['None']), logger=None, **kwargs):
        """Create multi directed networkx graph of agents from an edge list

        :param num_of_agents: number of agents in the network
        :type num_of_agents: int

        :param edge_list: edge list of network
        :type edge_list: iterable

        :param fig_path: figure path of output network image
        :type fig_path: str

        :param agent_type: type of agent, binary or lens
        :type agent_type: tuple

        :param logger: logger object, not used
        :type logger: None

        :param **kwargs: kwargs used for lens agent creation
        :type **kwargs: dict
        """
        logging.info('In mann.network_agent.'
                     'create_multidigraph_of_agents_from_edge_list()')

        # create the graph
        self.G = nx.MultiDiGraph()

        # dictonary container for agents, key values will be the agent.get_key
        all_agents = {}

        for i in range(number_of_agents):
            logging.info("creating agent # {}".format(i))

            # createing the different types of agents for the network
            if agent_type[0] == 'binary':
                new_agent = mann.agent_binary.BinaryAgent(agent_type[1],
                                                          agent_type[2])
            elif agent_type[0] == 'lens':
                if agent_type[2] == 'feed_forward_global_cascade':
                    new_agent = mann.agent.LensAgent(agent_type[1])
                    new_agent.create_weight_file(
                        kwargs.get('weight_in_file'),
                        kwargs.get('weight_dir'),
                        kwargs.get('base_example'),
                        kwargs.get('num_train_examples'),
                        kwargs.get('prototype_mutation_prob'),
                        kwargs.get('training_criterion'))
                elif agent_type[2] == 'recurrent_attitude':
                    # nothing really happens after the agent gets created
                    # this is more of a place holder for later training
                    # procedures
                    new_agent = mann.agent_lens_recurrent.LensAgentRecurrent(
                        agent_type[1])
                else:
                    raise ValueError('Unknown Lens Agent Type')
            else:
                raise mann.agent.UnknownAgentTypeError(
                    'Unknown agent specified as nodes for network')

            logging.info("agent {} created: type: {}".
                         format(new_agent.get_key(), type(new_agent)))
            logging.info("agent {} state: {}".
                         format(new_agent.get_key(), str(new_agent.state)))

            try:
                logging.info("agent {} threshold: {}".
                             format(new_agent.get_key(),
                                    str(new_agent.threshold)))
            except AttributeError:  # threshold is not a parameter for agent
                pass

            all_agents[new_agent.agent_id] = new_agent

        logging.info('total number of agents created: {}'.
                     format(new_agent.agent_count))

        self.G.add_nodes_from(all_agents.values())
        logging.info('number of nodes created: {}'.format(len(self.G)))

        logging.info('Creating edges')
        for edge in edge_list:
            u, v = edge
            self.G.add_edge(all_agents[u], all_agents[v])

        logging.info('Saving plot of mann copied graph')
        nx.draw_circular(self.G)
        # plt.show()
        plt.savefig(fig_path)

        return self.G

    def export_edge_list(self, export_file_dir, **kwargs):
        nx.write_edgelist(self.G, export_file_dir, **kwargs)

    def set_predecessors_for_each_node(self):
        # iterate through all nodes in network
        for node_agent in self.G.nodes_iter():
            # look up the predessors for each node
            predecessors = self.G.predecessors(node_agent)
            # since the nodes are an Agent class we can
            # assign the predecessors agent instance variable to the iter
            node_agent.set_predecessors(predecessors)
            logging.info('Agent {} predecessors assigned'.
                         format(node_agent.agent_id))
            print(node_agent.predecessors)

    def sample_network(self, number_of_agents_to_sample):
        '''
        From the random.sample documentation:
        Return a k length list of unique elements
        chosen from the population sequence or set.
        Used for random sampling without replacement.
        '''
        agents_picked = random.sample(self.G.nodes(),
                                      number_of_agents_to_sample)
        return agents_picked

    def str_list_with_out_brackets(self, list_to_str):
        # reg ex str replace multiple
        # http://stackoverflow.com/questions/6116978/python-replace-multiple-strings
        # dict.iteritems() is a python 2 syntax
        # python 3 has dict.itemd()
        rep = {"[": "", "]": "", "(": "", ")": ""}
        rep = dict((re.escape(k), v) for k, v in rep.items())
        pattern = re.compile("|".join(rep.keys()))
        text = pattern.sub(lambda m: rep[re.escape(m.group(0))],
                           str(list_to_str))
        return text

    def update_simultaneous(self, num_agents_update, update_algorithm,
                            update_type='simultaneous'):
        assert isinstance(num_agents_update, int)
        agents_for_update = self.sample_network(num_agents_update)

        logging.info('Num agents for update: {}'.
                     format(len(agents_for_update)))

        # assign new temp value
        for selected_agent in agents_for_update:
            logging.info('Updating: {}'.
                         format(self.G.nodes()[selected_agent.agent_id]))
            assert selected_agent.temp_new_state is None
            selected_agent.update_agent_state(update_type,
                                              update_algorithm)

        # assign new temp value as final state simultaneous update
        logging.info('Performing simultaneous update')
        for selected_agent in agents_for_update:
            if selected_agent.temp_new_state == 1:
                logging.info('Update agent {}: setting state to {}, was {}'.
                             format(selected_agent.agent_id,
                                    selected_agent.temp_new_state,
                                    selected_agent.state))
                assert selected_agent.temp_new_state is not None
                selected_agent.state = selected_agent.temp_new_state
                selected_agent.num_update += 1
            else:
                logging.info('Agent {}: No state change to {}, was {}'.
                             format(selected_agent.agent_id,
                                    selected_agent.temp_new_state,
                                    selected_agent.state))
            selected_agent.temp_new_state = None

    def update_sequential(self, num_agents_update, update_algorithm,
                          update_type='sequential'):
        assert isinstance(num_agents_update, int)
        agents_for_update = self.sample_network(num_agents_update)

        logging.info('Num agents for update: {}'.
                     format(len(agents_for_update)))

        # assign new temp value
        for selected_agent in agents_for_update:
            logging.info('Updating: {}'.
                         format(self.G.nodes()[selected_agent.agent_id]))
            selected_agent.update_agent_state(update_type, update_algorithm)

    def write_network_agent_step_info(self, time_step, file_to_write,
                                      file_mode, agent_type, **kwargs):
        """Write agent info for each time step

        Writes the following information respectively for binary agent
        - time step
        - agent id
        - total number of updates
        - update state for this time step
        - agent state at end of time

        Writes the following information respectively for lens agent
        - time step
        - agent id
        - total number of updates
        - update state for this time step
        - infl agent ID
        - agent state at end of time
        - input agent state
        - lens target
        - prototype
        """
        with open(file_to_write, mode=file_mode, encoding='utf-8') as f:
            for node in self.G.__iter__():
                if agent_type == 'binary':
                    f.write(",".join([
                        str(time_step),  # time step
                        str(node.agent_id),  # agent ID
                        str(node.num_update),  # total num updates
                        str(node.step_update_status),  # update state
                        # str(node.step_input_agent_id),  # infl ID
                        # agent state
                        self.str_list_with_out_brackets(node.state)
                        # input state
                        # self.str_list_with_out_brackets(
                        #     node.step_input_state_values),
                    ]) + "\n")

                elif agent_type == 'lens' and \
                    kwargs['lens_agent_type'] == \
                        'recurrent_attitude':
                    f.write(",".join([
                        str(time_step),  # time step
                        str(node.agent_id),  # agent ID
                        str(node.num_update),  # total num updates
                        # str(node.step_update_status),  # update state
                        # str(node.step_input_agent_id),  # infl ID
                        # agent state
                        self.str_list_with_out_brackets(node.state)  # ,
                        # input state
                        # self.str_list_with_out_brackets(
                        #     node.step_input_state_values),
                        # lens target
                        # self.str_list_with_out_brackets(node.step_lens_target),
                        # prototype
                        # self.str_list_with_out_brackets(node.prototype)
                    ]) + "\n")

                elif agent_type == 'lens' and \
                    kwargs['lens_agent_type'] == \
                        'feed_forward_global_cascade':
                    f.write(",".join([
                        str(time_step),  # time step
                        str(node.agent_id),  # agent ID
                        str(node.num_update),  # total num updates
                        str(node.step_update_status),  # update state
                        str(node.step_input_agent_id),  # infl ID
                        # agent state
                        self.str_list_with_out_brackets(node.state),
                        # input state
                        self.str_list_with_out_brackets(
                            node.step_input_state_values),
                        # lens target
                        self.str_list_with_out_brackets(node.step_lens_target),
                        # prototype
                        self.str_list_with_out_brackets(node.prototype)
                    ]) + "\n")
                node.reset_step_variables()
<|endoftext|>


<commit_before>import logging
from flask import request
from banal import ensure_list
from normality import stringify
from followthemoney import model

from aleph.core import url_for
from aleph.model import Role, Collection, Document, Entity, Events, Alert
from aleph.logic import resolver
from aleph.logic.util import collection_url, entity_url
from aleph.views.util import jsonify

log = logging.getLogger(__name__)

class Serializer(object):

    def __init__(self, reference=False):
        self.reference = reference

    def _collect(self, obj):
        pass

    def _serialize(self, obj):
        return obj

    def queue(self, clazz, key):
        if not self.reference:
            resolver.queue(request, clazz, key)

    def resolve(self, clazz, key, serializer=None):
        if self.reference:
            return
        data = resolver.get(request, clazz, key)
        if data is not None and serializer is not None:
            serializer = serializer(reference=True)
            data = serializer.serialize(data)
        return data

    def serialize(self, obj):
        obj = self._to_dict(obj)
        if obj is not None:
            self._collect(obj)
            resolver.resolve(request)
            return self._serialize(obj)

    def serialize_many(self, objs):
        collected = []
        for obj in ensure_list(objs):
            obj = self._to_dict(obj)
            if obj is not None:
                self._collect(obj)
                collected.append(obj)
        resolver.resolve(request)
        serialized = []
        for obj in collected:
            obj = self._serialize(obj)
            if obj is not None:
                serialized.append(obj)
        return serialized

    def _to_dict(self, obj):
        if hasattr(obj, 'to_dict'):
            obj = obj.to_dict()
        if hasattr(obj, '_asdict'):
            obj = obj._asdict()
        return obj

    @classmethod
    def jsonify(cls, obj, **kwargs):
        data = cls().serialize(obj)
        return jsonify(data, **kwargs)

    @classmethod
    def jsonify_result(cls, result, **kwargs):
        data = result.to_dict(serializer=cls)
        return jsonify(data, **kwargs)

class RoleSerializer(Serializer):

    def _serialize(self, obj):
        obj['links'] = {
            'self': url_for('roles_api.view', id=obj.get('id'))
        }
        obj['writeable'] = obj.get('id') == stringify(request.authz.id)
        if not obj['writeable']:
            obj.pop('has_password', None)
            obj.pop('api_key', None)
            obj.pop('email', None)
        if obj['type'] != Role.USER:
            obj.pop('api_key', None)
            obj.pop('email', None)
        obj.pop('password', None)
        return obj

class AlertSerializer(Serializer):

    # def _collect(self, obj):
    #     self.queue(Role, obj.get('role_id'))

    def _serialize(self, obj):
        pk = obj.get('id')
        obj['links'] = {
            'self': url_for('alerts_api.view', id=pk)
        }
        role_id = obj.pop('role_id', None)
        obj['writeable'] = role_id == stringify(request.authz.id)
        # obj['role'] = self.resolve(Role, role_id, RoleSerializer)
        return obj

class CollectionSerializer(Serializer):

    def _collect(self, obj):
        self.queue(Role, obj.get('creator_id'))

    def _serialize(self, obj):
        pk = obj.get('id')
        obj['links'] = {
            'self': url_for('collections_api.view', id=pk),
            'xref': url_for('xref_api.index', id=pk),
            'xref_csv': url_for('xref_api.csv_export', id=pk, _authorize=True),
            'ui': collection_url(pk)
        }
        obj['writeable'] = request.authz.can(pk, request.authz.WRITE)
        creator_id = obj.pop('creator_id', None)
        obj['creator'] = self.resolve(Role, creator_id, RoleSerializer)
        obj.pop('_index', None)
        return obj

class PermissionSerializer(Serializer):

    def _collect(self, obj):
        self.queue(Role, obj.get('role_id'))

    def _serialize(self, obj):
        obj.pop('collection_id', None)
        role_id = obj.pop('role_id', None)
        obj['role'] = self.resolve(Role, role_id, RoleSerializer)
        return obj

class EntitySerializer(Serializer):

    def _collect(self, obj):
        self.queue(Collection, obj.get('collection_id'))
        self.queue(Role, obj.get('uploader_id'))
        schema = model.get(obj.get('schema'))
        if schema is None:
            return
        # TODO associated entities

    def _serialize(self, obj):
        pk = obj.get('id')
        authz = request.authz
        collection_id = obj.pop('collection_id', None)
        obj['collection'] = self.resolve(Collection, collection_id,
                                         CollectionSerializer)
        schema = model.get(obj.get('schema'))
        if schema is None:
            return
        obj['schemata'] = schema.names
        links = {
            'self': url_for('entities_api.view', id=pk),
            'references': url_for('entities_api.references', id=pk),
            'tags': url_for('entities_api.tags', id=pk),
            'ui': entity_url(pk)
        }
        if schema.is_a(Document.SCHEMA):
            links['content'] = url_for('documents_api.content', document_id=pk)
        # if data.get('content_hash'):
        #     links['file'] = url_for('documents_api.file',
        #                             document_id=pk,
        #                             _authorize=True)
        # if schemata.intersection([Document.SCHEMA_PDF]):
        #     links['pdf'] = url_for('documents_api.pdf',
        #                            document_id=pk,
        #                            _authorize=True)

        obj['links'] = links
        obj['writeable'] = authz.can(collection_id, authz.WRITE)
        if obj.get('bulk'):
            obj['writeable'] = False
        obj.pop('_index', None)
        return obj

class MatchCollectionsSerializer(Serializer):

    def _serialize(self, obj):
        serializer = CollectionSerializer(reference=True)
        obj['collection'] = serializer.serialize(obj.get('collection'))
        return obj

class MatchSerializer(Serializer):

    def _collect(self, obj):
        self.queue(Entity, obj.get('entity_id'))
        self.queue(Entity, obj.get('match_id'))

    def _serialize(self, obj):
        entity_id = obj.pop('entity_id', None)
        obj['entity'] = self.resolve(Entity, entity_id, EntitySerializer)
        match_id = obj.pop('match_id', None)
        obj['match'] = self.resolve(Entity, match_id, EntitySerializer)
        return obj

class QueryLogSerializer(Serializer):
    pass

class RecordSerializer(Serializer):
    pass

class NotificationSerializer(Serializer):
    SERIALIZERS = {
        Alert: AlertSerializer,
        Entity: EntitySerializer,
        Collection: CollectionSerializer,
        Role: RoleSerializer
    }

    def _collect(self, obj):
        self.queue(Role, obj.get('actor_id'))
        event = Events.get(obj.get('event'))
        for name, clazz in event.params.items():
            key = obj.get('params', {}).get(name)
            self.queue(clazz, key)

    def _serialize(self, obj):
        event = Events.get(obj.get('event'))
        params = {
            'actor': self.resolve(Role, obj.get('actor_id'), RoleSerializer)
        }
        for name, clazz in event.params.items():
            key = obj.get('params', {}).get(name)
            serializer = self.SERIALIZERS.get(clazz)
            params[name] = self.resolve(clazz, key, serializer)
        obj['params'] = params
        obj['event'] = event.to_dict()
        return obj
<commit_msg>Implement referenced entity loading<commit_after>import logging
from flask import request
from banal import ensure_list
from normality import stringify
from followthemoney import model
from followthemoney.types import registry

from aleph.core import url_for
from aleph.model import Role, Collection, Document, Entity, Events, Alert
from aleph.logic import resolver
from aleph.logic.util import collection_url, entity_url
from aleph.views.util import jsonify

log = logging.getLogger(__name__)

class Serializer(object):

    def __init__(self, reference=False):
        self.reference = reference

    def _collect(self, obj):
        pass

    def _serialize(self, obj):
        return obj

    def queue(self, clazz, key):
        if not self.reference:
            resolver.queue(request, clazz, key)

    def resolve(self, clazz, key, serializer=None):
        if self.reference:
            return
        data = resolver.get(request, clazz, key)
        if data is not None and serializer is not None:
            serializer = serializer(reference=True)
            data = serializer.serialize(data)
        return data

    def serialize(self, obj):
        obj = self._to_dict(obj)
        if obj is not None:
            self._collect(obj)
            resolver.resolve(request)
            return self._serialize(obj)

    def serialize_many(self, objs):
        collected = []
        for obj in ensure_list(objs):
            obj = self._to_dict(obj)
            if obj is not None:
                self._collect(obj)
                collected.append(obj)
        resolver.resolve(request)
        serialized = []
        for obj in collected:
            obj = self._serialize(obj)
            if obj is not None:
                serialized.append(obj)
        return serialized

    def _to_dict(self, obj):
        if hasattr(obj, 'to_dict'):
            obj = obj.to_dict()
        if hasattr(obj, '_asdict'):
            obj = obj._asdict()
        return obj

    @classmethod
    def jsonify(cls, obj, **kwargs):
        data = cls().serialize(obj)
        return jsonify(data, **kwargs)

    @classmethod
    def jsonify_result(cls, result, **kwargs):
        data = result.to_dict(serializer=cls)
        return jsonify(data, **kwargs)

class RoleSerializer(Serializer):

    def _serialize(self, obj):
        obj['links'] = {
            'self': url_for('roles_api.view', id=obj.get('id'))
        }
        obj['writeable'] = obj.get('id') == stringify(request.authz.id)
        if not obj['writeable']:
            obj.pop('has_password', None)
            obj.pop('api_key', None)
            obj.pop('email', None)
        if obj['type'] != Role.USER:
            obj.pop('api_key', None)
            obj.pop('email', None)
        obj.pop('password', None)
        return obj

class AlertSerializer(Serializer):

    # def _collect(self, obj):
    #     self.queue(Role, obj.get('role_id'))

    def _serialize(self, obj):
        pk = obj.get('id')
        obj['links'] = {
            'self': url_for('alerts_api.view', id=pk)
        }
        role_id = obj.pop('role_id', None)
        obj['writeable'] = role_id == stringify(request.authz.id)
        # obj['role'] = self.resolve(Role, role_id, RoleSerializer)
        return obj

class CollectionSerializer(Serializer):

    def _collect(self, obj):
        self.queue(Role, obj.get('creator_id'))

    def _serialize(self, obj):
        pk = obj.get('id')
        obj['links'] = {
            'self': url_for('collections_api.view', id=pk),
            'xref': url_for('xref_api.index', id=pk),
            'xref_csv': url_for('xref_api.csv_export', id=pk, _authorize=True),
            'ui': collection_url(pk)
        }
        obj['writeable'] = request.authz.can(pk, request.authz.WRITE)
        creator_id = obj.pop('creator_id', None)
        obj['creator'] = self.resolve(Role, creator_id, RoleSerializer)
        obj.pop('_index', None)
        return obj

class PermissionSerializer(Serializer):

    def _collect(self, obj):
        self.queue(Role, obj.get('role_id'))

    def _serialize(self, obj):
        obj.pop('collection_id', None)
        role_id = obj.pop('role_id', None)
        obj['role'] = self.resolve(Role, role_id, RoleSerializer)
        return obj

class EntitySerializer(Serializer):

    def _collect(self, obj):
        self.queue(Collection, obj.get('collection_id'))
        self.queue(Role, obj.get('uploader_id'))
        schema = model.get(obj.get('schema'))
        if schema is None:
            return
        properties = obj.get('properties', {})
        for prop in schema.properties.values():
            if prop.type != registry.entity:
                continue
            values = ensure_list(properties.get(prop.name))
            for value in values:
                self.queue(Entity, value)

    def _serialize(self, obj):
        pk = obj.get('id')
        authz = request.authz
        collection_id = obj.pop('collection_id', None)
        obj['collection'] = self.resolve(Collection, collection_id,
                                         CollectionSerializer)
        schema = model.get(obj.get('schema'))
        obj['schemata'] = schema.names
        properties = obj.get('properties', {})
        for prop in schema.properties.values():
            if prop.type != registry.entity:
                continue
            values = ensure_list(properties.get(prop.name))
            properties[prop.name] = []
            for value in values:
                entity = self.resolve(Entity, value, EntitySerializer)
                properties[prop.name].append(entity)

        links = {
            'self': url_for('entities_api.view', id=pk),
            'references': url_for('entities_api.references', id=pk),
            'tags': url_for('entities_api.tags', id=pk),
            'ui': entity_url(pk)
        }
        if schema.is_a(Document.SCHEMA):
            links['content'] = url_for('documents_api.content', document_id=pk)

        for content_hash in ensure_list(properties.get('contentHash')):
            links['file'] = url_for('documents_api.file',
                                    document_id=pk,
                                    _authorize=True)

        for pdf_hash in ensure_list(properties.get('pdfHash')):
            links['pdf'] = url_for('documents_api.pdf',
                                   document_id=pk,
                                   _authorize=True)

        obj['links'] = links
        obj['writeable'] = authz.can(collection_id, authz.WRITE)
        if obj.get('bulk'):
            obj['writeable'] = False
        obj.pop('_index', None)
        return obj

class MatchCollectionsSerializer(Serializer):

    def _serialize(self, obj):
        serializer = CollectionSerializer(reference=True)
        obj['collection'] = serializer.serialize(obj.get('collection'))
        return obj

class MatchSerializer(Serializer):

    def _collect(self, obj):
        self.queue(Entity, obj.get('entity_id'))
        self.queue(Entity, obj.get('match_id'))

    def _serialize(self, obj):
        entity_id = obj.pop('entity_id', None)
        obj['entity'] = self.resolve(Entity, entity_id, EntitySerializer)
        match_id = obj.pop('match_id', None)
        obj['match'] = self.resolve(Entity, match_id, EntitySerializer)
        return obj

class QueryLogSerializer(Serializer):
    pass

class RecordSerializer(Serializer):
    pass

class NotificationSerializer(Serializer):
    SERIALIZERS = {
        Alert: AlertSerializer,
        Entity: EntitySerializer,
        Collection: CollectionSerializer,
        Role: RoleSerializer
    }

    def _collect(self, obj):
        self.queue(Role, obj.get('actor_id'))
        event = Events.get(obj.get('event'))
        for name, clazz in event.params.items():
            key = obj.get('params', {}).get(name)
            self.queue(clazz, key)

    def _serialize(self, obj):
        event = Events.get(obj.get('event'))
        params = {
            'actor': self.resolve(Role, obj.get('actor_id'), RoleSerializer)
        }
        for name, clazz in event.params.items():
            key = obj.get('params', {}).get(name)
            serializer = self.SERIALIZERS.get(clazz)
            params[name] = self.resolve(clazz, key, serializer)
        obj['params'] = params
        obj['event'] = event.to_dict()
        return obj
<|endoftext|>


<commit_before>"""
A class to compute a series of graph statistics over a sequence of subgraphs
"""
import gc
import numpy
import logging
from apgl.util.Util import Util
from apgl.util.Parameter import Parameter
from apgl.graph.GraphUtils import GraphUtils
from apgl.graph.AbstractSingleGraph import AbstractSingleGraph
from apgl.graph.AbstractMatrixGraph import AbstractMatrixGraph

class GraphStatistics(object):
    def __init__(self):
        self.numVerticesIndex = 0
        self.numEdgesIndex = 1
        self.numDirEdgesIndex = 2
        self.maxComponentSizeIndex = 3
        self.numComponentsIndex = 4
        self.meanComponentSizeIndex = 5
        self.meanDegreeIndex = 6
        self.diameterIndex = 7
        self.effectiveDiameterIndex = 8
        self.densityIndex = 9
        self.powerLawIndex = 10
        self.geodesicDistanceIndex = 11
        self.harmonicGeoDistanceIndex = 12
        self.geodesicDistMaxCompIndex = 13
        self.meanTreeSizeIndex = 14
        self.meanTreeDepthIndex = 15
        self.numNonSingletonComponentsIndex = 16
        self.maxComponentEdgesIndex = 17
        self.numTriOrMoreComponentsIndex = 18
        self.secondComponentSizeIndex = 19
        self.maxTreeSizeIndex = 20
        self.maxTreeDepthIndex = 21
        self.secondTreeSizeIndex = 22
        self.secondTreeDepthIndex = 23
        self.maxCompMeanDegreeIndex = 24

        self.numTreesIndex = 25
        self.numNonSingletonTreesIndex = 26

        self.numStats = 27
        self.q = 0.9
        self.printStep = 5
        self.vectorPrintStep = 1

        self.useFloydWarshall = False

    def getNumStats(self):
        return self.numStats

    def strScalarStatsArray(self, statsArray):
        """
        Get a string representation of the scalar statastics array.
        """
        s = "Num vertices: " + str(statsArray[self.numVerticesIndex]) + "\n"
        s += "Num edges: " + str(statsArray[self.numEdgesIndex]) + "\n"
        s += "Num directed edges: " + str(statsArray[self.numDirEdgesIndex]) + "\n"
        s += "Max component size: " + str(statsArray[self.maxComponentSizeIndex]) + "\n"
        s += "Max component edges: " + str(statsArray[self.maxComponentEdgesIndex]) + "\n"
        s += "Max component mean degree: " + str(statsArray[self.maxCompMeanDegreeIndex]) + "\n"
        s += "Max component geodesic disance: " + str(statsArray[self.geodesicDistMaxCompIndex]) + "\n"
        s += "2nd largest component size: " + str(statsArray[self.secondComponentSizeIndex]) + "\n"
        s += "Num components: " + str(statsArray[self.numComponentsIndex]) + "\n"
        s += "Mean component size: " + str(statsArray[self.meanComponentSizeIndex]) + "\n"
        s += "Mean degree: " + str(statsArray[self.meanDegreeIndex]) + "\n"
        s += "Diameter: " + str(statsArray[self.diameterIndex]) + "\n"
        s += "Effective diameter: " + str(statsArray[self.effectiveDiameterIndex]) + "\n"
        s += "Density: " + str(statsArray[self.densityIndex]) + "\n"
        s += "Power law exponent: " + str(statsArray[self.powerLawIndex]) + "\n"
        s += "Geodesic distance: " + str(statsArray[self.geodesicDistanceIndex]) + "\n"
        s += "Harmonic geodesic distance: " + str(statsArray[self.harmonicGeoDistanceIndex]) + "\n"
        s += "Mean tree size: " + str(statsArray[self.meanTreeSizeIndex]) + "\n"
        s += "Mean tree depth: " + str(statsArray[self.meanTreeDepthIndex]) + "\n"
        s += "Num non-singleton components: " + str(statsArray[self.numNonSingletonComponentsIndex]) + "\n"
        s += "Num tri-or-more components: " + str(statsArray[self.numTriOrMoreComponentsIndex]) + "\n"

        return s

    def scalarStatistics(self, graph, slowStats=True, treeStats=False):
        """
        Find a series of statistics for the given input graph which can be represented
        as scalar values. Return results as a vector.
        """
        #This method is a bit of a mess
        Parameter.checkClass(graph, AbstractSingleGraph)
        Parameter.checkBoolean(slowStats)
        Parameter.checkBoolean(treeStats)

        statsArray = numpy.ones(self.numStats)*-1
        statsArray[self.numVerticesIndex] = graph.getNumVertices()
        statsArray[self.numEdgesIndex] = graph.getNumEdges()
        statsArray[self.numDirEdgesIndex] = graph.getNumDirEdges()
        statsArray[self.densityIndex] = graph.density()

        if graph.isUndirected():
            logging.debug("Finding connected components")
            subComponents = graph.findConnectedComponents()
            logging.debug("Done")
            statsArray[self.numComponentsIndex] = len(subComponents)

            nonSingletonSubComponents = [c for c in subComponents if len(c) > 1]
            statsArray[self.numNonSingletonComponentsIndex] = len(nonSingletonSubComponents)

            triOrMoreSubComponents = [c for c in subComponents if len(c) > 2]
            statsArray[self.numTriOrMoreComponentsIndex] = len(triOrMoreSubComponents)

            logging.debug("Studying max component")
            if len(subComponents) != 0:
                maxCompGraph = graph.subgraph(list(subComponents[0]))
                statsArray[self.maxComponentSizeIndex] = len(subComponents[0])

                if len(subComponents) >= 2:
                    statsArray[self.secondComponentSizeIndex] = len(subComponents[1])

                statsArray[self.maxComponentEdgesIndex] = maxCompGraph.getNumEdges()
                statsArray[self.meanComponentSizeIndex] = sum([len(x) for x in subComponents])/float(statsArray[self.numComponentsIndex])
                statsArray[self.maxCompMeanDegreeIndex] = numpy.mean(maxCompGraph.outDegreeSequence())
            else:
                statsArray[self.maxComponentSizeIndex] = 0
                statsArray[self.maxComponentEdgesIndex] = 0
                statsArray[self.meanComponentSizeIndex] = 0
                statsArray[self.geodesicDistMaxCompIndex] = 0

        if graph.getNumVertices() != 0:
            statsArray[self.meanDegreeIndex] = numpy.mean(graph.outDegreeSequence())
        else:
            statsArray[self.meanDegreeIndex] = 0

        if slowStats:
            if self.useFloydWarshall:
                logging.debug("Running Floyd-Warshall")
                P = graph.floydWarshall(False)
            else:
                logging.debug("Running Dijkstra's algorithm")
                P = graph.findAllDistances(False)

            statsArray[self.diameterIndex] = graph.diameter(P=P)
            statsArray[self.effectiveDiameterIndex] = graph.effectiveDiameter(self.q, P=P)
            statsArray[self.powerLawIndex] = graph.fitPowerLaw()[0]
            statsArray[self.geodesicDistanceIndex] = graph.geodesicDistance(P=P)
            statsArray[self.harmonicGeoDistanceIndex] = graph.harmonicGeodesicDistance(P=P)

            if graph.isUndirected() and len(subComponents) != 0:
                statsArray[self.geodesicDistMaxCompIndex] = graph.geodesicDistance(P=P, vertexInds=list(subComponents[0]))

        if treeStats:
            logging.debug("Computing statistics on trees")
            trees = graph.findTrees()
            statsArray[self.numTreesIndex] = len(trees)

            nonSingletonTrees = [c for c in trees if len(c) > 1]
            statsArray[self.numNonSingletonTreesIndex] = len(nonSingletonTrees)

            statsArray[self.meanTreeSizeIndex] = numpy.mean([len(x) for x in trees])
            treeDepths = [GraphUtils.treeDepth((graph.subgraph(list(x)))) for x in trees]
            statsArray[self.meanTreeDepthIndex] = numpy.mean(treeDepths)

            if len(trees) != 0:
                maxTreeGraph = graph.subgraph(trees[0])
                statsArray[self.maxTreeSizeIndex] = len(trees[0])
                statsArray[self.maxTreeDepthIndex] = GraphUtils.treeDepth(maxTreeGraph)

                if len(trees) >= 2:
                    secondTreeGraph = graph.subgraph(trees[1])
                    statsArray[self.secondTreeSizeIndex] = len(trees[1])
                    statsArray[self.secondTreeDepthIndex] = GraphUtils.treeDepth(secondTreeGraph)

        return statsArray

    def vectorStatistics(self, graph, treeStats=False, eigenStats=True):
        """
        Find a series of statistics for the given input graph which can be represented
        as vector values.
        """
        Parameter.checkClass(graph, AbstractMatrixGraph)
        Parameter.checkBoolean(treeStats)
        statsDict = {}

        statsDict["inDegreeDist"] = graph.inDegreeDistribution()
        statsDict["outDegreeDist"] = graph.degreeDistribution()
        logging.debug("Computing hop counts")
        P = graph.findAllDistances(False)
        statsDict["hopCount"] = graph.hopCount(P)
        logging.debug("Computing triangle count")
        if graph.getNumVertices() != 0:
            statsDict["triangleDist"] = numpy.bincount(graph.triangleSequence())
        else:
            statsDict["triangleDist"] = numpy.array([])

        #Get the distribution of component sizes
        logging.debug("Finding distribution of component sizes")

        if graph.isUndirected():
            components = graph.findConnectedComponents()
            if len(components) != 0:
                statsDict["componentsDist"] = numpy.bincount(numpy.array([len(c) for c in components], numpy.int))

        #Make sure weight matrix is symmetric

        if graph.getNumVertices()!=0 and eigenStats:
            logging.debug("Computing eigenvalues/vectors")
            W = graph.getWeightMatrix()
            W = (W + W.T)/2
            eigenDistribution, V = numpy.linalg.eig(W)
            i = numpy.argmax(eigenDistribution)
            statsDict["maxEigVector"] = V[:, i]
            statsDict["eigenDist"] = numpy.flipud(numpy.sort(eigenDistribution[eigenDistribution>0]))
            gc.collect()
        else:
            statsDict["maxEigVector"] = numpy.array([])
            statsDict["eigenDist"] = numpy.array([])

        if treeStats:
            logging.debug("Computing statistics on trees")
            trees = graph.findTrees()
            statsDict["treeSizesDist"] = numpy.bincount([len(x) for x in trees])
            treeDepths = [GraphUtils.treeDepth((graph.subgraph(x))) for x in trees]
            statsDict["treeDepthsDist"] = numpy.bincount(treeDepths)

        return statsDict

    def sequenceScalarStats(self, graph, subgraphIndices, slowStats=True, treeStats=False):
        """
        Pass in a graph and list of subgraph indices and returns a series of statistics. Each row
        corresponds to the statistics on the subgraph.
        """
        Parameter.checkClass(graph, AbstractMatrixGraph)
        for inds in subgraphIndices:
            Parameter.checkList(inds, Parameter.checkInt, [0, graph.getNumVertices()])
        Parameter.checkBoolean(slowStats)
        Parameter.checkBoolean(treeStats)

        numGraphs = len(subgraphIndices)
        statsMatrix = numpy.zeros((numGraphs, self.numStats))

        for i in range(numGraphs):
            Util.printIteration(i, self.printStep, numGraphs)
            logging.debug("Subgraph size: " + str(len(subgraphIndices[i])))
            subgraph = graph.subgraph(subgraphIndices[i])
            statsMatrix[i, :] = self.scalarStatistics(subgraph, slowStats, treeStats)

        return statsMatrix

    def meanSeqScalarStats(self, graphList, slowStats=True, treeStats=False):
        """
        Pass in a list of tuples (graph, subgraphIndices) and returns a series of statistics. Each row
        corresponds to the statistics on the subgraph. All graphs must be the same size and computed
        from the same distribution, and the number of subgraphs must be the same.
        """
        Parameter.checkBoolean(slowStats)
        Parameter.checkBoolean(treeStats)
        if len(graphList)==0:
            return -1

        numGraphs = len(graphList)
        numSubgraphs = len(graphList[0][1])
        statsMatrix = numpy.zeros((numSubgraphs, self.numStats, numGraphs))

        for i in range(len(graphList)):
            (graph, subgraphIndices) = graphList[i]
            statsMatrix[:, :, i] = self.sequenceScalarStats(graph, subgraphIndices, slowStats, treeStats)

        return numpy.mean(statsMatrix, 2), numpy.std(statsMatrix, 2)

    def sequenceVectorStats(self, graph, subgraphIndices, treeStats=False, eigenStats=True):
        """
        Pass in a list of graphs are returns a series of statistics. Each list
        element is a dict of vector statistics.
        """
        Parameter.checkClass(graph, AbstractMatrixGraph)
        for inds in subgraphIndices:
            Parameter.checkList(inds, Parameter.checkInt, [0, graph.getNumVertices()])
        Parameter.checkBoolean(treeStats)

        numGraphs = len(subgraphIndices)
        statsDictList = []

        for i in range(numGraphs):
            Util.printIteration(i, self.vectorPrintStep, numGraphs)
            subgraph = graph.subgraph(subgraphIndices[i])
            statsDictList.append(self.vectorStatistics(subgraph, treeStats, eigenStats))

        return statsDictList

    def sequenceClustering(self, graph, subgraphIndices, clusterFunc, maxComponent=True):
        """
        Take a graph and a sequence of indices corresponding to subgraphs and
        compute some clusters indices for each one.
        """
        numGraphs = len(subgraphIndices)
        clusterList = []

        for i in range(numGraphs):
            Util.printIteration(i, self.vectorPrintStep, numGraphs)
            subgraph = graph.subgraph(subgraphIndices[i])

            if maxComponent:
                subComponents = subgraph.findConnectedComponents()
                subgraph = subgraph.subgraph(subComponents[-1])

            clusterList.append(clusterFunc(subgraph))

        return clusterList
        <commit_msg>Remove some logging debugs <commit_after>"""
A class to compute a series of graph statistics over a sequence of subgraphs
"""
import gc
import numpy
import logging
from apgl.util.Util import Util
from apgl.util.Parameter import Parameter
from apgl.graph.GraphUtils import GraphUtils
from apgl.graph.AbstractSingleGraph import AbstractSingleGraph
from apgl.graph.AbstractMatrixGraph import AbstractMatrixGraph

class GraphStatistics(object):
    def __init__(self):
        self.numVerticesIndex = 0
        self.numEdgesIndex = 1
        self.numDirEdgesIndex = 2
        self.maxComponentSizeIndex = 3
        self.numComponentsIndex = 4
        self.meanComponentSizeIndex = 5
        self.meanDegreeIndex = 6
        self.diameterIndex = 7
        self.effectiveDiameterIndex = 8
        self.densityIndex = 9
        self.powerLawIndex = 10
        self.geodesicDistanceIndex = 11
        self.harmonicGeoDistanceIndex = 12
        self.geodesicDistMaxCompIndex = 13
        self.meanTreeSizeIndex = 14
        self.meanTreeDepthIndex = 15
        self.numNonSingletonComponentsIndex = 16
        self.maxComponentEdgesIndex = 17
        self.numTriOrMoreComponentsIndex = 18
        self.secondComponentSizeIndex = 19
        self.maxTreeSizeIndex = 20
        self.maxTreeDepthIndex = 21
        self.secondTreeSizeIndex = 22
        self.secondTreeDepthIndex = 23
        self.maxCompMeanDegreeIndex = 24

        self.numTreesIndex = 25
        self.numNonSingletonTreesIndex = 26

        self.numStats = 27
        self.q = 0.9
        self.printStep = 5
        self.vectorPrintStep = 1

        self.useFloydWarshall = False

    def getNumStats(self):
        return self.numStats

    def strScalarStatsArray(self, statsArray):
        """
        Get a string representation of the scalar statastics array.
        """
        s = "Num vertices: " + str(statsArray[self.numVerticesIndex]) + "\n"
        s += "Num edges: " + str(statsArray[self.numEdgesIndex]) + "\n"
        s += "Num directed edges: " + str(statsArray[self.numDirEdgesIndex]) + "\n"
        s += "Max component size: " + str(statsArray[self.maxComponentSizeIndex]) + "\n"
        s += "Max component edges: " + str(statsArray[self.maxComponentEdgesIndex]) + "\n"
        s += "Max component mean degree: " + str(statsArray[self.maxCompMeanDegreeIndex]) + "\n"
        s += "Max component geodesic disance: " + str(statsArray[self.geodesicDistMaxCompIndex]) + "\n"
        s += "2nd largest component size: " + str(statsArray[self.secondComponentSizeIndex]) + "\n"
        s += "Num components: " + str(statsArray[self.numComponentsIndex]) + "\n"
        s += "Mean component size: " + str(statsArray[self.meanComponentSizeIndex]) + "\n"
        s += "Mean degree: " + str(statsArray[self.meanDegreeIndex]) + "\n"
        s += "Diameter: " + str(statsArray[self.diameterIndex]) + "\n"
        s += "Effective diameter: " + str(statsArray[self.effectiveDiameterIndex]) + "\n"
        s += "Density: " + str(statsArray[self.densityIndex]) + "\n"
        s += "Power law exponent: " + str(statsArray[self.powerLawIndex]) + "\n"
        s += "Geodesic distance: " + str(statsArray[self.geodesicDistanceIndex]) + "\n"
        s += "Harmonic geodesic distance: " + str(statsArray[self.harmonicGeoDistanceIndex]) + "\n"
        s += "Mean tree size: " + str(statsArray[self.meanTreeSizeIndex]) + "\n"
        s += "Mean tree depth: " + str(statsArray[self.meanTreeDepthIndex]) + "\n"
        s += "Num non-singleton components: " + str(statsArray[self.numNonSingletonComponentsIndex]) + "\n"
        s += "Num tri-or-more components: " + str(statsArray[self.numTriOrMoreComponentsIndex]) + "\n"

        return s

    def scalarStatistics(self, graph, slowStats=True, treeStats=False):
        """
        Find a series of statistics for the given input graph which can be represented
        as scalar values. Return results as a vector.
        """
        #This method is a bit of a mess
        Parameter.checkClass(graph, AbstractSingleGraph)
        Parameter.checkBoolean(slowStats)
        Parameter.checkBoolean(treeStats)

        statsArray = numpy.ones(self.numStats)*-1
        statsArray[self.numVerticesIndex] = graph.getNumVertices()
        statsArray[self.numEdgesIndex] = graph.getNumEdges()
        statsArray[self.numDirEdgesIndex] = graph.getNumDirEdges()
        statsArray[self.densityIndex] = graph.density()

        if graph.isUndirected():
            subComponents = graph.findConnectedComponents()
            statsArray[self.numComponentsIndex] = len(subComponents)

            nonSingletonSubComponents = [c for c in subComponents if len(c) > 1]
            statsArray[self.numNonSingletonComponentsIndex] = len(nonSingletonSubComponents)

            triOrMoreSubComponents = [c for c in subComponents if len(c) > 2]
            statsArray[self.numTriOrMoreComponentsIndex] = len(triOrMoreSubComponents)

            #logging.debug("Studying max component")
            if len(subComponents) != 0:
                maxCompGraph = graph.subgraph(list(subComponents[0]))
                statsArray[self.maxComponentSizeIndex] = len(subComponents[0])

                if len(subComponents) >= 2:
                    statsArray[self.secondComponentSizeIndex] = len(subComponents[1])

                statsArray[self.maxComponentEdgesIndex] = maxCompGraph.getNumEdges()
                statsArray[self.meanComponentSizeIndex] = sum([len(x) for x in subComponents])/float(statsArray[self.numComponentsIndex])
                statsArray[self.maxCompMeanDegreeIndex] = numpy.mean(maxCompGraph.outDegreeSequence())
            else:
                statsArray[self.maxComponentSizeIndex] = 0
                statsArray[self.maxComponentEdgesIndex] = 0
                statsArray[self.meanComponentSizeIndex] = 0
                statsArray[self.geodesicDistMaxCompIndex] = 0

        if graph.getNumVertices() != 0:
            statsArray[self.meanDegreeIndex] = numpy.mean(graph.outDegreeSequence())
        else:
            statsArray[self.meanDegreeIndex] = 0

        if slowStats:
            if self.useFloydWarshall:
                logging.debug("Running Floyd-Warshall")
                P = graph.floydWarshall(False)
            else:
                logging.debug("Running Dijkstra's algorithm")
                P = graph.findAllDistances(False)

            statsArray[self.diameterIndex] = graph.diameter(P=P)
            statsArray[self.effectiveDiameterIndex] = graph.effectiveDiameter(self.q, P=P)
            statsArray[self.powerLawIndex] = graph.fitPowerLaw()[0]
            statsArray[self.geodesicDistanceIndex] = graph.geodesicDistance(P=P)
            statsArray[self.harmonicGeoDistanceIndex] = graph.harmonicGeodesicDistance(P=P)

            if graph.isUndirected() and len(subComponents) != 0:
                statsArray[self.geodesicDistMaxCompIndex] = graph.geodesicDistance(P=P, vertexInds=list(subComponents[0]))

        if treeStats:
            logging.debug("Computing statistics on trees")
            trees = graph.findTrees()
            statsArray[self.numTreesIndex] = len(trees)

            nonSingletonTrees = [c for c in trees if len(c) > 1]
            statsArray[self.numNonSingletonTreesIndex] = len(nonSingletonTrees)

            statsArray[self.meanTreeSizeIndex] = numpy.mean([len(x) for x in trees])
            treeDepths = [GraphUtils.treeDepth((graph.subgraph(list(x)))) for x in trees]
            statsArray[self.meanTreeDepthIndex] = numpy.mean(treeDepths)

            if len(trees) != 0:
                maxTreeGraph = graph.subgraph(trees[0])
                statsArray[self.maxTreeSizeIndex] = len(trees[0])
                statsArray[self.maxTreeDepthIndex] = GraphUtils.treeDepth(maxTreeGraph)

                if len(trees) >= 2:
                    secondTreeGraph = graph.subgraph(trees[1])
                    statsArray[self.secondTreeSizeIndex] = len(trees[1])
                    statsArray[self.secondTreeDepthIndex] = GraphUtils.treeDepth(secondTreeGraph)

        return statsArray

    def vectorStatistics(self, graph, treeStats=False, eigenStats=True):
        """
        Find a series of statistics for the given input graph which can be represented
        as vector values.
        """
        Parameter.checkClass(graph, AbstractMatrixGraph)
        Parameter.checkBoolean(treeStats)
        statsDict = {}

        statsDict["inDegreeDist"] = graph.inDegreeDistribution()
        statsDict["outDegreeDist"] = graph.degreeDistribution()
        logging.debug("Computing hop counts")
        P = graph.findAllDistances(False)
        statsDict["hopCount"] = graph.hopCount(P)
        logging.debug("Computing triangle count")
        if graph.getNumVertices() != 0:
            statsDict["triangleDist"] = numpy.bincount(graph.triangleSequence())
        else:
            statsDict["triangleDist"] = numpy.array([])

        #Get the distribution of component sizes
        logging.debug("Finding distribution of component sizes")

        if graph.isUndirected():
            components = graph.findConnectedComponents()
            if len(components) != 0:
                statsDict["componentsDist"] = numpy.bincount(numpy.array([len(c) for c in components], numpy.int))

        #Make sure weight matrix is symmetric

        if graph.getNumVertices()!=0 and eigenStats:
            logging.debug("Computing eigenvalues/vectors")
            W = graph.getWeightMatrix()
            W = (W + W.T)/2
            eigenDistribution, V = numpy.linalg.eig(W)
            i = numpy.argmax(eigenDistribution)
            statsDict["maxEigVector"] = V[:, i]
            statsDict["eigenDist"] = numpy.flipud(numpy.sort(eigenDistribution[eigenDistribution>0]))
            gc.collect()
        else:
            statsDict["maxEigVector"] = numpy.array([])
            statsDict["eigenDist"] = numpy.array([])

        if treeStats:
            logging.debug("Computing statistics on trees")
            trees = graph.findTrees()
            statsDict["treeSizesDist"] = numpy.bincount([len(x) for x in trees])
            treeDepths = [GraphUtils.treeDepth((graph.subgraph(x))) for x in trees]
            statsDict["treeDepthsDist"] = numpy.bincount(treeDepths)

        return statsDict

    def sequenceScalarStats(self, graph, subgraphIndices, slowStats=True, treeStats=False):
        """
        Pass in a graph and list of subgraph indices and returns a series of statistics. Each row
        corresponds to the statistics on the subgraph.
        """
        Parameter.checkClass(graph, AbstractMatrixGraph)
        for inds in subgraphIndices:
            Parameter.checkList(inds, Parameter.checkInt, [0, graph.getNumVertices()])
        Parameter.checkBoolean(slowStats)
        Parameter.checkBoolean(treeStats)

        numGraphs = len(subgraphIndices)
        statsMatrix = numpy.zeros((numGraphs, self.numStats))

        for i in range(numGraphs):
            Util.printIteration(i, self.printStep, numGraphs)
            #logging.debug("Subgraph size: " + str(len(subgraphIndices[i])))
            subgraph = graph.subgraph(subgraphIndices[i])
            statsMatrix[i, :] = self.scalarStatistics(subgraph, slowStats, treeStats)

        return statsMatrix

    def meanSeqScalarStats(self, graphList, slowStats=True, treeStats=False):
        """
        Pass in a list of tuples (graph, subgraphIndices) and returns a series of statistics. Each row
        corresponds to the statistics on the subgraph. All graphs must be the same size and computed
        from the same distribution, and the number of subgraphs must be the same.
        """
        Parameter.checkBoolean(slowStats)
        Parameter.checkBoolean(treeStats)
        if len(graphList)==0:
            return -1

        numGraphs = len(graphList)
        numSubgraphs = len(graphList[0][1])
        statsMatrix = numpy.zeros((numSubgraphs, self.numStats, numGraphs))

        for i in range(len(graphList)):
            (graph, subgraphIndices) = graphList[i]
            statsMatrix[:, :, i] = self.sequenceScalarStats(graph, subgraphIndices, slowStats, treeStats)

        return numpy.mean(statsMatrix, 2), numpy.std(statsMatrix, 2)

    def sequenceVectorStats(self, graph, subgraphIndices, treeStats=False, eigenStats=True):
        """
        Pass in a list of graphs are returns a series of statistics. Each list
        element is a dict of vector statistics.
        """
        Parameter.checkClass(graph, AbstractMatrixGraph)
        for inds in subgraphIndices:
            Parameter.checkList(inds, Parameter.checkInt, [0, graph.getNumVertices()])
        Parameter.checkBoolean(treeStats)

        numGraphs = len(subgraphIndices)
        statsDictList = []

        for i in range(numGraphs):
            Util.printIteration(i, self.vectorPrintStep, numGraphs)
            subgraph = graph.subgraph(subgraphIndices[i])
            statsDictList.append(self.vectorStatistics(subgraph, treeStats, eigenStats))

        return statsDictList

    def sequenceClustering(self, graph, subgraphIndices, clusterFunc, maxComponent=True):
        """
        Take a graph and a sequence of indices corresponding to subgraphs and
        compute some clusters indices for each one.
        """
        numGraphs = len(subgraphIndices)
        clusterList = []

        for i in range(numGraphs):
            Util.printIteration(i, self.vectorPrintStep, numGraphs)
            subgraph = graph.subgraph(subgraphIndices[i])

            if maxComponent:
                subComponents = subgraph.findConnectedComponents()
                subgraph = subgraph.subgraph(subComponents[-1])

            clusterList.append(clusterFunc(subgraph))

        return clusterList
        <|endoftext|>


<commit_before><commit_msg>fake manage.py<commit_after>#!/bin/bash

if ! which leprikon &> /dev/null; then
    if ! test -e env/bin/activate; then
        virtualenv env
    fi
    . env/bin/activate
    if ! which leprikon &> /dev/null; then
        pip install -r requirements.txt
    fi
fi

if [ -n "$1" ]; then
    export DEBUG="${DEBUG:-TEMPLATE}"
else
    # bash-completion
    export DEBUG=""
fi

export PYTHONPATH=.
#export SITE_MODULE=leprikon

exec leprikon "$@"
<|endoftext|>


<commit_before>#!/bin/python

import dash_core_components as dcc

from evaluation.gekko.statistics import epochStatisticsNames, periodicStatisticsNames

def updateWorldGraph(app, WORLD):
    environmentData = [
        {
        }
    ]
    populationGroupData = [
        {
            'x': [locale.position[0]],
            'y': [locale.position[1]],
            'type': 'scatter',
            'name': locale.name,
            'showscale': False,
            'mode': 'markers',
            'marker': {
                'symbol': 'square'
            }

        } for locale in WORLD.locales
    ]

    fig = {
        'data': populationGroupData,
        'layout': {
            'title': "World Topology"
        }
    }

    G = dcc.Graph(id="WorldGraph", figure=fig)
    #app.layout.get("WorldGraphContainer").children = [G]
    app.WorldGraph = G
    return G

def updateLocaleGraph(app, LOCALE):

    GraphName = LOCALE.name
    print('Loading %s' % GraphName)
    Statistics = LOCALE.EvolutionStatistics
    ID = [s for s in GraphName if s.isdigit()]
    annotations = []

    oldLocaleGraph = None
    for lidx, localeGraph in enumerate(app.LocaleGraphs):
        if localeGraph.id == LOCALE.name:
            oldLocaleGraph = lidx
            break

    statisticsNames = {}
    statisticsNames.update(epochStatisticsNames)
    statisticsNames.update(periodicStatisticsNames)

    annotationFontDescription = {
        'family': 'Arial',
        'size': 12,
        'color': 'rgb(37,37,37)'
    }

    """
    for Statistic in Statistics:
        if 'dateRange' in Statistic.keys():
            if Statistic['dateRange']:
                for R, dateRange in enumerate(Statistic['dateRange']):
                    if dateRange is not None:
                        annotations.append(
                            {
                                'xref': 'axis',
                                'yref': 'paper',
                                'xanchor': 'left',
                                'yanchor': 'bottom',
                                'font': annotationFontDescription,
                                'x': R,
                                'y': 1 if not len(annotations) %
                                2 else 0.93,  # avoid label overlap;
                                'text': dateRange,
                            }
                        )
    """

    colorSequence = [
        (188, 189, 34),
        (100, 11, 182),
        (186, 3, 34),
        (45, 111, 45),
        (66, 128, 66),
        (128, 66, 66),
    ]
    statNames = [
        'avg', 'std', 'min',
        'max', 'evaluationScore',
        'evaluationScoreOnSecondary'
    ]
    """
    centralMarker = {
        'x': [0, Statistics[-1]["id"]],
        'y': [0],
        'type': 'line',
        'name': 'markzero',
        'line': {'color': 'rgb(0,0,0)'},
    }
    """
    DATA = []
    for Statistic in Statistics:
        DATA += [
            {
                'x': [Statistic['id']],
                'y': [Statistic[statNames[S]]],
                'type': 'line',
                'name': statisticsNames[statNames[S]],
                'line': {'color': 'rgb%s' % str(colorSequence[S])},
            }
            for S in range(len(statNames))
            if Statistic[statNames[S]] is not None
        ]

    fig = {
        'data': DATA,
        'layout': {
            'title': 'Evolution at %s' % GraphName,
            'annotations': annotations
        },
    }

    G = dcc.Graph(figure=fig, id=LOCALE.name)
    if oldLocaleGraph:
        app.LocaleGraphs[oldLocaleGraph] = G
    else:
        app.LocaleGraphs.append(G)

    return G

<commit_msg>v0.90<commit_after>#!/bin/python

import dash_core_components as dcc

from evaluation.gekko.statistics import epochStatisticsNames, periodicStatisticsNames

def updateWorldGraph(app, WORLD):
    environmentData = [
        {
        }
    ]
    populationGroupData = [
        {
            'x': [locale.position[0]],
            'y': [locale.position[1]],
            'type': 'scatter',
            'name': locale.name,
            'showscale': False,
            'mode': 'markers',
            'marker': {
                'symbol': 'square'
            }

        } for locale in WORLD.locales
    ]

    fig = {
        'data': populationGroupData,
        'layout': {
            'title': "World Topology"
        }
    }

    G = dcc.Graph(id="WorldGraph", figure=fig)
    #app.layout.get("WorldGraphContainer").children = [G]
    app.WorldGraph = G
    return G

def updateLocaleGraph(app, LOCALE):

    GraphName = LOCALE.name
    print('Loading %s' % GraphName)
    Statistics = LOCALE.EvolutionStatistics
    ID = [s for s in GraphName if s.isdigit()]
    annotations = []

    oldLocaleGraph = None
    for lidx, localeGraph in enumerate(app.LocaleGraphs):
        if localeGraph.id == LOCALE.name:
            oldLocaleGraph = lidx
            break

    statisticsNames = {}
    statisticsNames.update(epochStatisticsNames)
    statisticsNames.update(periodicStatisticsNames)

    annotationFontDescription = {
        'family': 'Arial',
        'size': 12,
        'color': 'rgb(37,37,37)'
    }

    """
    for Statistic in Statistics:
        if 'dateRange' in Statistic.keys():
            if Statistic['dateRange']:
                for R, dateRange in enumerate(Statistic['dateRange']):
                    if dateRange is not None:
                        annotations.append(
                            {
                                'xref': 'axis',
                                'yref': 'paper',
                                'xanchor': 'left',
                                'yanchor': 'bottom',
                                'font': annotationFontDescription,
                                'x': R,
                                'y': 1 if not len(annotations) %
                                2 else 0.93,  # avoid label overlap;
                                'text': dateRange,
                            }
                        )
    """

    colorSequence = [
        (188, 189, 34),
        (100, 11, 182),
        (186, 3, 34),
        (45, 111, 45),
        (66, 128, 66),
        (128, 66, 66),
    ]
    statNames = [
        'avg', 'std', 'min',
        'max', 'evaluationScore',
        'evaluationScoreOnSecondary'
    ]
    """
    centralMarker = {
        'x': [0, Statistics[-1]["id"]],
        'y': [0],
        'type': 'line',
        'name': 'markzero',
        'line': {'color': 'rgb(0,0,0)'},
    }
    """
    DATA = []
    for Statistic in Statistics:
        DATA += [
            {
                'x': [Statistic['id']],
                'y': [Statistic[statNames[S]]],
                'type': 'line',
                'name': statisticsNames[statNames[S]],
                'line': {'color': 'rgb%s' % str(colorSequence[S])},
            }
            for S in range(len(statNames))
            if Statistic[statNames[S]] is not None
        ]

    fig = {
        'data': DATA,
        'layout': {
            'title': 'Evolution at %s' % GraphName,
            'annotations': annotations
        },
    }

    G = dcc.Graph(figure=fig, id=LOCALE.name)
    if oldLocaleGraph is not None:
        app.LocaleGraphs[oldLocaleGraph] = G
    else:
        app.LocaleGraphs.append(G)

    return G

<|endoftext|>


<commit_before># Tai Sakuma <tai.sakuma@gmail.com>
import os
import sys
import pytest

from alphatwirl.selection.factories.expand import expand_path_cfg

##__________________________________________________________________||
@pytest.mark.parametrize('path_cfg, expected', [
    pytest.param(
        'ev : ev.nJets[0] >= 2',
        dict(
            factory='LambdaStrFactory',
            lambda_str='ev : ev.nJets[0] >= 2',
        ),
        id='string:lambda_str'
    ),
    pytest.param(
        'ev : ev.nJets[0] >= {n}',
        dict(
            factory='LambdaStrFactory',
            lambda_str='ev : ev.nJets[0] >= {n}',
        ),
        id='string:lambda_str-not-formatted'
    ),
    pytest.param(
        dict(All=()),
        {'factory': 'AllFactory', 'path_cfg_list': ()},
        id='dict-all-empty'
    ),
    pytest.param(
        dict(Any=()),
        {'factory': 'AnyFactory', 'path_cfg_list': ()},
        id='dict-any-empty'
    ),
    pytest.param(
        dict(All=(dict(factory='factory1'), dict(factory='factory2')), name='test_all', arg2=2, arg3=3),
        dict(
            factory='AllFactory',
            path_cfg_list=(dict(factory='factory1'), dict(factory='factory2')),
            name='test_all',
            arg2=2, arg3=3
        ),
        id='dict-all'
    ),
    pytest.param(
        dict(Any=(dict(factory='factory1'), dict(factory='factory2')), name='test_any', arg2=2, arg3=3),
        dict(
            factory='AnyFactory',
            path_cfg_list=(dict(factory='factory1'), dict(factory='factory2')),
            name='test_any',
            arg2=2, arg3=3
        ),
        id='dict-any'
    ),
    pytest.param(
        dict(Not=dict(factory='factory1'), name='test_not', arg2=2, arg3=3),
        dict(
            factory='NotFactory',
            path_cfg=dict(factory='factory1'),
            name='test_not',
            arg2=2, arg3=3
        ),
        id='dict-not'
    ),
    pytest.param(
        dict(Any=(
            'ev : ev.x[0] == 0',
            dict(All=(
                'ev : ev.x[0] >= 1',
                'ev : ev.y[0] >= 100',
                )),
            dict(Not=dict(
                Any=(
                    'ev : ev.z[0] == 0',
                    'ev : ev.w[0] >= 300',
                ),
            )),
        )),
        dict(
            factory='AnyFactory',
            path_cfg_list=(
                dict(
                    factory='LambdaStrFactory',
                    lambda_str='ev : ev.x[0] == 0',
                ),
                dict(
                    factory='AllFactory',
                    path_cfg_list=(
                        dict(
                            factory='LambdaStrFactory',
                            lambda_str='ev : ev.x[0] >= 1',
                        ),
                        dict(
                            factory='LambdaStrFactory',
                            lambda_str='ev : ev.y[0] >= 100',
                        )
                    )
                ),
                dict(
                    factory='NotFactory',
                    path_cfg=dict(
                        factory='AnyFactory',
                        path_cfg_list=(
                            dict(
                                factory='LambdaStrFactory',
                                lambda_str='ev : ev.z[0] == 0'
                            ),
                            dict(
                                factory='LambdaStrFactory',
                                lambda_str='ev : ev.w[0] >= 300',
                                ),
                        )
                    )
                )
            )
        ),
        id='example',
        ## marks=pytest.mark.skip(reason='not fully expanded')
    ),
])
def test_expand_path_cfg(path_cfg, expected):
    actual = expand_path_cfg(path_cfg=path_cfg)
    assert expected == actual

    # give expanded one
    actual = expand_path_cfg(path_cfg=actual)
    assert expected == actual

##__________________________________________________________________||
@pytest.mark.parametrize('path_cfg, error', [
    pytest.param(
        dict(All=(), Any=()), ValueError, id='multiple-vertices-All-Any'
    ),
    pytest.param(
        dict(All=(), Not=()), ValueError, id='multiple-vertices-All-Not'
    ),
    pytest.param(
        dict(Any=(), Not=()), ValueError, id='multiple-vertices-Any-Not'
    ),
    pytest.param(
        dict(), ValueError, id='empty-dict'
    ),
])
def test_expand_path_cfg_raise(path_cfg, error):
    with pytest.raises(error):
        expand_path_cfg(path_cfg=path_cfg)

##__________________________________________________________________||
<commit_msg>add test_factory() to test_expand_path_cfg.py<commit_after># Tai Sakuma <tai.sakuma@gmail.com>
import os
import sys
import pytest

from alphatwirl.selection.factories.expand import expand_path_cfg

from alphatwirl.selection.factories.factory import FactoryDispatcher
from alphatwirl.selection.modules.LambdaStr import LambdaStr
from alphatwirl.selection.modules import All, Any, Not

##__________________________________________________________________||
# path_cfg, expanded, obj
params = [
    pytest.param(
        'ev : ev.nJets[0] >= 2',
        dict(
            factory='LambdaStrFactory',
            lambda_str='ev : ev.nJets[0] >= 2',
        ),
        LambdaStr(
            name='ev : ev.nJets[0] >= 2',
            lambda_str='ev : ev.nJets[0] >= 2',
        ),
        id='string:lambda_str'
    ),
    pytest.param(
        'ev : ev.nJets[0] >= {n}',
        dict(
            factory='LambdaStrFactory',
            lambda_str='ev : ev.nJets[0] >= {n}',
        ),
        LambdaStr(
            name='ev : ev.nJets[0] >= 5242',
            lambda_str='ev : ev.nJets[0] >= 5242',
        ),
        id='string:lambda_str-not-formatted'
    ),
    pytest.param(
        dict(All=()),
        {'factory': 'AllFactory', 'path_cfg_list': ()},
        All(name='All', selections=[]),
        id='dict-all-empty'
    ),
    pytest.param(
        dict(Any=()),
        {'factory': 'AnyFactory', 'path_cfg_list': ()},
        Any(name='Any', selections=[]),
        id='dict-any-empty'
    ),
    pytest.param(
        dict(Any=(
            'ev : ev.x[0] == 0',
            dict(All=(
                'ev : ev.x[0] >= 1',
                'ev : ev.y[0] >= 100',
                )),
            dict(Not=dict(
                Any=(
                    'ev : ev.z[0] == 0',
                    'ev : ev.w[0] >= 300',
                ),
            )),
        )),
        dict(
            factory='AnyFactory',
            path_cfg_list=(
                dict(
                    factory='LambdaStrFactory',
                    lambda_str='ev : ev.x[0] == 0',
                ),
                dict(
                    factory='AllFactory',
                    path_cfg_list=(
                        dict(
                            factory='LambdaStrFactory',
                            lambda_str='ev : ev.x[0] >= 1',
                        ),
                        dict(
                            factory='LambdaStrFactory',
                            lambda_str='ev : ev.y[0] >= 100',
                        )
                    )
                ),
                dict(
                    factory='NotFactory',
                    path_cfg=dict(
                        factory='AnyFactory',
                        path_cfg_list=(
                            dict(
                                factory='LambdaStrFactory',
                                lambda_str='ev : ev.z[0] == 0'
                            ),
                            dict(
                                factory='LambdaStrFactory',
                                lambda_str='ev : ev.w[0] >= 300',
                                ),
                        )
                    )
                )
            )
        ),
        Any(
            name='Any',
            selections=[
                LambdaStr(
                    name='ev : ev.x[0] == 0',
                    lambda_str='ev : ev.x[0] == 0'
                ),
                All(
                    name='All',
                    selections=[
                        LambdaStr(
                            name='ev : ev.x[0] >= 1',
                            lambda_str='ev : ev.x[0] >= 1'),
                        LambdaStr(
                            name='ev : ev.y[0] >= 100',
                            lambda_str='ev : ev.y[0] >= 100')
                    ]
                ),
                Not(
                    name='Not',
                    selection=Any(
                        name='Any',
                        selections=[
                            LambdaStr(
                                name='ev : ev.z[0] == 0',
                                lambda_str='ev : ev.z[0] == 0'
                            ),
                            LambdaStr(
                                name='ev : ev.w[0] >= 300',
                                lambda_str='ev : ev.w[0] >= 300'
                            )
                        ]
                    )
                )
            ]
        ),
        id='example',
        ## marks=pytest.mark.skip(reason='not fully expanded')
    ),
]

@pytest.mark.parametrize('path_cfg, expected, _', params)
def test_expand_path_cfg(path_cfg, expected, _):
    actual = expand_path_cfg(path_cfg=path_cfg)
    assert expected == actual

    # give expanded one
    actual = expand_path_cfg(path_cfg=actual)
    assert expected == actual

@pytest.mark.parametrize('path_cfg, _, expected', params)
def test_factory(path_cfg, _, expected):

    kargs = dict(
        AllClass=All, AnyClass=Any, NotClass=Not,
        LambdaStrClass=LambdaStr,
        n=5242,
    )

    obj = FactoryDispatcher(path_cfg=path_cfg, **kargs)
    assert repr(expected) == repr(obj)
    assert str(expected) == str(obj)

##__________________________________________________________________||
@pytest.mark.parametrize('path_cfg, error', [
    pytest.param(
        dict(All=(), Any=()), ValueError, id='multiple-vertices-All-Any'
    ),
    pytest.param(
        dict(All=(), Not=()), ValueError, id='multiple-vertices-All-Not'
    ),
    pytest.param(
        dict(Any=(), Not=()), ValueError, id='multiple-vertices-Any-Not'
    ),
    pytest.param(
        dict(), ValueError, id='empty-dict'
    ),
])
def test_expand_path_cfg_raise(path_cfg, error):
    with pytest.raises(error):
        expand_path_cfg(path_cfg=path_cfg)

##__________________________________________________________________||
<|endoftext|>


<commit_before>import eventlet
import json
from flask import request, json, Flask
from st2reactor.sensor.base import Sensor

eventlet.monkey_patch(
    os=True,
    select=True,
    socket=True,
    thread=True,
    time=True)

class SmartThingsSensor(Sensor):
    def __init__(self, sensor_service, config=None):
        super(SmartThingsSensor, self).__init__(sensor_service=sensor_service,
                                                config=config)

        self._listen_ip = self._config.get('listen_ip', '0.0.0.0')
        self._listen_port = self._config.get('listen_port', '12000')
        self._api_key = self._config.get('api_key', None)

        self._trigger = 'smartthings.event'
        self._app = Flask(__name__)

    def setup(self):
        pass

    def run(self):
        if not self._api_key:
            raise Exception('[smartthings_sensor]: api_key config option not set')

        @self._app.route('/', methods=['PUT'])
        def process_incoming():
            if request.headers['X-Api-Key'] == self._api_key:
                return self._process_request(request)
            else:
                abort(401)

        self._app.run(host=self._listen_ip, port=self._listen_port)

    def cleanup(self):
        pass

    def add_trigger(self, trigger):
        pass

    def update_trigger(self, trigger):
        pass

    def remove_trigger(self, trigger):
        pass

    def _process_request(self, request):
        if request.headers['Content-Type'] == 'application/json':
            payload = json.dumps(request.json)
            self._sensor_service.dispatch(trigger=self._trigger, payload=payload)
        else:
            return "415 Unsupported Media Type"
<commit_msg>handle incoming payload from smartthings<commit_after>import eventlet
import json
from flask import request, json, Flask, Response
from st2reactor.sensor.base import Sensor

eventlet.monkey_patch(
    os=True,
    select=True,
    socket=True,
    thread=True,
    time=True)

class SmartThingsSensor(Sensor):
    def __init__(self, sensor_service, config=None):
        super(SmartThingsSensor, self).__init__(sensor_service=sensor_service,
                                                config=config)

        self._trigger = 'smartthings.event'
        self._logger = self._sensor_service.get_logger(__name__)

        self._listen_ip = self._config.get('listen_ip', '0.0.0.0')
        self._listen_port = self._config.get('listen_port', '12000')
        self._api_key = self._config.get('api_key', None)

        self._app = Flask(__name__)

    def setup(self):
        pass

    def run(self):
        if not self._api_key:
            raise Exception('[smartthings_sensor]: api_key config option not set')

        @self._app.route('/', methods=['PUT'])
        def process_incoming():
            response = None
            if request.headers['X-Api-Key'] == self._api_key:
		status = self._process_request(request)
                response = Response(status[0], status=status[1])
            else:
                response = Response('fail', status=401)

            return response

        self._app.run(host=self._listen_ip, port=self._listen_port)

    def cleanup(self):
        pass

    def add_trigger(self, trigger):
        pass

    def update_trigger(self, trigger):
        pass

    def remove_trigger(self, trigger):
        pass

    def _process_request(self, request):
        if request.headers['Content-Type'] == 'application/json':
            payload = request.json
            self._logger.debug('[smartthings_sensor]: processing request {}'.format(payload))

            self._sensor_service.dispatch(trigger=self._trigger, payload=payload)
            return ('ok', 200)
        else:
            return ('fail', 415)

<|endoftext|>