<commit_before># -*- encoding: utf-8 -*-
import collections
import re
import six
from discograph.library.Artist import Artist
from discograph.library.ArtistRole import ArtistRole
from discograph.library.Label import Label
from discograph.library.SQLArtist import SQLArtist
from discograph.library.SQLLabel import SQLLabel
from discograph.library.SQLRelation import SQLRelation

class RelationGrapher(object):

    ### CLASS VARIABLES ###

    word_pattern = re.compile('\s+')

    ### INITIALIZER ###

    def __init__(
        self,
        center_entity,
        cache=None,
        degree=3,
        max_links=None,
        max_nodes=None,
        role_names=None,
        ):
        prototype = (Artist, Label, SQLArtist, SQLLabel)
        assert isinstance(center_entity, prototype)
        self.center_entity = center_entity
        degree = int(degree)
        assert 0 < degree
        self.degree = degree
        self.cache = cache
        if max_links is not None:
            max_links = int(max_links)
            assert 0 < max_links
        self.max_links = max_links
        if max_nodes is not None:
            max_nodes = int(max_nodes)
            assert 0 < max_nodes
        self.max_nodes = max_nodes
        role_names = role_names or None
        if role_names:
            if isinstance(role_names, six.string_types):
                role_names = (role_names,)
            elif not isinstance(role_names, collections.Iterable):
                role_names = (role_names,)
            role_names = tuple(role_names)
            assert all(_ in ArtistRole._available_credit_roles
                for _ in role_names)
        self.role_names = role_names

    @classmethod
    def get_link_key(cls, link):
        source_type, source_id = link['source']
        if source_type == 1:
            source_type = 'artist'
        else:
            source_type = 'label'
        target_type, target_id = link['target']
        if target_type == 1:
            target_type = 'artist'
        else:
            target_type = 'label'
        return '{}-{}-{}-{}-{}'.format(
            source_type,
            source_id,
            cls.word_pattern.sub('-', link['role']).lower(),
            target_type,
            target_id,
            )

    def relation_to_link(self, relation):
        link = relation.copy()
        entity_one_id = link['entity_one_id']
        entity_one_type = link['entity_one_type']
        #entity_one_type = Relation.EntityType(entity_one_type)
        #entity_one_type = entity_one_type.name.lower()
        source_key = (entity_one_type, entity_one_id)
        link['source'] = source_key
        entity_two_id = link['entity_two_id']
        entity_two_type = link['entity_two_type']
        #entity_two_type = Relation.EntityType(entity_two_type)
        #entity_two_type = entity_two_type.name.lower()
        target_key = (entity_two_type, entity_two_id)
        link['target'] = target_key
        link['role'] = link['role_name']
        link['key'] = self.get_link_key(link)
        if '_id' in link: del(link['_id'])
        if 'country' in link: del(link['country'])
        if 'entity_one_id' in link: del(link['entity_one_id'])
        if 'entity_one_type' in link: del(link['entity_one_type'])
        if 'entity_two_id' in link: del(link['entity_two_id'])
        if 'entity_two_type' in link: del(link['entity_two_type'])
        if 'id' in link: del(link['id'])
        if 'role_name' in link: del(link['role_name'])
        if 'category' in link and not link.get('category'):
            del(link['category'])
        if 'subcategory' in link and not link.get('subcategory'):
            del(link['subcategory'])
        if 'genres' in link and not link.get('genres'):
            del(link['genres'])
        if 'random' in link:
            del(link['random'])
        if 'release_id' in link and not link.get('release_id'):
            del(link['release_id'])
        if 'styles' in link and not link.get('styles'):
            del(link['styles'])
        if 'year' in link and not link.get('year'):
            del(link['year'])
        return link

    def entity_key_to_node(self, entity_key, distance):
        node = dict(distance=distance, missing=0, members=set(), aliases=set())
        node['id'] = entity_key[1]
        if entity_key[0] == 1:
            node['type'] = 'artist'
        else:
            node['type'] = 'label'
        node['key'] = '{}-{}'.format(node['type'], node['id'])
        node['links'] = set()
        return node

    def collect_entities_2(self):
        original_role_names = self.role_names or ()
        provisional_role_names = set(original_role_names)
        provisional_role_names.update(['Alias', 'Member Of'])
        provisional_role_names = sorted(provisional_role_names)

        if type(self.center_entity).__name__.endswith('Artist'):
            initial_key = (1, self.center_entity.discogs_id)
        else:
            initial_key = (2, self.center_entity.discogs_id)
        entity_keys_to_visit = set([initial_key])

        links = dict()
        nodes = dict()

        entity_query_cap = 999
        entity_query_cap -= (1 + len(provisional_role_names)) * 2
        entity_query_cap //= 2

        for distance in range(self.degree + 1):
            current_entity_keys_to_visit = list(entity_keys_to_visit)
            for key in current_entity_keys_to_visit:
                nodes.setdefault(key, self.entity_key_to_node(key, distance))
            entity_keys_to_visit.clear()
            print('    Searching...', len(nodes), len(links))
            if self.max_nodes and self.max_nodes < len(nodes):
                break
            relations = []
            for i in range(0, len(current_entity_keys_to_visit), entity_query_cap):
                # Split into multiple queries to avoid variable maximum.
                entity_key_slice = current_entity_keys_to_visit[i:i + entity_query_cap]
                relations.extend(SQLRelation.search_multi(
                    entity_key_slice,
                    role_names=provisional_role_names,
                    ))
            for relation in relations:
                e1k = (relation['entity_one_type'], relation['entity_one_id'])
                e2k = (relation['entity_two_type'], relation['entity_two_id'])
                if e1k not in nodes:
                    entity_keys_to_visit.add(e1k)
                    nodes[e1k] = self.entity_key_to_node(e1k, distance + 1)
                if e2k not in nodes:
                    entity_keys_to_visit.add(e2k)
                    nodes[e2k] = self.entity_key_to_node(e2k, distance + 1)
                if relation['role_name'] == 'Alias':
                    nodes[e1k]['aliases'].add(e2k[1])
                    nodes[e2k]['aliases'].add(e1k[1])
                elif relation['role_name'] in ('Member Of', 'Sublabel Of'):
                    nodes[e2k]['members'].add(e1k[1])
                if relation['role_name'] not in original_role_names:
                    continue
                link = self.relation_to_link(relation)
                link['distance'] = min(
                    nodes[e1k]['distance'],
                    nodes[e2k]['distance'],
                    )
                links[link['key']] = link
                nodes[e1k]['links'].add(link['key'])
                nodes[e2k]['links'].add(link['key'])

        # Query node names.
        artist_ids = []
        label_ids = []
        for entity_type, entity_id in nodes.keys():
            if entity_type == 1:
                artist_ids.append(entity_id)
            else:
                label_ids.append(entity_id)
        artists = []
        for i in range(0, len(artist_ids), 999):
            query = (SQLArtist
                .select()
                .where(SQLArtist.id.in_(artist_ids[i:i + 999]))
                )
            artists.extend(query)
        labels = []
        for i in range(0, len(artist_ids), 999):
            query = (SQLLabel
                .select()
                .where(SQLLabel.id.in_(label_ids[i:i + 999]))
                )
            labels.extend(query)
        for artist in artists:
            nodes[(1, artist.id)]['name'] = artist.name
        for label in labels:
            nodes[(2, label.id)]['name'] = label.name

        # Prune nameless nodes.
        for node in tuple(nodes.values()):
            if not node.get('name'):
                self.prune_node(node, nodes, links, update_missing_count=False)
        print('    Pruning nameless...', len(nodes), len(links))

        # Prune unvisited nodes and links.
        for key in entity_keys_to_visit:
            node = nodes.get(key)
            self.prune_node(node, nodes, links)
        print('    Pruning unvisited...', len(nodes), len(links))

        # Prune nodes beyond maximum.
        if self.max_nodes:
            nodes_to_prune = sorted(nodes.values(),
                key=lambda x: (x['distance'], x['id']),
                )[self.max_nodes:]
            for node in nodes_to_prune:
                self.prune_node(node, nodes, links)
        print('    Pruning max nodes...', len(nodes), len(links))

        # Prune links beyond maximum.
        if self.max_links:
            links_to_prune = sorted(links.values(),
                key=self.link_sorter,
                )[self.max_links:]
            for link in links_to_prune:
                print(self.link_sorter(link))
                self.prune_link(link, nodes, links)
        print('    Pruning max links...', len(nodes), len(links))

        print('Found:', len(nodes), len(links))
        return nodes, links

    def prune_link(self, link, nodes, links, update_missing_count=True):
        if link is None:
            return
        if link['key'] in links:
            del(links[link['key']])
        source_node = nodes.get(link['source'])
        if source_node is not None:
            if update_missing_count:
                source_node['missing'] += 1
            source_node['links'].remove(link['key'])
            if not source_node['links']:
                self.prune_node(source_node, nodes, links,
                    update_missing_count=update_missing_count)
        target_node = nodes.get(link['target'])
        if target_node is not None:
            if update_missing_count:
                target_node['missing'] += 1
            target_node['links'].remove(link['key'])
            if not target_node['links']:
                self.prune_node(target_node, nodes, links,
                    update_missing_count=update_missing_count)

    def prune_node(self, node, nodes, links, update_missing_count=True):
        if node is None:
            return
        if node['type'] == 'artist':
            key = (1, node['id'])
        else:
            key = (2, node['id'])
        if key in nodes:
            del(nodes[key])
        for link_key in node['links'].copy():
            link = links.get(link_key)
            self.prune_link(link, nodes, links,
                update_missing_count=update_missing_count)

    def get_network_2(self):
        nodes, links = self.collect_entities_2()
        cluster_count = 0
        cluster_map = {}
        for node in nodes.values():
            cluster = None
            if node['aliases']:
                if node['id'] not in cluster_map:
                    cluster_count += 1
                    cluster_map[node['id']] = cluster_count
                    for alias_id in node['aliases']:
                        cluster_map[alias_id] = cluster_count
                cluster = cluster_map[node['id']]
            if not node['aliases']:
                del(node['aliases'])
            else:
                node['aliases'] = tuple(sorted(node['aliases']))
            if cluster is not None:
                node['cluster'] = cluster
            node['size'] = len(node.pop('members'))
            node['links'] = tuple(sorted(node['links']))
        links = tuple(sorted(links.values(),
            key=lambda x: (x['source'], x['role'], x['target'], x.get('release_id'))))
        for link in links:
            if link['source'][0] == 1:
                link['source'] = 'artist-{}'.format(link['source'][1])
            else:
                link['source'] = 'label-{}'.format(link['source'][1])
            if link['target'][0] == 1:
                link['target'] = 'artist-{}'.format(link['target'][1])
            else:
                link['target'] = 'label-{}'.format(link['target'][1])
        nodes = tuple(sorted(nodes.values(), key=lambda x: (x['type'], x['id'])))
        if type(self.center_entity) in (Artist, SQLArtist):
            center = 'artist-{}'.format(self.center_entity.discogs_id)
        else:
            center = 'label-{}'.format(self.center_entity.discogs_id)
        network = {
            'center': center,
            'nodes': nodes,
            'links': links,
            }
        return network

    @staticmethod
    def link_sorter(link):
        role = 2
        if link['role'] == 'Alias':
            role = 0
        elif link['role'] == 'Member Of':
            role = 1
        return link['distance'], role, link['key']<commit_msg>Improved messaging in RelationGrapher.<commit_after># -*- encoding: utf-8 -*-
import collections
import re
import six
from discograph.library.Artist import Artist
from discograph.library.ArtistRole import ArtistRole
from discograph.library.Label import Label
from discograph.library.SQLArtist import SQLArtist
from discograph.library.SQLLabel import SQLLabel
from discograph.library.SQLRelation import SQLRelation

class RelationGrapher(object):

    ### CLASS VARIABLES ###

    word_pattern = re.compile('\s+')

    ### INITIALIZER ###

    def __init__(
        self,
        center_entity,
        cache=None,
        degree=3,
        max_links=None,
        max_nodes=None,
        role_names=None,
        ):
        prototype = (Artist, Label, SQLArtist, SQLLabel)
        assert isinstance(center_entity, prototype)
        self.center_entity = center_entity
        degree = int(degree)
        assert 0 < degree
        self.degree = degree
        self.cache = cache
        if max_links is not None:
            max_links = int(max_links)
            assert 0 < max_links
        self.max_links = max_links
        if max_nodes is not None:
            max_nodes = int(max_nodes)
            assert 0 < max_nodes
        self.max_nodes = max_nodes
        role_names = role_names or None
        if role_names:
            if isinstance(role_names, six.string_types):
                role_names = (role_names,)
            elif not isinstance(role_names, collections.Iterable):
                role_names = (role_names,)
            role_names = tuple(role_names)
            assert all(_ in ArtistRole._available_credit_roles
                for _ in role_names)
        self.role_names = role_names

    @classmethod
    def get_link_key(cls, link):
        source_type, source_id = link['source']
        if source_type == 1:
            source_type = 'artist'
        else:
            source_type = 'label'
        target_type, target_id = link['target']
        if target_type == 1:
            target_type = 'artist'
        else:
            target_type = 'label'
        return '{}-{}-{}-{}-{}'.format(
            source_type,
            source_id,
            cls.word_pattern.sub('-', link['role']).lower(),
            target_type,
            target_id,
            )

    def relation_to_link(self, relation):
        link = relation.copy()
        entity_one_id = link['entity_one_id']
        entity_one_type = link['entity_one_type']
        #entity_one_type = Relation.EntityType(entity_one_type)
        #entity_one_type = entity_one_type.name.lower()
        source_key = (entity_one_type, entity_one_id)
        link['source'] = source_key
        entity_two_id = link['entity_two_id']
        entity_two_type = link['entity_two_type']
        #entity_two_type = Relation.EntityType(entity_two_type)
        #entity_two_type = entity_two_type.name.lower()
        target_key = (entity_two_type, entity_two_id)
        link['target'] = target_key
        link['role'] = link['role_name']
        link['key'] = self.get_link_key(link)
        if '_id' in link: del(link['_id'])
        if 'country' in link: del(link['country'])
        if 'entity_one_id' in link: del(link['entity_one_id'])
        if 'entity_one_type' in link: del(link['entity_one_type'])
        if 'entity_two_id' in link: del(link['entity_two_id'])
        if 'entity_two_type' in link: del(link['entity_two_type'])
        if 'id' in link: del(link['id'])
        if 'role_name' in link: del(link['role_name'])
        if 'category' in link and not link.get('category'):
            del(link['category'])
        if 'subcategory' in link and not link.get('subcategory'):
            del(link['subcategory'])
        if 'genres' in link and not link.get('genres'):
            del(link['genres'])
        if 'random' in link:
            del(link['random'])
        if 'release_id' in link and not link.get('release_id'):
            del(link['release_id'])
        if 'styles' in link and not link.get('styles'):
            del(link['styles'])
        if 'year' in link and not link.get('year'):
            del(link['year'])
        return link

    def entity_key_to_node(self, entity_key, distance):
        node = dict(distance=distance, missing=0, members=set(), aliases=set())
        node['id'] = entity_key[1]
        if entity_key[0] == 1:
            node['type'] = 'artist'
        else:
            node['type'] = 'label'
        node['key'] = '{}-{}'.format(node['type'], node['id'])
        node['links'] = set()
        return node

    def collect_entities_2(self):
        original_role_names = self.role_names or ()
        provisional_role_names = set(original_role_names)
        provisional_role_names.update(['Alias', 'Member Of'])
        provisional_role_names = sorted(provisional_role_names)

        if type(self.center_entity).__name__.endswith('Artist'):
            initial_key = (1, self.center_entity.discogs_id)
        else:
            initial_key = (2, self.center_entity.discogs_id)
        entity_keys_to_visit = set([initial_key])

        links = dict()
        nodes = dict()

        entity_query_cap = 999
        entity_query_cap -= (1 + len(provisional_role_names)) * 2
        entity_query_cap //= 2

        for distance in range(self.degree + 1):
            current_entity_keys_to_visit = list(entity_keys_to_visit)
            for key in current_entity_keys_to_visit:
                nodes.setdefault(key, self.entity_key_to_node(key, distance))
            print('    At distance {}:'.format(distance))
            print('        {} new nodes'.format(len(current_entity_keys_to_visit)))
            print('        {} old nodes'.format(len(nodes) - len(current_entity_keys_to_visit)))
            print('        {} old links'.format(len(links)))

            entity_keys_to_visit.clear()
            if self.max_nodes and self.max_nodes < len(nodes):
                print('        Maxed out node count.')
                break
            relations = []
            for i in range(0, len(current_entity_keys_to_visit), entity_query_cap):
                # Split into multiple queries to avoid variable maximum.
                print('        Querying: {} to {} of {} new nodes'.format(
                    i, i + entity_query_cap, len(current_entity_keys_to_visit)
                    ))
                entity_key_slice = current_entity_keys_to_visit[i:i + entity_query_cap]
                relations.extend(SQLRelation.search_multi(
                    entity_key_slice,
                    role_names=provisional_role_names,
                    ))
            for relation in relations:
                e1k = (relation['entity_one_type'], relation['entity_one_id'])
                e2k = (relation['entity_two_type'], relation['entity_two_id'])
                if e1k not in nodes:
                    entity_keys_to_visit.add(e1k)
                    nodes[e1k] = self.entity_key_to_node(e1k, distance + 1)
                if e2k not in nodes:
                    entity_keys_to_visit.add(e2k)
                    nodes[e2k] = self.entity_key_to_node(e2k, distance + 1)
                if relation['role_name'] == 'Alias':
                    nodes[e1k]['aliases'].add(e2k[1])
                    nodes[e2k]['aliases'].add(e1k[1])
                elif relation['role_name'] in ('Member Of', 'Sublabel Of'):
                    nodes[e2k]['members'].add(e1k[1])
                if relation['role_name'] not in original_role_names:
                    continue
                link = self.relation_to_link(relation)
                link['distance'] = min(
                    nodes[e1k]['distance'],
                    nodes[e2k]['distance'],
                    )
                links[link['key']] = link
                nodes[e1k]['links'].add(link['key'])
                nodes[e2k]['links'].add(link['key'])

        print('    Collected: {} / {}'.format(len(nodes), len(links)))

        # Query node names.
        artist_ids = []
        label_ids = []
        for entity_type, entity_id in nodes.keys():
            if entity_type == 1:
                artist_ids.append(entity_id)
            else:
                label_ids.append(entity_id)
        artists = []
        for i in range(0, len(artist_ids), 999):
            query = (SQLArtist
                .select()
                .where(SQLArtist.id.in_(artist_ids[i:i + 999]))
                )
            artists.extend(query)
        labels = []
        for i in range(0, len(artist_ids), 999):
            query = (SQLLabel
                .select()
                .where(SQLLabel.id.in_(label_ids[i:i + 999]))
                )
            labels.extend(query)
        for artist in artists:
            nodes[(1, artist.id)]['name'] = artist.name
        for label in labels:
            nodes[(2, label.id)]['name'] = label.name

        # Prune nameless nodes.
        for node in tuple(nodes.values()):
            if not node.get('name'):
                self.prune_node(node, nodes, links, update_missing_count=False)
        print('    Pruning nameless: {} / {}'.format(len(nodes), len(links)))

        # Prune unvisited nodes and links.
        for key in entity_keys_to_visit:
            node = nodes.get(key)
            self.prune_node(node, nodes, links)
        print('    Pruned unvisited: {} / {}'.format(len(nodes), len(links)))

        # Prune nodes beyond maximum.
        if self.max_nodes:
            nodes_to_prune = sorted(nodes.values(),
                key=lambda x: (x['distance'], x['id']),
                )[self.max_nodes:]
            for node in nodes_to_prune:
                self.prune_node(node, nodes, links)
        print('    Pruned by max nodes: {} / {}'.format(len(nodes), len(links)))

        # Prune links beyond maximum.
        if self.max_links:
            links_to_prune = sorted(links.values(),
                key=self.link_sorter,
                )[self.max_links:]
            for link in links_to_prune:
                print(self.link_sorter(link))
                self.prune_link(link, nodes, links)
        print('    Pruned by max links: {} / {}'.format(len(nodes), len(links)))

        print('Finally: {} / {}'.format(len(nodes), len(links)))
        return nodes, links

    def prune_link(self, link, nodes, links, update_missing_count=True):
        if link is None:
            return
        if link['key'] in links:
            del(links[link['key']])
        source_node = nodes.get(link['source'])
        if source_node is not None:
            if update_missing_count:
                source_node['missing'] += 1
            source_node['links'].remove(link['key'])
            if not source_node['links']:
                self.prune_node(source_node, nodes, links,
                    update_missing_count=update_missing_count)
        target_node = nodes.get(link['target'])
        if target_node is not None:
            if update_missing_count:
                target_node['missing'] += 1
            target_node['links'].remove(link['key'])
            if not target_node['links']:
                self.prune_node(target_node, nodes, links,
                    update_missing_count=update_missing_count)

    def prune_node(self, node, nodes, links, update_missing_count=True):
        if node is None:
            return
        if node['type'] == 'artist':
            key = (1, node['id'])
        else:
            key = (2, node['id'])
        if key in nodes:
            del(nodes[key])
        for link_key in node['links'].copy():
            link = links.get(link_key)
            self.prune_link(link, nodes, links,
                update_missing_count=update_missing_count)

    def get_network_2(self):
        nodes, links = self.collect_entities_2()
        cluster_count = 0
        cluster_map = {}
        for node in nodes.values():
            cluster = None
            if node['aliases']:
                if node['id'] not in cluster_map:
                    cluster_count += 1
                    cluster_map[node['id']] = cluster_count
                    for alias_id in node['aliases']:
                        cluster_map[alias_id] = cluster_count
                cluster = cluster_map[node['id']]
            if not node['aliases']:
                del(node['aliases'])
            else:
                node['aliases'] = tuple(sorted(node['aliases']))
            if cluster is not None:
                node['cluster'] = cluster
            node['size'] = len(node.pop('members'))
            node['links'] = tuple(sorted(node['links']))
        links = tuple(sorted(links.values(),
            key=lambda x: (x['source'], x['role'], x['target'], x.get('release_id'))))
        for link in links:
            if link['source'][0] == 1:
                link['source'] = 'artist-{}'.format(link['source'][1])
            else:
                link['source'] = 'label-{}'.format(link['source'][1])
            if link['target'][0] == 1:
                link['target'] = 'artist-{}'.format(link['target'][1])
            else:
                link['target'] = 'label-{}'.format(link['target'][1])
        nodes = tuple(sorted(nodes.values(), key=lambda x: (x['type'], x['id'])))
        if type(self.center_entity) in (Artist, SQLArtist):
            center = 'artist-{}'.format(self.center_entity.discogs_id)
        else:
            center = 'label-{}'.format(self.center_entity.discogs_id)
        network = {
            'center': center,
            'nodes': nodes,
            'links': links,
            }
        return network

    @staticmethod
    def link_sorter(link):
        role = 2
        if link['role'] == 'Alias':
            role = 0
        elif link['role'] == 'Member Of':
            role = 1
        return link['distance'], role, link['key']<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
System-level utilities and helper functions.
"""

from datetime import datetime, timedelta
import inspect
import logging
import os
import random
import subprocess
import socket
import sys

from nova import exception
from nova import flags

FLAGS = flags.FLAGS
TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"

def import_class(import_str):
    """Returns a class from a string including module and class"""
    mod_str, _sep, class_str = import_str.rpartition('.')
    try:
        __import__(mod_str)
        return getattr(sys.modules[mod_str], class_str)
    except (ImportError, AttributeError):
        raise exception.NotFound('Class %s cannot be found' % class_str)

def fetchfile(url, target):
    logging.debug("Fetching %s" % url)
#    c = pycurl.Curl()
#    fp = open(target, "wb")
#    c.setopt(c.URL, url)
#    c.setopt(c.WRITEDATA, fp)
#    c.perform()
#    c.close()
#    fp.close()
    execute("curl %s -o %s" % (url, target))

def execute(cmd, input=None, addl_env=None, check_exit_code=True):
    env = os.environ.copy()
    if addl_env:
        env.update(addl_env)
    obj = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
        stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
    result = None
    if input != None:
        result = obj.communicate(input)
    else:
        result = obj.communicate()
    obj.stdin.close()
    if obj.returncode:
        logging.debug("Result was %s" % (obj.returncode))
        if check_exit_code and obj.returncode <> 0:
            raise Exception("Unexpected exit code: %s.  result=%s" % (obj.returncode, result))
    return result

def abspath(s):
    return os.path.join(os.path.dirname(__file__), s)

def default_flagfile(filename='nova.conf'):
    for arg in sys.argv:
        if arg.find('flagfile') != -1:
            break
    else:
        if not os.path.isabs(filename):
            # turn relative filename into an absolute path
            script_dir = os.path.dirname(inspect.stack()[-1][1])
            filename = os.path.abspath(os.path.join(script_dir, filename))
        if os.path.exists(filename):
            sys.argv = sys.argv[:1] + ['--flagfile=%s' % filename] + sys.argv[1:]

def debug(arg):
    logging.debug('debug in callback: %s', arg)
    return arg

def runthis(prompt, cmd, check_exit_code = True):
    logging.debug("Running %s" % (cmd))
    exit_code = subprocess.call(cmd.split(" "))
    logging.debug(prompt % (exit_code))
    if check_exit_code and exit_code <> 0:
        raise Exception("Unexpected exit code: %s from cmd: %s" % (exit_code, cmd))

def generate_uid(topic, size=8):
    return '%s-%s' % (topic, ''.join([random.choice('01234567890abcdefghijklmnopqrstuvwxyz') for x in xrange(size)]))

def generate_mac():
    mac = [0x02, 0x16, 0x3e, random.randint(0x00, 0x7f),
           random.randint(0x00, 0xff), random.randint(0x00, 0xff)
           ]
    return ':'.join(map(lambda x: "%02x" % x, mac))

def last_octet(address):
    return int(address.split(".")[-1])

def get_my_ip():
    ''' returns the actual ip of the local machine.
    '''
    if getattr(FLAGS, 'fake_tests', None):
        return '127.0.0.1'
    csock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    csock.connect(('www.google.com', 80))
    (addr, port) = csock.getsockname()
    csock.close()
    return addr

def isotime(at=None):
    if not at:
        at = datetime.utcnow()
    return at.strftime(TIME_FORMAT)

def parse_isotime(timestr):
    return datetime.strptime(timestr, TIME_FORMAT)
<commit_msg>Added --fail argument to curl invocations, so that HTTP request fails get surfaced as non-zero exit codes<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
System-level utilities and helper functions.
"""

from datetime import datetime, timedelta
import inspect
import logging
import os
import random
import subprocess
import socket
import sys

from nova import exception
from nova import flags

FLAGS = flags.FLAGS
TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"

def import_class(import_str):
    """Returns a class from a string including module and class"""
    mod_str, _sep, class_str = import_str.rpartition('.')
    try:
        __import__(mod_str)
        return getattr(sys.modules[mod_str], class_str)
    except (ImportError, AttributeError):
        raise exception.NotFound('Class %s cannot be found' % class_str)

def fetchfile(url, target):
    logging.debug("Fetching %s" % url)
#    c = pycurl.Curl()
#    fp = open(target, "wb")
#    c.setopt(c.URL, url)
#    c.setopt(c.WRITEDATA, fp)
#    c.perform()
#    c.close()
#    fp.close()
    execute("curl --fail %s -o %s" % (url, target))

def execute(cmd, input=None, addl_env=None, check_exit_code=True):
    env = os.environ.copy()
    if addl_env:
        env.update(addl_env)
    obj = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
        stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
    result = None
    if input != None:
        result = obj.communicate(input)
    else:
        result = obj.communicate()
    obj.stdin.close()
    if obj.returncode:
        logging.debug("Result was %s" % (obj.returncode))
        if check_exit_code and obj.returncode <> 0:
            raise Exception("Unexpected exit code: %s.  result=%s" % (obj.returncode, result))
    return result

def abspath(s):
    return os.path.join(os.path.dirname(__file__), s)

def default_flagfile(filename='nova.conf'):
    for arg in sys.argv:
        if arg.find('flagfile') != -1:
            break
    else:
        if not os.path.isabs(filename):
            # turn relative filename into an absolute path
            script_dir = os.path.dirname(inspect.stack()[-1][1])
            filename = os.path.abspath(os.path.join(script_dir, filename))
        if os.path.exists(filename):
            sys.argv = sys.argv[:1] + ['--flagfile=%s' % filename] + sys.argv[1:]

def debug(arg):
    logging.debug('debug in callback: %s', arg)
    return arg

def runthis(prompt, cmd, check_exit_code = True):
    logging.debug("Running %s" % (cmd))
    exit_code = subprocess.call(cmd.split(" "))
    logging.debug(prompt % (exit_code))
    if check_exit_code and exit_code <> 0:
        raise Exception("Unexpected exit code: %s from cmd: %s" % (exit_code, cmd))

def generate_uid(topic, size=8):
    return '%s-%s' % (topic, ''.join([random.choice('01234567890abcdefghijklmnopqrstuvwxyz') for x in xrange(size)]))

def generate_mac():
    mac = [0x02, 0x16, 0x3e, random.randint(0x00, 0x7f),
           random.randint(0x00, 0xff), random.randint(0x00, 0xff)
           ]
    return ':'.join(map(lambda x: "%02x" % x, mac))

def last_octet(address):
    return int(address.split(".")[-1])

def get_my_ip():
    ''' returns the actual ip of the local machine.
    '''
    if getattr(FLAGS, 'fake_tests', None):
        return '127.0.0.1'
    csock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    csock.connect(('www.google.com', 80))
    (addr, port) = csock.getsockname()
    csock.close()
    return addr

def isotime(at=None):
    if not at:
        at = datetime.utcnow()
    return at.strftime(TIME_FORMAT)

def parse_isotime(timestr):
    return datetime.strptime(timestr, TIME_FORMAT)
<|endoftext|>


<commit_before>from io import BytesIO
from datetime import datetime
import logging
import pickle
import tarfile
import time
from uuid import uuid4

import numpy as np

from sklearn.base import BaseEstimator, ClassifierMixin
from sklearn.exceptions import NotFittedError

from ..utils import load_dictionary_from_file
from ..utils import Timer
from ..utils import parse_n_jobs

__all__ = ['BaseClassifier', 'load_classifier', 'get_thresholds_from_file']

logger = logging.getLogger(__name__)

class BaseClassifier(BaseEstimator, ClassifierMixin):
    def __init__(self, n_jobs=1, **kwargs):
        super(BaseClassifier, self).__init__()

        self.n_jobs = parse_n_jobs(n_jobs)
        logger.info('Using n_jobs={} for <{}>.'.format(self.n_jobs, self.name))
    #end def

    def fit(self, X, Y, **kwargs):
        self.uuid_ = str(uuid4())
        logger.debug('{} UUID is {}.'.format(self.name, self.uuid_))

        self.fitted_at_ = datetime.utcnow()

        timer = Timer()
        self._fit(X, Y, **kwargs)
        logger.info('{} fitting on {} instances complete {}.'.format(self.name, X.shape[0], timer))

        return self
    #end def

    def _fit(self, *args, **kwargs): raise NotImplementedError('_fit is not implemented.')

    def predict_proba(self, X_featurized, *, thresholds=0.5, denominators=None, rescale=False, **kwargs):
        timer = Timer()
        Y_proba = self._predict_proba(X_featurized, **kwargs)
        logger.debug('Computed prediction probabilities on {} instances {}.'.format(X_featurized.shape[0], timer))

        if not rescale: return Y_proba

        if isinstance(thresholds, (float, np.float)):
            thresholds = np.full(Y_proba.shape[1], thresholds)

        if denominators is None:
            denominators = 2.0 * (1.0 - thresholds)

        rescaled = np.zeros(Y_proba.shape)
        for i in range(Y_proba.shape[0]):
            for k in range(Y_proba.shape[1]):
                if Y_proba[i, k] > thresholds[k]: rescaled[i, k] = 0.5 + ((Y_proba[i, k] - thresholds[k]) / denominators[k])
                else: rescaled[i, k] = Y_proba[i, k] / (thresholds[k] + thresholds[k])
            #end for
        #end for

        return rescaled
    #end def

    def _predict_proba(self, X_featurized, **kwargs):
        raise NotImplementedError('_predict_proba is not implemented.')

    def predict(self, X_featurized, thresholds=0.5, **kwargs):
        _, Y_predict = self.predict_and_proba(X_featurized, thresholds=thresholds, rescale=False, **kwargs)

        return Y_predict
    #end def

    def predict_and_proba(self, X_featurized, *, thresholds=0.5, rescale=False, **kwargs):
        Y_proba = self.predict_proba(X_featurized, thresholds=thresholds, rescale=rescale, **kwargs)

        if rescale: Y_predict = Y_proba >= 0.5
        else:
            Y_predict = np.zeros(Y_proba.shape, dtype=np.bool)
            if thresholds is None: thresholds = np.full(Y_proba.shape[1], 0.5)
            for i in range(Y_proba.shape[0]):
                Y_predict[i, :] = Y_proba[i, :] >= thresholds
        #end def

        return Y_proba, Y_predict
    #end def

    def decision_function(self, *args, **kwargs): return self.predict_proba(*args, **kwargs)

    def save(self, f):
        if not hasattr(self, 'uuid_'):
            raise NotFittedError('This featurizer is not fitted yet.')

        with tarfile.open(fileobj=f, mode='w') as tf:
            with BytesIO() as model_f:
                pickle.dump(self, model_f, protocol=4)
                model_data = model_f.getvalue()
                model_f.seek(0)
                model_tarinfo = tarfile.TarInfo(name='model.pkl')
                model_tarinfo.size = len(model_data)
                model_tarinfo.mtime = int(time.time())
                tf.addfile(tarinfo=model_tarinfo, fileobj=model_f)
            #end with

            self.save_to_tarfile(tf)
        #end with

        logger.info('{} saved to <{}>.'.format(self, f.name))

        return self
    #end def

    def save_to_tarfile(self, tf): return self

    def load_from_tarfile(self, tf): return self

    @property
    def uuid(self):
        return self.uuid_
    #end def

    @property
    def name(self):
        return type(self).__name__

    @property
    def classes_(self): raise NotImplementedError('classes_ is not implemented.')

    def __str__(self):
        return '{}(UUID={})'.format(self.name, self.uuid_ if hasattr(self, 'uuid_') else 'None')
#end class

def load_classifier(f):
    with tarfile.open(fileobj=f, mode='r') as tf:
        classifier = pickle.load(tf.extractfile('model.pkl'))
        classifier.load_from_tarfile(tf)
    #end with

    logger.info('Loaded {} from <{}>.'.format(classifier, f.name))

    return classifier
#end def

def get_thresholds_from_file(f, classes, *, default=0.5):
    d = load_dictionary_from_file(f)
    return np.array([float(d.get(c, default)) for c in classes])
#end def
<commit_msg>Flushclassifier save<commit_after>from io import BytesIO
from datetime import datetime
import logging
import pickle
import tarfile
import time
from uuid import uuid4

import numpy as np

from sklearn.base import BaseEstimator, ClassifierMixin
from sklearn.exceptions import NotFittedError

from ..utils import load_dictionary_from_file
from ..utils import Timer
from ..utils import parse_n_jobs

__all__ = ['BaseClassifier', 'load_classifier', 'get_thresholds_from_file']

logger = logging.getLogger(__name__)

class BaseClassifier(BaseEstimator, ClassifierMixin):
    def __init__(self, n_jobs=1, **kwargs):
        super(BaseClassifier, self).__init__()

        self.n_jobs = parse_n_jobs(n_jobs)
        logger.info('Using n_jobs={} for <{}>.'.format(self.n_jobs, self.name))
    #end def

    def fit(self, X, Y, **kwargs):
        self.uuid_ = str(uuid4())
        logger.debug('{} UUID is {}.'.format(self.name, self.uuid_))

        self.fitted_at_ = datetime.utcnow()

        timer = Timer()
        self._fit(X, Y, **kwargs)
        logger.info('{} fitting on {} instances complete {}.'.format(self.name, X.shape[0], timer))

        return self
    #end def

    def _fit(self, *args, **kwargs): raise NotImplementedError('_fit is not implemented.')

    def predict_proba(self, X_featurized, *, thresholds=0.5, denominators=None, rescale=False, **kwargs):
        timer = Timer()
        Y_proba = self._predict_proba(X_featurized, **kwargs)
        logger.debug('Computed prediction probabilities on {} instances {}.'.format(X_featurized.shape[0], timer))

        if not rescale: return Y_proba

        if isinstance(thresholds, (float, np.float)):
            thresholds = np.full(Y_proba.shape[1], thresholds)

        if denominators is None:
            denominators = 2.0 * (1.0 - thresholds)

        rescaled = np.zeros(Y_proba.shape)
        for i in range(Y_proba.shape[0]):
            for k in range(Y_proba.shape[1]):
                if Y_proba[i, k] > thresholds[k]: rescaled[i, k] = 0.5 + ((Y_proba[i, k] - thresholds[k]) / denominators[k])
                else: rescaled[i, k] = Y_proba[i, k] / (thresholds[k] + thresholds[k])
            #end for
        #end for

        return rescaled
    #end def

    def _predict_proba(self, X_featurized, **kwargs):
        raise NotImplementedError('_predict_proba is not implemented.')

    def predict(self, X_featurized, thresholds=0.5, **kwargs):
        _, Y_predict = self.predict_and_proba(X_featurized, thresholds=thresholds, rescale=False, **kwargs)

        return Y_predict
    #end def

    def predict_and_proba(self, X_featurized, *, thresholds=0.5, rescale=False, **kwargs):
        Y_proba = self.predict_proba(X_featurized, thresholds=thresholds, rescale=rescale, **kwargs)

        if rescale: Y_predict = Y_proba >= 0.5
        else:
            Y_predict = np.zeros(Y_proba.shape, dtype=np.bool)
            if thresholds is None: thresholds = np.full(Y_proba.shape[1], 0.5)
            for i in range(Y_proba.shape[0]):
                Y_predict[i, :] = Y_proba[i, :] >= thresholds
        #end def

        return Y_proba, Y_predict
    #end def

    def decision_function(self, *args, **kwargs): return self.predict_proba(*args, **kwargs)

    def save(self, f):
        if not hasattr(self, 'uuid_'):
            raise NotFittedError('This featurizer is not fitted yet.')

        with tarfile.open(fileobj=f, mode='w') as tf:
            with BytesIO() as model_f:
                pickle.dump(self, model_f, protocol=4)
                model_data = model_f.getvalue()
                model_f.seek(0)
                model_tarinfo = tarfile.TarInfo(name='model.pkl')
                model_tarinfo.size = len(model_data)
                model_tarinfo.mtime = int(time.time())
                tf.addfile(tarinfo=model_tarinfo, fileobj=model_f)
            #end with

            self.save_to_tarfile(tf)
        #end with

        f.flush()

        logger.info('{} saved to <{}>.'.format(self, f.name))

        return self
    #end def

    def save_to_tarfile(self, tf): return self

    def load_from_tarfile(self, tf): return self

    @property
    def uuid(self):
        return self.uuid_
    #end def

    @property
    def name(self):
        return type(self).__name__

    @property
    def classes_(self): raise NotImplementedError('classes_ is not implemented.')

    def __str__(self):
        return '{}(UUID={})'.format(self.name, self.uuid_ if hasattr(self, 'uuid_') else 'None')
#end class

def load_classifier(f):
    with tarfile.open(fileobj=f, mode='r') as tf:
        classifier = pickle.load(tf.extractfile('model.pkl'))
        classifier.load_from_tarfile(tf)
    #end with

    logger.info('Loaded {} from <{}>.'.format(classifier, f.name))

    return classifier
#end def

def get_thresholds_from_file(f, classes, *, default=0.5):
    d = load_dictionary_from_file(f)
    return np.array([float(d.get(c, default)) for c in classes])
#end def
<|endoftext|>


<commit_before>from threading import Thread
from flask import current_app, render_template
from flask_mail import Message
from . import mail

def send_async_email(app, msg):
    with app.app_context():
        mail.send(msg)

def send_email(to, subject, template, **kwargs):
    app = current_app._get_current_object()
    msg = Message(
        subject=app.config['WEATHER_APP_MAIL_SUBJECT_PREFIX'] + ' ' + subject,
        recipients=[to],
        sender=app.config['WEATHER_APP_MAIL_SENDER'])
    msg.body = render_template(template + '.txt', **kwargs)
    msg.html = render_template(template + '.html', **kwargs)
    thr = Thread(target=send_async_email, args=[app, msg])
    thr.start()
    return thr
<commit_msg>Add email send fallback<commit_after>from threading import Thread
from flask import current_app, render_template
from flask_mail import Message
from . import mail
from .models import User
from app import db

def send_async_email(app, msg):
    with app.app_context():
        try:
            mail.send(msg)
        except:
            """If confirmation email cannot be send, user is confirmed automatically.
            I assume that senior dev won't have SSL configured locally so he (or she)
            won't set his email and password in environment variables for security reasons.
            To keep app usable, I confirm user without email but password reset
            and email change will not work."""
            recipients = msg.recipients
            for email in recipients:
                user = User.query.filter_by(email=email).first()
                user.confirmed = True
                db.session.add(user)
                db.session.commit()
                print('User ' + user.email + ' automatically confirmed.')

def send_email(to, subject, template, **kwargs):
    app = current_app._get_current_object()
    msg = Message(
        subject=app.config['WEATHER_APP_MAIL_SUBJECT_PREFIX'] + ' ' + subject,
        recipients=[to],
        sender=app.config['WEATHER_APP_MAIL_SENDER'])
    msg.body = render_template(template + '.txt', **kwargs)
    msg.html = render_template(template + '.html', **kwargs)
    thr = Thread(target=send_async_email, args=[app, msg])
    thr.start()
    return thr
<|endoftext|>


<commit_before>import re

from django.contrib.postgres.search import SearchQuery, SearchRank
from django.db.models import Count, F

from people.models import Person

def search_person_by_name(name, synonym=True):
    name = name.lower()
    name = re.sub(r"[^a-z ]", "", name)
    name = " ".join(name.strip().split())
    and_name = " & ".join(name.split(" "))
    or_name = " | ".join(name.split(" "))
    name = f"({and_name}) | ({or_name})"
    query = SearchQuery(name, search_type="raw", config="english")

    search_args = {}
    if synonym:
        search_args["name_search_vector__synonym"] = query
    else:
        search_args["name_search_vector"] = query

    qs = (
        Person.objects.annotate(membership_count=Count("memberships"))
        .filter(**search_args)
        .annotate(rank=SearchRank(F("name_search_vector"), query))
        .order_by("-rank", "membership_count")
        .defer("biography", "versions")
    )

    return qs
<commit_msg>Make sure we can search names with '<commit_after>import re

from django.contrib.postgres.search import SearchQuery, SearchRank
from django.db.models import Count, F

from people.models import Person

def search_person_by_name(name, synonym=True):
    name = name.lower()
    name = re.sub(r"[^a-z ]", " ", name)
    name = " ".join(name.strip().split())
    and_name = " & ".join(name.split(" "))
    or_name = " | ".join(name.split(" "))
    name = f"({and_name}) | ({or_name})"
    query = SearchQuery(name, search_type="raw", config="english")

    search_args = {}
    if synonym:
        search_args["name_search_vector__synonym"] = query
    else:
        search_args["name_search_vector"] = query

    qs = (
        Person.objects.annotate(membership_count=Count("memberships"))
        .filter(**search_args)
        .annotate(rank=SearchRank(F("name_search_vector"), query))
        .order_by("-rank", "membership_count")
        .defer("biography", "versions")
    )

    return qs
<|endoftext|>


<commit_before>import numpy

__author__ = 'Romain Tavenard romain.tavenard[at]univ-rennes2.fr'

def random_walks(n_ts=100, sz=256, d=1, mu=0., std=1.):
    """Random walk time series generator.

    Generate ``n_ts`` time series of size ``sz`` and dimensionality ``d``.
    Generated time series follow the model:
    $$ts[t] = ts[t - 1] + a$$
    where :math:`a` is drawn from a normal distribution of mean ``mu`` and standard deviation ``std``.

    Parameters
    ----------
    n_ts : int (default: 100)
        Number of time series
    sz : int (default: 256)
        Length of time series (number of time instants)
    d : int (default: 1)
        Dimensionality of time series
    mu : float (default: 0.)
        Mean of the normal distribution from which random walk steps are drawn
    std : float (default: 1.)
        Standard deviation of the normal distribution from which random walk steps are drawn

    Returns
    -------
    numpy.ndarray
        A dataset of random walk time series
    """
    ts = numpy.empty((n_ts, sz, d))
    rnd = numpy.random.randn(n_ts, sz, d) * std + mu
    ts[:, 0, :] = rnd[:, 0, :]
    for t in range(1, sz):
        ts[:, t, :] = ts[:, t - 1, :] + rnd[:, t, :]
    return ts

def random_walk_blobs(n_ts_per_blob=100, sz=256, d=1, n_blobs=2, noise_level=1.):
    """Blob-based random walk time series generator.

    Generate ``n_ts_per_blobs`` * ``n_blobs`` time series of size ``sz`` and dimensionality ``d``.
    Generated time series follow the model:
    $$ts[t] = ts[t - 1] + a$$
    where :math:`a` is drawn from a normal distribution of mean ``mu`` and standard deviation ``std``.

    Each blob contains time series derived from a same seed time series with added white noise at level ``noise_level``.

    Parameters
    ----------
    n_ts_per_blob : int (default: 100)
        Number of time series in each blob
    sz : int (default: 256)
        Length of time series (number of time instants)
    d : int (default: 1)
        Dimensionality of time series
    n_blobs : int (default: 2)
        Number of blobs
    noise_level : float (default: 1.)
        Standard deviation of white noise added to time series in each blob

    Returns
    -------
    numpy.ndarray
        A dataset of random walk time series
    """
    base_ts = random_walks(n_ts=n_blobs, sz=sz, d=d, std=1.0)
    rnd = numpy.random.randn(n_ts_per_blob * n_blobs, sz, d) * noise_level
    ts = numpy.repeat(base_ts, repeats=n_ts_per_blob, axis=0)
    y = numpy.repeat(range(n_blobs), repeats=n_ts_per_blob)
    return ts + rnd, y
<commit_msg>Doc harmonization<commit_after>import numpy

__author__ = 'Romain Tavenard romain.tavenard[at]univ-rennes2.fr'

def random_walks(n_ts=100, sz=256, d=1, mu=0., std=1.):
    """Random walk time series generator.

    Generate n_ts time series of size sz and dimensionality d.
    Generated time series follow the model:
    $$ts[t] = ts[t - 1] + a$$
    where :math:`a` is drawn from a normal distribution of mean mu and standard deviation std.

    Parameters
    ----------
    n_ts : int (default: 100)
        Number of time series
    sz : int (default: 256)
        Length of time series (number of time instants)
    d : int (default: 1)
        Dimensionality of time series
    mu : float (default: 0.)
        Mean of the normal distribution from which random walk steps are drawn
    std : float (default: 1.)
        Standard deviation of the normal distribution from which random walk steps are drawn

    Returns
    -------
    numpy.ndarray
        A dataset of random walk time series
    """
    ts = numpy.empty((n_ts, sz, d))
    rnd = numpy.random.randn(n_ts, sz, d) * std + mu
    ts[:, 0, :] = rnd[:, 0, :]
    for t in range(1, sz):
        ts[:, t, :] = ts[:, t - 1, :] + rnd[:, t, :]
    return ts

def random_walk_blobs(n_ts_per_blob=100, sz=256, d=1, n_blobs=2, noise_level=1.):
    """Blob-based random walk time series generator.

    Generate n_ts_per_blobs * n_blobs time series of size sz and dimensionality d.
    Generated time series follow the model:
    $$ts[t] = ts[t - 1] + a$$
    where :math:`a` is drawn from a normal distribution of mean mu and standard deviation std.

    Each blob contains time series derived from a same seed time series with added white noise.

    Parameters
    ----------
    n_ts_per_blob : int (default: 100)
        Number of time series in each blob
    sz : int (default: 256)
        Length of time series (number of time instants)
    d : int (default: 1)
        Dimensionality of time series
    n_blobs : int (default: 2)
        Number of blobs
    noise_level : float (default: 1.)
        Standard deviation of white noise added to time series in each blob

    Returns
    -------
    numpy.ndarray
        A dataset of random walk time series
    """
    base_ts = random_walks(n_ts=n_blobs, sz=sz, d=d, std=1.0)
    rnd = numpy.random.randn(n_ts_per_blob * n_blobs, sz, d) * noise_level
    ts = numpy.repeat(base_ts, repeats=n_ts_per_blob, axis=0)
    y = numpy.repeat(range(n_blobs), repeats=n_ts_per_blob)
    return ts + rnd, y
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import sys
import yaml
import argparse
try:
    import json
except:
    import simplejson as json

from ansible.inventory import expand_hosts as expand

def scan_inv_directory():
    """Function scans inventory directory for all files ending with .yml
    and tries to parse them into Python dict object."""
    invhosts = {}
    invfiles = os.listdir('inventory')

    # Scan every yml file found inside inventory directory
    for invfile in [x for x in invfiles if x.endswith('.yml')]:
        with open('inventory/{}'.format(invfile), 'r') as inventory:
            try:
                hosts = yaml.safe_load(inventory)
            except yaml.scanner.ScannerError as e:
                print('Inventory syntax error {}'.format(str(e.context_mark).lstrip()))
                sys.exit(1)

        # If there are hosts defined with range indicator [1:3], expand them
        hosts_expanded = {}
        for key, val in hosts.iteritems():
            if expand.detect_range(key):
                key = expand.expand_hostname_range(key)
                hosts_expanded.update({x:val for x in key})
            else:
                hosts_expanded[key] = val

        invhosts.update(hosts_expanded)

    if not invhosts:
        print('No hosts found.')
        sys.exit(1)
    return invhosts

def produce_json_object_list(invhosts):
    """Function parses Python dictionary object representing YAML data
    and produces valid JSON object for Ansible."""
    inventory = {}
    for key, val in invhosts.iteritems():
        groups = val.get('groups', None).replace(' ', '').split(',')
        for group in groups:
            if group in inventory.keys():
                inventory[group].append(key)
            else:
                inventory[group] = [key]

    return json.dumps(inventory, indent=2)

def produce_json_object_host(invhosts, host):
    """Function parses Python dictionary object representing YAML data
    and produces valid JSON object for Ansible."""
    host_vars = invhosts.get(host, {})
    return json.dumps(host_vars, indent=2)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Alternative YAML dynamic inventory')
    parser.add_argument('--list', default=False, dest="list", action="store_true",
            help="Produce a JSON consumable grouping of servers")
    parser.add_argument('--host', default=None, dest="host",
            help="Generate additional host specific details for given host")
    args = parser.parse_args()

    if args.list:
        print produce_json_object_list(scan_inv_directory())

    if args.host:
        print produce_json_object_host(scan_inv_directory(), args.host)
<commit_msg>include _meta dictionary within --list option output<commit_after>#!/usr/bin/env python

import os
import sys
import yaml
import argparse
try:
    import json
except:
    import simplejson as json

from ansible.inventory import expand_hosts as expand

def scan_inv_directory():
    """Function scans inventory directory for all files ending with .yml
    and tries to parse them into Python dict object."""
    invhosts = {}
    invfiles = os.listdir('inventory')

    # Scan every yml file found inside inventory directory
    for invfile in [x for x in invfiles if x.endswith('.yml')]:
        with open('inventory/{}'.format(invfile), 'r') as inventory:
            try:
                hosts = yaml.safe_load(inventory)
            except yaml.scanner.ScannerError as e:
                print('Inventory syntax error {}'.format(str(e.context_mark).lstrip()))
                sys.exit(1)

        # If there are hosts defined with range indicator [1:3], expand them
        hosts_expanded = {}
        for key, val in hosts.iteritems():
            if expand.detect_range(key):
                key = expand.expand_hostname_range(key)
                hosts_expanded.update({x:val for x in key})
            else:
                hosts_expanded[key] = val

        invhosts.update(hosts_expanded)

    if not invhosts:
        print('No hosts found.')
        sys.exit(1)
    return invhosts

def produce_json_object_list(invhosts):
    """Function parses Python dictionary object representing YAML data
    and produces valid JSON object for Ansible."""
    inventory = {'_meta': {'hostvars': {}}}
    for key, val in invhosts.iteritems():
        groups = val.get('groups', None).replace(' ', '').split(',')
        for group in groups:
            if group in inventory.keys():
                inventory[group].append(key)
            else:
                inventory[group] = [key]

        inventory['_meta']['hostvars'][key] = val

    return json.dumps(inventory, indent=2)

def produce_json_object_host(invhosts, host):
    """Function parses Python dictionary object representing YAML data
    and produces valid JSON object for Ansible."""
    host_vars = invhosts.get(host, {})
    return json.dumps(host_vars, indent=2)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Alternative YAML dynamic inventory')
    parser.add_argument('--list', default=False, dest="list", action="store_true",
            help="Produce a JSON consumable grouping of servers")
    parser.add_argument('--host', default=None, dest="host",
            help="Generate additional host specific details for given host")
    args = parser.parse_args()

    if args.list:
        print produce_json_object_list(scan_inv_directory())

    if args.host:
        print produce_json_object_host(scan_inv_directory(), args.host)
<|endoftext|>


<commit_before>import asyncio
import logging
import inspect
from copy import copy
from enum import Enum
from uuid import uuid4
from datetime import datetime, timezone

from tukio.event import EventSource
from tukio.utils import FutureState, SkipTask
from tukio.broker import get_broker, workflow_exec_topics

from .task import TaskRegistry

log = logging.getLogger(__name__)

class TaskExecState(Enum):

    BEGIN = 'task-begin'
    END = 'task-end'
    ERROR = 'task-error'
    SKIP = 'task-skip'
    PROGRESS = 'task-progress'
    TIMEOUT = 'task-timeout'

    @classmethod
    def values(cls):
        return [member.value for member in cls]

    @classmethod
    def from_exception(cls, exc):
        if isinstance(exc, SkipTask):
            return cls.SKIP
        else:
            return cls.ERROR

class TukioTaskError(Exception):

    def __init__(self, output):
        self.task_outputs = output

class TukioTask(asyncio.Task):

    """
    A simple subclass of `asyncio.Task()` to add an execution ID and optionally
    bind a task holder class.
    """

    __slots__ = (
        'holder', 'uid', '_broker', '_in_progress', '_template', '_workflow',
        '_source', '_start', '_end', '_inputs', '_outputs', '_queue',
        '_commited', '_timed_out',
    )

    def __init__(self, coro, *, loop=None):
        super().__init__(coro, loop=loop)
        self.holder = inspect.getcoroutinelocals(coro).get('self')
        try:
            self.uid = self.holder.uid
        except AttributeError:
            self.uid = str(uuid4())
        self._broker = get_broker(self._loop)
        self._in_progress = False
        self._template = None
        self._workflow = None
        self._source = None
        self._start = None
        self._end = None
        self._inputs = None
        self._outputs = None
        self._queue = asyncio.Queue(loop=self._loop)
        if self.holder:
            self.holder.queue = self._queue
        # A 'committed' task is a pending task not suspended
        self._committed = asyncio.Event()
        self._committed.set()
        self._timed_out = False

    @property
    def inputs(self):
        return self._inputs

    @inputs.setter
    def inputs(self, data):
        # Freeze input data (dict or event)
        self._inputs = copy(data)

    @property
    def outputs(self):
        return self._outputs

    @property
    def template(self):
        return self._template

    @property
    def workflow(self):
        return self._workflow

    @property
    def event_source(self):
        return self._source

    @property
    def queue(self):
        return self._queue

    @property
    def committed(self):
        return self._committed.is_set()

    @property
    def timed_out(self):
        return self._timed_out

    async def timeout(self):
        """
        Timeout the task, cancelling its execution.
        """
        self.cancel()
        self._timed_out = True
        if self.holder:
            self._outputs = await self.holder.teardown() or self._inputs
        else:
            self._outputs = self._inputs

    def suspend(self):
        """
        Suspend a task means, for now, to cancel the task but flag it as
        'suspended'. It shall be executed from scratch upon workflow resume.

        TODO: improve this method to avoid cancelling the task but to actually
        suspend transactionnal jobs inside the task.
        """
        self.cancel()
        self._committed.clear()

    def as_dict(self):
        """
        Returns the execution informations of this task.
        """
        return {
            'id': self.uid,
            'start': self._start,
            'end': self._end,
            'state': FutureState.get(self).value,
            'inputs': self._inputs,
            'outputs': self._outputs
        }

    async def data_received(self, event):
        """
        A handler that puts events into the task's own receiving queue. This
        handler shall be registered in the data broker so that any tukio task
        can receive and process events during execution.
        """
        await self._queue.put(event)

    def in_progress(self):
        """
        Returns True if the task execution started, else returns False.
        """
        return self._in_progress

    def result(self):
        """
        Wrapper around `Future.result()` to automatically dispatch a
        `TaskExecState.END` or `TaskExecState.ERROR` event.
        """
        try:
            result = super().result()
        except asyncio.CancelledError:
            # Task cancelled, do nothing
            if self._timed_out is False:
                raise
            # Task timed out
            self._end = datetime.now(timezone.utc)
            self._broker.dispatch(
                {'type': TaskExecState.TIMEOUT.value, 'content': self._outputs},
                topics=workflow_exec_topics(self._source._workflow_exec_id),
                source=self._source,
            )
            raise asyncio.TimeoutError
        except Exception as exc:
            self._end = datetime.now(timezone.utc)
            etype = TaskExecState.from_exception(exc)
            if isinstance(exc, TukioTaskError):
                content = exc.task_outputs
                self._outputs = content
            else:
                content = {'exception': exc}
            self._broker.dispatch(
                {'type': etype.value, 'content': content},
                topics=workflow_exec_topics(self._source._workflow_exec_id),
                source=self._source,
            )
            raise
        else:
            # Freeze output data (dict or event)
            self._outputs = copy(result)
            self._end = datetime.now(timezone.utc)
            self._broker.dispatch(
                {'type': TaskExecState.END.value, 'content': self._outputs},
                topics=workflow_exec_topics(self._source._workflow_exec_id),
                source=self._source,
            )
            return result

    def dispatch_progress(self, data, event_type=None):
        """
        Dispatch task progress events in the 'EXEC_TOPIC' from
        this tukio task.
        """
        if event_type is None:
            event_type = TaskExecState.PROGRESS.value
        event_data = {'type': event_type, 'content': data}
        self._broker.dispatch(
            event_data,
            topics=workflow_exec_topics(self._source._workflow_exec_id),
            source=self._source,
        )

    def _step(self, exc=None):
        """
        Wrapper around `Task._step()` to automatically dispatch a
        `TaskExecState.BEGIN` event.
        """
        if not self._in_progress:
            self._start = datetime.now(timezone.utc)
            source = {'task_exec_id': self.uid}
            if self._template:
                source['task_template_id'] = self._template.uid
            if self._workflow:
                source['workflow_template_id'] = self._workflow.template.uid
                source['workflow_exec_id'] = self._workflow.uid
            self._source = EventSource(**source)
            self._in_progress = True
            data = {
                'type': TaskExecState.BEGIN.value,
                'content': self._inputs
            }
            self._broker.dispatch(
                data,
                topics=workflow_exec_topics(self._source._workflow_exec_id),
                source=self._source,
            )
        super()._step(exc)

def tukio_factory(loop, coro):
    """
    A task factory for asyncio that creates `TukioTask()` instances for all
    coroutines registered as Tukio tasks and default `asyncio.Task()` instances
    for all others.
    """
    try:
        # Trigger exception if not valid
        TaskRegistry.codes()[coro.cr_code]
        task = TukioTask(coro, loop=loop)
    except (KeyError, AttributeError):
        # When the coroutine is not a registered Tukio task or when `coro` is a
        # simple generator (e.g. upon calling `asyncio.wait()`)
        task = asyncio.Task(coro, loop=loop)
    return task
<commit_msg>Timeout a task after retrieving its outputs<commit_after>import asyncio
import logging
import inspect
from copy import copy
from enum import Enum
from uuid import uuid4
from datetime import datetime, timezone

from tukio.event import EventSource
from tukio.utils import FutureState, SkipTask
from tukio.broker import get_broker, workflow_exec_topics

from .task import TaskRegistry

log = logging.getLogger(__name__)

class TaskExecState(Enum):

    BEGIN = 'task-begin'
    END = 'task-end'
    ERROR = 'task-error'
    SKIP = 'task-skip'
    PROGRESS = 'task-progress'
    TIMEOUT = 'task-timeout'

    @classmethod
    def values(cls):
        return [member.value for member in cls]

    @classmethod
    def from_exception(cls, exc):
        if isinstance(exc, SkipTask):
            return cls.SKIP
        else:
            return cls.ERROR

class TukioTaskError(Exception):

    def __init__(self, output):
        self.task_outputs = output

class TukioTask(asyncio.Task):

    """
    A simple subclass of `asyncio.Task()` to add an execution ID and optionally
    bind a task holder class.
    """

    __slots__ = (
        'holder', 'uid', '_broker', '_in_progress', '_template', '_workflow',
        '_source', '_start', '_end', '_inputs', '_outputs', '_queue',
        '_commited', '_timed_out',
    )

    def __init__(self, coro, *, loop=None):
        super().__init__(coro, loop=loop)
        self.holder = inspect.getcoroutinelocals(coro).get('self')
        try:
            self.uid = self.holder.uid
        except AttributeError:
            self.uid = str(uuid4())
        self._broker = get_broker(self._loop)
        self._in_progress = False
        self._template = None
        self._workflow = None
        self._source = None
        self._start = None
        self._end = None
        self._inputs = None
        self._outputs = None
        self._queue = asyncio.Queue(loop=self._loop)
        if self.holder:
            self.holder.queue = self._queue
        # A 'committed' task is a pending task not suspended
        self._committed = asyncio.Event()
        self._committed.set()
        self._timed_out = False

    @property
    def inputs(self):
        return self._inputs

    @inputs.setter
    def inputs(self, data):
        # Freeze input data (dict or event)
        self._inputs = copy(data)

    @property
    def outputs(self):
        return self._outputs

    @property
    def template(self):
        return self._template

    @property
    def workflow(self):
        return self._workflow

    @property
    def event_source(self):
        return self._source

    @property
    def queue(self):
        return self._queue

    @property
    def committed(self):
        return self._committed.is_set()

    @property
    def timed_out(self):
        return self._timed_out

    async def timeout(self):
        """
        Timeout the task, cancelling its execution.
        """
        if self.holder:
            self._outputs = await self.holder.teardown() or self._inputs
        else:
            self._outputs = self._inputs
        # Cancel after retrieving the task's outputs.
        self.cancel()
        self._timed_out = True

    def suspend(self):
        """
        Suspend a task means, for now, to cancel the task but flag it as
        'suspended'. It shall be executed from scratch upon workflow resume.

        TODO: improve this method to avoid cancelling the task but to actually
        suspend transactionnal jobs inside the task.
        """
        self.cancel()
        self._committed.clear()

    def as_dict(self):
        """
        Returns the execution informations of this task.
        """
        return {
            'id': self.uid,
            'start': self._start,
            'end': self._end,
            'state': FutureState.get(self).value,
            'inputs': self._inputs,
            'outputs': self._outputs
        }

    async def data_received(self, event):
        """
        A handler that puts events into the task's own receiving queue. This
        handler shall be registered in the data broker so that any tukio task
        can receive and process events during execution.
        """
        await self._queue.put(event)

    def in_progress(self):
        """
        Returns True if the task execution started, else returns False.
        """
        return self._in_progress

    def result(self):
        """
        Wrapper around `Future.result()` to automatically dispatch a
        `TaskExecState.END` or `TaskExecState.ERROR` event.
        """
        try:
            result = super().result()
        except asyncio.CancelledError:
            # Task cancelled, do nothing
            if self._timed_out is False:
                raise
            # Task timed out
            self._end = datetime.now(timezone.utc)
            self._broker.dispatch(
                {'type': TaskExecState.TIMEOUT.value, 'content': self._outputs},
                topics=workflow_exec_topics(self._source._workflow_exec_id),
                source=self._source,
            )
            raise asyncio.TimeoutError
        except Exception as exc:
            self._end = datetime.now(timezone.utc)
            etype = TaskExecState.from_exception(exc)
            if isinstance(exc, TukioTaskError):
                content = exc.task_outputs
                self._outputs = content
            else:
                content = {'exception': exc}
            self._broker.dispatch(
                {'type': etype.value, 'content': content},
                topics=workflow_exec_topics(self._source._workflow_exec_id),
                source=self._source,
            )
            raise
        else:
            # Freeze output data (dict or event)
            self._outputs = copy(result)
            self._end = datetime.now(timezone.utc)
            self._broker.dispatch(
                {'type': TaskExecState.END.value, 'content': self._outputs},
                topics=workflow_exec_topics(self._source._workflow_exec_id),
                source=self._source,
            )
            return result

    def dispatch_progress(self, data, event_type=None):
        """
        Dispatch task progress events in the 'EXEC_TOPIC' from
        this tukio task.
        """
        if event_type is None:
            event_type = TaskExecState.PROGRESS.value
        event_data = {'type': event_type, 'content': data}
        self._broker.dispatch(
            event_data,
            topics=workflow_exec_topics(self._source._workflow_exec_id),
            source=self._source,
        )

    def _step(self, exc=None):
        """
        Wrapper around `Task._step()` to automatically dispatch a
        `TaskExecState.BEGIN` event.
        """
        if not self._in_progress:
            self._start = datetime.now(timezone.utc)
            source = {'task_exec_id': self.uid}
            if self._template:
                source['task_template_id'] = self._template.uid
            if self._workflow:
                source['workflow_template_id'] = self._workflow.template.uid
                source['workflow_exec_id'] = self._workflow.uid
            self._source = EventSource(**source)
            self._in_progress = True
            data = {
                'type': TaskExecState.BEGIN.value,
                'content': self._inputs
            }
            self._broker.dispatch(
                data,
                topics=workflow_exec_topics(self._source._workflow_exec_id),
                source=self._source,
            )
        super()._step(exc)

def tukio_factory(loop, coro):
    """
    A task factory for asyncio that creates `TukioTask()` instances for all
    coroutines registered as Tukio tasks and default `asyncio.Task()` instances
    for all others.
    """
    try:
        # Trigger exception if not valid
        TaskRegistry.codes()[coro.cr_code]
        task = TukioTask(coro, loop=loop)
    except (KeyError, AttributeError):
        # When the coroutine is not a registered Tukio task or when `coro` is a
        # simple generator (e.g. upon calling `asyncio.wait()`)
        task = asyncio.Task(coro, loop=loop)
    return task
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
The purpose of numerical simulations is to simulate growing and dividing cells:
a process must be defined to sample instantaneous, single cell growth rate, to
relate the instantaneous growth rate (in units of time) to cell size, and then
to sample size regulation through division process.

This module deals with the following parameterization:

* global simulation parameters such as duration of experiment, number of
  samples, etc... are defined in :class:`SimuParams`,
* division control is defined in :class:`DivParams`,
* size sampling (only initial size sampling is needed) in
  :class:`SampleInitialSize`,

and with the :class:`Ecoli` objects that derive from
:class:`tunacell.base.cell.Cell` objects, only with precise information
regarding precise division timing.

The associated module, :module:`ou.py` contains classes and functions to
simulate the instantaneous, single cell growth rate as the
Ornstein-Uhlenbeck process (stationary Gaussian process). Both modules can be
used jointly to perform complete, tunable numerical simulations of growing
and dividing bacteria.
"""
from __future__ import print_function

import numpy as np

from tunacell.base.cell import CellError
from tunacell.base.cell import Cell

class SimuParams(object):
    """Stores parameters for simulation parameters

    Parameters
    ----------
    nbr_container : int
        Number of simulated containers
    nbr_colony_per_container : int
        Number of colonies per container
    start : float
        time at which simulation starts. Root cell age at initial time is set
        randomly. Root cell birth time is then less or equal to start time.
    stop : float
        time at which simulation stops. Leaf cell division time is greater or
        equal to stop time.
    period : float
        time interval at which measurements are recorded (sampling).
    seed : int (default None)
        seed for random number generation (given to numpy.random.seed())

    Notes
    -----
    A very informal note on the number of samples:
        A sample is a tree of cells. The tree depth depends on two things:
            - the difference between start and stop times
            - the average cell cycle time (set in DivisionParams)
        The number of evaluated values for an exaclty sampled process (e.g.
        the Ornstein Uhlenbeck process) is inversely proportional to the
        period at which data is recorded.

        The total number of operations depends:
            - exponentially on the difference 'stop - start'
            - inversely proportionally to 'period'
            - proportionally on nbr_container
            - proportionally on nbr_colony_per_container
    """

    def __init__(self, nbr_container=1, nbr_colony_per_container=1,
                 start=0., stop=100., period=5., seed=None):
        self.nbr_container = nbr_container
        self.nbr_colony_per_container = nbr_colony_per_container
        self.start = start
        self.stop = stop
        self.period = period
        self.seed = seed
        # metadata helper
        self.content = [('nbr_container', nbr_container),
                        ('nbr_colony_per_container', nbr_colony_per_container),
                        ('start', start),
                        ('stop', stop),
                        ('period', period),
                        ('seed', seed)]
        return

    def __str__(self):
        msg = ('Simulation parameters:\n'
               '----------------------\n')
        left_column_size = 25
        right_column_size = 10
        for key, val in self.content:
            msg += '{}'.format(key).ljust(left_column_size)
            msg += ': ' + '{}\n'.format(val).rjust(right_column_size)
        return msg

class DivisionParams(object):
    """Sets parameters for computing interdivision times.

    This settings is based on the linear results explained elsewhere.

    Parameters
    ----------
    div_lambda : float between 0 and 1
        lambda parameter (note trailing): mixing parameter between timer (0)
        and size (1). Adder is obtained with lambda_ 0.5.
    div_size_target : float
        X^* set the cutoff size for division. When lambda_ is 1, and there are
        no fluctuations, division is triggered when size reaches exactly X^*
        (fixed final size).
    div_mode : str {'gamma', 'none'}
        'fixed': only the linear response is taken;
        'gamma': adds fluctuations over linear response (needs kwargs)
    div_sd_to_mean : float (default 0.1)
        sets the standard deviation relative to mean value (default 10%)
    use_growth_rate : str {'parameter', 'birth'}
        division control scales as cell size growth rate, that can be given as
        a parameter (fixed), or from instanced value ('birth' value is taken
        as input)
    """

    def __init__(self, div_lambda=0, div_size_target=2.,
                 div_mode='gamma', div_sd_to_mean=.1,
                 use_growth_rate='parameter'):
        self.div_lambda = div_lambda
        self.div_size_target = div_size_target
        self.div_mode = div_mode
        self.div_sd_to_mean = div_sd_to_mean
        self.use_growth_rate = use_growth_rate
        self.content = [('div_lambda', div_lambda),
                        ('div_size_target', div_size_target),
                        ('div_mode', div_mode),
                        ('div_sd_to_mean', div_sd_to_mean),
                        ('use_growth_rate', use_growth_rate)]
        return

    def rv(self, birth_size=1., growth_rate=1./60*np.log(2.)):
        """Returns a random variate for interdivision time given X_0 and alpha.

        Parameters
        ----------
        birth_size : float (default 1.)
            birth size
        growth_rate : float (default 1./60*np.log(2.))
            default value corresponds to a size doubling time of one hour;
            must be used in 'per minutes'

        Returns
        -------
        tau : float
            interdivision time computed according to linear response with
            coefficient lambda plus fluctuations when mode is 'gamma'
        """
        tau = ((1. - self.div_lambda) * np.log(2.) +
               self.div_lambda * np.log(self.div_size_target/birth_size)) / growth_rate
        if self.div_mode == 'gamma':
            tau = _gamma(mean=tau, std=self.div_sd_to_mean*tau)
        return tau

    def __str__(self):
        msg = ('Division parameters:\n'
               '--------------------\n')
        left_column_size = 5
        right_column_size = 10
        for key, val in self.content:
            msg = '{}'.format(key).ljust(left_column_size)
            msg += ': ' + '{}\n'.format(val).rjust(right_column_size)
        return msg

def _gamma(mean=60, std=6.):
    theta = std**2 / mean
    k = (std / theta)**2 + 1.
    val = np.random.gamma(k, scale=theta)
    return val

class SampleInitialSize(object):
    """Initialize cell size given a target value for division and fluctuations.

    Parameters
    ----------
    birth_size_mean : float
    birth_size_mode : str {'fixed', 'lognormal'}
        initial distribution of birth size; fixed corresponds to a single value
        which is half the cutoff, lognormal returns a lognormal variate with
        parameter sigma
    birth_size_sd_to_mean : float
        relative fluctuations

    """

    def __init__(self, birth_size_mean=1., birth_size_mode='fixed',
                 birth_size_sd_to_mean=.1):
        self.birth_size_mean = birth_size_mean
        self.birth_size_mode = birth_size_mode
        self.birth_size_sd_to_mean = birth_size_sd_to_mean
        self.content = [('birth_size_mean', birth_size_mean),
                        ('birth_size_mode', birth_size_mode),
                        ('birth_size_sd_to_mean', birth_size_sd_to_mean)]
        return

    def rv(self):
        if self.birth_size_mode == 'fixed':
            return self.birth_size_mean
        elif self.birth_size_mode == 'lognormal':
            # getting normal parameters from average and relative fluct
            av = self.birth_size_mean
            sigma = np.sqrt(np.log(1. + self.birth_size_sd_to_mean**2))
            mu = np.log(av) - 0.5 * sigma**2
            lognorm = np.random.lognormal(mean=mu, sigma=sigma)
            return lognorm

class EcoliError(CellError):
    pass

class EcoliDivisionError(EcoliError):
    pass

class EcoliBirthError(EcoliError):
    pass

class Ecoli(Cell):
    """Ecoli class for numerical simulations of processes on trees.

    This class is based on `Cell` class, itself being built on
    `treelib.Node` class; so it is suitable for `treelib.Tree` structures.
    Each instance require a unique identifier, a parent cell (except for root
    cells), a birth time, and a given lifetime.

    Parameters
    ----------
    identifier : str
        unique identifier
    parent : str
        label of parent instance
    birth_time : float
        time at which cell cycle starts
    lifetime :
        float, duration of cell cycle
    """

    def __init__(self, identifier=None, parent=None,
                 birth_time=None, lifetime=None):

        Cell.__init__(self, identifier)

        self._lifetime = lifetime  # duration of cell cycle
        self._division_time = None  # time at cell division

        self.parent = parent
        if parent is not None:
            self.birth_time = parent.division_time
            self.birth_value = parent.division_value
        elif birth_time is not None:
            self.birth_time = birth_time
        else:
            raise EcoliBirthError

        if self.birth_time is not None and self.lifetime is not None:
            self.division_time = self.birth_time + self.lifetime

        self._rec_times = []  # recording times

        return

    @property
    def lifetime(self):
        "Get lifetime of Ecoli instance. See below for Setter."
        return self._lifetime

    @lifetime.setter
    def lifetime(self, value):
        "Set lifetime of Ecoli instance. See above for Getter."
        self._lifetime = value
        return

    @property
    def birth_value(self):
        "Get value of simulated process when cell cycle starts."
        return self._birth_value

    @birth_value.setter
    def birth_value(self, value):
        "Set value of simulated process when cell cycle starts."
        self._birth_value = value
        return

    @property
    def division_value(self):
        "Get value of simulated process when cell cycle ends."
        return self._division_value

    @division_value.setter
    def division_value(self, value):
        "Set value of simulated process when cell cycle ends."
        self._division_value = value
        return

    @property
    def rec_times(self):
        "Get recording times. See below for setter."
        return self._rec_times

    @rec_times.setter
    def rec_times(self, ndarray):
        "Set recording times. See above for getter."
        self._rec_times = ndarray
        return
<commit_msg>Edit docstrings; Black codestyle<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Base classes for numerical simulations of growing and dividing cells

This module defines base classes to parameterize numerical simulations regarding:
- global simulation parameters such as duration of experiment or the number of samples,
- division control parameters (how cell division is triggered as a function of other cell observables),
- initial cell size sampling

See also
---------

:mod:`tunacell.simu.ou` that defines classes and functions to simulate a particular
growth process using the Ornstein-Uhlenbeck model.
"""
from __future__ import print_function

import numpy as np

from tunacell.base.cell import CellError
from tunacell.base.cell import Cell

class SimuParams(object):
    """Stores global parameters for simulation parameters

    Parameters
    ----------
    nbr_container : int
        Number of simulated containers
    nbr_colony_per_container : int
        Number of colonies per container
    start : float
        time at which simulation starts. Root cell age at initial time is set
        randomly. Root cell birth time is then less or equal to start time.
    stop : float
        time at which simulation stops. Leaf cell division time is greater or
        equal to stop time.
    period : float
        time interval at which measurements are recorded (sampling).
    seed : int (default None)
        seed for random number generation (given to numpy.random.seed())

    Notes
    -----
    A very informal note on the number of samples:
        A sample is a tree of cells. The tree depth depends on two things:
            - the difference between start and stop times
            - the average cell cycle time (set in DivisionParams)
        The number of evaluated values for an exaclty sampled process (e.g.
        the Ornstein Uhlenbeck process) is inversely proportional to the
        period at which data is recorded.

        The total number of operations depends:
            - exponentially on the difference 'stop - start'
            - inversely proportionally to 'period'
            - proportionally on nbr_container
            - proportionally on nbr_colony_per_container
    """

    def __init__(
        self,
        nbr_container=1,
        nbr_colony_per_container=1,
        start=0.0,
        stop=100.0,
        period=5.0,
        seed=None,
    ):
        self.nbr_container = nbr_container
        self.nbr_colony_per_container = nbr_colony_per_container
        self.start = start
        self.stop = stop
        self.period = period
        self.seed = seed
        # metadata helper
        self.content = [
            ("nbr_container", nbr_container),
            ("nbr_colony_per_container", nbr_colony_per_container),
            ("start", start),
            ("stop", stop),
            ("period", period),
            ("seed", seed),
        ]
        return

    def __str__(self):
        msg = "Simulation parameters:\n" "----------------------\n"
        left_column_size = 25
        right_column_size = 10
        for key, val in self.content:
            msg += "{}".format(key).ljust(left_column_size)
            msg += ": " + "{}\n".format(val).rjust(right_column_size)
        return msg

class DivisionParams(object):
    """Parameterize when a given cell divides depending on its observables

    This settings is based on the linear results explained elsewhere.

    Parameters
    ----------
    div_lambda : float between 0 and 1
        the lambda parameter is the mixing parameter between timer (0)
        and sizer (1) models. The adder model is obtained with lambda_ 0.5.
    div_size_target : float
        the size target sets the cutoff size for division. With *div_lambda=1*, and
        no fluctuations, division is triggered when size reaches exactly the size target
    div_mode : str {'gamma', 'fixed'}
        'fixed': only the linear response is taken;
        'gamma': adds fluctuations over linear response
    div_sd_to_mean : float (default 0.1)
        sets the standard deviation relative to mean value (default 10%) when used
        with the *div_mode='gamma'* option
    use_growth_rate : str {'parameter', 'birth'}
        division control scales as cell size growth rate, that can be given as
        a parameter (fixed), or from the actual growth rate value **at cell birth**
    """

    def __init__(
        self,
        div_lambda=0,
        div_size_target=2.0,
        div_mode="gamma",
        div_sd_to_mean=0.1,
        use_growth_rate="parameter",
    ):
        self.div_lambda = div_lambda
        self.div_size_target = div_size_target
        self.div_mode = div_mode
        self.div_sd_to_mean = div_sd_to_mean
        self.use_growth_rate = use_growth_rate
        self.content = [
            ("div_lambda", div_lambda),
            ("div_size_target", div_size_target),
            ("div_mode", div_mode),
            ("div_sd_to_mean", div_sd_to_mean),
            ("use_growth_rate", use_growth_rate),
        ]
        return

    def rv(self, birth_size=1.0, growth_rate=1.0 / 60 * np.log(2.0)):
        """Returns a random variate for interdivision time given *birth_size* and *growth_rate*.

        Parameters
        ----------
        birth_size : float (default 1.)
            birth size
        growth_rate : float (default 1./60*np.log(2.))
            default value corresponds to a size doubling time of one hour;
            must be used in 'per minutes'

        Returns
        -------
        tau : float
            interdivision time computed according to linear response with
            coefficient lambda plus fluctuations when mode is 'gamma'
        """
        tau = (
            (1.0 - self.div_lambda) * np.log(2.0)
            + self.div_lambda * np.log(self.div_size_target / birth_size)
        ) / growth_rate
        if self.div_mode == "gamma":
            tau = _gamma(mean=tau, std=self.div_sd_to_mean * tau)
        return tau

    def __str__(self):
        msg = "Division parameters:\n" "--------------------\n"
        left_column_size = 5
        right_column_size = 10
        for key, val in self.content:
            msg = "{}".format(key).ljust(left_column_size)
            msg += ": " + "{}\n".format(val).rjust(right_column_size)
        return msg

def _gamma(mean=60, std=6.0):
    theta = std ** 2 / mean
    k = (std / theta) ** 2 + 1.0
    val = np.random.gamma(k, scale=theta)
    return val

class SampleInitialSize(object):
    """Initialize cell size given a target value for division and fluctuations.

    Parameters
    ----------
    birth_size_mean : float
    birth_size_mode : str {'fixed', 'lognormal'}
        initial distribution of birth size; fixed corresponds to a single value
        which is half the cutoff, lognormal returns a lognormal variate with
        parameter sigma
    birth_size_sd_to_mean : float
        relative fluctuations

    """

    def __init__(
        self, birth_size_mean=1.0, birth_size_mode="fixed", birth_size_sd_to_mean=0.1
    ):
        self.birth_size_mean = birth_size_mean
        self.birth_size_mode = birth_size_mode
        self.birth_size_sd_to_mean = birth_size_sd_to_mean
        self.content = [
            ("birth_size_mean", birth_size_mean),
            ("birth_size_mode", birth_size_mode),
            ("birth_size_sd_to_mean", birth_size_sd_to_mean),
        ]
        return

    def rv(self):
        if self.birth_size_mode == "fixed":
            return self.birth_size_mean
        elif self.birth_size_mode == "lognormal":
            # getting normal parameters from average and relative fluct
            av = self.birth_size_mean
            sigma = np.sqrt(np.log(1.0 + self.birth_size_sd_to_mean ** 2))
            mu = np.log(av) - 0.5 * sigma ** 2
            lognorm = np.random.lognormal(mean=mu, sigma=sigma)
            return lognorm

class EcoliError(CellError):
    pass

class EcoliDivisionError(EcoliError):
    pass

class EcoliBirthError(EcoliError):
    pass

class Ecoli(Cell):
    """Ecoli class for numerical simulations of processes on trees.

    This class is based on `Cell` class, itself being built on
    `treelib.Node` class; so it is suitable for `treelib.Tree` structures.
    Each instance require a unique identifier, a parent cell (except for root
    cells), a birth time, and a given lifetime.

    Parameters
    ----------
    identifier : str
        unique identifier
    parent : str
        label of parent instance
    birth_time : float
        time at which cell cycle starts
    lifetime :
        float, duration of cell cycle
    """

    def __init__(self, identifier=None, parent=None, birth_time=None, lifetime=None):

        Cell.__init__(self, identifier)

        self._lifetime = lifetime  # duration of cell cycle
        self._division_time = None  # time at cell division

        self.parent = parent
        if parent is not None:
            self.birth_time = parent.division_time
            self.birth_value = parent.division_value
        elif birth_time is not None:
            self.birth_time = birth_time
        else:
            raise EcoliBirthError

        if self.birth_time is not None and self.lifetime is not None:
            self.division_time = self.birth_time + self.lifetime

        self._rec_times = []  # recording times

        return

    @property
    def lifetime(self):
        "Get lifetime of Ecoli instance. See below for Setter."
        return self._lifetime

    @lifetime.setter
    def lifetime(self, value):
        "Set lifetime of Ecoli instance. See above for Getter."
        self._lifetime = value
        return

    @property
    def birth_value(self):
        "Get value of simulated process when cell cycle starts."
        return self._birth_value

    @birth_value.setter
    def birth_value(self, value):
        "Set value of simulated process when cell cycle starts."
        self._birth_value = value
        return

    @property
    def division_value(self):
        "Get value of simulated process when cell cycle ends."
        return self._division_value

    @division_value.setter
    def division_value(self, value):
        "Set value of simulated process when cell cycle ends."
        self._division_value = value
        return

    @property
    def rec_times(self):
        "Get recording times. See below for setter."
        return self._rec_times

    @rec_times.setter
    def rec_times(self, ndarray):
        "Set recording times. See above for getter."
        self._rec_times = ndarray
        return
<|endoftext|>