<commit_before>import networkx as nx
import numpy as np

def parent_panel(decision_class, resampling_panel):

    parent_panel = []
    for cycle_idx, cycle in enumerate(resampling_panel):

        # each stage in the resampling for that cycle
        # make a stage parent table
        parent_table = []

        # now iterate through the rest of the stages
        for step in cycle:

            # get the parents idxs for the children of this step
            step_parents = decision_class.parents(step)

            # for the full stage table save all the intermediate parents
            parent_table.append(step_parents)

        # for the full parent panel
        parent_panel.append(parent_table)

    return parent_panel

def cycle_tree_parent_table(decision_class, cycle_tree):
    """Determines the net parents for each cycle and sets them in-place to
    the cycle tree given."""

    # just go through each node individually in the tree
    for node in cycle_tree:
        # get the records for each step in this node
        node_recs = cycle_tree.nodes[node]['resampling_steps']

        # get the node parent table by using the parent panel method
        # on the node records
        node_parent_panel = parent_panel(decision_class, [node_recs])
        # then get the net parents from this parent panel, and slice
        # out the only entry from it
        node_parents = net_parent_table(node_parent_panel)[0]

        # put this back into the cycle_tree
        cycle_tree.nodes[node]['parent_idxs'] = node_parents

    return cycle_tree

def net_parent_table(parent_panel):

    net_parent_table = []

    # each cycle
    for cycle_idx, step_parent_table in enumerate(parent_panel):
        # for the net table we only want the end results,
        # we start at the last cycle and look at its parent
        step_net_parents = []
        n_steps = len(step_parent_table)
        for walker_idx, parent_idx in enumerate(step_parent_table[-1]):
            # initialize the root_parent_idx which will be updated
            root_parent_idx = parent_idx

            # if no resampling skip the loop and just return the idx
            if n_steps > 0:
                # go back through the steps getting the parent at each step
                for prev_step_idx in range(n_steps):
                    prev_step_parents = step_parent_table[-(prev_step_idx+1)]
                    root_parent_idx = prev_step_parents[root_parent_idx]

            # when this is done we should have the index of the root parent,
            # save this as the net parent index
            step_net_parents.append(root_parent_idx)

        # for this step save the net parents
        net_parent_table.append(step_net_parents)

    return net_parent_table

def parent_graph(parent_table):

    graph = nx.Graph()

    n_walkers = parent_table.shape[1]

    for step_idx, parent_idxs in enumerate(parent_table):

        # the first step is useless
        if step_idx == 0:
            continue

        # make edge between each walker of this step to the previous step
        for curr_walker_idx in range(n_walkers):

            # make an edge between the parent of this walker and this walker
            edge = ((step_idx, curr_walker_idx), (step_idx - 1, parent_idxs[curr_walker_idx]))

            graph.add_edge(*edge)

    return graph

def ancestors(parent_table, cycle_idx, walker_idx, ancestor_cycle=0):
    """Given a parent table, step_idx, and walker idx returns the
    ancestor at a given cycle of the walker.

    Input:

    parent: table (n_cycles x n_walkers numpy array): It describes
    how the walkers merged and cloned during the WExplore simulation .

    cycle_idx:

    walker_idx:

    ancestor_cycle:

    Output:

    ancestors: A list of 2x1 tuples indicating the
                   walker and cycle parents

    """

    lineage = [(walker_idx, cycle_idx)]

    previous_walker = walker_idx

    for curr_cycle_idx in range(cycle_idx, ancestor_cycle, -1):
            previous_walker = parent_table[curr_cycle_idx][previous_walker]

            # check for discontinuities, e.g. warping events
            if previous_walker == -1:
                # there are no more continuous ancestors for this
                # walker so we cannot return ancestors back to the
                # requested cycle just return the ancestors to this
                # point
                break

            previous_point = (previous_walker, curr_cycle_idx - 1)
            lineage.insert(0, previous_point)

    return lineage

def parent_table_from_tree(cycle_tree, run_idx, cycle_idx):
    # to make this easy we just first generate a parent table from the
    # tree that is relevant for the query
    parent_table = []

    # initialize the current node
    curr_node = (run_idx, cycle_idx)

    # stop the loop when we reach the root of the cycle tree
    at_root = False
    while not at_root:

        # first we get the cycle node previous to this one because it
        # has the parents for the current node
        parent_nodes = list(cycle_tree.adj[curr_node].keys())

        # if there is no parent then this is the root of the
        # cycle_tree and we should stop after this step of the loop
        if len(parent_nodes) == 0:
            at_root = True

        # if there are any parents there should only be one, since it is a tree
        prev_node = parent_nodes[0]

        # get the parents from the previous node
        parents = cycle_tree.nodes[prev_node]['parent_idxs']

        # add these parents to the parent table at the beginning
        parent_table.insert(0, parents)

        # then update the current node to the previous one
        curr_node = prev_node

    return parent_table

def run_cycle_to_spanning_contig_cycle(cycle_tree, run_idx, cycle_idx):
    # initialize the current node
    curr_node = (run_idx, cycle_idx)

    # the cycle idxs for the whole contig, this will tally the number
    # of movements back through the tree
    contig_cycle_idx = 0

    # stop the loop when we reach the root of the cycle tree
    at_root = False
    while not at_root:

        # first we get the cycle node previous to this one because it
        # has the parents for the current node
        parent_nodes = list(cycle_tree.adj[curr_node].keys())

        # if there is no parent then this is the root of the
        # cycle_tree and we should stop after this step of the loop
        if len(parent_nodes) == 0:
            at_root = True

        # if there are any parents there should only be one, since it is a tree
        prev_node = parent_nodes[0]

        # add to the count of cycles
        contig_cycle_idx += 1

        # then update the current node to the previous one
        curr_node = prev_node

    return contig_cycle_idx

def ancestors_from_tree(cycle_tree, run_idx, cycle_idx, walker_idx, ancestor_node=None):

    # first we generate the parent table given the walker we want
    # parents from
    parent_table = parent_table_from_tree(cycle_tree, run_idx, cycle_idx)

    # get the cycle index within the parent table (contig) and not the
    # one given for within the run
    contig_cycle_idx = run_cycle_to_spanning_contig_cycle(cycle_tree, run_idx, cycle_idx)

    # the same for the index of the ancestor node if it is given
    if ancestor_node is not None:
        ancestor_contig_cycle_idx = run_cycle_to_spanning_contig_cycle(cycle_tree, *ancestor_node)
    else:
        ancestor_contig_cycle_idx = 0

    # then we just perform a normal ancestors function on that table
    # using a corrected cycle index from the contig
    return ancestors(parent_table, contig_cycle_idx, walker_idx, ancestor_contig_cycle_idx)

def ancestor_table(parent_table, ancestor_cycle=0):
    """Given a parent table and a cycle index, will return a table that
    gives the index of the walker that was the ancestor to the walker.

    The table wil be of size (n_cycles - ancestor_cycle, n_walkers),
    because only walkers after the ancestor_cycle will be assigned
    values. Walkers at the ancestor cycle index will be assigned their own
    index.

    Defaults to the zeroth cycle which is the initial walkers.

    This function will create a table that tracks the ultimate
    parent of the walkers. Each row is a timestep, and the
    value in the table corresponds to the original walker
    that the walker was associated from at the start of the
    MD simulation.

    Inputs:

    parent_table (2D numpy array): A table that shows how walkers from different time
    steps are related to each other.

    Outputs:

    ancestor_table (2D numpy array): A table of the ancestor walker index
    for walkers after the ancestor cycle idx.

    """

    n_steps = parent_table.shape[0]
    n_walkers = parent_table.shape[1]
    ancestor_mat = np.zeros((n_steps - ancestor_cycle, n_walkers))

    for step_idx in range(ancestor_cycle, n_steps):

        for walker_idx in range(n_walkers):

            # if this is the ancestor cycel we set the ancestor idx to itself
            if step_idx == ancestor_cycle:
                ancestor_mat[step_idx, walker_idx] = walker_idx

            else:
                ancestor_mat[step_idx, walker_idx] = \
                                        ancestor_mat[step_idx - 1,
                                                        parent_table[step_idx, walker_idx]]

    return ancestor_mat

def sliding_window(parent_table, window_length):
    """Returns traces (lists of frames across a run) on a sliding window
    on the branching structure of a run of a WepyHDF5 file. There is
    no particular order guaranteed.

    """

    assert parent_table.dtype == np.int, \
        "parent table values must be integers, not {}".format(parent_table.dtype)

    assert window_length > 1, "window length must be greater than one"

    # we make a range iterator which goes from the last cycle to the
    # cycle which would be the end of the first possible sliding window
    for cycle_idx in range(len(parent_table)-1, window_length-2, -1):

        # then iterate for each walker at this cycle
        for walker_idx in range(len(parent_table[0])):

            # then get the ancestors according to the sliding window
            window = ancestors(parent_table, cycle_idx, walker_idx,
                               ancestor_cycle=cycle_idx-(window_length-1))

            # if the window is too short because the lineage has a
            # discontinuity in it skip to the next window
            if len(window) < window_length:
                continue

            yield window

def cycle_tree_leaves(root, cycle_tree):

    # get the reversed directions of the cycle tree as a view, we
    # don't need a copy
    rev_tree = cycle_tree.reverse(copy=False)

    # then

def cycle_tree_root(cycle_tree):

    # just pick a random node, the first one
    curr_node = list(cycle_tree.adjacency())[0][0]

    # we also get the adjacent nodes for this node
    adj_nodes = list(cycle_tree.adj[curr_node].keys())

    # there should only be one node in the dict
    assert len(adj_nodes) <= 1, "There should be at most 1 edge"

    # then we use this node as the starting point to move back to the
    # root, we end when there is no adjacent node
    while len(adj_nodes) > 0:

        # we take another step backwards, and choose the node in the
        # adjacency
        adj_nodes = list(cycle_tree.adj[curr_node])

        # there should only be 1 or none nodes
        assert len(adj_nodes) <= 1, "There should be at most 1 edge"

        # and reset the current node
        try:
            curr_node = adj_nodes[0]
        except IndexError:
            # this happens when this is the last node, inelegant apologies
            pass

    return curr_node

def cycle_tree_sliding_windows(cycle_tree, window_length):

    # to generate all the sliding windows over a connected cycle tree
    # it is useful to think of it as a braid, since within this tree
    # there is a forest of trees which are the lineages of the
    # walkers. To simplify things we can first generate window traces
    # over the cycle tree (ignoring the fin structure of the walkers),
    # which is a very similar process as to the original process of
    # the sliding windows over trees, and then treat each window trace
    # as it's own parent table, which can then be treated using the
    # original sliding window algorithm. As long as the correct contig
    # traces are generated this will be no problem, and all that is
    # left is to translate the cycle idxs properly such that the
    # windows generated are inter-run traces i.e. lists of tuples of
    # the form (run_idx, traj_idx, cycle_idx) where traj_idx and
    # cycle_idx are internal to the run specified by run_idx.

    # so we want to construct a list of contig windows, which are
    # traces with components (run_idx, cycle_idx)
    contig_windows = []

    # to do this we start at the leaves of the cycle tree, but first
    # we have to find them. The cycle tree should be directed with
    # edges pointing towards parents. Since we are working with a
    # single tree, we have a single root and we can just reverse the
    # directions of the edges and recursively walk the tree until we
    # find nodes with no adjacent edges

    # first we need to find the root

def cycle_forest_sliding_windows(cycle_forest, window_length):

    assert window_length > 1, "window length must be greater than one"

    # we can deal with each tree in this forest of trees separately,
    # that is runs that are not connected
    forest_windows = []
    for component_nodes in nx.weakly_connected_components(cycle_forest):

        # actually get the subtree from the main tree
        cycle_tree = cycle_forest.subgraph(component_nodes)

        # then we can get the windows on this connected tree
        subtree_windows = cycle_forest_sliding_windows(cycle_tree, window_length)

        forest_windows.extend(subtree_windows)

    return forest_windows
<commit_msg>cleanup, contig sliding windows<commit_after>import networkx as nx
import numpy as np

def parent_panel(decision_class, resampling_panel):

    parent_panel = []
    for cycle_idx, cycle in enumerate(resampling_panel):

        # each stage in the resampling for that cycle
        # make a stage parent table
        parent_table = []

        # now iterate through the rest of the stages
        for step in cycle:

            # get the parents idxs for the children of this step
            step_parents = decision_class.parents(step)

            # for the full stage table save all the intermediate parents
            parent_table.append(step_parents)

        # for the full parent panel
        parent_panel.append(parent_table)

    return parent_panel

def net_parent_table(parent_panel):

    net_parent_table = []

    # each cycle
    for cycle_idx, step_parent_table in enumerate(parent_panel):
        # for the net table we only want the end results,
        # we start at the last cycle and look at its parent
        step_net_parents = []
        n_steps = len(step_parent_table)
        for walker_idx, parent_idx in enumerate(step_parent_table[-1]):
            # initialize the root_parent_idx which will be updated
            root_parent_idx = parent_idx

            # if no resampling skip the loop and just return the idx
            if n_steps > 0:
                # go back through the steps getting the parent at each step
                for prev_step_idx in range(n_steps):
                    prev_step_parents = step_parent_table[-(prev_step_idx+1)]
                    root_parent_idx = prev_step_parents[root_parent_idx]

            # when this is done we should have the index of the root parent,
            # save this as the net parent index
            step_net_parents.append(root_parent_idx)

        # for this step save the net parents
        net_parent_table.append(step_net_parents)

    return net_parent_table

def ancestors(parent_table, cycle_idx, walker_idx, ancestor_cycle=0):
    """Given a parent table, step_idx, and walker idx returns the
    ancestor at a given cycle of the walker.

    Input:

    parent: table (n_cycles x n_walkers numpy array): It describes
    how the walkers merged and cloned during the WExplore simulation .

    cycle_idx:

    walker_idx:

    ancestor_cycle:

    Output:

    ancestors: A list of 2x1 tuples indicating the
                   walker and cycle parents

    """

    lineage = [(walker_idx, cycle_idx)]

    previous_walker = walker_idx

    for curr_cycle_idx in range(cycle_idx, ancestor_cycle, -1):
            previous_walker = parent_table[curr_cycle_idx][previous_walker]

            # check for discontinuities, e.g. warping events
            if previous_walker == -1:
                # there are no more continuous ancestors for this
                # walker so we cannot return ancestors back to the
                # requested cycle just return the ancestors to this
                # point
                break

            previous_point = (previous_walker, curr_cycle_idx - 1)
            lineage.insert(0, previous_point)

    return lineage

def sliding_window(parent_table, window_length):
    """Returns traces (lists of frames across a run) on a sliding window
    on the branching structure of a run of a WepyHDF5 file. There is
    no particular order guaranteed.

    """

    assert parent_table.dtype == np.int, \
        "parent table values must be integers, not {}".format(parent_table.dtype)

    assert window_length > 1, "window length must be greater than one"

    # we make a range iterator which goes from the last cycle to the
    # cycle which would be the end of the first possible sliding window
    for cycle_idx in range(len(parent_table)-1, window_length-2, -1):

        # then iterate for each walker at this cycle
        for walker_idx in range(len(parent_table[0])):

            # then get the ancestors according to the sliding window
            window = ancestors(parent_table, cycle_idx, walker_idx,
                               ancestor_cycle=cycle_idx-(window_length-1))

            # if the window is too short because the lineage has a
            # discontinuity in it skip to the next window
            if len(window) < window_length:
                continue

            yield window

## contig trees for of parent panels

def cycle_tree_parent_table(decision_class, cycle_tree):
    """Determines the net parents for each cycle and sets them in-place to
    the cycle tree given."""

    # just go through each node individually in the tree
    for node in cycle_tree:
        # get the records for each step in this node
        node_recs = cycle_tree.nodes[node]['resampling_steps']

        # get the node parent table by using the parent panel method
        # on the node records
        node_parent_panel = parent_panel(decision_class, [node_recs])
        # then get the net parents from this parent panel, and slice
        # out the only entry from it
        node_parents = net_parent_table(node_parent_panel)[0]

        # put this back into the cycle_tree
        cycle_tree.nodes[node]['parent_idxs'] = node_parents

    return cycle_tree

def cycle_tree_contig_trace(cycle_tree, run_idx, cycle_idx, start_contig_idx=0):
    """Given a single cycle tree and a (run_idx, cycle_idx) in that tree
    and a starting contig index generate a contig trace of (run_idx,
    cycle_idx) indices for that contig.

    """

    assert start_contig_idx >= 0, "start_contig_idx must be a valid index"

    # initialize the current node
    curr_node = (run_idx, cycle_idx)

    # make a trace of this contig
    contig_trace = [curr_node]

    # stop the loop when we reach the root of the cycle tree
    at_root = False
    while not at_root:

        # first we get the cycle node previous to this one because it
        # has the parents for the current node
        parent_nodes = list(cycle_tree.adj[curr_node].keys())

        # if there is no parent then this is the root of the
        # cycle_tree and we should stop after this step of the loop
        if len(parent_nodes) == 0:
            at_root = True

        # or else we put the node into the contig trace
        else:
            parent_node = parent_nodes[0]
            contig_trace.insert(0, parent_node)

            # if there are any parents there should only be one, since it is a tree
            prev_node = parent_nodes[0]

            # then update the current node to the previous one
            curr_node = prev_node

    return contig_trace

def contig_cycle(cycle_tree, run_idx, cycle_idx):
    """Get the contig cycle idx for a (run_idx, cycle_idx) pair."""

    # make the contig trace
    contig_trace = cycle_tree_contig_trace(cycle_tree, run_idx, cycle_idx)

    # get the length and subtract one for the index
    return len(contig_trace) - 1

def _tree_leaves(root, tree):

    # traverse the tree away from the root node until a branch point
    # is found then iterate over the subtrees from there and
    # recursively call this function on them
    branch_child_nodes = []
    curr_node = root
    leaves = []
    leaf_found = False
    while (len(branch_child_nodes) == 0) and (not leaf_found):

        # get the child nodes
        child_nodes = list(tree.adj[curr_node].keys())

        # if there is more than one child node, the current node is a
        # branch node
        if len(child_nodes) > 1:

            # we will use the branch child nodes as the roots of the
            # next recursion level
            branch_child_nodes = child_nodes

        # if there are no children then this is a leaf node
        elif len(child_nodes) == 0:
            # set the current node as the only leaf
            leaves = [curr_node]

            # and break out of the loop
            leaf_found = True

        # otherwise reset the current node
        else:
            # there will only be one child node
            curr_node = child_nodes[0]

    # this will run if any child nodes were found to find more leaves,
    # which won't happen when the loop ended upon finding a leaf node
    for branch_child_node in branch_child_nodes:
        branch_leaves = _tree_leaves(branch_child_node, tree)
        leaves.extend(branch_leaves)

    return leaves

def cycle_tree_leaves(root, cycle_tree):

    # get the reversed directions of the cycle tree as a view, we
    # don't need a copy
    rev_tree = cycle_tree.reverse(copy=False)

    # then we use the adjacencies to find the last node in the network
    # using a recursive algorithm
    leaves = _tree_leaves(root, rev_tree)

    return leaves

def cycle_tree_root(cycle_tree):

    # just pick a random node, the first one
    curr_node = list(cycle_tree.adjacency())[0][0]

    # we also get the adjacent nodes for this node
    adj_nodes = list(cycle_tree.adj[curr_node].keys())

    # there should only be one node in the dict
    assert len(adj_nodes) <= 1, "There should be at most 1 edge"

    # then we use this node as the starting point to move back to the
    # root, we end when there is no adjacent node
    while len(adj_nodes) > 0:

        # we take another step backwards, and choose the node in the
        # adjacency
        adj_nodes = list(cycle_tree.adj[curr_node])

        # there should only be 1 or none nodes
        assert len(adj_nodes) <= 1, "There should be at most 1 edge"

        # and reset the current node
        try:
            curr_node = adj_nodes[0]
        except IndexError:
            # this happens when this is the last node, inelegant apologies
            pass

    return curr_node

def contig_trace_parent_table(cycle_tree, contig_trace):
    """Given a cycle tree with parents and a contig trace returns a parent
    table for that contig."""

    parent_table = []
    for run_idx, cycle_idx in contig_trace:
        parent_idxs = cycle_tree.node[(run_idx, cycle_idx)]['parent_idxs']
        parent_table.append(parent_idxs)

    return parent_table

def ancestors_from_tree(cycle_tree, run_idx, cycle_idx, walker_idx, ancestor_node=None):

    raise NotImplementedError

    # make the contig trace for this to the root
    contig_trace = cycle_tree_contig_trace(cycle_tree, run_idx, cycle_idx)

    # first we generate the parent table given the walker we want
    # parents from
    parent_table = parent_table_from_tree(cycle_tree, run_idx, cycle_idx)

    # get the cycle index within the parent table (contig) and not the
    # one given for within the run
    contig_cycle_idx = run_cycle_to_spanning_contig_cycle(cycle_tree, run_idx, cycle_idx)

    # the same for the index of the ancestor node if it is given
    if ancestor_node is not None:
        ancestor_contig_cycle_idx = run_cycle_to_spanning_contig_cycle(cycle_tree, *ancestor_node)
    else:
        ancestor_contig_cycle_idx = 0

    # then we just perform a normal ancestors function on that table
    # using a corrected cycle index from the contig
    return ancestors(parent_table, contig_cycle_idx, walker_idx, ancestor_contig_cycle_idx)

def contig_sliding_windows(cycle_tree, window_length):

    # to generate all the sliding windows over a connected cycle tree
    # it is useful to think of it as a braid, since within this tree
    # there is a forest of trees which are the lineages of the
    # walkers. To simplify things we can first generate window traces
    # over the cycle tree (ignoring the fine structure of the walkers),
    # which is a very similar process as to the original process of
    # the sliding windows over trees, and then treat each window trace
    # as it's own parent table, which can then be treated using the
    # original sliding window algorithm. As long as the correct contig
    # traces are generated this will be no problem, and all that is
    # left is to translate the cycle idxs properly such that the
    # windows generated are inter-run traces i.e. lists of tuples of
    # the form (run_idx, traj_idx, cycle_idx) where traj_idx and
    # cycle_idx are internal to the run specified by run_idx.

    # so we want to construct a list of contig windows, which are
    # traces with components (run_idx, cycle_idx)
    contig_windows = []

    # to do this we start at the leaves of the cycle tree, but first
    # we have to find them. The cycle tree should be directed with
    # edges pointing towards parents. Since we are working with a
    # single tree, we have a single root and we can just reverse the
    # directions of the edges and recursively walk the tree until we
    # find nodes with no adjacent edges

    # first we need to find the root and leaves for this tree
    root = cycle_tree_root(cycle_tree)
    leaves = cycle_tree_root(root, cycle_tree)

    # now we use these leaves to move backwards in the tree with the
    # window length to get contiguous segments
    contig_windows = []

    # starting with the leaf nodes we generate contig trace windows
    # until the last nodes are the same as other windows from other
    # leaves, i.e. until a branch point has been arrived at between 2
    # or more leaf branches. To do this we start at the leaf of the
    # longest spanning contig and make windows until the endpoint is
    # no longer the largest contig cycle index. Then we alternate
    # between them until we find they have converged

    # initialize the list of active branches
    branch_contigs = [cycle_tree_contig_trace(cycle_tree, *leaf) for leaf in leaves]

    done = False
    while not done:

        # make a window for the largest endpoint, no need to break
        # ties since the next iteration will get it
        contig_lengths = [len(contig) for contig in branch_contigs]
        longest_branch_idx = np.argmax(contig_lengths)

        # if the branch is not long enough for the window we end this
        # process
        if window_length > branch_contigs[longest_branch_idx]:
            done = True

        # otherwise we get the next window and do the other processing
        else:
            # get the next window for this branch
            window = branch_contigs[longest_branch_idx][-window_length:]

            contig_windows.append(window)

            # pop the last element off of this branch contig
            last_node = branch_contigs[longest_branch_idx].pop()

            # if there are any other branches of the same length that have
            # this as their last node then we have reached a branch point
            # and that other branch must be eliminated
            for branch_idx, branch_contig in enumerate(branch_contigs):

                # compare the last node in contig and if it is the same as
                # the node that was just used as a window end
                if branch_contig[-1] == last_node:
                    # this branch is the same so we just get rid of it
                    _ = branch_contigs.pop(branch_idx)

    return contig_windows

def cycle_forest_sliding_windows(cycle_forest, window_length):

    assert window_length > 1, "window length must be greater than one"

    # we can deal with each tree in this forest of trees separately,
    # that is runs that are not connected
    forest_windows = []
    for component_nodes in nx.weakly_connected_components(cycle_forest):

        # actually get the subtree from the main tree
        cycle_tree = cycle_forest.subgraph(component_nodes)

        # then we can get the windows on this connected tree
        subtree_windows = cycle_tree_sliding_windows(cycle_tree, window_length)

        forest_windows.extend(subtree_windows)

    return forest_windows
<|endoftext|>


<commit_before>"""
Module to handle utility functions within AppDameon.
"""

import asyncio
import datetime
import traceback

import appdaemon.scheduler as scheduler
import appdaemon.utils as utils
from appdaemon.appdaemon import AppDaemon

class Utility:

    """
    Class that uncludes the utility loop.

    Checks for file changes, overdue threads, thread starvation, and schedules regular state refreshes
    """

    def __init__(self, ad: AppDaemon):

        """
        Constructor.

        :param ad: Reference to the AppDaemon object
        """

        self.AD = ad
        self.stopping = False
        self.logger = ad.logging.get_child("_utility")

    def stop(self):

        """
        Called by the AppDaemon object to terminate the loop cleanly
        """

        self.logger.debug("stop() called for utility")
        self.stopping = True

    async def loop(self):

        """
        The main utility loop.

        Loops until stop() is called, checks for file changes, overdue threads, thread starvation, and schedules regular state refreshes
        """

        #
        # Setup
        #

        await self.AD.threading.init_admin_stats()
        await self.AD.threading.create_initial_threads()
        await self.AD.app_management.create_sensors()

        #
        # Wait for all plugins to initialize
        #

        await self.AD.plugins.wait_for_plugins()

        # Check if we need to bail due to missing metadata

        if self.AD.plugins.required_meta_check() is False:
            if self.AD.stop_function is not None:
                self.AD.stop_function()
            else:
                self.stop()

        if not self.stopping:

            #
            # All plugins are loaded and we have initial state
            # We also have metadata so we can initialise the scheduler
            #

            self.AD.sched = scheduler.Scheduler(self.AD)

            # Create timer loop

            self.logger.debug("Starting timer loop")

            self.AD.loop.create_task(self.AD.sched.loop())

            if self.AD.apps is True:
                self.logger.debug("Reading Apps")

                await self.AD.app_management.check_app_updates()

                self.logger.info("App initialization complete")
                #
                # Fire APPD Started Event
                #
                await self.AD.events.process_event("global", {"event_type": "appd_started", "data": {}})

            self.booted = await self.AD.sched.get_now()
            await self.AD.state.add_entity("admin", "sensor.appdaemon_version", utils.__version__)
            await self.AD.state.add_entity("admin", "sensor.appdaemon_uptime", str(datetime.timedelta(0)))
            await self.AD.state.add_entity("admin", "sensor.appdaemon_booted", utils.dt_to_str((await self.AD.sched.get_now()).replace(microsecond=0), self.AD.tz))
            warning_step = 0
            warning_iterations = 0
            s1 = 0
            e1 = 0

            # Start the loop proper

            while not self.stopping:

                start_time = datetime.datetime.now().timestamp()

                try:

                    if self.AD.apps is True:

                        if self.AD.production_mode is False:
                            # Check to see if config has changed
                            s1 = datetime.datetime.now().timestamp()
                            await self.AD.app_management.check_app_updates()
                            e1 = datetime.datetime.now().timestamp()

                    # Call me suspicious, but lets update state from the plugins periodically

                    await self.AD.plugins.update_plugin_state()

                    # Check for thread starvation

                    warning_step, warning_iterations = await self.AD.threading.check_q_size(warning_step, warning_iterations)

                    # Check for any overdue threads

                    await self.AD.threading.check_overdue_and_dead_threads()

                    # Save any hybrid namespaces

                    self.AD.state.save_hybrid_namespaces()

                    # Run utility for each plugin

                    self.AD.plugins.run_plugin_utility()

                    # Update uptime sensor

                    uptime = (await self.AD.sched.get_now()).replace(microsecond=0) - self.booted.replace(microsecond=0)

                    await self.AD.state.set_state("_utility", "admin", "sensor.appdaemon_uptime", state=str(uptime))

                except:
                    self.logger.warning('-' * 60)
                    self.logger.warning("Unexpected error during utility()")
                    self.logger.warning('-' * 60)
                    self.logger.warning(traceback.format_exc())
                    self.logger.warning('-' * 60)

                end_time = datetime.datetime.now().timestamp()

                loop_duration = (int((end_time - start_time) * 1000) / 1000) * 1000
                check_app_updates_duration = (int((e1 - s1) * 1000) / 1000) * 1000

                self.logger.debug("Util loop compute time: %sms, check_config()=%sms, other=%sms", loop_duration, check_app_updates_duration, loop_duration - check_app_updates_duration)
                if self.AD.sched.realtime is True and loop_duration > (self.AD.max_utility_skew * 1000):
                    self.logger.warning("Excessive time spent in utility loop: %sms, %sms in check_app_updates(), %sms in other", loop_duration, check_app_updates_duration, loop_duration - check_app_updates_duration)
                    if self.AD.check_app_updates_profile is True:
                        self.logger.info("Profile information for Utility Loop")
                        self.logger.info(self.AD.app_management.check_app_updates_profile_stats)

                await asyncio.sleep(self.AD.utility_delay)

            if self.AD.app_management is not None:
                await self.AD.app_management.terminate()

    async def set_production_mode(self, mode=True):
        if mode is True:
            self.logger.info("AD Production Mode Activated")
        else:
            self.logger.info("AD Production Mode Deactivated")
        self.AD.production_mode = mode
<commit_msg>Update utility_loop.py<commit_after>"""
Module to handle utility functions within AppDameon.
"""

import asyncio
import datetime
import traceback

import appdaemon.scheduler as scheduler
import appdaemon.utils as utils
from appdaemon.appdaemon import AppDaemon

class Utility:

    """
    Class that uncludes the utility loop.

    Checks for file changes, overdue threads, thread starvation, and schedules regular state refreshes
    """

    def __init__(self, ad: AppDaemon):

        """
        Constructor.

        :param ad: Reference to the AppDaemon object
        """

        self.AD = ad
        self.stopping = False
        self.logger = ad.logging.get_child("_utility")

    def stop(self):

        """
        Called by the AppDaemon object to terminate the loop cleanly
        """

        self.logger.debug("stop() called for utility")
        self.stopping = True

    async def loop(self):

        """
        The main utility loop.

        Loops until stop() is called, checks for file changes, overdue threads, thread starvation, and schedules regular state refreshes
        """

        #
        # Setup
        #

        await self.AD.threading.init_admin_stats()
        await self.AD.threading.create_initial_threads()
        await self.AD.app_management.init_admin_stats()

        #
        # Wait for all plugins to initialize
        #

        await self.AD.plugins.wait_for_plugins()

        # Check if we need to bail due to missing metadata

        if self.AD.plugins.required_meta_check() is False:
            if self.AD.stop_function is not None:
                self.AD.stop_function()
            else:
                self.stop()

        if not self.stopping:

            #
            # All plugins are loaded and we have initial state
            # We also have metadata so we can initialise the scheduler
            #

            self.AD.sched = scheduler.Scheduler(self.AD)

            # Create timer loop

            self.logger.debug("Starting timer loop")

            self.AD.loop.create_task(self.AD.sched.loop())

            if self.AD.apps is True:
                self.logger.debug("Reading Apps")

                await self.AD.app_management.check_app_updates()

                self.logger.info("App initialization complete")
                #
                # Fire APPD Started Event
                #
                await self.AD.events.process_event("global", {"event_type": "appd_started", "data": {}})

            self.booted = await self.AD.sched.get_now()
            await self.AD.state.add_entity("admin", "sensor.appdaemon_version", utils.__version__)
            await self.AD.state.add_entity("admin", "sensor.appdaemon_uptime", str(datetime.timedelta(0)))
            await self.AD.state.add_entity("admin", "sensor.appdaemon_booted", utils.dt_to_str((await self.AD.sched.get_now()).replace(microsecond=0), self.AD.tz))
            warning_step = 0
            warning_iterations = 0
            s1 = 0
            e1 = 0

            # Start the loop proper

            while not self.stopping:

                start_time = datetime.datetime.now().timestamp()

                try:

                    if self.AD.apps is True:

                        if self.AD.production_mode is False:
                            # Check to see if config has changed
                            s1 = datetime.datetime.now().timestamp()
                            await self.AD.app_management.check_app_updates()
                            e1 = datetime.datetime.now().timestamp()

                    # Call me suspicious, but lets update state from the plugins periodically

                    await self.AD.plugins.update_plugin_state()

                    # Check for thread starvation

                    warning_step, warning_iterations = await self.AD.threading.check_q_size(warning_step, warning_iterations)

                    # Check for any overdue threads

                    await self.AD.threading.check_overdue_and_dead_threads()

                    # Save any hybrid namespaces

                    self.AD.state.save_hybrid_namespaces()

                    # Run utility for each plugin

                    self.AD.plugins.run_plugin_utility()

                    # Update uptime sensor

                    uptime = (await self.AD.sched.get_now()).replace(microsecond=0) - self.booted.replace(microsecond=0)

                    await self.AD.state.set_state("_utility", "admin", "sensor.appdaemon_uptime", state=str(uptime))

                except:
                    self.logger.warning('-' * 60)
                    self.logger.warning("Unexpected error during utility()")
                    self.logger.warning('-' * 60)
                    self.logger.warning(traceback.format_exc())
                    self.logger.warning('-' * 60)

                end_time = datetime.datetime.now().timestamp()

                loop_duration = (int((end_time - start_time) * 1000) / 1000) * 1000
                check_app_updates_duration = (int((e1 - s1) * 1000) / 1000) * 1000

                self.logger.debug("Util loop compute time: %sms, check_config()=%sms, other=%sms", loop_duration, check_app_updates_duration, loop_duration - check_app_updates_duration)
                if self.AD.sched.realtime is True and loop_duration > (self.AD.max_utility_skew * 1000):
                    self.logger.warning("Excessive time spent in utility loop: %sms, %sms in check_app_updates(), %sms in other", loop_duration, check_app_updates_duration, loop_duration - check_app_updates_duration)
                    if self.AD.check_app_updates_profile is True:
                        self.logger.info("Profile information for Utility Loop")
                        self.logger.info(self.AD.app_management.check_app_updates_profile_stats)

                await asyncio.sleep(self.AD.utility_delay)

            if self.AD.app_management is not None:
                await self.AD.app_management.terminate()

    async def set_production_mode(self, mode=True):
        if mode is True:
            self.logger.info("AD Production Mode Activated")
        else:
            self.logger.info("AD Production Mode Deactivated")
        self.AD.production_mode = mode
<|endoftext|>


<commit_before>import re

from django.db import models, IntegrityError

class NodeException(Exception):
    """ Base class for all Node exceptions """

class DuplicatePathException(NodeException):
    """ the path is already in use """

class InvalidPathException(NodeException):
    """ The path contains non-valid chars or is too short """

class NodeInUse(NodeException):
    """ the node already has content attached to it """

class CantRenameRoot(NodeException):
    """ the root's path is "" which cannot be changed """

class NodeBase(models.Model):
    ROOT_PATH = ""
    ALLOWED_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789_-"
    MAX_PATHLEN = 40
    POSITION_INTERVAL = 100

    validpathre = re.compile("^[%s]{1,%d}$" % (ALLOWED_CHARS, MAX_PATHLEN))

    path = models.CharField(max_length=MAX_PATHLEN, blank=False, unique=True)
    position = models.IntegerField(default=0)

    class Meta:
        abstract = True

    def content(self):
        from .content import Content
        try:
            return self.contentbase.content()
        except Content.DoesNotExist:
            return None

    @classmethod
    def get(cls, path):
        """ retrieve node directly by path. Returns None if not found """
        try:
            return cls.objects.get(path=path)
        except cls.DoesNotExist:
            if path == "":
                return cls.root()

            return None

    def set(self, content, replace=False):
        """
            Set content on the node, optionally replacing existing
            content. This is a more friendly way of setting the node
            on a Content directly
        """
        ## just to be sure, in case it's of type Content in stead of
        ## subclass
        from .content import Content
        content = content.content()

        old = None
        try:
            if self.contentbase:
                if replace:
                    old = self.content()
                    old.node = None
                    old.save()
                else:
                    raise NodeInUse()
        except Content.DoesNotExist:
            pass

        self.contentbase = content #.content_ptr  # XXX is _ptr documented?
        #content.node = self
        content.save()
        self.save()

        return old

    @classmethod
    def root(cls):
        return cls.objects.get_or_create(path=cls.ROOT_PATH)[0]

    def isroot(self):
        return self.path == self.ROOT_PATH

    def add(self, path, position=-1, after=None, before=None):
        """ handle invalid paths (invalid characters, empty, too long) """
        ## lowercasing is the only normalization we do
        path = path.lower()

        if not self.validpathre.match(path):
            raise InvalidPathException(path)

        children = self.children()
        positions = (c.position for c in self.children())

        if after:
            try:
                afterafter_all = self.childrenq(position__gt=after.position,
                                            order="position")
                afterafter = afterafter_all.get()
                position = (after.position + afterafter.position) / 2
                if position == after.position:
                    ## there's a conflict. the new position will be
                    ## "after.position + POSITION_INTERVAL", renumber
                    ## everything else
                    position = after.position + self.POSITION_INTERVAL
                    for i, n in enumerate(afterafter_all):
                        n.position = position + ((i + 1) * self.POSITION_INTERVAL)
                        n.save()
                    # XXX self.debug("repositioning children")
            except Node.DoesNotExist:
                ## after is the last childnode
                position = after.position + self.POSITION_INTERVAL
        elif before:
            try:
                beforebefore_all = self.childrenq(position__lt=before.position,
                                            order="position")
                beforebefore = beforebefore_all.latest("position")
                position = (before.position + beforebefore.position) / 2
                if position == beforebefore.position:
                    ## there's a conflict. the new position will be
                    ## "before.position", renumber before and everything
                    ## else after it.
                    position = before.position
                    everything_after = self.childrenq(
                                           position__gte=before.position,
                                           order="position")
                    for i, n in enumerate(everything_after):
                        n.position = position + ((i + 1) *
                                                 self.POSITION_INTERVAL)
                        n.save()
                    # XXX self.debug("repositioning children")
            except Node.DoesNotExist:
                ## before is the first childnode
                position = before.position - self.POSITION_INTERVAL
        elif position == -1:
            if children.count():
                position = max(positions) + self.POSITION_INTERVAL
            else:
                position = 0

        child = self.__class__(path=self.path + "/" + path,
                               position=position)
        try:
            child.save()
            # XXX child.info(action=create)
        except IntegrityError:
            raise DuplicatePathException(path)
        return child

    def parent(self):
        """ return the parent for this node """
        if self.isroot():
            return self
        parentpath, mypath = self.path.rsplit("/", 1)
        parent = self.__class__.objects.get(path=parentpath)
        return parent

    def childrenq(self, order="position", **kw):
        """ return the raw query for children """
        return self.__class__.objects.filter(
                  path__regex="^%s/[%s]+$" % (self.path, self.ALLOWED_CHARS),
                  **kw
                  ).order_by(order)

    def children(self, order="position"):
        return self.childrenq(order=order)

    def child(self, slug):
        """ return a specific child by its slug """
        childpath = self.path + '/' + slug

        return self.get(childpath)

    def slug(self):
        """ last part of self.path """
        return self.path.rsplit("/", 1)[-1]

    def rename(self, slug):
        """ change the slug """
        if self.isroot():
            raise CantRenameRoot()

        newpath = self.path.rsplit("/", 1)[0] + "/" + slug
        if Node.objects.filter(path=newpath).count():
            raise DuplicatePathException(newpath)
        ## do something transactionish?
        for childs in Node.objects.filter(path__startswith=self.path + '/'):
            remainder = childs.path[len(self.path):]
            childs.path = newpath + remainder
            childs.save()
        self.path = newpath
        self.save()

    def __unicode__(self):
        """ readable representation """
        return u"path %s pos %d" % (self.path or '/', self.position)

WHEEL_NODE_BASECLASS = NodeBase
class Node(WHEEL_NODE_BASECLASS):
    pass
<commit_msg>40 chars is not enough for the entire path<commit_after>import re

from django.db import models, IntegrityError

class NodeException(Exception):
    """ Base class for all Node exceptions """

class DuplicatePathException(NodeException):
    """ the path is already in use """

class InvalidPathException(NodeException):
    """ The path contains non-valid chars or is too short """

class NodeInUse(NodeException):
    """ the node already has content attached to it """

class CantRenameRoot(NodeException):
    """ the root's path is "" which cannot be changed """

class NodeBase(models.Model):
    ROOT_PATH = ""
    ALLOWED_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789_-"
    MAX_PATHLEN = 40
    POSITION_INTERVAL = 100

    validpathre = re.compile("^[%s]{1,%d}$" % (ALLOWED_CHARS, MAX_PATHLEN))

    path = models.CharField(max_length=1024, blank=False, unique=True)
    position = models.IntegerField(default=0)

    class Meta:
        abstract = True

    def content(self):
        from .content import Content
        try:
            return self.contentbase.content()
        except Content.DoesNotExist:
            return None

    @classmethod
    def get(cls, path):
        """ retrieve node directly by path. Returns None if not found """
        try:
            return cls.objects.get(path=path)
        except cls.DoesNotExist:
            if path == "":
                return cls.root()

            return None

    def set(self, content, replace=False):
        """
            Set content on the node, optionally replacing existing
            content. This is a more friendly way of setting the node
            on a Content directly
        """
        ## just to be sure, in case it's of type Content in stead of
        ## subclass
        from .content import Content
        content = content.content()

        old = None
        try:
            if self.contentbase:
                if replace:
                    old = self.content()
                    old.node = None
                    old.save()
                else:
                    raise NodeInUse()
        except Content.DoesNotExist:
            pass

        self.contentbase = content #.content_ptr  # XXX is _ptr documented?
        #content.node = self
        content.save()
        self.save()

        return old

    @classmethod
    def root(cls):
        return cls.objects.get_or_create(path=cls.ROOT_PATH)[0]

    def isroot(self):
        return self.path == self.ROOT_PATH

    def add(self, path, position=-1, after=None, before=None):
        """ handle invalid paths (invalid characters, empty, too long) """
        ## lowercasing is the only normalization we do
        path = path.lower()

        if not self.validpathre.match(path):
            raise InvalidPathException(path)

        children = self.children()
        positions = (c.position for c in self.children())

        if after:
            try:
                afterafter_all = self.childrenq(position__gt=after.position,
                                            order="position")
                afterafter = afterafter_all.get()
                position = (after.position + afterafter.position) / 2
                if position == after.position:
                    ## there's a conflict. the new position will be
                    ## "after.position + POSITION_INTERVAL", renumber
                    ## everything else
                    position = after.position + self.POSITION_INTERVAL
                    for i, n in enumerate(afterafter_all):
                        n.position = position + ((i + 1) * self.POSITION_INTERVAL)
                        n.save()
                    # XXX self.debug("repositioning children")
            except Node.DoesNotExist:
                ## after is the last childnode
                position = after.position + self.POSITION_INTERVAL
        elif before:
            try:
                beforebefore_all = self.childrenq(position__lt=before.position,
                                            order="position")
                beforebefore = beforebefore_all.latest("position")
                position = (before.position + beforebefore.position) / 2
                if position == beforebefore.position:
                    ## there's a conflict. the new position will be
                    ## "before.position", renumber before and everything
                    ## else after it.
                    position = before.position
                    everything_after = self.childrenq(
                                           position__gte=before.position,
                                           order="position")
                    for i, n in enumerate(everything_after):
                        n.position = position + ((i + 1) *
                                                 self.POSITION_INTERVAL)
                        n.save()
                    # XXX self.debug("repositioning children")
            except Node.DoesNotExist:
                ## before is the first childnode
                position = before.position - self.POSITION_INTERVAL
        elif position == -1:
            if children.count():
                position = max(positions) + self.POSITION_INTERVAL
            else:
                position = 0

        child = self.__class__(path=self.path + "/" + path,
                               position=position)
        try:
            child.save()
            # XXX child.info(action=create)
        except IntegrityError:
            raise DuplicatePathException(path)
        return child

    def parent(self):
        """ return the parent for this node """
        if self.isroot():
            return self
        parentpath, mypath = self.path.rsplit("/", 1)
        parent = self.__class__.objects.get(path=parentpath)
        return parent

    def childrenq(self, order="position", **kw):
        """ return the raw query for children """
        return self.__class__.objects.filter(
                  path__regex="^%s/[%s]+$" % (self.path, self.ALLOWED_CHARS),
                  **kw
                  ).order_by(order)

    def children(self, order="position"):
        return self.childrenq(order=order)

    def child(self, slug):
        """ return a specific child by its slug """
        childpath = self.path + '/' + slug

        return self.get(childpath)

    def slug(self):
        """ last part of self.path """
        return self.path.rsplit("/", 1)[-1]

    def rename(self, slug):
        """ change the slug """
        if self.isroot():
            raise CantRenameRoot()

        newpath = self.path.rsplit("/", 1)[0] + "/" + slug
        if Node.objects.filter(path=newpath).count():
            raise DuplicatePathException(newpath)
        ## do something transactionish?
        for childs in Node.objects.filter(path__startswith=self.path + '/'):
            remainder = childs.path[len(self.path):]
            childs.path = newpath + remainder
            childs.save()
        self.path = newpath
        self.save()

    def __unicode__(self):
        """ readable representation """
        return u"path %s pos %d" % (self.path or '/', self.position)

WHEEL_NODE_BASECLASS = NodeBase
class Node(WHEEL_NODE_BASECLASS):
    pass
<|endoftext|>


<commit_before>import os
from validate import ValidationError
from base import HubComponent
from genome import Genome
from genomes_file import GenomesFile

class GroupDefinition(object):

    def __init__(self, name, label=None, priority=1, default_is_closed=0):
        """
        Represents a group of tracks in a trackhub.

        Instances of this class are provided to an assembly.

        :param name:
            String; name for the group (e.g., "map").

        :param label:
            String; the label that will be displayed (e.g., "Mapping")

        :param priority:
            Orders this track group with the other track groups

        :param default_is_closed:
            Determines if this track group is expanded or closed by default. Values to use are 0 or 1
        """
        self.name = str(name)
        if label is None:
            label = name
        self.label = str(label)
        self.priority = int(priority)
        if default_is_closed in (0, 1):
            self.default_is_closed = default_is_closed
        else:
            raise ValueError("default_is_closed must be 1 or 0")

    def __str__(self):
        s = [
            'name %s' % self.name,
            'label %s' % self.label,
            'priority %s' % self.priority,
            'defaultIsClosed %s' % self.default_is_closed
        ]
        return '\n'.join(s) + '\n'

class GroupsFile(HubComponent):
    def __init__(self, groups):
        HubComponent.__init__(self)
        self._local_fn = None
        self._remote_fn = None
        self.groups = []
        self.add_groups(groups)

    def add_groups(self, groups):
        """
        Add a list of GroupDefinition objects.

        :param groups:
            List of GroupDefinition objects.
        """
        self.groups.extend(groups)

    def __str__(self):
        """
        Render groups.txt file.
        """
        return '\n'.join(g.__str__() for g in self.groups)

    @property
    def genomes_file(self):
        genomes_file, level = self.root(GenomesFile)
        if level is None:
            return None
        if level != -2:
            raise ValueError("GenomesFile is level %s, not -2" % level)
        return genomes_file

    @property
    def genome(self):
        genome, level = self.root(Genome)
        if level is None:
            return None
        if level != -1:
            raise ValueError('Genome is level %s, not -1' % level)
        return genome

    @property
    def local_fn(self):
        if self._local_fn is not None:
            return self._local_fn

        if self.genome is None:
            return None

        if self.genomes_file is None:
            return None

        else:
            return os.path.join(os.path.dirname(self.genomes_file.local_fn),
                                self.genome.genome, 'groups.txt')

    @local_fn.setter
    def local_fn(self, fn):
        self._local_fn = fn

    @property
    def remote_fn(self):
        if self._remote_fn is not None:
            return self._remote_fn

        if self.genome is None:
            return None

        if self.genomes_file is None:
            return None

        return os.path.join(os.path.dirname(self.genomes_file.remote_fn),
                            self.genome.genome, "groups.txt")

    @remote_fn.setter
    def remote_fn(self, fn):
        self._remote_fn = fn

    def validate(self):
        if self.genome is None:
            raise ValidationError("GroupsFile object must be attached to an Genome instance or subclass")
        pass

    def _render(self):
        """
        Renders the children GroupDefinition objects to file
        """
        fout = open(self.local_fn, 'w')
        fout.write(str(self))
        fout.close()
        return fout.name

<commit_msg>fixed bug allowing booleans in GroupDefinition<commit_after>import os
from validate import ValidationError
from base import HubComponent
from genome import Genome
from genomes_file import GenomesFile

class GroupDefinition(object):

    def __init__(self, name, label=None, priority=1, default_is_closed=0):
        """
        Represents a group of tracks in a trackhub.

        Instances of this class are provided to an assembly.

        :param name:
            String; name for the group (e.g., "map").

        :param label:
            String; the label that will be displayed (e.g., "Mapping")

        :param priority:
            Orders this track group with the other track groups

        :param default_is_closed:
            Determines if this track group is expanded or closed by default. Values to use are 0 or 1
        """
        self.name = str(name)
        if label is None:
            label = name
        self.label = str(label)
        self.priority = int(priority)
        if default_is_closed in (0, 1):
            self.default_is_closed = default_is_closed
        else:
            raise ValueError("default_is_closed must be 1 or 0")

    def __str__(self):
        s = [
            'name %s' % self.name,
            'label %s' % self.label,
            'priority %s' % self.priority,
            'defaultIsClosed %d' % self.default_is_closed
        ]
        return '\n'.join(s) + '\n'

class GroupsFile(HubComponent):
    def __init__(self, groups):
        HubComponent.__init__(self)
        self._local_fn = None
        self._remote_fn = None
        self.groups = []
        self.add_groups(groups)

    def add_groups(self, groups):
        """
        Add a list of GroupDefinition objects.

        :param groups:
            List of GroupDefinition objects.
        """
        self.groups.extend(groups)

    def __str__(self):
        """
        Render groups.txt file.
        """
        return '\n'.join(g.__str__() for g in self.groups)

    @property
    def genomes_file(self):
        genomes_file, level = self.root(GenomesFile)
        if level is None:
            return None
        if level != -2:
            raise ValueError("GenomesFile is level %s, not -2" % level)
        return genomes_file

    @property
    def genome(self):
        genome, level = self.root(Genome)
        if level is None:
            return None
        if level != -1:
            raise ValueError('Genome is level %s, not -1' % level)
        return genome

    @property
    def local_fn(self):
        if self._local_fn is not None:
            return self._local_fn

        if self.genome is None:
            return None

        if self.genomes_file is None:
            return None

        else:
            return os.path.join(os.path.dirname(self.genomes_file.local_fn),
                                self.genome.genome, 'groups.txt')

    @local_fn.setter
    def local_fn(self, fn):
        self._local_fn = fn

    @property
    def remote_fn(self):
        if self._remote_fn is not None:
            return self._remote_fn

        if self.genome is None:
            return None

        if self.genomes_file is None:
            return None

        return os.path.join(os.path.dirname(self.genomes_file.remote_fn),
                            self.genome.genome, "groups.txt")

    @remote_fn.setter
    def remote_fn(self, fn):
        self._remote_fn = fn

    def validate(self):
        if self.genome is None:
            raise ValidationError("GroupsFile object must be attached to an Genome instance or subclass")
        pass

    def _render(self):
        """
        Renders the children GroupDefinition objects to file
        """
        fout = open(self.local_fn, 'w')
        fout.write(str(self))
        fout.close()
        return fout.name

<|endoftext|>


<commit_before>import os
import platform

from distutils.version import StrictVersion

import cffi

if StrictVersion(cffi.__version__) < StrictVersion('1.0.0'):
    raise RuntimeError(
        'pyspotify-connect requires cffi >= 1.0, but found %s' %
        cffi.__version__)

# Options are armv6l or armv7l
machine = str(platform.machine())
#If travis deploy change x86_64 to armv7l to allow build the source package
if os.getenv('DEPLOY') == 'travis':
    machine = 'armv7l'
if machine == 'armv6l' or machine == 'armv7l':
    header = '#include "spotify.%s.h"' % machine
    header_processed = 'spotify.processed.%s.h' % machine
else:
    raise RuntimeError(
        'pyspotify-connect not available for platform %s' %
        machine)

header_file = os.path.join(os.path.dirname(__file__), header_processed)

with open(header_file) as fh:
    header = fh.read()

ffi = cffi.FFI()
ffi.cdef(header)
ffi.cdef("""
void *malloc(size_t size);
void exit(int status);
""")

ffi.set_source(
    'spotifyconnect._spotifyconnect',
    header,
    libraries=['spotify_embedded_shared'],
    include_dirs=[
        os.path.dirname(__file__)])

if __name__ == '__main__':
    ffi.compile()
<commit_msg>Fix format error<commit_after>import os
import platform

from distutils.version import StrictVersion

import cffi

if StrictVersion(cffi.__version__) < StrictVersion('1.0.0'):
    raise RuntimeError(
        'pyspotify-connect requires cffi >= 1.0, but found %s' %
        cffi.__version__)

# Options are armv6l or armv7l
machine = str(platform.machine())
# If travis deploy change x86_64 to armv7l to allow build the source package
if os.getenv('DEPLOY') == 'travis':
    machine = 'armv7l'
if machine == 'armv6l' or machine == 'armv7l':
    header = '#include "spotify.%s.h"' % machine
    header_processed = 'spotify.processed.%s.h' % machine
else:
    raise RuntimeError(
        'pyspotify-connect not available for platform %s' %
        machine)

header_file = os.path.join(os.path.dirname(__file__), header_processed)

with open(header_file) as fh:
    header = fh.read()

ffi = cffi.FFI()
ffi.cdef(header)
ffi.cdef("""
void *malloc(size_t size);
void exit(int status);
""")

ffi.set_source(
    'spotifyconnect._spotifyconnect',
    header,
    libraries=['spotify_embedded_shared'],
    include_dirs=[
        os.path.dirname(__file__)])

if __name__ == '__main__':
    ffi.compile()
<|endoftext|>


<commit_before>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 03-08-2017 15:44
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
These are the models for representing and dealing with trackhubs
DISCLAIMER - This is the first iteration of the software to create the trackhubs. In this iteration we are going for the
simple version of the hubs, "trackhub quickstart guide" at https://genome.ucsc.edu/goldenpath/help/hubQuickStart.html
"""

# Application imports
import config_manager

# Now, different trackhub builders for the different types of trackhubs (quickstart, custom, etc.)
# DISCLAIMER - I'm aware this first iteration may look a little bit patchy, but it needs to be acknowledged that the
# documentation for modeling this is not very developer friendly, this module it is not meant to be a general module for
# building all sorts of trackhubs and this is the first iteration, so we're not very sure what we want
# TODO - How and where to define the building directors?

# Modeling TrackHubs
class TrackHubGenomeAssembly:
    # TODO
    pass

class TrackCollector:
    # TODO
    pass

# Model for a Track
class BaseTrack:
    _SEPARATOR_CHAR = ' '

    def __init__(self, track, short_label, long_label):
        # Client side of the model
        self.__track = track
        self.__short_label = short_label
        self.__long_label = long_label
        # Builder side of the model
        self.__type = None
        self.__big_data_url = None

    def __str__(self):
        return "track{}{}\n" \
               "type{}{}\n" \
               "bigDataUrl{}{}\n" \
               "shortLabel{}{}\n" \
               "longLabel{}{}" \
            .format(self._SEPARATOR_CHAR, self.get_track(),
                    self._SEPARATOR_CHAR, self.get_type(),
                    self._SEPARATOR_CHAR, self.get_big_data_url(),
                    self._SEPARATOR_CHAR, self.get_short_label(),
                    self._SEPARATOR_CHAR, self.get_long_label())

    def __identify_track_type_from_file(self, file_path):
        # TODO - We return the default type right now, but this will change in the future
        return 'bed'

    def set_type(self, file_path):
        self.__type = self.__identify_track_type_from_file(file_path)

    def set_big_data_url(self, big_data_url):
        self.__big_data_url = big_data_url

    def get_track(self):
        return self.__track

    def get_short_label(self):
        return self.__short_label

    def get_long_label(self):
        return self.__long_label

    def get_type(self):
        return self.__type

    def get_big_data_url(self):
        return self.__big_data_url

    def dump_to_track_db_file(self, file_path):
        with open(file_path, 'w') as f:
            f.write(str(self))

# Model for a trackhub
class TrackHub:
    _SEPARATOR_CHAR = ' '

    def __init__(self, hub, short_label, long_label, email, description_url):
        self.__hub = hub
        self.__short_label = short_label
        self.__long_label = long_label
        self.__email = email
        self.__description_url = description_url
        # Default genomes file
        self.__genomes_file = 'genomes.txt'

    def __str__(self):
        return "hub{}{}\n" \
               "shortLabel{}{}\n" \
               "longLabel{}{}\n" \
               "genomesFile{}{}\n" \
               "email{}{}\n" \
               "descriptionUrl{}{}" \
            .format(self._SEPARATOR_CHAR, self.get_hub(),
                    self._SEPARATOR_CHAR, self.get_short_label(),
                    self._SEPARATOR_CHAR, self.get_long_label(),
                    self._SEPARATOR_CHAR, self.get_genomes_file(),
                    self._SEPARATOR_CHAR, self.get_email(),
                    self._SEPARATOR_CHAR, self.get_description_url())

    def get_hub(self):
        return self.__hub

    def get_short_label(self):
        return self.__short_label

    def get_long_label(self):
        return self.__long_label

    def get_email(self):
        return self.__email

    def get_description_url(self):
        return self.__description_url

    def get_genomes_file(self):
        return self.__genomes_file

    def dump_to_file(self, file_path):
        with open(file_path, 'w') as f:
            f.write(str(self))

if __name__ == '__main__':
    print("ERROR: This script is part of a pipeline collection and it is not meant to be run in stand alone mode")
<commit_msg>key_value_to_str_if_not_none helper<commit_after>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 03-08-2017 15:44
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
These are the models for representing and dealing with trackhubs
DISCLAIMER - This is the first iteration of the software to create the trackhubs. In this iteration we are going for the
simple version of the hubs, "trackhub quickstart guide" at https://genome.ucsc.edu/goldenpath/help/hubQuickStart.html
"""

# Application imports
import config_manager

# Now, different trackhub builders for the different types of trackhubs (quickstart, custom, etc.)
# DISCLAIMER - I'm aware this first iteration may look a little bit patchy, but it needs to be acknowledged that the
# documentation for modeling this is not very developer friendly, this module it is not meant to be a general module for
# building all sorts of trackhubs and this is the first iteration, so we're not very sure what we want
# TODO - How and where to define the building directors?

def key_value_to_str_if_not_none(key, value, separator=' ', suffix='\n'):
    if value:
        return "{}{}{}{}".format(key, separator, value, suffix)
    return ""

# Modeling TrackHubs
class TrackHubGenomeAssembly:
    # TODO
    pass

class TrackCollector:
    # TODO
    pass

# Model for a Track
class BaseTrack:
    _SEPARATOR_CHAR = ' '

    def __init__(self, track, short_label, long_label):
        # Client side of the model
        self.__track = track
        self.__short_label = short_label
        self.__long_label = long_label
        # Builder side of the model
        self.__type = None
        self.__big_data_url = None

    def __str__(self):
        return "track{}{}\n" \
               "type{}{}\n" \
               "bigDataUrl{}{}\n" \
               "shortLabel{}{}\n" \
               "longLabel{}{}" \
            .format(self._SEPARATOR_CHAR, self.get_track(),
                    self._SEPARATOR_CHAR, self.get_type(),
                    self._SEPARATOR_CHAR, self.get_big_data_url(),
                    self._SEPARATOR_CHAR, self.get_short_label(),
                    self._SEPARATOR_CHAR, self.get_long_label())

    def __identify_track_type_from_file(self, file_path):
        # TODO - We return the default type right now, but this will change in the future
        return 'bed'

    def set_type(self, file_path):
        self.__type = self.__identify_track_type_from_file(file_path)

    def set_big_data_url(self, big_data_url):
        self.__big_data_url = big_data_url

    def get_track(self):
        return self.__track

    def get_short_label(self):
        return self.__short_label

    def get_long_label(self):
        return self.__long_label

    def get_type(self):
        return self.__type

    def get_big_data_url(self):
        return self.__big_data_url

    def dump_to_track_db_file(self, file_path):
        with open(file_path, 'w') as f:
            f.write(str(self))

# Model for a trackhub
class TrackHub:
    _SEPARATOR_CHAR = ' '

    def __init__(self, hub, short_label, long_label, email, description_url):
        self.__hub = hub
        self.__short_label = short_label
        self.__long_label = long_label
        self.__email = email
        self.__description_url = description_url
        # Default genomes file
        self.__genomes_file = 'genomes.txt'

    def __str__(self):
        return "hub{}{}\n" \
               "shortLabel{}{}\n" \
               "longLabel{}{}\n" \
               "genomesFile{}{}\n" \
               "email{}{}\n" \
               "descriptionUrl{}{}" \
            .format(self._SEPARATOR_CHAR, self.get_hub(),
                    self._SEPARATOR_CHAR, self.get_short_label(),
                    self._SEPARATOR_CHAR, self.get_long_label(),
                    self._SEPARATOR_CHAR, self.get_genomes_file(),
                    self._SEPARATOR_CHAR, self.get_email(),
                    self._SEPARATOR_CHAR, self.get_description_url())

    def get_hub(self):
        return self.__hub

    def get_short_label(self):
        return self.__short_label

    def get_long_label(self):
        return self.__long_label

    def get_email(self):
        return self.__email

    def get_description_url(self):
        return self.__description_url

    def get_genomes_file(self):
        return self.__genomes_file

    def dump_to_file(self, file_path):
        with open(file_path, 'w') as f:
            f.write(str(self))

if __name__ == '__main__':
    print("ERROR: This script is part of a pipeline collection and it is not meant to be run in stand alone mode")
<|endoftext|>


<commit_before>from collections import defaultdict
import os
import re
from operator import attrgetter

from ryu.controller import ofp_event
from ryu.controller.handler import MAIN_DISPATCHER, DEAD_DISPATCHER
from ryu.controller.handler import set_ev_cls
from ryu.lib import hub
from ryu.base import app_manager
import time
import config
from rrd_data_source import RRDDataSource
from rrdmanager import RRDManager
from switch_stats import SwitchStats
import logging
import switch_stats

log = logging.getLogger('oshi.monitoring.traffic_monitor')
log.setLevel(config.LOG_LEVEL)
# create file handler which logs even debug messages
fh = logging.FileHandler(os.path.join(config.TRAFFIC_MONITOR_LOG_PATH, "traffic_monitor.log"))
fh.setLevel(config.LOG_LEVEL)
fh_complete = logging.FileHandler(os.path.join(config.RRD_LOG_PATH, "complete.log"))
fh_complete.setLevel(config.LOG_LEVEL)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(config.LOG_LEVEL)
# create formatter and add it to the handlers
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
ch.setFormatter(formatter)
fh.setFormatter(formatter)
fh_complete.setFormatter(formatter)
# add the handlers to logger
log.addHandler(ch)
log.addHandler(fh)
log.addHandler(fh_complete)
log.propagate = False

class SimpleMonitor(app_manager.RyuApp):
    def __init__(self, *args, **kwargs):
        super(SimpleMonitor, self).__init__(*args, **kwargs)
        self.switch_stats = {}
        self.rrd_managers = defaultdict()
        self.monitor_thread = hub.spawn(self._monitor)
        self.last_update_time = time.time()

    def _monitor(self):
        log.info("Started monitor thread.")
        while True:
            log.debug("Sending PORT stats requests")
            for switch_stat in self.switch_stats.values():
                data_path = switch_stat.data_path
                open_flow_protocol = data_path.ofproto
                parser = data_path.ofproto_parser
                req = parser.OFPPortStatsRequest(data_path, 0, open_flow_protocol.OFPP_ANY)
                log.debug("Sending PORT stats request for data_path: %s", data_path.id)
                data_path.send_msg(req)
            hub.sleep(config.REQUEST_INTERVAL)

    @set_ev_cls(ofp_event.EventOFPStateChange, [MAIN_DISPATCHER, DEAD_DISPATCHER])
    def _state_change_handler(self, ev):
        datapath = ev.datapath
        ofp_parser = datapath.ofproto_parser
        if ev.state == MAIN_DISPATCHER:
            if datapath.id not in self.switch_stats:
                log.info("Received MAIN_DISPATCHER event. Register datapath: %s", datapath.id)
                self.switch_stats[datapath.id] = SwitchStats(datapath)
                req = ofp_parser.OFPPortDescStatsRequest(datapath, 0)
                datapath.send_msg(req)
                open_flow_protocol = datapath.ofproto
                parser = datapath.ofproto_parser
                cookie = cookie_mask = 0
                req = parser.OFPFlowStatsRequest(datapath, 0,
                                                 open_flow_protocol.OFPTT_ALL,
                                                 open_flow_protocol.OFPP_ANY, open_flow_protocol.OFPG_ANY, cookie,
                                                 cookie_mask)
                datapath.send_msg(req)
        elif ev.state == DEAD_DISPATCHER:
            log.info("Received DEAD_DISPATCHER event.")
            if datapath.id in self.switch_stats:
                log.info("De-register %s datapath", datapath.id)
                del self.switch_stats[datapath.id]

    @staticmethod
    def _init_rrd_data_sources(port_stat_names, data_source_values=None):
        rrd_data_sources = []
        for port_stat_name in port_stat_names:
            log.debug("Building RRD data source for stat: %s", port_stat_name)
            rrd_data_source = RRDDataSource(port_stat_name, config.RRD_DATA_SOURCE_TYPE,
                                            config.RRD_DATA_SOURCE_HEARTBEAT)
            # add an eventual value to the data source. Useful for rrd updates
            if data_source_values is not None and port_stat_name in data_source_values:
                rrd_data_source.temp_value = data_source_values[port_stat_name]
            rrd_data_sources.append(rrd_data_source)
        return rrd_data_sources

    @set_ev_cls(ofp_event.EventOFPPortDescStatsReply, MAIN_DISPATCHER)
    def port_desc_stats_reply_handler(self, ev):
        log.debug("Received event (EventOFPPortDescStatsReply). Body: %s", str(ev.msg.body))
        log.debug("PORT stats received from %s", ev.msg.datapath.id)
        data_path_id = ev.msg.datapath.id
        device_name = data_path_id
        log.debug("Searching for device name for %s", str(data_path_id))
        # search for device name (PHY ports are named using device-port, while virtual ports are named just viX)
        for p in sorted(ev.msg.body, key=attrgetter('port_no')):
            if '-' in p.name and int(p.port_no) <= 1000:
                device_name = p.name.split('-')[0]
                break
        log.debug("Device name for %s: %s", str(data_path_id), device_name)

        ss = self.switch_stats[data_path_id]
        """ :type : SwitchStats """
        ss.device_name = device_name
        for p in sorted(ev.msg.body, key=attrgetter('port_no')):
            if int(p.port_no) > 1000:
                log.debug("Skipping port. Port number: %s", str(p.port_no))
                continue
            virtual = 'vi' in p.name
            port_name = (device_name + '-' + p.name) if virtual else p.name
            log.debug("Initializing port %s (%s). Virtual: %s", port_name, p.port_no, virtual)
            ss.add_port(p.port_no, virtual, port_name)
            if port_name not in self.rrd_managers:
                log.debug("Initializing RRD data sources for %s", port_name)
                port_stats_names = switch_stats.PORT_STATS
                """ :type : list """
                rrd_data_sources = self._init_rrd_data_sources(port_stats_names)
                log.debug("Initialized RRD data sources for %s", port_name)
                log.info("Creating RRD Manager for port %s of %s", port_name, device_name)
                self.rrd_managers[port_name] = RRDManager(port_name + '.rrd', rrd_data_sources)
            else:
                log.debug("Skip RRD Manager creation for port %s of %s as it's already available",
                          port_name, data_path_id)
            log.debug("Completed port %s (%s) initialization", port_name, p.port_no)

    @set_ev_cls(ofp_event.EventOFPFlowStatsReply, MAIN_DISPATCHER)
    def _flow_stats_reply_handler(self, ev):
        log.debug("Received event (EventOFPFlowStatsReply). Body: %s", str(ev.msg.body))
        log.debug("FLOW stats received from %s", ev.msg.datapath.id)
        ss = self.switch_stats[ev.msg.datapath.id]
        body = ev.msg.body
        log.debug("Setting IP partners info for %s", ss.device_name)
        for flow_stat in body:
            log.debug("Getting port info from flow stat: %s", str(flow_stat))
            try:
                in_port = flow_stat.match.fields[0].value
                log.debug("Got IN port: %s", str(in_port))
                if long(in_port) > 1000:
                    log.debug("Skipping flow_stat. IN port: %s", str(in_port))
                    continue
                out_port = flow_stat.instructions[0].actions[0].port
                if long(out_port) > 1000:
                    log.debug("Skipping flow_stat. OUT port: %s", str(out_port))
                    continue
                log.debug("Got OUT port: %s", str(out_port))
                out_port_name = ss.get_port_name(out_port)
                if len(re.findall(r"vi+[0-9]", out_port_name, flags=0)) == 1:
                    log.debug("Setting IN/OUT ports for %s, IN port: %s, OUT port: %s", ss.device_name, in_port,
                              out_port)
                    ss.set_ip_partner_port_number(in_port, out_port)
                    ss.set_ip_partner_port_number(out_port, in_port)
            except Exception:
                log.exception("Error while handling stat reply.")
                continue

    @set_ev_cls(ofp_event.EventOFPPortStatsReply, MAIN_DISPATCHER)
    def _port_stats_reply_handler(self, ev):
        data_path_id = ev.msg.datapath.id
        log.debug("Received event (EventOFPPortStatsReply). Body: %s", str(ev.msg.body))
        ss = self.switch_stats[data_path_id]
        log.info("PORT STATS reply for %s", ss.device_name)
        """ :type : SwitchStats """
        body = ev.msg.body
        log.info("Updating stats for %s", ss.device_name)
        for port in sorted(body, key=attrgetter('port_no')):
            if int(port.port_no) > 1000:
                log.debug("Skipping port. Port number: %s", str(port.port_no))
                continue
            log.info("Updating stats for %s", ss.get_port_name(port.port_no))
            ss.set_rx_bytes(port.port_no, port.rx_bytes)
            ss.set_tx_bytes(port.port_no, port.tx_bytes)
            ss.set_rx_packets(port.port_no, port.rx_packets)
            ss.set_tx_packets(port.port_no, port.tx_packets)

        port_numbers = ss.ports.keys()
        log.debug("Ports to update: %s", str(port_numbers))

        # update RRD if necessary
        current_time = time.time()
        if current_time - self.last_update_time >= config.RRD_STEP:
            log.debug("Updating RRDs")
            for port_number in port_numbers:
                log.debug("Updating port %s", ss.get_port_name(port_number))
                current_stats = ss.get_current_values(port_number)
                log.debug("Current stats for port %s (%s): %s", port_number, ss.get_port_name(port_number),
                          str(current_stats))
                rrd_data_sources_to_update = []
                log.debug("Building RRD data source for port %s and stats: %s", ss.get_port_name(port_number),
                          str(current_stats))
                for stat_name in current_stats:
                    log.debug("Building RRD data source to update %s", stat_name)
                    # use RRDDataSource object as DTO, so we need only data source name and data source current value
                    rrd_data_sources_to_update.append(RRDDataSource(stat_name, None, None, current_stats[stat_name]))
                log.debug("Completed RRD data sources initialization to update %s: %s", ss.device_name,
                          str(rrd_data_sources_to_update))
                if ss.get_port_name(port_number) in self.rrd_managers:
                    log.debug("Updating RRD for %s.", ss.device_name)
                    rrd_manager = self.rrd_managers[ss.get_port_name(port_number)]
                    """ :type : RRDManager """
                    rrd_manager.update(rrd_data_sources_to_update)
                else:
                    log.debug("Cannot find RRD manager for %s. Available managers: %s", ss.get_port_name(port_number),
                              str(self.rrd_managers))
            self.last_update_time = time.time()
        else:
            log.debug("Queueing up RRD update")
<commit_msg>Raised log level when updating RRD<commit_after>from collections import defaultdict
import os
import re
from operator import attrgetter

from ryu.controller import ofp_event
from ryu.controller.handler import MAIN_DISPATCHER, DEAD_DISPATCHER
from ryu.controller.handler import set_ev_cls
from ryu.lib import hub
from ryu.base import app_manager
import time
import config
from rrd_data_source import RRDDataSource
from rrdmanager import RRDManager
from switch_stats import SwitchStats
import logging
import switch_stats

log = logging.getLogger('oshi.monitoring.traffic_monitor')
log.setLevel(config.LOG_LEVEL)
# create file handler which logs even debug messages
fh = logging.FileHandler(os.path.join(config.TRAFFIC_MONITOR_LOG_PATH, "traffic_monitor.log"))
fh.setLevel(config.LOG_LEVEL)
fh_complete = logging.FileHandler(os.path.join(config.RRD_LOG_PATH, "complete.log"))
fh_complete.setLevel(config.LOG_LEVEL)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(config.LOG_LEVEL)
# create formatter and add it to the handlers
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
ch.setFormatter(formatter)
fh.setFormatter(formatter)
fh_complete.setFormatter(formatter)
# add the handlers to logger
log.addHandler(ch)
log.addHandler(fh)
log.addHandler(fh_complete)
log.propagate = False

class SimpleMonitor(app_manager.RyuApp):
    def __init__(self, *args, **kwargs):
        super(SimpleMonitor, self).__init__(*args, **kwargs)
        self.switch_stats = {}
        self.rrd_managers = defaultdict()
        self.monitor_thread = hub.spawn(self._monitor)
        self.last_update_time = time.time()

    def _monitor(self):
        log.info("Started monitor thread.")
        while True:
            log.debug("Sending PORT stats requests")
            for switch_stat in self.switch_stats.values():
                data_path = switch_stat.data_path
                open_flow_protocol = data_path.ofproto
                parser = data_path.ofproto_parser
                req = parser.OFPPortStatsRequest(data_path, 0, open_flow_protocol.OFPP_ANY)
                log.debug("Sending PORT stats request for data_path: %s", data_path.id)
                data_path.send_msg(req)
            hub.sleep(config.REQUEST_INTERVAL)

    @set_ev_cls(ofp_event.EventOFPStateChange, [MAIN_DISPATCHER, DEAD_DISPATCHER])
    def _state_change_handler(self, ev):
        datapath = ev.datapath
        ofp_parser = datapath.ofproto_parser
        if ev.state == MAIN_DISPATCHER:
            if datapath.id not in self.switch_stats:
                log.info("Received MAIN_DISPATCHER event. Register datapath: %s", datapath.id)
                self.switch_stats[datapath.id] = SwitchStats(datapath)
                req = ofp_parser.OFPPortDescStatsRequest(datapath, 0)
                datapath.send_msg(req)
                open_flow_protocol = datapath.ofproto
                parser = datapath.ofproto_parser
                cookie = cookie_mask = 0
                req = parser.OFPFlowStatsRequest(datapath, 0,
                                                 open_flow_protocol.OFPTT_ALL,
                                                 open_flow_protocol.OFPP_ANY, open_flow_protocol.OFPG_ANY, cookie,
                                                 cookie_mask)
                datapath.send_msg(req)
        elif ev.state == DEAD_DISPATCHER:
            log.info("Received DEAD_DISPATCHER event.")
            if datapath.id in self.switch_stats:
                log.info("De-register %s datapath", datapath.id)
                del self.switch_stats[datapath.id]

    @staticmethod
    def _init_rrd_data_sources(port_stat_names, data_source_values=None):
        rrd_data_sources = []
        for port_stat_name in port_stat_names:
            log.debug("Building RRD data source for stat: %s", port_stat_name)
            rrd_data_source = RRDDataSource(port_stat_name, config.RRD_DATA_SOURCE_TYPE,
                                            config.RRD_DATA_SOURCE_HEARTBEAT)
            # add an eventual value to the data source. Useful for rrd updates
            if data_source_values is not None and port_stat_name in data_source_values:
                rrd_data_source.temp_value = data_source_values[port_stat_name]
            rrd_data_sources.append(rrd_data_source)
        return rrd_data_sources

    @set_ev_cls(ofp_event.EventOFPPortDescStatsReply, MAIN_DISPATCHER)
    def port_desc_stats_reply_handler(self, ev):
        log.debug("Received event (EventOFPPortDescStatsReply). Body: %s", str(ev.msg.body))
        log.debug("PORT stats received from %s", ev.msg.datapath.id)
        data_path_id = ev.msg.datapath.id
        device_name = data_path_id
        log.debug("Searching for device name for %s", str(data_path_id))
        # search for device name (PHY ports are named using device-port, while virtual ports are named just viX)
        for p in sorted(ev.msg.body, key=attrgetter('port_no')):
            if '-' in p.name and int(p.port_no) <= 1000:
                device_name = p.name.split('-')[0]
                break
        log.debug("Device name for %s: %s", str(data_path_id), device_name)

        ss = self.switch_stats[data_path_id]
        """ :type : SwitchStats """
        ss.device_name = device_name
        for p in sorted(ev.msg.body, key=attrgetter('port_no')):
            if int(p.port_no) > 1000:
                log.debug("Skipping port. Port number: %s", str(p.port_no))
                continue
            virtual = 'vi' in p.name
            port_name = (device_name + '-' + p.name) if virtual else p.name
            log.debug("Initializing port %s (%s). Virtual: %s", port_name, p.port_no, virtual)
            ss.add_port(p.port_no, virtual, port_name)
            if port_name not in self.rrd_managers:
                log.debug("Initializing RRD data sources for %s", port_name)
                port_stats_names = switch_stats.PORT_STATS
                """ :type : list """
                rrd_data_sources = self._init_rrd_data_sources(port_stats_names)
                log.debug("Initialized RRD data sources for %s", port_name)
                log.info("Creating RRD Manager for port %s of %s", port_name, device_name)
                self.rrd_managers[port_name] = RRDManager(port_name + '.rrd', rrd_data_sources)
            else:
                log.debug("Skip RRD Manager creation for port %s of %s as it's already available",
                          port_name, data_path_id)
            log.debug("Completed port %s (%s) initialization", port_name, p.port_no)

    @set_ev_cls(ofp_event.EventOFPFlowStatsReply, MAIN_DISPATCHER)
    def _flow_stats_reply_handler(self, ev):
        log.debug("Received event (EventOFPFlowStatsReply). Body: %s", str(ev.msg.body))
        log.debug("FLOW stats received from %s", ev.msg.datapath.id)
        ss = self.switch_stats[ev.msg.datapath.id]
        body = ev.msg.body
        log.debug("Setting IP partners info for %s", ss.device_name)
        for flow_stat in body:
            log.debug("Getting port info from flow stat: %s", str(flow_stat))
            try:
                in_port = flow_stat.match.fields[0].value
                log.debug("Got IN port: %s", str(in_port))
                if long(in_port) > 1000:
                    log.debug("Skipping flow_stat. IN port: %s", str(in_port))
                    continue
                out_port = flow_stat.instructions[0].actions[0].port
                if long(out_port) > 1000:
                    log.debug("Skipping flow_stat. OUT port: %s", str(out_port))
                    continue
                log.debug("Got OUT port: %s", str(out_port))
                out_port_name = ss.get_port_name(out_port)
                if len(re.findall(r"vi+[0-9]", out_port_name, flags=0)) == 1:
                    log.debug("Setting IN/OUT ports for %s, IN port: %s, OUT port: %s", ss.device_name, in_port,
                              out_port)
                    ss.set_ip_partner_port_number(in_port, out_port)
                    ss.set_ip_partner_port_number(out_port, in_port)
            except Exception:
                log.exception("Error while handling stat reply.")
                continue

    @set_ev_cls(ofp_event.EventOFPPortStatsReply, MAIN_DISPATCHER)
    def _port_stats_reply_handler(self, ev):
        data_path_id = ev.msg.datapath.id
        log.debug("Received event (EventOFPPortStatsReply). Body: %s", str(ev.msg.body))
        ss = self.switch_stats[data_path_id]
        log.info("PORT STATS reply for %s", ss.device_name)
        """ :type : SwitchStats """
        body = ev.msg.body
        log.info("Updating stats for %s", ss.device_name)
        for port in sorted(body, key=attrgetter('port_no')):
            if int(port.port_no) > 1000:
                log.debug("Skipping port. Port number: %s", str(port.port_no))
                continue
            log.info("Updating stats for %s", ss.get_port_name(port.port_no))
            ss.set_rx_bytes(port.port_no, port.rx_bytes)
            ss.set_tx_bytes(port.port_no, port.tx_bytes)
            ss.set_rx_packets(port.port_no, port.rx_packets)
            ss.set_tx_packets(port.port_no, port.tx_packets)

        port_numbers = ss.ports.keys()
        log.debug("Ports to update: %s", str(port_numbers))

        # update RRD if necessary
        current_time = time.time()
        if current_time - self.last_update_time >= config.RRD_STEP:
            log.debug("Updating RRDs")
            for port_number in port_numbers:
                log.debug("Updating port %s", ss.get_port_name(port_number))
                current_stats = ss.get_current_values(port_number)
                log.debug("Current stats for port %s (%s): %s", port_number, ss.get_port_name(port_number),
                          str(current_stats))
                rrd_data_sources_to_update = []
                log.debug("Building RRD data source for port %s and stats: %s", ss.get_port_name(port_number),
                          str(current_stats))
                for stat_name in current_stats:
                    log.debug("Building RRD data source to update %s", stat_name)
                    # use RRDDataSource object as DTO, so we need only data source name and data source current value
                    rrd_data_sources_to_update.append(RRDDataSource(stat_name, None, None, current_stats[stat_name]))
                log.debug("Completed RRD data sources initialization to update %s: %s", ss.device_name,
                          str(rrd_data_sources_to_update))
                if ss.get_port_name(port_number) in self.rrd_managers:
                    log.info("Updating RRD for %s.", ss.device_name)
                    rrd_manager = self.rrd_managers[ss.get_port_name(port_number)]
                    """ :type : RRDManager """
                    rrd_manager.update(rrd_data_sources_to_update)
                else:
                    log.debug("Cannot find RRD manager for %s. Available managers: %s", ss.get_port_name(port_number),
                              str(self.rrd_managers))
            self.last_update_time = time.time()
        else:
            log.debug("Queueing up RRD update")
<|endoftext|>