<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import err
import heapq
# needed for testing

import random as rand

#  FIX heap duplicates used by different ksp!

def factory(graph_lib):
    if graph_lib == 'nx':
        from graphNX import GraphNX
        return GraphNX()
    elif graph_lib == 'gt':
        from graphGT import GraphGT
        return GraphGT()
    elif graph_lib == 'g':
        from graph_generic import Graph
        return Graph()
    else:
        raise err.Fatal('unknown graph library requested: {}'.format(graph_lib))

def random_graph(
    num_nodes,
    num_edges,
    dim,
    seed=None,
    directed=False,
    ):

    raise NotImplementedError('modify to work with new abstraction object')
    G = nx.gnm_random_graph(num_nodes, num_edges, seed, directed)
    abstract_G = nx.DiGraph()

    # create a mapping from G to abstract_G

    graph_map = {}

    # innane pos_dict for plotting only

    pos_dict = {}

    # create the nodes of abstract_G

    N_MAX = 5
    for n in G:
        n_ = (n, rand.randint(0, N_MAX))
        pos_dict[n_] = n_
        graph_map[n] = n_
        abstract_G.add_node(n_)

    # add edges to abstract_G

    edge_list = G.edges()
    edge_list_ = list(map(lambda e: (graph_map[e[0]],
                      graph_map[e[1]]), edge_list))
    abstract_G.add_edges_from(edge_list_)

#    nx.draw_networkx(abstract_G, pos=pos_dict, with_labels=True)
#    plt.show()

    return Graph(abstract_G, 'test_random')

# generates custom graphs to to test the correctness of the abstraction

def custom_graph(ID, num_dim):
    if num_dim != 2:
        raise NotImplementedError
    if ID == 1:
        return csg1()
    else:
        raise NotImplementedError

def csg1():
    G = nx.DiGraph()
    MAX_N = 10

    init_state = (0, 0)
    final_state = (MAX_N + 1, MAX_N + 1)
    edge_list = [((n, n), (n + 1, n + 1)) for n in range(1, MAX_N)]
    G.add_edge(init_state, edge_list[0][0])
    G.add_edges_from(edge_list)
    G.add_edge(edge_list[-1][-1], final_state)
    return Graph(G, 'test_csg1')

#    print(G.nodes())
#    print(G.edges())
#    plt.figure()
#    Graph(G).draw_2d()
#    plt.show()

# import networkx as nx
# import heapq
# from ModifiedDijkstra import ModifiedDijkstra

class YenKShortestPaths(object):

    """
     This is a straight forward implementation of Yen's K shortest loopless
     path algorithm. No attempt has been made to perform any optimization that
     have been suggested in the literature. Our main goal was to have a
     functioning K-shortest path  algorithm. This implementation should work
     for both undirected and directed  graphs. However it has only been tested
     so far against undirected graphs.
    """

    def __init__(
        self,
        graph,
        weight='weight',
        cap='capacity',
        ):
        """
        Constructor
        """

        self.wt = weight
        self.cap = cap
        self.g = graph
        self.pathHeap = []  # Use the heapq module functions heappush(pathHeap, item) and heappop(pathHeap, item)
        self.pathList = []  # Contains WeightedPath objects
        self.deletedEdges = set()
        self.deletedNodes = set()
        self.kPath = None

        # Make a copy of the graph tempG that we can manipulate

        if isinstance(graph, nx.Graph):

            # self.tempG = graph.copy()

            if nx.is_directed(graph):
                self.tempG = nx.DiGraph(graph)
            else:
                self.tempG = nx.Graph(graph)
        else:
            self.tempG = None

    def findFirstShortestPath(self, source, dest):
        """
        This function is called to initialize the k-shortest path algorithm.
        It also finds the shortest path in the network.
        You can use this function to restart the algorithm at anytime with
        possibly different source and destination values.
        @param source    The beginning node of the path.
        @param dest      The termination node of the path.
        @return          The shortest path or null if the path doesn't exist.
        """

        # Used to initialize or reinitialize the algorithm
        # Computes the shortest path via Dijsktra

        self.kPath = None
        self.pathHeap = []
        self.pathList = []
        self.source = source
        self.dest = dest

        # Compute the shortest path
        # nodeList = nx.dijkstra_path(self.g, source, dest, self.wt)

        alg = ModifiedDijkstra(self.g, self.wt)
        nodeList = alg.getPath(source, dest, as_nodes=True)
        if len(nodeList) == 0:
            return None
        deletedLinks = set()
        self.kPath = WeightedPath(nodeList, deletedLinks, self.g, wt=self.wt,
                                  cap=self.cap)
        self.kPath.dNode = source
        self.pathList.append(self.kPath)
        return self.kPath

    def getNextShortestPath(self):
        """
        Use this function to compute successive shortest path. Each one will have
        a length (cost) greater than or equal the previously generated algorithm.
        Returns null if no more paths can be found.
        You must first call findFirstShortestPath(source, dest) to initialize the
        algorithm and set the source and destination node.
        @return  the next shortest path (or the next longer path depending on
        how you want to think about things).
        """

        if self.kPath == None:
            raise UserWarning('Must call findFirstShortestPath before this method or no path exists'
                              )

        # Iterate over all the nodes in kPath from dNode to the node before the destination
        # and add candidate paths to the path heap.

        kNodes = self.kPath.nodeList
        index = kNodes.index(self.kPath.dNode)
        curNode = kNodes[index]
        while curNode != self.dest:
            self._removeEdgesNodes(curNode)
            candidate = self._computeCandidatePath(curNode)
            self._restoreGraph()
            if candidate != None:
                heapq.heappush(self.pathHeap, candidate)
            index = index + 1
            curNode = kNodes[index]

        if len(self.pathHeap) == 0:
            return None
        p = heapq.heappop(self.pathHeap)  # after iterations contains next shortest path
        self.pathList.append(p)
        self.kPath = p  # updates the kth path
        return p

    def _removeEdgesNodes(self, curNode):
        """
        Remove all nodes from source to the node before the current node in kPath.
        Delete the edge between curNode and the next node in kPath
        Delete any edges previously deleted in kPath starting at curNode
        add all deleted edges to the deleted edge list.
        """

        # Figure out all edges to be removed first then take them out of the temp graph
        # then remove all the nodes from the temp graph.
        # At the start the temp graph is equal to the initial graph.

        self.deletedEdges = set()
        self.deletedNodes = set()
        kNodes = self.kPath.nodeList
        index = 0
        tempNode = kNodes[index]
        index += 1
        while tempNode != curNode:
            edges = self.tempG.edges(tempNode)
            if len(edges) != 0:
                for edge in edges:
                    self.deletedEdges.add(edge)
                    self.tempG.remove_edge(edge[0], edge[1])

            #

            self.deletedNodes.add(tempNode)
            self.tempG.remove_node(tempNode)
            tempNode = kNodes[index]
            index += 1

        # Also need to remove those old deleted edges that start on curNode

        oldDelEdges = self.kPath.deletedEdges
        if self.g.is_directed():
            outEdges = self.g.out_edges(curNode)
        else:
            outEdges = self.g.edges(curNode)

        # outEdges = self.g.edges(curNode)

        for e in outEdges:
            if e in oldDelEdges:
                self.deletedEdges.add(e)
                self.tempG.remove_edge(e[0], e[1])

        # Now delete the edge from the curNode to the next in the path

        tempNode = kNodes[index]
        e = (curNode, tempNode)
        self.deletedEdges.add(e)
        self.tempG.remove_edge(curNode, tempNode)

    def _computeCandidatePath(self, curNode):
        """
        Compute the shortest path on the modified graph and then
        combines with the portion of kPath from the source up through
        the deviation node
        """

#        DijkstraShortestPath alg = new DijkstraShortestPath(tempG, wt);
#        List<E> ePath = alg.getPath(curNode, dest);

        # nodeList = nx.dijkstra_path(self.tempG, curNode, self.dest, self.wt)

        alg = ModifiedDijkstra(self.tempG, self.wt)
        nodeList = alg.getPath(curNode, self.dest, as_nodes=True)

        # Trying this out...

        if nodeList == None:
            return None

        # Get first part of the path from kPath

        nodePath = []
        if curNode in self.kPath.nodeList:
            index = self.kPath.nodeList.index(curNode)
            nodePath = self.kPath.nodeList[0:index]

        nodePath.extend(nodeList)
        wp = WeightedPath(nodePath, self.deletedEdges, self.g, wt=self.wt,
                          cap=self.cap)
        wp.dNode = curNode
        return wp

    def _restoreGraph(self):
        """
        Using the internal deleted node and deleted edge containers
        restores the temp graph to match the graph g.
        """

        # self.tempG = self.g.copy()

        if nx.is_directed(self.g):
            self.tempG = nx.DiGraph(self.g)
        else:
            self.tempG = nx.Graph(self.g)
        self.deletedEdges = []
        self.deletedNodes = []

class WeightedPath(object):

    """Used internally by the Yen k-shortest path algorithm.
    Also return to user as a result.
    """

    def __init__(
        self,
        pathNodeList,
        deletedEdges,
        g,
        wt='weight',
        cap='capacity',
        ):
        """
        Constructor
        """

        self.nodeList = pathNodeList
        self.deletedEdges = set(deletedEdges)
        self.g = g
        self.wt = wt
        self.dNode = None  # The deflection node
        self.cost = 0.0
        self.capacity = float('inf')

        # print("WtPath pathNodeList: {}".format(pathNodeList))

        for i in range(len(pathNodeList) - 1):
            self.cost = self.cost + g[pathNodeList[i]][pathNodeList[i + 1]][wt]

################################# ADI: Do we need this? ##############################
####            if not cap == None:
####                self.capacity = min(self.capacity, g[pathNodeList[i]][pathNodeList[i+1]][cap])
####            else:
####                self.capacity = None
######################################################################################

    def __cmp__(self, other):
        if other == None:
            return -1
        return cmp(self.cost, other.cost)

    def __str__(self):
        return 'nodeList: {}, cost: {}, capacity: {}'.format(self.nodeList,
                self.cost, self.capacity)

class ModifiedDijkstra(object):

    """
    The Modified Dijkstra algorithm from "Survivable Networks" by Ramesh Bhandari.
    This algorithm works with graphs that can have directed or undirected links.
    In addition, this algorithm can correctly function in some cases of negative
    arc lengths that arise in the disjoint path computations.

    Works with graphs, *g*, in NetworkX format. Specifically Graph and
    DiGraph classes.
    """

    def __init__(self, g, wt='weight'):
        """
        Constructor. Parameter *g* is a NetworkX Graph or DiGraph instance.
        The *wt* keyword argument sets the link attribute to be used in computing
        the path length.
        """

        self.dist = {}  # A map from nodes to their labels (float)
        self.predecessor = {}  # A map from a node to a node
        self.g = g
        self.wt = wt
        edges = g.edges()

        # Set the value for infinite distance in the graph

        self.inf = 0.0
        for e in edges:
            self.inf += abs(g[e[0]][e[1]][wt])
        self.inf += 1.0

    def getPath(
        self,
        source,
        dest,
        as_nodes=False,
        ):
        """
        Computes the shortest path in the graph between the given *source* and *dest*
        node (strings).  Returns the path as a list of links (default) or as a list of
        nodes by setting the *as_nodes* keyword argument to *True*.
        """

        self.dist = {}  # A map from nodes to their labels (float)
        self.predecessor = {}  # A map from a node to a node

        # Initialize the distance labels to "infinity"

        vertices = self.g.nodes()
        for vertex in vertices:
            self.dist[vertex] = self.inf
            self.predecessor[vertex] = source

        # Further set up the distance from the source to itself and
        # to all one hops away.

        self.dist[source] = 0.0
        if self.g.is_directed():
            outEdges = self.g.out_edges([source])
        else:
            outEdges = self.g.edges([source])
        for edge in outEdges:
            self.dist[edge[1]] = self.g[edge[0]][edge[1]][self.wt]

        s = set(vertices)
        s.remove(source)
        currentMin = self._findMinNode(s)
        if currentMin == None:
            return None
        s.remove(currentMin)
        while currentMin != dest and len(s) != 0 and currentMin != None:
            if self.g.is_directed():
                outEdges = self.g.out_edges([currentMin])
            else:
                outEdges = self.g.edges([currentMin])
            for edge in outEdges:
                opposite = edge[1]
                if self.dist[currentMin] + self.g[edge[0]][edge[1]][self.wt] \
                    < self.dist[opposite]:
                    self.dist[opposite] = self.dist[currentMin] \
                        + self.g[edge[0]][edge[1]][self.wt]
                    self.predecessor[opposite] = currentMin
                    s.add(opposite)

            currentMin = self._findMinNode(s)

            # print("Current min node {}, s = {}".format(currentMin, s))

            if currentMin == None:
                return None
            s.remove(currentMin)

        # Compute the path as a list of edges

        currentNode = dest
        predNode = self.predecessor.get(dest)
        node_list = [dest]
        done = False
        path = []
        while not done:
            path.append((predNode, currentNode))
            currentNode = predNode
            predNode = self.predecessor[predNode]
            node_list.append(currentNode)
            done = currentNode == source
        node_list.reverse()
        if as_nodes:
            return node_list
        else:
            return path

    def _findMinNode(self, s):
        """
        Finds the vertex with the minimum distance label in the set "s".
        returns the minimum vertex
        """

        minNode = None
        minVal = self.inf
        for vertex in s:
            if self.dist[vertex] < minVal:
                minVal = self.dist[vertex]
                minNode = vertex
        return minNode
<commit_msg>abs import fix<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import err
import heapq
# needed for testing

import random as rand

#  FIX heap duplicates used by different ksp!

def factory(graph_lib):
    if graph_lib == 'nx':
        from .graphNX import GraphNX
        return GraphNX()
    elif graph_lib == 'gt':
        from .graphGT import GraphGT
        return GraphGT()
    elif graph_lib == 'g':
        from .graph_generic import Graph
        return Graph()
    else:
        raise err.Fatal('unknown graph library requested: {}'.format(graph_lib))

def random_graph(
    num_nodes,
    num_edges,
    dim,
    seed=None,
    directed=False,
    ):

    raise NotImplementedError('modify to work with new abstraction object')
    G = nx.gnm_random_graph(num_nodes, num_edges, seed, directed)
    abstract_G = nx.DiGraph()

    # create a mapping from G to abstract_G

    graph_map = {}

    # innane pos_dict for plotting only

    pos_dict = {}

    # create the nodes of abstract_G

    N_MAX = 5
    for n in G:
        n_ = (n, rand.randint(0, N_MAX))
        pos_dict[n_] = n_
        graph_map[n] = n_
        abstract_G.add_node(n_)

    # add edges to abstract_G

    edge_list = G.edges()
    edge_list_ = list(map(lambda e: (graph_map[e[0]],
                      graph_map[e[1]]), edge_list))
    abstract_G.add_edges_from(edge_list_)

#    nx.draw_networkx(abstract_G, pos=pos_dict, with_labels=True)
#    plt.show()

    return Graph(abstract_G, 'test_random')

# generates custom graphs to to test the correctness of the abstraction

def custom_graph(ID, num_dim):
    if num_dim != 2:
        raise NotImplementedError
    if ID == 1:
        return csg1()
    else:
        raise NotImplementedError

def csg1():
    G = nx.DiGraph()
    MAX_N = 10

    init_state = (0, 0)
    final_state = (MAX_N + 1, MAX_N + 1)
    edge_list = [((n, n), (n + 1, n + 1)) for n in range(1, MAX_N)]
    G.add_edge(init_state, edge_list[0][0])
    G.add_edges_from(edge_list)
    G.add_edge(edge_list[-1][-1], final_state)
    return Graph(G, 'test_csg1')

#    print(G.nodes())
#    print(G.edges())
#    plt.figure()
#    Graph(G).draw_2d()
#    plt.show()

# import networkx as nx
# import heapq
# from ModifiedDijkstra import ModifiedDijkstra

class YenKShortestPaths(object):

    """
     This is a straight forward implementation of Yen's K shortest loopless
     path algorithm. No attempt has been made to perform any optimization that
     have been suggested in the literature. Our main goal was to have a
     functioning K-shortest path  algorithm. This implementation should work
     for both undirected and directed  graphs. However it has only been tested
     so far against undirected graphs.
    """

    def __init__(
        self,
        graph,
        weight='weight',
        cap='capacity',
        ):
        """
        Constructor
        """

        self.wt = weight
        self.cap = cap
        self.g = graph
        self.pathHeap = []  # Use the heapq module functions heappush(pathHeap, item) and heappop(pathHeap, item)
        self.pathList = []  # Contains WeightedPath objects
        self.deletedEdges = set()
        self.deletedNodes = set()
        self.kPath = None

        # Make a copy of the graph tempG that we can manipulate

        if isinstance(graph, nx.Graph):

            # self.tempG = graph.copy()

            if nx.is_directed(graph):
                self.tempG = nx.DiGraph(graph)
            else:
                self.tempG = nx.Graph(graph)
        else:
            self.tempG = None

    def findFirstShortestPath(self, source, dest):
        """
        This function is called to initialize the k-shortest path algorithm.
        It also finds the shortest path in the network.
        You can use this function to restart the algorithm at anytime with
        possibly different source and destination values.
        @param source    The beginning node of the path.
        @param dest      The termination node of the path.
        @return          The shortest path or null if the path doesn't exist.
        """

        # Used to initialize or reinitialize the algorithm
        # Computes the shortest path via Dijsktra

        self.kPath = None
        self.pathHeap = []
        self.pathList = []
        self.source = source
        self.dest = dest

        # Compute the shortest path
        # nodeList = nx.dijkstra_path(self.g, source, dest, self.wt)

        alg = ModifiedDijkstra(self.g, self.wt)
        nodeList = alg.getPath(source, dest, as_nodes=True)
        if len(nodeList) == 0:
            return None
        deletedLinks = set()
        self.kPath = WeightedPath(nodeList, deletedLinks, self.g, wt=self.wt,
                                  cap=self.cap)
        self.kPath.dNode = source
        self.pathList.append(self.kPath)
        return self.kPath

    def getNextShortestPath(self):
        """
        Use this function to compute successive shortest path. Each one will have
        a length (cost) greater than or equal the previously generated algorithm.
        Returns null if no more paths can be found.
        You must first call findFirstShortestPath(source, dest) to initialize the
        algorithm and set the source and destination node.
        @return  the next shortest path (or the next longer path depending on
        how you want to think about things).
        """

        if self.kPath == None:
            raise UserWarning('Must call findFirstShortestPath before this method or no path exists'
                              )

        # Iterate over all the nodes in kPath from dNode to the node before the destination
        # and add candidate paths to the path heap.

        kNodes = self.kPath.nodeList
        index = kNodes.index(self.kPath.dNode)
        curNode = kNodes[index]
        while curNode != self.dest:
            self._removeEdgesNodes(curNode)
            candidate = self._computeCandidatePath(curNode)
            self._restoreGraph()
            if candidate != None:
                heapq.heappush(self.pathHeap, candidate)
            index = index + 1
            curNode = kNodes[index]

        if len(self.pathHeap) == 0:
            return None
        p = heapq.heappop(self.pathHeap)  # after iterations contains next shortest path
        self.pathList.append(p)
        self.kPath = p  # updates the kth path
        return p

    def _removeEdgesNodes(self, curNode):
        """
        Remove all nodes from source to the node before the current node in kPath.
        Delete the edge between curNode and the next node in kPath
        Delete any edges previously deleted in kPath starting at curNode
        add all deleted edges to the deleted edge list.
        """

        # Figure out all edges to be removed first then take them out of the temp graph
        # then remove all the nodes from the temp graph.
        # At the start the temp graph is equal to the initial graph.

        self.deletedEdges = set()
        self.deletedNodes = set()
        kNodes = self.kPath.nodeList
        index = 0
        tempNode = kNodes[index]
        index += 1
        while tempNode != curNode:
            edges = self.tempG.edges(tempNode)
            if len(edges) != 0:
                for edge in edges:
                    self.deletedEdges.add(edge)
                    self.tempG.remove_edge(edge[0], edge[1])

            #

            self.deletedNodes.add(tempNode)
            self.tempG.remove_node(tempNode)
            tempNode = kNodes[index]
            index += 1

        # Also need to remove those old deleted edges that start on curNode

        oldDelEdges = self.kPath.deletedEdges
        if self.g.is_directed():
            outEdges = self.g.out_edges(curNode)
        else:
            outEdges = self.g.edges(curNode)

        # outEdges = self.g.edges(curNode)

        for e in outEdges:
            if e in oldDelEdges:
                self.deletedEdges.add(e)
                self.tempG.remove_edge(e[0], e[1])

        # Now delete the edge from the curNode to the next in the path

        tempNode = kNodes[index]
        e = (curNode, tempNode)
        self.deletedEdges.add(e)
        self.tempG.remove_edge(curNode, tempNode)

    def _computeCandidatePath(self, curNode):
        """
        Compute the shortest path on the modified graph and then
        combines with the portion of kPath from the source up through
        the deviation node
        """

#        DijkstraShortestPath alg = new DijkstraShortestPath(tempG, wt);
#        List<E> ePath = alg.getPath(curNode, dest);

        # nodeList = nx.dijkstra_path(self.tempG, curNode, self.dest, self.wt)

        alg = ModifiedDijkstra(self.tempG, self.wt)
        nodeList = alg.getPath(curNode, self.dest, as_nodes=True)

        # Trying this out...

        if nodeList == None:
            return None

        # Get first part of the path from kPath

        nodePath = []
        if curNode in self.kPath.nodeList:
            index = self.kPath.nodeList.index(curNode)
            nodePath = self.kPath.nodeList[0:index]

        nodePath.extend(nodeList)
        wp = WeightedPath(nodePath, self.deletedEdges, self.g, wt=self.wt,
                          cap=self.cap)
        wp.dNode = curNode
        return wp

    def _restoreGraph(self):
        """
        Using the internal deleted node and deleted edge containers
        restores the temp graph to match the graph g.
        """

        # self.tempG = self.g.copy()

        if nx.is_directed(self.g):
            self.tempG = nx.DiGraph(self.g)
        else:
            self.tempG = nx.Graph(self.g)
        self.deletedEdges = []
        self.deletedNodes = []

class WeightedPath(object):

    """Used internally by the Yen k-shortest path algorithm.
    Also return to user as a result.
    """

    def __init__(
        self,
        pathNodeList,
        deletedEdges,
        g,
        wt='weight',
        cap='capacity',
        ):
        """
        Constructor
        """

        self.nodeList = pathNodeList
        self.deletedEdges = set(deletedEdges)
        self.g = g
        self.wt = wt
        self.dNode = None  # The deflection node
        self.cost = 0.0
        self.capacity = float('inf')

        # print("WtPath pathNodeList: {}".format(pathNodeList))

        for i in range(len(pathNodeList) - 1):
            self.cost = self.cost + g[pathNodeList[i]][pathNodeList[i + 1]][wt]

################################# ADI: Do we need this? ##############################
####            if not cap == None:
####                self.capacity = min(self.capacity, g[pathNodeList[i]][pathNodeList[i+1]][cap])
####            else:
####                self.capacity = None
######################################################################################

    def __cmp__(self, other):
        if other == None:
            return -1
        return cmp(self.cost, other.cost)

    def __str__(self):
        return 'nodeList: {}, cost: {}, capacity: {}'.format(self.nodeList,
                self.cost, self.capacity)

class ModifiedDijkstra(object):

    """
    The Modified Dijkstra algorithm from "Survivable Networks" by Ramesh Bhandari.
    This algorithm works with graphs that can have directed or undirected links.
    In addition, this algorithm can correctly function in some cases of negative
    arc lengths that arise in the disjoint path computations.

    Works with graphs, *g*, in NetworkX format. Specifically Graph and
    DiGraph classes.
    """

    def __init__(self, g, wt='weight'):
        """
        Constructor. Parameter *g* is a NetworkX Graph or DiGraph instance.
        The *wt* keyword argument sets the link attribute to be used in computing
        the path length.
        """

        self.dist = {}  # A map from nodes to their labels (float)
        self.predecessor = {}  # A map from a node to a node
        self.g = g
        self.wt = wt
        edges = g.edges()

        # Set the value for infinite distance in the graph

        self.inf = 0.0
        for e in edges:
            self.inf += abs(g[e[0]][e[1]][wt])
        self.inf += 1.0

    def getPath(
        self,
        source,
        dest,
        as_nodes=False,
        ):
        """
        Computes the shortest path in the graph between the given *source* and *dest*
        node (strings).  Returns the path as a list of links (default) or as a list of
        nodes by setting the *as_nodes* keyword argument to *True*.
        """

        self.dist = {}  # A map from nodes to their labels (float)
        self.predecessor = {}  # A map from a node to a node

        # Initialize the distance labels to "infinity"

        vertices = self.g.nodes()
        for vertex in vertices:
            self.dist[vertex] = self.inf
            self.predecessor[vertex] = source

        # Further set up the distance from the source to itself and
        # to all one hops away.

        self.dist[source] = 0.0
        if self.g.is_directed():
            outEdges = self.g.out_edges([source])
        else:
            outEdges = self.g.edges([source])
        for edge in outEdges:
            self.dist[edge[1]] = self.g[edge[0]][edge[1]][self.wt]

        s = set(vertices)
        s.remove(source)
        currentMin = self._findMinNode(s)
        if currentMin == None:
            return None
        s.remove(currentMin)
        while currentMin != dest and len(s) != 0 and currentMin != None:
            if self.g.is_directed():
                outEdges = self.g.out_edges([currentMin])
            else:
                outEdges = self.g.edges([currentMin])
            for edge in outEdges:
                opposite = edge[1]
                if self.dist[currentMin] + self.g[edge[0]][edge[1]][self.wt] \
                    < self.dist[opposite]:
                    self.dist[opposite] = self.dist[currentMin] \
                        + self.g[edge[0]][edge[1]][self.wt]
                    self.predecessor[opposite] = currentMin
                    s.add(opposite)

            currentMin = self._findMinNode(s)

            # print("Current min node {}, s = {}".format(currentMin, s))

            if currentMin == None:
                return None
            s.remove(currentMin)

        # Compute the path as a list of edges

        currentNode = dest
        predNode = self.predecessor.get(dest)
        node_list = [dest]
        done = False
        path = []
        while not done:
            path.append((predNode, currentNode))
            currentNode = predNode
            predNode = self.predecessor[predNode]
            node_list.append(currentNode)
            done = currentNode == source
        node_list.reverse()
        if as_nodes:
            return node_list
        else:
            return path

    def _findMinNode(self, s):
        """
        Finds the vertex with the minimum distance label in the set "s".
        returns the minimum vertex
        """

        minNode = None
        minVal = self.inf
        for vertex in s:
            if self.dist[vertex] < minVal:
                minVal = self.dist[vertex]
                minNode = vertex
        return minNode
<|endoftext|>


<commit_before># coding=utf-8

"""This module, data_instance_to_c_code.py, converts DataInstance into usable prepared C code."""

from finance.finance_simulations.data_instance import *

class CCode(object):
	"""Represents a collection of c code."""
	def __init__(self):
		self._lines = []

	def add_line(self, s : str):
		"""Adds a line of c code."""
		self._lines.append(s)

	def add_define_statement(self, d, v):
		"""Adds a define statement."""
		self.add_line('#define ' + str(d) + ' ' + str(v) + '\n')

	def add_defined_1d_list_of_numbers(self, list_declaration, list_of_numbers):
		"""Utility function."""
		numbers = ''
		for n in list_of_numbers:
			numbers += str(n) + ','
		numbers = numbers[:-1]
		self.add_line(list_declaration.replace('ARG', numbers))

	def add_defined_2d_list_of_numbers(self, list_declaration, list_of_list_of_numbers):
		"""Utility function."""
		code_text = ''
		for row in list_of_list_of_numbers:
			single_line = '{'
			for e in row:
				single_line += str(e) + ','
			code_text += single_line[:-1] + '},\n'
		self.add_line(list_declaration.replace('ARG', code_text[:-2]))

	def __str__(self):
		s = ''
		for l in self._lines:
			s += l
		return s

def get_c_code_from_list_of_data_instances(data_instances, finance_model):
	"""Returns a text containing c_code that represents the data instances."""
	c_code = CCode()

	if finance_model.type_of_data_needed == DATA_REQUIREMENT_FULL_BOOK_ORDER:
		c_code.add_define_statement('NUMBER_OF_TIME_INSTANCES', len(data_instances))

		number_of_buy_orders_list = []
		number_of_sell_orders_list = []

		list_of_list_of_buy_prices = []
		list_of_list_of_buy_amounts = []
		list_of_list_of_sell_prices = []
		list_of_list_of_sell_amounts = []

		for i in data_instances:
			#print(str(data_instances))

			number_of_buy_orders_list.append(i.data[DATA_KEY_NUMBER_OF_BUY_ORDERS])
			number_of_sell_orders_list.append(i.data[DATA_KEY_NUMBER_OF_BUY_ORDERS])

			print('------------------------------------------------')
			print(str(i.data[DATA_KEY_BUY_PRICES]))
			print('------------------------------------------------')
			exit()
			list_of_list_of_buy_prices.append([])
			for e in i.data[DATA_KEY_BUY_PRICES]:
				list_of_list_of_buy_prices[-1].append(e)

				list_of_list_of_buy_amounts.append([])
			for e in i.data[DATA_KEY_BUY_AMOUNTS]:
				list_of_list_of_buy_amounts[-1].append(e)

				list_of_list_of_sell_prices.append([])
			for e in i.data[DATA_KEY_SELL_PRICES]:
				list_of_list_of_sell_prices[-1].append(e)

				list_of_list_of_sell_amounts.append([])
			for e in i.data[DATA_KEY_SELL_AMOUNTS]:
				list_of_list_of_sell_amounts[-1].append(e)

		c_code.add_defined_1d_list_of_numbers('const int number_of_buy_orders[NUMBER_OF_TIME_INSTANCES] = {ARG};\n', number_of_buy_orders_list)
		c_code.add_defined_1d_list_of_numbers('const int number_of_sell_orders[NUMBER_OF_TIME_INSTANCES] = {ARG};\n', number_of_sell_orders_list)

		c_code.add_defined_2d_list_of_numbers('const unsigned short all_buy_prices[NUMBER_OF_TIME_INSTANCES][50]   = {ARG};\n', list_of_list_of_buy_prices)
		c_code.add_defined_2d_list_of_numbers('const float          all_buy_amounts[NUMBER_OF_TIME_INSTANCES][50]  = {ARG};\n', list_of_list_of_buy_amounts)
		c_code.add_defined_2d_list_of_numbers('const unsigned short all_sell_prices[NUMBER_OF_TIME_INSTANCES][50]  = {ARG};\n', list_of_list_of_sell_prices)
		c_code.add_defined_2d_list_of_numbers('const float          all_sell_amounts[NUMBER_OF_TIME_INSTANCES][50] = {ARG};\n', list_of_list_of_sell_amounts)

		return str(c_code)

	else:
		print('SUPPORT THIS KIND OF DATA MODEL!!!')
<commit_msg>finance updating 139<commit_after># coding=utf-8

"""This module, data_instance_to_c_code.py, converts DataInstance into usable prepared C code."""

from finance.finance_simulations.data_instance import *

class CCode(object):
	"""Represents a collection of c code."""
	def __init__(self):
		self._lines = []

	def add_line(self, s: str):
		"""Adds a line of c code."""
		self._lines.append(s)

	def add_lines(self, s: list):
		"""Adds lines of c code."""
		for l in s:
			self.add_line(l)

	def add_define_statement(self, d, v):
		"""Adds a define statement."""
		self.add_line('#define ' + str(d) + ' ' + str(v) + '\n')

	def add_defined_1d_list_of_numbers(self, list_declaration, list_of_numbers):
		"""Utility function."""
		numbers = ''
		for n in list_of_numbers:
			numbers += str(n) + ','
		numbers = numbers[:-1]
		self.add_line(list_declaration.replace('ARG', numbers))

	def add_defined_2d_list_of_numbers(self, list_declaration, list_of_list_of_numbers):
		"""Utility function."""
		code_text = ''
		for row in list_of_list_of_numbers:
			single_line = '{'
			for e in row:
				single_line += str(e) + ','
			code_text += single_line[:-1] + '},\n'
		self.add_line(list_declaration.replace('ARG', code_text[:-2]))

	def __str__(self):
		s = ''
		for l in self._lines:
			s += l
		return s

def get_c_code_from_list_of_data_instances(data_instances, finance_model):
	"""Returns a text containing c_code that represents the data instances."""
	c_code = CCode()

	if finance_model.type_of_data_needed == DATA_REQUIREMENT_FULL_BOOK_ORDER:
		c_code.add_define_statement('NUMBER_OF_TIME_INSTANCES', len(data_instances))

		number_of_buy_orders_list = []
		number_of_sell_orders_list = []

		list_of_list_of_buy_prices = []
		list_of_list_of_buy_amounts = []
		list_of_list_of_sell_prices = []
		list_of_list_of_sell_amounts = []

		for i in data_instances:
			#print(str(data_instances))

			number_of_buy_orders_list.append(i.data[DATA_KEY_NUMBER_OF_BUY_ORDERS])
			number_of_sell_orders_list.append(i.data[DATA_KEY_NUMBER_OF_BUY_ORDERS])

			list_of_list_of_buy_prices.append([])
			for e in i.data[DATA_KEY_BUY_PRICES]:
				list_of_list_of_buy_prices[-1].append(e)

				list_of_list_of_buy_amounts.append([])
			for e in i.data[DATA_KEY_BUY_AMOUNTS]:
				list_of_list_of_buy_amounts[-1].append(e)

				list_of_list_of_sell_prices.append([])
			for e in i.data[DATA_KEY_SELL_PRICES]:
				list_of_list_of_sell_prices[-1].append(e)

				list_of_list_of_sell_amounts.append([])
			for e in i.data[DATA_KEY_SELL_AMOUNTS]:
				list_of_list_of_sell_amounts[-1].append(e)

		c_code.add_defined_1d_list_of_numbers('const int number_of_buy_orders[NUMBER_OF_TIME_INSTANCES] = {ARG};\n', number_of_buy_orders_list)
		c_code.add_defined_1d_list_of_numbers('const int number_of_sell_orders[NUMBER_OF_TIME_INSTANCES] = {ARG};\n', number_of_sell_orders_list)

		c_code.add_defined_2d_list_of_numbers('const unsigned short all_buy_prices[NUMBER_OF_TIME_INSTANCES][50]   = {ARG};\n', list_of_list_of_buy_prices)
		c_code.add_defined_2d_list_of_numbers('const float          all_buy_amounts[NUMBER_OF_TIME_INSTANCES][50]  = {ARG};\n', list_of_list_of_buy_amounts)
		c_code.add_defined_2d_list_of_numbers('const unsigned short all_sell_prices[NUMBER_OF_TIME_INSTANCES][50]  = {ARG};\n', list_of_list_of_sell_prices)
		c_code.add_defined_2d_list_of_numbers('const float          all_sell_amounts[NUMBER_OF_TIME_INSTANCES][50] = {ARG};\n', list_of_list_of_sell_amounts)

		return str(c_code)

	else:
		print('SUPPORT THIS KIND OF DATA MODEL!!!')
<|endoftext|>


<commit_before># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2018, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

from __future__ import print_function

import matplotlib
matplotlib.use('Agg')
import tarfile
from os import path
import os
import shutil

from htmresearch.frameworks.pytorch.sparse_speech_experiment import \
  SparseSpeechExperiment

# Run from root dir:
# domino run projects/speech_commands//run_experiment.py -c projects/speech_commands/experiments.cfg
#   OR
# python projects/speech_commands/run_experiment.py -c projects/speech_commands/experiments.cfg -d
if __name__ == '__main__':
  suite = SparseSpeechExperiment()
  #
  # suite.parse_opt()
  # projectDir = path.dirname(suite.options.config)
  # dataDir = path.join(path.abspath(projectDir), "data")
  #
  # if "DOMINO_WORKING_DIR" in os.environ:
  #   print("In domino")
  #   if path.isdir(path.join(dataDir, "speech_commands")):
  #     print("Removing speech_commands")
  #     shutil.rmtree(path.join(dataDir, "speech_commands"))
  #
  #   if not path.isdir(path.join(dataDir, "speech_commands")):
  #     print("Untarring dataset...")
  #     tar = tarfile.open(path.join(dataDir, "speech_commands.tar.gz"))
  #     tar.extractall(path=dataDir)
  #     tar.close()
  #   else:
  #     print("Apparently this still exists:", path.join(dataDir, "speech_commands"))
  #
  #   print("listing files:")
  #   files = os.listdir(path.join(dataDir, "speech_commands"))
  #   print(files)
  #   files = os.listdir(path.join(dataDir, "speech_commands", "train", "one"))
  #   print("train/one", len(files), files[0:10])

  suite.start()
<commit_msg>Remove commented out code<commit_after># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2018, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

from __future__ import print_function

import matplotlib
matplotlib.use('Agg')
import tarfile
from os import path
import os
import shutil

from htmresearch.frameworks.pytorch.sparse_speech_experiment import \
  SparseSpeechExperiment

# Run from root dir:
# domino run projects/speech_commands//run_experiment.py -c projects/speech_commands/experiments.cfg
#   OR
# python projects/speech_commands/run_experiment.py -c projects/speech_commands/experiments.cfg -d
if __name__ == '__main__':
  suite = SparseSpeechExperiment()
  suite.start()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""
Created on 5 Dec 2016

@author: Bruno Beloff (bruno.beloff@southcoastscience.com)

DESCRIPTION
The status_sampler utility is used to report on the configuration and condition of the host system and its
peripherals. Items included in the report vary depending on the hardware configuration of the equipment. Fields
which are always reported include:

* Sensing schedule
* Unix uptime report
* Temperature of the digital front-end board

Fields which may be reported include:

* Timezone
* GPS location
* Power supply condition
* Temperature of the host processor

The status_sampler writes its output to stdout. As for all sensing utilities, the output format is a JSON document with
fields for:

* the unique tag of the device (if the system ID is set)
* the recording date / time in ISO 8601 format
* a value field containing the sensed values

Command-line options allow for single-shot reading, multiple readings with specified time intervals, or readings
controlled by an independent scheduling process via a Unix semaphore.

SYNOPSIS
status_sampler.py [{ -s SEMAPHORE | -i INTERVAL [-n SAMPLES] }] [-v]

EXAMPLES
./status_sampler.py -i60

FILES
~/SCS/conf/schedule.json
~/SCS/conf/system_id.json

DOCUMENT EXAMPLE - OUTPUT
{"tag": "scs-be2-3", "rec": "2018-06-21T15:03:02.875+00:00", "val": {
"tz": {"name": "Etc/UCT", "utc-offset": "+00:00"},
"pos": {"lat": 50.823049, "lng": -0.123024, "alt": 48, "qual": 2},
"sch": {"scs-climate": {"interval": 60.0, "tally": 1}, "scs-gases": {"interval": 10.0, "tally": 1},
"scs-status": {"interval": 60.0, "tally": 1}}, "tmp": {"brd": 31.2},
"up": {"period": "01-05:29:00.000", "users": 2, "load": {"av1": 0.04, "av5": 0.01, "av15": 0.0}}}}

SEE ALSO
scs_dev/scheduler
scs_mfr/gps_conf
scs_mfr/psu_conf
scs_mfr/schedule
scs_mfr/system_id
scs_mfr/timezone

BUGS
If status_sampler is run in single-shot mode, GPS and PSU monitors may time out before being able to supply data.

RESOURCES
https://en.wikipedia.org/wiki/ISO_8601
"""

import sys

from scs_core.data.json import JSONify
from scs_core.data.localized_datetime import LocalizedDatetime

from scs_core.sync.timed_runner import TimedRunner

from scs_core.sys.system_id import SystemID

from scs_dev.cmd.cmd_sampler import CmdSampler
from scs_dev.sampler.status_sampler import StatusSampler

from scs_dfe.board.dfe_conf import DFEConf
from scs_dfe.gps.gps_conf import GPSConf

from scs_host.bus.i2c import I2C
from scs_host.sync.schedule_runner import ScheduleRunner
from scs_host.sys.host import Host

try:
    from scs_psu.psu.psu_conf import PSUConf
except ImportError:
    from scs_core.psu.psu_conf import PSUConf

# TODO: deal with the case of slow-to-start subsystem monitors

# --------------------------------------------------------------------------------------------------------------------

if __name__ == '__main__':

    sampler = None

    # ----------------------------------------------------------------------------------------------------------------
    # cmd...

    cmd = CmdSampler()

    if cmd.verbose:
        print("status_sampler: %s" % cmd, file=sys.stderr)

    try:
        I2C.open(Host.I2C_SENSORS)

        # ------------------------------------------------------------------------------------------------------------
        # resources...

        # SystemID...
        system_id = SystemID.load(Host)

        tag = None if system_id is None else system_id.message_tag()

        if system_id and cmd.verbose:
            print("status_sampler: %s" % system_id, file=sys.stderr)

        # board...
        dfe_conf = DFEConf.load(Host)
        board = None if dfe_conf is None else dfe_conf.board_temp_sensor()

        if cmd.verbose and dfe_conf:
            print("status_sampler: %s" % dfe_conf, file=sys.stderr)

        # GPS...
        gps_conf = GPSConf.load(Host)
        gps_monitor = None if gps_conf is None else gps_conf.gps_monitor(Host)

        if cmd.verbose and gps_monitor:
            print("status_sampler: %s" % gps_monitor, file=sys.stderr)

        # PSUMonitor...
        psu_conf = PSUConf.load(Host)
        psu_monitor = None if psu_conf is None else psu_conf.psu_monitor(Host)

        if cmd.verbose and psu_monitor:
            print("status_sampler: %s" % psu_monitor, file=sys.stderr)

        # sampler...
        runner = TimedRunner(cmd.interval, cmd.samples) if cmd.semaphore is None \
            else ScheduleRunner(cmd.semaphore, False)

        sampler = StatusSampler(runner, tag, board, gps_monitor, psu_monitor)

        if cmd.verbose:
            print("status_sampler: %s" % sampler, file=sys.stderr)
            sys.stderr.flush()

        # ------------------------------------------------------------------------------------------------------------
        # run...

        sampler.start()

        for sample in sampler.samples():
            if cmd.verbose:
                now = LocalizedDatetime.now()
                print("%s:       status: %s" % (now.as_time(), sample.rec.as_time()), file=sys.stderr)
                sys.stderr.flush()

            print(JSONify.dumps(sample))
            sys.stdout.flush()

    # ----------------------------------------------------------------------------------------------------------------
    # end...

    except KeyboardInterrupt:
        if cmd.verbose:
            print("status_sampler: KeyboardInterrupt", file=sys.stderr)

    finally:
        if sampler:
            sampler.stop()

        I2C.close()
<commit_msg>Updated documentation.<commit_after>#!/usr/bin/env python3

"""
Created on 5 Dec 2016

@author: Bruno Beloff (bruno.beloff@southcoastscience.com)

DESCRIPTION
The status_sampler utility is used to report on the configuration and condition of the host system and its
peripherals. Items included in the report vary depending on the hardware configuration of the equipment. Fields
which are always reported include:

* Sensing schedule
* Unix uptime report
* Temperature of the digital front-end board

Fields which may be reported include:

* Timezone
* GPS location
* Power supply condition
* Temperature of the host processor

The status_sampler writes its output to stdout. As for all sensing utilities, the output format is a JSON document with
fields for:

* the unique tag of the device (if the system ID is set)
* the recording date / time in ISO 8601 format
* a value field containing the sensed values

Command-line options allow for single-shot reading, multiple readings with specified time intervals, or readings
controlled by an independent scheduling process via a Unix semaphore.

SYNOPSIS
status_sampler.py [{ -s SEMAPHORE | -i INTERVAL [-n SAMPLES] }] [-v]

EXAMPLES
./status_sampler.py -i60

FILES
~/SCS/conf/schedule.json
~/SCS/conf/system_id.json

DOCUMENT EXAMPLE - OUTPUT
{"tag": "scs-bgx-500", "rec": "2019-01-05T12:04:10Z", "val": {
"tz": {"name": "Europe/London", "utc-offset": "+00:00"},
"gps": {"pos": [50.8230166, -0.1229759], "elv": 39.9, "qual": 1},
"sch": {"scs-climate": {"interval": 60.0, "tally": 1}, "scs-gases": {"interval": 10.0, "tally": 1},
"scs-status": {"interval": 60.0, "tally": 1}}, "tmp": {"brd": 29.6},
"up": {"period": "00-00:31:00", "users": 1, "load": {"av1": 0.06, "av5": 0.07, "av15": 0.22}},
"psu": {"rst": "00", "standby": false, "chg": "0000", "batt-flt": false, "host-3v3": 3.4, "pwr-in": 15.8,
"prot-batt": 0.1}}}

SEE ALSO
scs_dev/scheduler
scs_mfr/gps_conf
scs_mfr/psu_conf
scs_mfr/schedule
scs_mfr/system_id
scs_mfr/timezone

BUGS
If status_sampler is run in single-shot mode, GPS and PSU monitors may time out before being able to supply data.

RESOURCES
https://en.wikipedia.org/wiki/ISO_8601
"""

import sys

from scs_core.data.json import JSONify
from scs_core.data.localized_datetime import LocalizedDatetime

from scs_core.sync.timed_runner import TimedRunner

from scs_core.sys.system_id import SystemID

from scs_dev.cmd.cmd_sampler import CmdSampler
from scs_dev.sampler.status_sampler import StatusSampler

from scs_dfe.board.dfe_conf import DFEConf
from scs_dfe.gps.gps_conf import GPSConf

from scs_host.bus.i2c import I2C
from scs_host.sync.schedule_runner import ScheduleRunner
from scs_host.sys.host import Host

try:
    from scs_psu.psu.psu_conf import PSUConf
except ImportError:
    from scs_core.psu.psu_conf import PSUConf

# TODO: deal with the case of slow-to-start subsystem monitors

# --------------------------------------------------------------------------------------------------------------------

if __name__ == '__main__':

    sampler = None

    # ----------------------------------------------------------------------------------------------------------------
    # cmd...

    cmd = CmdSampler()

    if cmd.verbose:
        print("status_sampler: %s" % cmd, file=sys.stderr)

    try:
        I2C.open(Host.I2C_SENSORS)

        # ------------------------------------------------------------------------------------------------------------
        # resources...

        # SystemID...
        system_id = SystemID.load(Host)

        tag = None if system_id is None else system_id.message_tag()

        if system_id and cmd.verbose:
            print("status_sampler: %s" % system_id, file=sys.stderr)

        # board...
        dfe_conf = DFEConf.load(Host)
        board = None if dfe_conf is None else dfe_conf.board_temp_sensor()

        if cmd.verbose and dfe_conf:
            print("status_sampler: %s" % dfe_conf, file=sys.stderr)

        # GPS...
        gps_conf = GPSConf.load(Host)
        gps_monitor = None if gps_conf is None else gps_conf.gps_monitor(Host)

        if cmd.verbose and gps_monitor:
            print("status_sampler: %s" % gps_monitor, file=sys.stderr)

        # PSUMonitor...
        psu_conf = PSUConf.load(Host)
        psu_monitor = None if psu_conf is None else psu_conf.psu_monitor(Host)

        if cmd.verbose and psu_monitor:
            print("status_sampler: %s" % psu_monitor, file=sys.stderr)

        # sampler...
        runner = TimedRunner(cmd.interval, cmd.samples) if cmd.semaphore is None \
            else ScheduleRunner(cmd.semaphore, False)

        sampler = StatusSampler(runner, tag, board, gps_monitor, psu_monitor)

        if cmd.verbose:
            print("status_sampler: %s" % sampler, file=sys.stderr)
            sys.stderr.flush()

        # ------------------------------------------------------------------------------------------------------------
        # run...

        sampler.start()

        for sample in sampler.samples():
            if cmd.verbose:
                now = LocalizedDatetime.now()
                print("%s:       status: %s" % (now.as_time(), sample.rec.as_time()), file=sys.stderr)
                sys.stderr.flush()

            print(JSONify.dumps(sample))
            sys.stdout.flush()

    # ----------------------------------------------------------------------------------------------------------------
    # end...

    except KeyboardInterrupt:
        if cmd.verbose:
            print("status_sampler: KeyboardInterrupt", file=sys.stderr)

    finally:
        if sampler:
            sampler.stop()

        I2C.close()
<|endoftext|>


<commit_before><commit_msg>Fortinet's FortiOS report style (#52296)<commit_after><|endoftext|>


<commit_before>from appfd.models import *
from django.contrib.gis.gdal import DataSource
import os, urllib, zipfile
from urllib import urlretrieve

def run():

    if not os.path.isfile('/tmp/alternateNames.zip'):
        urlretrieve('http://download.geonames.org/export/dump/alternateNames.zip', '/tmp/alternateNames.zip')
        with zipfile.ZipFile('/tmp/alternateNames.zip', "r") as z:
            z.extractall('/tmp/')

    with open('/tmp/alternateNames.txt') as f:
        for index, line in enumerate(f):
          try:
            line_split = line.decode("utf-8").strip().split("\t")
            geonameid = int(line_split[0])
            language = line_split[2]
            alternate_name = line_split[3]

            #if index == 200:
            #    break

            if language != "link":
                print "line ", index, ": ", line_split

                alias = Alias.objects.get_or_create(alias=alternate_name)[0]
                alias.language = language[:2]
                alias.save()

                place = Place.objects.get(geonameid=geonameid)

                AliasPlace.objects.get_or_create(alias=alias, place=place)
          except Exception as e:
            print e
<commit_msg>added loadAlternateNames, which takes like 2 days by the way<commit_after>from appfd.models import *
from django.db import connection, transaction
from datetime import datetime
from os.path import isfile
from urllib import urlretrieve
from zipfile import ZipFile

def run():
    start = datetime.now()
    print "starting loadAlternateNames at ", start

    path_to_dir = '/home/usrfd/data/geonames/'
    path_to_zip = path_to_dir + 'alternateNames.zip'
    if not isfile(path_to_zip):
        print "retrieving alternateNames.zip . . . ",
        urlretrieve('http://download.geonames.org/export/dump/alternateNames.zip', path_to_zip)
        print "done"
        print "unzipping alternateNames.zip . . . ",
        with ZipFile(path_to_zip, "r") as z:
            z.extractall(path_to_dir)
        print "done"

    # truncate existing AlternateName table
    cursor = connection.cursor()
    cursor.execute("TRUNCATE appfd_alias CASCADE")
    cursor.execute("TRUNCATE appfd_aliasplace CASCADE")

    throughs_to_create = []
    with open(path_to_dir + 'alternateNames.txt') as f:
        counter = 0
        for line in f:
            counter += 1
            geonameid, _, language, alternate_name, _, _, _, _ = line.decode("utf-8").split("\t")
            alias_id = ((Alias.objects.filter(alias=alternate_name) or [None])[0] or Alias.objects.create(alias=alternate_name, language=language)).id
            place_id = (Place.objects.filter(geonameid=geonameid).values_list("id", flat=True) or [None])[0]
            if place_id:
                AliasPlace.objects.get_or_create(alias_id=alias_id, place_id=place_id)
            if counter % 1000 == 0:
                print "line: " + str(counter)

    end = datetime.now()
    total_seconds = (end - start).total_seconds()
    print "total_seconds = ", total_seconds
    with open("log.txt", "a") as f:
        f.write("Loading alternateNames took " + str(total_seconds) + " seconds.")
<|endoftext|>


<commit_before>from __future__ import absolute_import

import functools
import itertools
import logging
import random
import time
from contextlib import contextmanager
from datetime import datetime

import pytz
from django.conf import settings
from rb import Cluster
from redis.client import Script
from redis.exceptions import (
    ResponseError,
    WatchError,
)

from .base import (
    Backend,
    Record,
)

logger = logging.getLogger('sentry.timelines')

ADD_TO_SCHEDULE_SCRIPT = """\
-- Ensures an timeline is scheduled to be digested.
-- KEYS: {WATING, READY}
-- ARGV: {TIMELINE, TIMESTAMP}

-- Check to see if the timeline exists in the "waiting" set (heuristics tell us
-- that this should be more likely than it's presence in the "ready" set.)
local waiting = redis.call('ZSCORE', KEYS[1], ARGV[1])

if waiting ~= false then
    -- If the item already exists, update the score if the provided timestamp
    -- is less than the current score.
    if tonumber(waiting) > tonumber(ARGV[2]) then
        redis.call('ZADD', KEYS[1], ARGV[2], ARGV[1])
    end
    return
end

-- Otherwise, check to see if the timeline already exists in the "ready" set.
-- If it doesn't, it needs to be added to the "waiting" set to be scheduled.
if redis.call('ZSCORE', KEYS[2], ARGV[1]) == false then
    redis.call('ZADD', KEYS[1], ARGV[2], ARGV[1])
    return
end
"""

TRUNCATE_TIMELINE_SCRIPT = """\
-- Trims a timeline to a maximum number of records.
-- Returns the number of keys that were deleted.
-- KEYS: {TIMELINE}
-- ARGV: {LIMIT}

local keys = redis.call('ZREVRANGE', KEYS[1], ARGV[1], -1)
for i, record in pairs(keys) do
    -- TODO: This could probably be optimized into single operations.
    redis.call('DEL', KEYS[1] .. ':r:' .. record)
    redis.call('ZREM', KEYS[1], record)
end
return table.getn(keys)
"""

# XXX: Passing `None` as the first argument is a dirty hack to allow us to use
# this more easily with the cluster
add_to_schedule = Script(None, ADD_TO_SCHEDULE_SCRIPT)
truncate_timeline = Script(None, TRUNCATE_TIMELINE_SCRIPT)

def to_timestamp(value):
    return (value - datetime(1970, 1, 1, tzinfo=pytz.utc)).total_seconds()

WAITING_STATE = 'w'
READY_STATE = 'r'

def make_schedule_key(namespace, state):
    return '{0}:s:{1}'.format(namespace, state)

def make_timeline_key(namespace, target):
    return '{0}:t:{1}'.format(namespace, target)

def make_iteration_key(timeline_key):
    return '{0}:i'.format(timeline_key)

def make_digest_key(timeline_key):
    return '{0}:d'.format(timeline_key)

def make_record_key(timeline_key, record):
    # XXX: Don't update this without updating the Lua script!
    # TODO: Just interpolate the hierarchical part here into the script.
    return '{0}:r:{1}'.format(timeline_key, record)

class RedisBackend(Backend):
    def __init__(self, **options):
        super(RedisBackend, self).__init__(**options)

        self.cluster = Cluster(**options.pop('cluster', settings.SENTRY_REDIS_OPTIONS))
        self.namespace = options.pop('namespace', 'digests')
        self.record_ttl = options.pop('record_ttl', 60 * 60)

        if options:
            logger.warning('Discarding invalid options: %r', options)

    def add(self, target, record):
        timeline_key = make_timeline_key(self.namespace, target)
        record_key = make_record_key(timeline_key, record.key)

        connection = self.cluster.get_local_client_for_key(timeline_key)
        with connection.pipeline() as pipeline:
            pipeline.multi()

            pipeline.set(
                record_key,
                self.codec.encode(record.value),
                ex=self.record_ttl,
            )

            pipeline.set(make_iteration_key(timeline_key), 0, nx=True)

            # TODO: Prefix the entry with the timestamp (lexicographically
            # sortable) to ensure that we can maintain abitrary precision:
            # http://redis.io/commands/ZADD#elements-with-the-same-score
            pipeline.zadd(timeline_key, record.timestamp, record.key)

            add_to_schedule(
                map(
                    functools.partial(make_schedule_key, self.namespace),
                    (WAITING_STATE, READY_STATE),
                ),
                (target, record.timestamp),
                pipeline,
            )

            should_truncate = random.random() < self.trim_chance
            if should_truncate:
                truncate_timeline((timeline_key,), (self.capacity,), pipeline)

            results = pipeline.execute()
            if should_truncate:
                logger.info('Removed %s extra records from %s.', results[-1], target)

    def schedule(self, cutoff, chunk=1000):
        """
        Moves timelines that are ready to be digested from the ``WAITING`` to
        the ``READY`` state.
        """
        # TODO: This doesn't lead to a fair balancing of workers, ideally each
        # scheduling task would be executed by a different process for each
        # host.
        for connection in itertools.imap(self.cluster.get_local_client, self.cluster.hosts):
            # TODO: Scheduling for each host should be protected by a lease.
            while True:
                items = connection.zrangebyscore(
                    make_schedule_key(self.namespace, WAITING_STATE),
                    min=0,
                    max=cutoff,
                    withscores=True,
                    start=0,
                    num=chunk,
                )

                # XXX: Redis will error if we try and execute an empty
                # transaction. If there are no items to move between states, we
                # need to exit the loop now. (This can happen on the first
                # iteration of the loop if there is nothing to do, or on a
                # subsequent iteration if there was exactly the same number of
                # items to change states as the chunk size.)
                if not items:
                    break

                with connection.pipeline() as pipeline:
                    pipeline.multi()

                    pipeline.zrem(
                        make_schedule_key(self.namespace, WAITING_STATE),
                        *[key for key, timestamp in items]
                    )

                    pipeline.zadd(
                        make_schedule_key(self.namespace, READY_STATE),
                        *itertools.chain.from_iterable([(timestamp, key) for (key, timestamp) in items])
                    )

                    yield items

                    pipeline.execute()

                # If we retrieved less than the chunk size of items, we don't
                # need try to retrieve more items.
                if len(items) < chunk:
                    break

    def maintenance(self, timeout):
        raise NotImplementedError

    @contextmanager
    def digest(self, target):
        timeline_key = make_timeline_key(self.namespace, target)
        digest_key = make_digest_key(timeline_key)

        # TODO: Need to wrap this whole section in a lease to try and avoid data races.
        connection = self.cluster.get_local_client_for_key(timeline_key)

        if connection.zscore(make_schedule_key(self.namespace, READY_STATE), target) is None:
            raise Exception('Cannot digest timeline, timeline is not in the ready state.')

        with connection.pipeline() as pipeline:
            pipeline.watch(digest_key)  # This shouldn't be necessary, but better safe than sorry?

            if pipeline.exists(digest_key):
                pipeline.multi()
                pipeline.zunionstore(digest_key, (timeline_key, digest_key), aggregate='max')
            else:
                pipeline.multi()
                pipeline.rename(timeline_key, digest_key)
                try:
                    pipeline.execute()
                except ResponseError as error:
                    # TODO: This means that there are actually no contents in
                    # the original timeline -- we should be able to ignore this
                    # error, but it would be nice to be more specific.
                    logger.debug('Could not move timeline for digestion (likely has no contents.)')

        # XXX: This must select all records, even though not all of them will
        # be returned if they exceed the capacity, to ensure that all records
        # will be garbage collected.
        records = connection.zrevrange(digest_key, 0, -1, withscores=True)
        if not records:
            logger.info('Retrieved timeline containing no records.')

        # TODO: This should gracefully handle the iteration key being evicted.
        iteration = int(connection.get(make_iteration_key(timeline_key)))

        def get_records_for_digest():
            with connection.pipeline(transaction=False) as pipeline:
                for key, timestamp in records:
                    pipeline.get(make_record_key(timeline_key, key))

                for (key, timestamp), value in zip(records, pipeline.execute()):
                    # We have to handle failures if the key does not exist --
                    # this could happen due to evictions or race conditions
                    # where the record was added to a timeline while it was
                    # already being digested.
                    if value is None:
                        logger.warning('Could not retrieve event for timeline.')
                    else:
                        # TODO: Could make this lazy, but that might be unnecessary complexity.
                        yield Record(key, self.codec.decode(value), timestamp)

        yield itertools.islice(get_records_for_digest(), self.capacity)

        def reschedule():
            with connection.pipeline() as pipeline:
                pipeline.watch(digest_key)  # This shouldn't be necessary, but better safe than sorry?
                pipeline.multi()

                record_keys = [make_record_key(timeline_key, key) for key, score in records]
                pipeline.delete(digest_key, *record_keys)
                pipeline.zrem(make_schedule_key(self.namespace, READY_STATE), target)
                pipeline.zadd(make_schedule_key(self.namespace, WAITING_STATE), time.time() + self.backoff(iteration + 1), target)
                pipeline.set(make_iteration_key(timeline_key), iteration + 1)
                pipeline.execute()

        def unschedule():
            with connection.pipeline() as pipeline:
                # Watch the timeline to ensure that no other transactions add
                # events to the timeline while we are trying to delete it.
                pipeline.watch(timeline_key)
                pipeline.multi()
                if connection.zcard(timeline_key) is 0:
                    pipeline.delete(make_iteration_key(timeline_key))
                    pipeline.zrem(make_schedule_key(self.namespace, READY_STATE), target)
                    pipeline.zrem(make_schedule_key(self.namespace, WAITING_STATE), target)
                    pipeline.execute()

        # If there were records in the digest, we need to schedule it so that
        # we schedule any records that were added during digestion with the
        # appropriate backoff. If there were no items, we can try to remove the
        # timeline from the digestion schedule.
        if records:
            reschedule()
        else:
            try:
                unschedule()
            except WatchError:
                logger.debug('Could not remove timeline from schedule, rescheduling instead')
                reschedule()
<commit_msg>Handle missing iteration counter (due to evictions.)<commit_after>from __future__ import absolute_import

import functools
import itertools
import logging
import random
import time
from contextlib import contextmanager
from datetime import datetime

import pytz
from django.conf import settings
from rb import Cluster
from redis.client import Script
from redis.exceptions import (
    ResponseError,
    WatchError,
)

from .base import (
    Backend,
    Record,
)

logger = logging.getLogger('sentry.timelines')

ADD_TO_SCHEDULE_SCRIPT = """\
-- Ensures an timeline is scheduled to be digested.
-- KEYS: {WATING, READY}
-- ARGV: {TIMELINE, TIMESTAMP}

-- Check to see if the timeline exists in the "waiting" set (heuristics tell us
-- that this should be more likely than it's presence in the "ready" set.)
local waiting = redis.call('ZSCORE', KEYS[1], ARGV[1])

if waiting ~= false then
    -- If the item already exists, update the score if the provided timestamp
    -- is less than the current score.
    if tonumber(waiting) > tonumber(ARGV[2]) then
        redis.call('ZADD', KEYS[1], ARGV[2], ARGV[1])
    end
    return
end

-- Otherwise, check to see if the timeline already exists in the "ready" set.
-- If it doesn't, it needs to be added to the "waiting" set to be scheduled.
if redis.call('ZSCORE', KEYS[2], ARGV[1]) == false then
    redis.call('ZADD', KEYS[1], ARGV[2], ARGV[1])
    return
end
"""

TRUNCATE_TIMELINE_SCRIPT = """\
-- Trims a timeline to a maximum number of records.
-- Returns the number of keys that were deleted.
-- KEYS: {TIMELINE}
-- ARGV: {LIMIT}

local keys = redis.call('ZREVRANGE', KEYS[1], ARGV[1], -1)
for i, record in pairs(keys) do
    -- TODO: This could probably be optimized into single operations.
    redis.call('DEL', KEYS[1] .. ':r:' .. record)
    redis.call('ZREM', KEYS[1], record)
end
return table.getn(keys)
"""

# XXX: Passing `None` as the first argument is a dirty hack to allow us to use
# this more easily with the cluster
add_to_schedule = Script(None, ADD_TO_SCHEDULE_SCRIPT)
truncate_timeline = Script(None, TRUNCATE_TIMELINE_SCRIPT)

def to_timestamp(value):
    return (value - datetime(1970, 1, 1, tzinfo=pytz.utc)).total_seconds()

WAITING_STATE = 'w'
READY_STATE = 'r'

def make_schedule_key(namespace, state):
    return '{0}:s:{1}'.format(namespace, state)

def make_timeline_key(namespace, target):
    return '{0}:t:{1}'.format(namespace, target)

def make_iteration_key(timeline_key):
    return '{0}:i'.format(timeline_key)

def make_digest_key(timeline_key):
    return '{0}:d'.format(timeline_key)

def make_record_key(timeline_key, record):
    # XXX: Don't update this without updating the Lua script!
    # TODO: Just interpolate the hierarchical part here into the script.
    return '{0}:r:{1}'.format(timeline_key, record)

class RedisBackend(Backend):
    def __init__(self, **options):
        super(RedisBackend, self).__init__(**options)

        self.cluster = Cluster(**options.pop('cluster', settings.SENTRY_REDIS_OPTIONS))
        self.namespace = options.pop('namespace', 'digests')
        self.record_ttl = options.pop('record_ttl', 60 * 60)

        if options:
            logger.warning('Discarding invalid options: %r', options)

    def add(self, target, record):
        timeline_key = make_timeline_key(self.namespace, target)
        record_key = make_record_key(timeline_key, record.key)

        connection = self.cluster.get_local_client_for_key(timeline_key)
        with connection.pipeline() as pipeline:
            pipeline.multi()

            pipeline.set(
                record_key,
                self.codec.encode(record.value),
                ex=self.record_ttl,
            )

            pipeline.set(make_iteration_key(timeline_key), 0, nx=True)

            # TODO: Prefix the entry with the timestamp (lexicographically
            # sortable) to ensure that we can maintain abitrary precision:
            # http://redis.io/commands/ZADD#elements-with-the-same-score
            pipeline.zadd(timeline_key, record.timestamp, record.key)

            add_to_schedule(
                map(
                    functools.partial(make_schedule_key, self.namespace),
                    (WAITING_STATE, READY_STATE),
                ),
                (target, record.timestamp),
                pipeline,
            )

            should_truncate = random.random() < self.trim_chance
            if should_truncate:
                truncate_timeline((timeline_key,), (self.capacity,), pipeline)

            results = pipeline.execute()
            if should_truncate:
                logger.info('Removed %s extra records from %s.', results[-1], target)

    def schedule(self, cutoff, chunk=1000):
        """
        Moves timelines that are ready to be digested from the ``WAITING`` to
        the ``READY`` state.
        """
        # TODO: This doesn't lead to a fair balancing of workers, ideally each
        # scheduling task would be executed by a different process for each
        # host.
        for connection in itertools.imap(self.cluster.get_local_client, self.cluster.hosts):
            # TODO: Scheduling for each host should be protected by a lease.
            while True:
                items = connection.zrangebyscore(
                    make_schedule_key(self.namespace, WAITING_STATE),
                    min=0,
                    max=cutoff,
                    withscores=True,
                    start=0,
                    num=chunk,
                )

                # XXX: Redis will error if we try and execute an empty
                # transaction. If there are no items to move between states, we
                # need to exit the loop now. (This can happen on the first
                # iteration of the loop if there is nothing to do, or on a
                # subsequent iteration if there was exactly the same number of
                # items to change states as the chunk size.)
                if not items:
                    break

                with connection.pipeline() as pipeline:
                    pipeline.multi()

                    pipeline.zrem(
                        make_schedule_key(self.namespace, WAITING_STATE),
                        *[key for key, timestamp in items]
                    )

                    pipeline.zadd(
                        make_schedule_key(self.namespace, READY_STATE),
                        *itertools.chain.from_iterable([(timestamp, key) for (key, timestamp) in items])
                    )

                    yield items

                    pipeline.execute()

                # If we retrieved less than the chunk size of items, we don't
                # need try to retrieve more items.
                if len(items) < chunk:
                    break

    def maintenance(self, timeout):
        raise NotImplementedError

    @contextmanager
    def digest(self, target):
        timeline_key = make_timeline_key(self.namespace, target)
        digest_key = make_digest_key(timeline_key)

        # TODO: Need to wrap this whole section in a lease to try and avoid data races.
        connection = self.cluster.get_local_client_for_key(timeline_key)

        if connection.zscore(make_schedule_key(self.namespace, READY_STATE), target) is None:
            raise Exception('Cannot digest timeline, timeline is not in the ready state.')

        with connection.pipeline() as pipeline:
            pipeline.watch(digest_key)  # This shouldn't be necessary, but better safe than sorry?

            if pipeline.exists(digest_key):
                pipeline.multi()
                pipeline.zunionstore(digest_key, (timeline_key, digest_key), aggregate='max')
            else:
                pipeline.multi()
                pipeline.rename(timeline_key, digest_key)
                try:
                    pipeline.execute()
                except ResponseError as error:
                    # TODO: This means that there are actually no contents in
                    # the original timeline -- we should be able to ignore this
                    # error, but it would be nice to be more specific.
                    logger.debug('Could not move timeline for digestion (likely has no contents.)')

        # XXX: This must select all records, even though not all of them will
        # be returned if they exceed the capacity, to ensure that all records
        # will be garbage collected.
        records = connection.zrevrange(digest_key, 0, -1, withscores=True)
        if not records:
            logger.info('Retrieved timeline containing no records.')

        def get_iteration_count(default=0):
            value = connection.get(make_iteration_key(timeline_key))
            if not value:
                logger.warning('Could not retrieve iteration counter for %s, defaulting to %s.', target, default)
                return default
            return int(value)

        iteration = get_iteration_count()

        def get_records_for_digest():
            with connection.pipeline(transaction=False) as pipeline:
                for key, timestamp in records:
                    pipeline.get(make_record_key(timeline_key, key))

                for (key, timestamp), value in zip(records, pipeline.execute()):
                    # We have to handle failures if the key does not exist --
                    # this could happen due to evictions or race conditions
                    # where the record was added to a timeline while it was
                    # already being digested.
                    if value is None:
                        logger.warning('Could not retrieve event for timeline.')
                    else:
                        # TODO: Could make this lazy, but that might be unnecessary complexity.
                        yield Record(key, self.codec.decode(value), timestamp)

        yield itertools.islice(get_records_for_digest(), self.capacity)

        def reschedule():
            with connection.pipeline() as pipeline:
                pipeline.watch(digest_key)  # This shouldn't be necessary, but better safe than sorry?
                pipeline.multi()

                record_keys = [make_record_key(timeline_key, key) for key, score in records]
                pipeline.delete(digest_key, *record_keys)
                pipeline.zrem(make_schedule_key(self.namespace, READY_STATE), target)
                pipeline.zadd(make_schedule_key(self.namespace, WAITING_STATE), time.time() + self.backoff(iteration + 1), target)
                pipeline.set(make_iteration_key(timeline_key), iteration + 1)
                pipeline.execute()

        def unschedule():
            with connection.pipeline() as pipeline:
                # Watch the timeline to ensure that no other transactions add
                # events to the timeline while we are trying to delete it.
                pipeline.watch(timeline_key)
                pipeline.multi()
                if connection.zcard(timeline_key) is 0:
                    pipeline.delete(make_iteration_key(timeline_key))
                    pipeline.zrem(make_schedule_key(self.namespace, READY_STATE), target)
                    pipeline.zrem(make_schedule_key(self.namespace, WAITING_STATE), target)
                    pipeline.execute()

        # If there were records in the digest, we need to schedule it so that
        # we schedule any records that were added during digestion with the
        # appropriate backoff. If there were no items, we can try to remove the
        # timeline from the digestion schedule.
        if records:
            reschedule()
        else:
            try:
                unschedule()
            except WatchError:
                logger.debug('Could not remove timeline from schedule, rescheduling instead')
                reschedule()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Main Controller"""
import logging, string, os
from datetime import datetime
from collections import defaultdict

import pkg_resources
from tg import expose, flash, redirect, session, config, response, request, config
from tg.decorators import with_trailing_slash, without_trailing_slash
from tg.flash import TGFlash
from pylons import c, g, cache

import ew
import ming

import allura
from allura.app import SitemapEntry
from allura.lib.base import WsgiDispatchController
from allura.lib import helpers as h
from allura.lib import plugin
from allura.controllers.error import ErrorController
from allura import model as M
from allura.lib.widgets import project_list as plw
from .auth import AuthController
from .search import SearchController, ProjectBrowseController
from .static import NewForgeController
from .site_admin import SiteAdminController
from .project import NeighborhoodController, HostNeighborhoodController
from .rest import RestController

__all__ = ['RootController']

log = logging.getLogger(__name__)

TGFlash.static_template = '''$('#messages').notify('%(message)s', {status: '%(status)s'});'''

class W:
    project_summary = plw.ProjectSummary()

class RootController(WsgiDispatchController):
    """
    The root controller for the allura application.

    All the other controllers and WSGI applications should be mounted on this
    controller. For example::

        panel = ControlPanelController()
        another_app = AnotherWSGIApplication()

    Keep in mind that WSGI applications shouldn't be mounted directly: They
    must be wrapped around with :class:`tg.controllers.WSGIAppController`.

    """

    auth = AuthController()
    error = ErrorController()
    nf = NewForgeController()
    nf.admin = SiteAdminController()
    search = SearchController()
    rest = RestController()

    def __init__(self):
        n_url_prefix = '/%s/' % request.path.split('/')[1]
        n = M.Neighborhood.query.get(url_prefix=n_url_prefix)
        if n and not n.url_prefix.startswith('//'):
            n.bind_controller(self)
        self.browse = ProjectBrowseController()
        self.allura_sitemap = SitemapIndexController()
        super(RootController, self).__init__()

    def _setup_request(self):
        c.project = c.app = None
        c.memoize_cache = {}
        c.user = plugin.AuthenticationProvider.get(request).authenticate_request()
        assert c.user is not None, 'c.user should always be at least User.anonymous()'

    def _cleanup_request(self):
        pass

    @expose('jinja:allura:templates/project_list.html')
    @with_trailing_slash
    def index(self, **kw):
        """Handle the front-page."""
        c.project_summary = W.project_summary
        projects = M.Project.query.find(
            dict(is_root=True,
                 is_nbhd_project=False,
                 deleted=False)).sort('shortname').all()
        neighborhoods = M.Neighborhood.query.find().sort('name')
        psort = [ (n, [ p for p in projects if p.neighborhood_id==n._id ])
                  for n in neighborhoods ]
        categories = M.ProjectCategory.query.find({'parent_id':None}).sort('name').all()
        c.custom_sidebar_menu = [
            SitemapEntry(cat.label, '/browse/'+cat.name) for cat in categories
        ]
        return dict(projects=psort,title="All Projects",text=None)

class SitemapIndexController(object):
    projects_per_page = 1000

    @expose('jinja:allura:templates/sitemap_index.xml')
    def index(self, **kw):
        base_url = config.get('base_url', 'sourceforge.net')
        num_projects = M.Project.query.find().count()
        now = datetime.utcnow().date()
        return dict(
            now=now,
            sitemaps = [
                '%s/sitemap/%d' % (base_url, offset)
                for offset in range(0, num_projects, self.projects_per_page) ])

    @expose()
    def _lookup(self, offset, *remainder):
        return SitemapController(int(offset), self.projects_per_page), remainder

class SitemapController(object):

    def __init__(self, offset, limit):
        self.offset, self.limit = offset, limit

    @expose('jinja:allura:templates/sitemap.xml')
    def index(self, **kw):
        now = datetime.utcnow().date()
        base_url = config.get('base_url', 'sourceforge.net')
        locs = []
        for p in M.Project.query.find().skip(self.offset).limit(self.limit):
            c.project = p
            locs += [  base_url + s.url
                       for s in p.sitemap() ]
        return dict(
            now=now,
            locs=locs)
<commit_msg>[#4718] remove sitemap.xml URLs<commit_after># -*- coding: utf-8 -*-
"""Main Controller"""
import logging, string, os
from datetime import datetime
from collections import defaultdict

import pkg_resources
from tg import expose, flash, redirect, session, config, response, request, config
from tg.decorators import with_trailing_slash, without_trailing_slash
from tg.flash import TGFlash
from pylons import c, g, cache

import ew
import ming

import allura
from allura.app import SitemapEntry
from allura.lib.base import WsgiDispatchController
from allura.lib import helpers as h
from allura.lib import plugin
from allura.controllers.error import ErrorController
from allura import model as M
from allura.lib.widgets import project_list as plw
from .auth import AuthController
from .search import SearchController, ProjectBrowseController
from .static import NewForgeController
from .site_admin import SiteAdminController
from .project import NeighborhoodController, HostNeighborhoodController
from .rest import RestController

__all__ = ['RootController']

log = logging.getLogger(__name__)

TGFlash.static_template = '''$('#messages').notify('%(message)s', {status: '%(status)s'});'''

class W:
    project_summary = plw.ProjectSummary()

class RootController(WsgiDispatchController):
    """
    The root controller for the allura application.

    All the other controllers and WSGI applications should be mounted on this
    controller. For example::

        panel = ControlPanelController()
        another_app = AnotherWSGIApplication()

    Keep in mind that WSGI applications shouldn't be mounted directly: They
    must be wrapped around with :class:`tg.controllers.WSGIAppController`.

    """

    auth = AuthController()
    error = ErrorController()
    nf = NewForgeController()
    nf.admin = SiteAdminController()
    search = SearchController()
    rest = RestController()

    def __init__(self):
        n_url_prefix = '/%s/' % request.path.split('/')[1]
        n = M.Neighborhood.query.get(url_prefix=n_url_prefix)
        if n and not n.url_prefix.startswith('//'):
            n.bind_controller(self)
        self.browse = ProjectBrowseController()
        super(RootController, self).__init__()

    def _setup_request(self):
        c.project = c.app = None
        c.memoize_cache = {}
        c.user = plugin.AuthenticationProvider.get(request).authenticate_request()
        assert c.user is not None, 'c.user should always be at least User.anonymous()'

    def _cleanup_request(self):
        pass

    @expose('jinja:allura:templates/project_list.html')
    @with_trailing_slash
    def index(self, **kw):
        """Handle the front-page."""
        c.project_summary = W.project_summary
        projects = M.Project.query.find(
            dict(is_root=True,
                 is_nbhd_project=False,
                 deleted=False)).sort('shortname').all()
        neighborhoods = M.Neighborhood.query.find().sort('name')
        psort = [ (n, [ p for p in projects if p.neighborhood_id==n._id ])
                  for n in neighborhoods ]
        categories = M.ProjectCategory.query.find({'parent_id':None}).sort('name').all()
        c.custom_sidebar_menu = [
            SitemapEntry(cat.label, '/browse/'+cat.name) for cat in categories
        ]
        return dict(projects=psort,title="All Projects",text=None)

<|endoftext|>


<commit_before>import unittest
"""
Given an array, find the number of subarrays with even sum.
Input: 1 2 2 3 4 1
Output: 9
"""

"""
Approach:
1. If temp[i] is the cumulative sum of the array from arr[0...i], arr[i...j] is even sum subarray if
    temp[j]-temp[i] % 2 = 0.
2. So, we keep a running sum modulo 2 and count how many times the value is zero or one.
3. Value is zero for even sum and 1 for odd sum.
4. Since, a+b is even if both a and b are odd or both are even, temp[j]-temp[i] is even if both temp[j]
and temp[i] are even or both are odd.
5. So, we just have to chose two indices from the temp array: C(n,2) = n*(n-1)/2
"""

def subarrays_with_even_sum(list_of_nums):

    sum_count = [1, 0]  # sum[0] is even count and sum[1] is odd counts
    sum_val = 0

    for i in range(len(list_of_nums)):
        sum_val = ((sum_val + list_of_nums[i] % 2) + 2) % 2  # plus 2 is to handle odd values
        sum_count[sum_val] += 1

    return sum_count[0] * (sum_count[0] - 1) / 2 + sum_count[1] * (sum_count[1] - 1) / 2

class TestSubarrayEvenSum(unittest.TestCase):

    def test_subarray_even_sum(self):
        list_of_nums = [1, 2, 2, 3, 4, 1]
        self.assertEqual(subarrays_with_even_sum(list_of_nums), 9)
<commit_msg>Simplified logic<commit_after>import unittest
"""
Given an array, find the number of subarrays with even sum.
Input: 1 2 2 3 4 1
Output: 9
"""

"""
Approach:
1. If temp[i] is the cumulative sum of the array from arr[0...i], arr[i...j] is even sum subarray if
    temp[j]-temp[i] % 2 = 0.
2. So, we keep a running sum modulo 2 and count how many times the value is zero or one.
3. Value is zero for even sum and 1 for odd sum.
4. Since, a+b is even if both a and b are odd or both are even, temp[j]-temp[i] is even if both temp[j]
and temp[i] are even or both are odd.
5. So, we just have to chose two indices from the temp array: C(n,2) = n*(n-1)/2
"""

def subarrays_with_even_sum(list_of_nums):

    sum_count = [1, 0]  # sum_count[0] is even count and sum_count[1] is odd counts
    sum_val = 0

    for i in range(len(list_of_nums)):
        sum_val = sum_val + list_of_nums[i]
        if sum_val % 2 == 0:
            sum_count[0] += 1
        else:
            sum_count[1] += 1

    return sum_count[0] * (sum_count[0] - 1) / 2 + sum_count[1] * (sum_count[1] - 1) / 2

class TestSubarrayEvenSum(unittest.TestCase):

    def test_subarray_even_sum(self):
        list_of_nums = [1, 2, 2, 3, 4, 1]
        self.assertEqual(subarrays_with_even_sum(list_of_nums), 9)
<|endoftext|>