<commit_before>"""
https://www.hackerrank.com/contests/booking-passions-hacked-backend/challenges/good-memories/submissions/code/6536390

Problem reduction: do all friends' sequences form a directed acyclic graph (DAG)?

Runtime: O(V + E)
"""

import sys
from collections import defaultdict

sys.setrecursionlimit(2000)  # for recursive depth-first search on up to 1000 attractions

def big_union(sets): return frozenset().union(e for s in sets for e in s)

def is_dag(graph, n_nodes):
    # painting method from http://codeforces.com/blog/entry/4907?#comment-99802
    WHITE, GREY, BLACK = 0, 1, 2
    visited = [WHITE for _ in xrange(n_nodes)]

    def _has_back_edge(node):
        if visited[node] == BLACK:
            return False
        elif visited[node] == GREY:
            return True

        visited[node] = GREY
        next_nodes = graph.get(node)
        has_back_edge = any(_has_back_edge(node2) for node2 in next_nodes) if next_nodes else False
        visited[node] = BLACK
        return has_back_edge

    return not any(_has_back_edge(node) for node in graph)

def is_dag_alt(graph, n_nodes):
    "Explicitly manage stack, rather than rely on recursion."
    WHITE, GREY, BLACK = 0, 1, 2
    visited = [WHITE for _ in xrange(n_nodes)]

    def _has_back_edge(node):
        if visited[node] is BLACK:
            return False

        stack = [node]

        while stack:
            node = stack.pop()
            color = visited[node]
            if color is GREY:
                visited[node] = BLACK
            elif color is WHITE:
                visited[node] = GREY
                stack.append(node)
                for node2 in graph.get(node, ()):
                    color2 = visited[node2]
                    if color2 is GREY:
                        return True
                    elif color2 is WHITE:
                        stack.append(node2)

        return False

    return not any(_has_back_edge(node) for node in graph)

def create_graph(sequences):
    places = big_union(sequences)
    n_places = len(places)
    place_id_mapping = {p:i for i,p in enumerate(places)}
    graph = defaultdict(set)

    for seq in sequences:
        for a1,a2 in zip(seq, seq[1:]):
            graph[place_id_mapping[a1]].add(place_id_mapping[a2])

    return {k:tuple(v) for k,v in graph.iteritems()}, n_places

def solve(sequences):
    graph, n_nodes = create_graph(sequences)
    # return is_dag(graph, n_nodes)
    return is_dag_alt(graph, n_nodes)

def parse_sequence(line):
    places = (e.strip() for e in line.strip().split(','))
    return tuple(p for p in places if p)

def main():
    x = int(raw_input().strip())
    for _ in xrange(x):
        n = int(raw_input().strip())
        sequences = tuple(parse_sequence(raw_input()) for _ in xrange(n))
        print 'ORDER EXISTS' if solve(sequences) else 'ORDER VIOLATION'

if __name__ == '__main__':
    main()
<commit_msg>update good_memories.py with notes<commit_after>"""
https://www.hackerrank.com/contests/booking-passions-hacked-backend/challenges/good-memories/submissions/code/6536390

Problem reduction: do all friends' sequences form a directed acyclic graph (DAG)?

Runtime: O(V + E)
"""

import sys
from collections import defaultdict

sys.setrecursionlimit(2000)  # for recursive depth-first search on up to 1000 attractions

def big_union(sets): return frozenset().union(e for s in sets for e in s)

def is_dag(graph, n_nodes):
    # painting method from http://codeforces.com/blog/entry/4907?#comment-99802
    WHITE, GREY, BLACK = 0, 1, 2
    visited = [WHITE for _ in xrange(n_nodes)]

    def _has_back_edge(node):
        if visited[node] == BLACK:
            return False
        elif visited[node] == GREY:
            return True

        visited[node] = GREY
        next_nodes = graph.get(node)
        has_back_edge = any(_has_back_edge(node2) for node2 in next_nodes) if next_nodes else False
        visited[node] = BLACK
        return has_back_edge

    return not any(_has_back_edge(node) for node in graph)

def is_dag_alt(graph, n_nodes):
    "Explicitly manage stack, rather than rely on recursion."
    WHITE, GREY, BLACK = 0, 1, 2
    visited = [WHITE for _ in xrange(n_nodes)]
    # WHITE means you've never been popped off the stack before.
    # GREY means we're doing DFS from you right now.
    # BLACK means we've completed exhaustive DFS from you.

    # Therefore, it we run into a GREY node while adding nodes to
    # the stack in our DFS, that means there's a cycle.

    def _has_back_edge(node):
        if visited[node] is BLACK:
            return False

        stack = [node]

        while stack:
            node = stack.pop()
            color = visited[node]
            if color is GREY:
                visited[node] = BLACK
            elif color is WHITE:
                visited[node] = GREY
                stack.append(node)
                for node2 in graph.get(node, ()):
                    color2 = visited[node2]
                    if color2 is GREY:
                        return True
                    elif color2 is WHITE:
                        stack.append(node2)

        return False

    return not any(_has_back_edge(node) for node in graph)

def create_graph(sequences):
    places = big_union(sequences)
    n_places = len(places)
    place_id_mapping = {p:i for i,p in enumerate(places)}
    graph = defaultdict(set)

    for seq in sequences:
        for a1,a2 in zip(seq, seq[1:]):
            graph[place_id_mapping[a1]].add(place_id_mapping[a2])

    return {k:tuple(v) for k,v in graph.iteritems()}, n_places

def solve(sequences):
    graph, n_nodes = create_graph(sequences)
    # return is_dag(graph, n_nodes)
    return is_dag_alt(graph, n_nodes)

def parse_sequence(line):
    places = (e.strip() for e in line.strip().split(','))
    return tuple(p for p in places if p)

def main():
    x = int(raw_input().strip())
    for _ in xrange(x):
        n = int(raw_input().strip())
        sequences = tuple(parse_sequence(raw_input()) for _ in xrange(n))
        print 'ORDER EXISTS' if solve(sequences) else 'ORDER VIOLATION'

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-

from __future__ import print_function

import logging
import os
from subprocess import Popen, PIPE, STDOUT

CUR_DIR = os.path.abspath(os.path.dirname(__file__))
PATH_RIOTAM_BACKEND = os.path.normpath(os.path.join(CUR_DIR, ".."))
PROJECT_ROOT_DIR = os.path.normpath(os.path.join(CUR_DIR, ".."))

LOGFILE = os.path.join(PROJECT_ROOT_DIR, "log", "push_webhook_handler_log.txt")

def main():
    """
    Routine to update backend repository. Gets called by push_webhook_handler.py inside frontend"""

    """UPDATE GIT REPOSITORY"""
    output = execute_command(["git", "-C", PROJECT_ROOT_DIR, "pull"])
    logging.debug("PULL BACKEND REPO:\n" + output)

    output = execute_command(["git", "-C", PROJECT_ROOT_DIR, "submodule", "update", "--recursive", "--remote"])
    logging.debug("UPDATE SUBMODULES:\n" + output)

    """SETUP DATABASE"""
    output = execute_command(["python", "db_create.py"], os.path.join(PATH_RIOTAM_BACKEND, "riotam_backend", "setup"))
    logging.debug("DB_CREATE:\n" + output)

    output = execute_command(["python", "db_setup.py"], os.path.join(PATH_RIOTAM_BACKEND, "riotam_backend", "setup"))
    logging.debug("DB_SETUP:\n" + output)

    """UPDATE DATABASE"""
    output = execute_command(["python", "db_update.py"], os.path.join(PATH_RIOTAM_BACKEND, "riotam_backend", "tasks", "database"))
    logging.debug("DB_UPDATE:\n" + output)

    """CREATE STRIPPED RIOT REPOSITORY"""
    output = execute_command(["python", "strip_riot_repo.py"], os.path.join(PATH_RIOTAM_BACKEND, "riotam_backend"))
    logging.debug("STRIP_RIOT_REPO.py:\n" + output)

    # give calling script from frontend an answer
    print("updated backend sucessfully")

def execute_command(cmd, cwd=None):
    """
    Execute command with Popen

    Parameters
    ----------
    cmd: array_like
        List of strings of the command to execute. Needs to be split on spaces
    cwd: string
        working directory to change to (default is None)

    Returns
    -------
    string
        Commandline output

    """
    process = Popen(cmd, stdout=PIPE, stderr=STDOUT, cwd=cwd)
    return process.communicate()[0]

if __name__ == "__main__":

    logging.basicConfig(filename=LOGFILE, format="%(asctime)s [%(levelname)s]: %(message)s",
                        datefmt="%Y-%m-%d %H:%M:%S", level=logging.DEBUG)

    try:
        main()

    except Exception as e:
        logging.error(str(e), exc_info=True)<commit_msg>fixed paths<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-

from __future__ import print_function

import logging
import os
from subprocess import Popen, PIPE, STDOUT

CUR_DIR = os.path.abspath(os.path.dirname(__file__))
PROJECT_ROOT_DIR = os.path.normpath(os.path.join(CUR_DIR, ".."))

LOGFILE = os.path.join(PROJECT_ROOT_DIR, "log", "push_webhook_handler_log.txt")

def main():
    """
    Routine to update backend repository. Gets called by push_webhook_handler.py inside frontend"""

    """UPDATE GIT REPOSITORY"""
    output = execute_command(["git", "-C", PROJECT_ROOT_DIR, "pull"])
    logging.debug("PULL BACKEND REPO:\n" + output)

    output = execute_command(["git", "-C", PROJECT_ROOT_DIR, "submodule", "update", "--recursive", "--remote"])
    logging.debug("UPDATE SUBMODULES:\n" + output)

    """SETUP DATABASE"""
    output = execute_command(["python", "db_create.py"], os.path.join(PROJECT_ROOT_DIR, "riotam_backend", "setup"))
    logging.debug("DB_CREATE:\n" + output)

    output = execute_command(["python", "db_setup.py"], os.path.join(PROJECT_ROOT_DIR, "riotam_backend", "setup"))
    logging.debug("DB_SETUP:\n" + output)

    """UPDATE DATABASE"""
    output = execute_command(["python", "db_update.py"], os.path.join(PROJECT_ROOT_DIR, "riotam_backend", "tasks", "database"))
    logging.debug("DB_UPDATE:\n" + output)

    """CREATE STRIPPED RIOT REPOSITORY"""
    output = execute_command(["python", "strip_riot_repo.py"], os.path.join(PROJECT_ROOT_DIR, "riotam_backend"))
    logging.debug("STRIP_RIOT_REPO.py:\n" + output)

    # give calling script from frontend an answer
    print("updated backend sucessfully")

def execute_command(cmd, cwd=None):
    """
    Execute command with Popen

    Parameters
    ----------
    cmd: array_like
        List of strings of the command to execute. Needs to be split on spaces
    cwd: string
        working directory to change to (default is None)

    Returns
    -------
    string
        Commandline output

    """
    process = Popen(cmd, stdout=PIPE, stderr=STDOUT, cwd=cwd)
    return process.communicate()[0]

if __name__ == "__main__":

    logging.basicConfig(filename=LOGFILE, format="%(asctime)s [%(levelname)s]: %(message)s",
                        datefmt="%Y-%m-%d %H:%M:%S", level=logging.DEBUG)

    try:
        main()

    except Exception as e:
        logging.error(str(e), exc_info=True)<|endoftext|>


<commit_before>import re
import asyncio
from .issue_handler import IssueHandler
from .card_handler import CardHandler
from .enum_handler import EnumHandler

CMD_CARD = "!card "
CMD_CARD_COLLECTIBLE = "!cardc "
CMD_CARD_NONCOLLECTIBLE = "!cardn "
CMD_TAG = "!tag "
CMD_ENUM = "!enum "

def log(message, response, edited):
	print("[%s]" % (message.channel), message.content.encode("utf-8"))
	print("Reponse:", response.encode("utf-8"), edited)

class MessageHandler():
	def __init__(self, config, client):
		self.client = client
		self.issue_handler = IssueHandler(config["repos"])
		self.card_handler = CardHandler()
		self.enum_handler = EnumHandler(config)
		self.max_cards_public = int(config["max_cards_public"])
		self.max_cards_private = int(config["max_cards_private"])

	async def handle(self, message):
		if message.author.id == self.client.user.id:
			return
		if await self.handle_cmd(message):
			return

		matches = re.findall(r"(?<!<)(\w+)?#(\d+)", message.content)
		if len(matches):
			print("[%s]" % (message.channel), message.content)
		for match in matches:
			prefix = match[0]
			issue = match[1]
			response = self.issue_handler.handle(message.channel.name, prefix, issue)
			print("Reponse:", response.encode("utf-8") if response else "None")
			if response is not None:
				await self.client.send_message(message.channel, response)

	async def handle_cmd(self, message, my_message=None):
		if message.content.startswith(CMD_CARD):
			await self.handle_card(message, CMD_CARD, my_message)
			return True
		if message.content.startswith(CMD_CARD_COLLECTIBLE):
			await self.handle_card(message, CMD_CARD_COLLECTIBLE, my_message, True)
			return True
		if message.content.startswith(CMD_CARD_NONCOLLECTIBLE):
			await self.handle_card(message, CMD_CARD_NONCOLLECTIBLE, my_message, False)
			return True
		if message.content.startswith(CMD_TAG):
			response = self.enum_handler.handle("GameTag " + message.content[len(CMD_TAG):])
			await self.respond(message, response, my_message)
			return True
		if message.content.startswith(CMD_ENUM):
			response = self.enum_handler.handle(message.content[len(CMD_ENUM):])
			await self.respond(message, response, my_message)
			return True
		return False

	async def respond(self, message, response, my_message=None):
		log(message, response, my_message is not None)
		if my_message is None:
			my_message = await self.client.send_message(message.channel, response)
		else:
			await self.client.edit_message(my_message, response)
		await self.check_edit(message, my_message)

	async def handle_card(self, message, cmd, my_message, collectible=None):
		response = self.card_handler.handle(message.content[len(cmd):], self.max_response(message), collectible)
		await self.respond(message, response, my_message)

	async def check_edit(self, message, sent):
		original_content = message.content
		for _ in range(0, 30):
			if message.content != original_content:
				await self.handle_cmd(message, sent)
				return
			await asyncio.sleep(1)

	def max_response(self, message):
		return self.max_cards_private if message.channel.is_private else self.max_cards_public
<commit_msg>fix issue regex<commit_after>import re
import asyncio
from .issue_handler import IssueHandler
from .card_handler import CardHandler
from .enum_handler import EnumHandler

CMD_CARD = "!card "
CMD_CARD_COLLECTIBLE = "!cardc "
CMD_CARD_NONCOLLECTIBLE = "!cardn "
CMD_TAG = "!tag "
CMD_ENUM = "!enum "

def log(message, response, edited):
	print("[%s]" % (message.channel), message.content.encode("utf-8"))
	print("Reponse:", response.encode("utf-8"), edited)

class MessageHandler():
	def __init__(self, config, client):
		self.client = client
		self.issue_handler = IssueHandler(config["repos"])
		self.card_handler = CardHandler()
		self.enum_handler = EnumHandler(config)
		self.max_cards_public = int(config["max_cards_public"])
		self.max_cards_private = int(config["max_cards_private"])

	async def handle(self, message):
		if message.author.id == self.client.user.id:
			return
		if await self.handle_cmd(message):
			return

		matches = re.findall(r"(?<!<)(\w+)?#(\d+)($|\s)", message.content)
		if len(matches):
			print("[%s]" % (message.channel), message.content)
		for match in matches:
			prefix = match[0]
			issue = match[1]
			response = self.issue_handler.handle(message.channel.name, prefix, issue)
			print("Reponse:", response.encode("utf-8") if response else "None")
			if response is not None:
				await self.client.send_message(message.channel, response)

	async def handle_cmd(self, message, my_message=None):
		if message.content.startswith(CMD_CARD):
			await self.handle_card(message, CMD_CARD, my_message)
			return True
		if message.content.startswith(CMD_CARD_COLLECTIBLE):
			await self.handle_card(message, CMD_CARD_COLLECTIBLE, my_message, True)
			return True
		if message.content.startswith(CMD_CARD_NONCOLLECTIBLE):
			await self.handle_card(message, CMD_CARD_NONCOLLECTIBLE, my_message, False)
			return True
		if message.content.startswith(CMD_TAG):
			response = self.enum_handler.handle("GameTag " + message.content[len(CMD_TAG):])
			await self.respond(message, response, my_message)
			return True
		if message.content.startswith(CMD_ENUM):
			response = self.enum_handler.handle(message.content[len(CMD_ENUM):])
			await self.respond(message, response, my_message)
			return True
		return False

	async def respond(self, message, response, my_message=None):
		log(message, response, my_message is not None)
		if my_message is None:
			my_message = await self.client.send_message(message.channel, response)
		else:
			await self.client.edit_message(my_message, response)
		await self.check_edit(message, my_message)

	async def handle_card(self, message, cmd, my_message, collectible=None):
		response = self.card_handler.handle(message.content[len(cmd):], self.max_response(message), collectible)
		await self.respond(message, response, my_message)

	async def check_edit(self, message, sent):
		original_content = message.content
		for _ in range(0, 30):
			if message.content != original_content:
				await self.handle_cmd(message, sent)
				return
			await asyncio.sleep(1)

	def max_response(self, message):
		return self.max_cards_private if message.channel.is_private else self.max_cards_public
<|endoftext|>


<commit_before>#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import collections
import itertools

import six

from heat.common import exception
from heat.common.i18n import _
from heat.common.i18n import repr_wrapper

class CircularDependencyException(exception.HeatException):
    msg_fmt = _("Circular Dependency Found: %(cycle)s")

@repr_wrapper
@six.python_2_unicode_compatible
class Node(object):
    """A node in a dependency graph."""

    def __init__(self, requires=None, required_by=None):
        """Initialisation of the node.

        Initialise the node, optionally with a set of keys this node
        requires and/or a set of keys that this node is required by.
        """
        self.require = requires and requires.copy() or set()
        self.satisfy = required_by and required_by.copy() or set()

    def copy(self):
        """Return a copy of the node."""
        return Node(self.require, self.satisfy)

    def reverse_copy(self):
        """Return a copy of the node with the edge directions reversed."""
        return Node(self.satisfy, self.require)

    def required_by(self, source=None):
        """List the keys that require this node, and optionally add new one."""
        if source is not None:
            self.satisfy.add(source)
        return iter(self.satisfy)

    def requires(self, target=None):
        """Add a key that this node requires, and optionally add a new one."""
        if target is not None:
            self.require.add(target)
        return iter(self.require)

    def __isub__(self, target):
        """Remove a key that this node requires."""
        self.require.remove(target)
        return self

    def __nonzero__(self):
        """Return True if this node is not a leaf (it requires other nodes)."""
        return bool(self.require)

    def __bool__(self):
        """Return True if this node is not a leaf (it requires other nodes)."""
        return self.__nonzero__()

    def stem(self):
        """Return True if this node is a stem (required by nothing)."""
        return not bool(self.satisfy)

    def disjoint(self):
        """Return True if this node is both a leaf and a stem."""
        return (not self) and self.stem()

    def __len__(self):
        """Count the number of keys required by this node."""
        return len(self.require)

    def __iter__(self):
        """Iterate over the keys required by this node."""
        return iter(self.require)

    def __str__(self):
        """Return a human-readable string representation of the node."""
        text = '{%s}' % ', '.join(six.text_type(n) for n in self)
        return six.text_type(text)

    def __repr__(self):
        """Return a string representation of the node."""
        return repr(self.require)

@six.python_2_unicode_compatible
class Graph(collections.defaultdict):
    """A mutable mapping of objects to nodes in a dependency graph."""

    def __init__(self, *args):
        super(Graph, self).__init__(Node, *args)

    def map(self, func):
        """Map the supplied function onto each node in the graph.

        Return a dictionary derived from mapping the supplied function onto
        each node in the graph.
        """
        return dict((k, func(n)) for k, n in self.items())

    def copy(self):
        """Return a copy of the graph."""
        return Graph(self.map(lambda n: n.copy()))

    def reverse_copy(self):
        """Return a copy of the graph with the edges reversed."""
        return Graph(self.map(lambda n: n.reverse_copy()))

    def edges(self):
        """Return an iterator over all of the edges in the graph."""
        def outgoing_edges(rqr, node):
            if node.disjoint():
                yield (rqr, None)
            else:
                for rqd in node:
                    yield (rqr, rqd)
        return itertools.chain.from_iterable(outgoing_edges(*i)
                                             for i in six.iteritems(self))

    def __delitem__(self, key):
        """Delete the node given by the specified key from the graph."""
        node = self[key]

        for src in node.required_by():
            src_node = self[src]
            if key in src_node:
                src_node -= key

        return super(Graph, self).__delitem__(key)

    def __str__(self):
        """Convert the graph to a human-readable string."""
        pairs = ('%s: %s' % (six.text_type(k), six.text_type(v))
                 for k, v in six.iteritems(self))
        text = '{%s}' % ', '.join(pairs)
        return six.text_type(text)

    @staticmethod
    def toposort(graph):
        """Return a topologically sorted iterator over a dependency graph.

        This is a destructive operation for the graph.
        """
        for iteration in six.moves.xrange(len(graph)):
            for key, node in six.iteritems(graph):
                if not node:
                    yield key
                    del graph[key]
                    break
            else:
                # There are nodes remaining, but none without
                # dependencies: a cycle
                raise CircularDependencyException(cycle=six.text_type(graph))

@repr_wrapper
@six.python_2_unicode_compatible
class Dependencies(object):
    """Helper class for calculating a dependency graph."""

    def __init__(self, edges=None):
        """Initialise, optionally with a list of edges.

        Each edge takes the form of a (requirer, required) tuple.
        """
        edges = edges or []
        self._graph = Graph()
        for e in edges:
            self += e

    def __iadd__(self, edge):
        """Add another edge, in the form of a (requirer, required) tuple."""
        requirer, required = edge

        if required is None:
            # Just ensure the node is created by accessing the defaultdict
            self._graph[requirer]
        else:
            self._graph[required].required_by(requirer)
            self._graph[requirer].requires(required)

        return self

    def required_by(self, last):
        """List the keys that require the specified node."""
        if last not in self._graph:
            raise KeyError

        return self._graph[last].required_by()

    def requires(self, target):
        """List the keys that require the specified node."""
        if target not in self._graph:
            raise KeyError

        return self._graph[target].requires()

    def __getitem__(self, last):
        """Return a partial dependency graph starting with the specified node.

        Return a subset of the dependency graph consisting of the specified
        node and all those that require it only.
        """
        if last not in self._graph:
            raise KeyError

        def get_edges(key):
            def requirer_edges(rqr):
                # Concatenate the dependency on the current node with the
                # recursive generated list
                return itertools.chain([(rqr, key)], get_edges(rqr))

            # Get the edge list for each node that requires the current node
            edge_lists = six.moves.map(requirer_edges,
                                       self._graph[key].required_by())
            # Combine the lists into one long list
            return itertools.chain.from_iterable(edge_lists)

        if self._graph[last].stem():
            # Nothing requires this, so just add the node itself
            edges = [(last, None)]
        else:
            edges = get_edges(last)

        return Dependencies(edges)

    def leaves(self):
        """Return an iterator over all of the leaf nodes in the graph."""
        return (requirer for requirer, required in self._graph.items()
                if not required)

    def roots(self):
        """Return an iterator over all of the root nodes in the graph."""
        return (requirer for requirer, required in self.graph(
            reverse=True).items() if not required)

    def translate(self, transform):
        """Translate all of the nodes using a transform function.

        Returns a new Dependencies object.
        """
        def transform_key(key):
            return transform(key) if key is not None else None

        edges = self._graph.edges()
        return type(self)(tuple(map(transform_key, e)) for e in edges)

    def __str__(self):
        """Return a human-readable string repr of the dependency graph."""
        return six.text_type(self._graph)

    def __repr__(self):
        """Return a consistent string representation of the object."""
        edge_reprs = list(repr(e) for e in self._graph.edges())
        edge_reprs.sort()
        text = 'Dependencies([%s])' % ', '.join(edge_reprs)
        return text

    def graph(self, reverse=False):
        """Return a copy of the underlying dependency graph."""
        if reverse:
            return self._graph.reverse_copy()
        else:
            return self._graph.copy()

    def __iter__(self):
        """Return a topologically sorted iterator."""
        return Graph.toposort(self.graph())

    def __reversed__(self):
        """Return a reverse topologically sorted iterator."""
        return Graph.toposort(self.graph(reverse=True))
<commit_msg>Use __slots__ for dependencies.Node<commit_after>#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import collections
import itertools

import six

from heat.common import exception
from heat.common.i18n import _
from heat.common.i18n import repr_wrapper

class CircularDependencyException(exception.HeatException):
    msg_fmt = _("Circular Dependency Found: %(cycle)s")

@repr_wrapper
@six.python_2_unicode_compatible
class Node(object):
    """A node in a dependency graph."""

    __slots__ = ('require', 'satisfy')

    def __init__(self, requires=None, required_by=None):
        """Initialisation of the node.

        Initialise the node, optionally with a set of keys this node
        requires and/or a set of keys that this node is required by.
        """
        self.require = requires and requires.copy() or set()
        self.satisfy = required_by and required_by.copy() or set()

    def copy(self):
        """Return a copy of the node."""
        return Node(self.require, self.satisfy)

    def reverse_copy(self):
        """Return a copy of the node with the edge directions reversed."""
        return Node(self.satisfy, self.require)

    def required_by(self, source=None):
        """List the keys that require this node, and optionally add new one."""
        if source is not None:
            self.satisfy.add(source)
        return iter(self.satisfy)

    def requires(self, target=None):
        """Add a key that this node requires, and optionally add a new one."""
        if target is not None:
            self.require.add(target)
        return iter(self.require)

    def __isub__(self, target):
        """Remove a key that this node requires."""
        self.require.remove(target)
        return self

    def __nonzero__(self):
        """Return True if this node is not a leaf (it requires other nodes)."""
        return bool(self.require)

    def __bool__(self):
        """Return True if this node is not a leaf (it requires other nodes)."""
        return self.__nonzero__()

    def stem(self):
        """Return True if this node is a stem (required by nothing)."""
        return not bool(self.satisfy)

    def disjoint(self):
        """Return True if this node is both a leaf and a stem."""
        return (not self) and self.stem()

    def __len__(self):
        """Count the number of keys required by this node."""
        return len(self.require)

    def __iter__(self):
        """Iterate over the keys required by this node."""
        return iter(self.require)

    def __str__(self):
        """Return a human-readable string representation of the node."""
        text = '{%s}' % ', '.join(six.text_type(n) for n in self)
        return six.text_type(text)

    def __repr__(self):
        """Return a string representation of the node."""
        return repr(self.require)

@six.python_2_unicode_compatible
class Graph(collections.defaultdict):
    """A mutable mapping of objects to nodes in a dependency graph."""

    def __init__(self, *args):
        super(Graph, self).__init__(Node, *args)

    def map(self, func):
        """Map the supplied function onto each node in the graph.

        Return a dictionary derived from mapping the supplied function onto
        each node in the graph.
        """
        return dict((k, func(n)) for k, n in self.items())

    def copy(self):
        """Return a copy of the graph."""
        return Graph(self.map(lambda n: n.copy()))

    def reverse_copy(self):
        """Return a copy of the graph with the edges reversed."""
        return Graph(self.map(lambda n: n.reverse_copy()))

    def edges(self):
        """Return an iterator over all of the edges in the graph."""
        def outgoing_edges(rqr, node):
            if node.disjoint():
                yield (rqr, None)
            else:
                for rqd in node:
                    yield (rqr, rqd)
        return itertools.chain.from_iterable(outgoing_edges(*i)
                                             for i in six.iteritems(self))

    def __delitem__(self, key):
        """Delete the node given by the specified key from the graph."""
        node = self[key]

        for src in node.required_by():
            src_node = self[src]
            if key in src_node:
                src_node -= key

        return super(Graph, self).__delitem__(key)

    def __str__(self):
        """Convert the graph to a human-readable string."""
        pairs = ('%s: %s' % (six.text_type(k), six.text_type(v))
                 for k, v in six.iteritems(self))
        text = '{%s}' % ', '.join(pairs)
        return six.text_type(text)

    @staticmethod
    def toposort(graph):
        """Return a topologically sorted iterator over a dependency graph.

        This is a destructive operation for the graph.
        """
        for iteration in six.moves.xrange(len(graph)):
            for key, node in six.iteritems(graph):
                if not node:
                    yield key
                    del graph[key]
                    break
            else:
                # There are nodes remaining, but none without
                # dependencies: a cycle
                raise CircularDependencyException(cycle=six.text_type(graph))

@repr_wrapper
@six.python_2_unicode_compatible
class Dependencies(object):
    """Helper class for calculating a dependency graph."""

    def __init__(self, edges=None):
        """Initialise, optionally with a list of edges.

        Each edge takes the form of a (requirer, required) tuple.
        """
        edges = edges or []
        self._graph = Graph()
        for e in edges:
            self += e

    def __iadd__(self, edge):
        """Add another edge, in the form of a (requirer, required) tuple."""
        requirer, required = edge

        if required is None:
            # Just ensure the node is created by accessing the defaultdict
            self._graph[requirer]
        else:
            self._graph[required].required_by(requirer)
            self._graph[requirer].requires(required)

        return self

    def required_by(self, last):
        """List the keys that require the specified node."""
        if last not in self._graph:
            raise KeyError

        return self._graph[last].required_by()

    def requires(self, target):
        """List the keys that require the specified node."""
        if target not in self._graph:
            raise KeyError

        return self._graph[target].requires()

    def __getitem__(self, last):
        """Return a partial dependency graph starting with the specified node.

        Return a subset of the dependency graph consisting of the specified
        node and all those that require it only.
        """
        if last not in self._graph:
            raise KeyError

        def get_edges(key):
            def requirer_edges(rqr):
                # Concatenate the dependency on the current node with the
                # recursive generated list
                return itertools.chain([(rqr, key)], get_edges(rqr))

            # Get the edge list for each node that requires the current node
            edge_lists = six.moves.map(requirer_edges,
                                       self._graph[key].required_by())
            # Combine the lists into one long list
            return itertools.chain.from_iterable(edge_lists)

        if self._graph[last].stem():
            # Nothing requires this, so just add the node itself
            edges = [(last, None)]
        else:
            edges = get_edges(last)

        return Dependencies(edges)

    def leaves(self):
        """Return an iterator over all of the leaf nodes in the graph."""
        return (requirer for requirer, required in self._graph.items()
                if not required)

    def roots(self):
        """Return an iterator over all of the root nodes in the graph."""
        return (requirer for requirer, required in self.graph(
            reverse=True).items() if not required)

    def translate(self, transform):
        """Translate all of the nodes using a transform function.

        Returns a new Dependencies object.
        """
        def transform_key(key):
            return transform(key) if key is not None else None

        edges = self._graph.edges()
        return type(self)(tuple(map(transform_key, e)) for e in edges)

    def __str__(self):
        """Return a human-readable string repr of the dependency graph."""
        return six.text_type(self._graph)

    def __repr__(self):
        """Return a consistent string representation of the object."""
        edge_reprs = list(repr(e) for e in self._graph.edges())
        edge_reprs.sort()
        text = 'Dependencies([%s])' % ', '.join(edge_reprs)
        return text

    def graph(self, reverse=False):
        """Return a copy of the underlying dependency graph."""
        if reverse:
            return self._graph.reverse_copy()
        else:
            return self._graph.copy()

    def __iter__(self):
        """Return a topologically sorted iterator."""
        return Graph.toposort(self.graph())

    def __reversed__(self):
        """Return a reverse topologically sorted iterator."""
        return Graph.toposort(self.graph(reverse=True))
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import urllib2
import json

from heat.common import exception
from heat.engine import stack
from heat.db import api as db_api
from heat.engine import parser
from novaclient.exceptions import NotFound

from heat.openstack.common import log as logging

logger = logging.getLogger(__file__)

lb_template = '''
{
  "AWSTemplateFormatVersion": "2010-09-09",
  "Description": "Built in HAProxy server",
  "Parameters" : {
    "KeyName" : {
      "Type" : "String"
    }
  },
  "Resources": {
    "latency_watcher": {
     "Type": "AWS::CloudWatch::Alarm",
     "Properties": {
        "MetricName": "Latency",
        "Namespace": "AWS/ELB",
        "Statistic": "Average",
        "Period": "60",
        "EvaluationPeriods": "1",
        "Threshold": "2",
        "AlarmActions": [],
        "ComparisonOperator": "GreaterThanThreshold"
      }
    },
    "LB_instance": {
      "Type": "AWS::EC2::Instance",
      "Metadata": {
        "AWS::CloudFormation::Init": {
          "config": {
            "packages": {
              "yum": {
                "cronie"         : [],
                "haproxy"        : [],
                "socat"          : [],
                "python-psutil"  : []
              }
            },
            "services": {
              "systemd": {
                "crond"     : { "enabled" : "true", "ensureRunning" : "true" },
                "haproxy"   : { "enabled": "true", "ensureRunning": "true" }
              }
            },
            "files": {
              "/etc/cfn/cfn-hup.conf" : {
                "content" : { "Fn::Join" : ["", [
                  "[main]\\n",
                  "stack=", { "Ref" : "AWS::StackName" }, "\\n",
                  "credential-file=/etc/cfn/cfn-credentials\\n",
                  "region=", { "Ref" : "AWS::Region" }, "\\n",
                  "interval=60\\n"
                ]]},
                "mode"    : "000400",
                "owner"   : "root",
                "group"   : "root"
              },
              "/etc/cfn/hooks.conf" : {
                "content": { "Fn::Join" : ["", [
                  "[cfn-init]\\n",
                  "triggers=post.update\\n",
                  "path=Resources.LB_instance.Metadata\\n",
                  "action=/opt/aws/bin/cfn-init -s ",
                  { "Ref": "AWS::StackName" },
                  "    -r LB_instance ",
                  "    --region ", { "Ref": "AWS::Region" }, "\\n",
                  "runas=root\\n",
                  "\\n",
                  "[reload]\\n",
                  "triggers=post.update\\n",
                  "path=Resources.LB_instance.Metadata\\n",
                  "action=systemctl reload haproxy.service\\n",
                  "runas=root\\n"
                ]]},
                "mode"    : "000400",
                "owner"   : "root",
                "group"   : "root"
              },
              "/etc/haproxy/haproxy.cfg": {
                "content": "",
                "mode": "000644",
                "owner": "root",
                "group": "root"
              },
              "/tmp/cfn-hup-crontab.txt" : {
                "content" : { "Fn::Join" : ["", [
                "MAIL=\\"\\"\\n",
                "\\n",
                "* * * * * /opt/aws/bin/cfn-hup -f\\n",
                "* * * * * /opt/aws/bin/cfn-push-stats ",
                " --watch latency_watcher --haproxy\\n"
                ]]},
                "mode"    : "000600",
                "owner"   : "root",
                "group"   : "root"
              }
            }
          }
        }
      },
      "Properties": {
        "ImageId": "F16-x86_64-cfntools",
        "InstanceType": "m1.small",
        "KeyName": { "Ref": "KeyName" },
        "UserData": { "Fn::Base64": { "Fn::Join": ["", [
          "#!/bin/bash -v\\n",
          "/opt/aws/bin/cfn-init -s ",
          { "Ref": "AWS::StackName" },
          "    -r LB_instance ",
          "    --region ", { "Ref": "AWS::Region" }, "\\n",
          "touch /etc/cfn/cfn-credentials\\n",
          "# install cfn-hup crontab\\n",
          "crontab /tmp/cfn-hup-crontab.txt\\n"
        ]]}}
      }
    }
  },

  "Outputs": {
    "PublicIp": {
      "Value": { "Fn::GetAtt": [ "LB_instance", "PublicIp" ] },
      "Description": "instance IP"
    }
  }
}
'''

#
# TODO(asalkeld) the above inline template _could_ be placed in an external
# file at the moment this is because we will probably need to implement a
# LoadBalancer based on keepalived as well (for for ssl support).
#
class LoadBalancer(stack.Stack):

    listeners_schema = {
        'InstancePort': {'Type': 'Integer',
                         'Required': True},
        'LoadBalancerPort': {'Type': 'Integer',
                             'Required': True},
        'Protocol': {'Type': 'String',
                     'Required': True,
                     'AllowedValues': ['TCP', 'HTTP']},
        'SSLCertificateId': {'Type': 'String',
                             'Implemented': False},
        'PolicyNames': {'Type': 'Map',
                        'Implemented': False}
    }
    healthcheck_schema = {
        'HealthyThreshold': {'Type': 'Integer',
                             'Required': True},
        'Interval': {'Type': 'Integer',
                     'Required': True},
        'Target': {'Type': 'String',
                   'Required': True},
        'Timeout': {'Type': 'Integer',
                    'Required': True},
        'UnhealthyThreshold': {'Type': 'Integer',
                              'Required': True},
    }

    properties_schema = {
        'AvailabilityZones': {'Type': 'List',
                              'Required': True},
        'HealthCheck': {'Type': 'Map',
                        'Schema': healthcheck_schema},
        'Instances': {'Type': 'List'},
        'Listeners': {'Type': 'List',
                      'Schema': listeners_schema},
        'AppCookieStickinessPolicy': {'Type': 'String',
                                      'Implemented': False},
        'LBCookieStickinessPolicy': {'Type': 'String',
                                     'Implemented': False},
        'SecurityGroups': {'Type': 'String',
                           'Implemented': False},
        'Subnets': {'Type': 'List',
                    'Implemented': False}
    }

    def _params(self):
        # total hack - probably need an admin key here.
        params = {'KeyName': {'Ref': 'KeyName'}}
        p = self.stack.resolve_static_data(params)
        return p

    def _instance_to_ipaddress(self, inst):
        '''
        Return the server's IP address, fetching it from Nova
        '''
        try:
            server = self.nova().servers.get(inst)
        except NotFound as ex:
            logger.warn('Instance (%s) not found: %s' % (inst, str(ex)))
        else:
            for n in server.networks:
                return server.networks[n][0]

        return '0.0.0.0'

    def _haproxy_config(self, templ):
        # initial simplifications:
        # - only one Listener
        # - only http (no tcp or ssl)
        #
        # option httpchk HEAD /check.txt HTTP/1.0
        gl = '''
    global
        daemon
        maxconn 256
        stats socket /tmp/.haproxy-stats

    defaults
        mode http
        timeout connect 5000ms
        timeout client 50000ms
        timeout server 50000ms
'''

        listener = self.properties['Listeners'][0]
        lb_port = listener['LoadBalancerPort']
        inst_port = listener['InstancePort']
        spaces = '            '
        frontend = '''
        frontend http
            bind *:%s
''' % (lb_port)

        health_chk = self.properties['HealthCheck']
        if health_chk:
            check = 'check inter %ss fall %s rise %s' % (
                    health_chk['Interval'],
                    health_chk['UnhealthyThreshold'],
                    health_chk['HealthyThreshold'])
            timeout_check = 'timeout check %ds' % (health_chk['Timeout'])
        else:
            check = ''
            timeout_check = ''

        backend = '''
        default_backend servers

        backend servers
            balance roundrobin
            option http-server-close
            option forwardfor
            option httpchk
            %s
''' % timeout_check

        servers = []
        n = 1
        for i in self.properties['Instances']:
            ip = self._instance_to_ipaddress(i)
            logger.debug('haproxy server:%s' % ip)
            servers.append('%sserver server%d %s:%s %s' % (spaces, n,
                                                           ip, inst_port,
                                                           check))
            n = n + 1

        return '%s%s%s%s\n' % (gl, frontend, backend, '\n'.join(servers))

    def handle_create(self):
        templ = json.loads(lb_template)

        if self.properties['Instances']:
            md = templ['Resources']['LB_instance']['Metadata']
            files = md['AWS::CloudFormation::Init']['config']['files']
            cfg = self._haproxy_config(templ)
            files['/etc/haproxy/haproxy.cfg']['content'] = cfg

        self.create_with_template(templ)

    def validate(self):
        '''
        Validate any of the provided params
        '''
        res = super(LoadBalancer, self).validate()
        if res:
            return res

        health_chk = self.properties['HealthCheck']
        if health_chk:
            if float(health_chk['Interval']) < float(health_chk['Timeout']):
                return {'Error':
                        'Interval must be larger than Timeout'}

    def reload(self, inst_list):
        '''
        re-generate the Metadata
        save it to the db.
        rely on the cfn-hup to reconfigure HAProxy
        '''
        self.properties['Instances'] = inst_list
        templ = json.loads(lb_template)
        cfg = self._haproxy_config(templ)

        md = self.nested()['LB_instance'].metadata
        files = md['AWS::CloudFormation::Init']['config']['files']
        files['/etc/haproxy/haproxy.cfg']['content'] = cfg

        self.nested()['LB_instance'].metadata = md

    def FnGetRefId(self):
        return unicode(self.name)

    def handle_update(self):
        return self.UPDATE_REPLACE

    def FnGetAtt(self, key):
        '''
        We don't really support any of these yet.
        '''
        allow = ('CanonicalHostedZoneName',
                 'CanonicalHostedZoneNameID',
                 'DNSName',
                 'SourceSecurityGroupName',
                 'SourceSecurityGroupOwnerAlias')

        if not key in allow:
            raise exception.InvalidTemplateAttribute(resource=self.name,
                                                     key=key)

        if key == 'DNSName':
            return stack.Stack.FnGetAtt(self, 'Outputs.PublicIp')
        else:
            return ''
<commit_msg>Fix backtrace when using loadbalancer<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import urllib2
import json

from heat.common import exception
from heat.engine import stack
from heat.db import api as db_api
from heat.engine import parser
from novaclient.exceptions import NotFound

from heat.openstack.common import log as logging

logger = logging.getLogger(__file__)

lb_template = '''
{
  "AWSTemplateFormatVersion": "2010-09-09",
  "Description": "Built in HAProxy server",
  "Parameters" : {
    "KeyName" : {
      "Type" : "String"
    }
  },
  "Resources": {
    "latency_watcher": {
     "Type": "AWS::CloudWatch::Alarm",
     "Properties": {
        "MetricName": "Latency",
        "Namespace": "AWS/ELB",
        "Statistic": "Average",
        "Period": "60",
        "EvaluationPeriods": "1",
        "Threshold": "2",
        "AlarmActions": [],
        "ComparisonOperator": "GreaterThanThreshold"
      }
    },
    "LB_instance": {
      "Type": "AWS::EC2::Instance",
      "Metadata": {
        "AWS::CloudFormation::Init": {
          "config": {
            "packages": {
              "yum": {
                "cronie"         : [],
                "haproxy"        : [],
                "socat"          : [],
                "python-psutil"  : []
              }
            },
            "services": {
              "systemd": {
                "crond"     : { "enabled" : "true", "ensureRunning" : "true" },
                "haproxy"   : { "enabled": "true", "ensureRunning": "true" }
              }
            },
            "files": {
              "/etc/cfn/cfn-hup.conf" : {
                "content" : { "Fn::Join" : ["", [
                  "[main]\\n",
                  "stack=", { "Ref" : "AWS::StackName" }, "\\n",
                  "credential-file=/etc/cfn/cfn-credentials\\n",
                  "region=", { "Ref" : "AWS::Region" }, "\\n",
                  "interval=60\\n"
                ]]},
                "mode"    : "000400",
                "owner"   : "root",
                "group"   : "root"
              },
              "/etc/cfn/hooks.conf" : {
                "content": { "Fn::Join" : ["", [
                  "[cfn-init]\\n",
                  "triggers=post.update\\n",
                  "path=Resources.LB_instance.Metadata\\n",
                  "action=/opt/aws/bin/cfn-init -s ",
                  { "Ref": "AWS::StackName" },
                  "    -r LB_instance ",
                  "    --region ", { "Ref": "AWS::Region" }, "\\n",
                  "runas=root\\n",
                  "\\n",
                  "[reload]\\n",
                  "triggers=post.update\\n",
                  "path=Resources.LB_instance.Metadata\\n",
                  "action=systemctl reload haproxy.service\\n",
                  "runas=root\\n"
                ]]},
                "mode"    : "000400",
                "owner"   : "root",
                "group"   : "root"
              },
              "/etc/haproxy/haproxy.cfg": {
                "content": "",
                "mode": "000644",
                "owner": "root",
                "group": "root"
              },
              "/tmp/cfn-hup-crontab.txt" : {
                "content" : { "Fn::Join" : ["", [
                "MAIL=\\"\\"\\n",
                "\\n",
                "* * * * * /opt/aws/bin/cfn-hup -f\\n",
                "* * * * * /opt/aws/bin/cfn-push-stats ",
                " --watch latency_watcher --haproxy\\n"
                ]]},
                "mode"    : "000600",
                "owner"   : "root",
                "group"   : "root"
              }
            }
          }
        }
      },
      "Properties": {
        "ImageId": "F16-x86_64-cfntools",
        "InstanceType": "m1.small",
        "KeyName": { "Ref": "KeyName" },
        "UserData": { "Fn::Base64": { "Fn::Join": ["", [
          "#!/bin/bash -v\\n",
          "/opt/aws/bin/cfn-init -s ",
          { "Ref": "AWS::StackName" },
          "    -r LB_instance ",
          "    --region ", { "Ref": "AWS::Region" }, "\\n",
          "touch /etc/cfn/cfn-credentials\\n",
          "# install cfn-hup crontab\\n",
          "crontab /tmp/cfn-hup-crontab.txt\\n"
        ]]}}
      }
    }
  },

  "Outputs": {
    "PublicIp": {
      "Value": { "Fn::GetAtt": [ "LB_instance", "PublicIp" ] },
      "Description": "instance IP"
    }
  }
}
'''

#
# TODO(asalkeld) the above inline template _could_ be placed in an external
# file at the moment this is because we will probably need to implement a
# LoadBalancer based on keepalived as well (for for ssl support).
#
class LoadBalancer(stack.Stack):

    listeners_schema = {
        'InstancePort': {'Type': 'Integer',
                         'Required': True},
        'LoadBalancerPort': {'Type': 'Integer',
                             'Required': True},
        'Protocol': {'Type': 'String',
                     'Required': True,
                     'AllowedValues': ['TCP', 'HTTP']},
        'SSLCertificateId': {'Type': 'String',
                             'Implemented': False},
        'PolicyNames': {'Type': 'Map',
                        'Implemented': False}
    }
    healthcheck_schema = {
        'HealthyThreshold': {'Type': 'Integer',
                             'Required': True},
        'Interval': {'Type': 'Integer',
                     'Required': True},
        'Target': {'Type': 'String',
                   'Required': True},
        'Timeout': {'Type': 'Integer',
                    'Required': True},
        'UnhealthyThreshold': {'Type': 'Integer',
                              'Required': True},
    }

    properties_schema = {
        'AvailabilityZones': {'Type': 'List',
                              'Required': True},
        'HealthCheck': {'Type': 'Map',
                        'Schema': healthcheck_schema},
        'Instances': {'Type': 'List'},
        'Listeners': {'Type': 'List',
                      'Schema': listeners_schema},
        'AppCookieStickinessPolicy': {'Type': 'String',
                                      'Implemented': False},
        'LBCookieStickinessPolicy': {'Type': 'String',
                                     'Implemented': False},
        'SecurityGroups': {'Type': 'String',
                           'Implemented': False},
        'Subnets': {'Type': 'List',
                    'Implemented': False}
    }

    def _params(self):
        # total hack - probably need an admin key here.
        params = {'KeyName': {'Ref': 'KeyName'}}
        p = self.stack.resolve_static_data(params)
        return p

    def _instance_to_ipaddress(self, inst):
        '''
        Return the server's IP address, fetching it from Nova
        '''
        try:
            server = self.nova().servers.get(inst)
        except NotFound as ex:
            logger.warn('Instance (%s) not found: %s' % (inst, str(ex)))
        else:
            for n in server.networks:
                return server.networks[n][0]

        return '0.0.0.0'

    def _haproxy_config(self, templ):
        # initial simplifications:
        # - only one Listener
        # - only http (no tcp or ssl)
        #
        # option httpchk HEAD /check.txt HTTP/1.0
        gl = '''
    global
        daemon
        maxconn 256
        stats socket /tmp/.haproxy-stats

    defaults
        mode http
        timeout connect 5000ms
        timeout client 50000ms
        timeout server 50000ms
'''

        listener = self.properties['Listeners'][0]
        lb_port = listener['LoadBalancerPort']
        inst_port = listener['InstancePort']
        spaces = '            '
        frontend = '''
        frontend http
            bind *:%s
''' % (lb_port)

        health_chk = self.properties['HealthCheck']
        if health_chk:
            check = 'check inter %ss fall %s rise %s' % (
                    health_chk['Interval'],
                    health_chk['UnhealthyThreshold'],
                    health_chk['HealthyThreshold'])
            timeout_check = 'timeout check %ds' % int(health_chk['Timeout'])
        else:
            check = ''
            timeout_check = ''

        backend = '''
        default_backend servers

        backend servers
            balance roundrobin
            option http-server-close
            option forwardfor
            option httpchk
            %s
''' % timeout_check

        servers = []
        n = 1
        for i in self.properties['Instances']:
            ip = self._instance_to_ipaddress(i)
            logger.debug('haproxy server:%s' % ip)
            servers.append('%sserver server%d %s:%s %s' % (spaces, n,
                                                           ip, inst_port,
                                                           check))
            n = n + 1

        return '%s%s%s%s\n' % (gl, frontend, backend, '\n'.join(servers))

    def handle_create(self):
        templ = json.loads(lb_template)

        if self.properties['Instances']:
            md = templ['Resources']['LB_instance']['Metadata']
            files = md['AWS::CloudFormation::Init']['config']['files']
            cfg = self._haproxy_config(templ)
            files['/etc/haproxy/haproxy.cfg']['content'] = cfg

        self.create_with_template(templ)

    def validate(self):
        '''
        Validate any of the provided params
        '''
        res = super(LoadBalancer, self).validate()
        if res:
            return res

        health_chk = self.properties['HealthCheck']
        if health_chk:
            if float(health_chk['Interval']) < float(health_chk['Timeout']):
                return {'Error':
                        'Interval must be larger than Timeout'}

    def reload(self, inst_list):
        '''
        re-generate the Metadata
        save it to the db.
        rely on the cfn-hup to reconfigure HAProxy
        '''
        self.properties['Instances'] = inst_list
        templ = json.loads(lb_template)
        cfg = self._haproxy_config(templ)

        md = self.nested()['LB_instance'].metadata
        files = md['AWS::CloudFormation::Init']['config']['files']
        files['/etc/haproxy/haproxy.cfg']['content'] = cfg

        self.nested()['LB_instance'].metadata = md

    def FnGetRefId(self):
        return unicode(self.name)

    def handle_update(self):
        return self.UPDATE_REPLACE

    def FnGetAtt(self, key):
        '''
        We don't really support any of these yet.
        '''
        allow = ('CanonicalHostedZoneName',
                 'CanonicalHostedZoneNameID',
                 'DNSName',
                 'SourceSecurityGroupName',
                 'SourceSecurityGroupOwnerAlias')

        if not key in allow:
            raise exception.InvalidTemplateAttribute(resource=self.name,
                                                     key=key)

        if key == 'DNSName':
            return stack.Stack.FnGetAtt(self, 'Outputs.PublicIp')
        else:
            return ''
<|endoftext|>


<commit_before>from django.test import TestCase
from hello.twitter_api import TwitterCli, get_twitter_comments, json_into_table
import io
import json
from urllib.error import HTTPError

with io.open("hello/tests/example_twit.json") as samplejson:
    SAMPLE_JSON = json.loads(samplejson.read())

EXPECTED_TWITTER = {
    "symbols": [],
    "author": "Duane Newman",
    "hashtags": [
        "MSFT",
        "Build2016",
        "mountaindew",
        "bldwin",
        "BeverageFail",
        "HadToDrinkPepsi"
    ],
    "source": "twitter",
    "focus": "MSFT",
    "author_image": "http://pbs.twimg.com/profile_images/558825363619344386/gUN09sSf_normal.jpeg",
    "urls": [],
    "created_time": "2009-01-29T21:01:24Z",
    "popularity": 0,
    "content": "Hey #MSFT, I thought #Build2016 was a developer conference. Where's the #mountaindew? #bldwin #BeverageFail #HadToDrinkPepsi",
    "social_id": 19734130
}

class TwitterCase(TestCase):
    """Test Twitter API."""

    def test_dne_ticker(self):
        """Test access without auth."""
        wrong = get_twitter_comments("opwuirehe")
        self.assertEqual(wrong, [])

    def test_no_access(self):
        """Test acess without proper auth."""
        with self.assertRaises(HTTPError):
            client = TwitterCli("merp", "1333322")
            resp = client.request("https://api.twitter.com/1.1/search/tweets.json?q=%23MSFT")
            return resp

    def test_json_to_table(self):
        """Test the corrected format from json to db."""
        jsonified = json_into_table(SAMPLE_JSON, "MSFT")
        self.assertEqual(EXPECTED_TWITTER, jsonified)

    def test_invalid_json(self):
        """Test passing in message with invalid ticker."""
        with self.assertRaises(ValueError):
            so_true = False
            json_into_table(SAMPLE_JSON, so_true)
<commit_msg>all functions covered<commit_after>from django.test import TestCase
from hello.twitter_api import (
    TwitterCli,
    get_twitter_comments,
    json_into_table,
    save_tweets
)
import io
import json
from urllib.error import HTTPError

with io.open("hello/tests/example_twit.json") as samplejson:
    SAMPLE_JSON = json.loads(samplejson.read())

EXPECTED_TWITTER = {
    "symbols": [],
    "author": "Duane Newman",
    "hashtags": [
        "MSFT",
        "Build2016",
        "mountaindew",
        "bldwin",
        "BeverageFail",
        "HadToDrinkPepsi"
    ],
    "source": "twitter",
    "focus": "MSFT",
    "author_image": "http://pbs.twimg.com/profile_images/558825363619344386/gUN09sSf_normal.jpeg",
    "urls": [],
    "created_time": "2009-01-29T21:01:24Z",
    "popularity": 0,
    "content": "Hey #MSFT, I thought #Build2016 was a developer conference. Where's the #mountaindew? #bldwin #BeverageFail #HadToDrinkPepsi",
    "social_id": 19734130
}

class TwitterCase(TestCase):
    """Test Twitter API."""

    def test_dne_ticker(self):
        """Test access without auth."""
        wrong = get_twitter_comments("opwuirehe")
        self.assertEqual(wrong, [])

    def test_no_access(self):
        """Test acess without proper auth."""
        with self.assertRaises(HTTPError):
            client = TwitterCli("merp", "1333322")
            resp = client.request("https://api.twitter.com/1.1/search/tweets.json?q=%23MSFT")
            return resp

    def test_json_to_table(self):
        """Test the corrected format from json to db."""
        jsonified = json_into_table(SAMPLE_JSON, "MSFT")
        self.assertEqual(EXPECTED_TWITTER, jsonified)

    def test_invalid_json(self):
        """Test passing in message with invalid ticker."""
        with self.assertRaises(ValueError):
            so_true = False
            json_into_table(SAMPLE_JSON, so_true)

    def test_saving_tweetdict(self):
        """Test if dict of tweets are saved."""
        self.assertTrue(save_tweets(EXPECTED_TWITTER))
        self.assertFalse(save_tweets(EXPECTED_TWITTER))
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

''' Combine file from:
  * script argument 1
  with content of file from:
  * script argument 2
  using the beginning of line separators
  hardcoded using regexes in this file:

  We exclude any text using the separate
  regex specified here
'''

import re
import sys

insert_separator_regex = r'(.*?\[\^\]\:\ \(autogen_docs_start\))(.*?)(\n\[\^\]\:\ \(autogen_docs_end\).*?$)'
exclude_separator_regex = r'(.*?)Copyright 20\d\d Google LLC.*?limitations under the License.(.*?)$'

if len(sys.argv) != 3:
    sys.exit(1)

input = open(sys.argv[1], "r").read()
replace_content = open(sys.argv[2], "r").read()

# Exclude the specified content from the replacement content
groups = re.match(exclude_separator_regex, replace_content, re.DOTALL).groups(0)
replace_content = groups[0] + groups[1]

# Find where to put the replacement content, overwrite the input file
groups = re.match(insert_separator_regex, input, re.DOTALL).groups(0)
output = groups[0] + replace_content + groups[2]
open(sys.argv[1], "w").write(output)
<commit_msg>[E501] in combine_docfiles.py<commit_after>#!/usr/bin/env python3

# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
''' Combine file from:
  * script argument 1
  with content of file from:
  * script argument 2
  using the beginning of line separators
  hardcoded using regexes in this file:

  We exclude any text using the separate
  regex specified here
'''

import re
import sys

insert_separator_regex = r'(.*?\[\^\]\:\ \(autogen_docs_start\))(.*?)(\n\[\^\]\:\ \(autogen_docs_end\).*?$)'  # noqa: E501
exclude_separator_regex = r'(.*?)Copyright 20\d\d Google LLC.*?limitations under the License.(.*?)$'  # noqa: E501

if len(sys.argv) != 3:
    sys.exit(1)

input = open(sys.argv[1], "r").read()
replace_content = open(sys.argv[2], "r").read()

# Exclude the specified content from the replacement content
groups = re.match(exclude_separator_regex, replace_content,
                  re.DOTALL).groups(0)
replace_content = groups[0] + groups[1]

# Find where to put the replacement content, overwrite the input file
groups = re.match(insert_separator_regex, input, re.DOTALL).groups(0)
output = groups[0] + replace_content + groups[2]
open(sys.argv[1], "w").write(output)
<|endoftext|>


<commit_before>import os
from optparse import make_option
from multiprocessing import Process, Queue

from django.core.management.base import BaseCommand, CommandError

from corehq.apps.domain.models import Domain
from corehq.apps.domainsync.config import DocumentTransform
from dimagi.utils.couch.database import get_db, iter_docs

NUM_PROCESSES = 8

# This management command is closely modeled after copy_domain. It would be
# nice to reduce the code reuse.

class Command(BaseCommand):
    help = "Deletes the contents of a domain"
    args = '<domain>'

    option_list = BaseCommand.option_list + (
        make_option('--simulate',
                    action='store_true',
                    dest='simulate',
                    default=False,
                    help='Don\'t delete anything, print what would be deleted.'),
    )

    def handle(self, *args, **options):
        if len(args) != 1:
            raise CommandError('Usage is delete_domain %s' % self.args)

        sourcedb = get_db()
        domain = args[0].strip()
        simulate = options['simulate']

        if simulate:
            print "\nSimulated run, no data will be deleted.\n"

        domain_doc = Domain.get_by_name(domain)
        if domain_doc is None:
            # If this block not entered, domain will be deleted in delete_docs
            self.delete_domain(sourcedb, domain, simulate)

        startkey = [domain]
        endkey = [domain, {}]
        self.delete_docs(sourcedb, domain, simulate, startkey, endkey)

    def delete_domain(self, sourcedb, domain, simulate):

        result = sourcedb.view(
            "domain/domains",
            key=domain,
            reduce=False,
            include_docs=True
        ).first()

        if result and 'doc' in result:
            if not simulate:
                print "Deleting domain doc"
                domain_doc = Domain.wrap(result['doc'])
                sourcedb.delte_doc(domain_doc) #TODO: Attachements are deleted by this as well, right?
        else:
            print "Domain doc not found for domain %s." % domain

    def delete_docs(self, sourcedb, domain, simulate, startkey, endkey):
        # TODO: What will happen to documents in multiple domains?

        doc_ids = [ result["id"] for result in sourcedb.view("domain/docs",
                                                    startkey=startkey,
                                                    endkey=endkey,
                                                    reduce=False
                                                )]
        total = len(doc_ids)
        count = 0
        msg = "Found %s matching documents in domain: %s" % (total, domain)
        print msg

        err_log = self._get_err_log()

        queue = Queue(150)
        for i in range(NUM_PROCESSES):
            Worker(queue, sourcedb, total, simulate, err_log).start()

        for doc in iter_docs(sourcedb, doc_ids, chunksize=100):
            count += 1
            queue.put((doc, count))

        # shutdown workers
        for i in range(NUM_PROCESSES):
            queue.put(None)

        err_log.close()
        if os.stat(err_log.name)[6] == 0:
            os.remove(err_log.name)
        else:
            print 'Failed document IDs written to %s' % err_log.name

    def _get_err_log(self):
        name = 'copy_domain.err.%s'
        for i in range(1000):  # arbitrarily large number
            candidate = name % i
            if not os.path.isfile(candidate):
                return open(candidate, 'a', buffering=1)

class Worker(Process):

    def __init__(self, queue, sourcedb, total, simulate, err_log):
        super(Worker, self).__init__()
        self.queue = queue
        self.sourcedb = sourcedb
        self.total = total
        self.simulate = simulate
        self.err_log = err_log

    def run(self):
        for doc, count in iter(self.queue.get, None):
            try:
                if not self.simulate:
                    dt = DocumentTransform(doc, self.sourcedb)
                    self.sourcedb.delete_doc(doc) #TODO: This deletes attachements too, right?
                print "     Deleted %s/%s docs (%s: %s)" % (count, self.total, doc["doc_type"], doc["_id"])
            except Exception, e:
                self.err_log.write('%s\n' % doc["_id"])
                print "     Document %s failed! Error is: %s" % (doc["_id"], e)<commit_msg>Added dire warning. We probably shouldn't let this do that. We'll likely be using the other command though.<commit_after>import os
from optparse import make_option
from multiprocessing import Process, Queue

from django.core.management.base import BaseCommand, CommandError

from corehq.apps.domain.models import Domain
from corehq.apps.domainsync.config import DocumentTransform
from dimagi.utils.couch.database import get_db, iter_docs

NUM_PROCESSES = 8

# This management command is closely modeled after copy_domain. It would be
# nice to reduce the code reuse.

class Command(BaseCommand):
    help = "Deletes the contents of a domain. WARNING!!! It will also delete all admin users associated with this domain!!!"
    args = '<domain>'

    option_list = BaseCommand.option_list + (
        make_option('--simulate',
                    action='store_true',
                    dest='simulate',
                    default=False,
                    help='Don\'t delete anything, print what would be deleted.'),
    )

    def handle(self, *args, **options):
        if len(args) != 1:
            raise CommandError('Usage is delete_domain %s' % self.args)

        sourcedb = get_db()
        domain = args[0].strip()
        simulate = options['simulate']

        if simulate:
            print "\nSimulated run, no data will be deleted.\n"

        domain_doc = Domain.get_by_name(domain)
        if domain_doc is None:
            # If this block not entered, domain will be deleted in delete_docs
            self.delete_domain(sourcedb, domain, simulate)

        startkey = [domain]
        endkey = [domain, {}]
        self.delete_docs(sourcedb, domain, simulate, startkey, endkey)

    def delete_domain(self, sourcedb, domain, simulate):

        result = sourcedb.view(
            "domain/domains",
            key=domain,
            reduce=False,
            include_docs=True
        ).first()

        if result and 'doc' in result:
            if not simulate:
                print "Deleting domain doc"
                domain_doc = Domain.wrap(result['doc'])
                sourcedb.delte_doc(domain_doc) #TODO: Attachements are deleted by this as well, right?
        else:
            print "Domain doc not found for domain %s." % domain

    def delete_docs(self, sourcedb, domain, simulate, startkey, endkey):
        # TODO: What will happen to documents in multiple domains?

        doc_ids = [ result["id"] for result in sourcedb.view("domain/docs",
                                                    startkey=startkey,
                                                    endkey=endkey,
                                                    reduce=False
                                                )]
        total = len(doc_ids)
        count = 0
        msg = "Found %s matching documents in domain: %s" % (total, domain)
        print msg

        err_log = self._get_err_log()

        queue = Queue(150)
        for i in range(NUM_PROCESSES):
            Worker(queue, sourcedb, total, simulate, err_log).start()

        for doc in iter_docs(sourcedb, doc_ids, chunksize=100):
            count += 1
            queue.put((doc, count))

        # shutdown workers
        for i in range(NUM_PROCESSES):
            queue.put(None)

        err_log.close()
        if os.stat(err_log.name)[6] == 0:
            os.remove(err_log.name)
        else:
            print 'Failed document IDs written to %s' % err_log.name

    def _get_err_log(self):
        name = 'copy_domain.err.%s'
        for i in range(1000):  # arbitrarily large number
            candidate = name % i
            if not os.path.isfile(candidate):
                return open(candidate, 'a', buffering=1)

class Worker(Process):

    def __init__(self, queue, sourcedb, total, simulate, err_log):
        super(Worker, self).__init__()
        self.queue = queue
        self.sourcedb = sourcedb
        self.total = total
        self.simulate = simulate
        self.err_log = err_log

    def run(self):
        for doc, count in iter(self.queue.get, None):
            try:
                if not self.simulate:
                    dt = DocumentTransform(doc, self.sourcedb)
                    self.sourcedb.delete_doc(doc) #TODO: This deletes attachements too, right?
                print "     Deleted %s/%s docs (%s: %s)" % (count, self.total, doc["doc_type"], doc["_id"])
            except Exception, e:
                self.err_log.write('%s\n' % doc["_id"])
                print "     Document %s failed! Error is: %s" % (doc["_id"], e)<|endoftext|>


<commit_before>from __future__ import absolute_import, print_function
from __future__ import unicode_literals
import json
import os
import re
import six
import yaml
from django.contrib.staticfiles import finders
from django.conf import settings
from subprocess import call

from corehq.apps.hqwebapp.management.commands.resource_static import Command as ResourceStaticCommand
from io import open

class Command(ResourceStaticCommand):
    help = '''
        Runs RequireJS optimizer to concatenate, minify, and bundle JavaScript files
        and set them up with the CDN.
    '''

    root_dir = settings.FILEPATH

    def handle(self, **options):
        try:
            from resource_versions import resource_versions
        except (ImportError, SyntaxError):
            resource_versions = {}

        # Write build.js file to feed to r.js
        with open(os.path.join(self.root_dir, 'staticfiles', 'hqwebapp', 'yaml', 'requirejs.yaml'), 'r') as f:
            config = yaml.load(f)

            bundles = {}
            all_modules = []
            prefix = os.path.join(os.getcwd(), 'corehq')

            # Gather all directories of static files
            for finder in finders.get_finders():
                if isinstance(finder, finders.AppDirectoriesFinder):
                    for path, storage in finder.list(['.*', '*~', '* *', '*.*ss', '*.png']):
                        if not storage.location.startswith(prefix):
                            continue
                        if path.endswith(".js"):
                            directory = re.sub(r'/[^/]*$', '', path)
                            if directory not in bundles:
                                bundles[directory] = []
                            bundles[directory].append(path[:-3])
                            all_modules.append(path[:-3])

            # Go through customized bundles and expand any that include directories
            for module in config['modules']:
                if 'include_directories' in module:
                    if 'include' not in module:
                        module['include'] = []
                    for directory in module.pop('include_directories'):
                        if directory not in bundles:
                            raise Exception("Could not find directory to include: {}".format(directory))
                        module['include'] += bundles[directory]

            # Add a bundle for each directory that doesn't already have a custom module defined
            customized_directories = {re.sub(r'/[^/]*$', '', m['name']): True for m in config['modules']}
            for directory, inclusions in six.iteritems(bundles):
                if directory not in customized_directories and not directory.startswith("app_manager/js/vellum"):
                    # Add this module's config to build config
                    config['modules'].append({
                        'name': os.path.join(directory, 'bundle'),
                        'include': inclusions,
                        'excludeShallow': [name for name in all_modules if name not in inclusions],
                        'exclude': ['hqwebapp/js/common'],
                    })

            # Write a no-op .js file to staticfiles for each bundle, because r.js needs an actual file to overwrite
            for module in config['modules']:
                with open(os.path.join(self.root_dir, 'staticfiles', module['name'] + ".js"), 'w') as fout:
                    fout.write("define([], function() {});")

            # Write final r.js config out as a .js file
            with open(os.path.join(self.root_dir, 'staticfiles', 'build.js'), 'w') as fout:
                fout.write("({});".format(json.dumps(config, indent=4)))

        # Run r.js
        call(["node", "bower_components/r.js/dist/r.js", "-o", "staticfiles/build.js"])

        filename = os.path.join(self.root_dir, 'staticfiles', 'hqwebapp', 'js', 'requirejs_config.js')
        resource_versions["hqwebapp/js/requirejs_config.js"] = self.get_hash(filename)

        # Overwrite each bundle in resource_versions with the sha from the optimized version in staticfiles
        for module in config['modules']:
            filename = os.path.join(self.root_dir, 'staticfiles', module['name'] + ".js")

            # TODO: it'd be a performance improvement to do this after the `open` below
            # and pass in the file contents, since get_hash does another read.
            file_hash = self.get_hash(filename)

            # Overwrite source map reference. Source maps are accessed on the CDN,
            # so they need to have the version hash appended.
            with open(filename, 'r') as fin:
                lines = fin.readlines()
            with open(filename, 'w') as fout:
                for line in lines:
                    if re.search(r'sourceMappingURL=bundle.js.map', line):
                        line = re.sub(r'bundle.js.map', 'bundle.js.map?version=' + file_hash, line)
                    fout.write(line)
            resource_versions[module['name'] + ".js"] = file_hash

        # Write out resource_versions.js for all js files in resource_versions
        # Exclude formdesigner directory, which contains a ton of files, none of which are required by HQ
        filename = os.path.join(self.root_dir, 'staticfiles', 'hqwebapp', 'js', 'resource_versions.js')
        with open(filename, 'w') as fout:
            fout.write("requirejs.config({ paths: %s });" % json.dumps({
                file[:-3]: "{}{}{}{}".format(settings.STATIC_CDN, settings.STATIC_URL, file[:-3],
                                             ".js?version=%s" % version if version else "")
                for file, version in six.iteritems(resource_versions)
                if file.endswith(".js") and not file.startswith("formdesigner")
            }, indent=2))
        resource_versions["hqwebapp/js/resource_versions.js"] = self.get_hash(filename)

        self.overwrite_resources(resource_versions)
<commit_msg>Another comment/rename pass<commit_after>from __future__ import absolute_import, print_function
from __future__ import unicode_literals
import json
import os
import re
import six
import yaml
from django.contrib.staticfiles import finders
from django.conf import settings
from subprocess import call

from corehq.apps.hqwebapp.management.commands.resource_static import Command as ResourceStaticCommand
from io import open

class Command(ResourceStaticCommand):
    help = '''
        Runs RequireJS optimizer to concatenate, minify, and bundle JavaScript files
        and set them up with the CDN.
    '''

    root_dir = settings.FILEPATH

    def handle(self, **options):
        try:
            from resource_versions import resource_versions
        except (ImportError, SyntaxError):
            resource_versions = {}

        # Write build.js file to feed to r.js
        with open(os.path.join(self.root_dir, 'staticfiles', 'hqwebapp', 'yaml', 'requirejs.yaml'), 'r') as f:
            config = yaml.load(f)

            '''
            The default strategy is to make one "bundle" for every corehq directory of js files.
            Begin by gathering all corehq staticfiles directories into a dict like:
                {
                    'app_manager/js' => [
                        'app_manager/js/app_view',
                        'app_manager/js/add_ons',
                        ...
                    ],
                    'app_manager/js/forms' => [
                        'app_manager/js/forms/form_designer',
                        ...
                    ],
                    ...
                }
            '''
            prefix = os.path.join(os.getcwd(), 'corehq')
            corehq_dirs = {}
            all_js_files = []
            for finder in finders.get_finders():
                if isinstance(finder, finders.AppDirectoriesFinder):
                    for path, storage in finder.list(['.*', '*~', '* *', '*.*ss', '*.png']):
                        if not storage.location.startswith(prefix):
                            continue
                        if path.endswith(".js"):
                            directory = re.sub(r'/[^/]*$', '', path)
                            if directory not in corehq_dirs:
                                corehq_dirs[directory] = []
                            corehq_dirs[directory].append(path[:-3])
                            all_js_files.append(path[:-3])

            # Go through customized bundles and expand any that include directories
            for module in config['modules']:
                if 'include_directories' in module:
                    if 'include' not in module:
                        module['include'] = []
                    for directory in module.pop('include_directories'):
                        if directory not in corehq_dirs:
                            raise Exception("Could not find directory to include: {}".format(directory))
                        module['include'] += corehq_dirs[directory]

            # Add a bundle for each directory that doesn't already have a custom module defined
            customized_directories = {re.sub(r'/[^/]*$', '', m['name']): True for m in config['modules']}
            for directory, inclusions in six.iteritems(corehq_dirs):
                if directory not in customized_directories and not directory.startswith("app_manager/js/vellum"):
                    # Add this module's config to build config
                    config['modules'].append({
                        'name': os.path.join(directory, 'bundle'),
                        'include': inclusions,
                        'excludeShallow': [name for name in all_js_files if name not in inclusions],
                        'exclude': ['hqwebapp/js/common'],
                    })

            # Write a no-op .js file to staticfiles for each bundle, because r.js needs an actual file to overwrite
            for module in config['modules']:
                with open(os.path.join(self.root_dir, 'staticfiles', module['name'] + ".js"), 'w') as fout:
                    fout.write("define([], function() {});")

            # Write final r.js config out as a .js file
            with open(os.path.join(self.root_dir, 'staticfiles', 'build.js'), 'w') as fout:
                fout.write("({});".format(json.dumps(config, indent=4)))

        # Run r.js
        call(["node", "bower_components/r.js/dist/r.js", "-o", "staticfiles/build.js"])

        filename = os.path.join(self.root_dir, 'staticfiles', 'hqwebapp', 'js', 'requirejs_config.js')
        resource_versions["hqwebapp/js/requirejs_config.js"] = self.get_hash(filename)

        # Overwrite each bundle in resource_versions with the sha from the optimized version in staticfiles
        for module in config['modules']:
            filename = os.path.join(self.root_dir, 'staticfiles', module['name'] + ".js")

            # TODO: it'd be a performance improvement to do this after the `open` below
            # and pass in the file contents, since get_hash does another read.
            file_hash = self.get_hash(filename)

            # Overwrite source map reference. Source maps are accessed on the CDN,
            # so they need to have the version hash appended.
            with open(filename, 'r') as fin:
                lines = fin.readlines()
            with open(filename, 'w') as fout:
                for line in lines:
                    if re.search(r'sourceMappingURL=bundle.js.map', line):
                        line = re.sub(r'bundle.js.map', 'bundle.js.map?version=' + file_hash, line)
                    fout.write(line)
            resource_versions[module['name'] + ".js"] = file_hash

        # Write out resource_versions.js for all js files in resource_versions
        # Exclude formdesigner directory, which contains a ton of files, none of which are required by HQ
        filename = os.path.join(self.root_dir, 'staticfiles', 'hqwebapp', 'js', 'resource_versions.js')
        with open(filename, 'w') as fout:
            fout.write("requirejs.config({ paths: %s });" % json.dumps({
                file[:-3]: "{}{}{}{}".format(settings.STATIC_CDN, settings.STATIC_URL, file[:-3],
                                             ".js?version=%s" % version if version else "")
                for file, version in six.iteritems(resource_versions)
                if file.endswith(".js") and not file.startswith("formdesigner")
            }, indent=2))
        resource_versions["hqwebapp/js/resource_versions.js"] = self.get_hash(filename)

        self.overwrite_resources(resource_versions)
<|endoftext|>