<commit_before># vim: set fileencoding=utf-8 :
import os.path
import sys
import copy

sys.path.insert(0, os.path.realpath('external'))

from graph_drawing.graph import generate_graph
from graph_drawing.line import generate_line
from graph_drawing.node import Node, Leaf

from unicode_math import PI as u_PI

TYPE_OPERATOR = 1
TYPE_IDENTIFIER = 2
TYPE_INTEGER = 4
TYPE_FLOAT = 8

# Unary
OP_NEG = 1

# Binary
OP_ADD = 2
OP_SUB = 3
OP_MUL = 4
OP_DIV = 5
OP_POW = 6
OP_MOD = 7

# N-ary (functions)
OP_INT = 8
OP_COMMA = 9
OP_SQRT = 10

# Goniometry
OP_SIN = 11
OP_COS = 12
OP_TAN = 13

OP_SOLVE = 14
OP_EQ = 15

OP_POSSIBILITIES = 16
OP_HINT = 17
OP_REWRITE_ALL = 18
OP_REWRITE = 19

# Special identifierd
PI = 'pi'

TYPE_MAP = {
        int: TYPE_INTEGER,
        float: TYPE_FLOAT,
        str: TYPE_IDENTIFIER,
        }

OP_MAP = {
        ',': OP_COMMA,
        '+': OP_ADD,
        '-': OP_SUB,
        '*': OP_MUL,
        '/': OP_DIV,
        '^': OP_POW,
        'sin': OP_SIN,
        'cos': OP_COS,
        'tan': OP_TAN,
        'sqrt': OP_SQRT,
        'int': OP_INT,
        'solve': OP_SOLVE,
        '=': OP_EQ,
        '??': OP_POSSIBILITIES,
        '?': OP_HINT,
        '@@': OP_REWRITE_ALL,
        '@': OP_REWRITE,
        }

TOKEN_MAP = {
        OP_COMMA: 'COMMA',
        OP_ADD: 'PLUS',
        OP_SUB: 'MINUS',
        OP_MUL: 'TIMES',
        OP_DIV: 'DIVIDE',
        OP_POW: 'POW',
        OP_SQRT: 'FUNCTION',
        OP_SIN: 'FUNCTION',
        OP_COS: 'FUNCTION',
        OP_TAN: 'FUNCTION',
        OP_INT: 'FUNCTION',
        OP_SOLVE: 'FUNCTION',
        OP_EQ: 'EQ',
        OP_POSSIBILITIES: 'POSSIBILITIES',
        OP_HINT: 'HINT',
        OP_REWRITE_ALL: 'REWRITE_ALL',
        OP_REWRITE: 'REWRITE',
        }

def to_expression(obj):
    return obj if isinstance(obj, ExpressionBase) else ExpressionLeaf(obj)

class ExpressionBase(object):

    def __init__(self, *args, **kwargs):
        self.negated = 0

    def clone(self):
        return copy.deepcopy(self)

    def __lt__(self, other):
        """
        Comparison between this expression{node,leaf} and another
        expression{node,leaf}. This comparison will return True if this
        instance has less value than the other expression{node,leaf}.
        Otherwise, False is returned.

        The comparison is based on the following conditions:

        1. Both are leafs. String comparison of the value is used.
        2. This is a leaf and other is a node. This leaf has less value, thus
           True is returned.
        3. This is a node and other is a leaf. This leaf has more value, thus
           False is returned.
        4. Both are nodes. Compare the polynome properties of the nodes. True
           is returned if this node's root property is less than other's root
           property, or this node's exponent property is less than other's
           exponent property, or this node's coefficient property is less than
           other's coefficient property. Otherwise, False is returned.
        """
        if self.is_leaf:
            if other.is_leaf:
                # Both are leafs, string compare the value.
                self_value = '-' * (self.negated & 1) + str(self.value)
                other_value = '-' * (other.negated & 1) + str(other.value)

                return self_value < other_value

            # Self is a leaf, thus has less value than an expression node.
            return True

        if other.is_leaf:
            # Self is an expression node, and the other is a leaf. Thus, other
            # is greater than self.
            return False

        # Both are nodes, compare the polynome properties.
        s_coeff, s_root, s_exp = self.extract_polynome_properties()
        o_coeff, o_root, o_exp = other.extract_polynome_properties()

        return s_root < o_root or s_exp < o_exp or s_coeff < o_coeff

    def is_op(self, op):
        return not self.is_leaf and self.op == op

    def is_power(self, exponent=None):
        if self.is_leaf or self.op != OP_POW:
            return False

        return exponent == None or self[1] == exponent

    def is_nary(self):
        return not self.is_leaf and self.op in [OP_ADD, OP_SUB, OP_MUL]

    def is_identifier(self):
        return self.type == TYPE_IDENTIFIER

    def is_int(self):
        return self.type == TYPE_INTEGER

    def is_float(self):
        return self.type == TYPE_FLOAT

    def is_numeric(self):
        return self.type & (TYPE_FLOAT | TYPE_INTEGER)

    def __add__(self, other):
        return ExpressionNode('+', self, to_expression(other))

    def __sub__(self, other):
        return ExpressionNode('-', self, to_expression(other))

    def __mul__(self, other):
        return ExpressionNode('*', self, to_expression(other))

    def __div__(self, other):
        return ExpressionNode('/', self, to_expression(other))

    def __pow__(self, other):
        return ExpressionNode('^', self, to_expression(other))

    def __pos__(self):
        return self.reduce_negation()

    def reduce_negation(self, n=1):
        """Remove n negation flags from the node."""
        assert self.negated

        return self.negate(-n)

    def negate(self, n=1):
        """Negate the node n times."""
        return negate(self, self.negated + n)

class ExpressionNode(Node, ExpressionBase):
    def __init__(self, *args, **kwargs):
        super(ExpressionNode, self).__init__(*args, **kwargs)
        self.type = TYPE_OPERATOR
        self.op = OP_MAP[args[0]]

    def __str__(self):  # pragma: nocover
        return generate_line(self)

    def __eq__(self, other):
        """
        Check strict equivalence.
        """
        return isinstance(other, ExpressionNode) and self.op == other.op \
               and self.negated == other.negated and self.nodes == other.nodes

    def substitute(self, old_child, new_child):
        self.nodes[self.nodes.index(old_child)] = new_child

    def graph(self):  # pragma: nocover
        return generate_graph(self)

    def extract_polynome_properties(self):
        """
        Extract polynome properties into tuple format: (coefficient, root,
        exponent). Thus: c * r ^ e will be extracted into the tuple (c, r, e).

        This function will normalize the expression before extracting the
        properties. Therefore, the expression r ^ e * c results the same tuple
        (c, r, e) as the expression c * r ^ e.

        >>> from src.node import ExpressionNode as N, ExpressionLeaf as L
        >>> c, r, e = L('c'), L('r'), L('e')
        >>> n1 = N('*', c, N('^', r, e))
        >>> n1.extract_polynome()
        (c, r, e)
        >>> n2 = N('*', N('^', r, e), c)
        >>> n2.extract_polynome()
        (c, r, e)
        >>> n3 = N('-', r)
        >>> n3.extract_polynome()
        (1, -r, 1)
        """
        # TODO: change "get_polynome" -> "extract_polynome".
        # TODO: change retval of c * r ^ e to (c, r, e).
        # was: (root, exponent, coefficient, literal_exponent)

        # rule: r ^ e -> (1, r, e)
        if self.is_power():
            return (ExpressionLeaf(1), self[0], self[1])

        # rule: -r -> (1, r, 1)
        # rule: --r -> (1, r, 1)
        # rule: ---r -> (1, r, 1)
        if self.negated:
            return (ExpressionLeaf(1), self, ExpressionLeaf(1))

        if self.op != OP_MUL:
            return

        # rule: 3 * 7 ^ e | 'a' * 'b' ^ e

        # expression: c * r ^ e ; tree:
        #
        #    *
        #   ╭┴───╮
        #   c    ^
        #      ╭─┴╮
        #      r  e
        #
        # rule: c * r ^ e | (r ^ e) * c
        for i, j in ((0, 1), (1, 0)):
            if self[j].is_power():
                return (self[i], self[j][0], self[j][1])

        # Normalize c * r and r * c -> c * r. Otherwise, the tuple will not
        # match if the order of the expression is different. Example:
        #   r ^ e * c == c * r ^ e
        # without normalization, those expressions will not match.
        #
        # rule: c * r | r * c
        if self[0] < self[1]:
            return (self[0], self[1], ExpressionLeaf(1))
        return (self[1], self[0], ExpressionLeaf(1))

    def equals(self, other, ignore_negation=False):
        """
        Perform a non-strict equivalence check between two nodes:
        - If the other node is a leaf, it cannot be equal to this node.
        - If their operators differ, the nodes are not equal.
        - If both nodes are additions or both are multiplications, match each
          node in one scope to one in the other (an injective relationship).
          Any difference in order of the scopes is irrelevant.
        - If both nodes are divisions, the nominator and denominator have to be
          non-strictly equal.
        """
        if not isinstance(other, ExpressionNode) or other.op != self.op:
            return False

        if self.op in (OP_ADD, OP_MUL):
            s0 = Scope(self)
            s1 = set(Scope(other))

            # Scopes should be of equal size
            if len(s0) != len(s1):
                return False

            # Each node in one scope should have an image node in the other
            matched = set()

            for n0 in s0:
                found = False

                for n1 in s1 - matched:
                    if n0.equals(n1):
                        found = True
                        matched.add(n1)
                        break

                if not found:
                    return False
        else:
            # Check if all children are non-strictly equal, preserving order
            for i, child in enumerate(self):
                if not child.equals(other[i]):
                    return False

        if ignore_negation:
            return True

        return self.negated == other.negated

class ExpressionLeaf(Leaf, ExpressionBase):
    def __init__(self, *args, **kwargs):
        super(ExpressionLeaf, self).__init__(*args, **kwargs)
        self.type = TYPE_MAP[type(args[0])]

    def __eq__(self, other):
        """
        Check strict equivalence.
        """
        other_type = type(other)

        if other_type in TYPE_MAP:
            return TYPE_MAP[other_type] == self.type \
                   and self.actual_value() == other

        return self.negated == other.negated and self.type == other.type \
               and self.value == other.value

    def __str__(self):
        val = str(self.value)

        # Replace PI leaf by the Greek character
        if val == PI:
            val = u_PI

        return '-' * self.negated + val

    def __repr__(self):
        return str(self)

    def equals(self, other, ignore_negation=False):
        """
        Check non-strict equivalence.
        Between leaves, this is the same as strict equivalence, except when
        negations must be ignored.
        """
        if ignore_negation:
            other_type = type(other)

            if other_type in (int, float):
                return TYPE_MAP[other_type] == self.type \
                    and self.value == abs(other)
            elif other_type == str:
                return self.type == TYPE_IDENTIFIER and self.value == other

            return self.type == other.type and self.value == other.value
        else:
            return self == other

    def extract_polynome_properties(self):
        """
        An expression leaf will return the polynome tuple (1, r, 1), where r is
        the leaf itself. See also the method extract_polynome_properties in
        ExpressionBase.
        """
        # rule: 1 * r ^ 1 -> (1, r, 1)
        return (ExpressionLeaf(1), self, ExpressionLeaf(1))

    def actual_value(self):
        assert self.is_numeric()

        return (1 - 2 * (self.negated & 1)) * self.value

class Scope(object):

    def __init__(self, node):
        self.node = node
        self.nodes = get_scope(node)

    def __getitem__(self, key):
        return self.nodes[key]

    def __setitem__(self, key, value):
        self.nodes[key] = value

    def __len__(self):
        return len(self.nodes)

    def __iter__(self):
        return iter(self.nodes)

    def __eq__(self, other):
        return isinstance(other, Scope) and self.node == other.node \
               and self.nodes == other.nodes

    def __repr__(self):
        return '<Scope of "%s">' % repr(self.node)

    def remove(self, node, **kwargs):
        if node.is_leaf:
            node_cmp = hash(node)
        else:
            node_cmp = node

        for i, n in enumerate(self.nodes):
            if n.is_leaf:
                n_cmp = hash(n)
            else:
                n_cmp = n

            if n_cmp == node_cmp:
                if 'replacement' in kwargs:
                    self[i] = kwargs['replacement']
                else:
                    del self.nodes[i]

                return

        raise ValueError('Node "%s" is not in the scope of "%s".'
                         % (node, self.node))

    def replace(self, node, replacement):
        self.remove(node, replacement=replacement)

    def as_nary_node(self):
        return nary_node(self.node.value, self.nodes).negate(self.node.negated)

def nary_node(operator, scope):
    """
    Create a binary expression tree for an n-ary operator. Takes the operator
    and a list of expression nodes as arguments.
    """
    if len(scope) == 1:
        return scope[0]

    return ExpressionNode(operator, nary_node(operator, scope[:-1]), scope[-1])

def get_scope(node):
    """
    Find all n nodes within the n-ary scope of an operator node.
    """
    scope = []

    for child in node:
        if child.is_op(node.op):
            scope += get_scope(child)
        else:
            scope.append(child)

    return scope

def negate(node, n=1):
    """Negate the given node n times."""
    assert n >= 0

    new_node = node.clone()
    new_node.negated = n

    return new_node
<commit_msg>Added derivative operator.<commit_after># vim: set fileencoding=utf-8 :
import os.path
import sys
import copy

sys.path.insert(0, os.path.realpath('external'))

from graph_drawing.graph import generate_graph
from graph_drawing.line import generate_line
from graph_drawing.node import Node, Leaf

from unicode_math import PI as u_PI

TYPE_OPERATOR = 1
TYPE_IDENTIFIER = 2
TYPE_INTEGER = 4
TYPE_FLOAT = 8

# Unary
OP_NEG = 1

# Binary
OP_ADD = 2
OP_SUB = 3
OP_MUL = 4
OP_DIV = 5
OP_POW = 6
OP_MOD = 7

# N-ary (functions)
OP_INT = 8
OP_COMMA = 9
OP_SQRT = 10
OP_DERIV = 11

# Goniometry
OP_SIN = 12
OP_COS = 13
OP_TAN = 14

OP_SOLVE = 15
OP_EQ = 16

OP_POSSIBILITIES = 17
OP_HINT = 18
OP_REWRITE_ALL = 19
OP_REWRITE = 20

# Special identifierd
PI = 'pi'

TYPE_MAP = {
        int: TYPE_INTEGER,
        float: TYPE_FLOAT,
        str: TYPE_IDENTIFIER,
        }

OP_MAP = {
        ',': OP_COMMA,
        '+': OP_ADD,
        '-': OP_SUB,
        '*': OP_MUL,
        '/': OP_DIV,
        '^': OP_POW,
        'sin': OP_SIN,
        'cos': OP_COS,
        'tan': OP_TAN,
        'sqrt': OP_SQRT,
        'int': OP_INT,
        'der': OP_DERIV,
        'solve': OP_SOLVE,
        '=': OP_EQ,
        '??': OP_POSSIBILITIES,
        '?': OP_HINT,
        '@@': OP_REWRITE_ALL,
        '@': OP_REWRITE,
        }

TOKEN_MAP = {
        OP_COMMA: 'COMMA',
        OP_ADD: 'PLUS',
        OP_SUB: 'MINUS',
        OP_MUL: 'TIMES',
        OP_DIV: 'DIVIDE',
        OP_POW: 'POW',
        OP_SQRT: 'FUNCTION',
        OP_SIN: 'FUNCTION',
        OP_COS: 'FUNCTION',
        OP_TAN: 'FUNCTION',
        OP_INT: 'FUNCTION',
        OP_DERIV: 'FUNCTION',
        OP_SOLVE: 'FUNCTION',
        OP_EQ: 'EQ',
        OP_POSSIBILITIES: 'POSSIBILITIES',
        OP_HINT: 'HINT',
        OP_REWRITE_ALL: 'REWRITE_ALL',
        OP_REWRITE: 'REWRITE',
        }

def to_expression(obj):
    return obj if isinstance(obj, ExpressionBase) else ExpressionLeaf(obj)

class ExpressionBase(object):

    def __init__(self, *args, **kwargs):
        self.negated = 0

    def clone(self):
        return copy.deepcopy(self)

    def __lt__(self, other):
        """
        Comparison between this expression{node,leaf} and another
        expression{node,leaf}. This comparison will return True if this
        instance has less value than the other expression{node,leaf}.
        Otherwise, False is returned.

        The comparison is based on the following conditions:

        1. Both are leafs. String comparison of the value is used.
        2. This is a leaf and other is a node. This leaf has less value, thus
           True is returned.
        3. This is a node and other is a leaf. This leaf has more value, thus
           False is returned.
        4. Both are nodes. Compare the polynome properties of the nodes. True
           is returned if this node's root property is less than other's root
           property, or this node's exponent property is less than other's
           exponent property, or this node's coefficient property is less than
           other's coefficient property. Otherwise, False is returned.
        """
        if self.is_leaf:
            if other.is_leaf:
                # Both are leafs, string compare the value.
                self_value = '-' * (self.negated & 1) + str(self.value)
                other_value = '-' * (other.negated & 1) + str(other.value)

                return self_value < other_value

            # Self is a leaf, thus has less value than an expression node.
            return True

        if other.is_leaf:
            # Self is an expression node, and the other is a leaf. Thus, other
            # is greater than self.
            return False

        # Both are nodes, compare the polynome properties.
        s_coeff, s_root, s_exp = self.extract_polynome_properties()
        o_coeff, o_root, o_exp = other.extract_polynome_properties()

        return s_root < o_root or s_exp < o_exp or s_coeff < o_coeff

    def is_op(self, op):
        return not self.is_leaf and self.op == op

    def is_power(self, exponent=None):
        if self.is_leaf or self.op != OP_POW:
            return False

        return exponent == None or self[1] == exponent

    def is_nary(self):
        return not self.is_leaf and self.op in [OP_ADD, OP_SUB, OP_MUL]

    def is_identifier(self, identifier=None):
        return self.type == TYPE_IDENTIFIER \
               and (identifier == None or self.value == identifier)

    def is_variable(self):
        return self.type == TYPE_IDENTIFIER and self.value != PI

    def is_int(self):
        return self.type == TYPE_INTEGER

    def is_float(self):
        return self.type == TYPE_FLOAT

    def is_numeric(self):
        return self.type & (TYPE_FLOAT | TYPE_INTEGER)

    def __add__(self, other):
        return ExpressionNode('+', self, to_expression(other))

    def __sub__(self, other):
        return ExpressionNode('-', self, to_expression(other))

    def __mul__(self, other):
        return ExpressionNode('*', self, to_expression(other))

    def __div__(self, other):
        return ExpressionNode('/', self, to_expression(other))

    def __pow__(self, other):
        return ExpressionNode('^', self, to_expression(other))

    def __pos__(self):
        return self.reduce_negation()

    def reduce_negation(self, n=1):
        """Remove n negation flags from the node."""
        assert self.negated

        return self.negate(-n)

    def negate(self, n=1):
        """Negate the node n times."""
        return negate(self, self.negated + n)

class ExpressionNode(Node, ExpressionBase):
    def __init__(self, *args, **kwargs):
        super(ExpressionNode, self).__init__(*args, **kwargs)
        self.type = TYPE_OPERATOR
        self.op = OP_MAP[args[0]]

    def __str__(self):  # pragma: nocover
        return generate_line(self)

    def __eq__(self, other):
        """
        Check strict equivalence.
        """
        return isinstance(other, ExpressionNode) and self.op == other.op \
               and self.negated == other.negated and self.nodes == other.nodes

    def substitute(self, old_child, new_child):
        self.nodes[self.nodes.index(old_child)] = new_child

    def graph(self):  # pragma: nocover
        return generate_graph(self)

    def extract_polynome_properties(self):
        """
        Extract polynome properties into tuple format: (coefficient, root,
        exponent). Thus: c * r ^ e will be extracted into the tuple (c, r, e).

        This function will normalize the expression before extracting the
        properties. Therefore, the expression r ^ e * c results the same tuple
        (c, r, e) as the expression c * r ^ e.

        >>> from src.node import ExpressionNode as N, ExpressionLeaf as L
        >>> c, r, e = L('c'), L('r'), L('e')
        >>> n1 = N('*', c, N('^', r, e))
        >>> n1.extract_polynome()
        (c, r, e)
        >>> n2 = N('*', N('^', r, e), c)
        >>> n2.extract_polynome()
        (c, r, e)
        >>> n3 = N('-', r)
        >>> n3.extract_polynome()
        (1, -r, 1)
        """
        # TODO: change "get_polynome" -> "extract_polynome".
        # TODO: change retval of c * r ^ e to (c, r, e).
        # was: (root, exponent, coefficient, literal_exponent)

        # rule: r ^ e -> (1, r, e)
        if self.is_power():
            return (ExpressionLeaf(1), self[0], self[1])

        # rule: -r -> (1, r, 1)
        # rule: --r -> (1, r, 1)
        # rule: ---r -> (1, r, 1)
        if self.negated:
            return (ExpressionLeaf(1), self, ExpressionLeaf(1))

        if self.op != OP_MUL:
            return

        # rule: 3 * 7 ^ e | 'a' * 'b' ^ e

        # expression: c * r ^ e ; tree:
        #
        #    *
        #   ╭┴───╮
        #   c    ^
        #      ╭─┴╮
        #      r  e
        #
        # rule: c * r ^ e | (r ^ e) * c
        for i, j in ((0, 1), (1, 0)):
            if self[j].is_power():
                return (self[i], self[j][0], self[j][1])

        # Normalize c * r and r * c -> c * r. Otherwise, the tuple will not
        # match if the order of the expression is different. Example:
        #   r ^ e * c == c * r ^ e
        # without normalization, those expressions will not match.
        #
        # rule: c * r | r * c
        if self[0] < self[1]:
            return (self[0], self[1], ExpressionLeaf(1))
        return (self[1], self[0], ExpressionLeaf(1))

    def equals(self, other, ignore_negation=False):
        """
        Perform a non-strict equivalence check between two nodes:
        - If the other node is a leaf, it cannot be equal to this node.
        - If their operators differ, the nodes are not equal.
        - If both nodes are additions or both are multiplications, match each
          node in one scope to one in the other (an injective relationship).
          Any difference in order of the scopes is irrelevant.
        - If both nodes are divisions, the nominator and denominator have to be
          non-strictly equal.
        """
        if not isinstance(other, ExpressionNode) or other.op != self.op:
            return False

        if self.op in (OP_ADD, OP_MUL):
            s0 = Scope(self)
            s1 = set(Scope(other))

            # Scopes should be of equal size
            if len(s0) != len(s1):
                return False

            # Each node in one scope should have an image node in the other
            matched = set()

            for n0 in s0:
                found = False

                for n1 in s1 - matched:
                    if n0.equals(n1):
                        found = True
                        matched.add(n1)
                        break

                if not found:
                    return False
        else:
            # Check if all children are non-strictly equal, preserving order
            for i, child in enumerate(self):
                if not child.equals(other[i]):
                    return False

        if ignore_negation:
            return True

        return self.negated == other.negated

class ExpressionLeaf(Leaf, ExpressionBase):
    def __init__(self, *args, **kwargs):
        super(ExpressionLeaf, self).__init__(*args, **kwargs)
        self.type = TYPE_MAP[type(args[0])]

    def __eq__(self, other):
        """
        Check strict equivalence.
        """
        other_type = type(other)

        if other_type in TYPE_MAP:
            return TYPE_MAP[other_type] == self.type \
                   and self.actual_value() == other

        return self.negated == other.negated and self.type == other.type \
               and self.value == other.value

    def __str__(self):
        val = str(self.value)

        # Replace PI leaf by the Greek character
        if val == PI:
            val = u_PI

        return '-' * self.negated + val

    def __repr__(self):
        return str(self)

    def equals(self, other, ignore_negation=False):
        """
        Check non-strict equivalence.
        Between leaves, this is the same as strict equivalence, except when
        negations must be ignored.
        """
        if ignore_negation:
            other_type = type(other)

            if other_type in (int, float):
                return TYPE_MAP[other_type] == self.type \
                    and self.value == abs(other)
            elif other_type == str:
                return self.type == TYPE_IDENTIFIER and self.value == other

            return self.type == other.type and self.value == other.value
        else:
            return self == other

    def extract_polynome_properties(self):
        """
        An expression leaf will return the polynome tuple (1, r, 1), where r is
        the leaf itself. See also the method extract_polynome_properties in
        ExpressionBase.
        """
        # rule: 1 * r ^ 1 -> (1, r, 1)
        return (ExpressionLeaf(1), self, ExpressionLeaf(1))

    def actual_value(self):
        assert self.is_numeric()

        return (1 - 2 * (self.negated & 1)) * self.value

class Scope(object):

    def __init__(self, node):
        self.node = node
        self.nodes = get_scope(node)

    def __getitem__(self, key):
        return self.nodes[key]

    def __setitem__(self, key, value):
        self.nodes[key] = value

    def __len__(self):
        return len(self.nodes)

    def __iter__(self):
        return iter(self.nodes)

    def __eq__(self, other):
        return isinstance(other, Scope) and self.node == other.node \
               and self.nodes == other.nodes

    def __repr__(self):
        return '<Scope of "%s">' % repr(self.node)

    def remove(self, node, **kwargs):
        if node.is_leaf:
            node_cmp = hash(node)
        else:
            node_cmp = node

        for i, n in enumerate(self.nodes):
            if n.is_leaf:
                n_cmp = hash(n)
            else:
                n_cmp = n

            if n_cmp == node_cmp:
                if 'replacement' in kwargs:
                    self[i] = kwargs['replacement']
                else:
                    del self.nodes[i]

                return

        raise ValueError('Node "%s" is not in the scope of "%s".'
                         % (node, self.node))

    def replace(self, node, replacement):
        self.remove(node, replacement=replacement)

    def as_nary_node(self):
        return nary_node(self.node.value, self.nodes).negate(self.node.negated)

def nary_node(operator, scope):
    """
    Create a binary expression tree for an n-ary operator. Takes the operator
    and a list of expression nodes as arguments.
    """
    if len(scope) == 1:
        return scope[0]

    return ExpressionNode(operator, nary_node(operator, scope[:-1]), scope[-1])

def get_scope(node):
    """
    Find all n nodes within the n-ary scope of an operator node.
    """
    scope = []

    for child in node:
        if child.is_op(node.op):
            scope += get_scope(child)
        else:
            scope.append(child)

    return scope

def negate(node, n=1):
    """Negate the given node n times."""
    assert n >= 0

    new_node = node.clone()
    new_node.negated = n

    return new_node
<|endoftext|>


<commit_before>#!/usr/bin/python
import logging

#----------------------------------------------------------------------
# Copyright (c) 2010 Raytheon BBN Technologies
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and/or hardware specification (the "Work") to
# deal in the Work without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Work, and to permit persons to whom the Work
# is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Work.
#
# THE WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE WORK OR THE USE OR OTHER DEALINGS
# IN THE WORK.
#----------------------------------------------------------------------

""" The OMNI client
    This client is a GENI API client that is capable to connecting
    to the supported control frameworks for slice creation and deletion.
    It is also able to parse and create standard RSPECs of all supported
    control frameworks.

    The currently supported control frameworks are SFA and GCF.
"""

import optparse
import sys
import os
import pprint
import json
import dateutil.parser
from copy import copy
from geni.omni.xmlrpc.client import make_client
from geni.omni.omnispec.translation import rspec_to_omnispec, omnispec_to_rspec
from geni.omni.omnispec.omnispec import OmniSpec

class CallHandler(object):
    def __init__(self, framework, frame_config, omni_config):
        self.framework = framework
        self.frame_config = frame_config
        self.omni_config = omni_config
        frame_config['cert'] = os.path.expanduser(frame_config['cert'])
        frame_config['key'] = os.path.expanduser(frame_config['key'])

    def _handle(self, args):
        if len(args) == 0:
            sys.exit('Insufficient number of arguments')

        call = args[0].lower()
        if call.startswith('_'):
            return
        getattr(self,call)(args[1:])

    def _getclients(self):
        clients = []
        for (urn, url) in self._listaggregates([]).items():
            client = make_client(url, self.frame_config['key'], self.frame_config['cert'])
            client.urn = urn
            client.url = url
            clients.append(client)
        return clients

    def _listaggregates(self, args):
        aggs = self.framework.list_aggregates()
        return aggs

    def listresources(self, args):
        rspecs = {}
        options = {}

        # Get the credential for this query
        if len(args) == 0:
            cred = self.framework.get_user_cred()
        else:
            name = args[0]
            urn = self.framework.slice_name_to_urn(name)
            cred = self.framework.get_slice_cred(urn)
            options['geni_slice_urn'] = urn

        # Connect to each available GENI AM to list their resources
        for client in self._getclients():
            try:
                rspec = client.ListResources([cred], options)
                rspecs[(client.urn, client.url)] = rspec
            except Exception, exc:
                print "Failed to get resources from %s (%s)" % (client.urn, client.url)
                logger = logging.getLogger('omni')
                logger.debug(str(exc))

        # Convert the rspecs to omnispecs
        omnispecs = {}
        for ((urn,url), rspec) in rspecs.items():
            omnispecs[url] = rspec_to_omnispec(urn,rspec)

        jspecs = json.dumps(omnispecs, indent=4)
        print jspecs
        return omnispecs

    def createsliver(self, args):
        name = args[0]
        urn = self.framework.slice_name_to_urn(name)
        slice_cred = self.framework.get_slice_cred(urn)

        # Load up the user's edited omnispec
        specfile = args[1]
        jspecs = json.loads(file(specfile,'r').read())
        omnispecs = {}
        for url, spec_dict in jspecs.items():
            omnispecs[url] = OmniSpec('','',dictionary=spec_dict)

        # Copy the user config and read the keys from the files into the structure
        slice_users = copy(self.omni_config['slice_users'])
        for user in slice_users:
            newkeys = []
            for f in user['keys']:
                newkeys.append(file(os.path.expanduser(f)).read())
            user['keys'] = newkeys

        # Anything we need to allocate?
        for (url, ospec) in omnispecs.items():
            allocate = False
            for (_, r) in ospec.get_resources().items():
                if r.get_allocate():
                    allocate = True
                    break

            # Okay, send a message to the AM this resource came from
            if allocate:
                try:
                    client = make_client(url, self.frame_config['key'], self.frame_config['cert'])
                    rspec = omnispec_to_rspec(ospec, True)
                    client.CreateSliver(urn, [slice_cred], rspec, slice_users)
                except Exception, exc:
                    print "Unable to allocate from: %s" % (url)
                    logger = logging.getLogger('omni')
                    logger.debug(str(exc))

    def deletesliver(self, args):
        name = args[0]
        urn = self.framework.slice_name_to_urn(name)
        slice_cred = self.framework.get_slice_cred(urn)
        # Connect to each available GENI AM
        for client in self._getclients():
            try:
                client.DeleteSliver(urn, [slice_cred])
            except Exception, exc:
                print "Failed to delete sliver on %s (%s)" %(client.urn, client.url)
                logger = logging.getLogger('omni')
                logger.debug(str(exc))

    def renewsliver(self, args):
        name = args[0]
        urn = self.framework.slice_name_to_urn(name)
        slice_cred = self.framework.get_slice_cred(urn)
        time = dateutil.parser.parse(args[1])
        print time

        for client in self._getclients():
            try:
                client.RenewSliver(urn, [slice_cred], time.isoformat())
            except:
                print "Failed to renew sliver on %s" % client.urn

    def sliverstatus(self, args):
        name = args[0]
        urn = self.framework.slice_name_to_urn(name)
        slice_cred = self.framework.get_slice_cred(urn)
        for client in self._getclients():
            try:
                status = client.SliverStatus(urn, [slice_cred])
                print "%s (%s):" % (client.urn, client.url)
                pprint.pprint(status)
            except Exception, exc:
                print "Failed to retrieve status for: %s" % client.urn
                logger = logging.getLogger('omni')
                logger.debug(str(exc))

    def shutdown(self, args):
        name = args[0]
        urn = self.framework.slice_name_to_urn(name)
        slice_cred = self.framework.get_slice_cred(urn)
        for client in self._getclients():
            try:
                client.Shutdown(urn, [slice_cred])
            except:
                print "Failed to shutdown: %s at %s" % (client.urn, client.url)

    def getversion(self, args):
        for client in self._getclients():
            try:
                print "%s (%s) %s" % (client.urn, client.url, client.GetVersion())
            except:
                print "Failed to get version information for %s at (%s)" % (client.urn, client.url)

    def createslice(self, args):
        name = args[0]
        urn = self.framework.slice_name_to_urn(name)
        self.framework.create_slice(urn)

    def deleteslice(self, args):
        name = args[0]
        urn = self.framework.slice_name_to_urn(name)
        self.framework.delete_slice(urn)

    def getslicecred(self, args):
        name = args[0]
        urn = self.framework.slice_name_to_urn(name)
        cred = self.framework.get_slice_cred(urn)
        print cred

def parse_args(argv):
    parser = optparse.OptionParser()
    parser.add_option("-c", "--configfile", default="~/.omni/omni_config",
                      help="config file name", metavar="FILE")
    parser.add_option("-f", "--framework", default="",
                      help="control framework to use for creation/deletion of slices")
    parser.add_option("--debug", action="store_true", default=False,
                       help="enable debugging output")
    return parser.parse_args()

def configure_logging(opts):
    level = logging.INFO
    logging.basicConfig(level=level)
    if opts.debug:
        level = logging.DEBUG
    logger = logging.getLogger("omni")
    logger.setLevel(level)

def main(argv=None):
    if argv is None:
        argv = sys.argv
    opts, args = parse_args(argv)
    configure_logging(opts)

    logger = logging.getLogger('omni')
    # Load up the JSON formatted config file
    filename = os.path.expanduser(opts.configfile)
    logger.debug("Loading config file %s", filename)
    config = json.loads(file(filename, 'r').read())

    if not opts.framework:
        opts.framework = config['omni']['default_cf']

    # Dynamically load the selected control framework
    cf = opts.framework.lower()
    framework_mod = __import__('geni.omni.frameworks.framework_%s' % cf, fromlist=['geni.omni.frameworks'])
    framework = framework_mod.Framework(config[cf])

    # Process the user's call
    handler = CallHandler(framework, config[cf], config['omni'])
    handler._handle(args)

if __name__ == "__main__":
    sys.exit(main())
<commit_msg>Added compression on ListResources<commit_after>#!/usr/bin/python
import logging

#----------------------------------------------------------------------
# Copyright (c) 2010 Raytheon BBN Technologies
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and/or hardware specification (the "Work") to
# deal in the Work without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Work, and to permit persons to whom the Work
# is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Work.
#
# THE WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE WORK OR THE USE OR OTHER DEALINGS
# IN THE WORK.
#----------------------------------------------------------------------

""" The OMNI client
    This client is a GENI API client that is capable to connecting
    to the supported control frameworks for slice creation and deletion.
    It is also able to parse and create standard RSPECs of all supported
    control frameworks.

    The currently supported control frameworks are SFA and GCF.
"""

import optparse
import sys
import os
import pprint
import json
import dateutil.parser
from copy import copy
from geni.omni.xmlrpc.client import make_client
from geni.omni.omnispec.translation import rspec_to_omnispec, omnispec_to_rspec
from geni.omni.omnispec.omnispec import OmniSpec

class CallHandler(object):
    def __init__(self, framework, frame_config, omni_config):
        self.framework = framework
        self.frame_config = frame_config
        self.omni_config = omni_config
        frame_config['cert'] = os.path.expanduser(frame_config['cert'])
        frame_config['key'] = os.path.expanduser(frame_config['key'])

    def _handle(self, args):
        if len(args) == 0:
            sys.exit('Insufficient number of arguments')

        call = args[0].lower()
        if call.startswith('_'):
            return
        getattr(self,call)(args[1:])

    def _getclients(self):
        clients = []
        for (urn, url) in self._listaggregates([]).items():
            client = make_client(url, self.frame_config['key'], self.frame_config['cert'])
            client.urn = urn
            client.url = url
            clients.append(client)
        return clients

    def _listaggregates(self, args):
        aggs = self.framework.list_aggregates()
        return aggs

    def listresources(self, args):
        rspecs = {}
        options = {}

        options['geni_compressed'] = True;
        # Get the credential for this query
        if len(args) == 0:
            cred = self.framework.get_user_cred()
        else:
            name = args[0]
            urn = self.framework.slice_name_to_urn(name)
            cred = self.framework.get_slice_cred(urn)
            options['geni_slice_urn'] = urn

        # Connect to each available GENI AM to list their resources
        for client in self._getclients():
            try:
                rspec = client.ListResources([cred], options)
                rspecs[(client.urn, client.url)] = rspec
            except Exception, exc:
                print "Failed to get resources from %s (%s)" % (client.urn, client.url)
                logger = logging.getLogger('omni')
                logger.debug(str(exc))

        # Convert the rspecs to omnispecs
        omnispecs = {}
        for ((urn,url), rspec) in rspecs.items():
            omnispecs[url] = rspec_to_omnispec(urn,rspec)

        jspecs = json.dumps(omnispecs, indent=4)
        print jspecs
        return omnispecs

    def createsliver(self, args):
        name = args[0]
        urn = self.framework.slice_name_to_urn(name)
        slice_cred = self.framework.get_slice_cred(urn)

        # Load up the user's edited omnispec
        specfile = args[1]
        jspecs = json.loads(file(specfile,'r').read())
        omnispecs = {}
        for url, spec_dict in jspecs.items():
            omnispecs[url] = OmniSpec('','',dictionary=spec_dict)

        # Copy the user config and read the keys from the files into the structure
        slice_users = copy(self.omni_config['slice_users'])
        for user in slice_users:
            newkeys = []
            for f in user['keys']:
                newkeys.append(file(os.path.expanduser(f)).read())
            user['keys'] = newkeys

        # Anything we need to allocate?
        for (url, ospec) in omnispecs.items():
            allocate = False
            for (_, r) in ospec.get_resources().items():
                if r.get_allocate():
                    allocate = True
                    break

            # Okay, send a message to the AM this resource came from
            if allocate:
                try:
                    client = make_client(url, self.frame_config['key'], self.frame_config['cert'])
                    rspec = omnispec_to_rspec(ospec, True)
                    client.CreateSliver(urn, [slice_cred], rspec, slice_users)
                except Exception, exc:
                    print "Unable to allocate from: %s" % (url)
                    logger = logging.getLogger('omni')
                    logger.debug(str(exc))

    def deletesliver(self, args):
        name = args[0]
        urn = self.framework.slice_name_to_urn(name)
        slice_cred = self.framework.get_slice_cred(urn)
        # Connect to each available GENI AM
        for client in self._getclients():
            try:
                client.DeleteSliver(urn, [slice_cred])
            except Exception, exc:
                print "Failed to delete sliver on %s (%s)" %(client.urn, client.url)
                logger = logging.getLogger('omni')
                logger.debug(str(exc))

    def renewsliver(self, args):
        name = args[0]
        urn = self.framework.slice_name_to_urn(name)
        slice_cred = self.framework.get_slice_cred(urn)
        time = dateutil.parser.parse(args[1])
        print time

        for client in self._getclients():
            try:
                client.RenewSliver(urn, [slice_cred], time.isoformat())
            except:
                print "Failed to renew sliver on %s" % client.urn

    def sliverstatus(self, args):
        name = args[0]
        urn = self.framework.slice_name_to_urn(name)
        slice_cred = self.framework.get_slice_cred(urn)
        for client in self._getclients():
            try:
                status = client.SliverStatus(urn, [slice_cred])
                print "%s (%s):" % (client.urn, client.url)
                pprint.pprint(status)
            except Exception, exc:
                print "Failed to retrieve status for: %s" % client.urn
                logger = logging.getLogger('omni')
                logger.debug(str(exc))

    def shutdown(self, args):
        name = args[0]
        urn = self.framework.slice_name_to_urn(name)
        slice_cred = self.framework.get_slice_cred(urn)
        for client in self._getclients():
            try:
                client.Shutdown(urn, [slice_cred])
            except:
                print "Failed to shutdown: %s at %s" % (client.urn, client.url)

    def getversion(self, args):
        for client in self._getclients():
            try:
                print "%s (%s) %s" % (client.urn, client.url, client.GetVersion())
            except:
                print "Failed to get version information for %s at (%s)" % (client.urn, client.url)

    def createslice(self, args):
        name = args[0]
        urn = self.framework.slice_name_to_urn(name)
        self.framework.create_slice(urn)

    def deleteslice(self, args):
        name = args[0]
        urn = self.framework.slice_name_to_urn(name)
        self.framework.delete_slice(urn)

    def getslicecred(self, args):
        name = args[0]
        urn = self.framework.slice_name_to_urn(name)
        cred = self.framework.get_slice_cred(urn)
        print cred

def parse_args(argv):
    parser = optparse.OptionParser()
    parser.add_option("-c", "--configfile", default="~/.omni/omni_config",
                      help="config file name", metavar="FILE")
    parser.add_option("-f", "--framework", default="",
                      help="control framework to use for creation/deletion of slices")
    parser.add_option("--debug", action="store_true", default=False,
                       help="enable debugging output")
    return parser.parse_args()

def configure_logging(opts):
    level = logging.INFO
    logging.basicConfig(level=level)
    if opts.debug:
        level = logging.DEBUG
    logger = logging.getLogger("omni")
    logger.setLevel(level)

def main(argv=None):
    if argv is None:
        argv = sys.argv
    opts, args = parse_args(argv)
    configure_logging(opts)

    logger = logging.getLogger('omni')
    # Load up the JSON formatted config file
    filename = os.path.expanduser(opts.configfile)
    logger.debug("Loading config file %s", filename)
    config = json.loads(file(filename, 'r').read())

    if not opts.framework:
        opts.framework = config['omni']['default_cf']

    # Dynamically load the selected control framework
    cf = opts.framework.lower()
    framework_mod = __import__('geni.omni.frameworks.framework_%s' % cf, fromlist=['geni.omni.frameworks'])
    framework = framework_mod.Framework(config[cf])

    # Process the user's call
    handler = CallHandler(framework, config[cf], config['omni'])
    handler._handle(args)

if __name__ == "__main__":
    sys.exit(main())
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

from gevent import monkey
monkey.patch_all()

import json
import logging
import Queue
import select
import socket
import ssl
import time
from threading import Thread

import beanstalkc

import apns
import config

class Pipe(object):
    def __init__(
            self, beanstalkd_host, beanstalkd_port, tube,
            gateway_host, gateway_port, key_file, cert_file, master_worker):
        self.beanstalkd_host = beanstalkd_host
        self.beanstalkd_port = beanstalkd_port
        self.tube = tube
        self.gateway_host = gateway_host
        self.gateway_port = gateway_port
        self.key_file = key_file
        self.cert_file = cert_file
        self.master_worker = master_worker

        self.push_id = 0
        self.last_push_time = 0
        self.pushed_buffer = Queue.Queue(maxsize=1000)
        self.beanstalk = None
        self.gateway_connection = None
        self.gateway_invalid = False

    def init_beanstalk(self):
        # init beanstalk
        while True:
            try:
                if self.beanstalk:
                    self.beanstalk.close()

                self.beanstalk = beanstalkc.Connection(
                    self.beanstalkd_host, self.beanstalkd_port)
                logging.debug(
                    'Connect to %s:%s success' % (
                        self.beanstalkd_host, self.beanstalkd_port))
                self.beanstalk.watch(self.tube)
                for tube in self.beanstalk.watching():
                    if tube != self.tube:
                        self.beanstalk.ignore(tube)
                self.beanstalk.use(tube)
                return
            except beanstalkc.SocketError:
                logging.debug(
                    'Connect to %s:%s failed' % (
                        self.beanstalkd_host, self.beanstalkd_port))
                time.sleep(2)
                continue

    def init_gateway(self):
        while True:
            try:
                if not self.gateway_connection:
                    self.gateway_connection = apns.GatewayConnection(
                        host=self.gateway_host,
                        port=self.gateway_port,
                        cert_file=self.cert_file,
                        key_file=self.key_file,
                    )
                else:
                    self.gateway_connection.reconnect()
                return
            except ssl.SSLError as e:
                if e.errno == ssl.SSL_ERROR_SSL:
                    self.gateway_invalid = True
                    time.sleep(3600)
                    logging.debug('Invalid key')
            except (socket.error, IOError) as e:
                logging.debug('Gateway connect error %s' % e)
            time.sleep(2)
            continue

    def process_gateway_input(self):
        buff = self.gateway_connection.read(apns.ERROR_RESPONSE_LENGTH)
        if len(buff) == apns.ERROR_RESPONSE_LENGTH:
            command, status, error_identifier = \
                apns.unpack(apns.ERROR_RESPONSE_FORMAT, buff)

            if 8 == command:
                found = False
                while not self.pushed_buffer.empty():
                    identifier, job = self.pushed_buffer.get()
                    if found:
                        logging.debug('Reput failed job %s' % identifier)
                        self.beanstalk.put(json.dumps(job))
                    elif identifier == error_identifier:
                        logging.debug('Found error identifier %s' % identifier)
                        found = True
        elif len(buff) == 0:
            logging.debug('Close by server')
        else:
            logging.debug('Unexcepted read buf size %s' % len(buf))

    def push_job(self):
        job = self.beanstalk.reserve(timeout=10)
        if not job:
            logging.debug('No job found')
            return
        logging.debug('Reserved job: %s %s' % (job.jid, job.body))
        try:
            job_body = json.loads(job.body)
        except ValueError:
            logging.debug(
                'Failed to loads job body: %s %s' % (job.jid, job.body))
            job.bury()

        # push job
        self.push_id += 1
        try:
            logging.debug('Send notification: %s %s' % (job.jid, job.body))
            self.gateway_connection.send_notification(
                job_body['device_token'],
                apns.Payload(**job_body['payload']),
                self.push_id)
        except apns.InvalidTokenError:
            pass
        except Exception as e:
            logging.debug('Send notification error: %s' % e)
            job.release()
            raise
        if self.pushed_buffer.full():
            self.pushed_buffer.get()
        logging.debug('Enqueue pushed buffer: %s %s' % (job.jid, job.body))
        self.pushed_buffer.put((self.push_id, job_body))
        self.last_push_time = time.time()

        logging.debug('Delete job: %s %s' % (job.jid, job.body))
        job.delete()

    def reserve_and_push(self):
        while True:
            rlist, wlist, _ = select.select(
                [self.gateway_connection.connection()],
                [self.gateway_connection.connection()],
                [],
                10)
            if rlist:
                logging.debug('Start Reading from gateway')
                self.process_gateway_input()
                self.gateway_connection.reconnect()
            elif wlist:
                self.push_job()

            if self.ok_to_stop():
                break

    def need_to_start(self):
        if self.master_worker:
            return True
        tube_stat = self.beanstalk.stats_tube(self.tube)
        if tube_stat['current-jobs-ready'] > 100:
            return True
        return False

    def ok_to_stop(self):
        if self.master_worker:
            return False
        if time.time() - self.last_push_time > 10:
            return True
        return False

    def run(self):
        self.init_beanstalk()

        while True:
            try:
                if not self.need_to_start():
                    logging.debug('Sleepy')
                    time.sleep(30)
                    continue
                logging.debug('Start to reserve and push')
                self.init_gateway()
                self.reserve_and_push()
                self.gateway_connection.disconnect()
                logging.debug('Stop to reserve and push')
            except beanstalkc.SocketError:
                self.init_beanstalk()
            except (ssl.SSLError, socket.error, IOError) as e:
                pass

if __name__ == '__main__':
    logging.basicConfig(
        format=config.LOGGING_FORMAT, level=config.LOGGING_LEVEL)
    for app_name, pem_file, pip_count in config.APPS.items():
        for i in range(pip_count):
            pipe = Pipe(
                config.BEANSTALKD_HOST, config.BEANSTALKD_PORT,
                config.PUSH_TUBE % app_name, config.APNS_HOST,
                config.APNS_PORT, pem_file[1], pem_file[0], i == 0)
            t = Thread(target=pipe.run, name='%s.%d' % (app_name, i))
            t.start()
<commit_msg>update push.main<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

from gevent import monkey
monkey.patch_all()

import json
import logging
import Queue
import select
import socket
import ssl
import time
from threading import Thread

import beanstalkc

import apns
import config

class Pipe(object):
    def __init__(
            self, beanstalkd_host, beanstalkd_port, tube,
            gateway_host, gateway_port, key_file, cert_file, master_worker):
        self.beanstalkd_host = beanstalkd_host
        self.beanstalkd_port = beanstalkd_port
        self.tube = tube
        self.gateway_host = gateway_host
        self.gateway_port = gateway_port
        self.key_file = key_file
        self.cert_file = cert_file
        self.master_worker = master_worker

        self.push_id = 0
        self.last_push_time = 0
        self.pushed_buffer = Queue.Queue(maxsize=1000)
        self.beanstalk = None
        self.gateway_connection = None
        self.gateway_invalid = False

    def init_beanstalk(self):
        # init beanstalk
        while True:
            try:
                if self.beanstalk:
                    self.beanstalk.close()

                self.beanstalk = beanstalkc.Connection(
                    self.beanstalkd_host, self.beanstalkd_port)
                logging.debug(
                    'Connect to %s:%s success' % (
                        self.beanstalkd_host, self.beanstalkd_port))
                self.beanstalk.watch(self.tube)
                for tube in self.beanstalk.watching():
                    if tube != self.tube:
                        self.beanstalk.ignore(tube)
                self.beanstalk.use(tube)
                return
            except beanstalkc.SocketError:
                logging.debug(
                    'Connect to %s:%s failed' % (
                        self.beanstalkd_host, self.beanstalkd_port))
                time.sleep(2)
                continue

    def init_gateway(self):
        while True:
            try:
                if not self.gateway_connection:
                    self.gateway_connection = apns.GatewayConnection(
                        host=self.gateway_host,
                        port=self.gateway_port,
                        cert_file=self.cert_file,
                        key_file=self.key_file,
                    )
                else:
                    self.gateway_connection.reconnect()
                return
            except ssl.SSLError as e:
                if e.errno == ssl.SSL_ERROR_SSL:
                    self.gateway_invalid = True
                    time.sleep(3600)
                    logging.debug('Invalid key')
            except (socket.error, IOError) as e:
                logging.debug('Gateway connect error %s' % e)
            time.sleep(2)
            continue

    def process_gateway_input(self):
        buff = self.gateway_connection.read(apns.ERROR_RESPONSE_LENGTH)
        if len(buff) == apns.ERROR_RESPONSE_LENGTH:
            command, status, error_identifier = \
                apns.unpack(apns.ERROR_RESPONSE_FORMAT, buff)

            if 8 == command:
                found = False
                while not self.pushed_buffer.empty():
                    identifier, job = self.pushed_buffer.get()
                    if found:
                        logging.debug('Reput failed job %s' % identifier)
                        self.beanstalk.put(json.dumps(job))
                    elif identifier == error_identifier:
                        logging.debug('Found error identifier %s' % identifier)
                        found = True
        elif len(buff) == 0:
            logging.debug('Close by server')
        else:
            logging.debug('Unexcepted read buf size %s' % len(buf))

    def push_job(self):
        job = self.beanstalk.reserve(timeout=10)
        if not job:
            logging.debug('No job found')
            return
        logging.debug('Reserved job: %s %s' % (job.jid, job.body))
        try:
            job_body = json.loads(job.body)
        except ValueError:
            logging.debug(
                'Failed to loads job body: %s %s' % (job.jid, job.body))
            job.bury()

        # push job
        self.push_id += 1
        try:
            logging.debug('Send notification: %s %s' % (job.jid, job.body))
            self.gateway_connection.send_notification(
                job_body['device_token'],
                apns.Payload(**job_body['payload']),
                self.push_id)
        except apns.InvalidTokenError:
            pass
        except Exception as e:
            logging.debug('Send notification error: %s' % e)
            job.release()
            raise
        if self.pushed_buffer.full():
            self.pushed_buffer.get()
        logging.debug('Enqueue pushed buffer: %s %s' % (job.jid, job.body))
        self.pushed_buffer.put((self.push_id, job_body))
        self.last_push_time = time.time()

        logging.debug('Delete job: %s %s' % (job.jid, job.body))
        job.delete()

    def reserve_and_push(self):
        while True:
            rlist, wlist, _ = select.select(
                [self.gateway_connection.connection()],
                [self.gateway_connection.connection()],
                [],
                10)
            if rlist:
                logging.debug('Start Reading from gateway')
                self.process_gateway_input()
                self.gateway_connection.reconnect()
            elif wlist:
                self.push_job()

            if self.ok_to_stop():
                break

    def need_to_start(self):
        if self.master_worker:
            return True
        tube_stat = self.beanstalk.stats_tube(self.tube)
        if tube_stat['current-jobs-ready'] > 100:
            return True
        return False

    def ok_to_stop(self):
        if self.master_worker:
            return False
        if time.time() - self.last_push_time > 10:
            return True
        return False

    def run(self):
        self.init_beanstalk()

        while True:
            try:
                if not self.need_to_start():
                    logging.debug('Sleepy')
                    time.sleep(30)
                    continue
                logging.debug('Start to reserve and push')
                self.init_gateway()
                self.reserve_and_push()
                self.gateway_connection.disconnect()
                logging.debug('Stop to reserve and push')
            except beanstalkc.SocketError:
                self.init_beanstalk()
            except (ssl.SSLError, socket.error, IOError) as e:
                pass

if __name__ == '__main__':
    logging.basicConfig(
        format=config.LOGGING_FORMAT, level=config.LOGGING_LEVEL)
    for app_name, app_config in config.APPS.items():
        for i in range(app_config[2]):
            pipe = Pipe(
                config.BEANSTALKD_HOST, config.BEANSTALKD_PORT,
                config.PUSH_TUBE % app_name, config.APNS_HOST,
                config.APNS_PORT, app_config[1], app_config[0], i == 0)
            t = Thread(target=pipe.run, name='%s.%d' % (app_name, i))
            t.start()
<|endoftext|>


<commit_before>#!/usr/bin/python

from xml.sax.saxutils import XMLFilterBase, XMLGenerator
from xml.sax.xmlreader import AttributesImpl
from xml.sax import make_parser
import sys

def AttributesUnion(base, **values):
	baseitems = dict(base)
	baseitems.update(values)
	return AttributesImpl(baseitems)

class AnnotateSize(XMLFilterBase):
	types = {
		'BYTE': 1, 'BOOL': 1,
		'CARD8': 1, 'CARD16': 2, 'CARD32': 4,
		'INT8': 1, 'INT16': 2, 'INT32': 4,
		'char': 1, 'void': 1,
		'float': 4, 'double': 8,
		'XID': 4,
	}
	header = []
	def setTypeSize(self, name, size):
		assert not self.types.has_key(name)
		self.types[name] = size

	struct = None
	union = None
	def startElement(self, name, attrs):
		if name == 'xcb':
			self.header.insert(0, attrs['header'])
		elif name == 'field':
			size = self.types.get(attrs['type'], 0)
			if self.struct is not None:
				self.totalsize += size
			elif self.union is not None:
				self.totalsize = max(self.totalsize, size)
			attrs = AttributesUnion(attrs, bytes=str(size))
		elif name == 'xidtype':
			self.setTypeSize(attrs['name'], 4)
		elif name == 'typedef':
			self.setTypeSize(attrs['newname'], self.types[attrs['oldname']])
		elif name == 'struct' or name == 'union':
			assert self.struct is None and self.union is None
			setattr(self, name, attrs['name'])
			self.totalsize = 0

		if len(self.header) == 1 or name == 'xcb':
			XMLFilterBase.startElement(self, name, attrs)

	def characters(self, content):
		if len(self.header) == 1:
			XMLFilterBase.characters(self, content)

	def endElement(self, name):
		if len(self.header) == 1 or name == 'xcb':
			XMLFilterBase.endElement(self, name)

		if name == 'xcb':
			self.header.pop(0)
		elif name == 'struct' or name == 'union':
			assert getattr(self, name) is not None
			self.setTypeSize(getattr(self, name), self.totalsize)
			setattr(self, name, None)
			del self.totalsize

annotator = AnnotateSize(make_parser())
annotator.setContentHandler(XMLGenerator())
if len(sys.argv) > 1:
	annotator.parse(sys.argv[1])
else:
	annotator.parse(sys.stdin)
<commit_msg>Bugfix for size.py: count padding in struct size.<commit_after>#!/usr/bin/python

from xml.sax.saxutils import XMLFilterBase, XMLGenerator
from xml.sax.xmlreader import AttributesImpl
from xml.sax import make_parser
import sys

def AttributesUnion(base, **values):
	baseitems = dict(base)
	baseitems.update(values)
	return AttributesImpl(baseitems)

class AnnotateSize(XMLFilterBase):
	types = {
		'BYTE': 1, 'BOOL': 1,
		'CARD8': 1, 'CARD16': 2, 'CARD32': 4,
		'INT8': 1, 'INT16': 2, 'INT32': 4,
		'char': 1, 'void': 1,
		'float': 4, 'double': 8,
		'XID': 4,
	}
	header = []
	def setTypeSize(self, name, size):
		assert not self.types.has_key(name)
		self.types[name] = size

	struct = None
	union = None
	def startElement(self, name, attrs):
		if name == 'xcb':
			self.header.insert(0, attrs['header'])
		elif name == 'field':
			size = self.types.get(attrs['type'], 0)
			if self.struct is not None:
				self.totalsize += size
			elif self.union is not None:
				self.totalsize = max(self.totalsize, size)
			attrs = AttributesUnion(attrs, bytes=str(size))
		elif name == 'pad':
			assert self.union is None
			if self.struct is not None:
				self.totalsize += int(attrs['bytes'])
		elif name == 'xidtype':
			self.setTypeSize(attrs['name'], 4)
		elif name == 'typedef':
			self.setTypeSize(attrs['newname'], self.types[attrs['oldname']])
		elif name == 'struct' or name == 'union':
			assert self.struct is None and self.union is None
			setattr(self, name, attrs['name'])
			self.totalsize = 0

		if len(self.header) == 1 or name == 'xcb':
			XMLFilterBase.startElement(self, name, attrs)

	def characters(self, content):
		if len(self.header) == 1:
			XMLFilterBase.characters(self, content)

	def endElement(self, name):
		if len(self.header) == 1 or name == 'xcb':
			XMLFilterBase.endElement(self, name)

		if name == 'xcb':
			self.header.pop(0)
		elif name == 'struct' or name == 'union':
			assert getattr(self, name) is not None
			self.setTypeSize(getattr(self, name), self.totalsize)
			setattr(self, name, None)
			del self.totalsize

annotator = AnnotateSize(make_parser())
annotator.setContentHandler(XMLGenerator())
if len(sys.argv) > 1:
	annotator.parse(sys.argv[1])
else:
	annotator.parse(sys.stdin)
<|endoftext|>


<commit_before>import gym
env = gym.make('CartPole-v0')
env.reset()
for _ in range(1000):
    env.render()
    env.step(env.action_space.sample())<commit_msg>Ulala<commit_after>import gym

env = gym.make('CartPole-v0')
env.reset()
for _ in range(1000):
    env.render()
    env.step(env.action_space.sample())<|endoftext|>


<commit_before>#!/usr/bin/env python
#
#  This file is part of Bakefile (http://www.bakefile.org)
#
#  Copyright (C) 2009-2012 Vaclav Slavik
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to
#  deal in the Software without restriction, including without limitation the
#  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
#  sell copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in
#  all copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
#  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
#  IN THE SOFTWARE.
#

import sys
import logging
from optparse import OptionParser

class BklFormatter(logging.Formatter):

    def __init__(self):
        logging.Formatter.__init__(self, fmt=logging.BASIC_FORMAT)

    def format(self, record):
        level = record.levelno
        if level == logging.ERROR or level == logging.WARNING or level == logging.INFO:
            msg = ""
            if "pos" in dir(record) and record.pos:
                msg = "%s: " % record.pos
            if level != logging.INFO:
                msg += "%s: " % record.levelname.lower()
            msg += record.getMessage()
            return msg
        else:
            return logging.Formatter.format(self, record)

logger = logging.getLogger()
log_handler = logging.StreamHandler()
log_handler.setFormatter(BklFormatter())
logger.addHandler(log_handler)

parser = OptionParser()
parser.add_option("-v", "--verbose",
                  action="store_true", dest="verbose", default=False,
                  help="show verbose output")
parser.add_option("", "--dry-run",
                  action="store_true", dest="dry_run", default=False,
                  help="don't write any files, just pretend to do it")
parser.add_option("", "--debug",
                  action="store_true", dest="debug", default=False,
                  help="show debug log")
parser.add_option("", "--dump-model",
                  action="store_true", dest="dump", default=False,
                  help="dump project's model to stdout instead of generating output")

options, args = parser.parse_args(sys.argv[1:])

if len(args) != 1:
    sys.stderr.write("incorrect number of arguments, exactly 1 .bkl required\n")
    sys.exit(3)

if options.debug:
    log_level = logging.DEBUG
elif options.verbose:
    log_level = logging.INFO
else:
    log_level = logging.WARNING
logger.setLevel(log_level)

# note: we intentionally import bakefile this late so that the logging
# module is already initialized
import bkl.error
from bkl.interpreter import Interpreter
import bkl.dumper
import bkl.io

try:
    bkl.io.dry_run = options.dry_run
    if options.dump:
        intr = bkl.dumper.DumpingInterpreter()
    else:
        intr = Interpreter()
    intr.process_file(args[0])
except KeyboardInterrupt:
    sys.exit(2)
except bkl.error.Error as e:
    if options.debug:
        raise
    else:
        logging.error(e.msg, extra={"pos":e.pos})
        sys.exit(1)
<commit_msg>Report IOErrors nicely from the tool.<commit_after>#!/usr/bin/env python
#
#  This file is part of Bakefile (http://www.bakefile.org)
#
#  Copyright (C) 2009-2012 Vaclav Slavik
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to
#  deal in the Software without restriction, including without limitation the
#  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
#  sell copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#
#  The above copyright notice and this permission notice shall be included in
#  all copies or substantial portions of the Software.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
#  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
#  IN THE SOFTWARE.
#

import sys
import logging
from optparse import OptionParser

class BklFormatter(logging.Formatter):

    def __init__(self):
        logging.Formatter.__init__(self, fmt=logging.BASIC_FORMAT)

    def format(self, record):
        level = record.levelno
        if level == logging.ERROR or level == logging.WARNING or level == logging.INFO:
            msg = ""
            if "pos" in dir(record) and record.pos:
                msg = "%s: " % record.pos
            if level != logging.INFO:
                msg += "%s: " % record.levelname.lower()
            msg += record.getMessage()
            return msg
        else:
            return logging.Formatter.format(self, record)

logger = logging.getLogger()
log_handler = logging.StreamHandler()
log_handler.setFormatter(BklFormatter())
logger.addHandler(log_handler)

parser = OptionParser()
parser.add_option("-v", "--verbose",
                  action="store_true", dest="verbose", default=False,
                  help="show verbose output")
parser.add_option("", "--dry-run",
                  action="store_true", dest="dry_run", default=False,
                  help="don't write any files, just pretend to do it")
parser.add_option("", "--debug",
                  action="store_true", dest="debug", default=False,
                  help="show debug log")
parser.add_option("", "--dump-model",
                  action="store_true", dest="dump", default=False,
                  help="dump project's model to stdout instead of generating output")

options, args = parser.parse_args(sys.argv[1:])

if len(args) != 1:
    sys.stderr.write("incorrect number of arguments, exactly 1 .bkl required\n")
    sys.exit(3)

if options.debug:
    log_level = logging.DEBUG
elif options.verbose:
    log_level = logging.INFO
else:
    log_level = logging.WARNING
logger.setLevel(log_level)

# note: we intentionally import bakefile this late so that the logging
# module is already initialized
import bkl.error
from bkl.interpreter import Interpreter
import bkl.dumper
import bkl.io

try:
    bkl.io.dry_run = options.dry_run
    if options.dump:
        intr = bkl.dumper.DumpingInterpreter()
    else:
        intr = Interpreter()
    intr.process_file(args[0])
except KeyboardInterrupt:
    sys.exit(2)
except IOError as e:
    if options.debug:
        raise
    else:
        logging.error(e)
        sys.exit(1)
except bkl.error.Error as e:
    if options.debug:
        raise
    else:
        logging.error(e.msg, extra={"pos":e.pos})
        sys.exit(1)
<|endoftext|>


<commit_before>import matplotlib.pyplot as plt
from skimage import io, img_as_ubyte
from skimage.exposure import equalize_hist

from pysrm import srm

if __name__ == '__main__':
    img = img_as_ubyte(equalize_hist(io.imread('test.tif')))
    avg_out, lbl_out = srm.segment(img, q=15)
    plt.figure()
    plt.imshow(avg_out, cmap='gray')
    plt.show()
    plt.figure()
    plt.imshow(lbl_out)
    plt.show()
<commit_msg>Modified test plot format<commit_after>import matplotlib.pyplot as plt
from skimage import io, img_as_ubyte
from skimage.exposure import equalize_hist

from pysrm import srm

if __name__ == '__main__':
    img = img_as_ubyte(equalize_hist(io.imread('test.tif')))
    avg_out, lbl_out = srm.segment(img, q=10)
    f, ax = plt.subplots(2, 2)
    ax = ax.ravel()
    ax[0].imshow(img, cmap='gray')
    ax[1].imshow(avg_out, cmap='gray')
    ax[2].imshow(lbl_out)
    plt.show()
<|endoftext|>