<commit_before># -*- coding: utf-8 -*-
"""
    jinja2.nodes
    ~~~~~~~~~~~~

    This module implements additional nodes derived from the ast base node.

    It also provides some node tree helper functions like `in_lineno` and
    `get_nodes` used by the parser and translator in order to normalize
    python and jinja nodes.

    :copyright: 2007 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""
import operator
from itertools import chain, izip
from collections import deque
from copy import copy
from jinja2.runtime import Undefined, subscribe

_binop_to_func = {
    '*':        operator.mul,
    '/':        operator.truediv,
    '//':       operator.floordiv,
    '**':       operator.pow,
    '%':        operator.mod,
    '+':        operator.add,
    '-':        operator.sub
}

_uaop_to_func = {
    'not':      operator.not_,
    '+':        operator.pos,
    '-':        operator.neg
}

class Impossible(Exception):
    """Raised if the node could not perform a requested action."""

class NodeType(type):
    """A metaclass for nodes that handles the field and attribute
    inheritance.  fields and attributes from the parent class are
    automatically forwarded to the child."""

    def __new__(cls, name, bases, d):
        for attr in 'fields', 'attributes':
            storage = []
            for base in bases:
                storage.extend(getattr(base, attr, ()))
            storage.extend(d.get(attr, ()))
            assert len(storage) == len(set(storage))
            d[attr] = tuple(storage)
        return type.__new__(cls, name, bases, d)

class Node(object):
    """Baseclass for all Jinja nodes."""
    __metaclass__ = NodeType
    fields = ()
    attributes = ('lineno', 'environment')

    def __init__(self, *args, **kw):
        if args:
            if len(args) != len(self.fields):
                if not self.fields:
                    raise TypeError('%r takes 0 arguments' %
                                    self.__class__.__name__)
                raise TypeError('%r takes 0 or %d argument%s' % (
                    self.__class__.__name__,
                    len(self.fields),
                    len(self.fields) != 1 and 's' or ''
                ))
            for name, arg in izip(self.fields, args):
                setattr(self, name, arg)
        for attr in self.attributes:
            setattr(self, attr, kw.pop(attr, None))
        if kw:
            raise TypeError('unknown keyword argument %r' %
                            iter(kw).next())

    def iter_fields(self):
        """Iterate over all fields."""
        for name in self.fields:
            try:
                yield name, getattr(self, name)
            except AttributeError:
                pass

    def iter_child_nodes(self):
        """Iterate over all child nodes."""
        for field, item in self.iter_fields():
            if isinstance(item, list):
                for n in item:
                    if isinstance(n, Node):
                        yield n
            elif isinstance(item, Node):
                yield item

    def find(self, node_type):
        """Find the first node of a given type."""
        for result in self.find_all(node_type):
            return result

    def find_all(self, node_type):
        """Find all the nodes of a given type."""
        for child in self.iter_child_nodes():
            if isinstance(child, node_type):
                yield child
            for result in child.find_all(node_type):
                yield result

    def copy(self):
        """Return a deep copy of the node."""
        result = object.__new__(self.__class__)
        for field, value in self.iter_fields():
            if isinstance(value, Node):
                new_value = value.copy()
            elif isinstance(value, list):
                new_value = []
                for item in new_value:
                    if isinstance(item, Node):
                        item = item.copy()
                    else:
                        item = copy(item)
                    new_value.append(item)
            else:
                new_value = copy(new_value)
            setattr(result, field, new_value)
        for attr in self.attributes:
            try:
                setattr(result, attr, getattr(self, attr))
            except AttributeError:
                pass
        return result

    def set_ctx(self, ctx):
        """Reset the context of a node and all child nodes.  Per default the
        parser will all generate nodes that have a 'load' context as it's the
        most common one.  This method is used in the parser to set assignment
        targets and other nodes to a store context.
        """
        todo = deque([self])
        while todo:
            node = todo.popleft()
            if 'ctx' in node.fields:
                node.ctx = ctx
            todo.extend(node.iter_child_nodes())

    def set_environment(self, environment):
        """Set the environment for all nodes."""
        todo = deque([self])
        while todo:
            node = todo.popleft()
            node.environment = environment
            todo.extend(node.iter_child_nodes())

    def __repr__(self):
        return '%s(%s)' % (
            self.__class__.__name__,
            ', '.join('%s=%r' % (arg, getattr(self, arg, None)) for
                      arg in self.fields)
        )

class Stmt(Node):
    """Base node for all statements."""

class Helper(Node):
    """Nodes that exist in a specific context only."""

class Template(Node):
    """Node that represents a template."""
    fields = ('body',)

class Output(Stmt):
    """A node that holds multiple expressions which are then printed out.
    This is used both for the `print` statement and the regular template data.
    """
    fields = ('nodes',)

    def optimized_nodes(self):
        """Try to optimize the nodes."""
        buffer = []
        for node in self.nodes:
            try:
                const = unicode(node.as_const())
            except:
                buffer.append(node)
            else:
                if buffer and isinstance(buffer[-1], unicode):
                    buffer[-1] += const
                else:
                    buffer.append(const)
        return buffer

class Extends(Stmt):
    """Represents an extends statement."""
    fields = ('template',)

class For(Stmt):
    """A node that represents a for loop"""
    fields = ('target', 'iter', 'body', 'else_', 'recursive')

class If(Stmt):
    """A node that represents an if condition."""
    fields = ('test', 'body', 'else_')

class Macro(Stmt):
    """A node that represents a macro."""
    fields = ('name', 'args', 'defaults', 'body')

class CallBlock(Stmt):
    """A node that represents am extended macro call."""
    fields = ('call', 'body')

class Set(Stmt):
    """Allows defining own variables."""
    fields = ('name', 'expr')

class FilterBlock(Stmt):
    """Node for filter sections."""
    fields = ('body', 'filters')

class Block(Stmt):
    """A node that represents a block."""
    fields = ('name', 'body')

class Include(Stmt):
    """A node that represents the include tag."""
    fields = ('template', 'target')

class Trans(Stmt):
    """A node for translatable sections."""
    fields = ('singular', 'plural', 'indicator', 'replacements')

class ExprStmt(Stmt):
    """A statement that evaluates an expression to None."""
    fields = ('node',)

class Assign(Stmt):
    """Assigns an expression to a target."""
    fields = ('target', 'node')

class Expr(Node):
    """Baseclass for all expressions."""

    def as_const(self):
        """Return the value of the expression as constant or raise
        `Impossible` if this was not possible.
        """
        raise Impossible()

    def can_assign(self):
        """Check if it's possible to assign something to this node."""
        return False

class BinExpr(Expr):
    """Baseclass for all binary expressions."""
    fields = ('left', 'right')
    operator = None

    def as_const(self):
        f = _binop_to_func[self.operator]
        try:
            return f(self.left.as_const(), self.right.as_const())
        except:
            raise Impossible()

class UnaryExpr(Expr):
    """Baseclass for all unary expressions."""
    fields = ('node',)
    operator = None

    def as_const(self):
        f = _uaop_to_func[self.operator]
        try:
            return f(self.node.as_const())
        except:
            raise Impossible()

class Name(Expr):
    """any name such as {{ foo }}"""
    fields = ('name', 'ctx')

    def can_assign(self):
        return self.name not in ('true', 'false', 'none')

class Literal(Expr):
    """Baseclass for literals."""

class Const(Literal):
    """any constat such as {{ "foo" }}"""
    fields = ('value',)

    def as_const(self):
        return self.value

    @classmethod
    def from_untrusted(cls, value, lineno=None, environment=None):
        """Return a const object if the value is representable as
        constant value in the generated code, otherwise it will raise
        an `Impossible` exception."""
        from compiler import has_safe_repr
        if not has_safe_repr(value):
            if silent:
                return
            raise Impossible()
        return cls(value, lineno=lineno, environment=environment)

class Tuple(Literal):
    """For loop unpacking and some other things like multiple arguments
    for subscripts.
    """
    fields = ('items', 'ctx')

    def as_const(self):
        return tuple(x.as_const() for x in self.items)

    def can_assign(self):
        for item in self.items:
            if not item.can_assign():
                return False
        return True

class List(Literal):
    """any list literal such as {{ [1, 2, 3] }}"""
    fields = ('items',)

    def as_const(self):
        return [x.as_const() for x in self.items]

class Dict(Literal):
    """any dict literal such as {{ {1: 2, 3: 4} }}"""
    fields = ('items',)

    def as_const(self):
        return dict(x.as_const() for x in self.items)

class Pair(Helper):
    """A key, value pair for dicts."""
    fields = ('key', 'value')

    def as_const(self):
        return self.key.as_const(), self.value.as_const()

class Keyword(Helper):
    """A key, value pair for keyword arguments."""
    fields = ('key', 'value')

class CondExpr(Expr):
    """{{ foo if bar else baz }}"""
    fields = ('test', 'expr1', 'expr2')

    def as_const(self):
        if self.test.as_const():
            return self.expr1.as_const()
        return self.expr2.as_const()

class Filter(Expr):
    """{{ foo|bar|baz }}"""
    fields = ('node', 'name', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs')

    def as_const(self):
        filter = self.environment.filters.get(self.name)
        if filter is None or getattr(filter, 'contextfilter', False):
            raise nodes.Impossible()
        obj = self.node.as_const()
        args = [x.as_const() for x in self.args]
        kwargs = dict(x.as_const() for x in self.kwargs)
        if self.dyn_args is not None:
            try:
                args.extend(self.dyn_args.as_const())
            except:
                raise Impossible()
        if self.dyn_kwargs is not None:
            try:
                kwargs.update(self.dyn_kwargs.as_const())
            except:
                raise Impossible()
        try:
            return filter(obj, *args, **kwargs)
        except:
            raise nodes.Impossible()

class Test(Expr):
    """{{ foo is lower }}"""
    fields = ('node', 'name', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs')

class Call(Expr):
    """{{ foo(bar) }}"""
    fields = ('node', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs')

    def as_const(self):
        obj = self.node.as_const()
        args = [x.as_const() for x in self.args]
        kwargs = dict(x.as_const() for x in self.kwargs)
        if self.dyn_args is not None:
            try:
                args.extend(self.dyn_args.as_const())
            except:
                raise Impossible()
        if self.dyn_kwargs is not None:
            try:
                kwargs.update(self.dyn_kwargs.as_const())
            except:
                raise Impossible()
        try:
            return obj(*args, **kwargs)
        except:
            raise nodes.Impossible()

class Subscript(Expr):
    """{{ foo.bar }} and {{ foo['bar'] }} etc."""
    fields = ('node', 'arg', 'ctx')

    def as_const(self):
        if self.ctx != 'load':
            raise Impossible()
        try:
            return subscribe(self.node.as_const(), self.arg.as_const())
        except:
            raise Impossible()

    def can_assign(self):
        return True

class Slice(Expr):
    """1:2:3 etc."""
    fields = ('start', 'stop', 'step')

    def as_const(self):
        def const(obj):
            if obj is None:
                return obj
            return obj.as_const()
        return slice(const(self.start), const(self.stop), const(self.step))

class Concat(Expr):
    """For {{ foo ~ bar }}.  Concatenates strings."""
    fields = ('nodes',)

    def as_const(self):
        return ''.join(unicode(x.as_const()) for x in self.nodes)

class Compare(Expr):
    """{{ foo == bar }}, {{ foo >= bar }} etc."""
    fields = ('expr', 'ops')

class Operand(Helper):
    """Operator + expression."""
    fields = ('op', 'expr')

class Mul(BinExpr):
    """{{ foo * bar }}"""
    operator = '*'

class Div(BinExpr):
    """{{ foo / bar }}"""
    operator = '/'

class FloorDiv(BinExpr):
    """{{ foo // bar }}"""
    operator = '//'

class Add(BinExpr):
    """{{ foo + bar }}"""
    operator = '+'

class Sub(BinExpr):
    """{{ foo - bar }}"""
    operator = '-'

class Mod(BinExpr):
    """{{ foo % bar }}"""
    operator = '%'

class Pow(BinExpr):
    """{{ foo ** bar }}"""
    operator = '**'

class And(BinExpr):
    """{{ foo and bar }}"""
    operator = 'and'

    def as_const(self):
        return self.left.as_const() and self.right.as_const()

class Or(BinExpr):
    """{{ foo or bar }}"""
    operator = 'or'

    def as_const(self):
        return self.left.as_const() or self.right.as_const()

class Not(UnaryExpr):
    """{{ not foo }}"""
    operator = 'not'

class Neg(UnaryExpr):
    """{{ -foo }}"""
    operator = '-'

class Pos(UnaryExpr):
    """{{ +foo }}"""
    operator = '+'
<commit_msg>fixed deepcopy<commit_after># -*- coding: utf-8 -*-
"""
    jinja2.nodes
    ~~~~~~~~~~~~

    This module implements additional nodes derived from the ast base node.

    It also provides some node tree helper functions like `in_lineno` and
    `get_nodes` used by the parser and translator in order to normalize
    python and jinja nodes.

    :copyright: 2007 by Armin Ronacher.
    :license: BSD, see LICENSE for more details.
"""
import operator
from itertools import chain, izip
from collections import deque
from copy import copy
from jinja2.runtime import Undefined, subscribe

_binop_to_func = {
    '*':        operator.mul,
    '/':        operator.truediv,
    '//':       operator.floordiv,
    '**':       operator.pow,
    '%':        operator.mod,
    '+':        operator.add,
    '-':        operator.sub
}

_uaop_to_func = {
    'not':      operator.not_,
    '+':        operator.pos,
    '-':        operator.neg
}

class Impossible(Exception):
    """Raised if the node could not perform a requested action."""

class NodeType(type):
    """A metaclass for nodes that handles the field and attribute
    inheritance.  fields and attributes from the parent class are
    automatically forwarded to the child."""

    def __new__(cls, name, bases, d):
        for attr in 'fields', 'attributes':
            storage = []
            for base in bases:
                storage.extend(getattr(base, attr, ()))
            storage.extend(d.get(attr, ()))
            assert len(storage) == len(set(storage))
            d[attr] = tuple(storage)
        return type.__new__(cls, name, bases, d)

class Node(object):
    """Baseclass for all Jinja nodes."""
    __metaclass__ = NodeType
    fields = ()
    attributes = ('lineno', 'environment')

    def __init__(self, *args, **kw):
        if args:
            if len(args) != len(self.fields):
                if not self.fields:
                    raise TypeError('%r takes 0 arguments' %
                                    self.__class__.__name__)
                raise TypeError('%r takes 0 or %d argument%s' % (
                    self.__class__.__name__,
                    len(self.fields),
                    len(self.fields) != 1 and 's' or ''
                ))
            for name, arg in izip(self.fields, args):
                setattr(self, name, arg)
        for attr in self.attributes:
            setattr(self, attr, kw.pop(attr, None))
        if kw:
            raise TypeError('unknown keyword argument %r' %
                            iter(kw).next())

    def iter_fields(self):
        """Iterate over all fields."""
        for name in self.fields:
            try:
                yield name, getattr(self, name)
            except AttributeError:
                pass

    def iter_child_nodes(self):
        """Iterate over all child nodes."""
        for field, item in self.iter_fields():
            if isinstance(item, list):
                for n in item:
                    if isinstance(n, Node):
                        yield n
            elif isinstance(item, Node):
                yield item

    def find(self, node_type):
        """Find the first node of a given type."""
        for result in self.find_all(node_type):
            return result

    def find_all(self, node_type):
        """Find all the nodes of a given type."""
        for child in self.iter_child_nodes():
            if isinstance(child, node_type):
                yield child
            for result in child.find_all(node_type):
                yield result

    def copy(self):
        """Return a deep copy of the node."""
        result = object.__new__(self.__class__)
        for field, value in self.iter_fields():
            if isinstance(value, Node):
                new_value = value.copy()
            elif isinstance(value, list):
                new_value = []
                for item in value:
                    if isinstance(item, Node):
                        item = item.copy()
                    else:
                        item = copy(item)
                    new_value.append(item)
            else:
                new_value = copy(value)
            setattr(result, field, new_value)
        for attr in self.attributes:
            try:
                setattr(result, attr, getattr(self, attr))
            except AttributeError:
                pass
        return result

    def set_ctx(self, ctx):
        """Reset the context of a node and all child nodes.  Per default the
        parser will all generate nodes that have a 'load' context as it's the
        most common one.  This method is used in the parser to set assignment
        targets and other nodes to a store context.
        """
        todo = deque([self])
        while todo:
            node = todo.popleft()
            if 'ctx' in node.fields:
                node.ctx = ctx
            todo.extend(node.iter_child_nodes())

    def set_environment(self, environment):
        """Set the environment for all nodes."""
        todo = deque([self])
        while todo:
            node = todo.popleft()
            node.environment = environment
            todo.extend(node.iter_child_nodes())

    def __repr__(self):
        return '%s(%s)' % (
            self.__class__.__name__,
            ', '.join('%s=%r' % (arg, getattr(self, arg, None)) for
                      arg in self.fields)
        )

class Stmt(Node):
    """Base node for all statements."""

class Helper(Node):
    """Nodes that exist in a specific context only."""

class Template(Node):
    """Node that represents a template."""
    fields = ('body',)

class Output(Stmt):
    """A node that holds multiple expressions which are then printed out.
    This is used both for the `print` statement and the regular template data.
    """
    fields = ('nodes',)

    def optimized_nodes(self):
        """Try to optimize the nodes."""
        buffer = []
        for node in self.nodes:
            try:
                const = unicode(node.as_const())
            except:
                buffer.append(node)
            else:
                if buffer and isinstance(buffer[-1], unicode):
                    buffer[-1] += const
                else:
                    buffer.append(const)
        return buffer

class Extends(Stmt):
    """Represents an extends statement."""
    fields = ('template',)

class For(Stmt):
    """A node that represents a for loop"""
    fields = ('target', 'iter', 'body', 'else_', 'recursive')

class If(Stmt):
    """A node that represents an if condition."""
    fields = ('test', 'body', 'else_')

class Macro(Stmt):
    """A node that represents a macro."""
    fields = ('name', 'args', 'defaults', 'body')

class CallBlock(Stmt):
    """A node that represents am extended macro call."""
    fields = ('call', 'body')

class Set(Stmt):
    """Allows defining own variables."""
    fields = ('name', 'expr')

class FilterBlock(Stmt):
    """Node for filter sections."""
    fields = ('body', 'filters')

class Block(Stmt):
    """A node that represents a block."""
    fields = ('name', 'body')

class Include(Stmt):
    """A node that represents the include tag."""
    fields = ('template', 'target')

class Trans(Stmt):
    """A node for translatable sections."""
    fields = ('singular', 'plural', 'indicator', 'replacements')

class ExprStmt(Stmt):
    """A statement that evaluates an expression to None."""
    fields = ('node',)

class Assign(Stmt):
    """Assigns an expression to a target."""
    fields = ('target', 'node')

class Expr(Node):
    """Baseclass for all expressions."""

    def as_const(self):
        """Return the value of the expression as constant or raise
        `Impossible` if this was not possible.
        """
        raise Impossible()

    def can_assign(self):
        """Check if it's possible to assign something to this node."""
        return False

class BinExpr(Expr):
    """Baseclass for all binary expressions."""
    fields = ('left', 'right')
    operator = None

    def as_const(self):
        f = _binop_to_func[self.operator]
        try:
            return f(self.left.as_const(), self.right.as_const())
        except:
            raise Impossible()

class UnaryExpr(Expr):
    """Baseclass for all unary expressions."""
    fields = ('node',)
    operator = None

    def as_const(self):
        f = _uaop_to_func[self.operator]
        try:
            return f(self.node.as_const())
        except:
            raise Impossible()

class Name(Expr):
    """any name such as {{ foo }}"""
    fields = ('name', 'ctx')

    def can_assign(self):
        return self.name not in ('true', 'false', 'none')

class Literal(Expr):
    """Baseclass for literals."""

class Const(Literal):
    """any constat such as {{ "foo" }}"""
    fields = ('value',)

    def as_const(self):
        return self.value

    @classmethod
    def from_untrusted(cls, value, lineno=None, environment=None):
        """Return a const object if the value is representable as
        constant value in the generated code, otherwise it will raise
        an `Impossible` exception."""
        from compiler import has_safe_repr
        if not has_safe_repr(value):
            raise Impossible()
        return cls(value, lineno=lineno, environment=environment)

class Tuple(Literal):
    """For loop unpacking and some other things like multiple arguments
    for subscripts.
    """
    fields = ('items', 'ctx')

    def as_const(self):
        return tuple(x.as_const() for x in self.items)

    def can_assign(self):
        for item in self.items:
            if not item.can_assign():
                return False
        return True

class List(Literal):
    """any list literal such as {{ [1, 2, 3] }}"""
    fields = ('items',)

    def as_const(self):
        return [x.as_const() for x in self.items]

class Dict(Literal):
    """any dict literal such as {{ {1: 2, 3: 4} }}"""
    fields = ('items',)

    def as_const(self):
        return dict(x.as_const() for x in self.items)

class Pair(Helper):
    """A key, value pair for dicts."""
    fields = ('key', 'value')

    def as_const(self):
        return self.key.as_const(), self.value.as_const()

class Keyword(Helper):
    """A key, value pair for keyword arguments."""
    fields = ('key', 'value')

class CondExpr(Expr):
    """{{ foo if bar else baz }}"""
    fields = ('test', 'expr1', 'expr2')

    def as_const(self):
        if self.test.as_const():
            return self.expr1.as_const()
        return self.expr2.as_const()

class Filter(Expr):
    """{{ foo|bar|baz }}"""
    fields = ('node', 'name', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs')

    def as_const(self):
        filter = self.environment.filters.get(self.name)
        if filter is None or getattr(filter, 'contextfilter', False):
            raise nodes.Impossible()
        obj = self.node.as_const()
        args = [x.as_const() for x in self.args]
        kwargs = dict(x.as_const() for x in self.kwargs)
        if self.dyn_args is not None:
            try:
                args.extend(self.dyn_args.as_const())
            except:
                raise Impossible()
        if self.dyn_kwargs is not None:
            try:
                kwargs.update(self.dyn_kwargs.as_const())
            except:
                raise Impossible()
        try:
            return filter(obj, *args, **kwargs)
        except:
            raise nodes.Impossible()

class Test(Expr):
    """{{ foo is lower }}"""
    fields = ('node', 'name', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs')

class Call(Expr):
    """{{ foo(bar) }}"""
    fields = ('node', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs')

    def as_const(self):
        obj = self.node.as_const()
        args = [x.as_const() for x in self.args]
        kwargs = dict(x.as_const() for x in self.kwargs)
        if self.dyn_args is not None:
            try:
                args.extend(self.dyn_args.as_const())
            except:
                raise Impossible()
        if self.dyn_kwargs is not None:
            try:
                kwargs.update(self.dyn_kwargs.as_const())
            except:
                raise Impossible()
        try:
            return obj(*args, **kwargs)
        except:
            raise nodes.Impossible()

class Subscript(Expr):
    """{{ foo.bar }} and {{ foo['bar'] }} etc."""
    fields = ('node', 'arg', 'ctx')

    def as_const(self):
        if self.ctx != 'load':
            raise Impossible()
        try:
            return subscribe(self.node.as_const(), self.arg.as_const())
        except:
            raise Impossible()

    def can_assign(self):
        return True

class Slice(Expr):
    """1:2:3 etc."""
    fields = ('start', 'stop', 'step')

    def as_const(self):
        def const(obj):
            if obj is None:
                return obj
            return obj.as_const()
        return slice(const(self.start), const(self.stop), const(self.step))

class Concat(Expr):
    """For {{ foo ~ bar }}.  Concatenates strings."""
    fields = ('nodes',)

    def as_const(self):
        return ''.join(unicode(x.as_const()) for x in self.nodes)

class Compare(Expr):
    """{{ foo == bar }}, {{ foo >= bar }} etc."""
    fields = ('expr', 'ops')

class Operand(Helper):
    """Operator + expression."""
    fields = ('op', 'expr')

class Mul(BinExpr):
    """{{ foo * bar }}"""
    operator = '*'

class Div(BinExpr):
    """{{ foo / bar }}"""
    operator = '/'

class FloorDiv(BinExpr):
    """{{ foo // bar }}"""
    operator = '//'

class Add(BinExpr):
    """{{ foo + bar }}"""
    operator = '+'

class Sub(BinExpr):
    """{{ foo - bar }}"""
    operator = '-'

class Mod(BinExpr):
    """{{ foo % bar }}"""
    operator = '%'

class Pow(BinExpr):
    """{{ foo ** bar }}"""
    operator = '**'

class And(BinExpr):
    """{{ foo and bar }}"""
    operator = 'and'

    def as_const(self):
        return self.left.as_const() and self.right.as_const()

class Or(BinExpr):
    """{{ foo or bar }}"""
    operator = 'or'

    def as_const(self):
        return self.left.as_const() or self.right.as_const()

class Not(UnaryExpr):
    """{{ not foo }}"""
    operator = 'not'

class Neg(UnaryExpr):
    """{{ -foo }}"""
    operator = '-'

class Pos(UnaryExpr):
    """{{ +foo }}"""
    operator = '+'
<|endoftext|>


<commit_before># Store the version here so:
# 1) we don't load dependencies by storing it in __init__.py
# 2) we can import it in setup.py for the same reason
# 3) we can import it into the jira module
__version__ = '0.8'
<commit_msg>- bump to version 0.9 for summit release<commit_after># Store the version here so:
# 1) we don't load dependencies by storing it in __init__.py
# 2) we can import it in setup.py for the same reason
# 3) we can import it into the jira module
__version__ = '0.9'
<|endoftext|>


<commit_before># Store the version here so:
# 1) we don't load dependencies by storing it in __init__.py
# 2) we can import it in setup.py for the same reason
# 3) we can import it into the jira module
__version__ = '0.11'
<commit_msg>Bump to version 0.12 for imminent release<commit_after># Store the version here so:
# 1) we don't load dependencies by storing it in __init__.py
# 2) we can import it in setup.py for the same reason
# 3) we can import it into the jira module
__version__ = '0.12'
<|endoftext|>


<commit_before>import datetime
import tornado.gen
import tornado.ioloop

def delay(dur, callback=None):
    """Delay N seconds. This must be invoked as
    yield tornado.gen.Task(app.delay, dur)
    """
    delta = datetime.timedelta(seconds=dur)
    return tornado.ioloop.IOLoop.current().add_timeout(delta, callback)
<commit_msg>Fix comment.<commit_after>import datetime
import tornado.gen
import tornado.ioloop

def delay(dur, callback=None):
    """Delay N seconds. This must be invoked as
    yield tornado.gen.Task(twcommon.async.delay, dur)
    """
    delta = datetime.timedelta(seconds=dur)
    return tornado.ioloop.IOLoop.current().add_timeout(delta, callback)
<|endoftext|>


<commit_before>""" shrunk - Rutgers University URL Shortener

Unit tests for statutil functions.
"""

import operator

import pytest

from shrunk.util.stat import get_referer_domain, make_csv_for_links, \
    get_human_readable_referer_domain

from fixtures import app, db  # noqa: F401

@pytest.mark.parametrize('url,expected', [('https://google.com', 'google.com'),
                                          ('https://sld.google.com/abc', 'sld.google.com'),
                                          ('https://my.si.te:80', 'my.si.te'),
                                          ('https://www.example.com/pa/th/', 'example.com'),
                                          ('https://t.co/link', 'Twitter'),
                                          ('android-app://com.linkedin.android', 'LinkedIn App')])
def test_get_human_readable_referer_domain(url, expected):
    assert get_human_readable_referer_domain({'referer': url}) == expected

def test_get_referer_domain_no_referer():
    assert get_referer_domain({}) is None

def test_make_csv_for_links(db, app):
    with app.app_context():
        def shorten(long_url):
            return db.create_short_url(long_url, netid='shrunk_test')

        short0 = shorten('http://www.foobar.net/index')
        short1 = shorten('http://www.rutger.edu/coleg')

        for _ in range(4):
            # short url, source ip, useragent, referer
            db.visit(short0, '127.0.0.1', 'user agent', 'http://facebook.com')
            db.visit(short1, '127.0.0.1', 'user agent', 'http://reddit.com')

        header = 'short url,visitor id,location,referrer domain,user agent,time (eastern time)'

        csv0 = make_csv_for_links(db, [short0]).split('\r\n')
        assert len(csv0) == 6
        assert csv0[0] == header
        assert all('facebook.com' in l for l in csv0[1:] if l)

        csv1 = make_csv_for_links(db, [short1]).split('\r\n')
        assert len(csv1) == 6
        assert csv1[0] == header
        assert all('reddit.com' in l for l in csv1[1:] if l)

        csv01 = make_csv_for_links(db, [short0, short1]).split('\r\n')
        assert len(csv01) == 10
        assert csv01[0] == header

def test_make_geoip_csv(db, app):
    with app.app_context():
        short = db.create_short_url('google.com', netid='shrunk_test')

        ips = ['165.230.224.67', '34.201.163.243', '35.168.234.184',
               '107.77.70.130', '136.243.154.93', '94.130.167.121']
        for ip in ips:
            db.visit(short, ip, 'user agent', 'referer')

        expected = {
            'us': [{'code': 'NJ', 'value': 1},
                   {'code': 'NY', 'value': 1},
                   {'code': 'VA', 'value': 2}],
            'world': [{'code': 'US', 'value': 4},
                      {'code': 'DE', 'value': 2}]
        }

        actual = db.get_geoip_json(short)

        get_code = operator.itemgetter('code')
        assert sorted(expected['us'], key=get_code) == sorted(actual['us'], key=get_code)
        assert sorted(expected['world'], key=get_code) == sorted(actual['world'], key=get_code)
<commit_msg>add some tests<commit_after>""" shrunk - Rutgers University URL Shortener

Unit tests for statutil functions.
"""

import operator

import pytest

from shrunk.util.stat import get_referer_domain, make_csv_for_links, \
    get_human_readable_referer_domain, get_browser_platform

from fixtures import app, db  # noqa: F401

@pytest.mark.parametrize('url,expected', [('https://google.com', 'google.com'),
                                          ('https://sld.google.com/abc', 'sld.google.com'),
                                          ('https://my.si.te:80', 'my.si.te'),
                                          ('https://www.example.com/pa/th/', 'example.com'),
                                          ('https://t.co/link', 'Twitter'),
                                          ('android-app://com.linkedin.android', 'LinkedIn App')])
def test_get_human_readable_referer_domain(url, expected):
    assert get_human_readable_referer_domain({'referer': url}) == expected

def test_get_human_readable_referer_domain_no_referer():
    assert get_human_readable_referer_domain({}) == 'Unknown'

def test_get_referer_domain_no_referer():
    assert get_referer_domain({}) is None

def test_make_csv_for_links(db, app):
    with app.app_context():
        def shorten(long_url):
            return db.create_short_url(long_url, netid='shrunk_test')

        short0 = shorten('http://www.foobar.net/index')
        short1 = shorten('http://www.rutger.edu/coleg')

        for _ in range(4):
            # short url, source ip, useragent, referer
            db.visit(short0, '127.0.0.1', 'user agent', 'http://facebook.com')
            db.visit(short1, '127.0.0.1', 'user agent', 'http://reddit.com')

        header = 'short url,visitor id,location,referrer domain,user agent,time (eastern time)'

        csv0 = make_csv_for_links(db, [short0]).split('\r\n')
        assert len(csv0) == 6
        assert csv0[0] == header
        assert all('facebook.com' in l for l in csv0[1:] if l)

        csv1 = make_csv_for_links(db, [short1]).split('\r\n')
        assert len(csv1) == 6
        assert csv1[0] == header
        assert all('reddit.com' in l for l in csv1[1:] if l)

        csv01 = make_csv_for_links(db, [short0, short1]).split('\r\n')
        assert len(csv01) == 10
        assert csv01[0] == header

def test_make_geoip_csv(db, app):
    with app.app_context():
        short = db.create_short_url('google.com', netid='shrunk_test')

        ips = ['165.230.224.67', '34.201.163.243', '35.168.234.184',
               '107.77.70.130', '136.243.154.93', '94.130.167.121']
        for ip in ips:
            db.visit(short, ip, 'user agent', 'referer')

        expected = {
            'us': [{'code': 'NJ', 'value': 1},
                   {'code': 'NY', 'value': 1},
                   {'code': 'VA', 'value': 2}],
            'world': [{'code': 'US', 'value': 4},
                      {'code': 'DE', 'value': 2}]
        }

        actual = db.get_geoip_json(short)

        get_code = operator.itemgetter('code')
        assert sorted(expected['us'], key=get_code) == sorted(actual['us'], key=get_code)
        assert sorted(expected['world'], key=get_code) == sorted(actual['world'], key=get_code)

@pytest.mark.parametrize('useragent,platform', [
    ('Mozilla/5.0 (X11; U; OpenBSD amd64; en-US; rv:1.9.0.1) Gecko/2008081402 Firefox/3.0.1',
     ('Firefox', 'OpenBSD')),
    ('Mozilla/5.0 (X11; FreeBSD i686) Firefox/3.6', ('Firefox', 'FreeBSD')),
    ('Mozilla/5.0 (X11; U; NetBSD i386; en-US; rv:1.8) Gecko/20060104 Firefox/1.5',
     ('Firefox', 'NetBSD')),
    ('Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0', ('Firefox', 'Linux')),
    ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.82 Safari/537.36 Vivaldi/2.3.1440.41', ('Vivaldi', 'Windows'))
])
def test_get_browser_platform(useragent, platform):
    assert get_browser_platform(useragent) == platform
<|endoftext|>


<commit_before>import os
from pathlib import Path
from importlib import import_module
from Datasnakes.Manager import ProjectManagement
from Datasnakes.Orthologs.utils import attribute_config
from Datasnakes.Tools.ftp import NcbiFTPClient
from Datasnakes.Tools.logit import LogIt
from Datasnakes.Manager.BioSQL import biosql

class DatabaseManagement(object):

    def __init__(self, project, email, project_path=None, proj_mana=ProjectManagement, biosql_mana=biosql, **kwargs):
        self.dbmanalog = LogIt().default(logname="DatabaseManagement", logfile=None)
        self.config_options = {
            "GI_config": self.get_gi_lists,
            "Blast_config": self.download_blast_database,
            "Taxonomy_config": self.download_taxonomy_database,
            "GenBank_config": [self.download_refseq_release_files, self.upload_genbank_flatfiles]
                               }
        self.project = project
        self.email = email
        self.database_dict = {}
        self.ncbiftp = NcbiFTPClient(email=self.email)
        # TODO-ROB:  Configure this differently somehow
        self.biosql = biosql_mana

        # Configuration of class attributes.
        add_self = attribute_config(self, composer=proj_mana, checker=ProjectManagement, project=project, project_path=project_path)
        for var, attr in add_self.__dict__.items():
            setattr(self, var, attr)

        # Determine which database to update
        # And then run that script with the configuration.
        for config in self.config_options.keys():
            if config in kwargs.keys():
                db_config_type = config
                db_config_method = self.config_options[config]
                db_configuration = kwargs[config]
                # db_config_method(kwargs[config])
                self.database_dict[db_config_type] = [db_config_method, db_configuration]

    def get_gi_lists(self):
        print()

    def download_blast_database(self, database_name="refseq_rna", database_path=None):
        # <path>/<user or basic_project>/databases/NCBI/blast/db/<database_name>
        db_path = self.ncbi_db_repo / Path('blast') / Path('db')
        if database_path:
            db_path = Path(database_path)
        self.ncbiftp.getblastdb(database_name=database_name, download_path=db_path)
        # TODO-ROB Add email or slack notifications
        self.dbmanalog.critical("Please set the BLAST environment variables in your .bash_profile!!")
        self.dbmanalog.info("The appropriate environment variable is \'BLASTDB=%s\'." % str(db_path))
        self.dbmanalog.critical("Please set the BLAST environment variables in your .bash_profile!!")
        # TODO-ROB:  set up environment variables.  Also add CLI setup

    def download_taxonomy_database(self, db_type, dest_name=None, dest_path=None, driver=None):
        """
        This method gets the remote data and updates the local databases for ETE3, BioSQL, and PhyloDB taxonomy
        databases.  Most significant is the "biosql" and "phylodb" types.  The biosql databases use NCBI's taxonomy
        database along with the biosql schema.  And the phylodb databases use ITIS's taxonomy database along with the
        biosql schema.

        :param db_type:  The type of database.  ("ete3", "biosq", or "phylodb")
        :param dest_name:  The name of the new database.
        :param dest_path:  The location where the new database should go.
        :param driver:  The type of RDBMS.  ("SQLite", "MySQL", "PostGRE")
        :return:  An updated taxonomy database.
        """
        db_type = str(db_type).lower()
        if db_type == 'ete3':
            # DEFAULT_TAXADB = os.path.join(os.environ.get('HOME', '/'), '.etetoolkit', 'taxa.sqlite')
            ete3 = import_module("ete3")
            ete3.NCBITaxa.update_taxonomy_database()
        elif db_type == 'biosql':
            # Loads data from NCBI via ftp://ftp.ncbi.nlm.nih.gov/pub/taxonomy
            if driver.lower() == "sqlite3":
                db_path = self.ncbi_db_repo / Path('pub') / Path('taxonomy')
                ncbi_db = self.biosql.SQLiteBioSQL(database_path=db_path)
                ncbi_db.copy_template_database(dest_name=dest_name, dest_path=dest_path)

            elif driver.lower() == "mysql":
                db_path = self.ncbi_db_repo / Path('pub') / Path('taxonomy')
                ncbi_db = self.biosql.MySQLBioSQL()

        elif db_type == 'phylodb':
            # Loads data from ITIS via http://www.itis.gov/downloads/
            print('biosql_repo')

    # TODO-ROB:  Update ncbiftp class syntax to reflect NCBI's ftp site
    def download_refseq_release_files(self, database_name, database_path, collection_subset, seqtype, format, driver="sqlite3", extension=".gbk.db"):
        db_path = self.ncbi_db_repo / Path('refseq') / Path('release') / Path(collection_subset)
        self.ncbiftp.getrefseqrelease(database_name=collection_subset, seqtype=seqtype, filetype=format, download_path=db_path)
        if database_path:
            db_path = Path(database_path)

    def upload_refseq_release_files(self, database_name, database_path, collection_subset, seqtype, format,
                                    driver="sqlite3", extension=".gbk.db"):
        db_name = str(database_name) + str(extension)
        db_path = self.ncbi_db_repo / Path('refseq') / Path('release') / Path(collection_subset)
        self.download_taxonomy_database(db_type="biosql", dest_name=db_name, dest_path=db_path, driver=driver)
        pass

    def get_project_genbank_database(self):
        print()

<commit_msg>Finished the upload_refseq_release_files method.<commit_after>import os
from pathlib import Path
from importlib import import_module
from Datasnakes.Manager import ProjectManagement
from Datasnakes.Orthologs.utils import attribute_config
from Datasnakes.Tools.ftp import NcbiFTPClient
from Datasnakes.Tools.logit import LogIt
from Datasnakes.Manager.BioSQL import biosql

class DatabaseManagement(object):

    def __init__(self, project, email, project_path=None, proj_mana=ProjectManagement, biosql_mana=biosql, **kwargs):
        self.dbmanalog = LogIt().default(logname="DatabaseManagement", logfile=None)
        self.config_options = {
            "GI_config": self.get_gi_lists,
            "Blast_config": self.download_blast_database,
            "Taxonomy_config": self.download_taxonomy_database,
            "GenBank_config": [self.download_refseq_release_files, self.upload_genbank_flatfiles]
                               }
        self.project = project
        self.email = email
        self.database_dict = {}
        self.ncbiftp = NcbiFTPClient(email=self.email)
        # TODO-ROB:  Configure this differently somehow
        self.biosql = biosql_mana

        # Configuration of class attributes.
        add_self = attribute_config(self, composer=proj_mana, checker=ProjectManagement, project=project, project_path=project_path)
        for var, attr in add_self.__dict__.items():
            setattr(self, var, attr)

        # Determine which database to update
        # And then run that script with the configuration.
        for config in self.config_options.keys():
            if config in kwargs.keys():
                db_config_type = config
                db_config_method = self.config_options[config]
                db_configuration = kwargs[config]
                # db_config_method(kwargs[config])
                self.database_dict[db_config_type] = [db_config_method, db_configuration]

    def get_gi_lists(self):
        print()

    def download_blast_database(self, database_name="refseq_rna", database_path=None):
        # <path>/<user or basic_project>/databases/NCBI/blast/db/<database_name>
        db_path = self.ncbi_db_repo / Path('blast') / Path('db')
        if database_path:
            db_path = Path(database_path)
        self.ncbiftp.getblastdb(database_name=database_name, download_path=db_path)
        # TODO-ROB Add email or slack notifications
        self.dbmanalog.critical("Please set the BLAST environment variables in your .bash_profile!!")
        self.dbmanalog.info("The appropriate environment variable is \'BLASTDB=%s\'." % str(db_path))
        self.dbmanalog.critical("Please set the BLAST environment variables in your .bash_profile!!")
        # TODO-ROB:  set up environment variables.  Also add CLI setup

    def download_taxonomy_database(self, db_type, dest_name=None, dest_path=None, driver=None):
        """
        This method gets the remote data and updates the local databases for ETE3, BioSQL, and PhyloDB taxonomy
        databases.  Most significant is the "biosql" and "phylodb" types.  The biosql databases use NCBI's taxonomy
        database along with the biosql schema.  And the phylodb databases use ITIS's taxonomy database along with the
        biosql schema.

        :param db_type:  The type of database.  ("ete3", "biosq", or "phylodb")
        :param dest_name:  The name of the new database.
        :param dest_path:  The location where the new database should go.
        :param driver:  The type of RDBMS.  ("SQLite", "MySQL", "PostGRE")
        :return:  An updated taxonomy database.
        """
        db_type = str(db_type).lower()
        if db_type == 'ete3':
            # DEFAULT_TAXADB = os.path.join(os.environ.get('HOME', '/'), '.etetoolkit', 'taxa.sqlite')
            ete3 = import_module("ete3")
            ete3.NCBITaxa.update_taxonomy_database()
        elif db_type == 'biosql':
            # Loads data from NCBI via ftp://ftp.ncbi.nlm.nih.gov/pub/taxonomy
            if driver.lower() == "sqlite3":
                db_path = self.ncbi_db_repo / Path('pub') / Path('taxonomy')
                ncbi_db = self.biosql.SQLiteBioSQL(database_path=db_path)
                ncbi_db.copy_template_database(dest_name=dest_name, dest_path=dest_path)
                return ncbi_db

            elif driver.lower() == "mysql":
                db_path = self.ncbi_db_repo / Path('pub') / Path('taxonomy')
                ncbi_db = self.biosql.MySQLBioSQL()
                return ncbi_db

        elif db_type == 'phylodb':
            # Loads data from ITIS via http://www.itis.gov/downloads/
            print('biosql_repo')

    # TODO-ROB:  Update ncbiftp class syntax to reflect NCBI's ftp site
    def download_refseq_release_files(self, database_name, database_path, collection_subset, seqtype, filetype, driver="sqlite3", extension=".gbk.db"):
        db_path = self.ncbi_db_repo / Path('refseq') / Path('release') / Path(collection_subset)
        ncbiftp = self.ncbiftp.getrefseqrelease(database_name=collection_subset, seqtype=seqtype, filetype=filetype, download_path=db_path)
        if database_path:
            db_path = Path(database_path)
        return ncbiftp

    def upload_refseq_release_files(self, database_name, collection_subset, seqtype, filetype, upload_list,
                                    driver="sqlite3", extension=".gbk.db"):
        db_name = str(database_name) + str(extension)
        db_path = self.ncbi_db_repo / Path('refseq') / Path('release') / Path(collection_subset)
        ncbi_db = self.download_taxonomy_database(db_type="biosql", dest_name=db_name, dest_path=db_path, driver=driver)
        ncbi_db.upload_path = db_path
        ncbi_db.upload_list = upload_list
        ncbi_db.upload_files(seqtype=seqtype, filetype=filetype)
        pass

    def get_project_genbank_database(self):
        print()

<|endoftext|>


<commit_before>from datetime import datetime
import logging
import sys

FILE = "cli-count.txt"
UNIT = 1
log = logging.getLogger('cli-count')
ACTION_NEW = "new"
ACTION_ADD = "add"
ACTION_TOTAL = "total"
ACTION_LIST = "list"
OPTION_ALL = "all"

def help():
    print """
    cli-count ACTION TAG_NAME VALUE             :: general format
    cli-count new tag_name (start_value)        :: default start_value is 0
    cli-count add tag_name (value)              :: default value is 1
    cli-count total tag_name (start_date)       :: date format: dd.mm.year
    cli-count list (tag_name) (start_date)      :: date format: dd.mm.year
    cli-count tags (all)                        :: show tags (with all info)
    cli-count help                              :: examples on github
    """

def write(line):
    """ Append given line to FILE
    """
    with open(FILE, "a") as f:
        f.write(line)

def now():
    """ Return formatted now time. Example: Sun/17.09.2017/09:00:22
    """
    return datetime.now().strftime("%a/%d.%m.%Y/%H:%M:%S")

def new(action=None, tag_name=None, start_value=None):
    """ Create new tag and assign a start value
    """
    if tag_name is None:
        log.error("Missing tag name")
        return
    if start_value is None:
        start_value = 0
    line = '{} {} {} {} \n'.format(
        now(), ACTION_NEW, tag_name, str(start_value))
    write(line)
    log.info('{} {}'.format(ACTION_NEW, line))

def add(action=None, tag_name=None, value=None):
    """ Add value for given tag
    """
    if value is None:
        value = UNIT
    if tag_name is None:
        log.error("Missing tag name.")
        return

    # [TODO] Check if tag exist or it must be created.

    line = '{} {} {} {} \n'.format(now(), ACTION_ADD, tag_name, str(value))
    write(line)
    log.info('{} {}'.format(ACTION_ADD, line))

def total(action=None, tag_name=None, start_date=None):
    """ Show total value for a given tag
    """
    log.info("The total is...")

def list(action=None, tag_name=None, start_date=None):
    """ List records for a given tag (optional: starting from a given date)
    """
    log.info("Listing records...")
    if tag_name is not None:
        log.warning("[TODO] Implement tag filter.")
    if start_date is not None:
        log.warning("[TODO] Implement date filter.")
    with open(FILE, 'r') as f:
        lines = f.read().splitlines()
        for line in lines:
            print line

def tags(option=None):
    """ Show existing tag names. If show_all is True: show all details
    """
    log.info("Listing tags...")
    tags = []
    show_all = True if option == OPTION_ALL else False

    with open(FILE, 'r') as f:
        lines = f.read().splitlines()
        for line in lines:
            parts = line.split(" ")
            if parts[1] == ACTION_NEW:
                if show_all is True:
                    print line
                tags.append(parts[2])
    print tags

def create_file_if_missing():
    """ All records are added in this file. Make sure it exists.
    """
    try:
        f = open(FILE, 'r')
    except IOError:
        log.warning("Missing {} file.".format(FILE))
        f = open(FILE, 'w')
        log.info("Created {} file used to store everything.".format(FILE))

    f.close()

def do_operations(val1=None, val2=None, val3=None):
    """ Redirect to complete an action
    """
    if val1 == "new":
        new(action=val1, tag_name=val2, start_value=val3)
    elif val1 == "add":
        add(action=val1, tag_name=val2, value=val3)
    elif val1 == "total":
        total(action=val1, tag_name=val2, start_date=val3)
    elif val1 == "list":
        list(action=val1, tag_name=val2, start_date=val3)
    elif val1 == "tags":
        tags(option=val2)
    else:
        help()

def set_log():
    """ Settings related to logging
    """
    log.setLevel(logging.INFO)
    ch = logging.StreamHandler()
    ch.setLevel(logging.INFO)
    formatter = logging.Formatter('%(levelname)s - %(message)s')
    ch.setFormatter(formatter)
    log.addHandler(ch)

def init():
    """ Initialize first.
    """
    set_log()
    create_file_if_missing()

if __name__ == "__main__":
    """ Check for ACTION, TAG_NAME and VALUE as vals
        then do related operations
    """
    try:
        val1 = sys.argv[1]
    except Exception:
        val1 = None
    try:
        val2 = sys.argv[2]
    except Exception:
        val2 = None
    try:
        val3 = sys.argv[3]
    except Exception:
        val3 = None

    init()
    do_operations(val1=val1, val2=val2, val3=val3)
<commit_msg>Skip adding values for unknown tags.<commit_after>from datetime import datetime
import logging
import sys

FILE = "cli-count.txt"
DEFAULT_UNIT = 1
DEFAULT_START_VALUE_NEW = 0
log = logging.getLogger('cli-count')
ACTION_NEW = "new"
ACTION_ADD = "add"
ACTION_TOTAL = "total"
ACTION_LIST = "list"
OPTION_ALL = "all"

def help():
    """ Quick help
    """
    print """
    cli-count new tag_name (start_value)        :: default start_value is 0
    cli-count add tag_name (value)              :: default value is 1
    cli-count total tag_name (start_date)       :: date format: dd.mm.year
    cli-count list (tag_name) (start_date)      :: date format: dd.mm.year
    cli-count tags (all)                        :: show tags (with all info)
    cli-count help                              :: examples on github [TODO]
    cli-count rename tag_name                   :: [TODO]
    """

def write(line):
    """ Append given line to FILE
    """
    with open(FILE, "a") as f:
        f.write(line)

def now():
    """ Return formatted now time. Example: Sun/17.09.2017/09:00:22
    """
    return datetime.now().strftime("%a/%d.%m.%Y/%H:%M:%S")

def new(action=None, tag_name=None, start_value=None):
    """ Create new tag and assign a start value
    """
    if tag_name is None:
        log.error("Missing tag name")
        return

    if start_value is None:
        start_value = DEFAULT_START_VALUE_NEW

    line = '{} {} {} {} \n'.format(
        now(), ACTION_NEW, tag_name, str(start_value))
    write(line)
    log.info('{} {}'.format(ACTION_NEW, line))

def add(action=None, tag_name=None, value=None):
    """ Add value for given tag
    """
    if value is None:
        value = DEFAULT_UNIT

    if tag_name is None:
        log.error("Missing tag name.")
        return

    if tag_name not in get_tags():
        log.error("Unknown tag. Please create it before using.")
        return

    line = '{} {} {} {} \n'.format(now(), ACTION_ADD, tag_name, str(value))
    write(line)
    log.info('{} {}'.format(ACTION_ADD, line))

def total(action=None, tag_name=None, start_date=None):
    """ Show total value for a given tag
    """
    log.info("The total is...")

def list(action=None, tag_name=None, start_date=None):
    """ List records for a given tag (optional: starting from a given date)
    """
    log.info("Listing records...")
    if tag_name is not None:
        log.warning("[TODO] Implement tag filter.")
    if start_date is not None:
        log.warning("[TODO] Implement date filter.")
    with open(FILE, 'r') as f:
        lines = f.read().splitlines()
        for line in lines:
            print line

def get_tags():
    """ Return the list of tags
    """
    tags = []

    with open(FILE, 'r') as f:
        lines = f.read().splitlines()
        for line in lines:
            parts = line.split(" ")
            if parts[1] == ACTION_NEW:
                tags.append(parts[2])
    return tags

def tags(option=None):
    """ Show existing tag names.  option: all, for all info
    """
    log.info("Listing tags...")
    tags = []
    show_all = True if option == OPTION_ALL else False

    with open(FILE, 'r') as f:
        lines = f.read().splitlines()
        for line in lines:
            parts = line.split(" ")
            if parts[1] == ACTION_NEW:
                if show_all is True:
                    print line
                tags.append(parts[2])
    print tags

def create_file_if_missing():
    """ All records are added in this file. Make sure it exists.
    """
    try:
        f = open(FILE, 'r')
    except IOError:
        log.warning("Missing {} file.".format(FILE))
        f = open(FILE, 'w')
        log.info("Created {} file used to store everything.".format(FILE))

    f.close()

def do_operations(val1=None, val2=None, val3=None):
    """ Redirect to complete an action
    """
    if val1 == "new":
        new(action=val1, tag_name=val2, start_value=val3)
    elif val1 == "add":
        add(action=val1, tag_name=val2, value=val3)
    elif val1 == "total":
        total(action=val1, tag_name=val2, start_date=val3)
    elif val1 == "list":
        list(action=val1, tag_name=val2, start_date=val3)
    elif val1 == "tags":
        tags(option=val2)
    else:
        help()

def set_log():
    """ Settings related to logging
    """
    log.setLevel(logging.INFO)
    ch = logging.StreamHandler()
    ch.setLevel(logging.INFO)
    formatter = logging.Formatter('%(levelname)s - %(message)s')
    ch.setFormatter(formatter)
    log.addHandler(ch)

def init():
    """ Initialize first.
    """
    set_log()
    create_file_if_missing()

if __name__ == "__main__":
    """ Check for ACTION, TAG_NAME and VALUE as vals
        then do related operations
    """
    try:
        val1 = sys.argv[1]
    except Exception:
        val1 = None
    try:
        val2 = sys.argv[2]
    except Exception:
        val2 = None
    try:
        val3 = sys.argv[3]
    except Exception:
        val3 = None

    init()
    do_operations(val1=val1, val2=val2, val3=val3)
<|endoftext|>


<commit_before>import os
import re
import subprocess
import asyncio
import shlex
import magic
from alot.buffers import EnvelopeBuffer, SearchBuffer
from alot.commands import CommandCanceled
from alot.helper import mailto_to_envelope
from alot.settings.const import settings
from alot.settings.errors import NoMatchingAccount
from notmuch2 import Thread

def pre_buffer_focus(ui, dbm, buf):
    if isinstance(buf, SearchBuffer):
        buf.rebuild()

# TODO: async
# mark current message read at github
def github_mark_read(ui, msg=None):
    if msg is None:
        msg = ui.current_buffer.get_selected_message()
    msg = msg.get_email()

    if msg.is_multipart():
        msgtext = b""
        for msgpart in msg.get_payload():
            msgtext += msgpart.get_payload(decode=True)
    else:
        msgtext = msg.get_payload(decode=True)

    r = b'src="(https://github.com/notifications/beacon/.*.gif)"'
    beacons = re.findall(r, msgtext)

    if beacons:
        subprocess.Popen(['curl', '-s', beacons[0]], stdout=open(os.devnull, 'w'))
        ui.notify('removed from github notifications:\n %s' % beacons[0])
    else:
        ui.notify('no beacon found')

# automatically mark github notifications as read
async def post_search_select(ui, cmd, dbm):
    current_msg = ui.current_buffer.get_selected_message()
    if current_msg.get_author()[1] == 'notifications@github.com':
        last_msg = list(ui.current_buffer.messagetrees())[-1]._message
        if 'unread' in last_msg.get_tags():
            github_mark_read(ui, list(ui.current_buffer.messagetrees())[-1]._message)

# command to manually fetch mails
def getmails(ui):
    ui.notify("fetchinig email..")
    subprocess.call(['pkill', '-u', os.environ['USER'], '-USR1', 'maildaemon'])

def exit():
    subprocess.call(['notmuch-backup'])

def _sorted_func(func, key):
    def wrapper(*args, **kwargs):
        result = func(*args, **kwargs)
        return sorted(result, key=key)
    return wrapper

async def post_thread_save(ui, dbm, cmd):
    # we are only concerned when we saved a single focused attachment
    if cmd.all or not cmd.path:
        return

    if magic.from_file(cmd.path).endswith(' text, with CRLF line terminators'):
        if (await ui.choice("convert Windows text file?", select='yes', cancel='no')) == 'no':
            return
        process = await asyncio.create_subprocess_exec('dos2unix', cmd.path)
        await process.wait()
    if magic.from_file(cmd.path).startswith('ISO-8859 text'):
        if (await ui.choice("convert ISO-8859 text file?", select='yes', cancel='no')) == 'no':
            return
        process = await asyncio.create_subprocess_shell('iconv -f latin1 -t utf8 {0} | sponge {0}'.format(shlex.quote(cmd.path)))
        await process.wait()

def unsubscribe_list(ui):
    """
    Unsubscribe from a mailing list.

    This hook reads the 'List-Unsubscribe' header of a mail in thread mode,
    constructs a unsubsribe-mail according to any mailto-url it finds
    and opens the new mail in an envelope buffer.
    """
    msg = ui.current_buffer.get_selected_message()
    e = msg.get_email()
    uheader = e['List-Unsubscribe']
    dtheader = e.get('Delivered-To', None)

    if uheader is not None:
        mailto_match = re.search(r'<(mailto:\S*)>', uheader)
        if mailto_match is not None:
            envelope = mailto_to_envelope(mailto_match.group(1))
            if dtheader is not None:
                envelope['From'] = dtheader
                try:
                    envelope.account = settings.account_matching_address(dtheader)
                except NoMatchingAccount:
                    msg = 'Cannot compose mail - ' \
                          'no account found for `%s`' % dtheader
                    ui.notify(msg, priority='error')
                    raise CommandCanceled()
            ui.buffer_open(EnvelopeBuffer(ui, envelope))
    else:
        ui.notify('focussed mail contains no \'List-Unsubscribe\' header',
                  'error')

def _thread_destroy(self):
    self._thread_p = None

Thread._destroy = _thread_destroy
<commit_msg>alot: drop hack<commit_after>import os
import re
import subprocess
import asyncio
import shlex
import magic
from alot.buffers import EnvelopeBuffer, SearchBuffer
from alot.commands import CommandCanceled
from alot.helper import mailto_to_envelope
from alot.settings.const import settings
from alot.settings.errors import NoMatchingAccount

def pre_buffer_focus(ui, dbm, buf):
    if isinstance(buf, SearchBuffer):
        buf.rebuild()

# TODO: async
# mark current message read at github
def github_mark_read(ui, msg=None):
    if msg is None:
        msg = ui.current_buffer.get_selected_message()
    msg = msg.get_email()

    if msg.is_multipart():
        msgtext = b""
        for msgpart in msg.get_payload():
            msgtext += msgpart.get_payload(decode=True)
    else:
        msgtext = msg.get_payload(decode=True)

    r = b'src="(https://github.com/notifications/beacon/.*.gif)"'
    beacons = re.findall(r, msgtext)

    if beacons:
        subprocess.Popen(['curl', '-s', beacons[0]], stdout=open(os.devnull, 'w'))
        ui.notify('removed from github notifications:\n %s' % beacons[0])
    else:
        ui.notify('no beacon found')

# automatically mark github notifications as read
async def post_search_select(ui, cmd, dbm):
    current_msg = ui.current_buffer.get_selected_message()
    if current_msg.get_author()[1] == 'notifications@github.com':
        last_msg = list(ui.current_buffer.messagetrees())[-1]._message
        if 'unread' in last_msg.get_tags():
            github_mark_read(ui, list(ui.current_buffer.messagetrees())[-1]._message)

# command to manually fetch mails
def getmails(ui):
    ui.notify("fetchinig email..")
    subprocess.call(['pkill', '-u', os.environ['USER'], '-USR1', 'maildaemon'])

def exit():
    subprocess.call(['notmuch-backup'])

def _sorted_func(func, key):
    def wrapper(*args, **kwargs):
        result = func(*args, **kwargs)
        return sorted(result, key=key)
    return wrapper

async def post_thread_save(ui, dbm, cmd):
    # we are only concerned when we saved a single focused attachment
    if cmd.all or not cmd.path:
        return

    if magic.from_file(cmd.path).endswith(' text, with CRLF line terminators'):
        if (await ui.choice("convert Windows text file?", select='yes', cancel='no')) == 'no':
            return
        process = await asyncio.create_subprocess_exec('dos2unix', cmd.path)
        await process.wait()
    if magic.from_file(cmd.path).startswith('ISO-8859 text'):
        if (await ui.choice("convert ISO-8859 text file?", select='yes', cancel='no')) == 'no':
            return
        process = await asyncio.create_subprocess_shell('iconv -f latin1 -t utf8 {0} | sponge {0}'.format(shlex.quote(cmd.path)))
        await process.wait()

def unsubscribe_list(ui):
    """
    Unsubscribe from a mailing list.

    This hook reads the 'List-Unsubscribe' header of a mail in thread mode,
    constructs a unsubsribe-mail according to any mailto-url it finds
    and opens the new mail in an envelope buffer.
    """
    msg = ui.current_buffer.get_selected_message()
    e = msg.get_email()
    uheader = e['List-Unsubscribe']
    dtheader = e.get('Delivered-To', None)

    if uheader is not None:
        mailto_match = re.search(r'<(mailto:\S*)>', uheader)
        if mailto_match is not None:
            envelope = mailto_to_envelope(mailto_match.group(1))
            if dtheader is not None:
                envelope['From'] = dtheader
                try:
                    envelope.account = settings.account_matching_address(dtheader)
                except NoMatchingAccount:
                    msg = 'Cannot compose mail - ' \
                          'no account found for `%s`' % dtheader
                    ui.notify(msg, priority='error')
                    raise CommandCanceled()
            ui.buffer_open(EnvelopeBuffer(ui, envelope))
    else:
        ui.notify('focussed mail contains no \'List-Unsubscribe\' header',
                  'error')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Optimal Colour Stimuli - MacAdam Limits
=======================================

Defines objects related to optimal colour stimuli computations.

See Also
--------
`Optimal Colour Stimuli - MacAdam Limits IPython Notebook
<http://nbviewer.ipython.org/github/colour-science/colour-ipython/blob/master/notebooks/volume/macadam_limits.ipynb>`_  # noqa
"""

from __future__ import division, unicode_literals

import numpy as np

from colour.models import xyY_to_XYZ
from colour.volume import ILLUMINANTS_OPTIMAL_COLOUR_STIMULI
from colour.utilities import is_scipy_installed

__author__ = 'Colour Developers'
__copyright__ = 'Copyright (C) 2013 - 2015 - Colour Developers'
__license__ = 'New BSD License - http://opensource.org/licenses/BSD-3-Clause'
__maintainer__ = 'Colour Developers'
__email__ = 'colour-science@googlegroups.com'
__status__ = 'Production'

__all__ = ['is_within_macadam_limits']

_XYZ_OPTIMAL_COLOUR_STIMULI_CACHE = {}
_XYZ_OPTIMAL_COLOUR_STIMULI_TRIANGULATIONS_CACHE = {}

def _XYZ_optimal_colour_stimuli(illuminant):
    """
    Returns given illuminant optimal colour stimuli in *CIE XYZ* tristimulus
    values and caches it if not existing.

    Parameters
    ----------
    illuminant : unicode
        Illuminant.

    Returns
    -------
    tuple
        Illuminant optimal colour stimuli.
    """

    optimal_colour_stimuli = ILLUMINANTS_OPTIMAL_COLOUR_STIMULI.get(illuminant)

    if optimal_colour_stimuli is None:
        raise KeyError(
            '"{0}" not found in factory optimal colour stimuli: "{1}".'.format(
                illuminant, sorted(ILLUMINANTS_OPTIMAL_COLOUR_STIMULI.keys())))

    cached_ocs = _XYZ_OPTIMAL_COLOUR_STIMULI_CACHE.get(illuminant)
    if cached_ocs is None:
        _XYZ_OPTIMAL_COLOUR_STIMULI_CACHE[illuminant] = cached_ocs = (
            np.array([np.ravel(xyY_to_XYZ(x) / 100)
                      for x in optimal_colour_stimuli]))
    return cached_ocs

def is_within_macadam_limits(xyY, illuminant, tolerance=None):
    """
    Returns if given *CIE xyY* colourspace array is within MacAdam limits of
    given illuminant.

    Parameters
    ----------
    xyY : array_like
        *CIE xyY* colourspace array.
    illuminant : unicode
        Illuminant.
    tolerance : numeric, optional
        Tolerance allowed in the inside-triangle check.

    Returns
    -------
    bool
        Is within MacAdam limits.

    Notes
    -----
    -   Input *CIE xyY* colourspace array is in domain [0, 1].
    -   This definition requires *scipy* to be installed.

    Examples
    --------
    >>> is_within_macadam_limits(np.array([0.3205, 0.4131, 0.51]), 'A')
    array(True, dtype=bool)
    >>> a = np.array([[0.3205, 0.4131, 0.51],
    ...               [0.0005, 0.0031, 0.001]])
    >>> is_within_macadam_limits(a, 'A')
    array([ True, False], dtype=bool)
    """

    if is_scipy_installed(raise_exception=True):
        from scipy.spatial import Delaunay

        optimal_colour_stimuli = _XYZ_optimal_colour_stimuli(illuminant)
        triangulation = _XYZ_OPTIMAL_COLOUR_STIMULI_TRIANGULATIONS_CACHE.get(
            illuminant)
        if triangulation is None:
            _XYZ_OPTIMAL_COLOUR_STIMULI_TRIANGULATIONS_CACHE[illuminant] = \
                triangulation = Delaunay(optimal_colour_stimuli)

        simplex = triangulation.find_simplex(xyY_to_XYZ(xyY), tol=tolerance)
        simplex = np.where(simplex >= 0, True, False)

        return simplex
<commit_msg>Improve "colour.is_within_macadam_limits" definition.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Optimal Colour Stimuli - MacAdam Limits
=======================================

Defines objects related to optimal colour stimuli computations.

See Also
--------
`Optimal Colour Stimuli - MacAdam Limits IPython Notebook
<http://nbviewer.ipython.org/github/colour-science/colour-ipython/blob/master/notebooks/volume/macadam_limits.ipynb>`_  # noqa
"""

from __future__ import division, unicode_literals

import numpy as np

from colour.models import xyY_to_XYZ
from colour.volume import ILLUMINANTS_OPTIMAL_COLOUR_STIMULI
from colour.utilities import is_scipy_installed

__author__ = 'Colour Developers'
__copyright__ = 'Copyright (C) 2013 - 2015 - Colour Developers'
__license__ = 'New BSD License - http://opensource.org/licenses/BSD-3-Clause'
__maintainer__ = 'Colour Developers'
__email__ = 'colour-science@googlegroups.com'
__status__ = 'Production'

__all__ = ['is_within_macadam_limits']

_XYZ_OPTIMAL_COLOUR_STIMULI_CACHE = {}
_XYZ_OPTIMAL_COLOUR_STIMULI_TRIANGULATIONS_CACHE = {}

def _XYZ_optimal_colour_stimuli(illuminant):
    """
    Returns given illuminant optimal colour stimuli in *CIE XYZ* tristimulus
    values and caches it if not existing.

    Parameters
    ----------
    illuminant : unicode
        Illuminant.

    Returns
    -------
    tuple
        Illuminant optimal colour stimuli.
    """

    optimal_colour_stimuli = ILLUMINANTS_OPTIMAL_COLOUR_STIMULI.get(illuminant)

    if optimal_colour_stimuli is None:
        raise KeyError(
            '"{0}" not found in factory optimal colour stimuli: "{1}".'.format(
                illuminant, sorted(ILLUMINANTS_OPTIMAL_COLOUR_STIMULI.keys())))

    cached_ocs = _XYZ_OPTIMAL_COLOUR_STIMULI_CACHE.get(illuminant)
    if cached_ocs is None:
        _XYZ_OPTIMAL_COLOUR_STIMULI_CACHE[illuminant] = cached_ocs = (
            xyY_to_XYZ(optimal_colour_stimuli) / 100)
    return cached_ocs

def is_within_macadam_limits(xyY, illuminant, tolerance=None):
    """
    Returns if given *CIE xyY* colourspace array is within MacAdam limits of
    given illuminant.

    Parameters
    ----------
    xyY : array_like
        *CIE xyY* colourspace array.
    illuminant : unicode
        Illuminant.
    tolerance : numeric, optional
        Tolerance allowed in the inside-triangle check.

    Returns
    -------
    bool
        Is within MacAdam limits.

    Notes
    -----
    -   Input *CIE xyY* colourspace array is in domain [0, 1].
    -   This definition requires *scipy* to be installed.

    Examples
    --------
    >>> is_within_macadam_limits(np.array([0.3205, 0.4131, 0.51]), 'A')
    array(True, dtype=bool)
    >>> a = np.array([[0.3205, 0.4131, 0.51],
    ...               [0.0005, 0.0031, 0.001]])
    >>> is_within_macadam_limits(a, 'A')
    array([ True, False], dtype=bool)
    """

    if is_scipy_installed(raise_exception=True):
        from scipy.spatial import Delaunay

        optimal_colour_stimuli = _XYZ_optimal_colour_stimuli(illuminant)
        triangulation = _XYZ_OPTIMAL_COLOUR_STIMULI_TRIANGULATIONS_CACHE.get(
            illuminant)
        if triangulation is None:
            _XYZ_OPTIMAL_COLOUR_STIMULI_TRIANGULATIONS_CACHE[illuminant] = \
                triangulation = Delaunay(optimal_colour_stimuli)

        simplex = triangulation.find_simplex(xyY_to_XYZ(xyY), tol=tolerance)
        simplex = np.where(simplex >= 0, True, False)

        return simplex
<|endoftext|>


<commit_before>#!/usr/bin/python3
VERSION = '1.0.0'

"""The ok.py script runs tests, checks for updates, and saves your work.

Common uses:
  python3 ok.py          Run unlocked tests (and save your work).
  python3 ok.py -u       Unlock new tests interactively.
  python3 ok.py -h       Display full help documentation.

This script will search the current directory for test files. Make sure that
ok.py appears in the same directory as the assignment you wish to test.
Otherwise, use -t to specify a test file manually.
"""

# TODO(denero) Add mechanism for removing DEVELOPER INSTRUCTIONS.
DEVELOPER_INSTRUCTIONS = """

This multi-line string contains instructions for developers. It is removed
when the client is distributed to students.

This file is responsible for coordinating all communication with the ok-server.
Students should never need to modify this file.

Local and remote interactions are encapsulated as protocols.
Contributors should do the following to add a protocol to this autograder:

    1- Extend the Protocol class and implement on_start and on_interact.
    2- Add the classname of your protocol to the "protocols" list.
    3- If your protocol needs command line arguments, change parse_input.

A standard protocol lifecycle has only one round-trip communication with the
server, processed by the on_start method. If other interactions are required
outside of this lifecycle, the send_to_server function can be used to send and
receive information from the server outside of the default times. Such
communications should be limited to the body of an on_interact method.
"""

from auth import authenticate
from models import core
from urllib import request, error
import argparse
import config
import exceptions
import importlib.machinery
import json
import os
import sys
import utils

def send_to_server(messages, assignment, server, endpoint='submission'):
    """Send messages to server, along with user authentication."""
    data = {
        'assignment': assignment['name'],
        'messages': messages,
    }
    try:
        address = 'https://' + server + '/api/v1/' + endpoint
        serialized = json.dumps(data).encode(encoding='utf-8')
        # TODO(denero) Wrap in timeout (maybe use PR #51 timed execution).
        # TODO(denero) Send access token with the request
        access_token = authenticate()
        address += "?access_token=%s&client_version=%s" % (access_token, VERSION)
        req = request.Request(address)
        req.add_header("Content-Type", "application/json")
        response = request.urlopen(req, serialized)
        return json.loads(response.read().decode('utf-8'))
    except error.HTTPError as ex:
        print("Error while sending to server: {}".format(ex))
        try:
            response_json = json.loads(response)
            if response_json['status'] == 403:
                version = response_json['data']['correct_version']
                #TODO: handle updating client here
            message = response_json['message']
            indented = '\n'.join('\t' + line for line in message.split('\n'))
            print(indented)
            return {}
        except Exception as e:
            print("Couldn't connect to server")

######################
# Assignment loading #
######################

INFO_FILE = 'info.py'

def load_tests(test_dir, case_map):
    """Loads information and tests for the current assignment.

    PARAMETERS:
    test_dir -- str; a filepath to the test directory, 'tests' by default.
    case_map -- dict; a mapping of TestCase tags to TestCase classes

    RETURNS:
    assignment -- Assignment; contains information related to the
    assignment and its tests.
    """
    if not os.path.isdir(test_dir):
        raise exceptions.OkException(
                'Assignment must have a {} directory'.format(test_dir))
    info_file = os.path.join(test_dir, INFO_FILE)
    if not os.path.isfile(info_file):
        raise exceptions.OkException(
                'Directory {} must have a file called {}'.format(
                    test_dir, INFO_FILE))
    assignment = _get_info(info_file)
    _get_tests(test_dir, assignment, case_map)
    return assignment

def _get_info(filepath):
    """Loads information from an INFO file, given by the filepath.

    PARAMETERS:
    filepath -- str; filepath to an INFO file.

    RETURNS:
    dict; information contained in the INFO file.
    """
    # TODO(albert): add error handling in case no attribute info is
    # found.
    info_json = _import_module(filepath).info
    return core.Assignment.deserialize(info_json)

def _get_tests(directory, assignment, case_map):
    """Loads all tests in a tests directory and adds them to the given
    Assignment object.

    PARAMETER:
    directory  -- str; filepath to a directory that contains tests.
    assignment -- Assignment; top-level information about the
                  assignment, extracted from the info file.
    """
    test_files = os.listdir(directory)
    for file in test_files:
        if file == INFO_FILE or not file.endswith('.py'):
            continue
        path = os.path.normpath(os.path.join(directory, file))
        if os.path.isfile(path):
            try:
                test_json = _import_module(path).test
                test = core.Test.deserialize(test_json, assignment, case_map)
                assignment.add_test(test)
            except AttributeError as ex:
                # TODO(soumya): Do something here, but only for staff protocols.
                pass

def _import_module(path):
    """Attempt to load the source file at path. Returns None on failure."""
    loader = importlib.machinery.SourceFileLoader(path, path)
    test_module = loader.load_module()
    return test_module

######################
# Assignment dumping #
######################

def dump_tests(test_dir, assignment):
    """Writes an assignment into the given test directory.

    PARAMETERS:
    test_dir   -- str; filepath to the assignment's test directory.
    assignment -- dict; contains information, including Test objects,
                  for an assignment.
    """
    # TODO(albert): prettyify string formatting by using triple quotes.
    # TODO(albert): verify that assign_copy is serializable into json.
    info = utils.prettyformat(assignment.serialize())
    with open(os.path.join(test_dir, INFO_FILE), 'w') as f:
        f.write('info = ' + info)

    # TODO(albert): writing causes an error halfway, the tests
    # directory may be left in a corrupted state.
    # TODO(albert): might need to delete obsolete test files too.
    # TODO(albert): verify that test_json is serializable into json.
    for test in assignment.tests:
        test_json = utils.prettyformat(test.serialize())
        with open(os.path.join(test_dir, test.name + '.py'), 'w') as f:
            f.write('test = ' + test_json)

#####################
# Software Updating #
#####################

def get_latest_version(server, test_dir):
    """Check for the latest version of ok and update this file accordingly.
    """
    my_version = _get_info(os.path.join(test_dir, INFO_FILE)).version

    print("You are running version {0} of ok.py".format(my_version))

    # Get server version
    address = "https://" + server + "/api/v1" + "/version?name=okpy"

    req = request.Request(address)
    response = request.urlopen(req)

    full_response = json.loads(response.read())

    server_version = full_response['version']

    if server_version != my_version:
        print("A new version exists! Downloading now...")
        file_contents = full_response['file_data']
        new_file = open('ok')
        new_file.write(file_contents)

##########################
# Command-line Interface #
##########################

def parse_input():
    """Parses command line input."""
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('-q', '--question', type=str,
                        help="focus on a specific question")
    parser.add_argument('-s', '--server', type=str, default='ok-server.appspot.com',
                        help="server address")
    parser.add_argument('-t', '--tests', metavar='A', default='tests', type=str,
                        help="partial name or path to test file or directory")
    parser.add_argument('-u', '--unlock', action='store_true',
                        help="unlock tests interactively")
    parser.add_argument('-v', '--verbose', action='store_true',
                        help="print more output")
    parser.add_argument('-i', '--interactive', action='store_true',
                        help="toggles interactive mode")
    parser.add_argument('-l', '--lock', type=str,
                        help="partial name or path to test file or directory to lock")
    return parser.parse_args()

def ok_main(args):
    """Run all relevant aspects of ok.py."""
    assignment = load_tests(args.tests, config.cases)
    # TODO(soumya): uncomment this once ok.py is ready to ship, to hide
    # error messages.
    # try:
    #     assignment = load_tests(args.tests, config.cases)
    # except Exception as ex:
    #     print(ex)
    #     sys.exit(1)

    logger = sys.stdout = utils.OutputLogger()

    start_protocols = \
        [p(args, assignment, logger) for p in config.protocols.values()]
    interact_protocols = \
        [p(args, assignment, logger) for p in config.protocols.values()]

    messages = dict()
    for protocol in start_protocols:
        messages[protocol.name] = protocol.on_start()

    # TODO(denero) Send in a separate thread.
    try:
        send_to_server(messages, assignment, args.server)
    except error.URLError as ex:
        # TODO(soumya) Make a better error message
        print("Nothing was sent to the server!")

    for protocol in interact_protocols:
        protocol.on_interact()

    # TODO(denero) Print server responses.

    # TODO(albert): a premature error might prevent tests from being
    # dumped. Perhaps add this in a "finally" clause.
    dump_tests(args.tests, assignment)

if __name__ == '__main__':
    ok_main(parse_input())
<commit_msg>Add multithreading support and misc bug fixes.<commit_after>#!/usr/bin/python3
VERSION = '1.0.0'

"""The ok.py script runs tests, checks for updates, and saves your work.

Common uses:
  python3 ok.py          Run unlocked tests (and save your work).
  python3 ok.py -u       Unlock new tests interactively.
  python3 ok.py -h       Display full help documentation.

This script will search the current directory for test files. Make sure that
ok.py appears in the same directory as the assignment you wish to test.
Otherwise, use -t to specify a test file manually.
"""

# TODO(denero) Add mechanism for removing DEVELOPER INSTRUCTIONS.
DEVELOPER_INSTRUCTIONS = """

This multi-line string contains instructions for developers. It is removed
when the client is distributed to students.

This file is responsible for coordinating all communication with the ok-server.
Students should never need to modify this file.

Local and remote interactions are encapsulated as protocols.
Contributors should do the following to add a protocol to this autograder:

    1- Extend the Protocol class and implement on_start and on_interact.
    2- Add the classname of your protocol to the "protocols" list.
    3- If your protocol needs command line arguments, change parse_input.

A standard protocol lifecycle has only one round-trip communication with the
server, processed by the on_start method. If other interactions are required
outside of this lifecycle, the send_to_server function can be used to send and
receive information from the server outside of the default times. Such
communications should be limited to the body of an on_interact method.
"""

from auth import authenticate
from models import core
from urllib import request, error
import argparse
import config
import exceptions
import importlib.machinery
import json
import os
import sys
import utils
import threading

def send_to_server(messages, assignment, server, endpoint='submission'):
    """Send messages to server, along with user authentication."""
    data = {
        'assignment': assignment['name'],
        'messages': messages,
    }
    try:
        print("Sending stuff to server...")
        address = 'https://' + server + '/api/v1/' + endpoint
        serialized = json.dumps(data).encode(encoding='utf-8')
        # TODO(denero) Wrap in timeout (maybe use PR #51 timed execution).
        # TODO(denero) Send access token with the request
        access_token = authenticate()
        address += "?access_token=%s&client_version=%s" % (access_token, VERSION)
        req = request.Request(address)
        req.add_header("Content-Type", "application/json")
        response = request.urlopen(req, serialized)
        return json.loads(response.read().decode('utf-8'))
    except error.HTTPError as ex:
        print("Error while sending to server: {}".format(ex))
        try:
            response_json = json.loads(response)
            if response_json['status'] == 403:
                version = response_json['data']['correct_version']
                get_latest_version(server)
            message = response_json['message']
            indented = '\n'.join('\t' + line for line in message.split('\n'))
            print(indented)
            return {}
        except Exception as e:
            print(e)
            print("Couldn't connect to server")

######################
# Assignment loading #
######################

INFO_FILE = 'info.py'

def load_tests(test_dir, case_map):
    """Loads information and tests for the current assignment.

    PARAMETERS:
    test_dir -- str; a filepath to the test directory, 'tests' by default.
    case_map -- dict; a mapping of TestCase tags to TestCase classes

    RETURNS:
    assignment -- Assignment; contains information related to the
    assignment and its tests.
    """
    if not os.path.isdir(test_dir):
        raise exceptions.OkException(
                'Assignment must have a {} directory'.format(test_dir))
    info_file = os.path.join(test_dir, INFO_FILE)
    if not os.path.isfile(info_file):
        raise exceptions.OkException(
                'Directory {} must have a file called {}'.format(
                    test_dir, INFO_FILE))
    assignment = _get_info(info_file)
    _get_tests(test_dir, assignment, case_map)
    return assignment

def _get_info(filepath):
    """Loads information from an INFO file, given by the filepath.

    PARAMETERS:
    filepath -- str; filepath to an INFO file.

    RETURNS:
    dict; information contained in the INFO file.
    """
    # TODO(albert): add error handling in case no attribute info is
    # found.
    info_json = _import_module(filepath).info
    return core.Assignment.deserialize(info_json)

def _get_tests(directory, assignment, case_map):
    """Loads all tests in a tests directory and adds them to the given
    Assignment object.

    PARAMETER:
    directory  -- str; filepath to a directory that contains tests.
    assignment -- Assignment; top-level information about the
                  assignment, extracted from the info file.
    """
    test_files = os.listdir(directory)
    for file in test_files:
        if file == INFO_FILE or not file.endswith('.py'):
            continue
        path = os.path.normpath(os.path.join(directory, file))
        if os.path.isfile(path):
            try:
                test_json = _import_module(path).test
                test = core.Test.deserialize(test_json, assignment, case_map)
                assignment.add_test(test)
            except AttributeError as ex:
                # TODO(soumya): Do something here, but only for staff protocols.
                pass

def _import_module(path):
    """Attempt to load the source file at path. Returns None on failure."""
    loader = importlib.machinery.SourceFileLoader(path, path)
    test_module = loader.load_module()
    return test_module

######################
# Assignment dumping #
######################

def dump_tests(test_dir, assignment):
    """Writes an assignment into the given test directory.

    PARAMETERS:
    test_dir   -- str; filepath to the assignment's test directory.
    assignment -- dict; contains information, including Test objects,
                  for an assignment.
    """
    # TODO(albert): prettyify string formatting by using triple quotes.
    # TODO(albert): verify that assign_copy is serializable into json.
    info = utils.prettyformat(assignment.serialize())
    with open(os.path.join(test_dir, INFO_FILE), 'w') as f:
        f.write('info = ' + info)

    # TODO(albert): writing causes an error halfway, the tests
    # directory may be left in a corrupted state.
    # TODO(albert): might need to delete obsolete test files too.
    # TODO(albert): verify that test_json is serializable into json.
    for test in assignment.tests:
        test_json = utils.prettyformat(test.serialize())
        with open(os.path.join(test_dir, test.name + '.py'), 'w') as f:
            f.write('test = ' + test_json)

#####################
# Software Updating #
#####################

def get_latest_version(server):
    """Check for the latest version of ok and update this file accordingly.
    """
    print("You are running version {0} of ok.py".format(VERSION))

    # Get server version
    address = "https://" + server + "/api/v1" + "/version?name=okpy"

    try:
        req = request.Request(address)
        response = request.urlopen(req)

        full_response = json.loads(response.read())

        file_contents = full_response['file_data'].decode('base64')
        new_file = open('ok')
        new_file.write(file_contents)
        new_file.close()
    except error.HTTPError as ex:
        print("Error when downloading new version")

##########################
# Command-line Interface #
##########################

def parse_input():
    """Parses command line input."""
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument('-q', '--question', type=str,
                        help="focus on a specific question")
    parser.add_argument('-s', '--server', type=str, default='ok-server.appspot.com',
                        help="server address")
    parser.add_argument('-t', '--tests', metavar='A', default='tests', type=str,
                        help="partial name or path to test file or directory")
    parser.add_argument('-u', '--unlock', action='store_true',
                        help="unlock tests interactively")
    parser.add_argument('-v', '--verbose', action='store_true',
                        help="print more output")
    parser.add_argument('-i', '--interactive', action='store_true',
                        help="toggles interactive mode")
    parser.add_argument('-l', '--lock', type=str,
                        help="partial name or path to test file or directory to lock")
    return parser.parse_args()

def ok_main(args):
    """Run all relevant aspects of ok.py."""
    assignment = load_tests(args.tests, config.cases)
    # TODO(soumya): uncomment this once ok.py is ready to ship, to hide
    # error messages.
    # try:
    #     assignment = load_tests(args.tests, config.cases)
    # except Exception as ex:
    #     print(ex)
    #     sys.exit(1)

    logger = sys.stdout = utils.OutputLogger()

    start_protocols = \
        [p(args, assignment, logger) for p in config.protocols.values()]
    interact_protocols = \
        [p(args, assignment, logger) for p in config.protocols.values()]

    messages = dict()
    for protocol in start_protocols:
        messages[protocol.name] = protocol.on_start()

    try:
        server_thread = threading.Thread(target=send_to_server, args=(messages, assignment, args.server))
        server_thread.start()
    except error.URLError as ex:
        # TODO(soumya) Make a better error message
        print("Nothing was sent to the server!")

    for protocol in interact_protocols:
        protocol.on_interact()

    # TODO(denero) Print server responses.

    # TODO(albert): a premature error might prevent tests from being
    # dumped. Perhaps add this in a "finally" clause.
    dump_tests(args.tests, assignment)

if __name__ == '__main__':
    ok_main(parse_input())
<|endoftext|>