<commit_before>"""Dependency utilities

Authors:

* Min RK
"""
#-----------------------------------------------------------------------------
#  Copyright (C) 2013  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

from types import ModuleType

from IPython.parallel.client.asyncresult import AsyncResult
from IPython.parallel.error import UnmetDependency
from IPython.parallel.util import interactive
from IPython.utils import py3compat
from IPython.utils.py3compat import string_types
from IPython.utils.pickleutil import can, uncan

class depend(object):
    """Dependency decorator, for use with tasks.

    `@depend` lets you define a function for engine dependencies
    just like you use `apply` for tasks.

    Examples
    --------
    ::

        @depend(df, a,b, c=5)
        def f(m,n,p)

        view.apply(f, 1,2,3)

    will call df(a,b,c=5) on the engine, and if it returns False or
    raises an UnmetDependency error, then the task will not be run
    and another engine will be tried.
    """
    def __init__(self, f, *args, **kwargs):
        self.f = f
        self.args = args
        self.kwargs = kwargs

    def __call__(self, f):
        return dependent(f, self.f, *self.args, **self.kwargs)

class dependent(object):
    """A function that depends on another function.
    This is an object to prevent the closure used
    in traditional decorators, which are not picklable.
    """

    def __init__(self, f, df, *dargs, **dkwargs):
        self.f = f
        name = getattr(f, '__name__', 'f')
        if py3compat.PY3:
            self.__name__ = name
        else:
            self.func_name = name
        self.df = df
        self.dargs = dargs
        self.dkwargs = dkwargs

    def check_dependency(self):
        if self.df(*self.dargs, **self.dkwargs) is False:
            raise UnmetDependency()

    def __call__(self, *args, **kwargs):
        return self.f(*args, **kwargs)

    if not py3compat.PY3:
        @property
        def __name__(self):
            return self.func_name

@interactive
def _require(*modules, **mapping):
    """Helper for @require decorator."""
    from IPython.parallel.error import UnmetDependency
    from IPython.utils.pickleutil import uncan
    user_ns = globals()
    for name in modules:
        try:
            exec('import %s' % name, user_ns)
        except ImportError:
            raise UnmetDependency(name)

    for name, cobj in mapping.items():
        user_ns[name] = uncan(cobj, user_ns)
    return True

def require(*objects, **mapping):
    """Simple decorator for requiring local objects and modules to be available
    when the decorated function is called on the engine.

    Modules specified by name or passed directly will be imported
    prior to calling the decorated function.

    Objects other than modules will be pushed as a part of the task.
    Functions can be passed positionally,
    and will be pushed to the engine with their __name__.
    Other objects can be passed by keyword arg.

    Examples::

        In [1]: @require('numpy')
           ...: def norm(a):
           ...:     return numpy.linalg.norm(a,2)

        In [2]: foo = lambda x: x*x
        In [3]: @require(foo)
           ...: def bar(a):
           ...:     return foo(1-a)
    """
    names = []
    for obj in objects:
        if isinstance(obj, ModuleType):
            obj = obj.__name__

        if isinstance(obj, string_types):
            names.append(obj)
        elif hasattr(obj, '__name__'):
            mapping[obj.__name__] = obj
        else:
            raise TypeError("Objects other than modules and functions "
                "must be passed by kwarg, but got: %s" % type(obj)
            )

    for name, obj in mapping.items():
        mapping[name] = can(obj)
    return depend(_require, *names, **mapping)

class Dependency(set):
    """An object for representing a set of msg_id dependencies.

    Subclassed from set().

    Parameters
    ----------
    dependencies: list/set of msg_ids or AsyncResult objects or output of Dependency.as_dict()
        The msg_ids to depend on
    all : bool [default True]
        Whether the dependency should be considered met when *all* depending tasks have completed
        or only when *any* have been completed.
    success : bool [default True]
        Whether to consider successes as fulfilling dependencies.
    failure : bool [default False]
        Whether to consider failures as fulfilling dependencies.

    If `all=success=True` and `failure=False`, then the task will fail with an ImpossibleDependency
        as soon as the first depended-upon task fails.
    """

    all=True
    success=True
    failure=True

    def __init__(self, dependencies=[], all=True, success=True, failure=False):
        if isinstance(dependencies, dict):
            # load from dict
            all = dependencies.get('all', True)
            success = dependencies.get('success', success)
            failure = dependencies.get('failure', failure)
            dependencies = dependencies.get('dependencies', [])
        ids = []

        # extract ids from various sources:
        if isinstance(dependencies, string_types + (AsyncResult,)):
            dependencies = [dependencies]
        for d in dependencies:
            if isinstance(d, string_types):
                ids.append(d)
            elif isinstance(d, AsyncResult):
                ids.extend(d.msg_ids)
            else:
                raise TypeError("invalid dependency type: %r"%type(d))

        set.__init__(self, ids)
        self.all = all
        if not (success or failure):
            raise ValueError("Must depend on at least one of successes or failures!")
        self.success=success
        self.failure = failure

    def check(self, completed, failed=None):
        """check whether our dependencies have been met."""
        if len(self) == 0:
            return True
        against = set()
        if self.success:
            against = completed
        if failed is not None and self.failure:
            against = against.union(failed)
        if self.all:
            return self.issubset(against)
        else:
            return not self.isdisjoint(against)

    def unreachable(self, completed, failed=None):
        """return whether this dependency has become impossible."""
        if len(self) == 0:
            return False
        against = set()
        if not self.success:
            against = completed
        if failed is not None and not self.failure:
            against = against.union(failed)
        if self.all:
            return not self.isdisjoint(against)
        else:
            return self.issubset(against)

    def as_dict(self):
        """Represent this dependency as a dict. For json compatibility."""
        return dict(
            dependencies=list(self),
            all=self.all,
            success=self.success,
            failure=self.failure
        )

__all__ = ['depend', 'require', 'dependent', 'Dependency']

<commit_msg>don't use common names in require decorators<commit_after>"""Dependency utilities

Authors:

* Min RK
"""
#-----------------------------------------------------------------------------
#  Copyright (C) 2013  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

from types import ModuleType

from IPython.parallel.client.asyncresult import AsyncResult
from IPython.parallel.error import UnmetDependency
from IPython.parallel.util import interactive
from IPython.utils import py3compat
from IPython.utils.py3compat import string_types
from IPython.utils.pickleutil import can, uncan

class depend(object):
    """Dependency decorator, for use with tasks.

    `@depend` lets you define a function for engine dependencies
    just like you use `apply` for tasks.

    Examples
    --------
    ::

        @depend(df, a,b, c=5)
        def f(m,n,p)

        view.apply(f, 1,2,3)

    will call df(a,b,c=5) on the engine, and if it returns False or
    raises an UnmetDependency error, then the task will not be run
    and another engine will be tried.
    """
    def __init__(self, _wrapped_f, *args, **kwargs):
        self.f = _wrapped_f
        self.args = args
        self.kwargs = kwargs

    def __call__(self, f):
        return dependent(f, self.f, *self.args, **self.kwargs)

class dependent(object):
    """A function that depends on another function.
    This is an object to prevent the closure used
    in traditional decorators, which are not picklable.
    """

    def __init__(self, _wrapped_f, _wrapped_df, *dargs, **dkwargs):
        self.f = _wrapped_f
        name = getattr(f, '__name__', 'f')
        if py3compat.PY3:
            self.__name__ = name
        else:
            self.func_name = name
        self.df = _wrapped_df
        self.dargs = dargs
        self.dkwargs = dkwargs

    def check_dependency(self):
        if self.df(*self.dargs, **self.dkwargs) is False:
            raise UnmetDependency()

    def __call__(self, *args, **kwargs):
        return self.f(*args, **kwargs)

    if not py3compat.PY3:
        @property
        def __name__(self):
            return self.func_name

@interactive
def _require(*modules, **mapping):
    """Helper for @require decorator."""
    from IPython.parallel.error import UnmetDependency
    from IPython.utils.pickleutil import uncan
    user_ns = globals()
    for name in modules:
        try:
            exec('import %s' % name, user_ns)
        except ImportError:
            raise UnmetDependency(name)

    for name, cobj in mapping.items():
        user_ns[name] = uncan(cobj, user_ns)
    return True

def require(*objects, **mapping):
    """Simple decorator for requiring local objects and modules to be available
    when the decorated function is called on the engine.

    Modules specified by name or passed directly will be imported
    prior to calling the decorated function.

    Objects other than modules will be pushed as a part of the task.
    Functions can be passed positionally,
    and will be pushed to the engine with their __name__.
    Other objects can be passed by keyword arg.

    Examples::

        In [1]: @require('numpy')
           ...: def norm(a):
           ...:     return numpy.linalg.norm(a,2)

        In [2]: foo = lambda x: x*x
        In [3]: @require(foo)
           ...: def bar(a):
           ...:     return foo(1-a)
    """
    names = []
    for obj in objects:
        if isinstance(obj, ModuleType):
            obj = obj.__name__

        if isinstance(obj, string_types):
            names.append(obj)
        elif hasattr(obj, '__name__'):
            mapping[obj.__name__] = obj
        else:
            raise TypeError("Objects other than modules and functions "
                "must be passed by kwarg, but got: %s" % type(obj)
            )

    for name, obj in mapping.items():
        mapping[name] = can(obj)
    return depend(_require, *names, **mapping)

class Dependency(set):
    """An object for representing a set of msg_id dependencies.

    Subclassed from set().

    Parameters
    ----------
    dependencies: list/set of msg_ids or AsyncResult objects or output of Dependency.as_dict()
        The msg_ids to depend on
    all : bool [default True]
        Whether the dependency should be considered met when *all* depending tasks have completed
        or only when *any* have been completed.
    success : bool [default True]
        Whether to consider successes as fulfilling dependencies.
    failure : bool [default False]
        Whether to consider failures as fulfilling dependencies.

    If `all=success=True` and `failure=False`, then the task will fail with an ImpossibleDependency
        as soon as the first depended-upon task fails.
    """

    all=True
    success=True
    failure=True

    def __init__(self, dependencies=[], all=True, success=True, failure=False):
        if isinstance(dependencies, dict):
            # load from dict
            all = dependencies.get('all', True)
            success = dependencies.get('success', success)
            failure = dependencies.get('failure', failure)
            dependencies = dependencies.get('dependencies', [])
        ids = []

        # extract ids from various sources:
        if isinstance(dependencies, string_types + (AsyncResult,)):
            dependencies = [dependencies]
        for d in dependencies:
            if isinstance(d, string_types):
                ids.append(d)
            elif isinstance(d, AsyncResult):
                ids.extend(d.msg_ids)
            else:
                raise TypeError("invalid dependency type: %r"%type(d))

        set.__init__(self, ids)
        self.all = all
        if not (success or failure):
            raise ValueError("Must depend on at least one of successes or failures!")
        self.success=success
        self.failure = failure

    def check(self, completed, failed=None):
        """check whether our dependencies have been met."""
        if len(self) == 0:
            return True
        against = set()
        if self.success:
            against = completed
        if failed is not None and self.failure:
            against = against.union(failed)
        if self.all:
            return self.issubset(against)
        else:
            return not self.isdisjoint(against)

    def unreachable(self, completed, failed=None):
        """return whether this dependency has become impossible."""
        if len(self) == 0:
            return False
        against = set()
        if not self.success:
            against = completed
        if failed is not None and not self.failure:
            against = against.union(failed)
        if self.all:
            return not self.isdisjoint(against)
        else:
            return self.issubset(against)

    def as_dict(self):
        """Represent this dependency as a dict. For json compatibility."""
        return dict(
            dependencies=list(self),
            all=self.all,
            success=self.success,
            failure=self.failure
        )

__all__ = ['depend', 'require', 'dependent', 'Dependency']

<|endoftext|>


<commit_before>#!/usr/bin/env python2
# -*- coding: utf-8 -*-

from canopen_301_402.constants import *
from canopen_301_402.canopen_msgs.msgs import *
from canopen_301_402.canopen_301.obj import CanOpenObject

class CanOpen402(object):
    """docstring for CanOpen402"""
    def __init__(self, node):
        super(CanOpen402, self).__init__()
        self.node = node
        self.canopen = node.canopen

        self.controlword = None
        self.statusword = None
        self.modes_of_operation_set = None
        self.modes_of_operation_get = None
        self.target_position = None
        self.populate_with_defaults()

        self.state = Can402State.switch_on_disabled

        self.statusword.signal_value_updated.register(self._on_statusword_update)

    def populate_with_defaults(self):
        # init controlword and statusword
        self.node.obj_dict.objects[Can402Objects.controlword] = CanOpenObject(self.node, *Can402Objects.controlword)
        self.node.obj_dict.objects[Can402Objects.statusword] = CanOpenObject(self.node, *Can402Objects.statusword)
        self.node.obj_dict.objects[Can402Objects.modes_of_operation_set] = CanOpenObject(self.node, *Can402Objects.modes_of_operation_set)
        self.node.obj_dict.objects[Can402Objects.modes_of_operation_get] = CanOpenObject(self.node, *Can402Objects.modes_of_operation_get)
        self.node.obj_dict.objects[Can402Objects.target_position] = CanOpenObject(self.node, *Can402Objects.target_position)

        self.controlword = self.node.obj_dict.objects[Can402Objects.controlword]
        self.statusword = self.node.obj_dict.objects[Can402Objects.statusword]
        self.modes_of_operation_set = self.node.obj_dict.objects[Can402Objects.modes_of_operation_set]
        self.modes_of_operation_get = self.node.obj_dict.objects[Can402Objects.modes_of_operation_get]
        self.target_position = self.node.obj_dict.objects[Can402Objects.target_position]

        # set datatypes
        self.node.obj_dict.objects[Can402Objects.controlword].datatype_id = CanOpenBasicDatatypes.uint16
        self.node.obj_dict.objects[Can402Objects.statusword].datatype_id = CanOpenBasicDatatypes.uint16
        self.node.obj_dict.objects[Can402Objects.modes_of_operation_set].datatype_id = CanOpenBasicDatatypes.uint8
        self.node.obj_dict.objects[Can402Objects.modes_of_operation_get].datatype_id = CanOpenBasicDatatypes.uint8
        self.node.obj_dict.objects[Can402Objects.target_position].datatype_id = CanOpenBasicDatatypes.int32

    def _on_statusword_update(self):
        # todo: change state
        for state,bits in Can402StatuswordStateBits.iteritems():
            mask = Can402StatuswordStateMasks[state]
            if ((self.statusword.value & mask) & bits) == bits:
                print "changed state to", state
                self.state = state

    def set_mode(self, mode, callback_complete=None):
        '''
        @summary:
        @param mode: Can402ModesOfOperation
        @result:
        '''

        if callable(callback_complete):
            self.node.obj_dict.objects[Can402Objects.modes_of_operation_set].signal_value_updated.register_once(callback_complete)

        data = [Can402ModesOfOperationBits[mode]]

        msg = CanOpenMessageSdoWriteRequest(
            self.canopen,
            self.node.node_id,
            Can402Objects.modes_of_operation_set[0],
            Can402Objects.modes_of_operation_set[1],
            data)

        self.canopen.send_msg(msg)

    def change_state(self, command, callback_complete=None):
        '''
        @param command: Can402StateCommand
        '''

        if self.state not in Can402StateTransitions: raise Exception()
        if command not in Can402StateTransitions[self.state]: raise ValueError()
        new_state = Can402StateTransitions[self.state][command]

        state = self.controlword.value
        datatype = self.controlword.datatype

        assert datatype.identifier() == CanOpenBasicDatatypes.uint16

        bits = Can402StateCommandBits[command] # value of bits to be replaced
        mask = Can402StateCommandMask[command] # which bits shall be replaced
        # set all bits to zero that will be replaced
        state = state & (~mask & 0xFFFF)
        # replace bits
        state = state | (mask & bits & 0xFFFF)
        data = [state & 0xFF, (state >> 8) & 0xFF]

        def value_updated(raw_data):
            self.state = new_state
            if callable(callback_complete):
                callback_complete()

        self.node.obj_dict.objects[Can402Objects.controlword].signal_value_updated.register_once(value_updated)

        msg = CanOpenMessageSdoWriteRequest(
            self.canopen,
            self.node.node_id,
            Can402Objects.controlword[0],
            Can402Objects.controlword[1],
            data)

        self.canopen.send_msg(msg)

    def set_target_position(self, value, relative=False, immediatly=False):
        '''
        @summary: set new target position in
        @param value:
        @param relative:
        @param immediatly:
        @result:
        '''

        # todo check Voraussetzung: NMT-Zustand „Operational“, Antriebszustand „Operation Enabled“ und Modes of
        # Operation (0x6060) auf Profile Position Mode (1) gesetzt.

        # set target position value
        data = self.target_position.datatype.encode(value)
        msg = CanOpenMessageSdoWriteRequest(
            self.canopen,
            self.node.node_id,
            Can402Objects.target_position[0],
            Can402Objects.target_position[1],
            data)
        self.canopen.send_msg(msg)

        # notify device of new target by setting controlword
        state = self.controlword.value

        if relative:
            state |= Can402ControlwordBits.abs_rel << 1
        else:
            state &= ~(Can402ControlwordBits.abs_rel << 1)

        state |= Can402ControlwordBits.new_set_point << 1

        if immediatly:
            state |= Can402ControlwordBits.change_set_immediately << 1
        else:
            state &= ~(Can402ControlwordBits.change_set_immediately << 1)

        data = [state & 0xFF, (state >> 8) & 0xFF]

        msg = CanOpenMessageSdoWriteRequest(
            self.canopen,
            self.node.node_id,
            Can402Objects.controlword[0],
            Can402Objects.controlword[1],
            data)

        self.canopen.send_msg(msg)
<commit_msg>use correct signal<commit_after>#!/usr/bin/env python2
# -*- coding: utf-8 -*-

from canopen_301_402.constants import *
from canopen_301_402.canopen_msgs.msgs import *
from canopen_301_402.canopen_301.obj import CanOpenObject

class CanOpen402(object):
    """docstring for CanOpen402"""
    def __init__(self, node):
        super(CanOpen402, self).__init__()
        self.node = node
        self.canopen = node.canopen

        self.controlword = None
        self.statusword = None
        self.modes_of_operation_set = None
        self.modes_of_operation_get = None
        self.target_position = None
        self.populate_with_defaults()

        self.state = Can402State.switch_on_disabled

        self.statusword.signal_value_updated.register(self._on_statusword_update)

    def populate_with_defaults(self):
        # init controlword and statusword
        self.node.obj_dict.objects[Can402Objects.controlword] = CanOpenObject(self.node, *Can402Objects.controlword)
        self.node.obj_dict.objects[Can402Objects.statusword] = CanOpenObject(self.node, *Can402Objects.statusword)
        self.node.obj_dict.objects[Can402Objects.modes_of_operation_set] = CanOpenObject(self.node, *Can402Objects.modes_of_operation_set)
        self.node.obj_dict.objects[Can402Objects.modes_of_operation_get] = CanOpenObject(self.node, *Can402Objects.modes_of_operation_get)
        self.node.obj_dict.objects[Can402Objects.target_position] = CanOpenObject(self.node, *Can402Objects.target_position)

        self.controlword = self.node.obj_dict.objects[Can402Objects.controlword]
        self.statusword = self.node.obj_dict.objects[Can402Objects.statusword]
        self.modes_of_operation_set = self.node.obj_dict.objects[Can402Objects.modes_of_operation_set]
        self.modes_of_operation_get = self.node.obj_dict.objects[Can402Objects.modes_of_operation_get]
        self.target_position = self.node.obj_dict.objects[Can402Objects.target_position]

        # set datatypes
        self.node.obj_dict.objects[Can402Objects.controlword].datatype_id = CanOpenBasicDatatypes.uint16
        self.node.obj_dict.objects[Can402Objects.statusword].datatype_id = CanOpenBasicDatatypes.uint16
        self.node.obj_dict.objects[Can402Objects.modes_of_operation_set].datatype_id = CanOpenBasicDatatypes.uint8
        self.node.obj_dict.objects[Can402Objects.modes_of_operation_get].datatype_id = CanOpenBasicDatatypes.uint8
        self.node.obj_dict.objects[Can402Objects.target_position].datatype_id = CanOpenBasicDatatypes.int32

    def _on_statusword_update(self):
        # todo: change state
        for state,bits in Can402StatuswordStateBits.iteritems():
            mask = Can402StatuswordStateMasks[state]
            if ((self.statusword.value & mask) & bits) == bits:
                print "changed state to", state
                self.state = state

    def set_mode(self, mode, callback_complete=None):
        '''
        @summary:
        @param mode: Can402ModesOfOperation
        @result:
        '''

        if callable(callback_complete):
            self.node.obj_dict.objects[Can402Objects.modes_of_operation_set].signal_value_updated.register_once(callback_complete)

        data = [Can402ModesOfOperationBits[mode]]

        msg = CanOpenMessageSdoWriteRequest(
            self.canopen,
            self.node.node_id,
            Can402Objects.modes_of_operation_set[0],
            Can402Objects.modes_of_operation_set[1],
            data)

        self.canopen.send_msg(msg)

    def change_state(self, command, callback_complete=None):
        '''
        @param command: Can402StateCommand
        '''

        if self.state not in Can402StateTransitions: raise Exception()
        if command not in Can402StateTransitions[self.state]: raise ValueError()
        new_state = Can402StateTransitions[self.state][command]

        state = self.controlword.value
        datatype = self.controlword.datatype

        assert datatype.identifier() == CanOpenBasicDatatypes.uint16

        bits = Can402StateCommandBits[command] # value of bits to be replaced
        mask = Can402StateCommandMask[command] # which bits shall be replaced
        # set all bits to zero that will be replaced
        state = state & (~mask & 0xFFFF)
        # replace bits
        state = state | (mask & bits & 0xFFFF)
        data = [state & 0xFF, (state >> 8) & 0xFF]

        def value_updated(*args):
            self.state = new_state
            if callable(callback_complete):
                callback_complete()

        self.node.sdo.signal_write_complete[Can402Objects.controlword].register_once(value_updated)

        msg = CanOpenMessageSdoWriteRequest(
            self.canopen,
            self.node.node_id,
            Can402Objects.controlword[0],
            Can402Objects.controlword[1],
            data)

        self.canopen.send_msg(msg)

    def set_target_position(self, value, relative=False, immediatly=False):
        '''
        @summary: set new target position in
        @param value:
        @param relative:
        @param immediatly:
        @result:
        '''

        # todo check Voraussetzung: NMT-Zustand „Operational“, Antriebszustand „Operation Enabled“ und Modes of
        # Operation (0x6060) auf Profile Position Mode (1) gesetzt.

        # set target position value
        data = self.target_position.datatype.encode(value)
        msg = CanOpenMessageSdoWriteRequest(
            self.canopen,
            self.node.node_id,
            Can402Objects.target_position[0],
            Can402Objects.target_position[1],
            data)
        self.canopen.send_msg(msg)

        # notify device of new target by setting controlword
        state = self.controlword.value

        if relative:
            state |= Can402ControlwordBits.abs_rel << 1
        else:
            state &= ~(Can402ControlwordBits.abs_rel << 1)

        state |= Can402ControlwordBits.new_set_point << 1

        if immediatly:
            state |= Can402ControlwordBits.change_set_immediately << 1
        else:
            state &= ~(Can402ControlwordBits.change_set_immediately << 1)

        data = [state & 0xFF, (state >> 8) & 0xFF]

        msg = CanOpenMessageSdoWriteRequest(
            self.canopen,
            self.node.node_id,
            Can402Objects.controlword[0],
            Can402Objects.controlword[1],
            data)

        self.canopen.send_msg(msg)
<|endoftext|>


<commit_before>import argparse
import yaml
import sys
from collections import OrderedDict

from . import __version__, KbdgenException, Parser, gen, logger

# TODO move into base?
generators = OrderedDict((
    ("win", gen.WindowsGenerator),
    ("osx", gen.OSXGenerator),
    ("x11", gen.XKBGenerator),
    ("svg", gen.SVGGenerator),
    ("android", gen.AndroidGenerator),
    ("ios", gen.AppleiOSGenerator),
    ("json", gen.JSONGenerator)
))

def parse_args():
    def logging_type(string):
        n = {
            "critical": 50,
            "error": 40,
            "warning": 30,
            "info": 20,
            "debug": 10,
            "trace": 5
        }.get(string, None)

        if n is None:
            raise argparse.ArgumentTypeError("Invalid logging level.")
        return n

    p = argparse.ArgumentParser(prog="kbdgen")

    p.add_argument('--version', action='version',
                   version="%(prog)s " + __version__)
    p.add_argument('--logging', type=logging_type, default=20,
                   help="Logging level")
    p.add_argument('-K', '--key', nargs="*", dest='cfg_pairs',
                   help="Key-value overrides (eg -K target.thing.foo=42)")
    p.add_argument('-D', '--dry-run', action="store_true",
                   help="Don't build, just do sanity checks.")
    p.add_argument('-R', '--release', action='store_true',
                   help="Compile in 'release' mode (where necessary).")
    p.add_argument('-G', '--global', type=argparse.FileType('r'),
                   help="Override the global.yaml file")
    p.add_argument('-r', '--repo', help='Git repo to generate output from')
    p.add_argument('-b', '--branch', default='master',
                   help='Git branch (default: master)')
    p.add_argument('-t', '--target', required=True, choices=generators.keys(),
                   help="Target output.")
    p.add_argument('-o', '--output', default=".",
                   help="Output directory (default: current working directory)")
    p.add_argument('project', help="Keyboard generation project (yaml)",
                   type=argparse.FileType('r'),
                   default=sys.stdin)
    p.add_argument('-f', '--flag', nargs="*", dest="flags",
                   help="Generator-specific flags (for debugging)",
                   default=[])

    return p.parse_args()

def main():
    args = parse_args()
    logger.setLevel(args.logging)

    try:
        project = Parser().parse(args.project, args.cfg_pairs)
    except yaml.scanner.ScannerError as e:
        logger.critical("Error parsing project:\n%s %s" % (
                str(e.problem).strip(),
                str(e.problem_mark).strip()
            ))
        return 1
    except Exception as e:
        if logger.getEffectiveLevel() < 10:
            raise e
        logger.critical(e)
        return 1

    generator = generators.get(args.target, None)

    if generator is None:
        print("Error: '%s' is not a valid target." % args.target,
                file=sys.stderr)
        print("Valid targets: %s" % ", ".join(generators),
                file=sys.stderr)
        return 1

    x = generator(project, dict(args._get_kwargs()))

    try:
        x.generate(x.output_dir)
    except KbdgenException as e:
        logger.error(e)
    except KeyboardInterrupt:
        return 0

if __name__ == "__main__":
    sys.exit(main())
<commit_msg>Handle Ctrl+C a bit more gracefully<commit_after>import argparse
import yaml
import sys
from collections import OrderedDict

from . import __version__, KbdgenException, Parser, gen, logger

# TODO move into base?
generators = OrderedDict((
    ("win", gen.WindowsGenerator),
    ("osx", gen.OSXGenerator),
    ("x11", gen.XKBGenerator),
    ("svg", gen.SVGGenerator),
    ("android", gen.AndroidGenerator),
    ("ios", gen.AppleiOSGenerator),
    ("json", gen.JSONGenerator)
))

def parse_args():
    def logging_type(string):
        n = {
            "critical": 50,
            "error": 40,
            "warning": 30,
            "info": 20,
            "debug": 10,
            "trace": 5
        }.get(string, None)

        if n is None:
            raise argparse.ArgumentTypeError("Invalid logging level.")
        return n

    p = argparse.ArgumentParser(prog="kbdgen")

    p.add_argument('--version', action='version',
                   version="%(prog)s " + __version__)
    p.add_argument('--logging', type=logging_type, default=20,
                   help="Logging level")
    p.add_argument('-K', '--key', nargs="*", dest='cfg_pairs',
                   help="Key-value overrides (eg -K target.thing.foo=42)")
    p.add_argument('-D', '--dry-run', action="store_true",
                   help="Don't build, just do sanity checks.")
    p.add_argument('-R', '--release', action='store_true',
                   help="Compile in 'release' mode (where necessary).")
    p.add_argument('-G', '--global', type=argparse.FileType('r'),
                   help="Override the global.yaml file")
    p.add_argument('-r', '--repo', help='Git repo to generate output from')
    p.add_argument('-b', '--branch', default='master',
                   help='Git branch (default: master)')
    p.add_argument('-t', '--target', required=True, choices=generators.keys(),
                   help="Target output.")
    p.add_argument('-o', '--output', default=".",
                   help="Output directory (default: current working directory)")
    p.add_argument('project', help="Keyboard generation project (yaml)",
                   type=argparse.FileType('r'),
                   default=sys.stdin)
    p.add_argument('-f', '--flag', nargs="*", dest="flags",
                   help="Generator-specific flags (for debugging)",
                   default=[])

    return p.parse_args()

def main():
    args = parse_args()
    logger.setLevel(args.logging)

    try:
        project = Parser().parse(args.project, args.cfg_pairs)
    except yaml.scanner.ScannerError as e:
        logger.critical("Error parsing project:\n%s %s" % (
                str(e.problem).strip(),
                str(e.problem_mark).strip()
            ))
        return 1
    except Exception as e:
        if logger.getEffectiveLevel() < 10:
            raise e
        logger.critical(e)
        return 1

    generator = generators.get(args.target, None)

    if generator is None:
        print("Error: '%s' is not a valid target." % args.target,
                file=sys.stderr)
        print("Valid targets: %s" % ", ".join(generators),
                file=sys.stderr)
        return 1

    x = generator(project, dict(args._get_kwargs()))

    try:
        x.generate(x.output_dir)
    except KbdgenException as e:
        logger.error(e)

if __name__ == "__main__":
    try:
        sys.exit(main())
    except KeyboardInterrupt:
        sys.exit(255)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

DATE = '2015-09-14'
XLS_NAME = 'startup.xls'
SHEET_NAME = 'STARTUP_' + ''.join(reversed(DATE.split('-')))

BUSINESS_NAME = 'denominazione'
BUSINESS_PROV = 'pv'
BUSINESS_TYPE = 'nat.giuridica'
REVENUE_CLASS = 'classe di produzione ultimo anno (1)'
EMPLOYEE_CLASS = 'classe di addetti ultimo anno (2)'
CAPITAL_CLASS = 'classe di capitale (5)'
BEGIN_DATE = u'data inizio dell\'esercizio effettivo dell\'attività'

REVENUE_CLASSES = ['A', 'B', 'C', 'D', 'E']
REVENUE_LIMITS = {
    'A': {'lower': 0, 'upper': 100000},
    'B': {'lower': 100000, 'upper': 500000},
    'C': {'lower': 500000, 'upper': 1000000},
    'D': {'lower': 1000000, 'upper': 2000000},
    'E': {'lower': 2000000, 'upper': 5000000}
}

EMPLOYEE_CLASSES = ['A', 'B', 'C', 'D', 'E']
EMPLOYEE_LIMITS = {
    'A': {'lower': 0, 'upper': 4},
    'B': {'lower': 5, 'upper': 9},
    'C': {'lower': 10, 'upper': 19},
    'D': {'lower': 20, 'upper': 49},
    'E': {'lower': 50, 'upper': 50}
}

CAPITAL_CLASSES = [1, 2, 3, 4, 5, 6, 7, 8, 9]
CAPITAL_LIMITS = {
    1: {'lower': 1, 'upper': 1},
    2: {'lower': 1, 'upper': 5000},
    3: {'lower': 5000, 'upper': 10000},
    4: {'lower': 10000, 'upper': 50000},
    5: {'lower': 50000, 'upper': 100000},
    6: {'lower': 100000, 'upper': 250000},
    7: {'lower': 250000, 'upper': 500000},
    8: {'lower': 500000, 'upper': 1000000},
    9: {'lower': 1000000, 'upper': 1000000}
}

# From https://it.wikipedia.org/wiki/Province_d%27Italia.
# FC -> FO and PU -> PS to mimic corresponding errors in the xls.
POPULATION = {
    'AG': 453677, 'AL': 440481, 'AN': 481706, 'AO': 128376, 'AR': 350022,
    'AP': 213932, 'AT': 221871, 'AV': 427194, 'BA': 1245867, 'BT': 393002,
    'BL': 213242, 'BN': 283279, 'BG': 1101458, 'BI': 185701, 'BO': 1001574,
    'BZ': 508663, 'BS': 1259626, 'BR': 403135, 'CA': 563572, 'CL': 271242,
    'CB': 225714, 'CI': 129668, 'CE': 906113, 'CT': 1090462, 'CZ': 368381,
    'CH': 388208, 'CO': 596376, 'CS': 734414, 'CR': 363918, 'KR': 174532,
    'CN': 592782, 'EN': 172237, 'FM': 178243, 'FE': 359934, 'FI': 1000324,
    'FG': 640071, 'FO': 396158, 'FR': 498204, 'GE': 883419, 'GO': 142279,
    'GR': 228309, 'IM': 222807, 'IS': 86665, 'SP': 223357, 'AQ': 298161,
    'LT': 556934, 'LE': 815488, 'LC': 340470, 'LI': 342995, 'LO': 228102,
    'LU': 394252, 'MC': 325574, 'MN': 416230, 'MS': 203697, 'MT': 199916,
    'VS': 102202, 'ME': 652891, 'MI': 3170597, 'MO': 702487, 'MB': 852539,
    'NA': 3052763, 'NO': 372109, 'NU': 160399, 'OG': 57980, 'OT': 158144,
    'OR': 165931, 'PD': 936307, 'PA': 1250026, 'PR': 443136, 'PV': 549354,
    'PG': 657682, 'PS': 366931, 'PE': 315535, 'PC': 290215, 'PI': 418210,
    'PT': 321623, 'PN': 315631, 'PZ': 375884, 'PO': 250404, 'RG': 318935,
    'RA': 410333, 'RC': 566653, 'RE': 531433, 'RI': 160570, 'RN': 330112,
    'RM': 4208740, 'RO': 248195, 'SA': 1092452, 'SS': 337100, 'SV': 287566,
    'SI': 272756, 'SR': 403769, 'SO': 183158, 'TA': 579556, 'TE': 306750,
    'TR': 228416, 'TO': 2306881, 'TP': 436311, 'TN': 530671, 'TV': 889835,
    'TS': 236650, 'UD': 541173, 'VA': 885283, 'VE': 864189, 'VB': 163123,
    'VC': 179484, 'VR': 922210, 'VV': 166370, 'VI': 871965, 'VT': 321008
}
<commit_msg>Update for new data.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

DATE = '2015-09-21'
XLS_NAME = 'startup.xls'
SHEET_NAME = 'STARTUP_' + ''.join(reversed(DATE.split('-')))

BUSINESS_NAME = 'denominazione'
BUSINESS_PROV = 'pv'
BUSINESS_TYPE = 'nat.giuridica'
REVENUE_CLASS = 'classe di produzione ultimo anno (1)'
EMPLOYEE_CLASS = 'classe di addetti ultimo anno (2)'
CAPITAL_CLASS = 'classe di capitale (5)'
BEGIN_DATE = u'data inizio dell\'esercizio effettivo dell\'attività'

REVENUE_CLASSES = ['A', 'B', 'C', 'D', 'E']
REVENUE_LIMITS = {
    'A': {'lower': 0, 'upper': 100000},
    'B': {'lower': 100000, 'upper': 500000},
    'C': {'lower': 500000, 'upper': 1000000},
    'D': {'lower': 1000000, 'upper': 2000000},
    'E': {'lower': 2000000, 'upper': 5000000}
}

EMPLOYEE_CLASSES = ['A', 'B', 'C', 'D', 'E']
EMPLOYEE_LIMITS = {
    'A': {'lower': 0, 'upper': 4},
    'B': {'lower': 5, 'upper': 9},
    'C': {'lower': 10, 'upper': 19},
    'D': {'lower': 20, 'upper': 49},
    'E': {'lower': 50, 'upper': 50}
}

CAPITAL_CLASSES = [1, 2, 3, 4, 5, 6, 7, 8, 9]
CAPITAL_LIMITS = {
    1: {'lower': 1, 'upper': 1},
    2: {'lower': 1, 'upper': 5000},
    3: {'lower': 5000, 'upper': 10000},
    4: {'lower': 10000, 'upper': 50000},
    5: {'lower': 50000, 'upper': 100000},
    6: {'lower': 100000, 'upper': 250000},
    7: {'lower': 250000, 'upper': 500000},
    8: {'lower': 500000, 'upper': 1000000},
    9: {'lower': 1000000, 'upper': 1000000}
}

# From https://it.wikipedia.org/wiki/Province_d%27Italia.
# FC -> FO and PU -> PS to mimic corresponding errors in the xls.
POPULATION = {
    'AG': 453677, 'AL': 440481, 'AN': 481706, 'AO': 128376, 'AR': 350022,
    'AP': 213932, 'AT': 221871, 'AV': 427194, 'BA': 1245867, 'BT': 393002,
    'BL': 213242, 'BN': 283279, 'BG': 1101458, 'BI': 185701, 'BO': 1001574,
    'BZ': 508663, 'BS': 1259626, 'BR': 403135, 'CA': 563572, 'CL': 271242,
    'CB': 225714, 'CI': 129668, 'CE': 906113, 'CT': 1090462, 'CZ': 368381,
    'CH': 388208, 'CO': 596376, 'CS': 734414, 'CR': 363918, 'KR': 174532,
    'CN': 592782, 'EN': 172237, 'FM': 178243, 'FE': 359934, 'FI': 1000324,
    'FG': 640071, 'FO': 396158, 'FR': 498204, 'GE': 883419, 'GO': 142279,
    'GR': 228309, 'IM': 222807, 'IS': 86665, 'SP': 223357, 'AQ': 298161,
    'LT': 556934, 'LE': 815488, 'LC': 340470, 'LI': 342995, 'LO': 228102,
    'LU': 394252, 'MC': 325574, 'MN': 416230, 'MS': 203697, 'MT': 199916,
    'VS': 102202, 'ME': 652891, 'MI': 3170597, 'MO': 702487, 'MB': 852539,
    'NA': 3052763, 'NO': 372109, 'NU': 160399, 'OG': 57980, 'OT': 158144,
    'OR': 165931, 'PD': 936307, 'PA': 1250026, 'PR': 443136, 'PV': 549354,
    'PG': 657682, 'PS': 366931, 'PE': 315535, 'PC': 290215, 'PI': 418210,
    'PT': 321623, 'PN': 315631, 'PZ': 375884, 'PO': 250404, 'RG': 318935,
    'RA': 410333, 'RC': 566653, 'RE': 531433, 'RI': 160570, 'RN': 330112,
    'RM': 4208740, 'RO': 248195, 'SA': 1092452, 'SS': 337100, 'SV': 287566,
    'SI': 272756, 'SR': 403769, 'SO': 183158, 'TA': 579556, 'TE': 306750,
    'TR': 228416, 'TO': 2306881, 'TP': 436311, 'TN': 530671, 'TV': 889835,
    'TS': 236650, 'UD': 541173, 'VA': 885283, 'VE': 864189, 'VB': 163123,
    'VC': 179484, 'VR': 922210, 'VV': 166370, 'VI': 871965, 'VT': 321008
}
<|endoftext|>


<commit_before>"""
Collection of commonly used functions for general file input
and output

To import:
module_dir = os.path.join(os.environ['HOME'], 'phd', 'modules')
sys.path.insert(0, module_dir)

Included functions:
find_duplicates   -- Find duplicates in a list
read_dates        -- Read in a list of dates
set_outfile_date  -- Take an outfile name and replace existing date with new one
standard_datetime -- Convert any arbitrary date/time to standard format: YYYY-MM-DD
write_dates       -- Write a list of dates

"""

import os, sys, pdb
from datetime import datetime
from dateutil import parser
from collections import defaultdict
import re

try:
    from git import Repo  #doesn't come standard with uvcdat install
    cwd = os.getcwd()
    repo_dir = '/'
    for directory in cwd.split('/')[1:]:
        repo_dir = os.path.join(repo_dir, directory)
        if directory == 'phd':
            break
    MODULE_HASH = Repo(repo_dir).head.commit.hexsha
except ImportError:
    MODULE_HASH = 'unknown'

## Alternative provenance tracking, if netcdf_io.py
#  was under version control directly ##
#repo_dir = os.path.abspath(os.path.dirname(__file__))
#MODULE_HASH = Repo(repo_dir).head.commit.hexsha

def find_duplicates(inlist):
    """Return list of duplicates in a list"""

    D = defaultdict(list)
    for i,item in enumerate(mylist):
        D[item].append(i)
    D = {k:v for k,v in D.items() if len(v)>1}

    return D

def get_timestamp():
    """Return time stamp that incuded the command line entry"""

    time_stamp = """%s: %s %s (Git hash: %s)""" %(datetime.now().strftime("%a %b %d %H:%M:%S %Y"), sys.executable, " ".join(sys.argv), MODULE_HASH[0:7])

    return time_stamp

def read_dates(infile):
    """Read a file of dates (one per line) and write to a list"""

    fin = open(infile, 'r')
    date_list = []
    for line in fin:
        date_list.append(line.rstrip('\n'))
    fin.close()
    date_metadata = date_list.pop(0)

    return date_list, date_metadata

def set_outfile_date(outfile, new_date):
    """Take an outfile name and replace the existing date
    (in YYYY-MM-DD format) with new_date"""

    new_dt = parser.parse(str(new_date))

    date_pattern = '([0-9]{4})-([0-9]{1,2})-([0-9]{1,2})'
    assert re.search(date_pattern, outfile), \
    """Output file must contain the date of the final timestep in the format YYYY-MM-DD"""

    return re.sub(r'([0-9]{4})-([0-9]{1,2})-([0-9]{1,2})', new_dt.strftime("%Y-%m-%d"), outfile)

def write_dates(outfile, date_list):
    """Write a list of dates to file"""

    fout = open(outfile, 'w')
    timestamp = get_timestamp()
    fout.write('# '+timestamp+'\n')
    for date in date_list:
        fout.write(date+'\n')
    fout.close()

def standard_datetime(dt):
    """Take any arbitrarty date/time and convert to the standard
    I use for all outputs: YYYY-MM-DD"""

    new_dt = parser.parse(str(dt))

    return new_dt.strftime("%Y-%m-%d")

def write_metadata(ofile, file_info=None, extra_notes=None):
    """Write a metadata output file

    Arguments:
      ofile        --  name of output image file
      file_info    --  list of file info lists: ['fname', 'global atts history']
      extra_notes  --  list containing character strings of extra information (output is one list item per line)

    """

    # Open .met file
    fname, extension = ofile.split('.')
    fout = open(fname+'.met', 'w')

    # Write the timestamp
    time_stamp = get_timestamp()
    fout.write(time_stamp+'\n \n')

    # Write the extra info
    if extra_notes:
        fout.write('Extra notes: \n \n')
	for line in extra_notes:
	    fout.write(line+'\n')

    # Write the file details
    if file_info:
        fout.write('\n')
        for ifile in file_info:
	    fname, history = ifile
	    fout.write('Input file: %s \n \n' %(fname))
	    fout.write('History: %s \n \n' %(history))

    fout.close()
<commit_msg>write_metadata can now output a file or a string<commit_after>"""
Collection of commonly used functions for general file input
and output

To import:
module_dir = os.path.join(os.environ['HOME'], 'phd', 'modules')
sys.path.insert(0, module_dir)

Included functions:
find_duplicates   -- Find duplicates in a list
read_dates        -- Read in a list of dates
set_outfile_date  -- Take an outfile name and replace existing date with new one
standard_datetime -- Convert any arbitrary date/time to standard format: YYYY-MM-DD
write_dates       -- Write a list of dates

"""

import os, sys, pdb
from datetime import datetime
from dateutil import parser
from collections import defaultdict
import re

try:
    from git import Repo  #doesn't come standard with uvcdat install
    cwd = os.getcwd()
    repo_dir = '/'
    for directory in cwd.split('/')[1:]:
        repo_dir = os.path.join(repo_dir, directory)
        if directory == 'phd':
            break
    MODULE_HASH = Repo(repo_dir).head.commit.hexsha
except ImportError:
    MODULE_HASH = 'unknown'

def find_duplicates(inlist):
    """Return list of duplicates in a list"""

    D = defaultdict(list)
    for i,item in enumerate(mylist):
        D[item].append(i)
    D = {k:v for k,v in D.items() if len(v)>1}

    return D

def get_timestamp():
    """Return time stamp that incuded the command line entry"""

    time_stamp = """%s: %s %s (Git hash: %s)""" %(datetime.now().strftime("%a %b %d %H:%M:%S %Y"), sys.executable, " ".join(sys.argv), MODULE_HASH[0:7])

    return time_stamp

def read_dates(infile):
    """Read a file of dates (one per line) and write to a list"""

    fin = open(infile, 'r')
    date_list = []
    for line in fin:
        date_list.append(line.rstrip('\n'))
    fin.close()
    date_metadata = date_list.pop(0)

    return date_list, date_metadata

def set_outfile_date(outfile, new_date):
    """Take an outfile name and replace the existing date
    (in YYYY-MM-DD format) with new_date"""

    new_dt = parser.parse(str(new_date))

    date_pattern = '([0-9]{4})-([0-9]{1,2})-([0-9]{1,2})'
    assert re.search(date_pattern, outfile), \
    """Output file must contain the date of the final timestep in the format YYYY-MM-DD"""

    return re.sub(r'([0-9]{4})-([0-9]{1,2})-([0-9]{1,2})', new_dt.strftime("%Y-%m-%d"), outfile)

def standard_datetime(dt):
    """Take any arbitrarty date/time and convert to the standard
    I use for all outputs: YYYY-MM-DD"""

    new_dt = parser.parse(str(dt))

    return new_dt.strftime("%Y-%m-%d")

def write_dates(outfile, date_list):
    """Write a list of dates to file"""

    fout = open(outfile, 'w')
    timestamp = get_timestamp()
    fout.write('# '+timestamp+'\n')
    for date in date_list:
        fout.write(date+'\n')
    fout.close()

def write_metadata(ofile=None, file_info=None, extra_notes=None):
    """Write a metadata output file

    Arguments:
      ofile        --  name of output file that we want to create a .met file
                       alongside (i.e. new file with .met extension will be created)
      file_info    --  list of file info lists: ['fname', 'global atts history']
      extra_notes  --  list containing character strings of extra information (output is one list item per line)

    """

    result = ''

    # Write the timestamp
    time_stamp = get_timestamp()
    result += time_stamp + '\n \n'

    # Write the extra info
    if extra_notes:
        result += 'Extra notes: \n \n'
        for line in extra_notes:
            result += line + '\n'

    # Write the file details
    if file_info:
        result += '\n'
        for ifile in file_info:
            fname, history = ifile
            result += 'Input file: %s \n \n' %(fname)
            result += 'History: %s \n \n' %(history)

    # Create outfile or return string
    if ofile:
        fname, extension = ofile.split('.')
        fout = open(fname+'.met', 'w')
        fout.write(result)
        fout.close()
    else:
        return result<|endoftext|>


<commit_before># Module:   rss
# Date:     25th March 2007
# Author:   James Mills, prologic at shortcircuit dot net dot au

"""RSS

This plugin provides RSS aggregation functionality to the
user allowing the user to set personal and public RSS feeds
to be retrieved at regular intervals and messages to them.
"""

__version__ = "0.0.8"
__author__ = "James Mills, prologic at shortcircuit dot net dot au"

import os
import cPickle as pickle
from time import mktime, time

import feedparser

from circuits import Event, Timer
from circuits.lib.irc import Message
from pymills.utils import notags, decodeHTML

from kdb.plugin import BasePlugin

class RSSTick(Event): pass

class Feed(object):

    def __init__(self, url, target, interval=60):
        self.url = url
        self.target = target
        self.interval = interval

        self.next = time() + (self.interval * 60)
        self.entries = []
        self.title = ""
        self.link = ""

    def checkTime(self):
        if time() > self.next:
            self.next = time() + (self.interval * 60)
            return True
        else:
            return False

    def getItems(self):
        d = feedparser.parse(self.url)

        if self.title == "" and self.link == "":
            self.title = d.feed.title
            self.link = d.feed.link

        new = []
        for v in d.entries:
            e = {
                    "time": mktime(v.updated_parsed),
                    "title": v.title,
                    "summary": decodeHTML(
                        notags(v.summary).strip().split("\n")[0]),
                    "link": v.links[0].href
                    }

            if e not in self.entries:
                self.entries.append(e)
                new.append(e)

        if not new == []:
            s = []
            s.append("RSS: %s (%s)" % (
                self.title, self.link))
            for e in new[:3]:
                x = sum([
                    len(e["title"]),
                    len(e["summary"]),
                    len(e["link"])])
                if x > 450:
                    y = sum([
                        len(e["title"]),
                        len(e["link"])])
                    s.append(" * %s: %s ... <%s>" % (
                        e["title"],
                        e["summary"][:(450 - y)],
                        e["link"]))
                else:
                    s.append(" * %s: %s <%s>" % (
                        e["title"], e["summary"], e["link"]))
            return s
        else:
            return []

class RSS(BasePlugin):

    """RSS Aggregator plugin

    Provides RSS aggregation functions allowing you to
    create public or private RSS feeds that are downlaoded
    at regular intervals and checked and display.
    See: commands rss
    """

    def __init__(self, *args, **kwargs):
        super(RSS, self).__init__(*args, **kwargs)

        filename = os.path.join(self.env.path, "rss.bin")
        if os.path.exists(filename):
            fd = open(filename, "rb")
            try:
                self.entities = pickle.load(fd)
            except Exception, e:
                self.env.log.debug(
                        "ERROR: Could not load rss data from '%s'" \
                                " -> %s" % (filename, str(e)))
                self.entities = {}
            fd.close()
        else:
            self.entities = {}

        self.manager += Timer(60, RSSTick(), "rsstick", persist=True)

    def cleanup(self):
        filename = os.path.join(self.env.path, "rss.bin")
        fd = open(filename, "wb")
        pickle.dump(self.entities, fd)
        fd.close()

    @listener("rsstick")
    def onRSSTICK(self):
        for entity in self.entities:
            for f in self.entities[entity]:
                if f.checkTime():
                    for line in f.getItems():
                        self.push(Message(f.target, line), "PRIVMSG")

    def cmdRADD(self, source, url, interval="60"):
        """Add a new RSS feed to be checked at the given interval.
        Intervan is in minutes.

        Syntax: RADD <url> [<interval>]
        """

        try:
            interval = int(interval)
        except Exception, e:
            return "ERROR: Bad interval given '%s' --> %s" % (
                    interval, str(e))

        if interval > 60:
            return "ERROR: Given interval '%s' is too big. " \
                    "Interval is in minutes." % interval

        f = Feed(url, source, interval)
        if self.entities.has_key(source):
            self.entities[source].append(f)
        else:
            self.entities[source] = [f]

        return f.getItems()

    def cmdRDEL(self, source, n):
        """Delete an RSS feed.

        Syntax: RDEL <n>
        """

        if self.entities.has_key(source):

            try:
                n = int(n)
            except Exception, e:
                return "ERROR: Invalid feed no. given '%s' -> %s" % (
                        n, str(e))

            if n > 0 and n <= len(self.entities[source]):
                del self.entities[source][(n - 1)]
                msg = "Feed %d deleted." % n
            else:
                msg = "Given feed does not exist."
        else:
            msg = "No feeds to delete."

        return msg

    def cmdREAD(self, source, n):
        """Read an RSS feed.

        Syntax: READ <n>
        """

        if self.entities.has_key(source):

            try:
                n = int(n)
            except Exception, e:
                return "ERROR: Invalid feed no. given '%s' -> %s" % (
                        n, str(e))

            if n > 0 and n <= len(self.entities[source]):
                f = self.entities[source][(n - 1)]
                msg = f.getItems()
            else:
                msg = "Given feed does not exist."
        else:
            msg = "No feeds to read."

        return msg

    def cmdRLIST(self, source):
        """List all active RSS feeds.

        Syntax: RLIST
        """

        if self.entities.has_key(source):
            msg = ["RSS Feeds (%s):" % source]
            for i, f in enumerate(self.entities[source]):
                msg.append(" %d. %s (%s)/%dmins (next: %dmins)" % (
                    (i + 1), f.title, f.link,
                    f.interval, (f.next - time()) / 60))
        else:
            msg = "No feeds available."

        return msg
<commit_msg>plugins.rss: Renamed RSSTick Event to CheckFeeks.<commit_after># Module:   rss
# Date:     25th March 2007
# Author:   James Mills, prologic at shortcircuit dot net dot au

"""RSS

This plugin provides RSS aggregation functionality to the
user allowing the user to set personal and public RSS feeds
to be retrieved at regular intervals and messages to them.
"""

__version__ = "0.0.8"
__author__ = "James Mills, prologic at shortcircuit dot net dot au"

import os
import cPickle as pickle
from time import mktime, time

import feedparser

from circuits import Event, Timer
from circuits.lib.irc import Message
from pymills.utils import notags, decodeHTML

from kdb.plugin import BasePlugin

class CheckFeeds(Event): pass

class Feed(object):

    def __init__(self, url, target, interval=60):
        self.url = url
        self.target = target
        self.interval = interval

        self.next = time() + (self.interval * 60)
        self.entries = []
        self.title = ""
        self.link = ""

    def checkTime(self):
        if time() > self.next:
            self.next = time() + (self.interval * 60)
            return True
        else:
            return False

    def getItems(self):
        d = feedparser.parse(self.url)

        if self.title == "" and self.link == "":
            self.title = d.feed.title
            self.link = d.feed.link

        new = []
        for v in d.entries:
            e = {
                    "time": mktime(v.updated_parsed),
                    "title": v.title,
                    "summary": decodeHTML(
                        notags(v.summary).strip().split("\n")[0]),
                    "link": v.links[0].href
                    }

            if e not in self.entries:
                self.entries.append(e)
                new.append(e)

        if not new == []:
            s = []
            s.append("RSS: %s (%s)" % (
                self.title, self.link))
            for e in new[:3]:
                x = sum([
                    len(e["title"]),
                    len(e["summary"]),
                    len(e["link"])])
                if x > 450:
                    y = sum([
                        len(e["title"]),
                        len(e["link"])])
                    s.append(" * %s: %s ... <%s>" % (
                        e["title"],
                        e["summary"][:(450 - y)],
                        e["link"]))
                else:
                    s.append(" * %s: %s <%s>" % (
                        e["title"], e["summary"], e["link"]))
            return s
        else:
            return []

class RSS(BasePlugin):

    """RSS Aggregator plugin

    Provides RSS aggregation functions allowing you to
    create public or private RSS feeds that are downlaoded
    at regular intervals and checked and display.
    See: commands rss
    """

    def __init__(self, *args, **kwargs):
        super(RSS, self).__init__(*args, **kwargs)

        filename = os.path.join(self.env.path, "rss.bin")
        if os.path.exists(filename):
            fd = open(filename, "rb")
            try:
                self.entities = pickle.load(fd)
            except Exception, e:
                self.env.log.debug(
                        "ERROR: Could not load rss data from '%s'" \
                                " -> %s" % (filename, str(e)))
                self.entities = {}
            fd.close()
        else:
            self.entities = {}

        self += Timer(60, CheckFeeds(), "checkfeeds", persist=True)

    def cleanup(self):
        filename = os.path.join(self.env.path, "rss.bin")
        fd = open(filename, "wb")
        pickle.dump(self.entities, fd)
        fd.close()

    def checkfeeds(self):
        for entity in self.entities:
            for f in self.entities[entity]:
                if f.checkTime():
                    for line in f.getItems():
                        self.push(Message(f.target, line), "PRIVMSG")

    def cmdRADD(self, source, url, interval="60"):
        """Add a new RSS feed to be checked at the given interval.
        Intervan is in minutes.

        Syntax: RADD <url> [<interval>]
        """

        try:
            interval = int(interval)
        except Exception, e:
            return "ERROR: Bad interval given '%s' --> %s" % (
                    interval, str(e))

        if interval > 60:
            return "ERROR: Given interval '%s' is too big. " \
                    "Interval is in minutes." % interval

        f = Feed(url, source, interval)
        if self.entities.has_key(source):
            self.entities[source].append(f)
        else:
            self.entities[source] = [f]

        return f.getItems()

    def cmdRDEL(self, source, n):
        """Delete an RSS feed.

        Syntax: RDEL <n>
        """

        if self.entities.has_key(source):

            try:
                n = int(n)
            except Exception, e:
                return "ERROR: Invalid feed no. given '%s' -> %s" % (
                        n, str(e))

            if n > 0 and n <= len(self.entities[source]):
                del self.entities[source][(n - 1)]
                msg = "Feed %d deleted." % n
            else:
                msg = "Given feed does not exist."
        else:
            msg = "No feeds to delete."

        return msg

    def cmdREAD(self, source, n):
        """Read an RSS feed.

        Syntax: READ <n>
        """

        if self.entities.has_key(source):

            try:
                n = int(n)
            except Exception, e:
                return "ERROR: Invalid feed no. given '%s' -> %s" % (
                        n, str(e))

            if n > 0 and n <= len(self.entities[source]):
                f = self.entities[source][(n - 1)]
                msg = f.getItems()
            else:
                msg = "Given feed does not exist."
        else:
            msg = "No feeds to read."

        return msg

    def cmdRLIST(self, source):
        """List all active RSS feeds.

        Syntax: RLIST
        """

        if self.entities.has_key(source):
            msg = ["RSS Feeds (%s):" % source]
            for i, f in enumerate(self.entities[source]):
                msg.append(" %d. %s (%s)/%dmins (next: %dmins)" % (
                    (i + 1), f.title, f.link,
                    f.interval, (f.next - time()) / 60))
        else:
            msg = "No feeds available."

        return msg
<|endoftext|>


<commit_before>import shutil
import os
from subprocess import check_output
from git import Blob
from git import Repo
from itertools import count
from itertools import izip

from multiprocessing import (
                                Pool,
                                cpu_count
                            )

def commit_syntax_trees_worker(repo_dir, org_repo_dir, changed_commits, syntax_trees_dir):
    repo = Repo(repo_dir)
    org_repo = Repo(org_repo_dir)
    committer = SyntaxTreesCommitter(org_repo, syntax_trees_dir)
    committer.commit_syntax_trees(repo, changed_commits)

class SyntaxTreesCommitter:
    def __init__(self, org_repo, syntax_trees_dir):
        self.org_repo = org_repo
        self.syntax_trees_dir = syntax_trees_dir

    def remove_files(self, repo, index, removed_files):
        kwargs = {"r" : True}
        if len(removed_files) == 0:
            return
        index.remove(removed_files, **kwargs)
        index.write()

        for p in removed_files:
            shutil.rmtree(os.path.join(repo.working_dir, p))

    def add_files(self, repo, index, added_files):
        if len(added_files) == 0:
            return

        for path, hexsha in added_files.items():
            src = os.path.join(self.syntax_trees_dir, hexsha)
            dst = os.path.join(repo.working_dir, path)
            shutil.copytree(src, dst)

        repo.git.add(added_files.keys())
        index.update()

    def is_completed_parse(self, blob):
        path = os.path.join(self.syntax_trees_dir, blob.hexsha)
        cmd = ['find', path, '-type', 'f']
        output = check_output(cmd)
        if len(output) == 0:
            #print 'Interface?:', blob.path
            pass
        return len(output) > 0

    def construct_from_commit(self, repo, commit):
        added_files = {}
        for entry in commit.tree.traverse():
            if not isinstance(entry, Blob):
                continue

            if not entry.name.endswith('.java'):
                continue
            if self.is_completed_parse(entry):
                added_files[entry.path] = entry.hexsha

        self.add_files(repo, repo.index, added_files)
        repo.index.commit(commit.hexsha)

    def commit_syntax_trees(self, repo, changed_commits):
        start_commit = self.org_repo.commit(changed_commits.pop(0))
        self.construct_from_commit(repo, start_commit)

        total_commits = len(changed_commits)
        for (num, commit_hexsha) in izip(count(1), changed_commits):
            print '[%d/%d] commit to: %s' % (num, total_commits, repo.git_dir)
            commit = self.org_repo.commit(commit_hexsha)
            self.apply_change(repo, commit)

    def apply_change(self, new_repo, commit):
        assert len(commit.parents) < 2 # Not support branched repository

        index = new_repo.index
        removed_files = []
        added_files = {}
        for p in commit.parents:
            for diff in p.diff(commit):
                if(diff.a_blob):
                    if not diff.a_blob.name.endswith(".java"):
                        continue
                    if self.is_completed_parse(diff.a_blob):
                        removed_files.append(diff.a_blob.path)

                if(diff.b_blob):
                    if not diff.b_blob.name.endswith(".java"):
                        continue
                    if self.is_completed_parse(diff.b_blob):
                        added_files[diff.b_blob.path] = diff.b_blob.hexsha

            self.remove_files(new_repo, index, removed_files)
            self.add_files(new_repo, index, added_files)

        if len(index.diff(None, staged=True)):
            index.commit(commit.hexsha)

class SyntaxTreesParallelCommitter:
    def __init__(self, syntax_trees_dir, org_repo_dir, processes=None):
        self.syntax_trees_dir = syntax_trees_dir
        self.org_repo_dir = org_repo_dir
        self.processes = processes if processes else cpu_count()
        self.pool = Pool(self.processes)
        self.closed = False

    def commit_syntax_trees_parallel(self, repo_dir, changed_commits):
        args = [repo_dir, self.org_repo_dir, changed_commits, self.syntax_trees_dir]
        if(self.closed):
            self.pool = Pool(self.processes)
            self.closed = False

        self.pool.apply_async(commit_syntax_trees_worker, args=args)

    def join(self):
        self.pool.close()
        self.closed = True
        self.pool.join()
        self.pool = None
<commit_msg>フォルダ階層を正規化<commit_after>import shutil
import os
from subprocess import check_output
from git import Blob
from git import Repo
from itertools import count
from itertools import izip

from multiprocessing import (
                                Pool,
                                cpu_count
                            )

def commit_syntax_trees_worker(repo_dir, org_repo_dir, changed_commits, syntax_trees_dir):
    repo = Repo(repo_dir)
    org_repo = Repo(org_repo_dir)
    committer = SyntaxTreesCommitter(org_repo, syntax_trees_dir)
    committer.commit_syntax_trees(repo, changed_commits)

class SyntaxTreesCommitter:
    def __init__(self, org_repo, syntax_trees_dir):
        self.org_repo = org_repo
        self.syntax_trees_dir = syntax_trees_dir

    def remove_files(self, repo, index, removed_files):
        kwargs = {"r" : True}
        if len(removed_files) == 0:
            return
        index.remove(removed_files, **kwargs)
        index.write()

        for p in removed_files:
            shutil.rmtree(os.path.join(repo.working_dir, p))

    def add_files(self, repo, index, added_files):
        if len(added_files) == 0:
            return

        for path, hexsha in added_files.items():
            src = os.path.join(self.syntax_trees_dir, hexsha)
            dst = os.path.join(repo.working_dir, path)
            shutil.copytree(src, dst)

        repo.git.add(added_files.keys())
        index.update()

    def is_completed_parse(self, blob):
        path = os.path.join(self.syntax_trees_dir, blob.hexsha)
        cmd = ['find', path, '-type', 'f']
        output = check_output(cmd)
        if len(output) == 0:
            #print 'Interface?:', blob.path
            pass
        return len(output) > 0

    def construct_from_commit(self, repo, commit):
        added_files = {}
        for entry in commit.tree.traverse():
            if not isinstance(entry, Blob):
                continue

            if not entry.name.endswith('.java'):
                continue
            if self.is_completed_parse(entry):
                added_files[self.get_normalized_path(entry.path)] = entry.hexsha

        self.add_files(repo, repo.index, added_files)
        repo.index.commit(commit.hexsha)

    def commit_syntax_trees(self, repo, changed_commits):
        start_commit = self.org_repo.commit(changed_commits.pop(0))
        self.construct_from_commit(repo, start_commit)

        total_commits = len(changed_commits)
        for (num, commit_hexsha) in izip(count(1), changed_commits):
            print '[%d/%d] commit to: %s' % (num, total_commits, repo.git_dir)
            commit = self.org_repo.commit(commit_hexsha)
            self.apply_change(repo, commit)

    def apply_change(self, new_repo, commit):
        assert len(commit.parents) < 2 # Not support branched repository

        index = new_repo.index
        removed_files = []
        added_files = {}
        for p in commit.parents:
            for diff in p.diff(commit):
                if(diff.a_blob):
                    if not diff.a_blob.name.endswith(".java"):
                        continue
                    if self.is_completed_parse(diff.a_blob):
                        removed_files.append(self.get_normalized_path(diff.a_blob.path))

                if(diff.b_blob):
                    if not diff.b_blob.name.endswith(".java"):
                        continue
                    if self.is_completed_parse(diff.b_blob):
                        added_files[self.get_normalized_path(diff.b_blob.path)] = diff.b_blob.hexsha

            self.remove_files(new_repo, index, removed_files)
            self.add_files(new_repo, index, added_files)

        if len(index.diff(None, staged=True)):
            index.commit(commit.hexsha)

    def get_normalized_path(self, path):
        return path.replace("/", "_")

class SyntaxTreesParallelCommitter:
    def __init__(self, syntax_trees_dir, org_repo_dir, processes=None):
        self.syntax_trees_dir = syntax_trees_dir
        self.org_repo_dir = org_repo_dir
        self.processes = processes if processes else cpu_count()
        self.pool = Pool(self.processes)
        self.closed = False

    def commit_syntax_trees_parallel(self, repo_dir, changed_commits):
        args = [repo_dir, self.org_repo_dir, changed_commits, self.syntax_trees_dir]
        if(self.closed):
            self.pool = Pool(self.processes)
            self.closed = False

        self.pool.apply_async(commit_syntax_trees_worker, args=args)

    def join(self):
        self.pool.close()
        self.closed = True
        self.pool.join()
        self.pool = None
<|endoftext|>


<commit_before>from __future__ import absolute_import
import os
from tempfile import mkdtemp
from shutil import rmtree
from itertools import count, izip
from git.repo import Repo
from git.objects import Blob
from kenja.parser import ParserExecutor
from kenja.git.util import get_reversed_topological_ordered_commits
from kenja.committer import SyntaxTreesCommitter
from logging import getLogger

logger = getLogger(__name__)

class HistorageConverter:
    parser_jar_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'lib', 'java-parser.jar')

    def __init__(self, org_git_repo_dir, historage_dir, syntax_trees_dir=None):
        if org_git_repo_dir:
            self.org_repo = Repo(org_git_repo_dir)

        self.language = '.java'
        self.check_and_make_working_dir(historage_dir)
        self.historage_dir = historage_dir

        self.use_tempdir = syntax_trees_dir is None
        if self.use_tempdir:
            self.syntax_trees_dir = mkdtemp()
            logger.info(self.syntax_trees_dir)
        else:
            self.check_and_make_working_dir(syntax_trees_dir)
            self.syntax_trees_dir = syntax_trees_dir

        self.num_commits = 0

        self.is_bare_repo = False

    def check_and_make_working_dir(self, path):
        if os.path.isdir(path):
            if os.listdir(path):
                raise Exception('{0} is not an empty directory'.format(path))
        else:
            try:
                os.mkdir(path)
            except OSError:
                logger.error('Kenja cannot make a directory: {0}'.format(path))
                raise

    def is_target_blob(self, blob, ext):
        return blob and blob.name.endswith(ext)

    def parse_all_target_files(self):
        logger.info('create parser processes...')
        parser_executor = ParserExecutor(self.syntax_trees_dir, self.parser_jar_path)
        parsed_blob = set()
        for commit in get_reversed_topological_ordered_commits(self.org_repo, self.org_repo.refs):
            self.num_commits = self.num_commits + 1
            commit = self.org_repo.commit(commit)
            if commit.parents:
                for p in commit.parents:
                    for diff in p.diff(commit):
                        if self.is_target_blob(diff.b_blob, self.language):
                            if diff.b_blob.hexsha not in parsed_blob:
                                parser_executor.parse_blob(diff.b_blob)
                                parsed_blob.add(diff.b_blob.hexsha)
            else:
                for entry in commit.tree.traverse():
                    if isinstance(entry, Blob) and self.is_target_blob(entry, self.language):
                        if entry.hexsha not in parsed_blob:
                            parser_executor.parse_blob(entry)
                            parsed_blob.add(entry.hexsha)
        logger.info('waiting parser processes')
        parser_executor.join()

    def prepare_base_repo(self):
        base_repo = Repo.init(self.historage_dir, bare=self.is_bare_repo)
        self.set_git_config(base_repo)
        return base_repo

    def set_git_config(self, repo):
        reader = repo.config_reader()  # global config
        writer = repo.config_writer()  # local config
        user_key = 'user'
        if not reader.has_option(user_key, 'name'):
            if not writer.has_section(user_key):
                writer.add_section(user_key)
            writer.set(user_key, 'name', 'Kenja Converter')
        if not reader.has_option(user_key, 'email'):
            if not writer.has_section(user_key):
                writer.add_section(user_key)
            writer.set(user_key, 'email', 'default@example.com')

    def convert(self):
        self.parse_all_target_files()
        self.construct_historage()

    def construct_historage(self):
        logger.info('create historage...')

        base_repo = self.prepare_base_repo()
        committer = SyntaxTreesCommitter(Repo(self.org_repo.git_dir), base_repo, self.syntax_trees_dir)
        num_commits = self.num_commits if self.num_commits != 0 else '???'
        for num, commit in izip(count(), get_reversed_topological_ordered_commits(self.org_repo, self.org_repo.refs)):
            commit = self.org_repo.commit(commit)
            logger.info('[%d/%s] convert %s to: %s' % (num, num_commits, commit.hexsha, base_repo.git_dir))
            committer.apply_change(commit)
        committer.create_heads()
        committer.create_tags()
        if not self.is_bare_repo:
            base_repo.head.reset(working_tree=True)

    def __del__(self):
        if self.use_tempdir and os.path.exists(self.syntax_trees_dir):
            rmtree(self.syntax_trees_dir)
<commit_msg>modified log messages<commit_after>from __future__ import absolute_import
import os
from tempfile import mkdtemp
from shutil import rmtree
from itertools import count, izip
from git.repo import Repo
from git.objects import Blob
from kenja.parser import ParserExecutor
from kenja.git.util import get_reversed_topological_ordered_commits
from kenja.committer import SyntaxTreesCommitter
from logging import getLogger

logger = getLogger(__name__)

class HistorageConverter:
    parser_jar_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'lib', 'java-parser.jar')

    def __init__(self, org_git_repo_dir, historage_dir, syntax_trees_dir=None):
        if org_git_repo_dir:
            self.org_repo = Repo(org_git_repo_dir)

        self.language = '.java'
        self.check_and_make_working_dir(historage_dir)
        self.historage_dir = historage_dir

        self.use_tempdir = syntax_trees_dir is None
        if self.use_tempdir:
            self.syntax_trees_dir = mkdtemp()
            logger.info(self.syntax_trees_dir)
        else:
            self.check_and_make_working_dir(syntax_trees_dir)
            self.syntax_trees_dir = syntax_trees_dir

        self.num_commits = 0

        self.is_bare_repo = False

    def check_and_make_working_dir(self, path):
        if os.path.isdir(path):
            if os.listdir(path):
                raise Exception('{0} is not an empty directory'.format(path))
        else:
            try:
                os.mkdir(path)
            except OSError:
                logger.error('Kenja cannot make a directory: {0}'.format(path))
                raise

    def is_target_blob(self, blob, ext):
        return blob and blob.name.endswith(ext)

    def parse_all_target_files(self):
        logger.info('create parser processes...')
        parser_executor = ParserExecutor(self.syntax_trees_dir, self.parser_jar_path)
        parsed_blob = set()
        for commit in get_reversed_topological_ordered_commits(self.org_repo, self.org_repo.refs):
            self.num_commits = self.num_commits + 1
            commit = self.org_repo.commit(commit)
            if commit.parents:
                for p in commit.parents:
                    for diff in p.diff(commit):
                        if self.is_target_blob(diff.b_blob, self.language):
                            if diff.b_blob.hexsha not in parsed_blob:
                                parser_executor.parse_blob(diff.b_blob)
                                parsed_blob.add(diff.b_blob.hexsha)
            else:
                for entry in commit.tree.traverse():
                    if isinstance(entry, Blob) and self.is_target_blob(entry, self.language):
                        if entry.hexsha not in parsed_blob:
                            parser_executor.parse_blob(entry)
                            parsed_blob.add(entry.hexsha)
        logger.info('waiting parser processes')
        parser_executor.join()

    def prepare_base_repo(self):
        base_repo = Repo.init(self.historage_dir, bare=self.is_bare_repo)
        self.set_git_config(base_repo)
        return base_repo

    def set_git_config(self, repo):
        reader = repo.config_reader()  # global config
        writer = repo.config_writer()  # local config
        user_key = 'user'
        if not reader.has_option(user_key, 'name'):
            if not writer.has_section(user_key):
                writer.add_section(user_key)
            writer.set(user_key, 'name', 'Kenja Converter')
        if not reader.has_option(user_key, 'email'):
            if not writer.has_section(user_key):
                writer.add_section(user_key)
            writer.set(user_key, 'email', 'default@example.com')

    def convert(self):
        self.parse_all_target_files()
        self.construct_historage()

    def construct_historage(self):
        logger.info('convert a git repository to a  historage...')

        base_repo = self.prepare_base_repo()
        committer = SyntaxTreesCommitter(Repo(self.org_repo.git_dir), base_repo, self.syntax_trees_dir)
        num_commits = self.num_commits if self.num_commits != 0 else '???'
        for num, commit in izip(count(), get_reversed_topological_ordered_commits(self.org_repo, self.org_repo.refs)):
            commit = self.org_repo.commit(commit)
            logger.info('[%d/%s] convert %s to: %s' % (num, num_commits, commit.hexsha, base_repo.git_dir))
            committer.apply_change(commit)
        committer.create_heads()
        committer.create_tags()
        if not self.is_bare_repo:
            base_repo.head.reset(working_tree=True)
        logger.info('completed!')

    def __del__(self):
        if self.use_tempdir and os.path.exists(self.syntax_trees_dir):
            rmtree(self.syntax_trees_dir)
<|endoftext|>


<commit_before>from __future__ import absolute_import
import os
from itertools import count, izip
from git.repo import Repo
from git.objects import Commit, Blob
from kenja.parser import ParserExecutor
from kenja.git.util import get_reversed_topological_ordered_commits
from kenja.committer import SyntaxTreesParallelCommitter
from kenja.committer import SyntaxTreesCommitter

class HistorageConverter:
    parser_jar_path = "../target/kenja-0.0.1-SNAPSHOT-jar-with-dependencies.jar"

    def __init__(self, org_git_repo_dir, working_dir):
        if org_git_repo_dir:
            self.org_repo = Repo(org_git_repo_dir)

        if not(os.path.isdir(working_dir)):
            raise Exception('%s is not a directory' % (working_dir))

        self.working_dir = working_dir

        self.syntax_trees_dir = os.path.join(self.working_dir, 'syntax_trees')

        self.num_commits = 0

    def is_target_blob(self, blob, ext):
        return blob and blob.name.endswith(ext)

    def parse_all_java_files(self):
        print 'create paresr processes...'
        parser_executor = ParserExecutor(self.syntax_trees_dir, self.parser_jar_path)
        parsed_blob = set()
        for commit in get_reversed_topological_ordered_commits(self.org_repo, self.org_repo.refs):
            self.num_commits = self.num_commits + 1
            commit = self.org_repo.commit(commit)
            if commit.parents:
                for p in commit.parents:
                    for diff in p.diff(commit):
                        if self.is_target_blob(diff.b_blob, '.java'):
                            if not diff.b_blob.hexsha in parsed_blob:
                                parser_executor.parse_blob(diff.b_blob)
                                parsed_blob.add(diff.b_blob.hexsha)
            else:
                for entry in commit.tree.traverse():
                    if isinstance(entry, Blob) and self.is_target_blob(entry, '.java'):
                        if not entry.hexsha in parsed_blob:
                            parser_executor.parse_blob(entry)
                            parsed_blob.add(entry.hexsha)
        print 'waiting parser processes'
        parser_executor.join()

    def prepare_base_repo(self):
        base_repo_dir = os.path.join(self.working_dir, 'base_repo')
        base_repo = Repo.init(base_repo_dir)
        return base_repo

    def clone_working_repos(self, base_repo, num_working_repos):
        self.working_repo_dirs = []
        for i in range(num_working_repos):
            working_repo_dir = os.path.join(self.working_dir, 'work_repo%d' % (i))
            self.working_repo_dirs.append(working_repo_dir)
            base_repo.clone(working_repo_dir)

    def convert(self):
        self.parse_all_java_files()
        self.construct_historage()

    def construct_historage(self):
        print 'create historage...'

        base_repo = self.prepare_base_repo()
        committer = SyntaxTreesCommitter(Repo(self.org_repo.git_dir), base_repo, self.syntax_trees_dir)
        num_commits = self.num_commits if self.num_commits != 0 else '???'
        for num, commit in izip(count(), get_reversed_topological_ordered_commits(self.org_repo, self.org_repo.refs)):
            commit = self.org_repo.commit(commit)
            print '[%d/%s] convert %s to: %s' % (num, num_commits, commit.hexsha, base_repo.git_dir)
            committer.apply_change(commit)
        committer.create_heads()
        committer.create_tags()

<commit_msg>changed path of java-parser for installation<commit_after>from __future__ import absolute_import
import os
from itertools import count, izip
from git.repo import Repo
from git.objects import Commit, Blob
from kenja.parser import ParserExecutor
from kenja.git.util import get_reversed_topological_ordered_commits
from kenja.committer import SyntaxTreesParallelCommitter
from kenja.committer import SyntaxTreesCommitter

class HistorageConverter:
    parser_jar_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'lib', 'java-parser.jar')

    def __init__(self, org_git_repo_dir, working_dir):
        if org_git_repo_dir:
            self.org_repo = Repo(org_git_repo_dir)

        if not(os.path.isdir(working_dir)):
            raise Exception('%s is not a directory' % (working_dir))

        self.working_dir = working_dir

        self.syntax_trees_dir = os.path.join(self.working_dir, 'syntax_trees')

        self.num_commits = 0

    def is_target_blob(self, blob, ext):
        return blob and blob.name.endswith(ext)

    def parse_all_java_files(self):
        print 'create paresr processes...'
        parser_executor = ParserExecutor(self.syntax_trees_dir, self.parser_jar_path)
        parsed_blob = set()
        for commit in get_reversed_topological_ordered_commits(self.org_repo, self.org_repo.refs):
            self.num_commits = self.num_commits + 1
            commit = self.org_repo.commit(commit)
            if commit.parents:
                for p in commit.parents:
                    for diff in p.diff(commit):
                        if self.is_target_blob(diff.b_blob, '.java'):
                            if not diff.b_blob.hexsha in parsed_blob:
                                parser_executor.parse_blob(diff.b_blob)
                                parsed_blob.add(diff.b_blob.hexsha)
            else:
                for entry in commit.tree.traverse():
                    if isinstance(entry, Blob) and self.is_target_blob(entry, '.java'):
                        if not entry.hexsha in parsed_blob:
                            parser_executor.parse_blob(entry)
                            parsed_blob.add(entry.hexsha)
        print 'waiting parser processes'
        parser_executor.join()

    def prepare_base_repo(self):
        base_repo_dir = os.path.join(self.working_dir, 'base_repo')
        base_repo = Repo.init(base_repo_dir)
        return base_repo

    def clone_working_repos(self, base_repo, num_working_repos):
        self.working_repo_dirs = []
        for i in range(num_working_repos):
            working_repo_dir = os.path.join(self.working_dir, 'work_repo%d' % (i))
            self.working_repo_dirs.append(working_repo_dir)
            base_repo.clone(working_repo_dir)

    def convert(self):
        self.parse_all_java_files()
        self.construct_historage()

    def construct_historage(self):
        print 'create historage...'

        base_repo = self.prepare_base_repo()
        committer = SyntaxTreesCommitter(Repo(self.org_repo.git_dir), base_repo, self.syntax_trees_dir)
        num_commits = self.num_commits if self.num_commits != 0 else '???'
        for num, commit in izip(count(), get_reversed_topological_ordered_commits(self.org_repo, self.org_repo.refs)):
            commit = self.org_repo.commit(commit)
            print '[%d/%s] convert %s to: %s' % (num, num_commits, commit.hexsha, base_repo.git_dir)
            committer.apply_change(commit)
        committer.create_heads()
        committer.create_tags()

<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
$Id$

Copyright (c) 2006-2010 sqlmap developers (http://sqlmap.sourceforge.net/)
See the file 'doc/COPYING' for copying permission
"""

from lib.core.data import conf
from lib.core.data import kb
from lib.core.data import logger
from lib.core.settings import MSSQL_ALIASES
from lib.core.settings import MYSQL_ALIASES
from lib.core.settings import ORACLE_ALIASES
from lib.core.settings import PGSQL_ALIASES
from lib.core.settings import SQLITE_ALIASES
from lib.core.settings import ACCESS_ALIASES
from lib.core.settings import FIREBIRD_ALIASES
from lib.core.settings import MAXDB_ALIASES
from lib.core.settings import SYBASE_ALIASES

from plugins.dbms.mssqlserver import MSSQLServerMap
from plugins.dbms.mssqlserver.connector import Connector as MSSQLServerConn
from plugins.dbms.mysql import MySQLMap
from plugins.dbms.mysql.connector import Connector as MySQLConn
from plugins.dbms.oracle import OracleMap
from plugins.dbms.oracle.connector import Connector as OracleConn
from plugins.dbms.postgresql import PostgreSQLMap
from plugins.dbms.postgresql.connector import Connector as PostgreSQLConn
from plugins.dbms.sqlite import SQLiteMap
from plugins.dbms.sqlite.connector import Connector as SQLiteConn
from plugins.dbms.access import AccessMap
from plugins.dbms.access.connector import Connector as AccessConn
from plugins.dbms.firebird import FirebirdMap
from plugins.dbms.firebird.connector import Connector as FirebirdConn
from plugins.dbms.maxdb import MaxDBMap
from plugins.dbms.maxdb.connector import Connector as MaxDBConn
from plugins.dbms.sybase import SybaseMap
from plugins.dbms.sybase.connector import Connector as SybaseConn

def setHandler():
    """
    Detect which is the target web application back-end database
    management system.
    """

    count     = 0
    dbmsNames = ( "MySQL", "Oracle", "PostgreSQL", "Microsoft SQL Server", "SQLite", "Microsoft Access", "Firebird", "SAP MaxDB", "Sybase" )
    dbmsMap   = [
                  ( MYSQL_ALIASES, MySQLMap, MySQLConn ),
                  ( ORACLE_ALIASES, OracleMap, OracleConn ),
                  ( PGSQL_ALIASES, PostgreSQLMap, PostgreSQLConn ),
                  ( MSSQL_ALIASES, MSSQLServerMap, MSSQLServerConn ),
                  ( SQLITE_ALIASES, SQLiteMap, SQLiteConn ),
                  ( ACCESS_ALIASES, AccessMap, AccessConn ),
                  ( FIREBIRD_ALIASES, FirebirdMap, FirebirdConn ),
                  ( MAXDB_ALIASES, MaxDBMap, MaxDBConn ),
                  ( SYBASE_ALIASES, SybaseMap, SybaseConn ),
                ]

    if kb.htmlFp:
        for i in xrange(len(dbmsMap)):
            dbmsAliases, _, _ = dbmsMap[i]
            if kb.htmlFp[-1].lower() in dbmsAliases:
                if i > 0:
                    temp = dbmsMap[i]
                    dbmsMap.remove(temp)
                    dbmsMap.insert(0, temp)
                break

    for dbmsAliases, dbmsMap, dbmsConn in dbmsMap:
        if conf.dbms and conf.dbms not in dbmsAliases:
            debugMsg  = "skipping test for %s" % dbmsNames[count]
            logger.debug(debugMsg)

            count += 1

            continue

        handler = dbmsMap()
        conf.dbmsConnector = dbmsConn()

        if conf.direct:
            logger.debug("forcing timeout to 10 seconds")
            conf.timeout = 10

            conf.dbmsConnector.connect()

        if handler.checkDbms():
            if not conf.dbms or conf.dbms in dbmsAliases:
                kb.dbmsDetected = True

                conf.dbmsHandler = handler

                return
        else:
            conf.dbmsConnector = None
<commit_msg>using pushValue and popValue<commit_after>#!/usr/bin/env python

"""
$Id$

Copyright (c) 2006-2010 sqlmap developers (http://sqlmap.sourceforge.net/)
See the file 'doc/COPYING' for copying permission
"""

from lib.core.common import popValue
from lib.core.common import pushValue
from lib.core.data import conf
from lib.core.data import kb
from lib.core.data import logger
from lib.core.settings import MSSQL_ALIASES
from lib.core.settings import MYSQL_ALIASES
from lib.core.settings import ORACLE_ALIASES
from lib.core.settings import PGSQL_ALIASES
from lib.core.settings import SQLITE_ALIASES
from lib.core.settings import ACCESS_ALIASES
from lib.core.settings import FIREBIRD_ALIASES
from lib.core.settings import MAXDB_ALIASES
from lib.core.settings import SYBASE_ALIASES

from plugins.dbms.mssqlserver import MSSQLServerMap
from plugins.dbms.mssqlserver.connector import Connector as MSSQLServerConn
from plugins.dbms.mysql import MySQLMap
from plugins.dbms.mysql.connector import Connector as MySQLConn
from plugins.dbms.oracle import OracleMap
from plugins.dbms.oracle.connector import Connector as OracleConn
from plugins.dbms.postgresql import PostgreSQLMap
from plugins.dbms.postgresql.connector import Connector as PostgreSQLConn
from plugins.dbms.sqlite import SQLiteMap
from plugins.dbms.sqlite.connector import Connector as SQLiteConn
from plugins.dbms.access import AccessMap
from plugins.dbms.access.connector import Connector as AccessConn
from plugins.dbms.firebird import FirebirdMap
from plugins.dbms.firebird.connector import Connector as FirebirdConn
from plugins.dbms.maxdb import MaxDBMap
from plugins.dbms.maxdb.connector import Connector as MaxDBConn
from plugins.dbms.sybase import SybaseMap
from plugins.dbms.sybase.connector import Connector as SybaseConn

def setHandler():
    """
    Detect which is the target web application back-end database
    management system.
    """

    count     = 0
    dbmsNames = ( "MySQL", "Oracle", "PostgreSQL", "Microsoft SQL Server", "SQLite", "Microsoft Access", "Firebird", "SAP MaxDB", "Sybase" )
    dbmsMap   = [
                  ( MYSQL_ALIASES, MySQLMap, MySQLConn ),
                  ( ORACLE_ALIASES, OracleMap, OracleConn ),
                  ( PGSQL_ALIASES, PostgreSQLMap, PostgreSQLConn ),
                  ( MSSQL_ALIASES, MSSQLServerMap, MSSQLServerConn ),
                  ( SQLITE_ALIASES, SQLiteMap, SQLiteConn ),
                  ( ACCESS_ALIASES, AccessMap, AccessConn ),
                  ( FIREBIRD_ALIASES, FirebirdMap, FirebirdConn ),
                  ( MAXDB_ALIASES, MaxDBMap, MaxDBConn ),
                  ( SYBASE_ALIASES, SybaseMap, SybaseConn ),
                ]

    if kb.htmlFp:
        for i in xrange(len(dbmsMap)):
            dbmsAliases, _, _ = dbmsMap[i]
            if kb.htmlFp[-1].lower() in dbmsAliases:
                if i > 0:
                    pushValue(dbmsMap[i])
                    dbmsMap.remove(dbmsMap[i])
                    dbmsMap.insert(0, popValue())
                break

    for dbmsAliases, dbmsMap, dbmsConn in dbmsMap:
        if conf.dbms and conf.dbms not in dbmsAliases:
            debugMsg  = "skipping test for %s" % dbmsNames[count]
            logger.debug(debugMsg)

            count += 1

            continue

        handler = dbmsMap()
        conf.dbmsConnector = dbmsConn()

        if conf.direct:
            logger.debug("forcing timeout to 10 seconds")
            conf.timeout = 10

            conf.dbmsConnector.connect()

        if handler.checkDbms():
            if not conf.dbms or conf.dbms in dbmsAliases:
                kb.dbmsDetected = True

                conf.dbmsHandler = handler

                return
        else:
            conf.dbmsConnector = None
<|endoftext|>