<commit_before>import warnings
import sys
from contextlib import contextmanager
from functools import wraps

from twisted.internet.task import Clock
from twisted.internet.defer import Deferred

from spinoff.util.async import CancelledError
from spinoff.actor.actor import BaseActor
from spinoff.util.pattern_matching import match, ANY, IS_INSTANCE, NOT

__all__ = ['deferred', 'assert_raises', 'assert_not_raises', 'MockFunction', 'assert_num_warnings', 'assert_no_warnings', 'assert_one_warning']

def deferred(f):
    """This is almost exactly the same as nose.twistedtools.deferred, except it allows one to have simulated time in
    their test code by passing in a `Clock` instance. That `Clock` instance will be `advance`d as long as there are
    any deferred calls bound to it.

    """
    @wraps(f)
    def ret():
        error = [None]

        clock = Clock()

        d = f(clock)
        @d.addErrback
        def on_error(f):
            error[0] = sys.exc_info()

        while True:
            time_to_wait = max([0] + [call.getTime() - clock.seconds() for call in clock.getDelayedCalls()])
            if time_to_wait == 0:
                break
            else:
                clock.advance(time_to_wait)

        if error[0]:
            exc_info = error[0]
            raise exc_info[0], exc_info[1], exc_info[2]

    return ret

def immediate(d):
    assert d.called
    return d

def deferred_result(d):
    ret = [None]
    exc = [None]
    if isinstance(d, Deferred):
        d = immediate(d)
        d.addCallback(lambda result: ret.__setitem__(0, result))
        d.addErrback(lambda f: exc.__setitem__(0, f))
        if exc[0]:
            exc[0].raiseException()
        return ret[0]
    else:
        return d

def cancel_deferred(d):
    d.addErrback(lambda f: f.trap(CancelledError))
    d.cancel()

@contextmanager
def assert_not_raises(exc_class=Exception, message=None):
    if isinstance(exc_class, basestring):
        message, exc_class = exc_class, Exception
    try:
        yield
    except exc_class as e:
        raise AssertionError(message or "No exception should have been raised but instead %s was raised" % repr(e))

@contextmanager
def assert_raises(exc_class=Exception, message=None):
    if isinstance(exc_class, basestring):
        message, exc_class = exc_class, Exception
    try:
        yield
    except exc_class:
        pass
    else:
        raise AssertionError(message or "An exception should have been raised")

@contextmanager
def assert_num_warnings(n):
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter("always")
        yield
        assert len(w) == n

def assert_no_warnings():
    return assert_num_warnings(0)

def assert_one_warning():
    return assert_num_warnings(1)

class MockFunction(object):

    called = property(lambda self: len(self.argses))

    def __init__(self, return_values=None):
        self.reset()
        self.return_values = return_values and list(return_values)

    def __call__(self, *args, **kwargs):
        self.argses.append(args)
        self.kwargses.append(kwargs)
        if self.return_values is not None:
            return self.return_values.pop(0)

    def assert_called(self, n=None, message=None):
        if isinstance(n, basestring):
            n, message = None, n
        if n is not None:
            assert self.called == n
        else:
            assert self.called
        self.reset()

    def assert_called_with_signature(self, *args, **kwargs):
        args_, kwargs_ = self.argses.pop(0), self.kwargses.pop(0)
        assert args == args_ and kwargs == kwargs_, "mock function called with different arguments than expected"

    def reset(self):
        self.argses = []
        self.kwargses = []

def errback_called(d):
    mock_fn = MockFunction()
    d.addErrback(mock_fn)
    return mock_fn.called

def callback_called(d):
    mock_fn = MockFunction()
    d.addCallback(mock_fn)
    return mock_fn.called

class DebugActor(object):

    def __init__(self, name, debug_fn=lambda x: 'a message'):
        self.name = name
        self.debug_fn = debug_fn

    def deliver(self, message, inbox):
        print "%s: received %s into %s" % (self.name, self.debug_fn(message), inbox)

class MockActor(BaseActor):

    def __init__(self, *args, **kwargs):
        super(MockActor, self).__init__(*args, **kwargs)
        self.messages = []
        self.waiting = None

    def handle(self, message):
        if self.waiting:
            self.waiting.callback(message)
        else:
            self.messages.append(message)

    def clear(self):
        ret = self.messages
        self.messages = []
        return ret

    def wait(self):
        self.waiting = Deferred()
        return self.waiting

class Container(MockActor):

    def __init__(self, actor_cls=None, start_automatically=True):
        super(Container, self).__init__()
        self._actor_cls = actor_cls
        self._start_automatically = start_automatically

    def __enter__(self):
        assert self._actor_cls
        self.start()
        a = self._actor_cls() if isinstance(self._actor_cls, type) else self._actor_cls
        a._parent = self
        if self._start_automatically:
            a.start()
        return self, a

    def __exit__(self, exc_cls, exc, tb):
        if exc is None:
            self.raise_errors()
        else:
            raise exc_cls, exc, tb

    def consume_message(self, pattern, n=1):
        assert n is 'INF' or n >= 0
        consumed = 0
        while n > 0 or n is 'INF':
            for message in self.messages:
                if match(pattern, message):
                    self.messages.remove(message)
                    consumed += 1
                    break
            else:
                break
            if n is not 'INF':
                n -= 1
        assert consumed

    def raise_errors(self, only_asserts=False):
        assert not only_asserts
        for msg in self.messages:
            if msg[0] == 'error':
                # see: http://twistedmatrix.com/trac/ticket/5178
                if not isinstance(msg[2][1], basestring):
                    raise msg[2][0], None, msg[2][1]
                else:
                    print(msg[2][1])
                    raise msg[2][0]

class RootActor(Container):
    def __new__(cls, *args, **kwargs):
        warnings.warn("RootActor is deprecated; use Container instead", DeprecationWarning)
        return super(RootActor, cls).__new__(cls, *args, **kwargs)

def run(a_cls, raise_only_asserts=False):
    if raise_only_asserts:
        warnings.warn("raise_only_asserts is deprecated; use Container instead", DeprecationWarning)
    root = Container(a_cls)
    _, actor = root.__enter__()
    if raise_only_asserts:
        root.consume_message(('error', ANY, (NOT(IS_INSTANCE(AssertionError)), ANY), ANY), n='INF')
    root.__exit__(None, None, None)
    return root, actor
<commit_msg>Added the optional message argument to Container.consume_message<commit_after>import warnings
import sys
from contextlib import contextmanager
from functools import wraps

from twisted.internet.task import Clock
from twisted.internet.defer import Deferred

from spinoff.util.async import CancelledError
from spinoff.actor.actor import BaseActor
from spinoff.util.pattern_matching import match, ANY, IS_INSTANCE, NOT

__all__ = ['deferred', 'assert_raises', 'assert_not_raises', 'MockFunction', 'assert_num_warnings', 'assert_no_warnings', 'assert_one_warning']

def deferred(f):
    """This is almost exactly the same as nose.twistedtools.deferred, except it allows one to have simulated time in
    their test code by passing in a `Clock` instance. That `Clock` instance will be `advance`d as long as there are
    any deferred calls bound to it.

    """
    @wraps(f)
    def ret():
        error = [None]

        clock = Clock()

        d = f(clock)
        @d.addErrback
        def on_error(f):
            error[0] = sys.exc_info()

        while True:
            time_to_wait = max([0] + [call.getTime() - clock.seconds() for call in clock.getDelayedCalls()])
            if time_to_wait == 0:
                break
            else:
                clock.advance(time_to_wait)

        if error[0]:
            exc_info = error[0]
            raise exc_info[0], exc_info[1], exc_info[2]

    return ret

def immediate(d):
    assert d.called
    return d

def deferred_result(d):
    ret = [None]
    exc = [None]
    if isinstance(d, Deferred):
        d = immediate(d)
        d.addCallback(lambda result: ret.__setitem__(0, result))
        d.addErrback(lambda f: exc.__setitem__(0, f))
        if exc[0]:
            exc[0].raiseException()
        return ret[0]
    else:
        return d

def cancel_deferred(d):
    d.addErrback(lambda f: f.trap(CancelledError))
    d.cancel()

@contextmanager
def assert_not_raises(exc_class=Exception, message=None):
    if isinstance(exc_class, basestring):
        message, exc_class = exc_class, Exception
    try:
        yield
    except exc_class as e:
        raise AssertionError(message or "No exception should have been raised but instead %s was raised" % repr(e))

@contextmanager
def assert_raises(exc_class=Exception, message=None):
    if isinstance(exc_class, basestring):
        message, exc_class = exc_class, Exception
    try:
        yield
    except exc_class:
        pass
    else:
        raise AssertionError(message or "An exception should have been raised")

@contextmanager
def assert_num_warnings(n):
    with warnings.catch_warnings(record=True) as w:
        warnings.simplefilter("always")
        yield
        assert len(w) == n

def assert_no_warnings():
    return assert_num_warnings(0)

def assert_one_warning():
    return assert_num_warnings(1)

class MockFunction(object):

    called = property(lambda self: len(self.argses))

    def __init__(self, return_values=None):
        self.reset()
        self.return_values = return_values and list(return_values)

    def __call__(self, *args, **kwargs):
        self.argses.append(args)
        self.kwargses.append(kwargs)
        if self.return_values is not None:
            return self.return_values.pop(0)

    def assert_called(self, n=None, message=None):
        if isinstance(n, basestring):
            n, message = None, n
        if n is not None:
            assert self.called == n
        else:
            assert self.called
        self.reset()

    def assert_called_with_signature(self, *args, **kwargs):
        args_, kwargs_ = self.argses.pop(0), self.kwargses.pop(0)
        assert args == args_ and kwargs == kwargs_, "mock function called with different arguments than expected"

    def reset(self):
        self.argses = []
        self.kwargses = []

def errback_called(d):
    mock_fn = MockFunction()
    d.addErrback(mock_fn)
    return mock_fn.called

def callback_called(d):
    mock_fn = MockFunction()
    d.addCallback(mock_fn)
    return mock_fn.called

class DebugActor(object):

    def __init__(self, name, debug_fn=lambda x: 'a message'):
        self.name = name
        self.debug_fn = debug_fn

    def deliver(self, message, inbox):
        print "%s: received %s into %s" % (self.name, self.debug_fn(message), inbox)

class MockActor(BaseActor):

    def __init__(self, *args, **kwargs):
        super(MockActor, self).__init__(*args, **kwargs)
        self.messages = []
        self.waiting = None

    def handle(self, message):
        if self.waiting:
            self.waiting.callback(message)
        else:
            self.messages.append(message)

    def clear(self):
        ret = self.messages
        self.messages = []
        return ret

    def wait(self):
        self.waiting = Deferred()
        return self.waiting

class Container(MockActor):

    def __init__(self, actor_cls=None, start_automatically=True):
        super(Container, self).__init__()
        self._actor_cls = actor_cls
        self._start_automatically = start_automatically

    def __enter__(self):
        assert self._actor_cls
        self.start()
        a = self._actor_cls() if isinstance(self._actor_cls, type) else self._actor_cls
        a._parent = self
        if self._start_automatically:
            a.start()
        return self, a

    def __exit__(self, exc_cls, exc, tb):
        if exc is None:
            self.raise_errors()
        else:
            raise exc_cls, exc, tb

    def consume_message(self, pattern, n=1, message=None):
        assert n is 'INF' or n >= 0
        consumed = 0
        while n > 0 or n is 'INF':
            for msg in self.messages:
                if match(pattern, msg):
                    self.messages.remove(msg)
                    consumed += 1
                    break
            else:
                break
            if n is not 'INF':
                n -= 1
        assert consumed, message

    def raise_errors(self, only_asserts=False):
        assert not only_asserts
        for msg in self.messages:
            if msg[0] == 'error':
                # see: http://twistedmatrix.com/trac/ticket/5178
                if not isinstance(msg[2][1], basestring):
                    raise msg[2][0], None, msg[2][1]
                else:
                    print(msg[2][1])
                    raise msg[2][0]

class RootActor(Container):
    def __new__(cls, *args, **kwargs):
        warnings.warn("RootActor is deprecated; use Container instead", DeprecationWarning)
        return super(RootActor, cls).__new__(cls, *args, **kwargs)

def run(a_cls, raise_only_asserts=False):
    if raise_only_asserts:
        warnings.warn("raise_only_asserts is deprecated; use Container instead", DeprecationWarning)
    root = Container(a_cls)
    _, actor = root.__enter__()
    if raise_only_asserts:
        root.consume_message(('error', ANY, (NOT(IS_INSTANCE(AssertionError)), ANY), ANY), n='INF')
    root.__exit__(None, None, None)
    return root, actor
<|endoftext|>


<commit_before>'''
Package Management
==================
Salt can manage software packages via the pkg state module, packages can be
set up to be installed, latest, removed and purged. Package management
declarations are typically rather simple:

.. code-block:: yaml

    vim:
      pkg:
        - installed
'''

def installed(name):
    '''
    Verify that the package is installed, and only that it is installed. This
    state will not upgrade an existing package and only verify that it is
    installed

    name
        The name of the package to install
    '''
    if __salt__['pkg.version'](name):
        return {'name': name,
                'changes': {},
                'result': True,
                'comment': 'Package ' + name + ' is already installed'}
    changes = __salt__['pkg.install'](name, True)
    if not changes:
        return {'name': name,
                'changes': changes,
                'result': False,
                'comment': 'Package ' + name + ' failed to install'}
    return {'name': name,
            'changes': changes,
            'result': True,
            'comment': 'Package ' + name + ' installed'}

def latest(name):
    '''
    Verify that the named package is installed and the latest available
    package. If the package can be updated this state function will update
    the package. Generally it is better for the installed function to be
    used, as ``latest`` will update the package the package whenever a new
    package is available

    name
        The name of the package to maintain at the latest available version
    '''
    changes = {}
    version = __salt__['pkg.version'](name)
    avail = __salt__['pkg.available_version'](name)
    if avail > version:
        changes = __salt__['pkg.install'](name, True)
        if not changes:
            return {'name': name,
                    'changes': changes,
                    'result': False,
                    'comment': 'Package ' + name + ' failed to install'}
    return {'name': name,
            'changes': changes,
            'result': True,
            'comment': 'Package ' + name + ' installed'}

def removed(name):
    '''
    Verify that the package is removed, this will remove the package via
    the remove function in the salt pkg module for the platform.

    name
        The name of the package to be removed
    '''
    if not __salt__['pkg.version'](name):
        return {'name': name,
                'changes': {},
                'result': True,
                'comment': 'Package ' + name + ' is not installed'}
    else:
        changes = __salt__['pkg.remove'](name)
    if not changes:
        return {'name': name,
                'changes': changes,
                'result': False,
                'comment': 'Package ' + name + ' failed to remove'}
        return {'name': name,
            'changes': changes,
            'result': True,
            'comment': 'Package ' + name + ' removed'}

def purged(name):
    '''
    Verify that the package is purged, this will call the purge function in the
    salt pkg module for the platform.

    name
        The name of the package to be purged
    '''
    if not __salt__['pkg.version'](name):
        return {'name': name,
                'changes': {},
                'result': True,
                'comment': 'Package ' + name + ' is not installed'}
    else:
        changes = __salt__['pkg.purge'](name)

    if not changes:
        return {'name': name,
                'changes': changes,
                'result': False,
                'comment': 'Package ' + name + ' failed to purge'}
        # FIXME: this block will never be reached
        return {'name': name,
            'changes': changes,
            'result': True,
            'comment': 'Package ' + name + ' purged'}
<commit_msg>found another piece of unrechable code<commit_after>'''
Package Management
==================
Salt can manage software packages via the pkg state module, packages can be
set up to be installed, latest, removed and purged. Package management
declarations are typically rather simple:

.. code-block:: yaml

    vim:
      pkg:
        - installed
'''

def installed(name):
    '''
    Verify that the package is installed, and only that it is installed. This
    state will not upgrade an existing package and only verify that it is
    installed

    name
        The name of the package to install
    '''
    if __salt__['pkg.version'](name):
        return {'name': name,
                'changes': {},
                'result': True,
                'comment': 'Package ' + name + ' is already installed'}
    changes = __salt__['pkg.install'](name, True)
    if not changes:
        return {'name': name,
                'changes': changes,
                'result': False,
                'comment': 'Package ' + name + ' failed to install'}
    return {'name': name,
            'changes': changes,
            'result': True,
            'comment': 'Package ' + name + ' installed'}

def latest(name):
    '''
    Verify that the named package is installed and the latest available
    package. If the package can be updated this state function will update
    the package. Generally it is better for the installed function to be
    used, as ``latest`` will update the package the package whenever a new
    package is available

    name
        The name of the package to maintain at the latest available version
    '''
    changes = {}
    version = __salt__['pkg.version'](name)
    avail = __salt__['pkg.available_version'](name)
    if avail > version:
        changes = __salt__['pkg.install'](name, True)
        if not changes:
            return {'name': name,
                    'changes': changes,
                    'result': False,
                    'comment': 'Package ' + name + ' failed to install'}
    return {'name': name,
            'changes': changes,
            'result': True,
            'comment': 'Package ' + name + ' installed'}

def removed(name):
    '''
    Verify that the package is removed, this will remove the package via
    the remove function in the salt pkg module for the platform.

    name
        The name of the package to be removed
    '''
    if not __salt__['pkg.version'](name):
        return {'name': name,
                'changes': {},
                'result': True,
                'comment': 'Package ' + name + ' is not installed'}
    else:
        changes = __salt__['pkg.remove'](name)
    if not changes:
        return {'name': name,
                'changes': changes,
                'result': False,
                'comment': 'Package ' + name + ' failed to remove'}
        # FIXME: this block will never be reached
        return {'name': name,
            'changes': changes,
            'result': True,
            'comment': 'Package ' + name + ' removed'}

def purged(name):
    '''
    Verify that the package is purged, this will call the purge function in the
    salt pkg module for the platform.

    name
        The name of the package to be purged
    '''
    if not __salt__['pkg.version'](name):
        return {'name': name,
                'changes': {},
                'result': True,
                'comment': 'Package ' + name + ' is not installed'}
    else:
        changes = __salt__['pkg.purge'](name)

    if not changes:
        return {'name': name,
                'changes': changes,
                'result': False,
                'comment': 'Package ' + name + ' failed to purge'}
        # FIXME: this block will never be reached
        return {'name': name,
            'changes': changes,
            'result': True,
            'comment': 'Package ' + name + ' purged'}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import
import os

import lupa
from IPython.kernel.zmq.kernelapp import IPKernelApp
from IPython.kernel.zmq.eventloops import loop_qt4
from IPython.kernel.kernelspec import install_kernel_spec
from twisted.internet import defer

import splash
from splash.lua import get_version, get_main_sandboxed, get_main, parse_lua_error
from splash.browser_tab import BrowserTab
from splash.lua_runner import ScriptError
from splash.lua_runtime import SplashLuaRuntime
from splash.qtrender_lua import Splash, SplashScriptRunner
from splash.qtutils import init_qt_app
from splash.render_options import RenderOptions
from splash import network_manager
from splash import defaults
from splash import xvfb
from splash.kernel.kernelbase import Kernel
from splash.utils import BinaryCapsule
from splash.kernel.completer import Completer
from splash.kernel.inspections import Inspector

def install(user=True):
    """ Install IPython kernel specification """
    folder = os.path.join(os.path.dirname(__file__), 'kernels', 'splash')
    install_kernel_spec(folder, kernel_name="splash", user=user, replace=True)

def init_browser():
    # TODO: support the same command-line options as HTTP server.

    # from splash.server import start_logging
    # class opts(object):
    #    logfile = "./kernel.log"
    # start_logging(opts)

    manager = network_manager.create_default()
    proxy_factory = None  # TODO

    data = {}
    data['uid'] = id(data)

    tab = BrowserTab(
        network_manager=manager,
        splash_proxy_factory=proxy_factory,
        verbosity=2,  # TODO
        render_options=RenderOptions(data, defaults.MAX_TIMEOUT),  # TODO: timeout
        visible=True,
    )
    return tab

class DeferredSplashRunner(object):

    def __init__(self, lua, splash, sandboxed, log=None, render_options=None):
        self.lua = lua
        self.splash = splash
        self.sandboxed = sandboxed

        if log is None:
            self.log = self.splash.tab.logger.log
        else:
            self.log = log

        self.runner = SplashScriptRunner(
            lua=self.lua,
            log=self.log,
            splash=splash,
            sandboxed=self.sandboxed,
        )
        self.splash.init_dispatcher(self.runner.dispatch)

    def run(self, main_coro):
        """
        Run main_coro Lua coroutine, passing it a Splash
        instance as an argument. Return a Deferred.
        """
        d = defer.Deferred()

        def return_result(result):
            d.callback(result)

        def return_error(err):
            d.errback(err)

        self.runner.start(
            main_coro=main_coro,
            return_result=return_result,
            return_error=return_error,
        )
        return d

class SplashKernel(Kernel):
    implementation = 'Splash'
    implementation_version = splash.__version__
    language = 'Lua'
    language_version = get_version()
    language_info = {
        'name': 'Splash',
        'mimetype': 'application/x-lua',
        'display_name': 'Splash',
        'language': 'lua',
        'codemirror_mode': {
            "name": "text/x-lua",
        },
        'file_extension': '.lua',
        'pygments_lexer': 'lua'
    }
    banner = "Splash kernel - write browser automation scripts interactively"
    help_links = [
        {
            'text': "Splash Tutorial",
            'url': 'http://splash.readthedocs.org/en/latest/scripting-tutorial.html'
        },
        {
            'text': "Splash Reference",
            'url': 'http://splash.readthedocs.org/en/latest/scripting-ref.html'
        },
        {
            'text': "Programming in Lua",
            'url': 'http://www.lua.org/pil/contents.html'
        },
        {
            'text': "Lua 5.2 Manual",
            'url': 'http://www.lua.org/manual/5.2/'
        },
    ]

    sandboxed = False

    def __init__(self, **kwargs):
        super(SplashKernel, self).__init__(**kwargs)
        self.tab = init_browser()

        self.lua = SplashLuaRuntime(self.sandboxed, "", ())
        self.splash = Splash(lua=self.lua, tab=self.tab)
        self.lua.add_to_globals("splash", self.splash.get_wrapped())
        self.runner = DeferredSplashRunner(self.lua, self.splash, self.sandboxed) #, self.log_msg)
        self.completer = Completer(self.lua)
        self.inspector = Inspector(self.lua)
        #
        # try:
        #     sys.stdout.write = self._print
        #     sys.stderr.write = self._print
        # except:
        #     pass # Can't change stdout

    def send_execute_reply(self, stream, ident, parent, md, reply_content):
        def done(result):
            reply, result, ct = result
            if result:
                data = {
                    'text/plain': result if isinstance(result, basestring) else str(result),
                }
                if isinstance(result, BinaryCapsule):
                    data[result.content_type] = result.as_b64()
                self._publish_execute_result(parent, data, {}, self.execution_count)

            super(SplashKernel, self).send_execute_reply(stream, ident, parent, md, reply)

        assert isinstance(reply_content, defer.Deferred)
        reply_content.addCallback(done)

    def do_execute(self, code, silent, store_history=True, user_expressions=None,
                   allow_stdin=False):
        def success(res):
            result, content_type, headers = res
            reply = {
                'status': 'ok',
                'execution_count': self.execution_count,
                'payload': [],
                'user_expressions': {},
            }
            return reply, result, content_type or 'text/plain'

        def error(failure):
            text = "<unknown error>"
            try:
                failure.raiseException()
            except (lupa.LuaSyntaxError, lupa.LuaError, ScriptError) as e:
                tp, line_num, message = parse_lua_error(e)
                text = "<%s error> [input]:%s: %s" % (tp, line_num, message)
            except Exception as e:
                text = repr(e)
            reply = {
                'status': 'error',
                'execution_count': self.execution_count,
                'ename': '',
                'evalue': text,
                'traceback': []
            }
            return reply, text, 'text/plain'

        try:
            try:
                # XXX: this ugly formatting is important for exception
                # line numbers to be displayed properly!
                lua_source = 'local repr = require("repr"); function main(splash) return repr(%s) end' % code
                main_coro = self._get_main(lua_source)
            except lupa.LuaSyntaxError:
                try:
                    lines = code.splitlines(False)
                    lua_source = '''local repr = require("repr"); function main(splash) %s
                        return repr(%s)
                    end
                    ''' % ("\n".join(lines[:-1]), lines[-1])
                    main_coro = self._get_main(lua_source)
                except lupa.LuaSyntaxError:
                    lua_source = "function main(splash) %s end" % code
                    main_coro = self._get_main(lua_source)

        except (lupa.LuaSyntaxError, lupa.LuaError) as e:
            d = defer.Deferred()
            d.addCallbacks(success, error)
            d.errback(e)
            return d
        except Exception:
            d = defer.Deferred()
            d.addCallbacks(success, error)
            d.errback()
            return d

        d = self.runner.run(main_coro)
        d.addCallbacks(success, error)
        return d

    def do_complete(self, code, cursor_pos):
        return self.completer.complete(code, cursor_pos)

    def do_inspect(self, code, cursor_pos, detail_level=0):
        return self.inspector.help(code, cursor_pos, detail_level)

    def _publish_execute_result(self, parent, data, metadata, execution_count):
        msg = {
            u'data': data,
            u'metadata': metadata,
            u'execution_count': execution_count
        }
        self.session.send(self.iopub_socket, u'execute_result', msg,
                          parent=parent, ident=self._topic('execute_result')
        )

    def log_msg(self, text, min_level=2):
        self._print(text + "\n")

    def _print(self, message):
        stream_content = {'name': 'stdout', 'text': message, 'metadata': dict()}
        self.log.debug('Write: %s' % message)
        self.send_response(self.iopub_socket, 'stream', stream_content)

    def _get_main(self, lua_source):
        if self.sandboxed:
            main, env = get_main_sandboxed(self.lua, lua_source)
        else:
            main, env = get_main(self.lua, lua_source)
        return self.lua.create_coroutine(main)

def start():
    with xvfb.autostart():
        # FIXME: logs go to nowhere
        init_qt_app(verbose=False)
        kernel = IPKernelApp.instance(kernel_class=SplashKernel)
        kernel.initialize()
        kernel.kernel.eventloop = loop_qt4
        kernel.start()
<commit_msg>FIX Splash-Jupyter after https://github.com/scrapinghub/splash/commit/315e74150557d48fb40cd25006cd98546cc5832c<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import
import os

import lupa
from IPython.kernel.zmq.kernelapp import IPKernelApp
from IPython.kernel.zmq.eventloops import loop_qt4
from IPython.kernel.kernelspec import install_kernel_spec
from twisted.internet import defer

import splash
from splash.lua import get_version, get_main_sandboxed, get_main, parse_lua_error
from splash.browser_tab import BrowserTab
from splash.lua_runner import ScriptError
from splash.lua_runtime import SplashLuaRuntime
from splash.qtrender_lua import Splash, SplashScriptRunner
from splash.qtutils import init_qt_app
from splash.render_options import RenderOptions
from splash import network_manager
from splash import defaults
from splash import xvfb
from splash.kernel.kernelbase import Kernel
from splash.utils import BinaryCapsule
from splash.kernel.completer import Completer
from splash.kernel.inspections import Inspector

def install(user=True):
    """ Install IPython kernel specification """
    folder = os.path.join(os.path.dirname(__file__), 'kernels', 'splash')
    install_kernel_spec(folder, kernel_name="splash", user=user, replace=True)

def init_browser():
    # TODO: support the same command-line options as HTTP server.

    # from splash.server import start_logging
    # class opts(object):
    #    logfile = "./kernel.log"
    # start_logging(opts)

    manager = network_manager.create_default()
    proxy_factory = None  # TODO

    data = {}
    data['uid'] = id(data)

    tab = BrowserTab(
        network_manager=manager,
        splash_proxy_factory=proxy_factory,
        verbosity=2,  # TODO
        render_options=RenderOptions(data, defaults.MAX_TIMEOUT),  # TODO: timeout
        visible=True,
    )
    return tab

class DeferredSplashRunner(object):

    def __init__(self, lua, splash, sandboxed, log=None, render_options=None):
        self.lua = lua
        self.splash = splash
        self.sandboxed = sandboxed

        if log is None:
            self.log = self.splash.tab.logger.log
        else:
            self.log = log

        self.runner = SplashScriptRunner(
            lua=self.lua,
            log=self.log,
            splash=splash,
            sandboxed=self.sandboxed,
        )
        self.splash.init_dispatcher(self.runner.dispatch)

    def run(self, main_coro):
        """
        Run main_coro Lua coroutine, passing it a Splash
        instance as an argument. Return a Deferred.
        """
        d = defer.Deferred()

        def return_result(result):
            d.callback(result)

        def return_error(err):
            d.errback(err)

        self.runner.start(
            main_coro=main_coro,
            return_result=return_result,
            return_error=return_error,
        )
        return d

class SplashKernel(Kernel):
    implementation = 'Splash'
    implementation_version = splash.__version__
    language = 'Lua'
    language_version = get_version()
    language_info = {
        'name': 'Splash',
        'mimetype': 'application/x-lua',
        'display_name': 'Splash',
        'language': 'lua',
        'codemirror_mode': {
            "name": "text/x-lua",
        },
        'file_extension': '.lua',
        'pygments_lexer': 'lua'
    }
    banner = "Splash kernel - write browser automation scripts interactively"
    help_links = [
        {
            'text': "Splash Tutorial",
            'url': 'http://splash.readthedocs.org/en/latest/scripting-tutorial.html'
        },
        {
            'text': "Splash Reference",
            'url': 'http://splash.readthedocs.org/en/latest/scripting-ref.html'
        },
        {
            'text': "Programming in Lua",
            'url': 'http://www.lua.org/pil/contents.html'
        },
        {
            'text': "Lua 5.2 Manual",
            'url': 'http://www.lua.org/manual/5.2/'
        },
    ]

    sandboxed = False

    def __init__(self, **kwargs):
        super(SplashKernel, self).__init__(**kwargs)
        self.tab = init_browser()

        self.lua = SplashLuaRuntime(self.sandboxed, "", ())
        self.splash = Splash(lua=self.lua, tab=self.tab)
        self.lua.add_to_globals("splash", self.splash.get_wrapped())
        self.runner = DeferredSplashRunner(self.lua, self.splash, self.sandboxed) #, self.log_msg)
        self.completer = Completer(self.lua)
        self.inspector = Inspector(self.lua)
        #
        # try:
        #     sys.stdout.write = self._print
        #     sys.stderr.write = self._print
        # except:
        #     pass # Can't change stdout

    def send_execute_reply(self, stream, ident, parent, md, reply_content):
        def done(result):
            reply, result, ct = result
            if result:
                data = {
                    'text/plain': result if isinstance(result, basestring) else str(result),
                }
                if isinstance(result, BinaryCapsule):
                    data[result.content_type] = result.as_b64()
                self._publish_execute_result(parent, data, {}, self.execution_count)

            super(SplashKernel, self).send_execute_reply(stream, ident, parent, md, reply)

        assert isinstance(reply_content, defer.Deferred)
        reply_content.addCallback(done)

    def do_execute(self, code, silent, store_history=True, user_expressions=None,
                   allow_stdin=False):
        def success(res):
            result, content_type, headers, status_code = res
            reply = {
                'status': 'ok',
                'execution_count': self.execution_count,
                'payload': [],
                'user_expressions': {},
            }
            return reply, result, content_type or 'text/plain'

        def error(failure):
            text = "<unknown error>"
            try:
                failure.raiseException()
            except (lupa.LuaSyntaxError, lupa.LuaError, ScriptError) as e:
                tp, line_num, message = parse_lua_error(e)
                text = "<%s error> [input]:%s: %s" % (tp, line_num, message)
            except Exception as e:
                text = repr(e)
            reply = {
                'status': 'error',
                'execution_count': self.execution_count,
                'ename': '',
                'evalue': text,
                'traceback': []
            }
            return reply, text, 'text/plain'

        try:
            try:
                # XXX: this ugly formatting is important for exception
                # line numbers to be displayed properly!
                lua_source = 'local repr = require("repr"); function main(splash) return repr(%s) end' % code
                main_coro = self._get_main(lua_source)
            except lupa.LuaSyntaxError:
                try:
                    lines = code.splitlines(False)
                    lua_source = '''local repr = require("repr"); function main(splash) %s
                        return repr(%s)
                    end
                    ''' % ("\n".join(lines[:-1]), lines[-1])
                    main_coro = self._get_main(lua_source)
                except lupa.LuaSyntaxError:
                    lua_source = "function main(splash) %s end" % code
                    main_coro = self._get_main(lua_source)

        except (lupa.LuaSyntaxError, lupa.LuaError) as e:
            d = defer.Deferred()
            d.addCallbacks(success, error)
            d.errback(e)
            return d
        except Exception:
            d = defer.Deferred()
            d.addCallbacks(success, error)
            d.errback()
            return d

        d = self.runner.run(main_coro)
        d.addCallbacks(success, error)
        return d

    def do_complete(self, code, cursor_pos):
        return self.completer.complete(code, cursor_pos)

    def do_inspect(self, code, cursor_pos, detail_level=0):
        return self.inspector.help(code, cursor_pos, detail_level)

    def _publish_execute_result(self, parent, data, metadata, execution_count):
        msg = {
            u'data': data,
            u'metadata': metadata,
            u'execution_count': execution_count
        }
        self.session.send(self.iopub_socket, u'execute_result', msg,
                          parent=parent, ident=self._topic('execute_result')
        )

    def log_msg(self, text, min_level=2):
        self._print(text + "\n")

    def _print(self, message):
        stream_content = {'name': 'stdout', 'text': message, 'metadata': dict()}
        self.log.debug('Write: %s' % message)
        self.send_response(self.iopub_socket, 'stream', stream_content)

    def _get_main(self, lua_source):
        if self.sandboxed:
            main, env = get_main_sandboxed(self.lua, lua_source)
        else:
            main, env = get_main(self.lua, lua_source)
        return self.lua.create_coroutine(main)

def start():
    with xvfb.autostart():
        # FIXME: logs go to nowhere
        init_qt_app(verbose=False)
        kernel = IPKernelApp.instance(kernel_class=SplashKernel)
        kernel.initialize()
        kernel.kernel.eventloop = loop_qt4
        kernel.start()
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

from collections import defaultdict

class Flowmap(dict):
    def __init__(self):
        super(Flowmap, self).__init__()
        self.decorators = []
        self.annotations = {}

    def graph(self):
        result = defaultdict(set)
        backlog = set()
        processed = set()

        for port_out, port_in in self.iteritems():
            result[port_out].add(port_in)
            backlog.add(port_in)

        while len(backlog):
            node = backlog.pop()
            if node in processed:
                continue
            else:
                processed.add(node)

            try:
                arcs = tuple(node.dependencies)
            except AttributeError:
                continue

            for port_out, port_in in arcs:
                result[port_out].add(port_in)
                backlog.add(port_out)
                backlog.add(port_in)

        return result
<commit_msg>Refactor Flowmap into a MutableMapping<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

from collections import defaultdict, MutableMapping

class Flowmap(MutableMapping):
    def __init__(self):
        super(Flowmap, self).__init__()
        self.annotations = {}
        self.connections = {}
        self.decorators = []

    def __getitem__(self, key):
        return self.connections[key]

    def __setitem__(self, key, value):
        self.connections[key] = value

    def __delitem__(self, key):
        del self.connections[key]

    def __iter__(self):
        return iter(self.connections)

    def __len__(self):
        return len(self.connections)

    def graph(self):
        result = defaultdict(set)
        backlog = set()
        processed = set()

        for port_out, port_in in self.iteritems():
            result[port_out].add(port_in)
            backlog.add(port_in)

        while len(backlog):
            node = backlog.pop()
            if node in processed:
                continue
            else:
                processed.add(node)

            try:
                arcs = tuple(node.dependencies)
            except AttributeError:
                continue

            for port_out, port_in in arcs:
                result[port_out].add(port_in)
                backlog.add(port_out)
                backlog.add(port_in)

        return result
<|endoftext|>


<commit_before>import Queue
import pickle
import netaddr
import logging
import requests
import ripestat
import threading

class Probes(object):
    '''return a class with all atlas probes'''
    probe_api_url     = 'https://atlas.ripe.net/api/v1/probe/'
    probe_archive_url = 'https://atlas.ripe.net/api/v1/probe-archive/?format=json'

    def __init__(self, refresh=False, probes_file='probes.p', threads=200):
        '''initialise class'''
        self.probes_file = probes_file
        self.logger      = logging.getLogger('atlas-kibana.Probes')
        self.threads     = threads
        self.queue       = Queue.Queue(self.threads * 2)
        self.probes      = set()
        self.load()
        if refresh:
            self.refresh()
        self.save()

    def exists(self, probe_id):
        '''check if we already have this probe'''
        for probe in self.probes.copy():
            if probe.id == int(probe_id):
                return True
        return False

    def get(self, probe_id):
        '''get an probe element'''
        for probe in self.probes.copy():
            if probe.id == int(probe_id):
                return probe
        return False

    def load(self):
        '''load pickle data'''
        try:
            self.logger.info('Loading probes from pickle file')
            self.probes = pickle.load(open(self.probes_file, 'rb'))
            self.logger.info('Finished loading probes')
        except IOError:
            self.logger.warning('unable to load probes file {}'.format(self.probes_file))

    def load_probe(self, force=False):
        '''load a probe from the queue'''
        while True:
            probe = self.queue.get()
            try:
                if not self.exists(probe['id']):
                    self.probes.add(Probe(probe))
                    self.logger.info('{}:Added'.format(probe['id']))
                else:
                    self.logger.debug('{}:Already Exists'.format(probe['id']))
            except Exception as e:
                # This will likly come back and bite me :S
                self.logger.warning('{}:could not process\n{}'.format(probe['id'], e))
            self.queue.task_done()

    def refresh(self, force=False):
        '''load all atlas probes'''
        try:
            self.logger.info('fetching {}'.format(self.probe_archive_url))
            probes = requests.get(self.probe_archive_url, verify=False).json().get(
                    'objects', [])
            self.logger.info('fetched {}'.format(self.probe_archive_url))
        except Exception as e:
            self.logger.error('Could not fetch {0}:\n{1}'.format(self.probe_archive_url, e))
            return False
        for i in range(self.threads):
            thread        = threading.Thread(target=self.load_probe)
            thread.daemon = True
            thread.start()
        if force:
            self.logger.info('Clering all probes')
            self.probes.clear()
        for probe in probes:
            self.queue.put(probe)
        self.queue.join()
        return True

    def save(self):
        '''save the probes data'''
        pickle.dump(self.probes, open(self.probes_file, 'wb'))

class Probe(object):

    def __init__(self, probe):
        self.logger       = logging.getLogger('atlas-kibana.Probe')
        self.status       = probe.get('status_name',  None)
        self.status_since = probe.get('status_since', None)
        self.address_v4   = probe.get('address_v4',   None)
        self.address_v6   = probe.get('address_v6',   None)
        self.asn_v4       = probe.get('asn_v4',       None)
        self.asn_v6       = probe.get('asn_v6',       None)
        self.country_code = probe.get('country_code', None)
        self.latitude     = probe.get('latitude',     None)
        self.longitude    = probe.get('longitude',    None)
        self.prefix_v4    = probe.get('prefix_v4',    None)
        self.prefix_v6    = probe.get('prefix_v6',    None)
        self.id           = probe.get('id',           None)
        self.is_anchor    = probe.get('is_anchor',    None)
        self.is_public    = probe.get('is_public',    None)
        self.resource_uri = probe.get('resource_uri', None)
        self.tooltip      = 'Probe #{0}: https://atlas.ripe.net/probes/{0}/'.format(self.id)
        self.geojson      = [self.longitude, self.latitude]
        self.stat_api     = ripestat.StatAPI('Atlas-Kibana')
        self.tags         = []
        self.tags.extend(probe.get('tags', None))
        self.location     = dict()
        self.asn_v4_name  = None
        self.asn_v6_name  = None
        self.prefixlen_v4 = None
        self.rir_v4       = None
        self.prefixlen_v6 = None
        self.rir_v6       = None

        self.update()

    def __getstate__(self):
        '''remove logger and api from logger'''
        d = dict(self.__dict__)
        del d['logger']
        del d['stat_api']
        return d

    def __setstate__(self, d):
        '''recreate logger and stat_api'''
        d['stat_api'] = ripestat.StatAPI('Atlas-Kibana')
        d['logger']   = logging.getLogger('atlas-kibana.Probe')
        self.__dict__.update(d)

    def __eq__(self, other):
        '''Check equality'''
        if isinstance(other, Probe):
            return self.id == other.id
        return False

    def __ne__(self, other):
        '''Check inequality'''
        if isinstance(other, Probe):
            return not (self.id == other.id)
        return True

    def __repr__(self):
        '''represent object'''
        return 'Probe({})'.format(self.id)

    def __hash__(self):
        '''hash object'''
        return hash(self.__repr__())

    def get_location_meta(self, country_code):
        '''use restcountries.eu to get the location'''
        url = 'http://restcountries.eu/rest/v1/alpha/{}'.format(country_code.lower())
        try:
            location = requests.get(url, verify=False).json()
            self.logger.debug('{}:Add location for {}'.format(self.id, country_code))
            return location
        except Exception as e:
            self.logger.error('Could not fetch {}:{}\n{}'.format(url, location,  e))

    def get_location(self):
        location          = self.get_location_meta(self.country_code)
        return { 'country'  : location['name'],
                'region'    : location['region'],
                'subregion' : location['subregion']}

    def update(self):
        '''update probe metadata'''
        if self.country_code:
            self.location = self.get_location()
        if self.asn_v4:
            self.asn_v4_name  = self.get_asn_name(self.asn_v4)
        if self.asn_v6:
            if self.asn_v6 == self.asn_v4:
                self.asn_v6_name  = self.asn_v4_name
            else:
                self.asn_v6_name  = self.get_asn_name(self.asn_v6)
        if self.prefix_v4:
            self.prefixlen_v4 = self.get_prefix_len(self.prefix_v4)
            self.rir_v4       = self.get_rir(self.prefix_v4)
        if self.prefix_v6:
            self.prefixlen_v6 = self.get_prefix_len(self.prefix_v6)
            self.rir_v6       = self.get_rir(self.prefix_v6)

    def get_rir(self, prefix):
        '''use RIPEstat to get the rir name'''
        try:
            whois = self.stat_api.get_data('whois', {'resource': prefix})
            rir   = u','.join(whois.get('authorities', [])).encode('utf-8')
            self.logger.debug('{}:Add RIR "{}" for {}'.format(self.id, rir, prefix))
            return rir
        except Exception as e:
            self.logger.error('Could not fetch holder for {}:\n{}'.format(prefix, e))

    def get_asn_name(self, asn):
        '''use RIPEstat to get the asn name'''
        try:
            asn_name = self.stat_api.get_data('as-overview',
                {'resource': asn}).get('holder', None)
            self.logger.debug('{}:Add AS name "{}" for {}'.format(
                self.id, asn_name, asn))
            return asn_name
        except Exception as e:
            self.logger.error('Could not fetch holder for {}:\n{}'.format(asn, e))

    def get_prefix_len(self, prefix):
        try:
            prefix_len = int(netaddr.IPNetwork(prefix).prefixlen)
            self.logger.debug('{}:Add prefix length "{}" for {}'.format(
                self.id, prefix_len, prefix))
            return prefix_len
        except netaddr.AddrFormatError as e:
            self.logger.warning('could not get prefix length: {}'.format(e))
<commit_msg>change log message level\<commit_after>import Queue
import pickle
import netaddr
import logging
import requests
import ripestat
import threading

class Probes(object):
    '''return a class with all atlas probes'''
    probe_api_url     = 'https://atlas.ripe.net/api/v1/probe/'
    probe_archive_url = 'https://atlas.ripe.net/api/v1/probe-archive/?format=json'

    def __init__(self, refresh=False, probes_file='probes.p', threads=200):
        '''initialise class'''
        self.probes_file = probes_file
        self.logger      = logging.getLogger('atlas-kibana.Probes')
        self.threads     = threads
        self.queue       = Queue.Queue(self.threads * 2)
        self.probes      = set()
        self.load()
        if refresh:
            self.refresh()
        self.save()

    def exists(self, probe_id):
        '''check if we already have this probe'''
        for probe in self.probes.copy():
            if probe.id == int(probe_id):
                return True
        return False

    def get(self, probe_id):
        '''get an probe element'''
        for probe in self.probes.copy():
            if probe.id == int(probe_id):
                return probe
        return False

    def load(self):
        '''load pickle data'''
        try:
            self.logger.info('Loading probes from pickle file')
            self.probes = pickle.load(open(self.probes_file, 'rb'))
            self.logger.info('Finished loading probes')
        except IOError:
            self.logger.warning('unable to load probes file {}'.format(self.probes_file))

    def load_probe(self, force=False):
        '''load a probe from the queue'''
        while True:
            probe = self.queue.get()
            try:
                if not self.exists(probe['id']):
                    self.probes.add(Probe(probe))
                    self.logger.info('{}:Added'.format(probe['id']))
                else:
                    self.logger.debug('{}:Already Exists'.format(probe['id']))
            except Exception as e:
                # This will likly come back and bite me :S
                self.logger.warning('{}:could not process\n{}'.format(probe['id'], e))
            self.queue.task_done()

    def refresh(self, force=False):
        '''load all atlas probes'''
        try:
            self.logger.info('fetching {}'.format(self.probe_archive_url))
            probes = requests.get(self.probe_archive_url, verify=False).json().get(
                    'objects', [])
            self.logger.info('fetched {}'.format(self.probe_archive_url))
        except Exception as e:
            self.logger.error('Could not fetch {0}:\n{1}'.format(self.probe_archive_url, e))
            return False
        for i in range(self.threads):
            thread        = threading.Thread(target=self.load_probe)
            thread.daemon = True
            thread.start()
        if force:
            self.logger.info('Clering all probes')
            self.probes.clear()
        for probe in probes:
            self.queue.put(probe)
        self.queue.join()
        return True

    def save(self):
        '''save the probes data'''
        pickle.dump(self.probes, open(self.probes_file, 'wb'))

class Probe(object):

    def __init__(self, probe):
        self.logger       = logging.getLogger('atlas-kibana.Probe')
        self.status       = probe.get('status_name',  None)
        self.status_since = probe.get('status_since', None)
        self.address_v4   = probe.get('address_v4',   None)
        self.address_v6   = probe.get('address_v6',   None)
        self.asn_v4       = probe.get('asn_v4',       None)
        self.asn_v6       = probe.get('asn_v6',       None)
        self.country_code = probe.get('country_code', None)
        self.latitude     = probe.get('latitude',     None)
        self.longitude    = probe.get('longitude',    None)
        self.prefix_v4    = probe.get('prefix_v4',    None)
        self.prefix_v6    = probe.get('prefix_v6',    None)
        self.id           = probe.get('id',           None)
        self.is_anchor    = probe.get('is_anchor',    None)
        self.is_public    = probe.get('is_public',    None)
        self.resource_uri = probe.get('resource_uri', None)
        self.tooltip      = 'Probe #{0}: https://atlas.ripe.net/probes/{0}/'.format(self.id)
        self.geojson      = [self.longitude, self.latitude]
        self.stat_api     = ripestat.StatAPI('Atlas-Kibana')
        self.tags         = []
        self.tags.extend(probe.get('tags', None))
        self.location     = dict()
        self.asn_v4_name  = None
        self.asn_v6_name  = None
        self.prefixlen_v4 = None
        self.rir_v4       = None
        self.prefixlen_v6 = None
        self.rir_v6       = None

        self.update()

    def __getstate__(self):
        '''remove logger and api from logger'''
        d = dict(self.__dict__)
        del d['logger']
        del d['stat_api']
        return d

    def __setstate__(self, d):
        '''recreate logger and stat_api'''
        d['stat_api'] = ripestat.StatAPI('Atlas-Kibana')
        d['logger']   = logging.getLogger('atlas-kibana.Probe')
        self.__dict__.update(d)

    def __eq__(self, other):
        '''Check equality'''
        if isinstance(other, Probe):
            return self.id == other.id
        return False

    def __ne__(self, other):
        '''Check inequality'''
        if isinstance(other, Probe):
            return not (self.id == other.id)
        return True

    def __repr__(self):
        '''represent object'''
        return 'Probe({})'.format(self.id)

    def __hash__(self):
        '''hash object'''
        return hash(self.__repr__())

    def get_location_meta(self, country_code):
        '''use restcountries.eu to get the location'''
        url = 'http://restcountries.eu/rest/v1/alpha/{}'.format(country_code.lower())
        try:
            location = requests.get(url, verify=False).json()
            self.logger.debug('{}:Add location for {}'.format(self.id, country_code))
            return location
        except Exception as e:
            self.logger.error('Could not fetch {}:{}\n{}'.format(url, location,  e))

    def get_location(self):
        location          = self.get_location_meta(self.country_code)
        return { 'country'  : location['name'],
                'region'    : location['region'],
                'subregion' : location['subregion']}

    def update(self):
        '''update probe metadata'''
        if self.country_code:
            self.location = self.get_location()
        if self.asn_v4:
            self.asn_v4_name  = self.get_asn_name(self.asn_v4)
        if self.asn_v6:
            if self.asn_v6 == self.asn_v4:
                self.asn_v6_name  = self.asn_v4_name
            else:
                self.asn_v6_name  = self.get_asn_name(self.asn_v6)
        if self.prefix_v4:
            self.prefixlen_v4 = self.get_prefix_len(self.prefix_v4)
            self.rir_v4       = self.get_rir(self.prefix_v4)
        if self.prefix_v6:
            self.prefixlen_v6 = self.get_prefix_len(self.prefix_v6)
            self.rir_v6       = self.get_rir(self.prefix_v6)

    def get_rir(self, prefix):
        '''use RIPEstat to get the rir name'''
        try:
            whois = self.stat_api.get_data('whois', {'resource': prefix})
            self.logger.info('recived ripe stat info:\n{}'.format(whois))
            rir   = u','.join(whois.get('authorities', [])).encode('utf-8')
            self.logger.debug('{}:Add RIR "{}" for {}'.format(self.id, rir, prefix))
            return rir
        except Exception as e:
            self.logger.error('Could not fetch holder for {}:\n{}'.format(prefix, e))

    def get_asn_name(self, asn):
        '''use RIPEstat to get the asn name'''
        try:
            asn_name = self.stat_api.get_data('as-overview',
                {'resource': asn}).get('holder', None)
            self.logger.debug('{}:Add AS name "{}" for {}'.format(
                self.id, asn_name, asn))
            return asn_name
        except Exception as e:
            self.logger.error('Could not fetch holder for {}:\n{}'.format(asn, e))

    def get_prefix_len(self, prefix):
        try:
            prefix_len = int(netaddr.IPNetwork(prefix).prefixlen)
            self.logger.debug('{}:Add prefix length "{}" for {}'.format(
                self.id, prefix_len, prefix))
            return prefix_len
        except netaddr.AddrFormatError as e:
            self.logger.warning('could not get prefix length: {}'.format(e))
<|endoftext|>


<commit_before>'''
Manage the registry on Windows
'''

def __virtual__():
    '''
    Load this state if the reg module exists
    '''

    return 'reg' if 'reg.read_key' in __salt__ else False

def _parse_key(key):
    '''
    split the full path in the registry to the key and the rest
    '''

    splt = key.split('\\')
    hive = splt.pop(0)
    key = splt.pop(-1)
    path = '\\'.join(splt)
    return hive, path, key

def present(name, value, vtype='REG_DWORD'):
    '''
    Set a registry entry

    Example::

        'HKEY_CURRENT_USER\\SOFTWARE\\Salt\\version':
          reg.present:
            - value: 0.15.3
            - vtype: REG_SZ
    '''

    ret = {'name': name,
           'result': True,
           'changes': {},
           'comment': ''}

    # determine what to do
    hive, path, key = _parse_key(name)
    if value == __salt__['reg.read_key'](hive, path, key):
        ret['comment'] = '{0} is already configured'.format( name )
        return ret
    else:
        ret['changes'] = {'reg': 'configured to {}'.format(value) }

    if __opts__['test']:
        ret['result'] = None
        return ret

    # configure the key
    ret['result'] = __salt__['reg.set_key'](hive, path, key, value, vtype)
    if not ret:
        ret['changes'] = {}
        ret['comment'] = 'could not configure the registry key'

    return ret

<commit_msg>whitespace<commit_after>'''
Manage the registry on Windows
'''

def __virtual__():
    '''
    Load this state if the reg module exists
    '''
    return 'reg' if 'reg.read_key' in __salt__ else False

def _parse_key(key):
    '''
    split the full path in the registry to the key and the rest
    '''
    splt = key.split('\\')
    hive = splt.pop(0)
    key = splt.pop(-1)
    path = '\\'.join(splt)
    return hive, path, key

def present(name, value, vtype='REG_DWORD'):
    '''
    Set a registry entry

    Example::

        'HKEY_CURRENT_USER\\SOFTWARE\\Salt\\version':
          reg.present:
            - value: 0.15.3
            - vtype: REG_SZ
    '''
    ret = {'name': name,
           'result': True,
           'changes': {},
           'comment': ''}

    # determine what to do
    hive, path, key = _parse_key(name)
    if value == __salt__['reg.read_key'](hive, path, key):
        ret['comment'] = '{0} is already configured'.format( name )
        return ret
    else:
        ret['changes'] = {'reg': 'configured to {}'.format(value) }

    if __opts__['test']:
        ret['result'] = None
        return ret

    # configure the key
    ret['result'] = __salt__['reg.set_key'](hive, path, key, value, vtype)
    if not ret:
        ret['changes'] = {}
        ret['comment'] = 'could not configure the registry key'

    return ret
<|endoftext|>


<commit_before>import os

# Copyright (c) 2007-2008,
#   Bill McCloskey    <bill.mccloskey@gmail.com>
# All rights reserved.
# BSD license
# Borrow from http://web.archive.org/web/20090117001916/http://www.cs.berkeley.edu/~billm/memoize.py
def modtime(fname):
    """Return the modification time of a given filename, or None
    if there is a problem. (i.e: file doesn't exist.)
    """
    try:
        if os.path.isdir(fname):
            return 1
        else:
            return os.path.getmtime(fname)
    except:
        return None

def run(spec):
    print spec
    task = spec[0]
    map = spec[1]
    for src in map:
        print "build",map[src],"from",src,"using",task

def build(*tasks):
    print "tasks",tasks
    for task in tasks:
        for f in task.func_defaults:
            print f
        task()

def init():
    print "init"

def antlr(g, depends=(init,)):
    for d in depends:
        d()
    print "antlr4", g

def javac(f, depends=(antlr,)):
    print "javac", f

mantra       = (antlr, {"Mantra.g4": ["MantraParser.java", "MantraLexer.java"]})
mantrajava   = (javac, {"T.java": "T.class"})

def task_antlr(depends=(init,)):
    for d in depends:
        d()

run(mantra)

<commit_msg>must improved proto<commit_after>import os

# Copyright (c) 2007-2008,
#   Bill McCloskey    <bill.mccloskey@gmail.com>
# All rights reserved.
# BSD license
# Borrow from http://web.archive.org/web/20090117001916/http://www.cs.berkeley.edu/~billm/memoize.py
def modtime(fname):
	"""Return the modification time of a given filename, or None
	if there is a problem. (i.e: file doesn't exist.)
	"""
	try:
		if os.path.isdir(fname):
			return 1
		else:
			return os.path.getmtime(fname)
	except:
		return None

_ = None

completed = set() # which *targets* have been built.

def run(spec):
	if type(spec) is type([]):
		for target in spec:
			run(target)
		return
	task = spec[0]
	map = spec[1]
	dependencies = spec[2]
	if id(spec) in completed:
		return
	completed.add(id(spec))
	print "run", spec
	if dependencies is not None:
		if type(dependencies) is type([]):
			for d in dependencies:
				run(d)
		else:
			run(dependencies)
	if map is not None:
		for src in map:
			print "build", map[src], "from", src, "using", task, "depends on", dependencies
			task(src)
	else:
		task() # no src -> target, just exec

# def buildtasks(*tasks):
# 	print "tasks", tasks
# 	for task in tasks:
# 		for f in task.func_defaults:
# 			print f
# 		task()

def task_init():
	print "init"

def antlr(g):
	print "antlr4", g

def javac(f):
	print "javac", f

# target defs are tuples: (task-to-exec, {src:target}, dependencies)
init = (task_init,_,_)
mantra = (antlr, {"Mantra.g4": ["MantraParser.java", "MantraLexer.java"]}, init)
mantrajava = (javac, {"MantraParser.java": "MantraParser.class"}, [mantra,init])
all = [init,mantra] # can be list of targets

def task_antlr(depends=(init,)):
	for d in depends:
		d()

run(all)

<|endoftext|>


<commit_before><commit_msg>Fast sampling, with pytorch<commit_after><|endoftext|>


<commit_before># (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

import os
import pwd
import sys
import ConfigParser

# copied from utils, avoid circular reference fun :)
def mk_boolean(value):
    val = str(value)
    if val.lower() in [ "true", "t", "y", "1", "yes" ]:
        return True
    else:
        return False

def get_config(p, section, key, env_var, default, boolean=False, integer=False):
    ''' return a configuration variable with casting '''
    value = _get_config(p, section, key, env_var, default)
    if boolean:
        return mk_boolean(value)
    if integer:
        return int(value)
    return value

def _get_config(p, section, key, env_var, default, boolean=True):
    ''' helper function for get_config '''
    if env_var is not None:
        value = os.environ.get(env_var, None)
        if value is not None:
            return value
    if p is not None:
        try:
            return p.get(section, key)
        except:
            return default
    return default

def load_config_file():
    p = ConfigParser.ConfigParser()
    path1 = os.path.expanduser(os.environ.get('ANSIBLE_CONFIG', "~/.ansible.cfg"))
    path2 = os.getcwd() + "/ansible.cfg"
    path3 = "/etc/ansible/ansible.cfg"

    if os.path.exists(path1):
        p.read(path1)
    elif os.path.exists(path2):
        p.read(path2)
    elif os.path.exists(path3):
        p.read(path3)
    else:
        return None
    return p

def shell_expand_path(path):
    ''' shell_expand_path is needed as os.path.expanduser does not work
        when path is None, which is the default for ANSIBLE_PRIVATE_KEY_FILE '''
    if path:
        path = os.path.expanduser(path)
    return path

p = load_config_file()

active_user   = pwd.getpwuid(os.geteuid())[0]

# Needed so the RPM can call setup.py and have modules land in the
# correct location. See #1277 for discussion
if getattr(sys, "real_prefix", None):
    DIST_MODULE_PATH = os.path.join(sys.prefix, 'share/ansible/')
else:
    DIST_MODULE_PATH = '/usr/share/ansible/'

# sections in config file
DEFAULTS='defaults'

# configurable things
DEFAULT_HOST_LIST         = shell_expand_path(get_config(p, DEFAULTS, 'hostfile', 'ANSIBLE_HOSTS', '/etc/ansible/hosts'))
DEFAULT_MODULE_PATH       = get_config(p, DEFAULTS, 'library',          'ANSIBLE_LIBRARY',          DIST_MODULE_PATH)
DEFAULT_REMOTE_TMP        = shell_expand_path(get_config(p, DEFAULTS, 'remote_tmp',       'ANSIBLE_REMOTE_TEMP',      '$HOME/.ansible/tmp'))
DEFAULT_MODULE_NAME       = get_config(p, DEFAULTS, 'module_name',      None,                       'command')
DEFAULT_PATTERN           = get_config(p, DEFAULTS, 'pattern',          None,                       '*')
DEFAULT_FORKS             = get_config(p, DEFAULTS, 'forks',            'ANSIBLE_FORKS',            5)
DEFAULT_MODULE_ARGS       = get_config(p, DEFAULTS, 'module_args',      'ANSIBLE_MODULE_ARGS',      '')
DEFAULT_MODULE_LANG       = get_config(p, DEFAULTS, 'module_lang',      'ANSIBLE_MODULE_LANG',      'C')
DEFAULT_TIMEOUT           = get_config(p, DEFAULTS, 'timeout',          'ANSIBLE_TIMEOUT',          10)
DEFAULT_POLL_INTERVAL     = get_config(p, DEFAULTS, 'poll_interval',    'ANSIBLE_POLL_INTERVAL',    15)
DEFAULT_REMOTE_USER       = get_config(p, DEFAULTS, 'remote_user',      'ANSIBLE_REMOTE_USER',      active_user)
DEFAULT_ASK_PASS          = get_config(p, DEFAULTS, 'ask_pass',  'ANSIBLE_ASK_PASS',    False, boolean=True)
DEFAULT_PRIVATE_KEY_FILE  = shell_expand_path(get_config(p, DEFAULTS, 'private_key_file', 'ANSIBLE_PRIVATE_KEY_FILE', None))
DEFAULT_SUDO_USER         = get_config(p, DEFAULTS, 'sudo_user',        'ANSIBLE_SUDO_USER',        'root')
DEFAULT_ASK_SUDO_PASS     = get_config(p, DEFAULTS, 'ask_sudo_pass',    'ANSIBLE_ASK_SUDO_PASS',    False, boolean=True)
DEFAULT_REMOTE_PORT       = int(get_config(p, DEFAULTS, 'remote_port',      'ANSIBLE_REMOTE_PORT',      22))
DEFAULT_TRANSPORT         = get_config(p, DEFAULTS, 'transport',        'ANSIBLE_TRANSPORT',        'smart')
DEFAULT_SCP_IF_SSH        = get_config(p, 'ssh_connection', 'scp_if_ssh',       'ANSIBLE_SCP_IF_SSH',       False, boolean=True)
DEFAULT_MANAGED_STR       = get_config(p, DEFAULTS, 'ansible_managed',  None,           'Ansible managed: {file} modified on %Y-%m-%d %H:%M:%S by {uid} on {host}')
DEFAULT_SYSLOG_FACILITY   = get_config(p, DEFAULTS, 'syslog_facility',  'ANSIBLE_SYSLOG_FACILITY', 'LOG_USER')
DEFAULT_KEEP_REMOTE_FILES = get_config(p, DEFAULTS, 'keep_remote_files', 'ANSIBLE_KEEP_REMOTE_FILES', '0')
DEFAULT_SUDO              = get_config(p, DEFAULTS, 'sudo', 'ANSIBLE_SUDO', False, boolean=True)
DEFAULT_SUDO_EXE          = get_config(p, DEFAULTS, 'sudo_exe', 'ANSIBLE_SUDO_EXE', 'sudo')
DEFAULT_SUDO_FLAGS        = get_config(p, DEFAULTS, 'sudo_flags', 'ANSIBLE_SUDO_FLAGS', '-H')
DEFAULT_HASH_BEHAVIOUR    = get_config(p, DEFAULTS, 'hash_behaviour', 'ANSIBLE_HASH_BEHAVIOUR', 'replace')
DEFAULT_LEGACY_PLAYBOOK_VARIABLES = get_config(p, DEFAULTS, 'legacy_playbook_variables', 'ANSIBLE_LEGACY_PLAYBOOK_VARIABLES', True, boolean=True)
DEFAULT_JINJA2_EXTENSIONS = get_config(p, DEFAULTS, 'jinja2_extensions', 'ANSIBLE_JINJA2_EXTENSIONS', None)
DEFAULT_EXECUTABLE        = get_config(p, DEFAULTS, 'executable', 'ANSIBLE_EXECUTABLE', '/bin/sh')

DEFAULT_ACTION_PLUGIN_PATH     = get_config(p, DEFAULTS, 'action_plugins',     'ANSIBLE_ACTION_PLUGINS', '/usr/share/ansible_plugins/action_plugins')
DEFAULT_CALLBACK_PLUGIN_PATH   = get_config(p, DEFAULTS, 'callback_plugins',   'ANSIBLE_CALLBACK_PLUGINS', '/usr/share/ansible_plugins/callback_plugins')
DEFAULT_CONNECTION_PLUGIN_PATH = get_config(p, DEFAULTS, 'connection_plugins', 'ANSIBLE_CONNECTION_PLUGINS', '/usr/share/ansible_plugins/connection_plugins')
DEFAULT_LOOKUP_PLUGIN_PATH     = get_config(p, DEFAULTS, 'lookup_plugins',     'ANSIBLE_LOOKUP_PLUGINS', '/usr/share/ansible_plugins/lookup_plugins')
DEFAULT_VARS_PLUGIN_PATH       = get_config(p, DEFAULTS, 'vars_plugins',       'ANSIBLE_VARS_PLUGINS', '/usr/share/ansible_plugins/vars_plugins')
DEFAULT_FILTER_PLUGIN_PATH     = get_config(p, DEFAULTS, 'filter_plugins',     'ANSIBLE_FILTER_PLUGINS', '/usr/share/ansible_plugins/filter_plugins')
DEFAULT_LOG_PATH               = shell_expand_path(get_config(p, DEFAULTS, 'log_path',           'ANSIBLE_LOG_PATH', ''))

ANSIBLE_NOCOWS                 = get_config(p, DEFAULTS, 'nocows', 'ANSIBLE_NOCOWS', None)
ANSIBLE_SSH_ARGS               = get_config(p, 'ssh_connection', 'ssh_args', 'ANSIBLE_SSH_ARGS', None)
PARAMIKO_RECORD_HOST_KEYS      = get_config(p, 'paramiko_connection', 'record_host_keys', 'ANSIBLE_PARAMIKO_RECORD_HOST_KEYS', True, boolean=True)
ZEROMQ_PORT                    = int(get_config(p, 'fireball_connection', 'zeromq_port', 'ANSIBLE_ZEROMQ_PORT', 5099))

DEFAULT_UNDEFINED_VAR_BEHAVIOR = get_config(p, DEFAULTS, 'error_on_undefined_vars', 'ANSIBLE_ERROR_ON_UNDEFINED_VARS', False, boolean=True)
HOST_KEY_CHECKING              = get_config(p, DEFAULTS, 'host_key_checking',  'ANSIBLE_HOST_KEY_CHECKING',    True, boolean=True)

# non-configurable things
DEFAULT_SUDO_PASS         = None
DEFAULT_REMOTE_PASS       = None
DEFAULT_SUBSET            = None

<commit_msg>expand vars like $HOME used in config values too<commit_after># (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

import os
import pwd
import sys
import ConfigParser

# copied from utils, avoid circular reference fun :)
def mk_boolean(value):
    val = str(value)
    if val.lower() in [ "true", "t", "y", "1", "yes" ]:
        return True
    else:
        return False

def get_config(p, section, key, env_var, default, boolean=False, integer=False):
    ''' return a configuration variable with casting '''
    value = _get_config(p, section, key, env_var, default)
    if boolean:
        return mk_boolean(value)
    if integer:
        return int(value)
    return value

def _get_config(p, section, key, env_var, default, boolean=True):
    ''' helper function for get_config '''
    if env_var is not None:
        value = os.environ.get(env_var, None)
        if value is not None:
            return value
    if p is not None:
        try:
            return p.get(section, key)
        except:
            return default
    return default

def load_config_file():
    p = ConfigParser.ConfigParser()
    path1 = os.path.expanduser(os.environ.get('ANSIBLE_CONFIG', "~/.ansible.cfg"))
    path2 = os.getcwd() + "/ansible.cfg"
    path3 = "/etc/ansible/ansible.cfg"

    if os.path.exists(path1):
        p.read(path1)
    elif os.path.exists(path2):
        p.read(path2)
    elif os.path.exists(path3):
        p.read(path3)
    else:
        return None
    return p

def shell_expand_path(path):
    ''' shell_expand_path is needed as os.path.expanduser does not work
        when path is None, which is the default for ANSIBLE_PRIVATE_KEY_FILE '''
    if path:
        path = os.path.expandvars(os.path.expanduser(path))
    return path

p = load_config_file()

active_user   = pwd.getpwuid(os.geteuid())[0]

# Needed so the RPM can call setup.py and have modules land in the
# correct location. See #1277 for discussion
if getattr(sys, "real_prefix", None):
    DIST_MODULE_PATH = os.path.join(sys.prefix, 'share/ansible/')
else:
    DIST_MODULE_PATH = '/usr/share/ansible/'

# sections in config file
DEFAULTS='defaults'

# configurable things
DEFAULT_HOST_LIST         = shell_expand_path(get_config(p, DEFAULTS, 'hostfile', 'ANSIBLE_HOSTS', '/etc/ansible/hosts'))
DEFAULT_MODULE_PATH       = get_config(p, DEFAULTS, 'library',          'ANSIBLE_LIBRARY',          DIST_MODULE_PATH)
DEFAULT_REMOTE_TMP        = shell_expand_path(get_config(p, DEFAULTS, 'remote_tmp',       'ANSIBLE_REMOTE_TEMP',      '$HOME/.ansible/tmp'))
DEFAULT_MODULE_NAME       = get_config(p, DEFAULTS, 'module_name',      None,                       'command')
DEFAULT_PATTERN           = get_config(p, DEFAULTS, 'pattern',          None,                       '*')
DEFAULT_FORKS             = get_config(p, DEFAULTS, 'forks',            'ANSIBLE_FORKS',            5)
DEFAULT_MODULE_ARGS       = get_config(p, DEFAULTS, 'module_args',      'ANSIBLE_MODULE_ARGS',      '')
DEFAULT_MODULE_LANG       = get_config(p, DEFAULTS, 'module_lang',      'ANSIBLE_MODULE_LANG',      'C')
DEFAULT_TIMEOUT           = get_config(p, DEFAULTS, 'timeout',          'ANSIBLE_TIMEOUT',          10)
DEFAULT_POLL_INTERVAL     = get_config(p, DEFAULTS, 'poll_interval',    'ANSIBLE_POLL_INTERVAL',    15)
DEFAULT_REMOTE_USER       = get_config(p, DEFAULTS, 'remote_user',      'ANSIBLE_REMOTE_USER',      active_user)
DEFAULT_ASK_PASS          = get_config(p, DEFAULTS, 'ask_pass',  'ANSIBLE_ASK_PASS',    False, boolean=True)
DEFAULT_PRIVATE_KEY_FILE  = shell_expand_path(get_config(p, DEFAULTS, 'private_key_file', 'ANSIBLE_PRIVATE_KEY_FILE', None))
DEFAULT_SUDO_USER         = get_config(p, DEFAULTS, 'sudo_user',        'ANSIBLE_SUDO_USER',        'root')
DEFAULT_ASK_SUDO_PASS     = get_config(p, DEFAULTS, 'ask_sudo_pass',    'ANSIBLE_ASK_SUDO_PASS',    False, boolean=True)
DEFAULT_REMOTE_PORT       = int(get_config(p, DEFAULTS, 'remote_port',      'ANSIBLE_REMOTE_PORT',      22))
DEFAULT_TRANSPORT         = get_config(p, DEFAULTS, 'transport',        'ANSIBLE_TRANSPORT',        'smart')
DEFAULT_SCP_IF_SSH        = get_config(p, 'ssh_connection', 'scp_if_ssh',       'ANSIBLE_SCP_IF_SSH',       False, boolean=True)
DEFAULT_MANAGED_STR       = get_config(p, DEFAULTS, 'ansible_managed',  None,           'Ansible managed: {file} modified on %Y-%m-%d %H:%M:%S by {uid} on {host}')
DEFAULT_SYSLOG_FACILITY   = get_config(p, DEFAULTS, 'syslog_facility',  'ANSIBLE_SYSLOG_FACILITY', 'LOG_USER')
DEFAULT_KEEP_REMOTE_FILES = get_config(p, DEFAULTS, 'keep_remote_files', 'ANSIBLE_KEEP_REMOTE_FILES', '0')
DEFAULT_SUDO              = get_config(p, DEFAULTS, 'sudo', 'ANSIBLE_SUDO', False, boolean=True)
DEFAULT_SUDO_EXE          = get_config(p, DEFAULTS, 'sudo_exe', 'ANSIBLE_SUDO_EXE', 'sudo')
DEFAULT_SUDO_FLAGS        = get_config(p, DEFAULTS, 'sudo_flags', 'ANSIBLE_SUDO_FLAGS', '-H')
DEFAULT_HASH_BEHAVIOUR    = get_config(p, DEFAULTS, 'hash_behaviour', 'ANSIBLE_HASH_BEHAVIOUR', 'replace')
DEFAULT_LEGACY_PLAYBOOK_VARIABLES = get_config(p, DEFAULTS, 'legacy_playbook_variables', 'ANSIBLE_LEGACY_PLAYBOOK_VARIABLES', True, boolean=True)
DEFAULT_JINJA2_EXTENSIONS = get_config(p, DEFAULTS, 'jinja2_extensions', 'ANSIBLE_JINJA2_EXTENSIONS', None)
DEFAULT_EXECUTABLE        = get_config(p, DEFAULTS, 'executable', 'ANSIBLE_EXECUTABLE', '/bin/sh')

DEFAULT_ACTION_PLUGIN_PATH     = get_config(p, DEFAULTS, 'action_plugins',     'ANSIBLE_ACTION_PLUGINS', '/usr/share/ansible_plugins/action_plugins')
DEFAULT_CALLBACK_PLUGIN_PATH   = get_config(p, DEFAULTS, 'callback_plugins',   'ANSIBLE_CALLBACK_PLUGINS', '/usr/share/ansible_plugins/callback_plugins')
DEFAULT_CONNECTION_PLUGIN_PATH = get_config(p, DEFAULTS, 'connection_plugins', 'ANSIBLE_CONNECTION_PLUGINS', '/usr/share/ansible_plugins/connection_plugins')
DEFAULT_LOOKUP_PLUGIN_PATH     = get_config(p, DEFAULTS, 'lookup_plugins',     'ANSIBLE_LOOKUP_PLUGINS', '/usr/share/ansible_plugins/lookup_plugins')
DEFAULT_VARS_PLUGIN_PATH       = get_config(p, DEFAULTS, 'vars_plugins',       'ANSIBLE_VARS_PLUGINS', '/usr/share/ansible_plugins/vars_plugins')
DEFAULT_FILTER_PLUGIN_PATH     = get_config(p, DEFAULTS, 'filter_plugins',     'ANSIBLE_FILTER_PLUGINS', '/usr/share/ansible_plugins/filter_plugins')
DEFAULT_LOG_PATH               = shell_expand_path(get_config(p, DEFAULTS, 'log_path',           'ANSIBLE_LOG_PATH', ''))

ANSIBLE_NOCOWS                 = get_config(p, DEFAULTS, 'nocows', 'ANSIBLE_NOCOWS', None)
ANSIBLE_SSH_ARGS               = get_config(p, 'ssh_connection', 'ssh_args', 'ANSIBLE_SSH_ARGS', None)
PARAMIKO_RECORD_HOST_KEYS      = get_config(p, 'paramiko_connection', 'record_host_keys', 'ANSIBLE_PARAMIKO_RECORD_HOST_KEYS', True, boolean=True)
ZEROMQ_PORT                    = int(get_config(p, 'fireball_connection', 'zeromq_port', 'ANSIBLE_ZEROMQ_PORT', 5099))

DEFAULT_UNDEFINED_VAR_BEHAVIOR = get_config(p, DEFAULTS, 'error_on_undefined_vars', 'ANSIBLE_ERROR_ON_UNDEFINED_VARS', False, boolean=True)
HOST_KEY_CHECKING              = get_config(p, DEFAULTS, 'host_key_checking',  'ANSIBLE_HOST_KEY_CHECKING',    True, boolean=True)

# non-configurable things
DEFAULT_SUDO_PASS         = None
DEFAULT_REMOTE_PASS       = None
DEFAULT_SUBSET            = None

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
###############################################################################
#
#    Copyright (C) 2001-2014 Micronaet SRL (<http://www.micronaet.it>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
import os
import sys
import logging
import openerp
import openerp.netsvc as netsvc
import openerp.addons.decimal_precision as dp
from openerp.osv import fields, osv, expression, orm
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
from openerp import SUPERUSER_ID, api
from openerp import tools
from openerp.tools.translate import _
from openerp.tools.float_utils import float_round as round
from openerp.tools import (DEFAULT_SERVER_DATE_FORMAT,
    DEFAULT_SERVER_DATETIME_FORMAT,
    DATETIME_FORMATS_MAP,
    float_compare)

_logger = logging.getLogger(__name__)

class MRPBom(orm.Model):
    """ Model name: MRPBom
    """
    _inherit = 'mrp.bom'

    # EXTRA BLOCK -------------------------------------------------------------
    def migrate_assign_parent_bom(self, cr, uid, ids, context=None):
        ''' Migrate bom in dynamic way
        '''
        assert len(ids) == 1, 'Works only with one record a time'

        line_pool = self.pool.get('mrp.bom.line')

        # Create rule in dynamic:
        bom_proxy = self.browse(cr, uid, ids, context=context)[0]
        structure = bom_proxy.product_id.structure_id

        for line in bom_proxy.bom_line_ids:
            line_pool.create(cr, uid, {
                'bom_id': structure.dynamic_bom_id.id,
                'product_id': line.product_id.id,
                'dynamic_mask': '%s%s' % (bom_proxy.product_id.code, '%'),
                'product_qty': line.product_qty,
                'product_uom': line.product_uom.id,
                }, context=context)

        # Move in to be remove category
        self.write(cr, uid, ids, {
            'bom_category': 'remove',
            }, context=context)
        return True

    def migrate_assign_product_bom_product(self, cr, uid, ids, context=None):
        ''' Loop button:
        '''
        product_pool = self.pool.get('product.product')

        product_ids = self.search(cr, uid, [
            ('bom_category', '=', 'product')], context=context)

        import pdb; pdb.set_trace()
        # LOG operation (TODO remove)
        log_f = open(os.path.expanduser('~/bom.csv'), 'w')

        for item in product_ids:
            message = self.migrate_assign_product_bom_product1(
                cr, uid, [item], context=context)
            log_f.write(message)
        log_f.close()
        return True

    def migrate_assign_product_bom_product1(self, cr, uid, ids, context=None):
        ''' Migrate bom in dynamic way
            Create half work element
            Create BOM
            Create move in dynamic
        '''
        assert len(ids) == 1, 'Works only with one record a time'

        line_pool = self.pool.get('mrp.bom.line')
        product_pool = self.pool.get('product.product')

        bom_proxy = self.browse(cr, uid, ids, context=context)[0]

        log = ''
        # ---------------------------------------------------------------------
        # Create dynamic mask from code:
        # ---------------------------------------------------------------------
        if not bom_proxy.product_id:
            pr_ids = product_pool.search(cr, uid, [
                ('product_tmpl_id', '=', bom_proxy.product_tmpl_id.id),
                ], context=context)
            if pr_ids:
                default_code = product_pool.browse(
                    cr, uid, pr_ids, context=context)[0]
        default_code = bom_proxy.product_id.default_code
        if not default_code:
            log += '|%s|No default_code for BOM' % bom_proxy.name
            return log

        # Set mask for unique element S and no S are the same
        if default_code[12:13] == 'S':
            dynamic_mask = default_code[:12] + '%'
        else:
            dynamic_mask = default_code + '%'
        log += '\n%s|Mask: %s (%s) # comp: %s|New BOM\n' % (
            default_code,
            dynamic_mask,
            bom_proxy.name,
            len(bom_proxy.bom_line_ids),
            )

        # TODO Check if dynamic and product are jet present

        # ---------------------------------------------------------------------
        # Create TL element:
        # ---------------------------------------------------------------------
        for line in bom_proxy.bom_line_ids:
            component_code = line.product_id.default_code
            if not component_code:
                log += '|%s|No component_code found\n' % bom_proxy.name
                return log

            TL_code = 'TL%s%s%s' % (
                component_code[:3],
                component_code[3:6],
                component_code[6:8],
                )
            component_ids = product_pool.search(cr, uid, [
                ('default_code', '=', TL_code)], context=context)
            if component_ids:
                if len(component_ids) > 1:
                    log += '|%s|Component more than one\n' % TL_code
                else:
                    log += '|%s|\n' % TL_code
            else:
                log += '|%s|Component not found in ODOO\n' % TL_code
        print log
        return log

        # ---------------------------------------------------------------------
        # Create TL BOM:
        # ---------------------------------------------------------------------

        # ---------------------------------------------------------------------
        # Create rule in dynamic:
        # ---------------------------------------------------------------------
        structure = bom_proxy.product_id.structure_id

        #for line in bom_proxy.bom_line_ids:
        #    line_pool.create(cr, uid, {
        #        'bom_id': structure.dynamic_bom_id.id,
        #        'product_id': line.product_id.id,
        #        'dynamic_mask': '%s%s' % (bom_proxy.product_id.code, '%'), # TODO remove S finale
        #        'product_qty': line.product_qty,
        #        'product_uom': line.product_uom.id,
        #        }, context=context)

        # Move in to be remove category
        #self.write(cr, uid, ids, {
        #    'bom_category': 'remove',
        #    }, context=context)
        return True
    # EXTRA BLOCK -------------------------------------------------------------

    def get_config_parameter_list(self, cr, uid, context=None):
        ''' Read parameter:
        '''
        key = 'product.default.product.parent.bom'
        config_pool = self.pool.get('ir.config_parameter')
        config_ids = config_pool.search(cr, uid, [
            ('key', '=', key)], context=context)
        if not config_ids:
            _logger.warning('Parameter not found: %s' % key)
            return []
        config_proxy = config_pool.browse(
            cr, uid, config_ids, context=context)[0]
        return eval(config_proxy.value)

    def assign_parent_bom(self, cr, uid, ids, context=None):
        ''' Assign bom depend on code format
        '''
        bom_code_split = self.get_config_parameter_list(
            cr, uid, context=context)
        if not bom_code_split:
            raise osv.except_osv(
                _('Error'),
                _('Setup config paremeter!'))

        bom_proxy = self.browse(cr, uid, ids, context=context)[0]
        default_code = bom_proxy.product_id.default_code
        if not default_code:
            raise osv.except_osv(
                _('Error'),
                _('No default code in product!'))

        bom_ids = False
        for to in bom_code_split:
            if len(default_code) <= to:
                continue
            partial = default_code[0:to]
            bom_ids = self.search(cr, uid, [
                ('bom_category', '=', 'half'),
                ('product_id.default_code', '=', partial),
                ], context=context)
            if bom_ids:
                break

        if not bom_ids:
            raise osv.except_osv(
                _('Error'),
                _('No default code in product!'))

        if len(bom_ids) > 1:
            _logger.error('Found more parent bom!')

        return self.write(cr, uid, ids, {
            'subparent_id': bom_ids[0],
            }, context=context)

    _columns = {
        'subparent_id': fields.many2one(
            'mrp.bom', 'Sub parent bom'),
        }
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>try format beetter<commit_after># -*- coding: utf-8 -*-
###############################################################################
#
#    Copyright (C) 2001-2014 Micronaet SRL (<http://www.micronaet.it>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
import os
import sys
import logging
import openerp
import openerp.netsvc as netsvc
import openerp.addons.decimal_precision as dp
from openerp.osv import fields, osv, expression, orm
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
from openerp import SUPERUSER_ID, api
from openerp import tools
from openerp.tools.translate import _
from openerp.tools.float_utils import float_round as round
from openerp.tools import (DEFAULT_SERVER_DATE_FORMAT,
    DEFAULT_SERVER_DATETIME_FORMAT,
    DATETIME_FORMATS_MAP,
    float_compare)

_logger = logging.getLogger(__name__)

class MRPBom(orm.Model):
    """ Model name: MRPBom
    """
    _inherit = 'mrp.bom'

    # EXTRA BLOCK -------------------------------------------------------------
    def migrate_assign_parent_bom(self, cr, uid, ids, context=None):
        ''' Migrate bom in dynamic way
        '''
        assert len(ids) == 1, 'Works only with one record a time'

        line_pool = self.pool.get('mrp.bom.line')

        # Create rule in dynamic:
        bom_proxy = self.browse(cr, uid, ids, context=context)[0]
        structure = bom_proxy.product_id.structure_id

        for line in bom_proxy.bom_line_ids:
            line_pool.create(cr, uid, {
                'bom_id': structure.dynamic_bom_id.id,
                'product_id': line.product_id.id,
                'dynamic_mask': '%s%s' % (bom_proxy.product_id.code, '%'),
                'product_qty': line.product_qty,
                'product_uom': line.product_uom.id,
                }, context=context)

        # Move in to be remove category
        self.write(cr, uid, ids, {
            'bom_category': 'remove',
            }, context=context)
        return True

    def migrate_assign_product_bom_product(self, cr, uid, ids, context=None):
        ''' Loop button:
        '''
        product_pool = self.pool.get('product.product')

        product_ids = self.search(cr, uid, [
            ('bom_category', '=', 'product')], context=context)

        # LOG operation (TODO remove)
        log_f = open(os.path.expanduser('~/bom.csv'), 'w')

        for item in product_ids:
            message = self.migrate_assign_product_bom_product1(
                cr, uid, [item], context=context)
            log_f.write(message)
        log_f.close()
        return True

    def migrate_assign_product_bom_product1(self, cr, uid, ids, context=None):
        ''' Migrate bom in dynamic way
            Create half work element
            Create BOM
            Create move in dynamic
        '''
        assert len(ids) == 1, 'Works only with one record a time'

        line_pool = self.pool.get('mrp.bom.line')
        product_pool = self.pool.get('product.product')

        bom_proxy = self.browse(cr, uid, ids, context=context)[0]

        log = ''
        # ---------------------------------------------------------------------
        # Create dynamic mask from code:
        # ---------------------------------------------------------------------
        if not bom_proxy.product_id:
            pr_ids = product_pool.search(cr, uid, [
                ('product_tmpl_id', '=', bom_proxy.product_tmpl_id.id),
                ], context=context)
            if pr_ids:
                default_code = product_pool.browse(
                    cr, uid, pr_ids, context=context)[0]
        default_code = bom_proxy.product_id.default_code
        if not default_code:
            log += '\n???|%s||No codice x BOM\n' % (
                bom_proxy.name, len(bom_proxy.bom_line_ids))
            return log

        # Set mask for unique element S and no S are the same
        if default_code[12:13] == 'S':
            dynamic_mask = default_code[:12] + '%'
        else:
            dynamic_mask = default_code + '%'
        log += '\n%s|%s|%s|%s\n' % (
            default_code,
            len(bom_proxy.bom_line_ids),
            dynamic_mask,
            bom_proxy.name or '???', # or bom_proxy.product_tmpl_id.name,
            )

        # TODO Check if dynamic and product are jet present

        # ---------------------------------------------------------------------
        # Create TL element:
        # ---------------------------------------------------------------------
        for line in bom_proxy.bom_line_ids:
            component_code = line.product_id.default_code
            if not component_code:
                log += '|||%s||No codice componente\n' % bom_proxy.name
                return log

            TL_code = 'TL%s%s%s' % (
                default_code[:3],
                default_code[3:6],
                default_code[8:12],
                )
            component_ids = product_pool.search(cr, uid, [
                ('default_code', '=', TL_code)], context=context)
            if component_ids:
                if len(component_ids) > 1:
                    log += '||||%s|%s|Piu componenti|NO\n' % (
                        component_code, TL_code)
                else:
                    log += '||||%s|%s||SI\n' % (
                        component_code, TL_code)
            else:
                log += '||||%s|%s|Non trovato in ODOO|NO\n' % (
                    component_code, TL_code)
        print log
        return log

        # ---------------------------------------------------------------------
        # Create TL BOM:
        # ---------------------------------------------------------------------

        # ---------------------------------------------------------------------
        # Create rule in dynamic:
        # ---------------------------------------------------------------------
        structure = bom_proxy.product_id.structure_id

        #for line in bom_proxy.bom_line_ids:
        #    line_pool.create(cr, uid, {
        #        'bom_id': structure.dynamic_bom_id.id,
        #        'product_id': line.product_id.id,
        #        'dynamic_mask': '%s%s' % (bom_proxy.product_id.code, '%'), # TODO remove S finale
        #        'product_qty': line.product_qty,
        #        'product_uom': line.product_uom.id,
        #        }, context=context)

        # Move in to be remove category
        #self.write(cr, uid, ids, {
        #    'bom_category': 'remove',
        #    }, context=context)
        return True
    # EXTRA BLOCK -------------------------------------------------------------

    def get_config_parameter_list(self, cr, uid, context=None):
        ''' Read parameter:
        '''
        key = 'product.default.product.parent.bom'
        config_pool = self.pool.get('ir.config_parameter')
        config_ids = config_pool.search(cr, uid, [
            ('key', '=', key)], context=context)
        if not config_ids:
            _logger.warning('Parameter not found: %s' % key)
            return []
        config_proxy = config_pool.browse(
            cr, uid, config_ids, context=context)[0]
        return eval(config_proxy.value)

    def assign_parent_bom(self, cr, uid, ids, context=None):
        ''' Assign bom depend on code format
        '''
        bom_code_split = self.get_config_parameter_list(
            cr, uid, context=context)
        if not bom_code_split:
            raise osv.except_osv(
                _('Error'),
                _('Setup config paremeter!'))

        bom_proxy = self.browse(cr, uid, ids, context=context)[0]
        default_code = bom_proxy.product_id.default_code
        if not default_code:
            raise osv.except_osv(
                _('Error'),
                _('No default code in product!'))

        bom_ids = False
        for to in bom_code_split:
            if len(default_code) <= to:
                continue
            partial = default_code[0:to]
            bom_ids = self.search(cr, uid, [
                ('bom_category', '=', 'half'),
                ('product_id.default_code', '=', partial),
                ], context=context)
            if bom_ids:
                break

        if not bom_ids:
            raise osv.except_osv(
                _('Error'),
                _('No default code in product!'))

        if len(bom_ids) > 1:
            _logger.error('Found more parent bom!')

        return self.write(cr, uid, ids, {
            'subparent_id': bom_ids[0],
            }, context=context)

    _columns = {
        'subparent_id': fields.many2one(
            'mrp.bom', 'Sub parent bom'),
        }
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>