<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4
#
#   Copyright 2011 Chris Behrens
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

"""Pyhole Plugin Library"""

import functools
import os
import sys

def _reset_variables():
    """
    Local function to init some variables that are common between
    load and reload
    """

    global _plugin_instances
    global _plugin_hooks
    _plugin_instances = []
    _plugin_hooks = {}
    for x in _hook_names:
        _plugin_hooks[x] = []

# Decorator for adding a hook
def hook_add(hookname, arg):
    """
    Generic decorator to add hooks.  Generally, this is not called
    directly by plugins.  Decorators that plugins use are automatically
    generated below with the setattrs you'll see
    """

    def wrap(f):
        setattr(f, "_is_%s_hook" % hookname, True)
        f._hook_arg = arg
        return f
    return wrap

def hook_get(hookname):
    """
    Function to return the list of hooks of a particular type.  Genearlly
    this is not called directly.  Callers tend to use the dynamically
    generated calls 'hook_get_*' that are created below with the setattrs
    """

    return _plugin_hooks[hookname]

def active_get(hookname):
    """
    Function to return the list of hook arguments.  Genearlly
    this is not called directly.  Callers tend to use the dynamically
    generated calls 'active_get_*' that are created below with the
    setattrs
    """

    return [x[2] for x in _plugin_hooks[hookname]]

_hook_names = ["keyword", "command", "msg_regex"]
_reset_variables()
_this_mod = sys.modules[__name__]

for x in _hook_names:
    # Dynamically create the decorators and functions for various hooks
    setattr(_this_mod, "hook_add_%s" % x, functools.partial(hook_add, x))
    setattr(_this_mod, "hook_get_%ss" % x, functools.partial(hook_get, x))
    setattr(_this_mod, "active_%ss" % x, functools.partial(active_get, x))

class PluginMetaClass(type):
    """
    The metaclass that makes all of the plugin magic work.  All subclassing
    gets caught here, which we can use to have plugins automagically
    register themselves
    """

    def __init__(cls, name, bases, attrs):
        """
        Catch subclassing.  If the class doesn't yet have _plugin_classes,
        it means it's the Plugin class itself, otherwise it's a class
        that's been subclassed from Plugin (ie, a real plugin class)
        """

        if not hasattr(cls, "_plugin_classes"):
            cls._plugin_classes = []
        else:
            cls._plugin_classes.append(cls)
        cls.__name__ = name

class Plugin(object):
    """
    The class that all plugin classes should inherit from
    """

    # Set the metaclass
    __metaclass__ = PluginMetaClass

    def __init__(self, irc, *args, **kwargs):
        """
        Default constructor for Plugin.  Stores the IRC instance, etc
        """

        self.irc = irc
        self.name = self.__class__.__name__

def _init_plugins(*args, **kwargs):
    """
    Create instances of the plugin classes and create a cache
    of their hook functions
    """

    for cls in Plugin._plugin_classes:
        # Create instance of 'p'
        instance = cls(*args, **kwargs)
        # Store the instance
        _plugin_instances.append(instance)

        # Setup _keyword_hooks by looking at all of the attributes
        # in the class and finding the ones that have a _is_*_hook
        # attribute
        for attr_name in dir(instance):
            attr = getattr(instance, attr_name)

            for hook_key in _hook_names:
                if getattr(attr, "_is_%s_hook" % hook_key, False):
                    hook_arg = getattr(attr, "_hook_arg", None)
                    # Append (module, method, arg) tuple
                    _plugin_hooks[hook_key].append(
                            (attr.__module__, attr, hook_arg))

def load_plugins(plugindir, *args, **kwargs):
    """
    Module function that loads plugins from a particular directory
    """

    plugins = os.path.abspath(plugindir)
    plugin_names = (x[:-3] for x in os.listdir(plugins) if x.endswith(".py")
            and not x.startswith("_"))
    for p in plugin_names:
        try:
            __import__(os.path.basename(plugindir), globals(), locals(), [p])
        except ImportError:
            # log something here?
            pass
    _init_plugins(*args, **kwargs)

def reload_plugins(plugins, *args, **kwargs):
    """
    Module function that'll reload all of the plugins
    """

    # When the modules are reloaded, the meta class will append
    # all of the classes again, so we need to make sure this is empty
    Plugin._plugin_classes = []
    _reset_variables()

    # Now reload all of the plugins
    plugins_to_reload = []
    plugindir = os.path.basename(plugins)

    # reload plugins
    for mod, val in sys.modules.items():
        if plugindir in mod and val and mod != plugindir:
            mod_file = val.__file__
            if not os.path.isfile(mod_file):
                continue
            if mod_file.endswith('.pyc') or mod_file.endswith('.pyo'):
                source_file_guess = mod_file[:-1]
                if not os.path.isfile(source_file_guess):
                    os.unlink(mod_file)
            plugins_to_reload.append(mod)
    for p in plugins_to_reload:
        try:
            reload(sys.modules[p])
        except Exception, err:
            # log something here?
            pass
    # look for new plugins to load
    plugin_names = (x[:-3] for x in os.listdir(plugins) if x.endswith(".py")
            and not x.startswith("_") and x not in plugins_to_reload)
    for p in plugin_names:
        try:
            __import__(os.path.basename(plugindir), globals(), locals(), [p])
        except ImportError:
            # log something here?
            pass
    _init_plugins(*args, **kwargs)

def active_plugins():
    """
    Get the loaded plugin names
    """

    return [x.__name__ for x in Plugin._plugin_classes]

def active_plugin_classes():
    """
    Get the loaded plugin classes
    """

    return Plugin._plugin_classes
<commit_msg>Updated some comments.<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4
#
#   Copyright 2011 Chris Behrens
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

"""Pyhole Plugin Library"""

import functools
import os
import sys

def _reset_variables():
    """
    Local function to init some variables that are common between
    load and reload
    """

    global _plugin_instances
    global _plugin_hooks
    _plugin_instances = []
    _plugin_hooks = {}
    for x in _hook_names:
        _plugin_hooks[x] = []

# Decorator for adding a hook
def hook_add(hookname, arg):
    """
    Generic decorator to add hooks.  Generally, this is not called
    directly by plugins.  Decorators that plugins use are automatically
    generated below with the setattrs you'll see
    """

    def wrap(f):
        setattr(f, "_is_%s_hook" % hookname, True)
        f._hook_arg = arg
        return f
    return wrap

def hook_get(hookname):
    """
    Function to return the list of hooks of a particular type.  Genearlly
    this is not called directly.  Callers tend to use the dynamically
    generated calls 'hook_get_*' that are created below with the setattrs
    """

    return _plugin_hooks[hookname]

def active_get(hookname):
    """
    Function to return the list of hook arguments.  Genearlly
    this is not called directly.  Callers tend to use the dynamically
    generated calls 'active_get_*' that are created below with the
    setattrs
    """

    return [x[2] for x in _plugin_hooks[hookname]]

_hook_names = ["keyword", "command", "msg_regex"]
_reset_variables()
_this_mod = sys.modules[__name__]

for x in _hook_names:
    # Dynamically create the decorators and functions for various hooks
    setattr(_this_mod, "hook_add_%s" % x, functools.partial(hook_add, x))
    setattr(_this_mod, "hook_get_%ss" % x, functools.partial(hook_get, x))
    setattr(_this_mod, "active_%ss" % x, functools.partial(active_get, x))

class PluginMetaClass(type):
    """
    The metaclass that makes all of the plugin magic work.  All subclassing
    gets caught here, which we can use to have plugins automagically
    register themselves
    """

    def __init__(cls, name, bases, attrs):
        """
        Catch subclassing.  If the class doesn't yet have _plugin_classes,
        it means it's the Plugin class itself, otherwise it's a class
        that's been subclassed from Plugin (ie, a real plugin class)
        """

        if not hasattr(cls, "_plugin_classes"):
            cls._plugin_classes = []
        else:
            cls._plugin_classes.append(cls)
        cls.__name__ = name

class Plugin(object):
    """
    The class that all plugin classes should inherit from
    """

    # Set the metaclass
    __metaclass__ = PluginMetaClass

    def __init__(self, irc, *args, **kwargs):
        """
        Default constructor for Plugin.  Stores the IRC instance, etc
        """

        self.irc = irc
        self.name = self.__class__.__name__

def _init_plugins(*args, **kwargs):
    """
    Create instances of the plugin classes and create a cache
    of their hook functions
    """

    for cls in Plugin._plugin_classes:
        # Create instance of 'p'
        instance = cls(*args, **kwargs)
        # Store the instance
        _plugin_instances.append(instance)

        # Setup _keyword_hooks by looking at all of the attributes
        # in the class and finding the ones that have a _is_*_hook
        # attribute
        for attr_name in dir(instance):
            attr = getattr(instance, attr_name)

            for hook_key in _hook_names:
                if getattr(attr, "_is_%s_hook" % hook_key, False):
                    hook_arg = getattr(attr, "_hook_arg", None)
                    # Append (module, method, arg) tuple
                    _plugin_hooks[hook_key].append(
                            (attr.__module__, attr, hook_arg))

def load_plugins(plugindir, *args, **kwargs):
    """
    Module function that loads plugins from a particular directory
    """

    plugins = os.path.abspath(plugindir)
    plugin_names = (x[:-3] for x in os.listdir(plugins) if x.endswith(".py")
            and not x.startswith("_"))

    for p in plugin_names:
        try:
            __import__(os.path.basename(plugindir), globals(), locals(), [p])
        except ImportError:
            # log something here?
            pass

    _init_plugins(*args, **kwargs)

def reload_plugins(plugins, *args, **kwargs):
    """
    Module function that'll reload all of the plugins
    """

    # When the modules are reloaded, the meta class will append
    # all of the classes again, so we need to make sure this is empty
    Plugin._plugin_classes = []
    _reset_variables()

    # Now reload all of the plugins
    plugins_to_reload = []
    plugindir = os.path.basename(plugins)

    # Reload existing plugins
    for mod, val in sys.modules.items():
        if plugindir in mod and val and mod != plugindir:
            mod_file = val.__file__
            if not os.path.isfile(mod_file):
                continue
            if mod_file.endswith('.pyc') or mod_file.endswith('.pyo'):
                source_file_guess = mod_file[:-1]
                if not os.path.isfile(source_file_guess):
                    os.unlink(mod_file)
            plugins_to_reload.append(mod)

    for p in plugins_to_reload:
        try:
            reload(sys.modules[p])
        except Exception, err:
            # log something here?
            pass

    # Load new plugins
    plugin_names = (x[:-3] for x in os.listdir(plugins) if x.endswith(".py")
            and not x.startswith("_") and x not in plugins_to_reload)
    for p in plugin_names:
        try:
            __import__(os.path.basename(plugindir), globals(), locals(), [p])
        except ImportError:
            # log something here?
            pass

    _init_plugins(*args, **kwargs)

def active_plugins():
    """
    Get the loaded plugin names
    """

    return [x.__name__ for x in Plugin._plugin_classes]

def active_plugin_classes():
    """
    Get the loaded plugin classes
    """

    return Plugin._plugin_classes
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
u"""Useful operations for unit tests

:copyright: Copyright (c) 2015 RadiaSoft LLC.  All Rights Reserved.
:license: http://www.apache.org/licenses/LICENSE-2.0.html
"""
from __future__ import absolute_import, division, print_function
from pykern import pkcollections
from pykern import pkinspect
from pykern import pkio
# defer importing pkconfig
import contextlib
import importlib
import inspect
import json
import os
import py
import pytest
import re
import sys

#: Environment var set by pykern.pkcli.test for each module under test
TEST_FILE_ENV = 'PYKERN_PKUNIT_TEST_FILE'

#: Where persistent input files are stored (test_base_name_data)
_DATA_DIR_SUFFIX = '_data'

#: Where to write temporary files (test_base_name_work)
_WORK_DIR_SUFFIX = '_work'

#: Set to the most recent test module by `pykern.pytest_plugin`
module_under_test = None

#: Type of a regular expression
_RE_TYPE = type(re.compile(''))

#: _test_file initialized?
_init = False

#: module being run by `pykern.pkcli.test`
_test_file = None

def assert_object_with_json(basename, actual):
    """Converts actual to JSON and compares with data_dir/basename.json

    Reads data_dir/basename.json and compares with actual
    converted to json. Trailing newline is managed properly. The
    keys are sorted and indentation is 4. actual written to work_dir.

    Args:
        expected_basename (str): file to be found in data_dir with json suffix
        actual (object): to be serialized as json
    """
    from pykern.pkdebug import pkdpretty

    actual = pkdpretty(actual)
    fn = '{}.json'.format(basename)
    pkio.write_text(work_dir().join(fn), actual)
    expect = pkio.read_text(data_dir().join(fn))
    assert expect == actual, \
        '{}: unexpected result'.format(basename)

def data_dir():
    """Compute the data directory based on the test name

    The test data directory is always ``<test>_data``, where ``<test>``
    is the name of the test's python module with the ``_test`` or
    ``test_`` removed.  For example, if the test file is
    ``setup_test.py`` then the directory will be ``setup_data``.

    Returns:
        py.path.local: data directory

    """
    return _base_dir(_DATA_DIR_SUFFIX)

def data_yaml(base_name):
    """Load base_name.yml from data_dir

    Args:
        base_name (str): name of YAML file with ``.yml`` extension

    Returns:
        object: YAML data structure, usually dict or array
    """
    from pykern import pkyaml

    return pkyaml.load_file(data_dir().join(base_name) + '.yml')

def empty_work_dir():
    """Remove `work_dir` if it exists and create.

    All contents of the test directory will be removed.

    Returns:
        py.path.local: empty work directory

    """
    d = work_dir()
    if os.path.exists(str(d)):
        # doesn't ignore "not found" errors
        d.remove(rec=1, ignore_errors=True)
    return d.ensure(dir=True)

def import_module_from_data_dir(module_name):
    """Add `data_dir` to sys.path and import module_name.

    Note that `module_name` with be removed from the sys.modules cache
    before loading in case the module was loaded by another test.

    Args:
        module_name (str): module relative to `data_dir` to import.

    Returns:
        module: imported module
    """
    d = str(data_dir())
    prev_path = sys.path
    try:
        sys.path = [d]
        try:
            del sys.modules[module_name]
        except KeyError:
            pass
        m = importlib.import_module(module_name)
        return m
    finally:
        sys.path = prev_path

@contextlib.contextmanager
def pkexcept(exc_or_re, *fmt_and_args, **kwargs):
    """Expect an exception to be thrown and match or output msg

    If `fmt_and_args` is falsey, will generate a message saying
    what was expected and what was received.

    Examples::

        # Expect an exception (or its subclass)
        with pkexcept(AssertionError, 'did not expect this'):
            assert 0

        # Expect exception to contain a specific message
        with pkexcept('match this', 'problem with matching'):
            assert 0, 'some string with "match this" in it'

        # Use a default output message
        with pkexcept(KeyError):
            something['key will not be found']

    Args:
        exc_or_re (object): BaseException, re, or str; if str, compiled with `re.IGNORECASE`
        fmt_and_args (tuple): passed to format
        kwargs (dict): passed to format

    Yields:
        None: just for context manager
    """
    try:
        yield None
    except BaseException as e:
        e_str = '{} {}'.format(type(e), e)
        if isinstance(exc_or_re, type) and issubclass(exc_or_re, BaseException):
            if isinstance(e, exc_or_re):
                return
            if not fmt_and_args:
                fmt_and_args=(
                    '{}: an exception was raised, but expected it to be {}',
                    e_str,
                    exc_or_re,
                )
        else:
            if not isinstance(exc_or_re, _RE_TYPE):
                exc_or_re = re.compile(exc_or_re, flags=re.IGNORECASE)
            if exc_or_re.search(e_str):
                return
            if not fmt_and_args:
                fmt_and_args=(
                    '{}: an exception was raised, but did not match "{}"',
                    e_str,
                    exc_or_re.pattern,
                )
    else:
        if not fmt_and_args:
            fmt_and_args=('Exception was not raised: expecting={}', exc_or_re)
    pkfail(*fmt_and_args, **kwargs)

def pkeq(expect, actual, *args, **kwargs):
    """If actual is not expect, throw assertion with calling context.

    Args:
        expect (object): what to test for
        actual (object): run-time value
        args (tuple): passed to pkfail()
        kwargs (dict): passed to pkfail()
    """
    if expect != actual:
        if args or kwargs:
            pkfail(*args, **kwargs)
        else:
            pkfail('expect={} != actual={}', expect, actual)

def pkfail(fmt, *args, **kwargs):
    """Format message and raise AssertionError.

    Args:
        fmt (str): to be passed to `string.format`
        args (tuple): passed to format
        kwargs (dict): passed to format
    """
    msg = fmt.format(*args, **kwargs)
    call = pkinspect.caller(ignore_modules=[contextlib])
    raise AssertionError('{} {}'.format(call, msg))

def pkok(cond, fmt, *args, **kwargs):
    """If cond is not true, throw assertion with calling context

    Args:
        cond (object): expression which should evaluate to true
        fmt (str): to be passed to `string.format`
        args (tuple): passed to format
        kwargs (dict): passed to format

    Returns:
        object: `obj` value
    """
    if not cond:
        pkfail(fmt, *args, **kwargs)
    return cond

def pkre(expect_re, actual, flags=re.IGNORECASE + re.DOTALL):
    """If actual does not match (re.search) expect_re, throw assertion with calling context.

    Args:
        expect_re (object): string or re object
        actual (object): run-time value
        flags: passed on to re.search [IGNORECASE + DOTALL]
    """
    if not re.search(expect_re, actual, flags=flags):
        pkfail('expect_re={} != actual={}', expect_re, actual)

def random_alpha(length=6):
    """Random lowercase alpha string

    Args:
        length (int): how many chars

    Returns:
        str: lower case alpha string
    """
    import random
    import string
    return ''.join(random.choice(string.ascii_lowercase) for _ in range(length))

def save_chdir_work(is_pkunit_prefix=False):
    """Create empty work_dir and chdir

    Args:
        is_pkunit_prefix (bool): use as root of (most) file I/O (optional)

    Returns:
        py.path.local: empty work directory

    """
    return pkio.save_chdir(empty_work_dir(), is_pkunit_prefix=is_pkunit_prefix)

def work_dir():
    """Returns ephemeral work directory, created if necessary.

    To enable easier debugging, the test directory is always
    ``<test>_work``, where ``<test>`` is the name of the test's python
    module with the ``_test`` or ``test_`` removed.  For example, if the
    test file is ``setup_test.py`` then the directory will be
    ``setup_work``.

    The name "work" distinguishes from "tmp", which could imply
    anything. Also, with editor autocomplete, "setup_work" and
    "setup_test" are more easily distinguishable.

    Returns:
        py.path: directory name
    """
    return _base_dir(_WORK_DIR_SUFFIX).ensure(dir=True)

def _base_dir(postfix):
    """Base name with directory.

    Args:
        postfix (str): what to append to base (``_data`` or ``_work``).

    Returns:
        py.path.local: base directory with postfix
    """
    global _init, _test_file
    if not _init:
        _init = True
        # pykern.pkcli.test
        t = os.environ.get(TEST_FILE_ENV)
        if t:
            _test_file = py.path.local(t)
    if _test_file:
        f = _test_file
    elif module_under_test:
        # pykern.pytest_plugin
        m = module_under_test
        f = py.path.local(m.__file__)
    else:
        # py.test alone, just guess
        s = inspect.currentframe().f_back.f_back
        f = None
        for _ in range(100):
            if s.f_code.co_filename.endswith('_test.py'):
                f = py.path.local(s.f_code.co_filename)
                break
            s = s.f_back
            if not s:
                break
        if not f:
            raise AssertionError('unable to find test module')
    b = re.sub(r'_test$|^test_', '', f.purebasename)
    assert b != f.purebasename, \
        '{}: module name must end in _test'.format(f)
    return f.new(basename=b + postfix).realpath()
<commit_msg>improve quality of pkexcept reporting<commit_after># -*- coding: utf-8 -*-
u"""Useful operations for unit tests

:copyright: Copyright (c) 2015 RadiaSoft LLC.  All Rights Reserved.
:license: http://www.apache.org/licenses/LICENSE-2.0.html
"""
from __future__ import absolute_import, division, print_function
from pykern import pkcollections
from pykern import pkinspect
from pykern import pkio
# defer importing pkconfig
import contextlib
import importlib
import inspect
import json
import os
import py
import pytest
import re
import sys

#: Environment var set by pykern.pkcli.test for each module under test
TEST_FILE_ENV = 'PYKERN_PKUNIT_TEST_FILE'

#: Where persistent input files are stored (test_base_name_data)
_DATA_DIR_SUFFIX = '_data'

#: Where to write temporary files (test_base_name_work)
_WORK_DIR_SUFFIX = '_work'

#: Set to the most recent test module by `pykern.pytest_plugin`
module_under_test = None

#: Type of a regular expression
_RE_TYPE = type(re.compile(''))

#: _test_file initialized?
_init = False

#: module being run by `pykern.pkcli.test`
_test_file = None

def assert_object_with_json(basename, actual):
    """Converts actual to JSON and compares with data_dir/basename.json

    Reads data_dir/basename.json and compares with actual
    converted to json. Trailing newline is managed properly. The
    keys are sorted and indentation is 4. actual written to work_dir.

    Args:
        expected_basename (str): file to be found in data_dir with json suffix
        actual (object): to be serialized as json
    """
    from pykern.pkdebug import pkdpretty

    actual = pkdpretty(actual)
    fn = '{}.json'.format(basename)
    pkio.write_text(work_dir().join(fn), actual)
    expect = pkio.read_text(data_dir().join(fn))
    assert expect == actual, \
        '{}: unexpected result'.format(basename)

def data_dir():
    """Compute the data directory based on the test name

    The test data directory is always ``<test>_data``, where ``<test>``
    is the name of the test's python module with the ``_test`` or
    ``test_`` removed.  For example, if the test file is
    ``setup_test.py`` then the directory will be ``setup_data``.

    Returns:
        py.path.local: data directory

    """
    return _base_dir(_DATA_DIR_SUFFIX)

def data_yaml(base_name):
    """Load base_name.yml from data_dir

    Args:
        base_name (str): name of YAML file with ``.yml`` extension

    Returns:
        object: YAML data structure, usually dict or array
    """
    from pykern import pkyaml

    return pkyaml.load_file(data_dir().join(base_name) + '.yml')

def empty_work_dir():
    """Remove `work_dir` if it exists and create.

    All contents of the test directory will be removed.

    Returns:
        py.path.local: empty work directory

    """
    d = work_dir()
    if os.path.exists(str(d)):
        # doesn't ignore "not found" errors
        d.remove(rec=1, ignore_errors=True)
    return d.ensure(dir=True)

def import_module_from_data_dir(module_name):
    """Add `data_dir` to sys.path and import module_name.

    Note that `module_name` with be removed from the sys.modules cache
    before loading in case the module was loaded by another test.

    Args:
        module_name (str): module relative to `data_dir` to import.

    Returns:
        module: imported module
    """
    d = str(data_dir())
    prev_path = sys.path
    try:
        sys.path = [d]
        try:
            del sys.modules[module_name]
        except KeyError:
            pass
        m = importlib.import_module(module_name)
        return m
    finally:
        sys.path = prev_path

@contextlib.contextmanager
def pkexcept(exc_or_re, *fmt_and_args, **kwargs):
    """Expect an exception to be thrown and match or output msg

    If `fmt_and_args` is falsey, will generate a message saying
    what was expected and what was received.

    Examples::

        # Expect an exception (or its subclass)
        with pkexcept(AssertionError, 'did not expect this'):
            assert 0

        # Expect exception to contain a specific message
        with pkexcept('match this', 'problem with matching'):
            assert 0, 'some string with "match this" in it'

        # Use a default output message
        with pkexcept(KeyError):
            something['key will not be found']

    Args:
        exc_or_re (object): BaseException, re, or str; if str, compiled with `re.IGNORECASE`
        fmt_and_args (tuple): passed to format
        kwargs (dict): passed to format

    Yields:
        None: just for context manager
    """
    try:
        yield None
    except BaseException as e:
        from pykern.pkdebug import pkdexc

        e_str = '{} {}'.format(type(e), e)
        if isinstance(exc_or_re, type) and issubclass(exc_or_re, BaseException):
            if isinstance(e, exc_or_re):
                return
            if not fmt_and_args:
                fmt_and_args=(
                    '{}: an exception was raised, but expected it to be {}; stack={}',
                    e_str,
                    exc_or_re,
                    pkdexc(),
                )
        else:
            if not isinstance(exc_or_re, _RE_TYPE):
                exc_or_re = re.compile(exc_or_re, flags=re.IGNORECASE)
            if exc_or_re.search(e_str):
                return
            if not fmt_and_args:
                fmt_and_args=(
                    '{}: an exception was raised, but did not match "{}"; stack={}',
                    e_str,
                    exc_or_re.pattern,
                    pkdexc(),
                )
    else:
        if not fmt_and_args:
            fmt_and_args=('Exception was not raised: expecting={}', exc_or_re)
    pkfail(*fmt_and_args, **kwargs)

def pkeq(expect, actual, *args, **kwargs):
    """If actual is not expect, throw assertion with calling context.

    Args:
        expect (object): what to test for
        actual (object): run-time value
        args (tuple): passed to pkfail()
        kwargs (dict): passed to pkfail()
    """
    if expect != actual:
        if args or kwargs:
            pkfail(*args, **kwargs)
        else:
            pkfail('expect={} != actual={}', expect, actual)

def pkfail(fmt, *args, **kwargs):
    """Format message and raise AssertionError.

    Args:
        fmt (str): to be passed to `string.format`
        args (tuple): passed to format
        kwargs (dict): passed to format
    """
    msg = fmt.format(*args, **kwargs)
    call = pkinspect.caller(ignore_modules=[contextlib])
    raise AssertionError('{} {}'.format(call, msg))

def pkok(cond, fmt, *args, **kwargs):
    """If cond is not true, throw assertion with calling context

    Args:
        cond (object): expression which should evaluate to true
        fmt (str): to be passed to `string.format`
        args (tuple): passed to format
        kwargs (dict): passed to format

    Returns:
        object: `obj` value
    """
    if not cond:
        pkfail(fmt, *args, **kwargs)
    return cond

def pkre(expect_re, actual, flags=re.IGNORECASE + re.DOTALL):
    """If actual does not match (re.search) expect_re, throw assertion with calling context.

    Args:
        expect_re (object): string or re object
        actual (object): run-time value
        flags: passed on to re.search [IGNORECASE + DOTALL]
    """
    if not re.search(expect_re, actual, flags=flags):
        pkfail('expect_re={} != actual={}', expect_re, actual)

def random_alpha(length=6):
    """Random lowercase alpha string

    Args:
        length (int): how many chars

    Returns:
        str: lower case alpha string
    """
    import random
    import string
    return ''.join(random.choice(string.ascii_lowercase) for _ in range(length))

def save_chdir_work(is_pkunit_prefix=False):
    """Create empty work_dir and chdir

    Args:
        is_pkunit_prefix (bool): use as root of (most) file I/O (optional)

    Returns:
        py.path.local: empty work directory

    """
    return pkio.save_chdir(empty_work_dir(), is_pkunit_prefix=is_pkunit_prefix)

def work_dir():
    """Returns ephemeral work directory, created if necessary.

    To enable easier debugging, the test directory is always
    ``<test>_work``, where ``<test>`` is the name of the test's python
    module with the ``_test`` or ``test_`` removed.  For example, if the
    test file is ``setup_test.py`` then the directory will be
    ``setup_work``.

    The name "work" distinguishes from "tmp", which could imply
    anything. Also, with editor autocomplete, "setup_work" and
    "setup_test" are more easily distinguishable.

    Returns:
        py.path: directory name
    """
    return _base_dir(_WORK_DIR_SUFFIX).ensure(dir=True)

def _base_dir(postfix):
    """Base name with directory.

    Args:
        postfix (str): what to append to base (``_data`` or ``_work``).

    Returns:
        py.path.local: base directory with postfix
    """
    global _init, _test_file
    if not _init:
        _init = True
        # pykern.pkcli.test
        t = os.environ.get(TEST_FILE_ENV)
        if t:
            _test_file = py.path.local(t)
    if _test_file:
        f = _test_file
    elif module_under_test:
        # pykern.pytest_plugin
        m = module_under_test
        f = py.path.local(m.__file__)
    else:
        # py.test alone, just guess
        s = inspect.currentframe().f_back.f_back
        f = None
        for _ in range(100):
            if s.f_code.co_filename.endswith('_test.py'):
                f = py.path.local(s.f_code.co_filename)
                break
            s = s.f_back
            if not s:
                break
        if not f:
            raise AssertionError('unable to find test module')
    b = re.sub(r'_test$|^test_', '', f.purebasename)
    assert b != f.purebasename, \
        '{}: module name must end in _test'.format(f)
    return f.new(basename=b + postfix).realpath()
<|endoftext|>


<commit_before># http://xkcd.com/353/

import argparse
import subprocess
import xml.etree.cElementTree as ET
import sqlite3 as db

if __name__ == '__main__':

	parser = argparse.ArgumentParser(description='A Subversion log parser written in Python')
	parser.add_argument('-url', required=True, help='SVN Repository project URL')
	parser.add_argument('-target', required=True, help='Target path to checkout the project to')
	args = parser.parse_args()

	# result = subprocess.call(['svn', 'checkout', '--non-interactive', args.url, args.target])
	result = subprocess.check_output(['svn', 'log', '--non-interactive', '--xml', '-l', '5', args.target])

	print result

	conn = db.connect(':memory:')

	c = conn.cursor()

	c.execute('''CREATE TABLE logentry (revision integer, author text, date text, msg text)''')
	c.execute('''CREATE TABLE revision (revision integer, file text, added integer, removed integer)''')

	root = ET.fromstring(result)

	for logentry in root:
		row = (logentry.attrib['revision'],
				logentry[0].text,
				logentry[1].text,
				logentry[2].text)
		# print row
		c.execute('INSERT INTO logentry VALUES (?, ?, ?, ?)', row)

	# c.execute('SELECT * FROM logentry')

	# print c.fetchall()

	conn.close()
	# print root

# 138806 161995
# svn log -r 1:HEAD --limit 1 http://svn.apache.org/repos/asf/commons/proper/logging/trunk/
# http://svn.apache.org/repos/asf/commons/proper/logging/trunk/
	# result = call(['svn', 'log', '-v', '-g', '--xml','-l','2', args.url])

#1362964
#1362962
#
#    <?xml version="1.0"?>
#    <log>
#    <logentry
#       revision="1612041">
#    <author>ggregory</author>
#    <date>2014-07-20T06:54:48.146304Z</date>
#    <msg>Use the more modern, compact, and flexible Javadoc "{@code ...}" instead of
#     the HTML "<tt>...</tt>".</msg>
#    </logentry>
#    <logentry
#       revision="1609299">
#    <author>tn</author>
#    <date>2014-07-09T21:06:14.727009Z</date>
#    <msg>Update to current snapshot version.</msg>
#    </logentry>
#    </log>
#<commit_msg>Adding references<commit_after># http://xkcd.com/353/

# http://en.wikipedia.org/wiki/Diff#Unified_format
# http://www.gnu.org/software/diffutils/manual/diffutils.html#Detailed-Unified

import argparse
import subprocess
import xml.etree.cElementTree as ET
import sqlite3 as db

if __name__ == '__main__':

	parser = argparse.ArgumentParser(description='A Subversion log parser written in Python')
	parser.add_argument('-url', required=True, help='SVN Repository project URL')
	parser.add_argument('-target', required=True, help='Target path to checkout the project to')
	args = parser.parse_args()

	# result = subprocess.call(['svn', 'checkout', '--non-interactive', args.url, args.target])
	result = subprocess.check_output(['svn', 'log', '--non-interactive', '--xml', '-l', '5', args.target])

	print result

	conn = db.connect(':memory:')

	c = conn.cursor()

	c.execute('''CREATE TABLE logentry (revision integer, author text, date text, msg text)''')
	c.execute('''CREATE TABLE revision (revision integer, file text, added integer, removed integer)''')

	root = ET.fromstring(result)

	for logentry in root:
		row = (logentry.attrib['revision'],
				logentry[0].text,
				logentry[1].text,
				logentry[2].text)
		# print row
		c.execute('INSERT INTO logentry VALUES (?, ?, ?, ?)', row)

	# c.execute('SELECT * FROM logentry')

	# print c.fetchall()

	conn.close()
	# print root

# 138806 161995
# svn log -r 1:HEAD --limit 1 http://svn.apache.org/repos/asf/commons/proper/logging/trunk/
# http://svn.apache.org/repos/asf/commons/proper/logging/trunk/
	# result = call(['svn', 'log', '-v', '-g', '--xml','-l','2', args.url])

#1362964
#1362962
#
#    <?xml version="1.0"?>
#    <log>
#    <logentry
#       revision="1612041">
#    <author>ggregory</author>
#    <date>2014-07-20T06:54:48.146304Z</date>
#    <msg>Use the more modern, compact, and flexible Javadoc "{@code ...}" instead of
#     the HTML "<tt>...</tt>".</msg>
#    </logentry>
#    <logentry
#       revision="1609299">
#    <author>tn</author>
#    <date>2014-07-09T21:06:14.727009Z</date>
#    <msg>Update to current snapshot version.</msg>
#    </logentry>
#    </log>
#<|endoftext|>


<commit_before># Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This module is used to configure gcp-devrel-py-tools run-pylint."""

# Library configuration

# library_additions = {}
# library_replacements = {}

# Test configuration

# test_additions = copy.deepcopy(library_additions)
# test_replacements = copy.deepcopy(library_replacements)
<commit_msg>Making all LICENSE headers "uniform". (#3563)<commit_after># Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This module is used to configure gcp-devrel-py-tools run-pylint."""

# Library configuration

# library_additions = {}
# library_replacements = {}

# Test configuration

# test_additions = copy.deepcopy(library_additions)
# test_replacements = copy.deepcopy(library_replacements)
<|endoftext|>


<commit_before>_DEFAULT_TEMPLATE = Label("//third_party/com_github_robotlocomotion_drake:tools/workspace/pkg_config.BUILD.tpl")

_DEFAULT_STATIC = False

def _run_pkg_config(repository_ctx, command_line, pkg_config_paths):
    """Run command_line with PKG_CONFIG_PATH = pkg_config_paths and return its
    tokenized output."""
    pkg_config_path = ":".join(pkg_config_paths)
    result = repository_ctx.execute(command_line,
                                    environment = {
                                        "PKG_CONFIG_PATH": pkg_config_path,
                                    })
    if result.return_code != 0:
        return struct(error = "error {} from {}: {}{}".format(
            result.return_code, command_line, result.stdout, result.stderr))
    tokens = [x for x in result.stdout.strip().split(" ") if x]
    return struct(tokens = tokens, error = None)

def setup_pkg_config_repository(repository_ctx):
    """This is the macro form of the pkg_config_repository() rule below.
    Refer to that rule's API documentation for details.

    This flavor of this rule is intended to be called by other repository_rule
    implementation functions.  The pkg_config_repository flavor of this rule is
    intended to be called directly from the WORKSPACE file, or from a macro
    that was called by the WORKSPACE file.
    """
    # First locate pkg-config on the $PATH.
    tool_path = repository_ctx.which("pkg-config")
    if not tool_path:
        return struct(error = "Could not find pkg-config on PATH={}".format(
            repository_ctx.os.environ["PATH"]))
    args = [tool_path, repository_ctx.attr.modname]

    pkg_config_paths = getattr(repository_ctx.attr,
                               "pkg_config_paths",
                               [])

    # Check if we can find the required *.pc file of any version.
    result = _run_pkg_config(repository_ctx, args, pkg_config_paths)
    if result.error != None:
        return result

    # If we have a minimum version, enforce that.
    atleast_version = getattr(repository_ctx.attr, "atleast_version", "")
    if atleast_version:
        result = _run_pkg_config(repository_ctx, args + [
            "--atleast-version", atleast_version], pkg_config_paths)
        if result.error != None:
            return struct(error = result.error + "during version check")

    # Determine linkopts.
    static = getattr(repository_ctx.attr, "static", _DEFAULT_STATIC)
    libs_args = args + ["--libs"]
    if static:
        libs_args = libs_args + ["--static"]
    result = _run_pkg_config(repository_ctx, libs_args, pkg_config_paths)
    if result.error != None:
        return result
    linkopts = result.tokens
    # Bazel "linkopts=" must be either switches ("-foo"), variables ("$(FOO)"),
    # or labels ("foo").  We should only get switches from `pkg-config --libs`.
    # However, sometimes it produces "-framework CoreFoundation" or similar,
    # which is *supposed* to be a single switch, but our split heuristic
    # chopped it up.  We recombine non-switch args with their preceeding arg as
    # a repair.  We process args in reserve order to keep our loop index
    # unchanged by a pop.
    for i in reversed(range(len(linkopts))):
        linkopt = linkopts[i]
        # Absolute system paths to *.so files get turned into -l instead.
        # This fixup is only implemented for Ubuntu (not macOS) so far.
        if linkopt.endswith(".so"):
            possible_libdirs = [
                "/usr/lib",
                "/usr/lib/x86_64-linux-gnu",
            ]
            for dir in possible_libdirs:
                prefix = dir + "/lib"
                if linkopt.startswith(prefix):
                    name = linkopt[len(prefix):-len(".so")]
                    if "/" not in name:
                        linkopt = "-l" + name
                        linkopts[i] = linkopt
                        break
        # Add `-Wl,-rpath <path>` for `-L<path>`.
        # See https://github.com/RobotLocomotion/drake/issues/7387#issuecomment-359952616  # noqa
        if linkopt.startswith("-L"):
            linkopts[i] = "-Wl,-rpath " + linkopt[2:] + " " + linkopt
            continue

        # On macOS, we need to pass `-undefined dynamic_lookup`
        # instead of `-lpython2.7` or `lpython3.x`.
        os_name = repository_ctx.os.name
        if os_name == "mac os x" and linkopt.startswith("-lpython"):
            linkopts[i] = "-undefined dynamic_lookup"
            continue

        # On Linux, skip "-lpython..."
        if os_name == "linux" and linkopt.startswith("-lpython"):
            linkopts[i] = ""
            continue

        # Switches stay put.
        if linkopt.startswith("-"):
            continue
        # A non-switch arg should be recombined with the preceding arg.
        non_switch_arg = linkopts.pop(i)
        if i == 0:
            return struct(error = "malformed linkopts: " + repr(linkopts))
        linkopts[i - 1] += " " + non_switch_arg

    # Determine cflags; we'll split into includes and defines in a moment.
    result = _run_pkg_config(repository_ctx,
                             args + ["--cflags"],
                             pkg_config_paths)
    if result.error != None:
        return result
    cflags = result.tokens
    # Placate whiny compilers.
    if "-pthread" in cflags and "-pthread" in linkopts:
        cflags.remove("-pthread")

    # Split cflags into includes and defines.  The -I paths from pkg-config
    # will be absolute paths; we'll make them relative in a moment.
    absolute_includes = []
    defines = []
    copts = []
    unknown_cflags = []
    # We process in reserve order to keep our loop index unchanged by a pop.
    for cflag in cflags:
        if cflag.startswith("-I"):
            absolute_includes += [cflag[2:]]
        elif cflag.startswith("-D"):
            defines += [cflag[2:]]
        else:
            copts += [cflag]

    # Symlink the absolute include paths into our repository, to obtain
    # relative paths for them as required by cc_library's attributes.
    includes = []
    hdrs_path = repository_ctx.path("include")
    for item in absolute_includes:
        symlink_dest = item.replace('/', '_')
        repository_ctx.symlink(
            repository_ctx.path(item),
            hdrs_path.get_child(symlink_dest))
        includes += ["include/" + symlink_dest]
    hdrs_prologue = "glob([\"include/**\"]) + "

    # Write out the BUILD file.
    substitutions = {
        "%{name}": repr(
            repository_ctx.name),
        "%{srcs}": repr(
            getattr(repository_ctx.attr, "extra_srcs", [])),
        "%{hdrs}": (
            hdrs_prologue + repr(
                getattr(repository_ctx.attr, "extra_hdrs", []))),
        "%{copts}": repr(
            copts + getattr(repository_ctx.attr, "extra_copts", [])),
        "%{defines}": repr(
            defines + getattr(repository_ctx.attr, "extra_defines", [])),
        "%{includes}": repr(
            includes + getattr(repository_ctx.attr, "extra_includes", [])),
        "%{linkopts}": repr(
            linkopts + getattr(repository_ctx.attr, "extra_linkopts", [])),
        "%{deps}": repr(
            getattr(repository_ctx.attr, "extra_deps", [])),
    }
    template = getattr(
        repository_ctx.attr, "build_file_template", _DEFAULT_TEMPLATE)
    repository_ctx.template("BUILD", template, substitutions)

    return struct(value = True, error = None)

def _impl(repository_ctx):
    result = setup_pkg_config_repository(repository_ctx)
    if result.error != None:
        fail("Unable to complete pkg-config setup for @{} repository: {}".
             format(repository_ctx.name, result.error))

pkg_config_repository = repository_rule(
    attrs = {
        "modname": attr.string(mandatory = True),
        "atleast_version": attr.string(),
        "static": attr.bool(default = _DEFAULT_STATIC),
        "build_file_template": attr.label(
            default = _DEFAULT_TEMPLATE,
            allow_files = True,
        ),
        "extra_srcs": attr.string_list(),
        "extra_hdrs": attr.string_list(),
        "extra_copts": attr.string_list(),
        "extra_defines": attr.string_list(),
        "extra_includes": attr.string_list(),
        "extra_linkopts": attr.string_list(),
        "extra_deps": attr.string_list(),
        "pkg_config_paths": attr.string_list(),
    },
    environ = [
        "PATH",
    ],
    local = True,
    implementation = _impl,
)

"""Creates a repository that contains a single library target, based on the
results of invoking pkg-config.

The pkg_config_repository flavor of this rule is intended to be called directly
from the WORKSPACE file, or from a macro that was called by the WORKSPACE file.
The setup_pkg_config_repository flavor of this rule is intended to be called by
other repository_rule implementation functions.

Example:
    WORKSPACE:
        load("@drake//tools/workspace:pkg_config.bzl", "pkg_config_repository")
        pkg_config_repository(
            name = "foo",
            modname = "foo-2.0",
        )

    BUILD:
        cc_library(
            name = "foobar",
            deps = ["@foo"],
            srcs = ["bar.cc"],
        )

Args:
    name: A unique name for this rule.
    modname: The library name as known to pkg-config.
    atleast_version: (Optional) The --atleast-version to pkg-config.
    static: (Optional) Add linkopts for static linking to the library target.
    build_file_template: (Optional) (Advanced) Override the BUILD template.
    extra_srcs: (Optional) Extra items to add to the library target.
    extra_hdrs: (Optional) Extra items to add to the library target.
    extra_copts: (Optional) Extra items to add to the library target.
    extra_defines: (Optional) Extra items to add to the library target.
    extra_includes: (Optional) Extra items to add to the library target.
    extra_linkopts: (Optional) Extra items to add to the library target.
    extra_deps: (Optional) Extra items to add to the library target.
    pkg_config_paths: (Optional) Paths to find pkg-config files (.pc). Note
                      that we ignore the enviornment variable PKG_CONFIG_PATH
                      set by the user.
"""
<commit_msg>bazel(third_party/pkg_config.bzl): Use rpath hack only for linux<commit_after>_DEFAULT_TEMPLATE = Label("//third_party/com_github_robotlocomotion_drake:tools/workspace/pkg_config.BUILD.tpl")

_DEFAULT_STATIC = False

def _run_pkg_config(repository_ctx, command_line, pkg_config_paths):
    """Run command_line with PKG_CONFIG_PATH = pkg_config_paths and return its
    tokenized output."""
    pkg_config_path = ":".join(pkg_config_paths)
    result = repository_ctx.execute(command_line,
                                    environment = {
                                        "PKG_CONFIG_PATH": pkg_config_path,
                                    })
    if result.return_code != 0:
        return struct(error = "error {} from {}: {}{}".format(
            result.return_code, command_line, result.stdout, result.stderr))
    tokens = [x for x in result.stdout.strip().split(" ") if x]
    return struct(tokens = tokens, error = None)

def setup_pkg_config_repository(repository_ctx):
    """This is the macro form of the pkg_config_repository() rule below.
    Refer to that rule's API documentation for details.

    This flavor of this rule is intended to be called by other repository_rule
    implementation functions.  The pkg_config_repository flavor of this rule is
    intended to be called directly from the WORKSPACE file, or from a macro
    that was called by the WORKSPACE file.
    """
    # First locate pkg-config on the $PATH.
    tool_path = repository_ctx.which("pkg-config")
    if not tool_path:
        return struct(error = "Could not find pkg-config on PATH={}".format(
            repository_ctx.os.environ["PATH"]))
    args = [tool_path, repository_ctx.attr.modname]

    pkg_config_paths = getattr(repository_ctx.attr,
                               "pkg_config_paths",
                               [])

    # Check if we can find the required *.pc file of any version.
    result = _run_pkg_config(repository_ctx, args, pkg_config_paths)
    if result.error != None:
        return result

    # If we have a minimum version, enforce that.
    atleast_version = getattr(repository_ctx.attr, "atleast_version", "")
    if atleast_version:
        result = _run_pkg_config(repository_ctx, args + [
            "--atleast-version", atleast_version], pkg_config_paths)
        if result.error != None:
            return struct(error = result.error + "during version check")

    # Determine linkopts.
    static = getattr(repository_ctx.attr, "static", _DEFAULT_STATIC)
    libs_args = args + ["--libs"]
    if static:
        libs_args = libs_args + ["--static"]
    result = _run_pkg_config(repository_ctx, libs_args, pkg_config_paths)
    if result.error != None:
        return result
    linkopts = result.tokens
    # Bazel "linkopts=" must be either switches ("-foo"), variables ("$(FOO)"),
    # or labels ("foo").  We should only get switches from `pkg-config --libs`.
    # However, sometimes it produces "-framework CoreFoundation" or similar,
    # which is *supposed* to be a single switch, but our split heuristic
    # chopped it up.  We recombine non-switch args with their preceeding arg as
    # a repair.  We process args in reserve order to keep our loop index
    # unchanged by a pop.
    for i in reversed(range(len(linkopts))):
        linkopt = linkopts[i]
        # Absolute system paths to *.so files get turned into -l instead.
        # This fixup is only implemented for Ubuntu (not macOS) so far.
        if linkopt.endswith(".so"):
            possible_libdirs = [
                "/usr/lib",
                "/usr/lib/x86_64-linux-gnu",
            ]
            for dir in possible_libdirs:
                prefix = dir + "/lib"
                if linkopt.startswith(prefix):
                    name = linkopt[len(prefix):-len(".so")]
                    if "/" not in name:
                        linkopt = "-l" + name
                        linkopts[i] = linkopt
                        break

        os_name = repository_ctx.os.name

        # Add `-Wl,-rpath <path>` for `-L<path>`.
        # See https://github.com/RobotLocomotion/drake/issues/7387#issuecomment-359952616  # noqa
        if os_name == "linux" and linkopt.startswith("-L"):
            linkopts[i] = "-Wl,-rpath " + linkopt[2:] + " " + linkopt
            continue

        # On macOS, we need to pass `-undefined dynamic_lookup`
        # instead of `-lpython2.7` or `lpython3.x`.
        if os_name == "mac os x" and linkopt.startswith("-lpython"):
            linkopts[i] = "-undefined dynamic_lookup"
            continue

        # On Linux, skip "-lpython..."
        if os_name == "linux" and linkopt.startswith("-lpython"):
            linkopts[i] = ""
            continue

        # Switches stay put.
        if linkopt.startswith("-"):
            continue
        # A non-switch arg should be recombined with the preceding arg.
        non_switch_arg = linkopts.pop(i)
        if i == 0:
            return struct(error = "malformed linkopts: " + repr(linkopts))
        linkopts[i - 1] += " " + non_switch_arg

    # Determine cflags; we'll split into includes and defines in a moment.
    result = _run_pkg_config(repository_ctx,
                             args + ["--cflags"],
                             pkg_config_paths)
    if result.error != None:
        return result
    cflags = result.tokens
    # Placate whiny compilers.
    if "-pthread" in cflags and "-pthread" in linkopts:
        cflags.remove("-pthread")

    # Split cflags into includes and defines.  The -I paths from pkg-config
    # will be absolute paths; we'll make them relative in a moment.
    absolute_includes = []
    defines = []
    copts = []
    unknown_cflags = []
    # We process in reserve order to keep our loop index unchanged by a pop.
    for cflag in cflags:
        if cflag.startswith("-I"):
            absolute_includes += [cflag[2:]]
        elif cflag.startswith("-D"):
            defines += [cflag[2:]]
        else:
            copts += [cflag]

    # Symlink the absolute include paths into our repository, to obtain
    # relative paths for them as required by cc_library's attributes.
    includes = []
    hdrs_path = repository_ctx.path("include")
    for item in absolute_includes:
        symlink_dest = item.replace('/', '_')
        repository_ctx.symlink(
            repository_ctx.path(item),
            hdrs_path.get_child(symlink_dest))
        includes += ["include/" + symlink_dest]
    hdrs_prologue = "glob([\"include/**\"]) + "

    # Write out the BUILD file.
    substitutions = {
        "%{name}": repr(
            repository_ctx.name),
        "%{srcs}": repr(
            getattr(repository_ctx.attr, "extra_srcs", [])),
        "%{hdrs}": (
            hdrs_prologue + repr(
                getattr(repository_ctx.attr, "extra_hdrs", []))),
        "%{copts}": repr(
            copts + getattr(repository_ctx.attr, "extra_copts", [])),
        "%{defines}": repr(
            defines + getattr(repository_ctx.attr, "extra_defines", [])),
        "%{includes}": repr(
            includes + getattr(repository_ctx.attr, "extra_includes", [])),
        "%{linkopts}": repr(
            linkopts + getattr(repository_ctx.attr, "extra_linkopts", [])),
        "%{deps}": repr(
            getattr(repository_ctx.attr, "extra_deps", [])),
    }
    template = getattr(
        repository_ctx.attr, "build_file_template", _DEFAULT_TEMPLATE)
    repository_ctx.template("BUILD", template, substitutions)

    return struct(value = True, error = None)

def _impl(repository_ctx):
    result = setup_pkg_config_repository(repository_ctx)
    if result.error != None:
        fail("Unable to complete pkg-config setup for @{} repository: {}".
             format(repository_ctx.name, result.error))

pkg_config_repository = repository_rule(
    attrs = {
        "modname": attr.string(mandatory = True),
        "atleast_version": attr.string(),
        "static": attr.bool(default = _DEFAULT_STATIC),
        "build_file_template": attr.label(
            default = _DEFAULT_TEMPLATE,
            allow_files = True,
        ),
        "extra_srcs": attr.string_list(),
        "extra_hdrs": attr.string_list(),
        "extra_copts": attr.string_list(),
        "extra_defines": attr.string_list(),
        "extra_includes": attr.string_list(),
        "extra_linkopts": attr.string_list(),
        "extra_deps": attr.string_list(),
        "pkg_config_paths": attr.string_list(),
    },
    environ = [
        "PATH",
    ],
    local = True,
    implementation = _impl,
)

"""Creates a repository that contains a single library target, based on the
results of invoking pkg-config.

The pkg_config_repository flavor of this rule is intended to be called directly
from the WORKSPACE file, or from a macro that was called by the WORKSPACE file.
The setup_pkg_config_repository flavor of this rule is intended to be called by
other repository_rule implementation functions.

Example:
    WORKSPACE:
        load("@drake//tools/workspace:pkg_config.bzl", "pkg_config_repository")
        pkg_config_repository(
            name = "foo",
            modname = "foo-2.0",
        )

    BUILD:
        cc_library(
            name = "foobar",
            deps = ["@foo"],
            srcs = ["bar.cc"],
        )

Args:
    name: A unique name for this rule.
    modname: The library name as known to pkg-config.
    atleast_version: (Optional) The --atleast-version to pkg-config.
    static: (Optional) Add linkopts for static linking to the library target.
    build_file_template: (Optional) (Advanced) Override the BUILD template.
    extra_srcs: (Optional) Extra items to add to the library target.
    extra_hdrs: (Optional) Extra items to add to the library target.
    extra_copts: (Optional) Extra items to add to the library target.
    extra_defines: (Optional) Extra items to add to the library target.
    extra_includes: (Optional) Extra items to add to the library target.
    extra_linkopts: (Optional) Extra items to add to the library target.
    extra_deps: (Optional) Extra items to add to the library target.
    pkg_config_paths: (Optional) Paths to find pkg-config files (.pc). Note
                      that we ignore the enviornment variable PKG_CONFIG_PATH
                      set by the user.
"""
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    DWX_ZeroMQ_Connector_v2_0_1_RC8.py
    --
    @author: Darwinex Labs (www.darwinex.com)

    Copyright (c) 2017-2019, Darwinex. All rights reserved.

    Licensed under the BSD 3-Clause License, you may not use this file except
    in compliance with the License.

    You may obtain a copy of the License at:
    https://opensource.org/licenses/BSD-3-Clause
"""

# IMPORT zmq library
import zmq
from pandas import Timestamp, Series
from threading import Thread

class DWX_ZeroMQ_Connector():

    """
    Setup ZeroMQ -> MetaTrader Connector
    """
    def __init__(self,
                 _ClientID='DLabs_Python',  # Unique ID for this client
                 _host='localhost',         # Host to connect to
                 _protocol='tcp',           # Connection protocol
                 _PUSH_PORT=32768,           # Port for Sending commands
                 _PULL_PORT=32769,           # Port for Receiving responses
                 _SUB_PORT=32770,            # Port for Subscribing for prices
                 _delimiter=';'):           # String delimiter

        # Strategy Status (if this is False, ZeroMQ will not listen for data)
        self._ACTIVE = True

        # Client ID
        self._ClientID = _ClientID

        # ZeroMQ Host
        self._host = _host

        # Connection Protocol
        self._protocol = _protocol

        # ZeroMQ Context
        self._ZMQ_CONTEXT = zmq.Context()

        # TCP Connection URL Template
        self._URL = self._protocol + "://" + self._host + ":"

        # Ports for PUSH, PULL and SUB sockets respectively
        self._PUSH_PORT = _PUSH_PORT
        self._PULL_PORT = _PULL_PORT
        self._SUB_PORT = _SUB_PORT

        # Create Sockets
        self._PUSH_SOCKET = self._ZMQ_CONTEXT.socket(zmq.PUSH)
        self._PULL_SOCKET = self._ZMQ_CONTEXT.socket(zmq.PULL)
        self._SUB_SOCKET = self._ZMQ_CONTEXT.socket(zmq.SUB)

        # Bind PUSH Socket to send commands to MetaTrader
        self._PUSH_SOCKET.connect(self._URL + str(self._PUSH_PORT))
        print("[INIT] Ready to send commands to METATRADER (PUSH): " + str(self._PUSH_PORT))

        # Connect PULL Socket to receive command responses from MetaTrader
        self._PULL_SOCKET.connect(self._URL + str(self._PULL_PORT))
        print("[INIT] Listening for responses from METATRADER (PULL): " + str(self._PULL_PORT))

        # Connect SUB Socket to receive market data from MetaTrader
        self._SUB_SOCKET.connect(self._URL + str(self._SUB_PORT))

        # Initialize POLL set and register PULL and SUB sockets
        self._poller = zmq.Poller()
        self._poller.register(self._PULL_SOCKET, zmq.POLLIN)
        self._poller.register(self._SUB_SOCKET, zmq.POLLIN)

        # Start listening for responses to commands and new market data
        self._string_delimiter = _delimiter

        # BID/ASK Market Data Subscription Threads ({SYMBOL: Thread})
        self._MarketData_Thread = None

        # Begin polling for PULL / SUB data
        self._MarketData_Thread = Thread(target=self._DWX_ZMQ_Poll_Data_, args=(self._string_delimiter))
        self._MarketData_Thread.start()

        # Market Data Dictionary by Symbol (holds tick data)
        self._Market_Data_DB = {}   # {SYMBOL: {TIMESTAMP: (BID, ASK)}}

        # Temporary Order STRUCT for convenience wrappers later.
        self.temp_order_dict = self._generate_default_order_dict()

        # Thread returns the most recently received DATA block here
        self._thread_data_output = None

    ##########################################################################

    """
    Set Status (to enable/disable strategy manually)
    """
    def _setStatus(self, _new_status=False):

        self._ACTIVE = _new_status
        print("\n**\n[KERNEL] Setting Status to {} - Deactivating Threads.. please wait a bit.\n**".format(_new_status))

    ##########################################################################

    """
    Function to send commands to MetaTrader (PUSH)
    """
    def remote_send(self, _socket, _data):
        _socket.send_string(_data, zmq.DONTWAIT)

    ##########################################################################

    """
    Function to retrieve data from MetaTrader (PULL or SUB)
    """
    def remote_recv(self, _socket):
        msg = _socket.recv_string(zmq.DONTWAIT)
        return msg

    ##########################################################################

    # Convenience functions to permit easy trading via underlying functions.

    # OPEN ORDER
    def _DWX_MTX_NEW_TRADE_(self, _order=None):

        if _order is None:
            _order = self._generate_default_order_dict()

        # Execute
        self._DWX_MTX_SEND_COMMAND_(**_order)

    # MODIFY ORDER
    def _DWX_MTX_MODIFY_TRADE_BY_TICKET_(self, _ticket, _SL, _TP): # in points

        try:
            self.temp_order_dict['_action'] = 'MODIFY'
            self.temp_order_dict['_SL'] = _SL
            self.temp_order_dict['_TP'] = _TP
            self.temp_order_dict['_ticket'] = _ticket

            # Execute
            self._DWX_MTX_SEND_COMMAND_(**self.temp_order_dict)

        except KeyError:
            print("[ERROR] Order Ticket {} not found!".format(_ticket))

    # CLOSE ORDER
    def _DWX_MTX_CLOSE_TRADE_BY_TICKET_(self, _ticket):

        try:
            self.temp_order_dict['_action'] = 'CLOSE'
            self.temp_order_dict['_ticket'] = _ticket

            # Execute
            self._DWX_MTX_SEND_COMMAND_(**self.temp_order_dict)

        except KeyError:
            print("[ERROR] Order Ticket {} not found!".format(_ticket))

    # CLOSE PARTIAL
    def _DWX_MTX_CLOSE_PARTIAL_BY_TICKET_(self, _ticket, _lots):

        try:
            self.temp_order_dict['_action'] = 'CLOSE_PARTIAL'
            self.temp_order_dict['_ticket'] = _ticket
            self.temp_order_dict['_lots'] = _lots

            # Execute
            self._DWX_MTX_SEND_COMMAND_(**self.temp_order_dict)

        except KeyError:
            print("[ERROR] Order Ticket {} not found!".format(_ticket))

    # CLOSE MAGIC
    def _DWX_MTX_CLOSE_TRADES_BY_MAGIC_(self, _magic):

        try:
            self.temp_order_dict['_action'] = 'CLOSE_MAGIC'
            self.temp_order_dict['_magic'] = _magic

            # Execute
            self._DWX_MTX_SEND_COMMAND_(**self.temp_order_dict)

        except KeyError:
            pass

    # CLOSE ALL TRADES
    def _DWX_MTX_CLOSE_ALL_TRADES_(self):

        try:
            self.temp_order_dict['_action'] = 'CLOSE_ALL'

            # Execute
            self._DWX_MTX_SEND_COMMAND_(**self.temp_order_dict)

        except KeyError:
            pass

    # GET OPEN TRADES
    def _DWX_MTX_GET_ALL_OPEN_TRADES_(self):

        try:
            self.temp_order_dict['_action'] = 'GET_OPEN_TRADES'

            # Execute
            self._DWX_MTX_SEND_COMMAND_(**self.temp_order_dict)

        except KeyError:
            pass

    # DEFAULT ORDER DICT
    def _generate_default_order_dict(self):
        return({'_action': 'OPEN',
                  '_type': 0,
                  '_symbol': 'EURUSD',
                  '_price': 0.0,
                  '_SL': 500, # SL/TP in POINTS, not pips.
                  '_TP': 500,
                  '_comment': 'DWX_Python_to_MT',
                  '_lots': 0.01,
                  '_magic': 123456,
                  '_ticket': 0})

    # DEFAULT DATA REQUEST DICT
    def _generate_default_data_dict(self):
        return({'_action': 'DATA',
                  '_symbol': 'EURUSD',
                  '_timeframe': 1440, # M1 = 1, M5 = 5, and so on..
                  '_start': '2018.12.21 17:00:00', # timestamp in MT4 recognized format
                  '_end': '2018.12.21 17:05:00'})

    ##########################################################################
    """
    Function to construct messages for sending DATA commands to MetaTrader
    """
    def _DWX_MTX_SEND_MARKETDATA_REQUEST_(self,
                                 _symbol='EURUSD',
                                 _timeframe=1,
                                 _start='2019.01.04 17:00:00',
                                 _end=Timestamp.now().strftime('%Y.%m.%d %H:%M:00')):
                                 #_end='2019.01.04 17:05:00'):

        _msg = "{};{};{};{};{}".format('DATA',
                                     _symbol,
                                     _timeframe,
                                     _start,
                                     _end)
        # Send via PUSH Socket
        self.remote_send(self._PUSH_SOCKET, _msg)

    ##########################################################################
    """
    Function to construct messages for sending Trade commands to MetaTrader
    """
    def _DWX_MTX_SEND_COMMAND_(self, _action='OPEN', _type=0,
                                 _symbol='EURUSD', _price=0.0,
                                 _SL=50, _TP=50, _comment="Python-to-MT",
                                 _lots=0.01, _magic=123456, _ticket=0):

        _msg = "{};{};{};{};{};{};{};{};{};{};{}".format('TRADE',_action,_type,
                                                         _symbol,_price,
                                                         _SL,_TP,_comment,
                                                         _lots,_magic,
                                                         _ticket)

        # Send via PUSH Socket
        self.remote_send(self._PUSH_SOCKET, _msg)

        """
         compArray[0] = TRADE or DATA
         compArray[1] = ACTION (e.g. OPEN, MODIFY, CLOSE)
         compArray[2] = TYPE (e.g. OP_BUY, OP_SELL, etc - only used when ACTION=OPEN)

         For compArray[0] == DATA, format is:
             DATA|SYMBOL|TIMEFRAME|START_DATETIME|END_DATETIME

         // ORDER TYPES:
         // https://docs.mql4.com/constants/tradingconstants/orderproperties

         // OP_BUY = 0
         // OP_SELL = 1
         // OP_BUYLIMIT = 2
         // OP_SELLLIMIT = 3
         // OP_BUYSTOP = 4
         // OP_SELLSTOP = 5

         compArray[3] = Symbol (e.g. EURUSD, etc.)
         compArray[4] = Open/Close Price (ignored if ACTION = MODIFY)
         compArray[5] = SL
         compArray[6] = TP
         compArray[7] = Trade Comment
         compArray[8] = Lots
         compArray[9] = Magic Number
         compArray[10] = Ticket Number (MODIFY/CLOSE)
         """
        # pass

    ##########################################################################

    """
    Function to check Poller for new reponses (PULL) and market data (SUB)
    """

    def _DWX_ZMQ_Poll_Data_(self,
                           string_delimiter=';',
                           _verbose=False):

        while self._ACTIVE:

            sockets = dict(self._poller.poll())

            # Process response to commands sent to MetaTrader
            if self._PULL_SOCKET in sockets and sockets[self._PULL_SOCKET] == zmq.POLLIN:

                try:

                    msg = self._PULL_SOCKET.recv_string(zmq.DONTWAIT)

                    # If data is returned, store as pandas Series
                    if msg != '' and msg != None:

                        try:
                            _data = eval(msg)

                            self._thread_data_output = _data
                            print(_data) # default logic

                        except Exception as ex:
                            _exstr = "Exception Type {0}. Args:\n{1!r}"
                            _msg = _exstr.format(type(ex).__name__, ex.args)
                            print(_msg)

                except zmq.error.Again:
                    pass # resource temporarily unavailable, nothing to print
                except ValueError:
                    pass # No data returned, passing iteration.
                except UnboundLocalError:
                    pass # _symbol may sometimes get referenced before being assigned.

            # Receive new market data from MetaTrader
            if self._SUB_SOCKET in sockets and sockets[self._SUB_SOCKET] == zmq.POLLIN:

                try:
                    msg = self._SUB_SOCKET.recv_string(zmq.DONTWAIT)

                    if msg != "":
                        _symbol, _data = msg.split(" ")
                        _bid, _ask = _data.split(string_delimiter)
                        _timestamp = str(Timestamp.now('UTC'))[:-6]

                        if _verbose:
                            print("\n[" + _symbol + "] " + _timestamp + " (" + _bid + "/" + _ask + ") BID/ASK")

                        # Update Market Data DB
                        if _symbol not in self._Market_Data_DB.keys():
                            self._Market_Data_DB[_symbol] = {}

                        self._Market_Data_DB[_symbol][_timestamp] = (float(_bid), float(_ask))

                except zmq.error.Again:
                    pass # resource temporarily unavailable, nothing to print
                except ValueError:
                    pass # No data returned, passing iteration.
                except UnboundLocalError:
                    pass # _symbol may sometimes get referenced before being assigned.

    ##########################################################################

    """
    Function to subscribe to given Symbol's BID/ASK feed from MetaTrader
    """
    def _DWX_MTX_SUBSCRIBE_MARKETDATA_(self, _symbol, _string_delimiter=';'):

        # Subscribe to SYMBOL first.
        self._SUB_SOCKET.setsockopt_string(zmq.SUBSCRIBE, _symbol)

        if self._MarketData_Thread is None:

            self._MarketData_Thread = Thread(target=self._DWX_ZMQ_Poll_Data, args=(_string_delimiter))
            self._MarketData_Thread.start()

        print("[KERNEL] Subscribed to {} BID/ASK updates. See self._Market_Data_DB.".format(_symbol))

    """
    Function to unsubscribe to given Symbol's BID/ASK feed from MetaTrader
    """
    def _DWX_MTX_UNSUBSCRIBE_MARKETDATA_(self, _symbol):

        self._SUB_SOCKET.setsockopt_string(zmq.UNSUBSCRIBE, _symbol)
        print("\n**\n[KERNEL] Unsubscribing from " + _symbol + "\n**\n")

    """
    Function to unsubscribe from ALL MetaTrader Symbols
    """
    def _DWX_MTX_UNSUBSCRIBE_ALL_MARKETDATA_REQUESTS_(self):

        self._setStatus(False)
        self._MarketData_Thread = None

    ##########################################################################
<commit_msg>Update DWX_ZeroMQ_Connector_v2_0_1_RC8.py<commit_after># -*- coding: utf-8 -*-
"""
    DWX_ZeroMQ_Connector_v2_0_1_RC8.py
    --
    @author: Darwinex Labs (www.darwinex.com)

    Copyright (c) 2017-2019, Darwinex. All rights reserved.

    Licensed under the BSD 3-Clause License, you may not use this file except
    in compliance with the License.

    You may obtain a copy of the License at:
    https://opensource.org/licenses/BSD-3-Clause
"""

# IMPORT zmq library
import zmq
from pandas import Timestamp
from threading import Thread

class DWX_ZeroMQ_Connector():

    """
    Setup ZeroMQ -> MetaTrader Connector
    """
    def __init__(self,
                 _ClientID='DLabs_Python',  # Unique ID for this client
                 _host='localhost',         # Host to connect to
                 _protocol='tcp',           # Connection protocol
                 _PUSH_PORT=32768,           # Port for Sending commands
                 _PULL_PORT=32769,           # Port for Receiving responses
                 _SUB_PORT=32770,            # Port for Subscribing for prices
                 _delimiter=';',
                 _verbose=False):           # String delimiter

        # Strategy Status (if this is False, ZeroMQ will not listen for data)
        self._ACTIVE = True

        # Client ID
        self._ClientID = _ClientID

        # ZeroMQ Host
        self._host = _host

        # Connection Protocol
        self._protocol = _protocol

        # ZeroMQ Context
        self._ZMQ_CONTEXT = zmq.Context()

        # TCP Connection URL Template
        self._URL = self._protocol + "://" + self._host + ":"

        # Ports for PUSH, PULL and SUB sockets respectively
        self._PUSH_PORT = _PUSH_PORT
        self._PULL_PORT = _PULL_PORT
        self._SUB_PORT = _SUB_PORT

        # Create Sockets
        self._PUSH_SOCKET = self._ZMQ_CONTEXT.socket(zmq.PUSH)
        self._PULL_SOCKET = self._ZMQ_CONTEXT.socket(zmq.PULL)
        self._SUB_SOCKET = self._ZMQ_CONTEXT.socket(zmq.SUB)

        # Bind PUSH Socket to send commands to MetaTrader
        self._PUSH_SOCKET.connect(self._URL + str(self._PUSH_PORT))
        print("[INIT] Ready to send commands to METATRADER (PUSH): " + str(self._PUSH_PORT))

        # Connect PULL Socket to receive command responses from MetaTrader
        self._PULL_SOCKET.connect(self._URL + str(self._PULL_PORT))
        print("[INIT] Listening for responses from METATRADER (PULL): " + str(self._PULL_PORT))

        # Connect SUB Socket to receive market data from MetaTrader
        self._SUB_SOCKET.connect(self._URL + str(self._SUB_PORT))

        # Initialize POLL set and register PULL and SUB sockets
        self._poller = zmq.Poller()
        self._poller.register(self._PULL_SOCKET, zmq.POLLIN)
        self._poller.register(self._SUB_SOCKET, zmq.POLLIN)

        # Start listening for responses to commands and new market data
        self._string_delimiter = _delimiter

        # BID/ASK Market Data Subscription Threads ({SYMBOL: Thread})
        self._MarketData_Thread = None

        # Begin polling for PULL / SUB data
        self._MarketData_Thread = Thread(target=self._DWX_ZMQ_Poll_Data_, args=(self._string_delimiter))
        self._MarketData_Thread.start()

        # Market Data Dictionary by Symbol (holds tick data)
        self._Market_Data_DB = {}   # {SYMBOL: {TIMESTAMP: (BID, ASK)}}

        # Temporary Order STRUCT for convenience wrappers later.
        self.temp_order_dict = self._generate_default_order_dict()

        # Thread returns the most recently received DATA block here
        self._thread_data_output = None

        # Verbosity
        self._verbose = _verbose

    ##########################################################################

    """
    Set Status (to enable/disable strategy manually)
    """
    def _setStatus(self, _new_status=False):

        self._ACTIVE = _new_status
        print("\n**\n[KERNEL] Setting Status to {} - Deactivating Threads.. please wait a bit.\n**".format(_new_status))

    ##########################################################################

    """
    Function to send commands to MetaTrader (PUSH)
    """
    def remote_send(self, _socket, _data):
        _socket.send_string(_data, zmq.DONTWAIT)

    ##########################################################################

    """
    Function to retrieve data from MetaTrader (PULL or SUB)
    """
    def remote_recv(self, _socket):
        msg = _socket.recv_string(zmq.DONTWAIT)
        return msg

    ##########################################################################

    # Convenience functions to permit easy trading via underlying functions.

    # OPEN ORDER
    def _DWX_MTX_NEW_TRADE_(self, _order=None):

        if _order is None:
            _order = self._generate_default_order_dict()

        # Execute
        self._DWX_MTX_SEND_COMMAND_(**_order)

    # MODIFY ORDER
    def _DWX_MTX_MODIFY_TRADE_BY_TICKET_(self, _ticket, _SL, _TP): # in points

        try:
            self.temp_order_dict['_action'] = 'MODIFY'
            self.temp_order_dict['_SL'] = _SL
            self.temp_order_dict['_TP'] = _TP
            self.temp_order_dict['_ticket'] = _ticket

            # Execute
            self._DWX_MTX_SEND_COMMAND_(**self.temp_order_dict)

        except KeyError:
            print("[ERROR] Order Ticket {} not found!".format(_ticket))

    # CLOSE ORDER
    def _DWX_MTX_CLOSE_TRADE_BY_TICKET_(self, _ticket):

        try:
            self.temp_order_dict['_action'] = 'CLOSE'
            self.temp_order_dict['_ticket'] = _ticket

            # Execute
            self._DWX_MTX_SEND_COMMAND_(**self.temp_order_dict)

        except KeyError:
            print("[ERROR] Order Ticket {} not found!".format(_ticket))

    # CLOSE PARTIAL
    def _DWX_MTX_CLOSE_PARTIAL_BY_TICKET_(self, _ticket, _lots):

        try:
            self.temp_order_dict['_action'] = 'CLOSE_PARTIAL'
            self.temp_order_dict['_ticket'] = _ticket
            self.temp_order_dict['_lots'] = _lots

            # Execute
            self._DWX_MTX_SEND_COMMAND_(**self.temp_order_dict)

        except KeyError:
            print("[ERROR] Order Ticket {} not found!".format(_ticket))

    # CLOSE MAGIC
    def _DWX_MTX_CLOSE_TRADES_BY_MAGIC_(self, _magic):

        try:
            self.temp_order_dict['_action'] = 'CLOSE_MAGIC'
            self.temp_order_dict['_magic'] = _magic

            # Execute
            self._DWX_MTX_SEND_COMMAND_(**self.temp_order_dict)

        except KeyError:
            pass

    # CLOSE ALL TRADES
    def _DWX_MTX_CLOSE_ALL_TRADES_(self):

        try:
            self.temp_order_dict['_action'] = 'CLOSE_ALL'

            # Execute
            self._DWX_MTX_SEND_COMMAND_(**self.temp_order_dict)

        except KeyError:
            pass

    # GET OPEN TRADES
    def _DWX_MTX_GET_ALL_OPEN_TRADES_(self):

        try:
            self.temp_order_dict['_action'] = 'GET_OPEN_TRADES'

            # Execute
            self._DWX_MTX_SEND_COMMAND_(**self.temp_order_dict)

        except KeyError:
            pass

    # DEFAULT ORDER DICT
    def _generate_default_order_dict(self):
        return({'_action': 'OPEN',
                  '_type': 0,
                  '_symbol': 'EURUSD',
                  '_price': 0.0,
                  '_SL': 500, # SL/TP in POINTS, not pips.
                  '_TP': 500,
                  '_comment': 'DWX_Python_to_MT',
                  '_lots': 0.01,
                  '_magic': 123456,
                  '_ticket': 0})

    # DEFAULT DATA REQUEST DICT
    def _generate_default_data_dict(self):
        return({'_action': 'DATA',
                  '_symbol': 'EURUSD',
                  '_timeframe': 1440, # M1 = 1, M5 = 5, and so on..
                  '_start': '2018.12.21 17:00:00', # timestamp in MT4 recognized format
                  '_end': '2018.12.21 17:05:00'})

    ##########################################################################
    """
    Function to construct messages for sending DATA commands to MetaTrader
    """
    def _DWX_MTX_SEND_MARKETDATA_REQUEST_(self,
                                 _symbol='EURUSD',
                                 _timeframe=1,
                                 _start='2019.01.04 17:00:00',
                                 _end=Timestamp.now().strftime('%Y.%m.%d %H:%M:00')):
                                 #_end='2019.01.04 17:05:00'):

        _msg = "{};{};{};{};{}".format('DATA',
                                     _symbol,
                                     _timeframe,
                                     _start,
                                     _end)
        # Send via PUSH Socket
        self.remote_send(self._PUSH_SOCKET, _msg)

    ##########################################################################
    """
    Function to construct messages for sending Trade commands to MetaTrader
    """
    def _DWX_MTX_SEND_COMMAND_(self, _action='OPEN', _type=0,
                                 _symbol='EURUSD', _price=0.0,
                                 _SL=50, _TP=50, _comment="Python-to-MT",
                                 _lots=0.01, _magic=123456, _ticket=0):

        _msg = "{};{};{};{};{};{};{};{};{};{};{}".format('TRADE',_action,_type,
                                                         _symbol,_price,
                                                         _SL,_TP,_comment,
                                                         _lots,_magic,
                                                         _ticket)

        # Send via PUSH Socket
        self.remote_send(self._PUSH_SOCKET, _msg)

        """
         compArray[0] = TRADE or DATA
         compArray[1] = ACTION (e.g. OPEN, MODIFY, CLOSE)
         compArray[2] = TYPE (e.g. OP_BUY, OP_SELL, etc - only used when ACTION=OPEN)

         For compArray[0] == DATA, format is:
             DATA|SYMBOL|TIMEFRAME|START_DATETIME|END_DATETIME

         // ORDER TYPES:
         // https://docs.mql4.com/constants/tradingconstants/orderproperties

         // OP_BUY = 0
         // OP_SELL = 1
         // OP_BUYLIMIT = 2
         // OP_SELLLIMIT = 3
         // OP_BUYSTOP = 4
         // OP_SELLSTOP = 5

         compArray[3] = Symbol (e.g. EURUSD, etc.)
         compArray[4] = Open/Close Price (ignored if ACTION = MODIFY)
         compArray[5] = SL
         compArray[6] = TP
         compArray[7] = Trade Comment
         compArray[8] = Lots
         compArray[9] = Magic Number
         compArray[10] = Ticket Number (MODIFY/CLOSE)
         """
        # pass

    ##########################################################################

    """
    Function to check Poller for new reponses (PULL) and market data (SUB)
    """

    def _DWX_ZMQ_Poll_Data_(self,
                           string_delimiter=';'):

        while self._ACTIVE:

            sockets = dict(self._poller.poll())

            # Process response to commands sent to MetaTrader
            if self._PULL_SOCKET in sockets and sockets[self._PULL_SOCKET] == zmq.POLLIN:

                try:

                    msg = self._PULL_SOCKET.recv_string(zmq.DONTWAIT)

                    # If data is returned, store as pandas Series
                    if msg != '' and msg != None:

                        try:
                            _data = eval(msg)

                            self._thread_data_output = _data
                            if self._verbose:
                                print(_data) # default logic

                        except Exception as ex:
                            _exstr = "Exception Type {0}. Args:\n{1!r}"
                            _msg = _exstr.format(type(ex).__name__, ex.args)
                            print(_msg)

                except zmq.error.Again:
                    pass # resource temporarily unavailable, nothing to print
                except ValueError:
                    pass # No data returned, passing iteration.
                except UnboundLocalError:
                    pass # _symbol may sometimes get referenced before being assigned.

            # Receive new market data from MetaTrader
            if self._SUB_SOCKET in sockets and sockets[self._SUB_SOCKET] == zmq.POLLIN:

                try:
                    msg = self._SUB_SOCKET.recv_string(zmq.DONTWAIT)

                    if msg != "":
                        _symbol, _data = msg.split(" ")
                        _bid, _ask = _data.split(string_delimiter)
                        _timestamp = str(Timestamp.now('UTC'))[:-6]

                        if self._verbose:
                            print("\n[" + _symbol + "] " + _timestamp + " (" + _bid + "/" + _ask + ") BID/ASK")

                        # Update Market Data DB
                        if _symbol not in self._Market_Data_DB.keys():
                            self._Market_Data_DB[_symbol] = {}

                        self._Market_Data_DB[_symbol][_timestamp] = (float(_bid), float(_ask))

                except zmq.error.Again:
                    pass # resource temporarily unavailable, nothing to print
                except ValueError:
                    pass # No data returned, passing iteration.
                except UnboundLocalError:
                    pass # _symbol may sometimes get referenced before being assigned.

    ##########################################################################

    """
    Function to subscribe to given Symbol's BID/ASK feed from MetaTrader
    """
    def _DWX_MTX_SUBSCRIBE_MARKETDATA_(self, _symbol, _string_delimiter=';'):

        # Subscribe to SYMBOL first.
        self._SUB_SOCKET.setsockopt_string(zmq.SUBSCRIBE, _symbol)

        if self._MarketData_Thread is None:

            self._MarketData_Thread = Thread(target=self._DWX_ZMQ_Poll_Data, args=(_string_delimiter))
            self._MarketData_Thread.start()

        print("[KERNEL] Subscribed to {} BID/ASK updates. See self._Market_Data_DB.".format(_symbol))

    """
    Function to unsubscribe to given Symbol's BID/ASK feed from MetaTrader
    """
    def _DWX_MTX_UNSUBSCRIBE_MARKETDATA_(self, _symbol):

        self._SUB_SOCKET.setsockopt_string(zmq.UNSUBSCRIBE, _symbol)
        print("\n**\n[KERNEL] Unsubscribing from " + _symbol + "\n**\n")

    """
    Function to unsubscribe from ALL MetaTrader Symbols
    """
    def _DWX_MTX_UNSUBSCRIBE_ALL_MARKETDATA_REQUESTS_(self):

        self._setStatus(False)
        self._MarketData_Thread = None

    ##########################################################################
<|endoftext|>


<commit_before># @author:  Kevin S Hahn

"""
API request handlers for process-job-handling.

represents the /nimsapi/jobs route
"""

import os
import bson
import json
import logging
import datetime
log = logging.getLogger('nimsapi.jobs')

import base

# TODO: what should this whitelist contain? protocol + FQDN?
# ex. https://coronal.stanford.edu
PROCESSOR_WHITELIST = [
    'dockerhost',
]

JOB_STATES = [
    'pending',      # created but not started
    'queued',       # job claimed by a processor
    'running',      # job running on a processor
    'done',         # job completed successfully
    'failed',       # some error occurred,
    'paused',       # job paused.  can't think when this would be useful...
]

# Jobs must now how they affect the various components of a file description
# some "special" case things will reset state from 'orig' to 'pending'
# but the usual case will be to append an item to the state list.

# TODO: create job function should live here
# where it can be editted with the route that consume and modify the jobs

# GET  /jobs full list of jobs, allow specifiers, status=
# POST /jobs creates a new job. this will be used by webapp to add new jobs
# GET  /jobs/<_id> get information about one job
# PUT  /jobs/<_id> update informabout about one job
# GET  /jobs/next, special route to get the 'next job'

class Jobs(base.RequestHandler):

    """Provide /jobs API routes."""

    def get(self):
        """
        Return one Job that needs processing.

        TODO: allow querying for group
        TODO: allow querying for project
        TODO: allow querying by other meta data. can this be generalized?

        """
        # TODO: auth
        return list(self.app.db.jobs.find())

    def count(self):
        """Return the total number of jobs."""
        # no auth?
        return self.app.db.jobs.count()

    def counts(self):
        """Return more information about the jobs."""
        counts = {
            'total': self.app.db.jobs.count(),
            'failed': self.app.db.jobs.find({'status': 'failed'}).count(),
            'pending': self.app.db.jobs.find({'status': 'pending'}).count(),
            'done': self.app.db.jobs.find({'status': 'done'}).count(),
        }
        return counts

    def next(self):
        """Return the next job in the queue that matches the query parameters."""
        # TODO: add ability to query on things like psd type or psd name
        try:
            query_params = self.request.json
        except ValueError as e:
            self.abort(400, str(e))

        query = {'status': 'pending'}
        try:
            query_params = self.request.json
        except ValueError as e:
            self.abort(400, str(e))

        project_query = query_params.get('project')
        group_query = query_params.get('group')
        query = {'status': 'pending'}
        if project_query:
            query.update({'project': project_query})
        if group_query:
            query.update({'group': group_query})

        # TODO: how to guarantee the 'oldest' jobs pending jobs are given out first
        job_spec = self.app.db.jobs.find_and_modify(
            query,
            {'$set': {'status': 'queued', 'modified': datetime.datetime.now()}},
            sort=[('modified', -1)],
            new=True
        )
        return job_spec

class Job(base.RequestHandler):

    """Provides /Jobs/<jid> routes."""

    # TODO flesh out the job schema
    json_schema = {
        '$schema': 'http://json-schema.org/draft-04/schema#',
        'title': 'User',
        'type': 'object',
        'properties': {
            '_id': {
                'title': 'Job ID',
                'type': 'string',
            },
        },
        'required': ['_id'],
        'additionalProperties': True,
    }

    def get(self, _id):
        return self.app.db.jobs.find_one({'_id': int(_id)})

    def put(self, _id):
        """Update a single job."""
        payload = self.request.json
        # TODO: validate the json before updating the db
        self.app.db.jobs.update({'_id': int(_id)}, {'$set': {'status': payload.get('status'), 'activity': payload.get('activity')}})
<commit_msg>remove unused imports<commit_after># @author:  Kevin S Hahn

"""
API request handlers for process-job-handling.

represents the /nimsapi/jobs route
"""

import logging
import datetime
log = logging.getLogger('nimsapi.jobs')

import base

# TODO: what should this whitelist contain? protocol + FQDN?
# ex. https://coronal.stanford.edu
PROCESSOR_WHITELIST = [
    'dockerhost',
]

JOB_STATES = [
    'pending',      # created but not started
    'queued',       # job claimed by a processor
    'running',      # job running on a processor
    'done',         # job completed successfully
    'failed',       # some error occurred,
    'paused',       # job paused.  can't think when this would be useful...
]

# Jobs must now how they affect the various components of a file description
# some "special" case things will reset state from 'orig' to 'pending'
# but the usual case will be to append an item to the state list.

# TODO: create job function should live here
# where it can be editted with the route that consume and modify the jobs

# GET  /jobs full list of jobs, allow specifiers, status=
# POST /jobs creates a new job. this will be used by webapp to add new jobs
# GET  /jobs/<_id> get information about one job
# PUT  /jobs/<_id> update informabout about one job
# GET  /jobs/next, special route to get the 'next job'

class Jobs(base.RequestHandler):

    """Provide /jobs API routes."""

    def get(self):
        """
        Return one Job that needs processing.

        TODO: allow querying for group
        TODO: allow querying for project
        TODO: allow querying by other meta data. can this be generalized?

        """
        # TODO: auth
        return list(self.app.db.jobs.find())

    def count(self):
        """Return the total number of jobs."""
        # no auth?
        return self.app.db.jobs.count()

    def counts(self):
        """Return more information about the jobs."""
        counts = {
            'total': self.app.db.jobs.count(),
            'failed': self.app.db.jobs.find({'status': 'failed'}).count(),
            'pending': self.app.db.jobs.find({'status': 'pending'}).count(),
            'done': self.app.db.jobs.find({'status': 'done'}).count(),
        }
        return counts

    def next(self):
        """Return the next job in the queue that matches the query parameters."""
        # TODO: add ability to query on things like psd type or psd name
        try:
            query_params = self.request.json
        except ValueError as e:
            self.abort(400, str(e))

        query = {'status': 'pending'}
        try:
            query_params = self.request.json
        except ValueError as e:
            self.abort(400, str(e))

        project_query = query_params.get('project')
        group_query = query_params.get('group')
        query = {'status': 'pending'}
        if project_query:
            query.update({'project': project_query})
        if group_query:
            query.update({'group': group_query})

        # TODO: how to guarantee the 'oldest' jobs pending jobs are given out first
        job_spec = self.app.db.jobs.find_and_modify(
            query,
            {'$set': {'status': 'queued', 'modified': datetime.datetime.now()}},
            sort=[('modified', -1)],
            new=True
        )
        return job_spec

class Job(base.RequestHandler):

    """Provides /Jobs/<jid> routes."""

    # TODO flesh out the job schema
    json_schema = {
        '$schema': 'http://json-schema.org/draft-04/schema#',
        'title': 'User',
        'type': 'object',
        'properties': {
            '_id': {
                'title': 'Job ID',
                'type': 'string',
            },
        },
        'required': ['_id'],
        'additionalProperties': True,
    }

    def get(self, _id):
        return self.app.db.jobs.find_one({'_id': int(_id)})

    def put(self, _id):
        """Update a single job."""
        payload = self.request.json
        # TODO: validate the json before updating the db
        self.app.db.jobs.update({'_id': int(_id)}, {'$set': {'status': payload.get('status'), 'activity': payload.get('activity')}})
<|endoftext|>