<commit_before>from __future__ import absolute_import, print_function

import errno
import os
import sys
import json

import time
from typing import TYPE_CHECKING

import psutil

MAX_ARGUMENTS_JSON_LENGTH = 65536

try:
    import subprocess32 as subprocess
except ImportError:
    import subprocess
import functools
import logging
import tempfile
import traceback

# noinspection PyCompatibility
from builtins import map

from future.utils import iteritems

from simpleflow import compat, format
from simpleflow import logger as simpleflow_logger
from simpleflow.exceptions import ExecutionError, ExecutionTimeoutError
from simpleflow.utils import json_dumps

if TYPE_CHECKING:
    from typing import Any, Iterable  # NOQA
    import inspect  # NOQA

__all__ = ['program', 'python']

class RequiredArgument(object):
    pass

def format_arguments(*args, **kwargs):
    """
    Returns a string that contains the values of *args* and *kwargs* as command
    line options.

    :param args: that can be converted to strings.
    :type  args: tuple.
    :param kwargs: whose keys and values can be converted to strings.
    :type  kwargs: dict.

    :returns:
        :rtype: str.

    The elements args must be convertible to strings and will be used as
    positional arguments.

    The items of *kwargs* are translated to key/value options (-c=1). Their
    format follows the convention of one hyphen for short options (-c) and two
    hyphens for long options (--val).

    Examples:

        >>> sorted(format_arguments('a', 'b', c=1, val=2))
        ['--val="2"', '-c="1"', 'a', 'b']

    """

    def arg(key):
        if len(key) == 1:
            return '-' + str(key)  # short option -c
        return '--' + str(key)  # long option --val

    return ['{}="{}"'.format(arg(k), v) for k, v in
            iteritems(kwargs)] + list(map(str, args))

def zip_arguments_defaults(argspec):
    # type: (inspect.ArgSpec) -> Iterable
    if not argspec.defaults:
        return []

    return zip(
        argspec.args[-len(argspec.defaults):],
        argspec.defaults)

def check_arguments(argspec, args):
    # type: (inspect.ArgSpec, Any) -> None
    """Validates there is the right number of arguments"""
    # func() or func(**kwargs) or func(a=1, b=2)
    if not argspec.varargs and not argspec.args and args:
        raise TypeError('command does not take varargs')

    # Calling func(a, b) with func(1, 2, 3)
    if not argspec.varargs and argspec.args and len(args) != len(argspec.args):
        raise TypeError('command takes {} arguments: {} passed'.format(
            len(argspec.args),
            len(args)))

def check_keyword_arguments(argspec, kwargs):
    # type: (inspect.ArgSpec, dict) -> None
    # func() or func(*args) or func(a, b)
    if not argspec.keywords and not argspec.defaults and kwargs:
        raise TypeError('command does not take keyword arguments')

    arguments_defaults = zip_arguments_defaults(argspec)
    not_found = (set(name for name, value in arguments_defaults if
                     value is RequiredArgument) -
                 set(kwargs))
    # Calling func(a=1, b) with func(2) instead of func(a=0, 2)
    if not_found:
        raise TypeError('argument{} "{}" not found'.format(
            's' if len(not_found) > 1 else '',
            ', '.join(not_found)))

def format_arguments_json(*args, **kwargs):
    dump = json_dumps({
        'args': args,
        'kwargs': kwargs,
    })
    return dump

def get_name(func):
    """
    Returns the name of a callable.

    It handles different types of callable: function, callable object with
    ``__call__`` method and callable objects that provide their name in the
    ``name`` attributes.

    :type func: callable.
    :returns:
        :rtype: str.

    """
    prefix = func.__module__

    if not callable(func):
        raise ValueError('{} is not callable'.format(
            func))

    if hasattr(func, 'name'):
        name = func.name
    elif hasattr(func, '__name__'):
        name = func.__name__
    else:
        name = func.__class__.__name__

    return '.'.join([prefix, name])

def wait_subprocess(process, timeout=None, command_info=None):
    """
    Wait for a process, raise if timeout.
    :param process: the process to wait
    :param timeout: timeout after 'timeout' seconds
    :type timeout: int | None
    :param command_info:
        :returns: return code
        :rtype: int.
    """
    if timeout:
        t_start = time.time()
        rc = process.poll()
        while time.time() - t_start < timeout and rc is None:
            time.sleep(1)
            rc = process.poll()

        if rc is None:
            try:
                process.terminate()  # send SIGTERM
            except OSError as e:
                # Ignore that exception the case the sub-process already terminated after last poll() call.
                if e.errno == errno.ESRCH:
                    return process.poll()
                else:
                    raise
            raise ExecutionTimeoutError(command=command_info, timeout_value=timeout)
        return rc
    return process.wait()

def python(interpreter='python', logger_name=__name__, timeout=None, kill_children=False):
    """
    Execute a callable as an external Python program.

    One of the use cases is to use a different interpreter than the current one
    such as pypy.

    Arguments of the decorated callable must be serializable in JSON.

    """

    def wrap_callable(func):
        @functools.wraps(func)
        def execute(*args, **kwargs):
            logger = logging.getLogger(logger_name)
            command = 'simpleflow.execute'  # name of a module.
            sys.stdout.flush()
            sys.stderr.flush()
            result_str = None  # useless
            context = kwargs.pop('context', {})
            with tempfile.TemporaryFile() as result_fd, tempfile.TemporaryFile() as error_fd:
                dup_result_fd = os.dup(result_fd.fileno())  # remove FD_CLOEXEC
                dup_error_fd = os.dup(error_fd.fileno())  # remove FD_CLOEXEC
                arguments_json = format_arguments_json(*args, **kwargs)
                full_command = [
                    interpreter, '-m', command,  # execute module a script.
                    get_name(func),
                    '--logger-name={}'.format(logger_name),
                    '--result-fd={}'.format(dup_result_fd),
                    '--error-fd={}'.format(dup_error_fd),
                    '--context={}'.format(json_dumps(context)),
                ]
                if len(arguments_json) < MAX_ARGUMENTS_JSON_LENGTH:  # command-line limit on Linux: 128K
                    full_command.append(arguments_json)
                    arg_file = None
                    arg_fd = None
                else:
                    arg_file = tempfile.TemporaryFile()
                    arg_file.write(arguments_json.encode('utf-8'))
                    arg_file.flush()
                    arg_file.seek(0)
                    arg_fd = os.dup(arg_file.fileno())
                    full_command.append('--arguments-json-fd={}'.format(arg_fd))
                    full_command.append('foo')  # dummy funcarg
                if kill_children:
                    full_command.append('--kill-children')
                if compat.PY2:  # close_fds doesn't work with python2 (using its C _posixsubprocess helper)
                    close_fds = False
                    pass_fds = []
                else:
                    close_fds = True
                    pass_fds = [dup_result_fd, dup_error_fd]
                    if arg_file:
                        pass_fds.append(arg_fd)
                process = subprocess.Popen(
                    full_command,
                    bufsize=-1,
                    close_fds=close_fds,
                    pass_fds=pass_fds,
                )
                rc = wait_subprocess(process, timeout=timeout, command_info=full_command)
                os.close(dup_result_fd)
                os.close(dup_error_fd)
                if arg_file:
                    arg_file.close()
                if rc:
                    error_fd.seek(0)
                    err_output = error_fd.read()
                    if err_output:
                        if not compat.PY2:
                            err_output = err_output.decode('utf-8', errors='replace')
                    raise ExecutionError(err_output)

                result_fd.seek(0)
                result_str = result_fd.read()

            if not result_str:
                return None
            try:
                if not compat.PY2:
                    result_str = result_str.decode('utf-8', errors='replace')
                result = format.decode(result_str)
                return result
            except BaseException as ex:
                logger.exception('Exception in python.execute: {} {}'.format(ex.__class__.__name__, ex))
                logger.warning('%r', result_str)

        # Not automatically assigned in python < 3.2.
        execute.__wrapped__ = func
        execute.add_context_in_kwargs = True
        return execute

    return wrap_callable

def program(path=None, argument_format=format_arguments):
    r"""
    Decorate a callable to execute it as an external program.

    :param path: of the program to execute. If it is ``None`` the name of the
                 executable will be the name of the callable.
    :type  path: str.
    :param argument_format: takes the arguments of the callable and converts
                            them to command line arguments.
    :type  argument_format: callable(*args, **kwargs).

    :returns:
        :rtype: callable(*args, **kwargs).

    Examples
    --------

    >>> @program()
    ... def ls(path):
    ...     pass
    >>> ls('/etc/resolv.conf')
    '/etc/resolv.conf\n'

    It will execute the ``ls`` command and requires a single positional
    argument *path*.

    """
    import inspect

    def wrap_callable(func):
        @functools.wraps(func)
        def execute(*args, **kwargs):
            check_arguments(argspec, args)
            check_keyword_arguments(argspec, kwargs)

            command = path or func.__name__
            return subprocess.check_output(
                [command] + argument_format(*args, **kwargs),
                universal_newlines=True)

        try:
            args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = inspect.getfullargspec(func)
            argspec = inspect.ArgSpec(args, varargs, varkw, defaults)
        except AttributeError:
            # noinspection PyDeprecation
            argspec = inspect.getargspec(func)

        # Not automatically assigned in python < 3.2.
        execute.__wrapped__ = func
        return execute

    return wrap_callable

def make_callable(funcname):
    """
    Return a callable object from a string.

    This function resolves a name into a callable object. It automatically
    loads the required modules. If there is no module path, it considers the
    callable is a builtin.

    :param funcname: name of the callable.
    :type  funcname: str.

    :returns:
        :rtype: callable.

    Examples
    --------

    Loading a function from a library:

    >>> func = make_callable('itertools.chain')
    >>> list(func(range(3), range(4)))
    [0, 1, 2, 0, 1, 2, 3]

    Loading a builtin:

    >>> func = make_callable('map')
    >>> list(func(lambda x: x + 1, range(4)))
    [1, 2, 3, 4]

    """
    if '.' not in funcname:
        module_name = 'builtins'
        object_name = funcname
    else:
        module_name, object_name = funcname.rsplit('.', 1)

    module = __import__(module_name, fromlist=['*'])
    try:
        callable_ = getattr(module, object_name)
    except AttributeError:
        raise AttributeError('module {} has no attribute {}'.format(
            module.__name__,
            object_name,
        ))
    return callable_

def main():
    """
    When executed as a script, this module expects the name of a callable as
    its first argument and the arguments of the callable encoded in a JSON
    string as its second argument. It then executes the callable with the
    arguments after decoding them into Python objects. It finally encodes the
    value returned by the callable into a JSON string and prints it on stdout.

    the arguments of the callable are stored in a dict with the following
    format: ::

        {'args': [...],
         'kwargs': {
            ...,
         }
         }

    Synopsis
    --------

    ::
        usage: execute.py [-h] funcname funcargs

        positional arguments:
          funcname    name of the callable to execute
          funcargs    callable arguments in JSON

        optional arguments:
          -h, --help  show this help message and exit

    Examples
    --------

    ::
        $ python -m simpleflow.execute "os.path.exists" '{"args": ["/tmp"]}'
        true

    """
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument(
        'funcname',
        help='name of the callable to execute',
    )
    parser.add_argument(
        'funcargs',
        help='callable arguments in JSON',
    )
    parser.add_argument(
        '--context',
        help='Activity Context',
    )
    parser.add_argument(
        '--logger-name',
        help='logger name',
    )
    parser.add_argument(
        '--result-fd',
        type=int,
        default=1,
        metavar='N',
        help='result file descriptor',
    )
    parser.add_argument(
        '--error-fd',
        type=int,
        default=2,
        metavar='N',
        help='error file descriptor',
    )
    parser.add_argument(
        '--arguments-json-fd',
        type=int,
        default=None,
        metavar='N',
        help='JSON input file descriptor',
    )
    parser.add_argument(
        '--kill-children',
        action='store_true',
        help='kill child processes on exit',
    )
    cmd_arguments = parser.parse_args()

    def kill_child_processes():
        process = psutil.Process(os.getpid())
        children = process.children(recursive=True)

        for child in children:
            try:
                child.terminate()
            except psutil.NoSuchProcess:
                pass
        _, still_alive = psutil.wait_procs(children, timeout=0.3)
        for child in still_alive:
            try:
                child.kill()
            except psutil.NoSuchProcess:
                pass

    funcname = cmd_arguments.funcname
    if cmd_arguments.arguments_json_fd is None:
        content = cmd_arguments.funcargs
        if content is None:
            parser.error('the following arguments are required: funcargs')
    else:
        with os.fdopen(cmd_arguments.arguments_json_fd) as arguments_json_file:
            content = arguments_json_file.read()
    try:
        arguments = format.decode(content)
    except Exception:
        raise ValueError('cannot load arguments from {}'.format(
            content))
    if cmd_arguments.logger_name:
        logger = logging.getLogger(cmd_arguments.logger_name)
    else:
        logger = simpleflow_logger
    callable_ = make_callable(funcname)
    if hasattr(callable_, '__wrapped__'):
        callable_ = callable_.__wrapped__
    args = arguments.get('args', ())
    kwargs = arguments.get('kwargs', {})
    context = json.loads(cmd_arguments.context) if cmd_arguments.context is not None else None
    try:
        if hasattr(callable_, 'execute'):
            inst = callable_(*args, **kwargs)
            if context is not None:
                inst.context = context
            result = inst.execute()
            if hasattr(inst, 'post_execute'):
                inst.post_execute()
        else:
            if context is not None:
                callable_.context = context
            result = callable_(*args, **kwargs)
    except Exception as err:
        logger.error('Exception: {}'.format(err))
        exc_type, exc_value, exc_traceback = sys.exc_info()
        tb = traceback.format_tb(exc_traceback)
        details = json_dumps(
            {
                'error': exc_type.__name__,
                'message': str(exc_value),
                'traceback': tb,
            },
            default=repr,
        )
        if cmd_arguments.error_fd == 2:
            sys.stderr.flush()
        if not compat.PY2:
            details = details.encode('utf-8')
        os.write(cmd_arguments.error_fd, details)
        if cmd_arguments.kill_children:
            kill_child_processes()
        sys.exit(1)

    if cmd_arguments.result_fd == 1:  # stdout (legacy)
        sys.stdout.flush()  # may have print's in flight
        os.write(cmd_arguments.result_fd, b'\n')
    result = json_dumps(result)
    if not compat.PY2:
        result = result.encode('utf-8')
    os.write(cmd_arguments.result_fd, result)
    if cmd_arguments.kill_children:
        kill_child_processes()

if __name__ == '__main__':
    main()
<commit_msg>Check whether subprocess32 is buggy<commit_after>from __future__ import absolute_import, print_function

import errno
import os
import sys
import json

import time
from typing import TYPE_CHECKING

import psutil

try:
    import subprocess32 as subprocess
except ImportError:
    import subprocess
import functools
import logging
import tempfile
import traceback

# noinspection PyCompatibility
from builtins import map

from future.utils import iteritems

from simpleflow import compat, format
from simpleflow import logger as simpleflow_logger
from simpleflow.exceptions import ExecutionError, ExecutionTimeoutError
from simpleflow.utils import json_dumps

if TYPE_CHECKING:
    from typing import Any, Iterable  # NOQA
    import inspect  # NOQA

MAX_ARGUMENTS_JSON_LENGTH = 65536

__all__ = ['program', 'python']

class RequiredArgument(object):
    pass

def format_arguments(*args, **kwargs):
    """
    Returns a string that contains the values of *args* and *kwargs* as command
    line options.

    :param args: that can be converted to strings.
    :type  args: tuple.
    :param kwargs: whose keys and values can be converted to strings.
    :type  kwargs: dict.

    :returns:
        :rtype: str.

    The elements args must be convertible to strings and will be used as
    positional arguments.

    The items of *kwargs* are translated to key/value options (-c=1). Their
    format follows the convention of one hyphen for short options (-c) and two
    hyphens for long options (--val).

    Examples:

        >>> sorted(format_arguments('a', 'b', c=1, val=2))
        ['--val="2"', '-c="1"', 'a', 'b']

    """

    def arg(key):
        if len(key) == 1:
            return '-' + str(key)  # short option -c
        return '--' + str(key)  # long option --val

    return ['{}="{}"'.format(arg(k), v) for k, v in
            iteritems(kwargs)] + list(map(str, args))

def zip_arguments_defaults(argspec):
    # type: (inspect.ArgSpec) -> Iterable
    if not argspec.defaults:
        return []

    return zip(
        argspec.args[-len(argspec.defaults):],
        argspec.defaults)

def check_arguments(argspec, args):
    # type: (inspect.ArgSpec, Any) -> None
    """Validates there is the right number of arguments"""
    # func() or func(**kwargs) or func(a=1, b=2)
    if not argspec.varargs and not argspec.args and args:
        raise TypeError('command does not take varargs')

    # Calling func(a, b) with func(1, 2, 3)
    if not argspec.varargs and argspec.args and len(args) != len(argspec.args):
        raise TypeError('command takes {} arguments: {} passed'.format(
            len(argspec.args),
            len(args)))

def check_keyword_arguments(argspec, kwargs):
    # type: (inspect.ArgSpec, dict) -> None
    # func() or func(*args) or func(a, b)
    if not argspec.keywords and not argspec.defaults and kwargs:
        raise TypeError('command does not take keyword arguments')

    arguments_defaults = zip_arguments_defaults(argspec)
    not_found = (set(name for name, value in arguments_defaults if
                     value is RequiredArgument) -
                 set(kwargs))
    # Calling func(a=1, b) with func(2) instead of func(a=0, 2)
    if not_found:
        raise TypeError('argument{} "{}" not found'.format(
            's' if len(not_found) > 1 else '',
            ', '.join(not_found)))

def format_arguments_json(*args, **kwargs):
    dump = json_dumps({
        'args': args,
        'kwargs': kwargs,
    })
    return dump

def get_name(func):
    """
    Returns the name of a callable.

    It handles different types of callable: function, callable object with
    ``__call__`` method and callable objects that provide their name in the
    ``name`` attributes.

    :type func: callable.
    :returns:
        :rtype: str.

    """
    prefix = func.__module__

    if not callable(func):
        raise ValueError('{} is not callable'.format(
            func))

    if hasattr(func, 'name'):
        name = func.name
    elif hasattr(func, '__name__'):
        name = func.__name__
    else:
        name = func.__class__.__name__

    return '.'.join([prefix, name])

def wait_subprocess(process, timeout=None, command_info=None):
    """
    Wait for a process, raise if timeout.
    :param process: the process to wait
    :param timeout: timeout after 'timeout' seconds
    :type timeout: int | None
    :param command_info:
        :returns: return code
        :rtype: int.
    """
    if timeout:
        t_start = time.time()
        rc = process.poll()
        while time.time() - t_start < timeout and rc is None:
            time.sleep(1)
            rc = process.poll()

        if rc is None:
            try:
                process.terminate()  # send SIGTERM
            except OSError as e:
                # Ignore that exception the case the sub-process already terminated after last poll() call.
                if e.errno == errno.ESRCH:
                    return process.poll()
                else:
                    raise
            raise ExecutionTimeoutError(command=command_info, timeout_value=timeout)
        return rc
    return process.wait()

def python(interpreter='python', logger_name=__name__, timeout=None, kill_children=False):
    """
    Execute a callable as an external Python program.

    One of the use cases is to use a different interpreter than the current one
    such as pypy.

    Arguments of the decorated callable must be serializable in JSON.

    """

    def wrap_callable(func):
        @functools.wraps(func)
        def execute(*args, **kwargs):
            logger = logging.getLogger(logger_name)
            command = 'simpleflow.execute'  # name of a module.
            sys.stdout.flush()
            sys.stderr.flush()
            result_str = None  # useless
            context = kwargs.pop('context', {})
            with tempfile.TemporaryFile() as result_fd, tempfile.TemporaryFile() as error_fd:
                dup_result_fd = os.dup(result_fd.fileno())  # remove FD_CLOEXEC
                dup_error_fd = os.dup(error_fd.fileno())  # remove FD_CLOEXEC
                arguments_json = format_arguments_json(*args, **kwargs)
                full_command = [
                    interpreter, '-m', command,  # execute module a script.
                    get_name(func),
                    '--logger-name={}'.format(logger_name),
                    '--result-fd={}'.format(dup_result_fd),
                    '--error-fd={}'.format(dup_error_fd),
                    '--context={}'.format(json_dumps(context)),
                ]
                if len(arguments_json) < MAX_ARGUMENTS_JSON_LENGTH:  # command-line limit on Linux: 128K
                    full_command.append(arguments_json)
                    arg_file = None
                    arg_fd = None
                else:
                    arg_file = tempfile.TemporaryFile()
                    arg_file.write(arguments_json.encode('utf-8'))
                    arg_file.flush()
                    arg_file.seek(0)
                    arg_fd = os.dup(arg_file.fileno())
                    full_command.append('--arguments-json-fd={}'.format(arg_fd))
                    full_command.append('foo')  # dummy funcarg
                if kill_children:
                    full_command.append('--kill-children')
                if is_buggy_subprocess32():  # close_fds doesn't work with subprocess32 < 3.5.0
                    close_fds = False
                    pass_fds = []
                else:
                    close_fds = True
                    pass_fds = [dup_result_fd, dup_error_fd]
                    if arg_file:
                        pass_fds.append(arg_fd)
                process = subprocess.Popen(
                    full_command,
                    bufsize=-1,
                    close_fds=close_fds,
                    pass_fds=pass_fds,
                )
                rc = wait_subprocess(process, timeout=timeout, command_info=full_command)
                os.close(dup_result_fd)
                os.close(dup_error_fd)
                if arg_file:
                    arg_file.close()
                if rc:
                    error_fd.seek(0)
                    err_output = error_fd.read()
                    if err_output:
                        if not compat.PY2:
                            err_output = err_output.decode('utf-8', errors='replace')
                    raise ExecutionError(err_output)

                result_fd.seek(0)
                result_str = result_fd.read()

            if not result_str:
                return None
            try:
                if not compat.PY2:
                    result_str = result_str.decode('utf-8', errors='replace')
                result = format.decode(result_str)
                return result
            except BaseException as ex:
                logger.exception('Exception in python.execute: {} {}'.format(ex.__class__.__name__, ex))
                logger.warning('%r', result_str)

        # Not automatically assigned in python < 3.2.
        execute.__wrapped__ = func
        execute.add_context_in_kwargs = True
        return execute

    return wrap_callable

def is_buggy_subprocess32():
    """
    subprocess32 < 3.5.0:
    * doesn't support close_fds
    * has its _subprocess C helper named _subprocess
    """
    if not compat.PY2:
        return False
    return (
            subprocess.__name__ == "subprocess32"
            and hasattr(subprocess, "_posixsubprocess")
            and subprocess._posixsubprocess.__name__ == "_posixsubprocess"
    )

def program(path=None, argument_format=format_arguments):
    r"""
    Decorate a callable to execute it as an external program.

    :param path: of the program to execute. If it is ``None`` the name of the
                 executable will be the name of the callable.
    :type  path: str.
    :param argument_format: takes the arguments of the callable and converts
                            them to command line arguments.
    :type  argument_format: callable(*args, **kwargs).

    :returns:
        :rtype: callable(*args, **kwargs).

    Examples
    --------

    >>> @program()
    ... def ls(path):
    ...     pass
    >>> ls('/etc/resolv.conf')
    '/etc/resolv.conf\n'

    It will execute the ``ls`` command and requires a single positional
    argument *path*.

    """
    import inspect

    def wrap_callable(func):
        @functools.wraps(func)
        def execute(*args, **kwargs):
            check_arguments(argspec, args)
            check_keyword_arguments(argspec, kwargs)

            command = path or func.__name__
            return subprocess.check_output(
                [command] + argument_format(*args, **kwargs),
                universal_newlines=True)

        try:
            args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = inspect.getfullargspec(func)
            argspec = inspect.ArgSpec(args, varargs, varkw, defaults)
        except AttributeError:
            # noinspection PyDeprecation
            argspec = inspect.getargspec(func)

        # Not automatically assigned in python < 3.2.
        execute.__wrapped__ = func
        return execute

    return wrap_callable

def make_callable(funcname):
    """
    Return a callable object from a string.

    This function resolves a name into a callable object. It automatically
    loads the required modules. If there is no module path, it considers the
    callable is a builtin.

    :param funcname: name of the callable.
    :type  funcname: str.

    :returns:
        :rtype: callable.

    Examples
    --------

    Loading a function from a library:

    >>> func = make_callable('itertools.chain')
    >>> list(func(range(3), range(4)))
    [0, 1, 2, 0, 1, 2, 3]

    Loading a builtin:

    >>> func = make_callable('map')
    >>> list(func(lambda x: x + 1, range(4)))
    [1, 2, 3, 4]

    """
    if '.' not in funcname:
        module_name = 'builtins'
        object_name = funcname
    else:
        module_name, object_name = funcname.rsplit('.', 1)

    module = __import__(module_name, fromlist=['*'])
    try:
        callable_ = getattr(module, object_name)
    except AttributeError:
        raise AttributeError('module {} has no attribute {}'.format(
            module.__name__,
            object_name,
        ))
    return callable_

def main():
    """
    When executed as a script, this module expects the name of a callable as
    its first argument and the arguments of the callable encoded in a JSON
    string as its second argument. It then executes the callable with the
    arguments after decoding them into Python objects. It finally encodes the
    value returned by the callable into a JSON string and prints it on stdout.

    the arguments of the callable are stored in a dict with the following
    format: ::

        {'args': [...],
         'kwargs': {
            ...,
         }
         }

    Synopsis
    --------

    ::
        usage: execute.py [-h] funcname funcargs

        positional arguments:
          funcname    name of the callable to execute
          funcargs    callable arguments in JSON

        optional arguments:
          -h, --help  show this help message and exit

    Examples
    --------

    ::
        $ python -m simpleflow.execute "os.path.exists" '{"args": ["/tmp"]}'
        true

    """
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument(
        'funcname',
        help='name of the callable to execute',
    )
    parser.add_argument(
        'funcargs',
        help='callable arguments in JSON',
    )
    parser.add_argument(
        '--context',
        help='Activity Context',
    )
    parser.add_argument(
        '--logger-name',
        help='logger name',
    )
    parser.add_argument(
        '--result-fd',
        type=int,
        default=1,
        metavar='N',
        help='result file descriptor',
    )
    parser.add_argument(
        '--error-fd',
        type=int,
        default=2,
        metavar='N',
        help='error file descriptor',
    )
    parser.add_argument(
        '--arguments-json-fd',
        type=int,
        default=None,
        metavar='N',
        help='JSON input file descriptor',
    )
    parser.add_argument(
        '--kill-children',
        action='store_true',
        help='kill child processes on exit',
    )
    cmd_arguments = parser.parse_args()

    def kill_child_processes():
        process = psutil.Process(os.getpid())
        children = process.children(recursive=True)

        for child in children:
            try:
                child.terminate()
            except psutil.NoSuchProcess:
                pass
        _, still_alive = psutil.wait_procs(children, timeout=0.3)
        for child in still_alive:
            try:
                child.kill()
            except psutil.NoSuchProcess:
                pass

    funcname = cmd_arguments.funcname
    if cmd_arguments.arguments_json_fd is None:
        content = cmd_arguments.funcargs
        if content is None:
            parser.error('the following arguments are required: funcargs')
    else:
        with os.fdopen(cmd_arguments.arguments_json_fd) as arguments_json_file:
            content = arguments_json_file.read()
    try:
        arguments = format.decode(content)
    except Exception:
        raise ValueError('cannot load arguments from {}'.format(
            content))
    if cmd_arguments.logger_name:
        logger = logging.getLogger(cmd_arguments.logger_name)
    else:
        logger = simpleflow_logger
    callable_ = make_callable(funcname)
    if hasattr(callable_, '__wrapped__'):
        callable_ = callable_.__wrapped__
    args = arguments.get('args', ())
    kwargs = arguments.get('kwargs', {})
    context = json.loads(cmd_arguments.context) if cmd_arguments.context is not None else None
    try:
        if hasattr(callable_, 'execute'):
            inst = callable_(*args, **kwargs)
            if context is not None:
                inst.context = context
            result = inst.execute()
            if hasattr(inst, 'post_execute'):
                inst.post_execute()
        else:
            if context is not None:
                callable_.context = context
            result = callable_(*args, **kwargs)
    except Exception as err:
        logger.error('Exception: {}'.format(err))
        exc_type, exc_value, exc_traceback = sys.exc_info()
        tb = traceback.format_tb(exc_traceback)
        details = json_dumps(
            {
                'error': exc_type.__name__,
                'message': str(exc_value),
                'traceback': tb,
            },
            default=repr,
        )
        if cmd_arguments.error_fd == 2:
            sys.stderr.flush()
        if not compat.PY2:
            details = details.encode('utf-8')
        os.write(cmd_arguments.error_fd, details)
        if cmd_arguments.kill_children:
            kill_child_processes()
        sys.exit(1)

    if cmd_arguments.result_fd == 1:  # stdout (legacy)
        sys.stdout.flush()  # may have print's in flight
        os.write(cmd_arguments.result_fd, b'\n')
    result = json_dumps(result)
    if not compat.PY2:
        result = result.encode('utf-8')
    os.write(cmd_arguments.result_fd, result)
    if cmd_arguments.kill_children:
        kill_child_processes()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import oelite.fetch
import oelite.git
import os
import re
import warnings
import string
import sys

import bb.utils

class GitFetcher():

    SUPPORTED_SCHEMES = ("git")
    COMMIT_ID_RE = re.compile("([0-9a-f]{1,40})")

    def __init__(self, uri, d):
        if not uri.scheme in self.SUPPORTED_SCHEMES:
            raise Exception(
                "Scheme %s not supported by oelite.fetch.GitFetcher"%(scheme))
        uri.fdepends.append("native:git")
        self.uri = uri
        try:
            protocol = uri.params["protocol"]
        except KeyError:
            protocol = "git"
        self.url = "%s://%s"%(protocol, uri.location)
        repo_name = protocol + "_" + \
            self.uri.location.rstrip("/").translate(string.maketrans("/", "_"))
        self.repo = os.path.join(uri.ingredients, uri.isubdir, 'git', repo_name)
        self.mirror_name = repo_name
        if self.mirror_name.endswith(".git"):
            self.mirror_name = self.mirror_name[:-4]
        self.commit = None
        self.tag = None
        self.branch = None
        if "commit" in uri.params:
            self.commit = uri.params["commit"]
            if not self.COMMIT_ID_RE.match(self.commit):
                raise oelite.fetch.InvalidURI(
                    self.uri, "invalid commit id %s"%(repr(self.commit)))
        if "tag" in uri.params:
            self.tag = uri.params["tag"]
            self.signature_name = "git://" + uri.location
            if protocol != "git":
                self.signature_name += ";protocol=" + protocol
            self.signature_name += ";tag=" + self.tag
        if "branch" in uri.params:
            self.branch = uri.params["branch"]
        i = bool(self.commit) + bool(self.tag) + bool(self.branch)
        if i == 0:
            self.branch = "HEAD"
        elif i != 1:
            raise oelite.fetch.InvalidURI(
                self.uri, "cannot mix commit, tag and branch parameters")
        if "track" in uri.params:
            self.track = uri.params["track"].split(",")
            warnings.warn("track parameter not implemented yet")
        else:
            self.track = None
        repo_name = uri.location.strip("/").split("/")[-1]
        if repo_name.endswith(".git"):
            repo_name = repo_name[:-4]
        if "subdir" in uri.params:
            self.dest = uri.params["subdir"]
            if self.dest[-1] == "/":
                self.dest += repo_name
        else:
            self.dest = repo_name
        self.signatures = d.get("FILE") + ".sig"
        self.fetch_signatures = d["__fetch_signatures"]
        return

    def signature(self):
        if self.commit:
            return self.commit
        elif self.tag:
            try:
                self._signature = self.fetch_signatures[self.signature_name]
                return self._signature
            except KeyError:
                raise oelite.fetch.NoSignature(self.uri, "signature unknown")
        elif self.branch:
            warnings.warn("fetching git branch head, causing source signature to not be sufficient for proper signature handling (%s)"%(self.uri))
            return ""
        raise Exception("this should not be reached")

    def fetch(self):
        # TODO: add special handling of file:// git repos, which should not
        # be cloned/mirrored to ingredients
        if not os.path.exists(self.repo):
            if not self.fetch_clone():
                return False
        repo = oelite.git.GitRepository(self.repo)
        if not self.has_rev(repo):
            if not self.fetch_update(repo):
                return False
        if self.tag:
            commit = repo.get_tag(self.tag)
            if not commit:
                raise oelite.fetch.FetchError(
                    self.uri, "unknown tag: %s"%(self.tag))
            if not "_signature" in dir(self):
                return (self.signature_name, commit)
            if (commit != self._signature):
                print "Error signature mismatch "+self.tag
                print "  expected: %s"%self._signature
                print "  obtained: %s"%commit
            return commit == self._signature
        return True

    def fetch_clone(self):
        basedir = os.path.dirname(self.repo)
        repodir = os.path.basename(self.repo)
        bb.utils.mkdirhier(basedir)
        options = ['--mirror']
        if self.uri.params.get('recursive', '0') != '0':
            options.append('--recursive')
        fetched = False
        for url in self.uri.premirrors + [self.url] + self.uri.mirrors:
            if not isinstance(url, basestring):
                if url[0].endswith("//"):
                    url = os.path.join(url[0].rstrip("/"), self.mirror_name)
                    url += ".git"
                else:
                    url = os.path.join(url[0], url[1])
            if not self.uri.allow_url(url):
                print "Skipping", url
                continue
            cmd = ['git', 'clone'] + options + [ url, repodir ]
            print "Cloning from", url
            if oelite.util.shcmd(cmd, dir=basedir) is True:
                return True
            print "fetching from %s failed"%(url)
        print "Error: git clone failed"
        return False

    def fetch_update(self, repo):
        fetched = False
        for url in self.uri.premirrors + [self.url] + self.uri.mirrors:
            if not isinstance(url, basestring):
                if url[0].endswith("//"):
                    url = os.path.join(url[0].rstrip("/"), self.mirror_name)
                    url += ".git"
                else:
                    url = os.path.join(url[0], url[1])
            if not self.uri.allow_url(url):
                print "Skipping", url
                continue
            print "Updating from %s"%(url)
            repo.remote_update(url)
            if self.has_rev(repo):
                return True
        print "Error: git update failed"
        return False

    def has_rev(self, repo):
        if self.commit:
            return repo.has_commit(self.commit)
        elif self.tag:
            return repo.has_tag(self.tag)
        elif self.branch:
            return repo.has_head(self.branch)
        return False

    def unpack(self, d):
        wc = os.path.join(d.get("SRCDIR"), self.dest)
        basedir = os.path.dirname(wc)
        bb.utils.mkdirhier(basedir)
        if self.branch:
            branch = self.resolve_head(self.branch)
            cmd = "git clone --shared -b %s %s %s"%(branch, self.repo, wc)
            if not oelite.util.shcmd(cmd):
                print "Error: git clone failed"
                return False
            return True
        cmd = "git clone --shared --no-checkout %s %s"%(self.repo, wc)
        if not oelite.util.shcmd(cmd):
            print "Error: git clone failed"
            return False
        cmd = "git checkout -q "
        if self.commit:
            cmd += self.commit
        elif self.tag:
            cmd += "refs/tags/%s"%(self.tag)
        else:
            print "Error: WTF! no commit, tag or branch!!"
            return False
        if not oelite.util.shcmd(cmd, dir=self.dest):
            print "Error: git checkout failed"
            return False
        return True

    def mirror(self, mirror=os.getcwd()):
        path = os.path.join(self.uri.isubdir, "git", self.mirror_name) + ".git"
        basedir = os.path.dirname(path)
        if not os.path.exists(path):
            print "Creating git mirror", path
            bb.utils.mkdirhier(basedir)
            options = ['--mirror']
            if self.uri.params.get('recursive', '0') != '0':
                options.append('--recursive')
            cmd = ['git', 'clone'] + options + [self.repo, path]
            return oelite.util.shcmd(cmd) is True
        else:
            print "Updating git mirror", path
            cmd = "git remote update"
            return oelite.util.shcmd(cmd, dir=path) is True
<commit_msg>lib/oelite: Improve error message in git fetcher<commit_after>import oelite.fetch
import oelite.git
import os
import re
import warnings
import string
import sys

import bb.utils

class GitFetcher():

    SUPPORTED_SCHEMES = ("git")
    COMMIT_ID_RE = re.compile("([0-9a-f]{1,40})")

    def __init__(self, uri, d):
        if not uri.scheme in self.SUPPORTED_SCHEMES:
            raise Exception(
                "Scheme %s not supported by oelite.fetch.GitFetcher"%(scheme))
        uri.fdepends.append("native:git")
        self.uri = uri
        try:
            protocol = uri.params["protocol"]
        except KeyError:
            protocol = "git"
        self.url = "%s://%s"%(protocol, uri.location)
        repo_name = protocol + "_" + \
            self.uri.location.rstrip("/").translate(string.maketrans("/", "_"))
        self.repo = os.path.join(uri.ingredients, uri.isubdir, 'git', repo_name)
        self.mirror_name = repo_name
        if self.mirror_name.endswith(".git"):
            self.mirror_name = self.mirror_name[:-4]
        self.commit = None
        self.tag = None
        self.branch = None
        if "commit" in uri.params:
            self.commit = uri.params["commit"]
            if not self.COMMIT_ID_RE.match(self.commit):
                raise oelite.fetch.InvalidURI(
                    self.uri, "invalid commit id %s"%(repr(self.commit)))
        if "tag" in uri.params:
            self.tag = uri.params["tag"]
            self.signature_name = "git://" + uri.location
            if protocol != "git":
                self.signature_name += ";protocol=" + protocol
            self.signature_name += ";tag=" + self.tag
        if "branch" in uri.params:
            self.branch = uri.params["branch"]
        i = bool(self.commit) + bool(self.tag) + bool(self.branch)
        if i == 0:
            self.branch = "HEAD"
        elif i != 1:
            raise oelite.fetch.InvalidURI(
                self.uri, "cannot mix commit, tag and branch parameters")
        if "track" in uri.params:
            self.track = uri.params["track"].split(",")
            warnings.warn("track parameter not implemented yet")
        else:
            self.track = None
        repo_name = uri.location.strip("/").split("/")[-1]
        if repo_name.endswith(".git"):
            repo_name = repo_name[:-4]
        if "subdir" in uri.params:
            self.dest = uri.params["subdir"]
            if self.dest[-1] == "/":
                self.dest += repo_name
        else:
            self.dest = repo_name
        self.signatures = d.get("FILE") + ".sig"
        self.fetch_signatures = d["__fetch_signatures"]
        return

    def signature(self):
        if self.commit:
            return self.commit
        elif self.tag:
            try:
                self._signature = self.fetch_signatures[self.signature_name]
                return self._signature
            except KeyError:
                raise oelite.fetch.NoSignature(self.uri, "signature unknown")
        elif self.branch:
            warnings.warn("fetching git branch head, causing source signature to not be sufficient for proper signature handling (%s)"%(self.uri))
            return ""
        raise Exception("this should not be reached")

    def fetch(self):
        # TODO: add special handling of file:// git repos, which should not
        # be cloned/mirrored to ingredients
        if not os.path.exists(self.repo):
            if not self.fetch_clone():
                return False
        repo = oelite.git.GitRepository(self.repo)
        if not self.has_rev(repo):
            if not self.fetch_update(repo):
                return False
        if self.tag:
            commit = repo.get_tag(self.tag)
            if not commit:
                raise oelite.fetch.FetchError(
                    self.uri, "unknown tag: %s"%(self.tag))
            if not "_signature" in dir(self):
                return (self.signature_name, commit)
            if (commit != self._signature):
                print "Error signature mismatch "+self.tag
                print "  expected: %s"%self._signature
                print "  obtained: %s"%commit
            return commit == self._signature
        return True

    def fetch_clone(self):
        basedir = os.path.dirname(self.repo)
        repodir = os.path.basename(self.repo)
        bb.utils.mkdirhier(basedir)
        options = ['--mirror']
        if self.uri.params.get('recursive', '0') != '0':
            options.append('--recursive')
        fetched = False
        for url in self.uri.premirrors + [self.url] + self.uri.mirrors:
            if not isinstance(url, basestring):
                if url[0].endswith("//"):
                    url = os.path.join(url[0].rstrip("/"), self.mirror_name)
                    url += ".git"
                else:
                    url = os.path.join(url[0], url[1])
            if not self.uri.allow_url(url):
                print "Skipping", url
                continue
            cmd = ['git', 'clone'] + options + [ url, repodir ]
            print "Cloning from", url
            if oelite.util.shcmd(cmd, dir=basedir) is True:
                return True
            print "fetching from %s failed"%(url)
        print "Error: git clone failed"
        return False

    def fetch_update(self, repo):
        fetched = False
        for url in self.uri.premirrors + [self.url] + self.uri.mirrors:
            if not isinstance(url, basestring):
                if url[0].endswith("//"):
                    url = os.path.join(url[0].rstrip("/"), self.mirror_name)
                    url += ".git"
                else:
                    url = os.path.join(url[0], url[1])
            if not self.uri.allow_url(url):
                print "Skipping", url
                continue
            print "Updating from %s"%(url)
            repo.remote_update(url)
            if self.has_rev(repo):
                return True
        print "Error: git update failed (required rev could not be fetched)"
        return False

    def has_rev(self, repo):
        if self.commit:
            return repo.has_commit(self.commit)
        elif self.tag:
            return repo.has_tag(self.tag)
        elif self.branch:
            return repo.has_head(self.branch)
        return False

    def unpack(self, d):
        wc = os.path.join(d.get("SRCDIR"), self.dest)
        basedir = os.path.dirname(wc)
        bb.utils.mkdirhier(basedir)
        if self.branch:
            branch = self.resolve_head(self.branch)
            cmd = "git clone --shared -b %s %s %s"%(branch, self.repo, wc)
            if not oelite.util.shcmd(cmd):
                print "Error: git clone failed"
                return False
            return True
        cmd = "git clone --shared --no-checkout %s %s"%(self.repo, wc)
        if not oelite.util.shcmd(cmd):
            print "Error: git clone failed"
            return False
        cmd = "git checkout -q "
        if self.commit:
            cmd += self.commit
        elif self.tag:
            cmd += "refs/tags/%s"%(self.tag)
        else:
            print "Error: WTF! no commit, tag or branch!!"
            return False
        if not oelite.util.shcmd(cmd, dir=self.dest):
            print "Error: git checkout failed"
            return False
        return True

    def mirror(self, mirror=os.getcwd()):
        path = os.path.join(self.uri.isubdir, "git", self.mirror_name) + ".git"
        basedir = os.path.dirname(path)
        if not os.path.exists(path):
            print "Creating git mirror", path
            bb.utils.mkdirhier(basedir)
            options = ['--mirror']
            if self.uri.params.get('recursive', '0') != '0':
                options.append('--recursive')
            cmd = ['git', 'clone'] + options + [self.repo, path]
            return oelite.util.shcmd(cmd) is True
        else:
            print "Updating git mirror", path
            cmd = "git remote update"
            return oelite.util.shcmd(cmd, dir=path) is True
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Copyright (c) 2006-2014 sqlmap developers (http://sqlmap.org/)
See the file 'doc/COPYING' for copying permission
"""

import codecs

from ConfigParser import MissingSectionHeaderError

from lib.core.common import checkFile
from lib.core.common import unArrayizeValue
from lib.core.common import UnicodeRawConfigParser
from lib.core.data import conf
from lib.core.data import logger
from lib.core.exception import SqlmapMissingMandatoryOptionException
from lib.core.exception import SqlmapSyntaxException
from lib.core.optiondict import optDict
from lib.core.settings import UNICODE_ENCODING

config = None

def configFileProxy(section, option, boolean=False, integer=False):
    """
    Parse configuration file and save settings into the configuration
    advanced dictionary.
    """

    global config

    if config.has_option(section, option):
        if boolean:
            value = config.getboolean(section, option) if config.get(section, option) else False
        elif integer:
            value = config.getint(section, option) if config.get(section, option) else 0
        else:
            value = config.get(section, option)

        if value:
            conf[option] = value
        else:
            conf[option] = None
    else:
        debugMsg = "missing requested option '%s' (section " % option
        debugMsg += "'%s') into the configuration file, " % section
        debugMsg += "ignoring. Skipping to next."
        logger.debug(debugMsg)

def configFileParser(configFile):
    """
    Parse configuration file and save settings into the configuration
    advanced dictionary.
    """

    global config

    debugMsg = "parsing configuration file"
    logger.debug(debugMsg)

    checkFile(configFile)
    configFP = codecs.open(configFile, "rb", UNICODE_ENCODING)

    try:
        config = UnicodeRawConfigParser()
        config.readfp(configFP)
    except MissingSectionHeaderError:
        errMsg = "you have provided an invalid configuration file"
        raise SqlmapSyntaxException(errMsg)

    if not config.has_section("Target"):
        errMsg = "missing a mandatory section 'Target' in the configuration file"
        raise SqlmapMissingMandatoryOptionException(errMsg)

    condition = not config.has_option("Target", "direct")
    condition &= not config.has_option("Target", "url")
    condition &= not config.has_option("Target", "logFile")
    condition &= not config.has_option("Target", "bulkFile")
    condition &= not config.has_option("Target", "googleDork")
    condition &= not config.has_option("Target", "requestFile")
    condition &= not config.has_option("Target", "wizard")

    if condition:
        errMsg = "missing a mandatory option in the configuration file "
        errMsg += "(direct, url, logFile, bulkFile, googleDork, requestFile or wizard)"
        raise SqlmapMissingMandatoryOptionException(errMsg)

    for family, optionData in optDict.items():
        for option, datatype in optionData.items():
            datatype = unArrayizeValue(datatype)

            boolean = datatype == "boolean"
            integer = datatype == "integer"

            configFileProxy(family, option, boolean, integer)
<commit_msg>Graceful abort in case of an invalid configuration file<commit_after>#!/usr/bin/env python

"""
Copyright (c) 2006-2014 sqlmap developers (http://sqlmap.org/)
See the file 'doc/COPYING' for copying permission
"""

import codecs

from ConfigParser import MissingSectionHeaderError
from ConfigParser import ParsingError

from lib.core.common import checkFile
from lib.core.common import unArrayizeValue
from lib.core.common import UnicodeRawConfigParser
from lib.core.data import conf
from lib.core.data import logger
from lib.core.exception import SqlmapMissingMandatoryOptionException
from lib.core.exception import SqlmapSyntaxException
from lib.core.optiondict import optDict
from lib.core.settings import UNICODE_ENCODING

config = None

def configFileProxy(section, option, boolean=False, integer=False):
    """
    Parse configuration file and save settings into the configuration
    advanced dictionary.
    """

    global config

    if config.has_option(section, option):
        if boolean:
            value = config.getboolean(section, option) if config.get(section, option) else False
        elif integer:
            value = config.getint(section, option) if config.get(section, option) else 0
        else:
            value = config.get(section, option)

        if value:
            conf[option] = value
        else:
            conf[option] = None
    else:
        debugMsg = "missing requested option '%s' (section " % option
        debugMsg += "'%s') into the configuration file, " % section
        debugMsg += "ignoring. Skipping to next."
        logger.debug(debugMsg)

def configFileParser(configFile):
    """
    Parse configuration file and save settings into the configuration
    advanced dictionary.
    """

    global config

    debugMsg = "parsing configuration file"
    logger.debug(debugMsg)

    checkFile(configFile)
    configFP = codecs.open(configFile, "rb", UNICODE_ENCODING)

    try:
        config = UnicodeRawConfigParser()
        config.readfp(configFP)
    except (MissingSectionHeaderError, ParsingError), ex:
        errMsg = "you have provided an invalid configuration file ('%s')" % str(ex)
        raise SqlmapSyntaxException(errMsg)

    if not config.has_section("Target"):
        errMsg = "missing a mandatory section 'Target' in the configuration file"
        raise SqlmapMissingMandatoryOptionException(errMsg)

    condition = not config.has_option("Target", "direct")
    condition &= not config.has_option("Target", "url")
    condition &= not config.has_option("Target", "logFile")
    condition &= not config.has_option("Target", "bulkFile")
    condition &= not config.has_option("Target", "googleDork")
    condition &= not config.has_option("Target", "requestFile")
    condition &= not config.has_option("Target", "wizard")

    if condition:
        errMsg = "missing a mandatory option in the configuration file "
        errMsg += "(direct, url, logFile, bulkFile, googleDork, requestFile or wizard)"
        raise SqlmapMissingMandatoryOptionException(errMsg)

    for family, optionData in optDict.items():
        for option, datatype in optionData.items():
            datatype = unArrayizeValue(datatype)

            boolean = datatype == "boolean"
            integer = datatype == "integer"

            configFileProxy(family, option, boolean, integer)
<|endoftext|>


<commit_before>class PipelineSchema(object):
	def __init__(self, name, tag, config):  # config must be an instance of pipelines.utils.PipelinesConfig
		self.name = name
		self.tag = tag
		self._schema = {
			"name": name,
			"tag": tag,
			"children": [],
			"request": {
				"pipelineArgs": {
					"projectId": config.project_id,
					"resources": {
						"disks": [],
						"zones": config.zones.split(',')
					},
					"logging": {},
					"inputs": {},
					"outputs": {},
					"serviceAccount": {
						"email": config.service_account_email,
						"scopes": config.scopes.split(',')
					}
				},
				"ephemeralPipeline": {
					"name": name,
					"projectId": config.project_id,
					"inputParameters": [],
					"outputParameters": [],
					"resources": {
						"disks": [],
						"zones": config.zones.split(',')
					},
					"docker": {}
				}
			}
		}
		self._metadata = {}

	def getSchema(self):
		return self._schema

	def addChild(self, child):
		self._schema["children"].append(child.name)

	def addInput(self, name, disk, localPath, gcsPath):
		self._schema["request"]["pipelineArgs"]["inputs"][name] = gcsPath

		self._schema["request"]["ephemeralPipeline"]["inputParameters"].append({
			"name": name,
			"localCopy": {
				"path": localPath,
				"disk": disk
			}
		})

	def addOutput(self, name, disk, localPath, gcsPath):
		self._schema["request"]["pipelineArgs"]["outputs"][name] = gcsPath

		self._schema["request"]["ephemeralPipeline"]["outputParameters"].append({
			"name": name,
			"localCopy": {
				"path": localPath,
				"disk": disk
			}
		})

	def addDisk(self, name, diskType, sizeGb, mountPath, autoDelete=True, readOnly=False):
		self._schema["request"]["pipelineArgs"]["resources"]["disks"].append({
			"name": name,
			"type": diskType,
			"sizeGb": sizeGb,
			"autoDelete": autoDelete,
			"readOnly": readOnly
		})
		self._schema["request"]["ephemeralPipeline"]["resources"]["disks"].append({
			"name": name,
			"type": diskType,
			"sizeGb": sizeGb,
			"autoDelete": autoDelete,
			"readOnly": readOnly,
			"mountPoint": mountPath
		})

	def setLogOutput(self, gcsPath):
		self._schema["request"]["pipelineArgs"]["logging"]["gcsPath"] = gcsPath

	def setMem(self, memGb):
		self._schema["request"]["pipelineArgs"]["resources"]["minimumRamGb"] = memGb
		self._schema["request"]["ephemeralPipeline"]["resources"]["minimumRamGb"] = memGb

	def setCpu(self, cores):
		self._schema["request"]["pipelineArgs"]["resources"]["minimumCpuCores"] = cores
		self._schema["request"]["ephemeralPipeline"]["resources"]["minimumCpuCores"] = cores

	def setBootDiskSize(self, sizeGb):
		self._schema["request"]["pipelineArgs"]["resources"]["bootDiskSizeGb"] = sizeGb
		self._schema["request"]["ephemeralPipeline"]["resources"]["bootDiskSizeGb"]

	def setPreemptible(self, preemptible):
		self._schema["request"]["pipelineArgs"]["resources"]["preemptible"] = preemptible
		self._schema["request"]["ephemeralPipeline"]["resources"]["preemptible"] = preemptible

	def setImage(self, imageName):
		self._schema["request"]["ephemeralPipeline"]["docker"]["imageName"] = imageName

	def setCmd(self, cmdString):
		self._schema["request"]["ephemeralPipeline"]["docker"]["cmd"] = cmdString

	def addSchemaMetadata(self, **kwargs):
		self._metadata.update(**kwargs)

	def getSchemaMetadata(self, key):
		if key in self._metadata.keys():
			return self._metadata[key]
		else:
			raise LookupError("Metadata key '{k}' not found".format(k=key))

<commit_msg>Fixed a bug<commit_after>class PipelineSchema(object):
	def __init__(self, name, tag, config):  # config must be an instance of pipelines.utils.PipelinesConfig
		self.name = name
		self.tag = tag
		self._schema = {
			"name": name,
			"tag": tag,
			"children": [],
			"request": {
				"pipelineArgs": {
					"projectId": config.project_id,
					"resources": {
						"disks": [],
						"zones": config.zones.split(',')
					},
					"logging": {},
					"inputs": {},
					"outputs": {},
					"serviceAccount": {
						"email": config.service_account_email,
						"scopes": config.scopes.split(',')
					}
				},
				"ephemeralPipeline": {
					"name": name,
					"projectId": config.project_id,
					"inputParameters": [],
					"outputParameters": [],
					"resources": {
						"disks": [],
						"zones": config.zones.split(',')
					},
					"docker": {}
				}
			}
		}
		self._metadata = {}

	def getSchema(self):
		return self._schema

	def addChild(self, child):
		self._schema["children"].append(child.name)

	def addInput(self, name, disk, localPath, gcsPath):
		self._schema["request"]["pipelineArgs"]["inputs"][name] = gcsPath

		self._schema["request"]["ephemeralPipeline"]["inputParameters"].append({
			"name": name,
			"localCopy": {
				"path": localPath,
				"disk": disk
			}
		})

	def addOutput(self, name, disk, localPath, gcsPath):
		self._schema["request"]["pipelineArgs"]["outputs"][name] = gcsPath

		self._schema["request"]["ephemeralPipeline"]["outputParameters"].append({
			"name": name,
			"localCopy": {
				"path": localPath,
				"disk": disk
			}
		})

	def addDisk(self, name, diskType, sizeGb, mountPath, autoDelete=True, readOnly=False):
		self._schema["request"]["pipelineArgs"]["resources"]["disks"].append({
			"name": name,
			"type": diskType,
			"sizeGb": sizeGb,
			"autoDelete": autoDelete,
			"readOnly": readOnly
		})
		self._schema["request"]["ephemeralPipeline"]["resources"]["disks"].append({
			"name": name,
			"type": diskType,
			"sizeGb": sizeGb,
			"autoDelete": autoDelete,
			"readOnly": readOnly,
			"mountPoint": mountPath
		})

	def setLogOutput(self, gcsPath):
		self._schema["request"]["pipelineArgs"]["logging"]["gcsPath"] = gcsPath

	def setMem(self, memGb):
		self._schema["request"]["pipelineArgs"]["resources"]["minimumRamGb"] = memGb
		self._schema["request"]["ephemeralPipeline"]["resources"]["minimumRamGb"] = memGb

	def setCpu(self, cores):
		self._schema["request"]["pipelineArgs"]["resources"]["minimumCpuCores"] = cores
		self._schema["request"]["ephemeralPipeline"]["resources"]["minimumCpuCores"] = cores

	def setBootDiskSize(self, sizeGb):
		self._schema["request"]["pipelineArgs"]["resources"]["bootDiskSizeGb"] = sizeGb
		self._schema["request"]["ephemeralPipeline"]["resources"]["bootDiskSizeGb"] = sizeGb

	def setPreemptible(self, preemptible):
		self._schema["request"]["pipelineArgs"]["resources"]["preemptible"] = preemptible
		self._schema["request"]["ephemeralPipeline"]["resources"]["preemptible"] = preemptible

	def setImage(self, imageName):
		self._schema["request"]["ephemeralPipeline"]["docker"]["imageName"] = imageName

	def setCmd(self, cmdString):
		self._schema["request"]["ephemeralPipeline"]["docker"]["cmd"] = cmdString

	def addSchemaMetadata(self, **kwargs):
		self._metadata.update(**kwargs)

	def getSchemaMetadata(self, key):
		if key in self._metadata.keys():
			return self._metadata[key]
		else:
			raise LookupError("Metadata key '{k}' not found".format(k=key))

<|endoftext|>


<commit_before>#!/usr/bin/env python
# vim: ai ts=4 sts=4 et sw=4

import copy

from rapidsms.connection import Connection
from rapidsms.person import Person
from datetime import datetime

class Message(object):
    def __init__(self, connection=None, text=None, person=None, date=datetime.now()):
        if connection == None and person == None:
            raise Exception("Message __init__() must take one of: connection, person")
        self._connection = connection
        self.text = text
        self.date = date
        self.person = person
        self.responses = []

        # a message is considered "unprocessed" until
        # rapidsms has dispatched it to all apps, and
        # flushed the responses out
        self.processed = False

    def __unicode__(self):
        return self.text

    @property
    def connection(self):
        # connection is read-only, since it's an
        # immutable property of this object
        if self._connection is not None:
            return self._connection
        else:
            return self.person.connection

    @property
    def peer (self):
        # return the identity (e.g. phone number) of
        # the other end of this message's connection
        return self.connection.identity

    def send(self):
        """Send this message via self.connection.backend, returning
           True if the message was sent successfully."""
        return self.connection.backend.router.outgoing(self)

    def flush_responses (self):
        """Sends all responses added to this message (via the
           Message.respond method) in the order which they were
           added, and clears self.responses"""

        # keep on iterating until all of
        # the messages have been sent
        while self.responses:
            self.responses.pop(0).send()

    def error(self, text, level):
        """Apps send error messages here rather than through respond
           so users only receive one - the with the highest level of specificity"""
        #TODO implement this
        pass

    def respond(self, text):
        """Send the given text back to the original caller of this
           message on the same route that it came in on"""
        if self.connection:
            response = copy.copy(self)
            response.text = text
            self.responses.append(response)
            return True
        else:
            return False

    def forward (self, identity, text=None):
        if self.connection:
            target = self.connection.fork(identity)
            if text is None: text = self.text
            message = type(self)(connection=target, text=text)
            self.responses.append(message)
            return True
        else:
            return False

<commit_msg>Adding a status code to the message object and response API.<commit_after>#!/usr/bin/env python
# vim: ai ts=4 sts=4 et sw=4

import copy

from rapidsms.connection import Connection
from rapidsms.person import Person
from datetime import datetime

class StatusCodes:
    '''Enum for representing status types of a message or response.'''
    NONE = "None" # we don't know.  the default
    OK  = "Ok" # is great success!
    APP_ERROR = "Application Error" # application specific errors - e.g. bad data
    GENERIC_ERROR = "Generic error" # generic errors - e.g. a catch all responder

class Message(object):
    def __init__(self, connection=None, text=None, person=None, date=datetime.now()):
        if connection == None and person == None:
            raise Exception("Message __init__() must take one of: connection, person")
        self._connection = connection
        self.text = text
        self.date = date
        self.person = person
        self.responses = []
        self.status = StatusCodes.NONE

        # a message is considered "unprocessed" until
        # rapidsms has dispatched it to all apps, and
        # flushed the responses out
        self.processed = False

    def __unicode__(self):
        return self.text

    @property
    def connection(self):
        # connection is read-only, since it's an
        # immutable property of this object
        if self._connection is not None:
            return self._connection
        else:
            return self.person.connection

    @property
    def peer (self):
        # return the identity (e.g. phone number) of
        # the other end of this message's connection
        return self.connection.identity

    def send(self):
        """Send this message via self.connection.backend, returning
           True if the message was sent successfully."""
        return self.connection.backend.router.outgoing(self)

    def flush_responses (self):
        """Sends all responses added to this message (via the
           Message.respond method) in the order which they were
           added, and clears self.responses"""

        # keep on iterating until all of
        # the messages have been sent
        while self.responses:
            self.responses.pop(0).send()

    def error(self, text, level):
        """Apps send error messages here rather than through respond
           so users only receive one - the with the highest level of specificity"""
        #TODO implement this
        pass

    def respond(self, text, status = StatusCodes.NONE):
        """Send the given text back to the original caller of this
           message on the same route that it came in on"""
        if self.connection:
            response = copy.copy(self)
            response.text = text
            response.status = status
            self.responses.append(response)
            return True
        else:
            return False

    def forward (self, identity, text=None):
        if self.connection:
            target = self.connection.fork(identity)
            if text is None: text = self.text
            message = type(self)(connection=target, text=text)
            self.responses.append(message)
            return True
        else:
            return False

<|endoftext|>


<commit_before>if __name__ == '__main__':
    import sys
    sys.path.append('./src/')

# TODO Implement test of apply_single_freq with invalid params.

import discretegreenop
import greenop
import itertools
import numpy as np
import numpy.random as rnd

from numpy.testing import assert_array_max_ulp
from matprop import IsotropicLinearElasticMaterial as Material
from nose.tools import nottest
from nose.tools import raises

# All tests are performed with discrete Green operators based on these
# grids
GRID_SIZES = [(8, 8), (8, 16), (8, 8, 8), (8, 16, 32)]
#GRID_SIZES = [(8, 8), (8, 16)]

def discrete_green_operator(n, h):
    """Create a discrete Green operator with default material constants
    g = 0.75 and nu = 0.3. The gridsize is specified by the tuple n, the
    length of which gives the dimension of the physical space (2 or 3). h is
    the voxel size (can be fixed to 1., as it plays no role in linear
    elasticity.
    """

    mat = Material(0.75, 0.3, len(n))
    greenc = greenop.create(mat)
    if mat.dim == 2:
        return discretegreenop.TruncatedGreenOperator2D(greenc, n, h)
    else:
        return discretegreenop.TruncatedGreenOperator(greenc, n, h)

def get_base(a):
    # TODO This is a hack to avoid writing uggly things like a.base.base.base.
    if a.base is None:
        return a
    else:
        return get_base(a.base)

@nottest
def do_test_as_array(n, inplace):
    mat = Material(0.75, 0.3, len(n))

    n_arr = np.asarray(n)

    # Continuous Green operator
    greenc = greenop.create(mat)

    # Discrete Green operator
    greend = discretegreenop.TruncatedGreenOperator(greenc, n, 1.0)

    # Wave-vector
    k = np.empty((len(n),), dtype=np.float64)

    iterables = [range(ni) for ni in n]
    for b in itertools.product(*iterables):
        b_arr = np.asarray(b)
        i = 2 * b_arr > n_arr
        k[i] = 2. * np.pi * (b_arr[i] - n_arr[i]) / n_arr[i]
        k[~i] = 2. * np.pi * b_arr[~i] / n_arr[~i]

        expected = greenc.as_array(k)

        if inplace:
            base = np.empty_like(expected)
            actual = greend.as_array(b_arr, base)
            assert get_base(actual) is base
        else:
            actual = greend.as_array(b_arr)
        assert_array_max_ulp(expected, actual, 1)

def test_as_array():
    for n in GRID_SIZES:
        for inplace in [True, False]:
            yield do_test_as_array, n, inplace

@nottest
@raises(ValueError)
def do_test_as_array_invalid_b(n):
    b = np.zeros((len(n) + 1,), dtype=np.intp)
    greend = discrete_green_operator(n, 1.)
    greend.as_array(b)

def test_as_array_invalid_b():
    for dim in [2, 3]:
        n = tuple(itertools.repeat(16, dim))
        yield do_test_as_array_invalid_b, n

@nottest
@raises(ValueError)
def do_test_as_array_invalid_output(dim, shape):
    n = tuple(itertools.repeat(16, dim))
    b = np.zeros((dim,), dtype=np.intp)
    out = np.zeros(shape, dtype=np.float64)
    discrete_green_operator(n, 1.).as_array(b, out)

def test_as_array_invalid_output():
    for dim in [2, 3]:
        sym = (dim * (dim + 1)) // 2
        shapes = [(sym, sym, sym), (sym + 1, sym), (sym, sym + 1)]
        for shape in shapes:
            yield do_test_as_array_invalid_output, dim, shape

@nottest
def do_test_apply_single_freq(n, tau, inplace):
    dim = len(n)
    sym = (dim * (dim + 1)) // 2
    mat = Material(0.75, 0.3, dim)

    n_arr = np.asarray(n)
    tau_vec = tau.reshape(sym, 1)

    greenc = greenop.create(mat)
    greend = discretegreenop.TruncatedGreenOperator(greenc, n, 1.0)

    iterables = [range(ni) for ni in n]
    for b in itertools.product(*iterables):
        b_arr = np.asarray(b)

        g = np.asmatrix(greend.as_array(b_arr))
        # expected is by default a matrix, so that it has two dimensions.
        # First convert to ndarray so as to reshape is to a 1D array.
        expected = np.asarray(g * tau_vec).reshape((sym,))

        if inplace:
            base = np.empty_like(expected)
            actual = greend.apply_single_freq(b_arr, tau, base)
            assert get_base(actual) is base
        else:
            actual = greend.apply_single_freq(b_arr, tau)

        actual = np.asarray(actual)
        assert_array_max_ulp(expected, actual, 1)

def test_apply_single_freq():
    shapes = GRID_SIZES

    tau = [np.array([0.3, -0.4, 0.5]),
           np.array([0.1, -0.2, 0.3, -0.4, 0.5, -0.6])]

    for n in shapes:
        for inplace in [False, True]:
            yield do_test_apply_single_freq, n, tau[len(n) - 2], inplace

@nottest
@raises(ValueError)
def do_test_apply_single_freq_invalid_params(n, b, tau, eta):
    discrete_green_operator(n, 1.).apply_single_freq(b, tau, eta)
    b = np.zeros((greend.dim + 1,), dtype=np.intp)
    tau = np.zeros((greend.sym,), dtype=np.float64)
    greend.apply_single_freq(b, tau)

def test_apply_single_freq_invalid_params():
    for dim in [2, 3]:
        sym = (dim * (dim + 1)) // 2
        n = tuple(itertools.repeat(16, dim))
        params = [(np.zeros((dim + 1,), dtype=np.intp),
                   np.zeros((sym, sym), dtype=np.float64),
                   None),
                   (np.zeros((dim,), dtype=np.intp),
                   np.zeros((sym + 1, sym), dtype=np.float64),
                   None),
                   (np.zeros((dim,), dtype=np.intp),
                   np.zeros((sym, sym + 1), dtype=np.float64),
                   None),
                   (np.zeros((dim,), dtype=np.intp),
                   np.zeros((sym, sym), dtype=np.float64),
                   np.zeros((sym + 1, sym), dtype=np.float64)),
                   (np.zeros((dim,), dtype=np.intp),
                    np.zeros((sym, sym), dtype=np.float64),
                    np.zeros((sym, sym + 1), dtype=np.float64))]
        for b, tau, eta in params:
            yield do_test_apply_single_freq_invalid_params, n, b, tau, eta

"""
@nottest
def do_test_apply_all_freqs(n, inplace):
    greend = discrete_green_operator(n, 1.)
    dim = len(n)
    sym = (dim * (dim + 1)) // 2
    tau = rnd.rand(n[1], n[0], sym)
    expected = np.empty_like(tau)
    b = np.empty((dim,), dtype=np.intp)
    for b[0] in range(n[0]):
        for b[1] in range(n[1]):
            greend.apply_single_freq(b,
                                     tau[b[1], b[0], :],
                                     expected[b[1], b[0], :])
    if inplace:
        base = np.empty_like(tau)
        actual = greend.apply_all_freqs(tau, base)
        assert get_base(actual) is base
    else:
        actual = greend.apply_all_freqs(tau)

    assert_array_max_ulp(expected, actual, 0)

def test_apply_all_freqs():
    shapes = ((8, 8), (8, 16))
    for n in GRID_SIZES:
        for inplace in [True, False]:
            yield do_test_apply_all_freqs, n, inplace

@raises(ValueError)
def test_apply_all_freqs_tau_wrong_num_dims():
    green = discrete_green_operator((8, 16), 1.)
    green.apply_all_freqs(np.empty((8, 8), dtype=np.float64))

@nottest
@raises(ValueError)
def do_test_apply_all_freqs_tau_invalid_shape(n, shape):
    green = discrete_green_operator(n, 1.)
    dim = len(n)
    sym = (dim * (dim + 1)) // 2
    tau = np.empty(shape + (sym,), dtype=np.float64)
    green.apply_all_freqs(tau)

def incr_elem(n, i):
    n_list = list(n)
    n_list[i] += 1
    return tuple(n_list)

def test_apply_all_freqs_tau_invalid_shape():
    for n in GRID_SIZES:
        n_mirror = list(n)[::-1]
        for shape in [incr_elem(n_mirror, i) for i in range(len(n))]:
            yield do_test_apply_all_freqs_tau_invalid_shape, n, shape

if __name__ == '__main__':
    do_test_apply_all_freqs_tau_invalid_shape((8, 16), (17, 8))
"""
<commit_msg>Refactored unit tests for TruncatedGreenOperator.as_array().<commit_after>if __name__ == '__main__':
    import sys
    sys.path.append('./src/')

# TODO Implement test of apply_single_freq with invalid params.

import discretegreenop
import greenop
import itertools
import numpy as np
import numpy.random as rnd

from numpy.testing import assert_array_max_ulp
from matprop import IsotropicLinearElasticMaterial as Material
from nose.tools import nottest
from nose.tools import raises

# All tests are performed with discrete Green operators based on these
# grids
GRID_SIZES = [(8, 8), (8, 16), (8, 8, 8), (8, 16, 32)]
#GRID_SIZES = [(8, 8), (8, 16)]

def discrete_green_operator(n, h):
    """Create a discrete Green operator with default material constants
    g = 0.75 and nu = 0.3. The gridsize is specified by the tuple n, the
    length of which gives the dimension of the physical space (2 or 3). h is
    the voxel size (can be fixed to 1., as it plays no role in linear
    elasticity.
    """

    mat = Material(0.75, 0.3, len(n))
    greenc = greenop.create(mat)
    if mat.dim == 2:
        return discretegreenop.TruncatedGreenOperator2D(greenc, n, h)
    else:
        return discretegreenop.TruncatedGreenOperator(greenc, n, h)

def get_base(a):
    # TODO This is a hack to avoid writing uggly things like a.base.base.base.
    if a.base is None:
        return a
    else:
        return get_base(a.base)

def invalid_frequency_multi_indices(n):
    """Return a list of frequency multi-indices which are incompatible with the
    specified grid-size (too many indices, negative and too large indices).

    """
    def f1(a, i):
        a[i] += 1
        return a

    def f2(a, i):
        a[i] = -1
        return a

    ret = [f(np.copy(n).astype(np.intp), i)
            for i in range(len(n)) for f in [f1, f2]]
    ret.append(np.zeros((len(n) + 1,), dtype=np.intp))
    return ret

#
# 1 Test of the method as_array
#   ===========================
#
# 1.1 Valid parameters
#     ----------------
#

@nottest
def do_test_as_array(n, inplace):
    mat = Material(0.75, 0.3, len(n))

    n_arr = np.asarray(n)

    # Continuous Green operator
    greenc = greenop.create(mat)

    # Discrete Green operator
    greend = discretegreenop.TruncatedGreenOperator(greenc, n, 1.0)

    # Wave-vector
    k = np.empty((len(n),), dtype=np.float64)

    iterables = [range(ni) for ni in n]
    for b in itertools.product(*iterables):
        b_arr = np.asarray(b)
        i = 2 * b_arr > n_arr
        k[i] = 2. * np.pi * (b_arr[i] - n_arr[i]) / n_arr[i]
        k[~i] = 2. * np.pi * b_arr[~i] / n_arr[~i]

        expected = greenc.as_array(k)

        if inplace:
            base = np.empty_like(expected)
            actual = greend.as_array(b_arr, base)
            assert get_base(actual) is base
        else:
            actual = greend.as_array(b_arr)
        assert_array_max_ulp(expected, actual, 1)

def test_as_array():
    for n in GRID_SIZES:
        for inplace in [True, False]:
            yield do_test_as_array, n, inplace

#
# 1.2 Invalid parameters
#     ------------------
#

def test_as_array_invalid_parameters():
    for dim in [2, 3]:
        n = tuple(itertools.repeat(16, dim))
        green = discrete_green_operator(n, 1.)
        b0 = np.zeros((green.dim,), dtype=np.intp)
        out1 = np.zeros((green.nrows + 1, green.ncols), dtype=np.float64)
        out2 = np.zeros((green.nrows, green.ncols + 1), dtype=np.float64)
        params = ([(b, None) for b in invalid_frequency_multi_indices(n)]
                  + [(b0, out1), (b0, out2)])

        @raises(ValueError)
        def test(b, out):
            green.as_array(b, out)

        for b, out in params:
            yield test, b, out

"""
@nottest
def do_test_apply_single_freq(n, tau, inplace):
    dim = len(n)
    sym = (dim * (dim + 1)) // 2
    mat = Material(0.75, 0.3, dim)

    n_arr = np.asarray(n)
    tau_vec = tau.reshape(sym, 1)

    greenc = greenop.create(mat)
    greend = discretegreenop.TruncatedGreenOperator(greenc, n, 1.0)

    iterables = [range(ni) for ni in n]
    for b in itertools.product(*iterables):
        b_arr = np.asarray(b)

        g = np.asmatrix(greend.as_array(b_arr))
        # expected is by default a matrix, so that it has two dimensions.
        # First convert to ndarray so as to reshape is to a 1D array.
        expected = np.asarray(g * tau_vec).reshape((sym,))

        if inplace:
            base = np.empty_like(expected)
            actual = greend.apply_single_freq(b_arr, tau, base)
            assert get_base(actual) is base
        else:
            actual = greend.apply_single_freq(b_arr, tau)

        actual = np.asarray(actual)
        assert_array_max_ulp(expected, actual, 1)

def test_apply_single_freq():
    shapes = GRID_SIZES

    tau = [np.array([0.3, -0.4, 0.5]),
           np.array([0.1, -0.2, 0.3, -0.4, 0.5, -0.6])]

    for n in shapes:
        for inplace in [False, True]:
            yield do_test_apply_single_freq, n, tau[len(n) - 2], inplace

@nottest
@raises(ValueError)
def do_test_apply_single_freq_invalid_params(n, b, tau, eta):
    discrete_green_operator(n, 1.).apply_single_freq(b, tau, eta)
    b = np.zeros((greend.dim + 1,), dtype=np.intp)
    tau = np.zeros((greend.sym,), dtype=np.float64)
    greend.apply_single_freq(b, tau)

def test_apply_single_freq_invalid_params():
    for dim in [2, 3]:
        n = tuple(itertools.repeat(16, dim))
        green = discrete_green_operator(n, 1.)
        nrows = green.sym
        ncols = green.sym

        b0 = np.zeros((dim,), dtype=np.intp)
        tau0 = np.zeros((nrows, ncols), dtype=np.float64)
        tau1 = np.zeros((nrows + 1, ncols), dtype=np.float64)
        tau2 = np.zeros((nrows, ncols + 1), dtype=np.float64)
        eta0 = np.zeros((nrows, ncols), dtype=np.float64)
        eta1 = np.zeros((nrows + 1, ncols), dtype=np.float64)
        eta2 = np.zeros((nrows, ncols + 1), dtype=np.float64)

        params = ([(b, tau1, None) for b in invalid_bs(n)]
                  + [(b0, tau1, None), (b0, tau2, None),
                     (b0, tau0, eta1), (b0, tau0, eta2)])

        def test(b, tau, eta):
            greend.apply_single_freq(b, tau, eta)

        for b, tau, eta in params:
            yield test, b, tau, eta
"""

"""
@nottest
def do_test_apply_all_freqs(n, inplace):
    greend = discrete_green_operator(n, 1.)
    dim = len(n)
    sym = (dim * (dim + 1)) // 2
    tau = rnd.rand(n[1], n[0], sym)
    expected = np.empty_like(tau)
    b = np.empty((dim,), dtype=np.intp)
    for b[0] in range(n[0]):
        for b[1] in range(n[1]):
            greend.apply_single_freq(b,
                                     tau[b[1], b[0], :],
                                     expected[b[1], b[0], :])
    if inplace:
        base = np.empty_like(tau)
        actual = greend.apply_all_freqs(tau, base)
        assert get_base(actual) is base
    else:
        actual = greend.apply_all_freqs(tau)

    assert_array_max_ulp(expected, actual, 0)

def test_apply_all_freqs():
    shapes = ((8, 8), (8, 16))
    for n in GRID_SIZES:
        for inplace in [True, False]:
            yield do_test_apply_all_freqs, n, inplace

@raises(ValueError)
def test_apply_all_freqs_tau_wrong_num_dims():
    green = discrete_green_operator((8, 16), 1.)
    green.apply_all_freqs(np.empty((8, 8), dtype=np.float64))

@nottest
@raises(ValueError)
def do_test_apply_all_freqs_tau_invalid_shape(n, shape):
    green = discrete_green_operator(n, 1.)
    dim = len(n)
    sym = (dim * (dim + 1)) // 2
    tau = np.empty(shape + (sym,), dtype=np.float64)
    green.apply_all_freqs(tau)

def incr_elem(n, i):
    n_list = list(n)
    n_list[i] += 1
    return tuple(n_list)

def test_apply_all_freqs_tau_invalid_shape():
    for n in GRID_SIZES:
        n_mirror = list(n)[::-1]
        for shape in [incr_elem(n_mirror, i) for i in range(len(n))]:
            yield do_test_apply_all_freqs_tau_invalid_shape, n, shape

if __name__ == '__main__':
    do_test_apply_all_freqs_tau_invalid_shape((8, 16), (17, 8))
"""
<|endoftext|>


<commit_before># Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Visualization tools for tensorflow_docs.

Use this module for plotting and visualization code that is too long to inline
into a notebook.
"""
import matplotlib.pyplot as plt
import numpy as np

prop_cycle = plt.rcParams['axes.prop_cycle']
COLOR_CYCLE = prop_cycle.by_key()['color']

def _smooth(values, std):
  """Smooths a list of values by convolving with a gussian.

  Assumes equal spacing.

  Args:
    values: A 1D array of values to smooth.
    std: The standard devistion of the gussian. The units are array elements.

  Returns:
    The smoothed array.
  """
  width = std * 4
  x = np.linspace(-width, width, 2 * width + 1)
  kernel = np.exp(-(x / 5)**2)

  values = np.array(values)
  weights = np.ones_like(values)

  smoothed_values = np.convolve(values, kernel, mode='same')
  smoothed_weights = np.convolve(weights, kernel, mode='same')

  return smoothed_values / smoothed_weights

class HistoryPlotter(object):
  """A class for plotting named set of keras-histories.

  The class maintains colors for each key from plot to plot.
  """

  def __init__(self, metric=None, smoothing_std=None):
    self.color_table = {}
    self.metric = metric
    self.smoothing_std = smoothing_std

  def plot(self, histories, metric=None, smoothing_std=None):
    """Plots a {name: history} dictionary of keras histories.

    Colors are assigned to the name-key, and maintained from call to call.
    Training metrics are shown as a solid line, validation metrics dashed.

    Args:
      histories: {name: history} dictionary of keras histories.
      metric: which metric to plot from all the histories.
      smoothing_std: the standard-deviaation of the smoothing kernel applied
        before plotting. The units are in array-indices.
    """
    if metric is None:
      metric = self.metric
    if smoothing_std is None:
      smoothing_std = self.smoothing_std

    for name, history in histories.items():
      # Remember name->color asociations.
      if name in self.color_table:
        color = self.color_table[name]
      else:
        color = COLOR_CYCLE[len(self.color_table) % len(COLOR_CYCLE)]
        self.color_table[name] = color

      train_value = history.history[metric]
      val_value = history.history['val_' + metric]
      if smoothing_std is not None:
        train_value = _smooth(train_value, std=smoothing_std)
        val_value = _smooth(val_value, std=smoothing_std)

      plt.plot(
          history.epoch,
          train_value,
          color=color,
          label=name.title() + ' Train')
      plt.plot(
          history.epoch,
          val_value,
          '--',
          label=name.title() + ' Val',
          color=color)

    plt.xlabel('Epochs')
    plt.ylabel(metric.replace('_', ' ').title())
    plt.legend()

    plt.xlim(
        [0, max([history.epoch[-1] for name, history in histories.items()])])
    plt.grid(True)
<commit_msg>Update __init__.py<commit_after># Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Visualization tools for tensorflow_docs.

Use this module for plotting and visualization code that is too long to inline
into a notebook.
"""
import matplotlib.pyplot as plt
import numpy as np

prop_cycle = plt.rcParams['axes.prop_cycle']
COLOR_CYCLE = prop_cycle.by_key()['color']

def _smooth(values, std):
  """Smooths a list of values by convolving with a gussian.

  Assumes equal spacing.

  Args:
    values: A 1D array of values to smooth.
    std: The standard devistion of the gussian. The units are array elements.

  Returns:
    The smoothed array.
  """
  width = std * 4
  x = np.linspace(-width, width, min(2 * width + 1, len(values)))
  kernel = np.exp(-(x / 5)**2)

  values = np.array(values)
  weights = np.ones_like(values)

  smoothed_values = np.convolve(values, kernel, mode='same')
  smoothed_weights = np.convolve(weights, kernel, mode='same')

  return smoothed_values / smoothed_weights

class HistoryPlotter(object):
  """A class for plotting named set of keras-histories.

  The class maintains colors for each key from plot to plot.
  """

  def __init__(self, metric=None, smoothing_std=None):
    self.color_table = {}
    self.metric = metric
    self.smoothing_std = smoothing_std

  def plot(self, histories, metric=None, smoothing_std=None):
    """Plots a {name: history} dictionary of keras histories.

    Colors are assigned to the name-key, and maintained from call to call.
    Training metrics are shown as a solid line, validation metrics dashed.

    Args:
      histories: {name: history} dictionary of keras histories.
      metric: which metric to plot from all the histories.
      smoothing_std: the standard-deviaation of the smoothing kernel applied
        before plotting. The units are in array-indices.
    """
    if metric is None:
      metric = self.metric
    if smoothing_std is None:
      smoothing_std = self.smoothing_std

    for name, history in histories.items():
      # Remember name->color asociations.
      if name in self.color_table:
        color = self.color_table[name]
      else:
        color = COLOR_CYCLE[len(self.color_table) % len(COLOR_CYCLE)]
        self.color_table[name] = color

      train_value = history.history[metric]
      val_value = history.history['val_' + metric]
      if smoothing_std is not None:
        train_value = _smooth(train_value, std=smoothing_std)
        val_value = _smooth(val_value, std=smoothing_std)

      plt.plot(
          history.epoch,
          train_value,
          color=color,
          label=name.title() + ' Train')
      plt.plot(
          history.epoch,
          val_value,
          '--',
          label=name.title() + ' Val',
          color=color)

    plt.xlabel('Epochs')
    plt.ylabel(metric.replace('_', ' ').title())
    plt.legend()

    plt.xlim(
        [0, max([history.epoch[-1] for name, history in histories.items()])])
    plt.grid(True)
<|endoftext|>