<commit_before>from binascii import hexlify, unhexlify
from hashlib import md5, sha1, sha256

from ..exceptions import SSLError, InsecurePlatformWarning

SSLContext = None
HAS_SNI = False
create_default_context = None

import errno
import ssl
import warnings

try:  # Test for SSL features
    from ssl import wrap_socket, CERT_NONE, PROTOCOL_SSLv23
    from ssl import HAS_SNI  # Has SNI?
except ImportError:
    pass

try:
    from ssl import OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION
except ImportError:
    OP_NO_SSLv2, OP_NO_SSLv3 = 0x1000000, 0x2000000
    OP_NO_COMPRESSION = 0x20000

try:
    from ssl import _DEFAULT_CIPHERS
except ImportError:
    _DEFAULT_CIPHERS = (
        'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:'
        'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:!aNULL:'
        '!eNULL:!MD5'
    )

try:
    from ssl import SSLContext  # Modern SSL?
except ImportError:
    import sys

    class SSLContext(object):  # Platform-specific: Python 2 & 3.1
        supports_set_ciphers = sys.version_info >= (2, 7)

        def __init__(self, protocol_version):
            self.protocol = protocol_version
            # Use default values from a real SSLContext
            self.check_hostname = False
            self.verify_mode = ssl.CERT_NONE
            self.ca_certs = None
            self.options = 0
            self.certfile = None
            self.keyfile = None
            self.ciphers = None

        def load_cert_chain(self, certfile, keyfile):
            self.certfile = certfile
            self.keyfile = keyfile

        def load_verify_locations(self, location):
            self.ca_certs = location

        def set_ciphers(self, cipher_suite):
            if not self.supports_set_ciphers:
                raise TypeError(
                    'Your version of Python does not support setting '
                    'a custom cipher suite. Please upgrade to Python '
                    '2.7, 3.2, or later if you need this functionality.'
                )
            self.ciphers = cipher_suite

        def wrap_socket(self, socket, server_hostname=None):
            warnings.warn(
                'A true SSLContext object is not available. This prevents '
                'urllib3 from configuring SSL appropriately and may cause '
                'certain SSL connections to fail. For more information, see '
                'https://urllib3.readthedocs.org/en/latest/security.html'
                '#insecureplatformwarning.',
                InsecurePlatformWarning
            )
            kwargs = {
                'keyfile': self.keyfile,
                'certfile': self.certfile,
                'ca_certs': self.ca_certs,
                'cert_reqs': self.verify_mode,
                'ssl_version': self.protocol,
            }
            if self.supports_set_ciphers:  # Platform-specific: Python 2.7+
                return wrap_socket(socket, ciphers=self.ciphers, **kwargs)
            else:  # Platform-specific: Python 2.6
                return wrap_socket(socket, **kwargs)

def assert_fingerprint(cert, fingerprint):
    """
    Checks if given fingerprint matches the supplied certificate.

    :param cert:
        Certificate as bytes object.
    :param fingerprint:
        Fingerprint as string of hexdigits, can be interspersed by colons.
    """

    # Maps the length of a digest to a possible hash function producing
    # this digest.
    hashfunc_map = {
        16: md5,
        20: sha1,
        32: sha256,
    }

    fingerprint = fingerprint.replace(':', '').lower()
    digest_length, odd = divmod(len(fingerprint), 2)

    if odd or digest_length not in hashfunc_map:
        raise SSLError('Fingerprint is of invalid length.')

    # We need encode() here for py32; works on py2 and p33.
    fingerprint_bytes = unhexlify(fingerprint.encode())

    hashfunc = hashfunc_map[digest_length]

    cert_digest = hashfunc(cert).digest()

    if not cert_digest == fingerprint_bytes:
        raise SSLError('Fingerprints did not match. Expected "{0}", got "{1}".'
                       .format(hexlify(fingerprint_bytes),
                               hexlify(cert_digest)))

def resolve_cert_reqs(candidate):
    """
    Resolves the argument to a numeric constant, which can be passed to
    the wrap_socket function/method from the ssl module.
    Defaults to :data:`ssl.CERT_NONE`.
    If given a string it is assumed to be the name of the constant in the
    :mod:`ssl` module or its abbrevation.
    (So you can specify `REQUIRED` instead of `CERT_REQUIRED`.
    If it's neither `None` nor a string we assume it is already the numeric
    constant which can directly be passed to wrap_socket.
    """
    if candidate is None:
        return CERT_NONE

    if isinstance(candidate, str):
        res = getattr(ssl, candidate, None)
        if res is None:
            res = getattr(ssl, 'CERT_' + candidate)
        return res

    return candidate

def resolve_ssl_version(candidate):
    """
    like resolve_cert_reqs
    """
    if candidate is None:
        return PROTOCOL_SSLv23

    if isinstance(candidate, str):
        res = getattr(ssl, candidate, None)
        if res is None:
            res = getattr(ssl, 'PROTOCOL_' + candidate)
        return res

    return candidate

def create_urllib3_context(ssl_version=None, cert_reqs=ssl.CERT_REQUIRED,
                           options=None, ciphers=None):
    """All arguments have the same meaning as ``ssl_wrap_socket``.

    By default, this function does a lot of the same work that
    ``ssl.create_default_context`` does on Python 3.4+. It:

    - Disables SSLv2, SSLv3, and compression
    - Sets a restricted set of server ciphers

    If you wish to enable SSLv3, you can do::

        from urllib3.util import ssl_
        context = ssl_.create_urllib3_context()
        context.options &= ~ssl_.OP_NO_SSLv3

    You can do the same to enable compression (substituting ``COMPRESSION``
    for ``SSLv3`` in the last line above).

    :param ssl_version:
        The desired protocol version to use. This will default to
        PROTOCOL_SSLv23 which will negotiate the highest protocol that both
        the server and your installation of OpenSSL support.
    :param cert_reqs:
        Whether to require the certificate verification. This defaults to
        ``ssl.CERT_REQUIRED``.
    :param options:
        Specific OpenSSL options. These default to ``ssl.OP_NO_SSLv2``,
        ``ssl.OP_NO_SSLv3``, ``ssl.OP_NO_COMPRESSION``.
    :param ciphers:
        Which cipher suites to allow the server to select.
    :returns:
        Constructed SSLContext object with specified options
    :rtype: SSLContext
    """
    context = SSLContext(ssl_version or ssl.PROTOCOL_SSLv23)

    if options is None:
        options = 0
        # SSLv2 is easily broken and is considered harmful and dangerous
        options |= OP_NO_SSLv2
        # SSLv3 has several problems and is now dangerous
        options |= OP_NO_SSLv3
        # Disable compression to prevent CRIME attacks for OpenSSL 1.0+
        # (issue #309)
        options |= OP_NO_COMPRESSION

    context.options |= options

    if getattr(context, 'supports_set_ciphers', True):  # Platform-specific: Python 2.6
        context.set_ciphers(ciphers or _DEFAULT_CIPHERS)

    context.verify_mode = cert_reqs
    if getattr(context, 'check_hostname', None) is not None:  # Platform-specific: Python 3.2
        # We do our own verification, including fingerprints and alternative
        # hostnames. So disable it here
        context.check_hostname = False
    return context

def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
                    ca_certs=None, server_hostname=None,
                    ssl_version=None, ciphers=None, ssl_context=None):
    """
    All arguments except for server_hostname and ssl_context have the same
    meaning as they do when using :func:`ssl.wrap_socket`.

    :param server_hostname:
        When SNI is supported, the expected hostname of the certificate
    :param ssl_context:
        A pre-made :class:`SSLContext` object. If none is provided, one will
        be created using :func:`create_urllib3_context`.
    :param ciphers:
        A string of ciphers we wish the client to support. This is not
        supported on Python 2.6 as the ssl module does not support it.
    """
    context = ssl_context
    if context is None:
        context = create_urllib3_context(ssl_version, cert_reqs,
                                         ciphers=ciphers)

    if ca_certs:
        try:
            context.load_verify_locations(ca_certs)
        except IOError as e:  # Platform-specific: Python 2.6, 2.7, 3.2
            raise SSLError(e)
        # Py33 raises FileNotFoundError which subclasses OSError
        # These are not equivalent unless we check the errno attribute
        except OSError as e:  # Platform-specific: Python 3.3 and beyond
            if e.errno == errno.ENOENT:
                raise SSLError(e)
            raise
    if certfile:
        context.load_cert_chain(certfile, keyfile)
    if HAS_SNI:  # Platform-specific: OpenSSL with enabled SNI
        return context.wrap_socket(sock, server_hostname=server_hostname)
    return context.wrap_socket(sock)
<commit_msg>Properly detect Python 3.0-3.1 in SSLContext<commit_after>from binascii import hexlify, unhexlify
from hashlib import md5, sha1, sha256

from ..exceptions import SSLError, InsecurePlatformWarning

SSLContext = None
HAS_SNI = False
create_default_context = None

import errno
import ssl
import warnings

try:  # Test for SSL features
    from ssl import wrap_socket, CERT_NONE, PROTOCOL_SSLv23
    from ssl import HAS_SNI  # Has SNI?
except ImportError:
    pass

try:
    from ssl import OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION
except ImportError:
    OP_NO_SSLv2, OP_NO_SSLv3 = 0x1000000, 0x2000000
    OP_NO_COMPRESSION = 0x20000

try:
    from ssl import _DEFAULT_CIPHERS
except ImportError:
    _DEFAULT_CIPHERS = (
        'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:'
        'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:!aNULL:'
        '!eNULL:!MD5'
    )

try:
    from ssl import SSLContext  # Modern SSL?
except ImportError:
    import sys

    class SSLContext(object):  # Platform-specific: Python 2 & 3.1
        supports_set_ciphers = (2, 7) <= sys.version_info < (3,) or \
                               (3, 2) <= sys.version_info

        def __init__(self, protocol_version):
            self.protocol = protocol_version
            # Use default values from a real SSLContext
            self.check_hostname = False
            self.verify_mode = ssl.CERT_NONE
            self.ca_certs = None
            self.options = 0
            self.certfile = None
            self.keyfile = None
            self.ciphers = None

        def load_cert_chain(self, certfile, keyfile):
            self.certfile = certfile
            self.keyfile = keyfile

        def load_verify_locations(self, location):
            self.ca_certs = location

        def set_ciphers(self, cipher_suite):
            if not self.supports_set_ciphers:
                raise TypeError(
                    'Your version of Python does not support setting '
                    'a custom cipher suite. Please upgrade to Python '
                    '2.7, 3.2, or later if you need this functionality.'
                )
            self.ciphers = cipher_suite

        def wrap_socket(self, socket, server_hostname=None):
            warnings.warn(
                'A true SSLContext object is not available. This prevents '
                'urllib3 from configuring SSL appropriately and may cause '
                'certain SSL connections to fail. For more information, see '
                'https://urllib3.readthedocs.org/en/latest/security.html'
                '#insecureplatformwarning.',
                InsecurePlatformWarning
            )
            kwargs = {
                'keyfile': self.keyfile,
                'certfile': self.certfile,
                'ca_certs': self.ca_certs,
                'cert_reqs': self.verify_mode,
                'ssl_version': self.protocol,
            }
            if self.supports_set_ciphers:  # Platform-specific: Python 2.7+
                return wrap_socket(socket, ciphers=self.ciphers, **kwargs)
            else:  # Platform-specific: Python 2.6
                return wrap_socket(socket, **kwargs)

def assert_fingerprint(cert, fingerprint):
    """
    Checks if given fingerprint matches the supplied certificate.

    :param cert:
        Certificate as bytes object.
    :param fingerprint:
        Fingerprint as string of hexdigits, can be interspersed by colons.
    """

    # Maps the length of a digest to a possible hash function producing
    # this digest.
    hashfunc_map = {
        16: md5,
        20: sha1,
        32: sha256,
    }

    fingerprint = fingerprint.replace(':', '').lower()
    digest_length, odd = divmod(len(fingerprint), 2)

    if odd or digest_length not in hashfunc_map:
        raise SSLError('Fingerprint is of invalid length.')

    # We need encode() here for py32; works on py2 and p33.
    fingerprint_bytes = unhexlify(fingerprint.encode())

    hashfunc = hashfunc_map[digest_length]

    cert_digest = hashfunc(cert).digest()

    if not cert_digest == fingerprint_bytes:
        raise SSLError('Fingerprints did not match. Expected "{0}", got "{1}".'
                       .format(hexlify(fingerprint_bytes),
                               hexlify(cert_digest)))

def resolve_cert_reqs(candidate):
    """
    Resolves the argument to a numeric constant, which can be passed to
    the wrap_socket function/method from the ssl module.
    Defaults to :data:`ssl.CERT_NONE`.
    If given a string it is assumed to be the name of the constant in the
    :mod:`ssl` module or its abbrevation.
    (So you can specify `REQUIRED` instead of `CERT_REQUIRED`.
    If it's neither `None` nor a string we assume it is already the numeric
    constant which can directly be passed to wrap_socket.
    """
    if candidate is None:
        return CERT_NONE

    if isinstance(candidate, str):
        res = getattr(ssl, candidate, None)
        if res is None:
            res = getattr(ssl, 'CERT_' + candidate)
        return res

    return candidate

def resolve_ssl_version(candidate):
    """
    like resolve_cert_reqs
    """
    if candidate is None:
        return PROTOCOL_SSLv23

    if isinstance(candidate, str):
        res = getattr(ssl, candidate, None)
        if res is None:
            res = getattr(ssl, 'PROTOCOL_' + candidate)
        return res

    return candidate

def create_urllib3_context(ssl_version=None, cert_reqs=ssl.CERT_REQUIRED,
                           options=None, ciphers=None):
    """All arguments have the same meaning as ``ssl_wrap_socket``.

    By default, this function does a lot of the same work that
    ``ssl.create_default_context`` does on Python 3.4+. It:

    - Disables SSLv2, SSLv3, and compression
    - Sets a restricted set of server ciphers

    If you wish to enable SSLv3, you can do::

        from urllib3.util import ssl_
        context = ssl_.create_urllib3_context()
        context.options &= ~ssl_.OP_NO_SSLv3

    You can do the same to enable compression (substituting ``COMPRESSION``
    for ``SSLv3`` in the last line above).

    :param ssl_version:
        The desired protocol version to use. This will default to
        PROTOCOL_SSLv23 which will negotiate the highest protocol that both
        the server and your installation of OpenSSL support.
    :param cert_reqs:
        Whether to require the certificate verification. This defaults to
        ``ssl.CERT_REQUIRED``.
    :param options:
        Specific OpenSSL options. These default to ``ssl.OP_NO_SSLv2``,
        ``ssl.OP_NO_SSLv3``, ``ssl.OP_NO_COMPRESSION``.
    :param ciphers:
        Which cipher suites to allow the server to select.
    :returns:
        Constructed SSLContext object with specified options
    :rtype: SSLContext
    """
    context = SSLContext(ssl_version or ssl.PROTOCOL_SSLv23)

    if options is None:
        options = 0
        # SSLv2 is easily broken and is considered harmful and dangerous
        options |= OP_NO_SSLv2
        # SSLv3 has several problems and is now dangerous
        options |= OP_NO_SSLv3
        # Disable compression to prevent CRIME attacks for OpenSSL 1.0+
        # (issue #309)
        options |= OP_NO_COMPRESSION

    context.options |= options

    if getattr(context, 'supports_set_ciphers', True):  # Platform-specific: Python 2.6
        context.set_ciphers(ciphers or _DEFAULT_CIPHERS)

    context.verify_mode = cert_reqs
    if getattr(context, 'check_hostname', None) is not None:  # Platform-specific: Python 3.2
        # We do our own verification, including fingerprints and alternative
        # hostnames. So disable it here
        context.check_hostname = False
    return context

def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
                    ca_certs=None, server_hostname=None,
                    ssl_version=None, ciphers=None, ssl_context=None):
    """
    All arguments except for server_hostname and ssl_context have the same
    meaning as they do when using :func:`ssl.wrap_socket`.

    :param server_hostname:
        When SNI is supported, the expected hostname of the certificate
    :param ssl_context:
        A pre-made :class:`SSLContext` object. If none is provided, one will
        be created using :func:`create_urllib3_context`.
    :param ciphers:
        A string of ciphers we wish the client to support. This is not
        supported on Python 2.6 as the ssl module does not support it.
    """
    context = ssl_context
    if context is None:
        context = create_urllib3_context(ssl_version, cert_reqs,
                                         ciphers=ciphers)

    if ca_certs:
        try:
            context.load_verify_locations(ca_certs)
        except IOError as e:  # Platform-specific: Python 2.6, 2.7, 3.2
            raise SSLError(e)
        # Py33 raises FileNotFoundError which subclasses OSError
        # These are not equivalent unless we check the errno attribute
        except OSError as e:  # Platform-specific: Python 3.3 and beyond
            if e.errno == errno.ENOENT:
                raise SSLError(e)
            raise
    if certfile:
        context.load_cert_chain(certfile, keyfile)
    if HAS_SNI:  # Platform-specific: OpenSSL with enabled SNI
        return context.wrap_socket(sock, server_hostname=server_hostname)
    return context.wrap_socket(sock)
<|endoftext|>


<commit_before>"""
a bunch of simple, composable nodes - not necessarily simple in their
implementation, but simple in that they do a single thing
"""

from __future__ import division, absolute_import
from __future__ import print_function, unicode_literals

import toolz
import theano
import theano.tensor as T

from .. import utils
from .. import core

@core.register_node("reference")
class ReferenceNode(core.NodeImpl):

    """
    provides dependencies into separate parts of the tree, allowing
    separate branches to have computational graph dependencies between them

    reference:
    node name to take output from as input to the reference node
    """

    hyperparameter_names = ("reference",)
    input_keys = ("reference_input",)

    def init_long_range_dependencies(self, network):
        network.take_output_from(network.find_hyperparameter(["reference"]),
                                 to_key="reference_input")

@core.register_node("send_to")
class SendToNode(core.NodeImpl):

    """
    sends the input of the node into separate parts of the tree, allowing
    separate branches to have computational graph dependencies between them

    reference:
    node name to take output from as input to the reference node

    to_key:
    input key for the reference node of the SendToNode's output
    """

    hyperparameter_names = ("send_to_reference",
                            "reference",
                            "to_key")

    def init_long_range_dependencies(self, network):
        network.forward_output_to(
            network.find_hyperparameter(["send_to_reference",
                                         "reference"]),
            to_key=network.find_hyperparameter(["to_key"],
                                               "default"))

@core.register_node("hyperparameter")
class HyperparameterNode(core.Wrapper1NodeImpl):

    """
    for providing hyperparameters to a subtree
    """

    def __init__(self, name, children, **hyperparameters):
        # set hyperparameter keys to be all passed in keys
        self.hyperparameter_names = tuple(hyperparameters.keys())
        # override init to allow for using keyword arguments
        super(HyperparameterNode, self).__init__(name,
                                                 children,
                                                 **hyperparameters)

@core.register_node("input")
class InputNode(core.NodeImpl):

    """
    an entry point into the network
    """

    hyperparameter_names = ("input_shape",
                            "shape",
                            "input_dtype",
                            "dtype",
                            "input_broadcastable",
                            "broadcastable")
    input_keys = ()

    def compute_output(self, network):
        network.create_vw(
            name="default",
            shape=network.find_hyperparameter(["input_shape",
                                               "shape"]),
            dtype=network.find_hyperparameter(["input_dtype",
                                               "dtype"],
                                              theano.config.floatX),
            broadcastable=network.find_hyperparameter(["input_broadcastable",
                                                       "broadcastable"],
                                                      None),
            is_shared=False,
            tags=["input"],
        )

@core.register_node("identity")
class IdentityNode(core.NodeImpl):

    """
    returns input
    """
    # NOTE: default implementation of NodeImpl is Identity

@core.register_node("constant")
class ConstantNode(core.NodeImpl):

    """
    node that returns a constant value
    """

    hyperparameter_names = ("constant_value", "value")
    input_keys = ()

    def compute_output(self, network):
        value = network.find_hyperparameter(["constant_value", "value"])
        if utils.is_nonshared_variable(value):
            variable = value
            shape = None
        elif utils.is_shared_variable(value):
            variable = value
            shape = value.get_value().shape
        else:
            variable = T.constant(value)
            shape = value.shape if hasattr(value, "shape") else ()
        network.create_vw(
            name="default",
            variable=variable,
            shape=shape,
            tags={"output"},
        )

@core.register_node("add_bias")
class AddBiasNode(core.NodeImpl):

    """
    node that adds a bias parameter to its input

    takes in either a list of axes (as key broadcastable_axes) to broadcast
    over, or as a tuple of booleans
    """

    hyperparameter_names = ("bias_inits",
                            "inits",
                            "broadcastable_axes",
                            "broadcastable",)

    def compute_output(self, network, in_vw):
        inits = list(toolz.concat(network.find_hyperparameters(
            ["bias_inits",
             "inits"],
            [])))
        # gather hyperparameters
        broadcastable = network.find_hyperparameter(["broadcastable"],
                                                    None)
        broadcastable_axes = network.find_hyperparameter(
            ["broadcastable_axes"],
            None)
        batch_axis = network.find_hyperparameter(["batch_axis"])
        # have broadcastable as a tuple take precedence over broadcastable_axes
        if broadcastable is None:
            if broadcastable_axes is None:
                if batch_axis is None:
                    # no minibatch axis = no default broadcasting
                    broadcastable_axes = []
                else:
                    # by default, broadcast over minibatch axis, if any
                    broadcastable_axes = [batch_axis]
            broadcastable = [False] * in_vw.ndim
            for axis in broadcastable_axes:
                broadcastable[axis] = True

        assert len(broadcastable) == in_vw.ndim
        shape = tuple([1 if is_broadcastable else size
                       for is_broadcastable, size in zip(broadcastable,
                                                         in_vw.shape)])
        b = network.create_vw(
            name="bias",
            is_shared=True,
            shape=shape,
            tags={"parameter", "bias"},
            inits=inits,
        )
        b_var = b.variable
        # not calling patternbroadcast if not broadcastable, because it seems
        # to have a small overhead
        if any(broadcastable):
            b_var = T.patternbroadcast(b_var, broadcastable)
        network.create_vw(
            name="default",
            variable=(in_vw.variable + b_var),
            shape=in_vw.shape,
            tags={"output"},
        )

@core.register_node("linear_mapping")
class LinearMappingNode(core.NodeImpl):

    """
    node that applies a linear mapping to the last dimension of its input
    (a dot product with a parameter)
    """

    hyperparameter_names = ("linear_mapping_inits",
                            "inits",
                            "output_dim")

    def compute_output(self, network, in_vw):
        inits = list(toolz.concat(network.find_hyperparameters(
            ["linear_mapping_inits",
             "inits"],
            [])))
        output_dim = network.find_hyperparameter(["output_dim"])
        weight_shape = (in_vw.shape[-1], output_dim)
        output_shape = tuple(in_vw.shape[:-1]) + (output_dim, )
        W = network.create_vw(
            name="weight",
            is_shared=True,
            shape=weight_shape,
            tags={"parameter", "weight"},
            inits=inits,
        )
        network.create_vw(
            name="default",
            variable=T.dot(in_vw.variable, W.variable),
            shape=output_shape,
            tags={"output"},
        )

@core.register_node("apply")
class ApplyNode(core.NodeImpl):

    """
    applies a pure theano function
    """

    hyperparameter_names = ("fn", "shape_fn")

    def compute_output(self, network, in_vw):
        fn = network.find_hyperparameter(["fn"])
        shape_fn = network.find_hyperparameter(["shape_fn"], None)
        if shape_fn is None:
            shape = None
        else:
            shape = shape_fn(in_vw.shape)
        network.create_vw(
            name="default",
            variable=fn(in_vw.variable),
            shape=shape,
            tags={"output"},
        )

@core.register_node("add_constant")
class AddConstantNode(core.NodeImpl):

    """
    adds a constant value to its input
    """

    hyperparameter_names = ("value", )

    def compute_output(self, network, in_vw):
        value = network.find_hyperparameter(["value"])
        network.create_vw(
            name="default",
            variable=in_vw.variable + value,
            shape=in_vw.shape,
            tags={"output"},
        )

@core.register_node("multiply_constant")
class MultiplyConstantNode(core.NodeImpl):

    """
    adds a constant value to its input
    """

    hyperparameter_names = ("value", )

    def compute_output(self, network, in_vw):
        value = network.find_hyperparameter(["value"])
        network.create_vw(
            name="default",
            variable=in_vw.variable * value,
            shape=in_vw.shape,
            tags={"output"},
        )
<commit_msg>handle scalar input default case for add bias node<commit_after>"""
a bunch of simple, composable nodes - not necessarily simple in their
implementation, but simple in that they do a single thing
"""

from __future__ import division, absolute_import
from __future__ import print_function, unicode_literals

import toolz
import theano
import theano.tensor as T

from .. import utils
from .. import core

@core.register_node("reference")
class ReferenceNode(core.NodeImpl):

    """
    provides dependencies into separate parts of the tree, allowing
    separate branches to have computational graph dependencies between them

    reference:
    node name to take output from as input to the reference node
    """

    hyperparameter_names = ("reference",)
    input_keys = ("reference_input",)

    def init_long_range_dependencies(self, network):
        network.take_output_from(network.find_hyperparameter(["reference"]),
                                 to_key="reference_input")

@core.register_node("send_to")
class SendToNode(core.NodeImpl):

    """
    sends the input of the node into separate parts of the tree, allowing
    separate branches to have computational graph dependencies between them

    reference:
    node name to take output from as input to the reference node

    to_key:
    input key for the reference node of the SendToNode's output
    """

    hyperparameter_names = ("send_to_reference",
                            "reference",
                            "to_key")

    def init_long_range_dependencies(self, network):
        network.forward_output_to(
            network.find_hyperparameter(["send_to_reference",
                                         "reference"]),
            to_key=network.find_hyperparameter(["to_key"],
                                               "default"))

@core.register_node("hyperparameter")
class HyperparameterNode(core.Wrapper1NodeImpl):

    """
    for providing hyperparameters to a subtree
    """

    def __init__(self, name, children, **hyperparameters):
        # set hyperparameter keys to be all passed in keys
        self.hyperparameter_names = tuple(hyperparameters.keys())
        # override init to allow for using keyword arguments
        super(HyperparameterNode, self).__init__(name,
                                                 children,
                                                 **hyperparameters)

@core.register_node("input")
class InputNode(core.NodeImpl):

    """
    an entry point into the network
    """

    hyperparameter_names = ("input_shape",
                            "shape",
                            "input_dtype",
                            "dtype",
                            "input_broadcastable",
                            "broadcastable")
    input_keys = ()

    def compute_output(self, network):
        network.create_vw(
            name="default",
            shape=network.find_hyperparameter(["input_shape",
                                               "shape"]),
            dtype=network.find_hyperparameter(["input_dtype",
                                               "dtype"],
                                              theano.config.floatX),
            broadcastable=network.find_hyperparameter(["input_broadcastable",
                                                       "broadcastable"],
                                                      None),
            is_shared=False,
            tags=["input"],
        )

@core.register_node("identity")
class IdentityNode(core.NodeImpl):

    """
    returns input
    """
    # NOTE: default implementation of NodeImpl is Identity

@core.register_node("constant")
class ConstantNode(core.NodeImpl):

    """
    node that returns a constant value
    """

    hyperparameter_names = ("constant_value", "value")
    input_keys = ()

    def compute_output(self, network):
        value = network.find_hyperparameter(["constant_value", "value"])
        if utils.is_nonshared_variable(value):
            variable = value
            shape = None
        elif utils.is_shared_variable(value):
            variable = value
            shape = value.get_value().shape
        else:
            variable = T.constant(value)
            shape = value.shape if hasattr(value, "shape") else ()
        network.create_vw(
            name="default",
            variable=variable,
            shape=shape,
            tags={"output"},
        )

@core.register_node("add_bias")
class AddBiasNode(core.NodeImpl):

    """
    node that adds a bias parameter to its input

    takes in either a list of axes (as key broadcastable_axes) to broadcast
    over, or as a tuple of booleans
    """

    hyperparameter_names = ("bias_inits",
                            "inits",
                            "broadcastable_axes",
                            "broadcastable",)

    def compute_output(self, network, in_vw):
        inits = list(toolz.concat(network.find_hyperparameters(
            ["bias_inits",
             "inits"],
            [])))
        # gather hyperparameters
        broadcastable = network.find_hyperparameter(["broadcastable"],
                                                    None)
        broadcastable_axes = network.find_hyperparameter(
            ["broadcastable_axes"],
            None)
        batch_axis = network.find_hyperparameter(["batch_axis"])
        # have broadcastable as a tuple take precedence over broadcastable_axes
        if broadcastable is None:
            if broadcastable_axes is None:
                if batch_axis is None:
                    # no minibatch axis = no default broadcasting
                    broadcastable_axes = []
                elif batch_axis >= in_vw.ndim:
                    # scalar input = no broadcasting
                    broadcastable_axes = []
                else:
                    # by default, broadcast over minibatch axis, if any
                    broadcastable_axes = [batch_axis]
            broadcastable = [False] * in_vw.ndim
            for axis in broadcastable_axes:
                broadcastable[axis] = True

        assert len(broadcastable) == in_vw.ndim
        shape = tuple([1 if is_broadcastable else size
                       for is_broadcastable, size in zip(broadcastable,
                                                         in_vw.shape)])
        b = network.create_vw(
            name="bias",
            is_shared=True,
            shape=shape,
            tags={"parameter", "bias"},
            inits=inits,
        )
        b_var = b.variable
        # not calling patternbroadcast if not broadcastable, because it seems
        # to have a small overhead
        if any(broadcastable):
            b_var = T.patternbroadcast(b_var, broadcastable)
        network.create_vw(
            name="default",
            variable=(in_vw.variable + b_var),
            shape=in_vw.shape,
            tags={"output"},
        )

@core.register_node("linear_mapping")
class LinearMappingNode(core.NodeImpl):

    """
    node that applies a linear mapping to the last dimension of its input
    (a dot product with a parameter)
    """

    hyperparameter_names = ("linear_mapping_inits",
                            "inits",
                            "output_dim")

    def compute_output(self, network, in_vw):
        inits = list(toolz.concat(network.find_hyperparameters(
            ["linear_mapping_inits",
             "inits"],
            [])))
        output_dim = network.find_hyperparameter(["output_dim"])
        weight_shape = (in_vw.shape[-1], output_dim)
        output_shape = tuple(in_vw.shape[:-1]) + (output_dim, )
        W = network.create_vw(
            name="weight",
            is_shared=True,
            shape=weight_shape,
            tags={"parameter", "weight"},
            inits=inits,
        )
        network.create_vw(
            name="default",
            variable=T.dot(in_vw.variable, W.variable),
            shape=output_shape,
            tags={"output"},
        )

@core.register_node("apply")
class ApplyNode(core.NodeImpl):

    """
    applies a pure theano function
    """

    hyperparameter_names = ("fn", "shape_fn")

    def compute_output(self, network, in_vw):
        fn = network.find_hyperparameter(["fn"])
        shape_fn = network.find_hyperparameter(["shape_fn"], None)
        if shape_fn is None:
            shape = None
        else:
            shape = shape_fn(in_vw.shape)
        network.create_vw(
            name="default",
            variable=fn(in_vw.variable),
            shape=shape,
            tags={"output"},
        )

@core.register_node("add_constant")
class AddConstantNode(core.NodeImpl):

    """
    adds a constant value to its input
    """

    hyperparameter_names = ("value", )

    def compute_output(self, network, in_vw):
        value = network.find_hyperparameter(["value"])
        network.create_vw(
            name="default",
            variable=in_vw.variable + value,
            shape=in_vw.shape,
            tags={"output"},
        )

@core.register_node("multiply_constant")
class MultiplyConstantNode(core.NodeImpl):

    """
    adds a constant value to its input
    """

    hyperparameter_names = ("value", )

    def compute_output(self, network, in_vw):
        value = network.find_hyperparameter(["value"])
        network.create_vw(
            name="default",
            variable=in_vw.variable * value,
            shape=in_vw.shape,
            tags={"output"},
        )
<|endoftext|>


<commit_before>"""
mbed SDK
Copyright (c) 2011-2013 ARM Limited

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

One repository to update them all
On mbed.org the mbed SDK is split up in multiple repositories, this script takes
care of updating them all.
"""
import sys
from copy import copy
from os import walk, remove, makedirs
from os.path import join, abspath, dirname, relpath, exists, isfile
from shutil import copyfile
from optparse import OptionParser
import re
import string

ROOT = abspath(join(dirname(__file__), ".."))
sys.path.insert(0, ROOT)

from workspace_tools.settings import MBED_ORG_PATH, MBED_ORG_USER, BUILD_DIR
from workspace_tools.paths import *
from workspace_tools.utils import run_cmd

MBED_URL = "mbed.org"
MBED_USER = "mbed_official"

changed = []
push_remote = True
quiet = False
commit_msg = ''

# Code that does have a mirror in the mbed SDK
# Tuple data: (repo_name, list_of_code_dirs, [team])
# team is optional - if not specified, the code is published under mbed_official
OFFICIAL_CODE = (
    ("mbed-dev" , MBED_BASE),
    ("mbed-rtos", RTOS),
    ("mbed-dsp" , DSP),
    ("mbed-rpc" , MBED_RPC),

    ("lwip"    , LWIP_SOURCES+"/lwip"),
    ("lwip-sys", LWIP_SOURCES+"/lwip-sys"),
    ("Socket"  , LWIP_SOURCES+"/Socket"),

    ("lwip-eth"         , LWIP_SOURCES+"/lwip-eth"),
    ("EthernetInterface", ETH_SOURCES+"/EthernetInterface"),

    ("USBDevice", USB),
    ("USBHost"  , USB_HOST),

    ("CellularModem", CELLULAR_SOURCES),
    ("CellularUSBModem", CELLULAR_USB_SOURCES),
    ("UbloxUSBModem", UBLOX_SOURCES),
    ("UbloxModemHTTPClientTest", [TEST_DIR+"/net/cellular/http/common", TEST_DIR+"/net/cellular/http/ubloxusb"]),
    ("UbloxModemSMSTest", [TEST_DIR+"/net/cellular/sms/common", TEST_DIR+"/net/cellular/sms/ubloxusb"]),
    ("FATFileSystem", FAT_FS, "mbed-official"),
)

# Code that does have dependencies to libraries should point to
# the latest revision. By default, they point to a specific revision.
CODE_WITH_DEPENDENCIES = (
    # Libraries
    "EthernetInterface",

    # RTOS Examples
    "rtos_basic",
    "rtos_isr",
    "rtos_mail",
    "rtos_mutex",
    "rtos_queue",
    "rtos_semaphore",
    "rtos_signals",
    "rtos_timer",

    # Net Examples
    "TCPEchoClient",
    "TCPEchoServer",
    "TCPSocket_HelloWorld",
    "UDPSocket_HelloWorld",
    "UDPEchoClient",
    "UDPEchoServer",
    "BroadcastReceive",
    "BroadcastSend",

    # mbed sources
    "mbed-src-program",
)

# A list of regular expressions that will be checked against each directory
# name and skipped if they match.
IGNORE_DIRS = (
)

IGNORE_FILES = (
    'COPYING',
    '\.md',
    "\.lib",
    "\.bld"
)

def ignore_path(name, reg_exps):
    for r in reg_exps:
        if re.search(r, name):
            return True
    return False

class MbedRepository:
    @staticmethod
    def run_and_print(command, cwd):
        stdout, _, _ = run_cmd(command, wd=cwd, redirect=True)
        print(stdout)

    def __init__(self, name, team = None):
        self.name = name
        self.path = join(MBED_ORG_PATH, name)
        if team is None:
            self.url = "http://" + MBED_URL + "/users/" + MBED_USER + "/code/%s/"
        else:
            self.url = "http://" + MBED_URL + "/teams/" + team + "/code/%s/"
        if not exists(self.path):
            # Checkout code
            if not exists(MBED_ORG_PATH):
                makedirs(MBED_ORG_PATH)

            self.run_and_print(['hg', 'clone', self.url % name], cwd=MBED_ORG_PATH)

        else:
            # Update
            self.run_and_print(['hg', 'pull'], cwd=self.path)
            self.run_and_print(['hg', 'update'], cwd=self.path)

    def publish(self):
        # The maintainer has to evaluate the changes first and explicitly accept them
        self.run_and_print(['hg', 'addremove'], cwd=self.path)
        stdout, _, _ = run_cmd(['hg', 'status'], wd=self.path)
        if stdout == '':
            print "No changes"
            return False
        print stdout
        if quiet:
            commit = 'Y'
        else:
            commit = raw_input(push_remote and "Do you want to commit and push? Y/N: " or "Do you want to commit? Y/N: ")
        if commit == 'Y':
            args = ['hg', 'commit', '-u', MBED_ORG_USER]
            if commit_msg:
                args = args + ['-m', commit_msg]
            self.run_and_print(args, cwd=self.path)
            if push_remote:
                self.run_and_print(['hg', 'push'], cwd=self.path)
        return True

# Check if a file is a text file or a binary file
# Taken from http://code.activestate.com/recipes/173220/
text_characters = "".join(map(chr, range(32, 127)) + list("\n\r\t\b"))
_null_trans = string.maketrans("", "")
def is_text_file(filename):
    block_size = 1024
    def istext(s):
        if "\0" in s:
            return 0

        if not s:  # Empty files are considered text
            return 1

        # Get the non-text characters (maps a character to itself then
        # use the 'remove' option to get rid of the text characters.)
        t = s.translate(_null_trans, text_characters)

        # If more than 30% non-text characters, then
        # this is considered a binary file
        if float(len(t))/len(s) > 0.30:
            return 0
        return 1
    with open(filename) as f:
        res = istext(f.read(block_size))
    return res

# Return the line ending type for the given file ('cr' or 'crlf')
def get_line_endings(f):
  examine_size = 1024
  try:
    tf = open(f, "rb")
    lines, ncrlf = tf.readlines(examine_size), 0
    tf.close()
    for l in lines:
      if l.endswith("\r\n"):
        ncrlf = ncrlf + 1
    return 'crlf' if ncrlf > len(lines) >> 1 else 'cr'
  except:
    return 'cr'

# Copy file to destination, but preserve destination line endings if possible
# This prevents very annoying issues with huge diffs that appear because of
# differences in line endings
def copy_with_line_endings(sdk_file, repo_file):
    if not isfile(repo_file):
        copyfile(sdk_file, repo_file)
        return
    is_text = is_text_file(repo_file)
    if is_text:
        sdk_le = get_line_endings(sdk_file)
        repo_le = get_line_endings(repo_file)
    if not is_text or sdk_le == repo_le:
        copyfile(sdk_file, repo_file)
    else:
        print "Converting line endings in '%s' to '%s'" % (abspath(repo_file), repo_le)
        f = open(sdk_file, "rb")
        data = f.read()
        f.close()
        f = open(repo_file, "wb")
        data = data.replace("\r\n", "\n") if repo_le == 'cr' else data.replace('\n','\r\n')
        f.write(data)
        f.close()

def visit_files(path, visit):
    for root, dirs, files in walk(path):
        # Ignore hidden directories
        for d in copy(dirs):
            full = join(root, d)
            if d.startswith('.'):
                dirs.remove(d)
            if ignore_path(full, IGNORE_DIRS):
                print "Skipping '%s'" % full
                dirs.remove(d)

        for file in files:
            if ignore_path(file, IGNORE_FILES):
                continue

            visit(join(root, file))

def update_repo(repo_name, sdk_paths, team_name):
    repo = MbedRepository(repo_name, team_name)
    # copy files from mbed SDK to mbed_official repository
    def visit_mbed_sdk(sdk_file):
        repo_file = join(repo.path, relpath(sdk_file, sdk_path))

        repo_dir = dirname(repo_file)
        if not exists(repo_dir):
            makedirs(repo_dir)

        copy_with_line_endings(sdk_file, repo_file)
    for sdk_path in sdk_paths:
        visit_files(sdk_path, visit_mbed_sdk)

    # remove repository files that do not exist in the mbed SDK
    def visit_repo(repo_file):
        for sdk_path in sdk_paths:
            sdk_file = join(sdk_path, relpath(repo_file, repo.path))
            if exists(sdk_file):
                break
        else:
            remove(repo_file)
            print "remove: %s" % repo_file
    visit_files(repo.path, visit_repo)

    if repo.publish():
        changed.append(repo_name)

def update_code(repositories):
    for r in repositories:
        repo_name, sdk_dir = r[0], r[1]
        team_name = r[2] if len(r) == 3 else None
        print '\n=== Updating "%s" ===' % repo_name
        sdk_dirs = [sdk_dir] if type(sdk_dir) != type([]) else sdk_dir
        update_repo(repo_name, sdk_dirs, team_name)

def update_single_repo(repo):
    repos = [r for r in OFFICIAL_CODE if r[0] == repo]
    if not repos:
        print "Repository '%s' not found" % repo
    else:
        update_code(repos)

def update_dependencies(repositories):
    for repo_name in repositories:
        print '\n=== Updating "%s" ===' % repo_name
        repo = MbedRepository(repo_name)

        # point to the latest libraries
        def visit_repo(repo_file):
            with open(repo_file, "r") as f:
                url = f.read()
            with open(repo_file, "w") as f:
                f.write(url[:(url.rindex('/')+1)])
        visit_files(repo.path, visit_repo, None, MBED_REPO_EXT)

        if repo.publish():
            changed.append(repo_name)

def update_mbed():
    update_repo("mbed", [join(BUILD_DIR, "mbed")], None)

def do_sync(options):
    global push_remote, quiet, commit_msg, changed

    push_remote = not options.nopush
    quiet = options.quiet
    commit_msg = options.msg
    chnaged = []

    if options.code:
        update_code(OFFICIAL_CODE)

    if options.dependencies:
        update_dependencies(CODE_WITH_DEPENDENCIES)

    if options.mbed:
        update_mbed()

    if options.repo:
        update_single_repo(options.repo)

    if changed:
        print "Repositories with changes:", changed

    return changed

if __name__ == '__main__':
    parser = OptionParser()

    parser.add_option("-c", "--code",
                  action="store_true",  default=False,
                  help="Update the mbed_official code")

    parser.add_option("-d", "--dependencies",
                  action="store_true",  default=False,
                  help="Update the mbed_official code dependencies")

    parser.add_option("-m", "--mbed",
                  action="store_true",  default=False,
                  help="Release a build of the mbed library")

    parser.add_option("-n", "--nopush",
                  action="store_true", default=False,
                  help="Commit the changes locally only, don't push them")

    parser.add_option("", "--commit_message",
                  action="store", type="string", default='', dest='msg',
                  help="Commit message to use for all the commits")

    parser.add_option("-r", "--repository",
                  action="store", type="string", default='', dest='repo',
                  help="Synchronize only the given repository")

    parser.add_option("-q", "--quiet",
                  action="store_true", default=False,
                  help="Don't ask for confirmation before commiting or pushing")

    (options, args) = parser.parse_args()

    do_sync(options)

<commit_msg>Synch - fix lwip-eth path<commit_after>"""
mbed SDK
Copyright (c) 2011-2013 ARM Limited

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

One repository to update them all
On mbed.org the mbed SDK is split up in multiple repositories, this script takes
care of updating them all.
"""
import sys
from copy import copy
from os import walk, remove, makedirs
from os.path import join, abspath, dirname, relpath, exists, isfile
from shutil import copyfile
from optparse import OptionParser
import re
import string

ROOT = abspath(join(dirname(__file__), ".."))
sys.path.insert(0, ROOT)

from workspace_tools.settings import MBED_ORG_PATH, MBED_ORG_USER, BUILD_DIR
from workspace_tools.paths import *
from workspace_tools.utils import run_cmd

MBED_URL = "mbed.org"
MBED_USER = "mbed_official"

changed = []
push_remote = True
quiet = False
commit_msg = ''

# Code that does have a mirror in the mbed SDK
# Tuple data: (repo_name, list_of_code_dirs, [team])
# team is optional - if not specified, the code is published under mbed_official
OFFICIAL_CODE = (
    ("mbed-dev" , MBED_BASE),
    ("mbed-rtos", RTOS),
    ("mbed-dsp" , DSP),
    ("mbed-rpc" , MBED_RPC),

    ("lwip"    , LWIP_SOURCES+"/lwip"),
    ("lwip-sys", LWIP_SOURCES+"/lwip-sys"),
    ("Socket"  , LWIP_SOURCES+"/Socket"),

    ("lwip-eth"         , ETH_SOURCES+"/lwip-eth"),
    ("EthernetInterface", ETH_SOURCES+"/EthernetInterface"),

    ("USBDevice", USB),
    ("USBHost"  , USB_HOST),

    ("CellularModem", CELLULAR_SOURCES),
    ("CellularUSBModem", CELLULAR_USB_SOURCES),
    ("UbloxUSBModem", UBLOX_SOURCES),
    ("UbloxModemHTTPClientTest", [TEST_DIR+"/net/cellular/http/common", TEST_DIR+"/net/cellular/http/ubloxusb"]),
    ("UbloxModemSMSTest", [TEST_DIR+"/net/cellular/sms/common", TEST_DIR+"/net/cellular/sms/ubloxusb"]),
    ("FATFileSystem", FAT_FS, "mbed-official"),
)

# Code that does have dependencies to libraries should point to
# the latest revision. By default, they point to a specific revision.
CODE_WITH_DEPENDENCIES = (
    # Libraries
    "EthernetInterface",

    # RTOS Examples
    "rtos_basic",
    "rtos_isr",
    "rtos_mail",
    "rtos_mutex",
    "rtos_queue",
    "rtos_semaphore",
    "rtos_signals",
    "rtos_timer",

    # Net Examples
    "TCPEchoClient",
    "TCPEchoServer",
    "TCPSocket_HelloWorld",
    "UDPSocket_HelloWorld",
    "UDPEchoClient",
    "UDPEchoServer",
    "BroadcastReceive",
    "BroadcastSend",

    # mbed sources
    "mbed-src-program",
)

# A list of regular expressions that will be checked against each directory
# name and skipped if they match.
IGNORE_DIRS = (
)

IGNORE_FILES = (
    'COPYING',
    '\.md',
    "\.lib",
    "\.bld"
)

def ignore_path(name, reg_exps):
    for r in reg_exps:
        if re.search(r, name):
            return True
    return False

class MbedRepository:
    @staticmethod
    def run_and_print(command, cwd):
        stdout, _, _ = run_cmd(command, wd=cwd, redirect=True)
        print(stdout)

    def __init__(self, name, team = None):
        self.name = name
        self.path = join(MBED_ORG_PATH, name)
        if team is None:
            self.url = "http://" + MBED_URL + "/users/" + MBED_USER + "/code/%s/"
        else:
            self.url = "http://" + MBED_URL + "/teams/" + team + "/code/%s/"
        if not exists(self.path):
            # Checkout code
            if not exists(MBED_ORG_PATH):
                makedirs(MBED_ORG_PATH)

            self.run_and_print(['hg', 'clone', self.url % name], cwd=MBED_ORG_PATH)

        else:
            # Update
            self.run_and_print(['hg', 'pull'], cwd=self.path)
            self.run_and_print(['hg', 'update'], cwd=self.path)

    def publish(self):
        # The maintainer has to evaluate the changes first and explicitly accept them
        self.run_and_print(['hg', 'addremove'], cwd=self.path)
        stdout, _, _ = run_cmd(['hg', 'status'], wd=self.path)
        if stdout == '':
            print "No changes"
            return False
        print stdout
        if quiet:
            commit = 'Y'
        else:
            commit = raw_input(push_remote and "Do you want to commit and push? Y/N: " or "Do you want to commit? Y/N: ")
        if commit == 'Y':
            args = ['hg', 'commit', '-u', MBED_ORG_USER]
            if commit_msg:
                args = args + ['-m', commit_msg]
            self.run_and_print(args, cwd=self.path)
            if push_remote:
                self.run_and_print(['hg', 'push'], cwd=self.path)
        return True

# Check if a file is a text file or a binary file
# Taken from http://code.activestate.com/recipes/173220/
text_characters = "".join(map(chr, range(32, 127)) + list("\n\r\t\b"))
_null_trans = string.maketrans("", "")
def is_text_file(filename):
    block_size = 1024
    def istext(s):
        if "\0" in s:
            return 0

        if not s:  # Empty files are considered text
            return 1

        # Get the non-text characters (maps a character to itself then
        # use the 'remove' option to get rid of the text characters.)
        t = s.translate(_null_trans, text_characters)

        # If more than 30% non-text characters, then
        # this is considered a binary file
        if float(len(t))/len(s) > 0.30:
            return 0
        return 1
    with open(filename) as f:
        res = istext(f.read(block_size))
    return res

# Return the line ending type for the given file ('cr' or 'crlf')
def get_line_endings(f):
  examine_size = 1024
  try:
    tf = open(f, "rb")
    lines, ncrlf = tf.readlines(examine_size), 0
    tf.close()
    for l in lines:
      if l.endswith("\r\n"):
        ncrlf = ncrlf + 1
    return 'crlf' if ncrlf > len(lines) >> 1 else 'cr'
  except:
    return 'cr'

# Copy file to destination, but preserve destination line endings if possible
# This prevents very annoying issues with huge diffs that appear because of
# differences in line endings
def copy_with_line_endings(sdk_file, repo_file):
    if not isfile(repo_file):
        copyfile(sdk_file, repo_file)
        return
    is_text = is_text_file(repo_file)
    if is_text:
        sdk_le = get_line_endings(sdk_file)
        repo_le = get_line_endings(repo_file)
    if not is_text or sdk_le == repo_le:
        copyfile(sdk_file, repo_file)
    else:
        print "Converting line endings in '%s' to '%s'" % (abspath(repo_file), repo_le)
        f = open(sdk_file, "rb")
        data = f.read()
        f.close()
        f = open(repo_file, "wb")
        data = data.replace("\r\n", "\n") if repo_le == 'cr' else data.replace('\n','\r\n')
        f.write(data)
        f.close()

def visit_files(path, visit):
    for root, dirs, files in walk(path):
        # Ignore hidden directories
        for d in copy(dirs):
            full = join(root, d)
            if d.startswith('.'):
                dirs.remove(d)
            if ignore_path(full, IGNORE_DIRS):
                print "Skipping '%s'" % full
                dirs.remove(d)

        for file in files:
            if ignore_path(file, IGNORE_FILES):
                continue

            visit(join(root, file))

def update_repo(repo_name, sdk_paths, team_name):
    repo = MbedRepository(repo_name, team_name)
    # copy files from mbed SDK to mbed_official repository
    def visit_mbed_sdk(sdk_file):
        repo_file = join(repo.path, relpath(sdk_file, sdk_path))

        repo_dir = dirname(repo_file)
        if not exists(repo_dir):
            makedirs(repo_dir)

        copy_with_line_endings(sdk_file, repo_file)
    for sdk_path in sdk_paths:
        visit_files(sdk_path, visit_mbed_sdk)

    # remove repository files that do not exist in the mbed SDK
    def visit_repo(repo_file):
        for sdk_path in sdk_paths:
            sdk_file = join(sdk_path, relpath(repo_file, repo.path))
            if exists(sdk_file):
                break
        else:
            remove(repo_file)
            print "remove: %s" % repo_file
    visit_files(repo.path, visit_repo)

    if repo.publish():
        changed.append(repo_name)

def update_code(repositories):
    for r in repositories:
        repo_name, sdk_dir = r[0], r[1]
        team_name = r[2] if len(r) == 3 else None
        print '\n=== Updating "%s" ===' % repo_name
        sdk_dirs = [sdk_dir] if type(sdk_dir) != type([]) else sdk_dir
        update_repo(repo_name, sdk_dirs, team_name)

def update_single_repo(repo):
    repos = [r for r in OFFICIAL_CODE if r[0] == repo]
    if not repos:
        print "Repository '%s' not found" % repo
    else:
        update_code(repos)

def update_dependencies(repositories):
    for repo_name in repositories:
        print '\n=== Updating "%s" ===' % repo_name
        repo = MbedRepository(repo_name)

        # point to the latest libraries
        def visit_repo(repo_file):
            with open(repo_file, "r") as f:
                url = f.read()
            with open(repo_file, "w") as f:
                f.write(url[:(url.rindex('/')+1)])
        visit_files(repo.path, visit_repo, None, MBED_REPO_EXT)

        if repo.publish():
            changed.append(repo_name)

def update_mbed():
    update_repo("mbed", [join(BUILD_DIR, "mbed")], None)

def do_sync(options):
    global push_remote, quiet, commit_msg, changed

    push_remote = not options.nopush
    quiet = options.quiet
    commit_msg = options.msg
    chnaged = []

    if options.code:
        update_code(OFFICIAL_CODE)

    if options.dependencies:
        update_dependencies(CODE_WITH_DEPENDENCIES)

    if options.mbed:
        update_mbed()

    if options.repo:
        update_single_repo(options.repo)

    if changed:
        print "Repositories with changes:", changed

    return changed

if __name__ == '__main__':
    parser = OptionParser()

    parser.add_option("-c", "--code",
                  action="store_true",  default=False,
                  help="Update the mbed_official code")

    parser.add_option("-d", "--dependencies",
                  action="store_true",  default=False,
                  help="Update the mbed_official code dependencies")

    parser.add_option("-m", "--mbed",
                  action="store_true",  default=False,
                  help="Release a build of the mbed library")

    parser.add_option("-n", "--nopush",
                  action="store_true", default=False,
                  help="Commit the changes locally only, don't push them")

    parser.add_option("", "--commit_message",
                  action="store", type="string", default='', dest='msg',
                  help="Commit message to use for all the commits")

    parser.add_option("-r", "--repository",
                  action="store", type="string", default='', dest='repo',
                  help="Synchronize only the given repository")

    parser.add_option("-q", "--quiet",
                  action="store_true", default=False,
                  help="Don't ask for confirmation before commiting or pushing")

    (options, args) = parser.parse_args()

    do_sync(options)

<|endoftext|>


<commit_before># encoding: utf-8

from attest import (
    Tests, assert_hook,
    contextmanager, raises,
)
import rdflib
from lkbutils import nodeprovider
from lkbutils.nodeprovider import kakasicall

nameprovider_unit = Tests()
kakasi_unit = Tests()
nodeprovider_unit = Tests()

@contextmanager
def empty_nameprovider(romanize=False):
    try:
        yield nodeprovider.NameProvider(romanize=romanize)
    finally:
        pass

@contextmanager
def empty_rdflib_nodeprovider(romanize=True):
    try:
        yield nodeprovider.RDFLibNodeProvider(
            romanize=romanize,
        )
    finally:
        pass

class Fixtures:
    """Namespace for test fixtures."""

    class simple_names:
        formalized_map = {
            u'John': u'john',
            u'Ken': u'ken',
            u'Yumi is waiting at the door': u'yumi_is_waiting_at_the_door',
        }
        not_added = u'poupou'
        invalid_names = [
            u'3way',
            u'魂',
            u"I'm fine",
        ]

    class kana_kanji_names:
        formalized_map = {
            u'魂': u'tamashii',
            u'抵当権': u'teitouken',
            u'補助開始の審判': u'hojokaishinoshinpan',
        }
        not_added = u'hitanposaiken'

    class simple_nodenames:
        formalized_map = {
            u'John': u'john',
            u'魂': u'tamashii',
        }
        invalid_name = u'3way'
        not_added = u'詐害行為取消権'

    class simple_properties:
        names = [u'part_of', u'hyper']

    class kakasi_conversion:
        desired_conversion = {
            u'抵当権': u'teitouken',
            u'国民の休日': u'kokuminnokyuujitsu',
            u'公共のベランダ': u'koukyounoberanda',
        }

# Adding names.

@nameprovider_unit.test
def add_simple_names():
    """NameProvider.add."""
    with empty_nameprovider() as provider:

        for name in Fixtures.simple_names.formalized_map:
            ret = provider.add(name)
            expected_formalized = Fixtures.simple_names.formalized_map[name]
            assert ret == expected_formalized

        for name in Fixtures.simple_names.formalized_map:
            with raises(nodeprovider.NameConflict):
                provider.add(name)

        namespace = provider.ns
        for name in Fixtures.simple_names.formalized_map:
            formalized_name = Fixtures.simple_names.formalized_map[name]
            assert formalized_name in namespace
            assert getattr(namespace, formalized_name) == name
            assert provider.get_ns_identifier(name) == formalized_name

        with raises(nodeprovider.NameNotRegistered):
            provider.get_ns_identifier(Fixtures.simple_names.not_added)

@nameprovider_unit.test
def add_invalid_names():
    """NameProvider.add refuses invalid names."""
    with empty_nameprovider() as provider:
        for invalid_name in Fixtures.simple_names.invalid_names:
            with raises(nodeprovider.InvalidName):
                provider.add(invalid_name)

@nameprovider_unit.test
def handle_kana_name_addition():
    """
    NameProvider with romanize=True accepts kanji/kana names.
    """
    with empty_nameprovider(romanize=True) as provider:

        for name in Fixtures.kana_kanji_names.formalized_map:
            ret = provider.add(name)
            expected_formalized = Fixtures.kana_kanji_names.formalized_map[name]
            assert ret == expected_formalized

        for name in Fixtures.kana_kanji_names.formalized_map:
            formalized_name = Fixtures.kana_kanji_names.formalized_map[name]
            with raises(nodeprovider.NameConflict):
                provider.add(name)
            with raises(nodeprovider.NameConflict):
                provider.add(formalized_name)

        namespace = provider.ns
        for name in Fixtures.kana_kanji_names.formalized_map:
            formalized_name = Fixtures.kana_kanji_names.formalized_map[name]
            assert getattr(namespace, formalized_name) == name
            assert provider.get_ns_identifier(name) == formalized_name

        with raises(nodeprovider.NameNotRegistered):
            provider.get_ns_identifier(Fixtures.kana_kanji_names.not_added)

# simple kakasi caller.
@kakasi_unit.test
def kakasi_conversino():
    """Roughly check kakasi conversion."""
    for target in Fixtures.kakasi_conversion.desired_conversion:
        yomi = Fixtures.kakasi_conversion.desired_conversion[target]
        assert kakasicall.romanize(target) == yomi

# Adding nodes.

@nodeprovider_unit.test
def nodeprovider_on_toplevel():
    """NodeProvider subclasses are accessible on pkg's toplevel ns."""
    from lkbutils import RDFLibNodeProvider

@nodeprovider_unit.test
def add_nodes():
    """(.*)NodeProvider.add."""

    with empty_rdflib_nodeprovider() as provider:

        for name in Fixtures.simple_nodenames.formalized_map:
            ret = provider.add(name)
            formalized_name = Fixtures.simple_nodenames.formalized_map[name]
            assert isinstance(ret, provider.classes['bnode'])
            assert getattr(provider.ns, formalized_name) == ret
            assert formalized_name in provider.ns
            assert (
                (ret, rdflib.RDFS.label, rdflib.Literal(name))
                in list(provider.graph.triples((None, None, None)))
            )
            assert provider.get(name) == ret

        with raises(nodeprovider.InvalidName):
            provider.add(Fixtures.simple_nodenames.invalid_name)

        with raises(nodeprovider.NameNotRegistered):
            provider.get(Fixtures.simple_nodenames.invalid_name)

@nodeprovider_unit.test
def add_nodes_as_properties():
    """(.*)NodeProvider.add(..., as_property=True)."""

    with empty_rdflib_nodeprovider() as provider:

        for propname in Fixtures.simple_properties.names:
            ret = provider.add(propname, as_property=True)
            assert isinstance(ret, provider.classes['bnode'])
            assert getattr(provider.ns, propname) == ret
            assert propname in provider.ns
            assert (
                (ret, rdflib.RDFS.label, rdflib.Literal(propname))
                in list(provider.graph.triples((None, None, None)))
            )
            assert (
                (ret, rdflib.RDF.type, rdflib.RDF.Property)
                in list(provider.graph.triples((None, None, None)))
            )
            assert provider.get(propname) == ret
<commit_msg>A few annotations + typo fix for tests.nodeprovider.<commit_after># encoding: utf-8

from attest import (
    Tests, assert_hook,
    contextmanager, raises,
)
import rdflib
from lkbutils import nodeprovider
from lkbutils.nodeprovider import kakasicall

nameprovider_unit = Tests()
kakasi_unit = Tests()
nodeprovider_unit = Tests()

@contextmanager
def empty_nameprovider(romanize=False):
    try:
        yield nodeprovider.NameProvider(romanize=romanize)
    finally:
        pass

@contextmanager
def empty_rdflib_nodeprovider(romanize=True):
    try:
        yield nodeprovider.RDFLibNodeProvider(
            romanize=romanize,
        )
    finally:
        pass

class Fixtures:
    """Namespace for test fixtures."""

    class simple_names:
        formalized_map = {
            u'John': u'john',
            u'Ken': u'ken',
            u'Yumi is waiting at the door': u'yumi_is_waiting_at_the_door',
        }
        not_added = u'poupou'
        invalid_names = [
            u'3way',
            u'魂',
            u"I'm fine",
        ]

    class kana_kanji_names:
        formalized_map = {
            u'魂': u'tamashii',
            u'抵当権': u'teitouken',
            u'補助開始の審判': u'hojokaishinoshinpan',
        }
        not_added = u'hitanposaiken'

    class simple_nodenames:
        formalized_map = {
            u'John': u'john',
            u'魂': u'tamashii',
        }
        invalid_name = u'3way'
        not_added = u'詐害行為取消権'

    class simple_properties:
        names = [u'part_of', u'hyper']

    class kakasi_conversion:
        desired_conversion = {
            u'抵当権': u'teitouken',
            u'国民の休日': u'kokuminnokyuujitsu',
            u'公共のベランダ': u'koukyounoberanda',
        }

# Adding names.

@nameprovider_unit.test
def add_simple_names():
    """NameProvider.add."""
    with empty_nameprovider() as provider:

        # additions & returned values
        for name in Fixtures.simple_names.formalized_map:
            ret = provider.add(name)
            expected_formalized = Fixtures.simple_names.formalized_map[name]
            assert ret == expected_formalized

        # avoiding name conflicts
        for name in Fixtures.simple_names.formalized_map:
            with raises(nodeprovider.NameConflict):
                provider.add(name)

        # namespace & reverse lookup
        namespace = provider.ns
        for name in Fixtures.simple_names.formalized_map:
            formalized_name = Fixtures.simple_names.formalized_map[name]
            assert formalized_name in namespace
            assert getattr(namespace, formalized_name) == name
            assert provider.get_ns_identifier(name) == formalized_name

        # reference error
        with raises(nodeprovider.NameNotRegistered):
            provider.get_ns_identifier(Fixtures.simple_names.not_added)

@nameprovider_unit.test
def add_invalid_names():
    """NameProvider.add refuses invalid names."""
    with empty_nameprovider() as provider:
        for invalid_name in Fixtures.simple_names.invalid_names:
            with raises(nodeprovider.InvalidName):
                provider.add(invalid_name)

@nameprovider_unit.test
def handle_kana_name_addition():
    """
    NameProvider with romanize=True accepts kanji/kana names.
    """
    with empty_nameprovider(romanize=True) as provider:

        # additions & returned values
        for name in Fixtures.kana_kanji_names.formalized_map:
            ret = provider.add(name)
            expected_formalized = Fixtures.kana_kanji_names.formalized_map[name]
            assert ret == expected_formalized

        # avoiding name conflicts
        for name in Fixtures.kana_kanji_names.formalized_map:
            formalized_name = Fixtures.kana_kanji_names.formalized_map[name]
            with raises(nodeprovider.NameConflict):
                provider.add(name)
            with raises(nodeprovider.NameConflict):
                provider.add(formalized_name)

        # namespace & reverse lookup
        namespace = provider.ns
        for name in Fixtures.kana_kanji_names.formalized_map:
            formalized_name = Fixtures.kana_kanji_names.formalized_map[name]
            assert getattr(namespace, formalized_name) == name
            assert provider.get_ns_identifier(name) == formalized_name

        # reference error
        with raises(nodeprovider.NameNotRegistered):
            provider.get_ns_identifier(Fixtures.kana_kanji_names.not_added)

# simple kakasi caller.
@kakasi_unit.test
def kakasi_conversion():
    """Roughly check kakasi conversion."""
    for target in Fixtures.kakasi_conversion.desired_conversion:
        yomi = Fixtures.kakasi_conversion.desired_conversion[target]
        assert kakasicall.romanize(target) == yomi

# Adding nodes.

@nodeprovider_unit.test
def nodeprovider_on_toplevel():
    """NodeProvider subclasses are accessible on pkg's toplevel ns."""
    from lkbutils import RDFLibNodeProvider

@nodeprovider_unit.test
def add_nodes():
    """(.*)NodeProvider.add."""

    with empty_rdflib_nodeprovider() as provider:

        # additions & returned values
        for name in Fixtures.simple_nodenames.formalized_map:
            ret = provider.add(name)
            formalized_name = Fixtures.simple_nodenames.formalized_map[name]
            assert isinstance(ret, provider.classes['bnode'])
            assert getattr(provider.ns, formalized_name) == ret
            assert formalized_name in provider.ns
            assert (
                (ret, rdflib.RDFS.label, rdflib.Literal(name))
                in list(provider.graph.triples((None, None, None)))
            )
            assert provider.get(name) == ret

        # avoiding name conflicts
        with raises(nodeprovider.InvalidName):
            provider.add(Fixtures.simple_nodenames.invalid_name)

        # reverse lookup
        with raises(nodeprovider.NameNotRegistered):
            provider.get(Fixtures.simple_nodenames.invalid_name)

@nodeprovider_unit.test
def add_nodes_as_properties():
    """(.*)NodeProvider.add(..., as_property=True)."""

    with empty_rdflib_nodeprovider() as provider:

        for propname in Fixtures.simple_properties.names:
            ret = provider.add(propname, as_property=True)
            assert isinstance(ret, provider.classes['bnode'])
            assert getattr(provider.ns, propname) == ret
            assert propname in provider.ns
            assert (
                (ret, rdflib.RDFS.label, rdflib.Literal(propname))
                in list(provider.graph.triples((None, None, None)))
            )
            assert (
                (ret, rdflib.RDF.type, rdflib.RDF.Property)
                in list(provider.graph.triples((None, None, None)))
            )
            assert provider.get(propname) == ret
<|endoftext|>


<commit_before>"""Utility methods for computing evaluating metrics. All methods assumes greater
scores for better matches, and assumes label == 1 means match.
"""
import operator

def ErrorRateAt95Recall(labels, scores):
    recall_point = 0.95
    # Sort label-score tuples by the score in descending order.
    sorted_scores = zip(labels, scores)
    sorted_scores.sort(key=operator.itemgetter(1), reverse=True)

    # Compute error rate
    n_match = sum(1 for x in sorted_scores if x[0] == 1)
    n_thresh = recall_point * n_match
    tp = 0
    count = 0
    for label, score in sorted_scores:
        count += 1
        if label == 1:
            tp += 1
        if tp >= n_thresh:
            break

    return float(count - tp) / count
<commit_msg>fix reverse parameter and add reference<commit_after>"""Utility methods for computing evaluating metrics. All methods assumes greater
scores for better matches, and assumes label == 1 means match.

Extracted from:
https://github.com/hanxf/matchnet/blob/master/eval_metrics.py
"""
import operator

def ErrorRateAt95Recall(labels, scores):
    recall_point = 0.95
    # Sort label-score tuples by the score in descending order.
    sorted_scores = zip(labels, scores)
    sorted_scores.sort(key=operator.itemgetter(1), reverse=False)

    # Compute error rate
    n_match = sum(1 for x in sorted_scores if x[0] == 1)
    n_thresh = recall_point * n_match
    tp = 0
    count = 0
    for label, score in sorted_scores:
        count += 1
        if label == 1:
            tp += 1
        if tp >= n_thresh:
            break

    return float(count - tp) / count
<|endoftext|>


<commit_before>"""
mbed SDK
Copyright (c) 2011-2013 ARM Limited

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import sys
import inspect
import os
from os import listdir, remove, makedirs
from shutil import copyfile
from os.path import isdir, join, exists, split, relpath, splitext
from subprocess import Popen, PIPE, STDOUT, call

def cmd(l, check=True, verbose=False, shell=False, cwd=None):
    text = l if shell else ' '.join(l)
    if verbose:
        print text
    rc = call(l, shell=shell, cwd=cwd)
    if check and rc != 0:
        raise Exception('ERROR %d: "%s"' % (rc, text))

def run_cmd(command, wd=None, redirect=False):
    assert is_cmd_valid(command[0])
    p = Popen(command, stdout=PIPE, stderr=STDOUT if redirect else PIPE, cwd=wd)
    _stdout, _stderr = p.communicate()
    return _stdout, _stderr, p.returncode

def run_cmd_ext(command):
    assert is_cmd_valid(command[0])
    p = Popen(command, stdout=PIPE, stderr=PIPE)
    _stdout, _stderr = p.communicate()
    return _stdout, _stderr, p.returncode

def is_cmd_valid(cmd):
    caller = get_caller_name()
    abspath = find_cmd_abspath(cmd)
    if not abspath:
        error("%s: Command '%s' can't be found" % (caller, cmd))
    if not is_exec(abspath):
        error("%s: Command '%s' resolves to file '%s' which is not executable" % (caller, cmd, abspath))
    return True

def is_exec(path):
    return os.access(path, os.X_OK)

def find_cmd_abspath(cmd):
    """ Returns the absolute path to a command.
        None is returned if no absolute path was found.
    """
    if exists(cmd):
        return '%s/%s' % (os.getcwd(), cmd)
    if not 'PATH' in os.environ:
        raise Exception("Can't find command path for current platform ('%s')" % sys.platform)
    PATH=os.environ['PATH']
    for path in PATH.split(os.pathsep):
        abspath = '%s/%s' % (path, cmd)
        if exists(abspath):
            return abspath

def mkdir(path):
    if not exists(path):
        makedirs(path)

def copy_file(src, dst):
    """ Implement the behaviour of "shutil.copy(src, dst)" without copying the
        permissions (this was causing errors with directories mounted with samba)
    """
    if isdir(dst):
        _, file = split(src)
        dst = join(dst, file)
    copyfile(src, dst)

def delete_dir_files(dir):
    if not exists(dir):
        return

    for f in listdir(dir):
        file = join(dir, f)
        if not isdir(file):
            remove(file)

def get_caller_name(steps=2):
    """
    When called inside a function, it returns the name
    of the caller of that function.
    """
    return inspect.stack()[steps][3]

def error(msg):
    print msg
    sys.exit(1)

def rel_path(path, base, dot=False):
    p = relpath(path, base)
    if dot and not p.startswith('.'):
        p = './' + p
    return p

class ToolException(Exception):
    pass

def split_path(path):
    base, file = split(path)
    name, ext = splitext(file)
    return base, name, ext

def args_error(parser, message):
    print "\n\n%s\n\n" % message
    parser.print_help()
    sys.exit()

def construct_enum(**enums):
    """ Create your own pseudo-enums """
    return type('Enum', (), enums)

def check_required_modules(required_modules, verbose=True):
    """ Function checks for Python modules which should be "importable" (installed)
        before test suite can be used.
        @return returns True if all modules are installed already
    """
    import imp
    not_installed_modules = []
    for module_name in required_modules:
        try:
            imp.find_module(module_name)
        except ImportError as e:
            # We also test against a rare case: module is an egg file
            try:
                __import__(module_name)
            except ImportError as e:
                not_installed_modules.append(module_name)
                if verbose:
                    print "Error: %s" % e

    if verbose:
        if not_installed_modules:
            print "Warning: Module(s) %s not installed. Please install required module(s) before using this script."% (', '.join(not_installed_modules))

    if not_installed_modules:
        return False
    else:
        return True
<commit_msg>Small refactoring<commit_after>"""
mbed SDK
Copyright (c) 2011-2013 ARM Limited

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import sys
import inspect
import os
from os import listdir, remove, makedirs
from shutil import copyfile
from os.path import isdir, join, exists, split, relpath, splitext
from subprocess import Popen, PIPE, STDOUT, call

def cmd(l, check=True, verbose=False, shell=False, cwd=None):
    text = l if shell else ' '.join(l)
    if verbose:
        print text
    rc = call(l, shell=shell, cwd=cwd)
    if check and rc != 0:
        raise Exception('ERROR %d: "%s"' % (rc, text))

def run_cmd(command, wd=None, redirect=False):
    assert is_cmd_valid(command[0])
    p = Popen(command, stdout=PIPE, stderr=STDOUT if redirect else PIPE, cwd=wd)
    _stdout, _stderr = p.communicate()
    return _stdout, _stderr, p.returncode

def run_cmd_ext(command):
    assert is_cmd_valid(command[0])
    p = Popen(command, stdout=PIPE, stderr=PIPE)
    _stdout, _stderr = p.communicate()
    return _stdout, _stderr, p.returncode

def is_cmd_valid(cmd):
    caller = get_caller_name()
    abspath = find_cmd_abspath(cmd)
    if not abspath:
        error("%s: Command '%s' can't be found" % (caller, cmd))
    if not is_exec(abspath):
        error("%s: Command '%s' resolves to file '%s' which is not executable" % (caller, cmd, abspath))
    return True

def is_exec(path):
    return os.access(path, os.X_OK)

def find_cmd_abspath(cmd):
    """ Returns the absolute path to a command.
        None is returned if no absolute path was found.
    """
    if exists(cmd):
        return os.path.abspath(cmd)
    if not 'PATH' in os.environ:
        raise Exception("Can't find command path for current platform ('%s')" % sys.platform)
    PATH=os.environ['PATH']
    for path in PATH.split(os.pathsep):
        abspath = '%s/%s' % (path, cmd)
        if exists(abspath):
            return abspath

def mkdir(path):
    if not exists(path):
        makedirs(path)

def copy_file(src, dst):
    """ Implement the behaviour of "shutil.copy(src, dst)" without copying the
        permissions (this was causing errors with directories mounted with samba)
    """
    if isdir(dst):
        _, file = split(src)
        dst = join(dst, file)
    copyfile(src, dst)

def delete_dir_files(dir):
    if not exists(dir):
        return

    for f in listdir(dir):
        file = join(dir, f)
        if not isdir(file):
            remove(file)

def get_caller_name(steps=2):
    """
    When called inside a function, it returns the name
    of the caller of that function.
    """
    return inspect.stack()[steps][3]

def error(msg):
    print("ERROR: %s" % msg)
    sys.exit(1)

def rel_path(path, base, dot=False):
    p = relpath(path, base)
    if dot and not p.startswith('.'):
        p = './' + p
    return p

class ToolException(Exception):
    pass

def split_path(path):
    base, file = split(path)
    name, ext = splitext(file)
    return base, name, ext

def args_error(parser, message):
    print "\n\n%s\n\n" % message
    parser.print_help()
    sys.exit()

def construct_enum(**enums):
    """ Create your own pseudo-enums """
    return type('Enum', (), enums)

def check_required_modules(required_modules, verbose=True):
    """ Function checks for Python modules which should be "importable" (installed)
        before test suite can be used.
        @return returns True if all modules are installed already
    """
    import imp
    not_installed_modules = []
    for module_name in required_modules:
        try:
            imp.find_module(module_name)
        except ImportError as e:
            # We also test against a rare case: module is an egg file
            try:
                __import__(module_name)
            except ImportError as e:
                not_installed_modules.append(module_name)
                if verbose:
                    print "Error: %s" % e

    if verbose:
        if not_installed_modules:
            print "Warning: Module(s) %s not installed. Please install required module(s) before using this script."% (', '.join(not_installed_modules))

    if not_installed_modules:
        return False
    else:
        return True
<|endoftext|>