<commit_before># -*- coding: utf-8 -*-
from __future__ import (
    division, absolute_import, print_function, unicode_literals,
)
from builtins import *                  # noqa
from future.builtins.disabled import *  # noqa
from future.utils import with_metaclass

from abc import ABCMeta
# collections.abc dosn't esist in Python 2.x.
import collections as abc

from magic_constraints.exception import MagicTypeError
from magic_constraints.utils import (
    type_object, nontype_object,
    repr_return, conditional_repr,
)

def unbound_getattr(target, name):
    return target.__dict__.get(name, None)

def CreateMetaMagicType(generator_cls):

    class MetaMagicType(ABCMeta):

        def __getitem__(cls, type_decl):

            type_decl_checker = unbound_getattr(
                generator_cls,
                'check_getitem_type_decl',
            )
            if type_decl_checker and not type_decl_checker(type_decl):
                raise MagicTypeError(
                    'invalid type.',
                    type_decl=type_decl,
                )

            ret_cls = generator_cls(cls.main_cls)
            ret_cls.partial_cls = type_decl
            return ret_cls

        # fix unbound error of Python 2.x.
        __instancecheck__ = unbound_getattr(generator_cls, '__instancecheck__')

        def __subclasscheck__(cls, subclass):
            if nontype_object(subclass):
                return False

            if unbound_getattr(generator_cls, '__subclasshook__'):
                subclasshook = unbound_getattr(
                    generator_cls, '__subclasshook__',
                )
                if not subclasshook(cls, subclass):
                    return False

            # corner case, subclass isn't MagicType.
            if not hasattr(subclass, 'partial_cls'):
                return issubclass(subclass, cls.main_cls)

            # subclass is MagicType.
            if cls.partial_cls or subclass.partial_cls:
                # if subclass has partial_cls, return False.
                return False
            else:
                # 1. subclass is normal type object.
                # 2. subclass is a MagicType.
                return issubclass(subclass.main_cls, cls.main_cls)

        def __repr__(cls):

            name = conditional_repr(cls.main_cls)
            if cls.partial_cls:
                partial = ', '.join(
                    map(
                        conditional_repr,
                        cls.partial_cls
                        if isinstance(cls.partial_cls, tuple)
                        else [cls.partial_cls],
                    ),
                )

                name = '{0}[{1}]'.format(
                    name, partial,
                )

            return repr_return(name)

    return MetaMagicType

def CreateMagicType(generator_cls, MetaMagicType, ABC):

    class MagicType(with_metaclass(MetaMagicType, object)):

        main_cls = ABC
        partial_cls = None

        for attr in [
            '__new__',
            '__init__',
            # iterator.
            '__iter__',
            '__next__',
            # callable.
            '__call__',
        ]:
            if unbound_getattr(generator_cls, attr):
                locals()[attr] = unbound_getattr(generator_cls, attr)

    return MagicType

class MagicTypeGenerator(type):

    def __new__(cls, ABC):
        MetaMagicType = CreateMetaMagicType(cls)
        return CreateMagicType(cls, MetaMagicType, ABC)

def check_type_of_instance(cls, instance):
    return any(
        issubclass(T, cls)
        # handle old style class.
        for T in (instance.__class__, type(instance))
    )

def check_getitem_tuple(type_decl, n):
    # type_decl should be a n-tuple.
    if not isinstance(type_decl, tuple):
        return False
    return len(type_decl) == n

def generate_immutable_abc(supercls, mutable_subclass):

    class ABCImmutableMeta(ABCMeta):

        def __subclasscheck__(cls, subclass):
            if not issubclass(subclass, supercls):
                return False
            return not issubclass(subclass, mutable_subclass)

    class ABCImmutable(with_metaclass(ABCImmutableMeta, object)):
        pass

    # dirty hack to assert issubclass(ABCImmutable, supercls).
    supercls._abc_cache.add(ABCImmutable)

    return ABCImmutable

class SequenceGenerator(MagicTypeGenerator):

    def check_getitem_type_decl(type_decl):
        if type_object(type_decl):
            return True

        if isinstance(type_decl, tuple):
            for T in type_decl:
                if nontype_object(T):
                    return False
            return True
        else:
            return False

    def __instancecheck__(cls, instance):
        if not check_type_of_instance(cls, instance):
            return False

        if not cls.partial_cls:
            return True

        if type_object(cls.partial_cls):
            for e in instance:
                if not isinstance(e, cls.partial_cls):
                    return False
        else:
            if len(cls.partial_cls) != len(instance):
                return False
            for i in range(len(instance)):
                if not isinstance(instance[i], cls.partial_cls[i]):
                    return False

        return True

class SetGenerator(MagicTypeGenerator):

    def check_getitem_type_decl(type_decl):
        return type_object(type_decl)

    def __instancecheck__(cls, instance):
        if not check_type_of_instance(cls, instance):
            return False

        if not cls.partial_cls:
            return True

        if type_object(cls.partial_cls):
            for e in instance:
                if not isinstance(e, cls.partial_cls):
                    return False

        return True

class MappingGenerator(MagicTypeGenerator):

    def check_getitem_type_decl(type_decl):
        if not check_getitem_tuple(type_decl, 2):
            return False

        return type_object(type_decl[0]) and type_object(type_decl[1])

    def __instancecheck__(cls, instance):
        if not check_type_of_instance(cls, instance):
            return False

        if cls.partial_cls:
            key_cls, val_cls = cls.partial_cls
            for key, val in instance.items():
                if not (isinstance(key, key_cls) and isinstance(val, val_cls)):
                    return False
        return True

class IteratorGenerator(MagicTypeGenerator):

    def check_getitem_type_decl(type_decl):
        return type_object(type_decl)

    def __init__(self, iterator):
        if self.partial_cls is None:
            raise MagicTypeError(
                'require T on Iterator[T].'
            )
        if not isinstance(iterator, abc.Iterator):
            raise MagicTypeError(
                'require Iterator.',
                iterator=iterator,
            )

        self.iterator = iterator

    def __iter__(self):
        return self

    def __next__(self):
        element = next(self.iterator)
        if isinstance(element, self.partial_cls):
            return element
        else:
            raise MagicTypeError(
                'type unmatched.',
                element=element,
                type_=self.partial_cls,
            )

    def __instancecheck__(cls, instance):
        return check_type_of_instance(cls, instance)

class IterableGenerator(MagicTypeGenerator):

    def check_getitem_type_decl(type_decl):
        return type_object(type_decl)

    def __init__(self, iterable):
        if self.partial_cls is None:
            raise MagicTypeError(
                'require T on Iterable[T].'
            )
        if not isinstance(iterable, abc.Iterable):
            raise MagicTypeError(
                'require Iterable.',
                iterable=iterable,
            )
        self.iterable = iterable

    def __iter__(self):
        return Iterator[self.partial_cls](
            iter(self.iterable),
        )

    def __instancecheck__(cls, instance):
        return check_type_of_instance(cls, instance)

class CallableGenerator(MagicTypeGenerator):

    def check_getitem_type_decl(type_decl):
        # Callable[[T, ...], T]
        if not check_getitem_tuple(type_decl, 2):
            return False
        if not isinstance(type_decl[0], abc.Iterable) or\
                nontype_object(type_decl[1]):
            return False
        for T in type_decl[0]:
            if nontype_object(T):
                return False

        return True

    def __init__(self, instance):
        # 1. not Callable.
        if not isinstance(instance, self.main_cls):
            raise MagicTypeError(
                'instance should be a Callable.',
                instance=instance,
            )
        # 2. with no specification.
        if not self.partial_cls:
            raise MagicSyntaxError(
                'Callable should be specified to wrap the callable.',
            )

        parameters_types, return_type = self.partial_cls
        self._wrapper = function_constraints(
            *parameters_types,
            return_type=return_type,
        )(instance)

    def __call__(self, *args, **kwargs):
        return self._wrapper(*args, **kwargs)

    def __instancecheck__(cls, instance):
        if cls.partial_cls or not check_type_of_instance(cls, instance):
            return False
        else:
            # is callable and not Callable[T, ...].
            return True

class AnyMeta(ABCMeta):

    def __instancecheck__(cls, instance):
        return True

    def __subclasscheck__(cls, subclass):
        return type_object(subclass)

class Any(with_metaclass(AnyMeta, object)):
    pass

class UnionGenerator(MagicTypeGenerator):

    def check_getitem_type_decl(type_decl):
        if type_object(type_decl) or not isinstance(type_decl, tuple):
            return False

        for T in type_decl:
            if nontype_object(T):
                return False
        else:
            return True

    def __subclasshook__(cls, subclass):
        return False

    def __instancecheck__(cls, instance):
        if cls.partial_cls is None:
            return False

        for candidate_cls in cls.partial_cls:
            if isinstance(instance, candidate_cls):
                return True
        return False

class OptionalGenerator(MagicTypeGenerator):

    def check_getitem_type_decl(type_decl):
        return type_object(type_decl)

    def __subclasshook__(cls, subclass):
        return False

    def __instancecheck__(cls, instance):
        if cls.partial_cls is None:
            return False

        if instance is None:
            return True
        else:
            return isinstance(instance, cls.partial_cls)

ABCImmutableSequence = generate_immutable_abc(
    abc.Sequence, abc.MutableSequence,
)
ABCImmutableSet = generate_immutable_abc(
    abc.Set, abc.MutableSet,
)
ABCImmutableMapping = generate_immutable_abc(
    abc.Mapping, abc.MutableMapping,
)

Sequence = SequenceGenerator(abc.Sequence)
MutableSequence = SequenceGenerator(abc.MutableSequence)
ImmutableSequence = SequenceGenerator(ABCImmutableSequence)

Set = SetGenerator(abc.Set)
MutableSet = SetGenerator(abc.MutableSet)
ImmutableSet = SetGenerator(ABCImmutableSet)

Mapping = MappingGenerator(abc.Mapping)
MutableMapping = MappingGenerator(abc.MutableMapping)
ImmutableMapping = MappingGenerator(ABCImmutableMapping)

Iterator = IteratorGenerator(abc.Iterator)
Iterable = IterableGenerator(abc.Iterable)

Callable = CallableGenerator(abc.Callable)

Union = UnionGenerator(Any)
Optional = OptionalGenerator(Any)
NoneType = type(None)

from magic_constraints.decorator import function_constraints  # noqa
<commit_msg>Bug fixed.<commit_after># -*- coding: utf-8 -*-
from __future__ import (
    division, absolute_import, print_function, unicode_literals,
)
from builtins import *                  # noqa
from future.builtins.disabled import *  # noqa
from future.utils import with_metaclass

from abc import ABCMeta
# collections.abc dosn't esist in Python 2.x.
import collections as abc

from magic_constraints.exception import MagicTypeError
from magic_constraints.utils import (
    type_object, nontype_object,
    repr_return, conditional_repr,
)

def unbound_getattr(target, name):
    return target.__dict__.get(name, None)

def CreateMetaMagicType(generator_cls):

    class MetaMagicType(ABCMeta):

        def __getitem__(cls, type_decl):

            type_decl_checker = unbound_getattr(
                generator_cls,
                'check_getitem_type_decl',
            )
            if type_decl_checker and not type_decl_checker(type_decl):
                raise MagicTypeError(
                    'invalid type.',
                    type_decl=type_decl,
                )

            ret_cls = generator_cls(cls.main_cls)
            ret_cls.partial_cls = type_decl
            return ret_cls

        # fix unbound error of Python 2.x.
        __instancecheck__ = unbound_getattr(generator_cls, '__instancecheck__')

        def __subclasscheck__(cls, subclass):
            if nontype_object(subclass):
                return False

            if unbound_getattr(generator_cls, '__subclasshook__'):
                subclasshook = unbound_getattr(
                    generator_cls, '__subclasshook__',
                )
                if not subclasshook(cls, subclass):
                    return False

            # corner case, subclass isn't MagicType.
            if not hasattr(subclass, 'partial_cls'):
                return issubclass(subclass, cls.main_cls)

            # subclass is MagicType.
            if cls.partial_cls or subclass.partial_cls:
                # if subclass has partial_cls, return False.
                return False
            else:
                # 1. subclass is normal type object.
                # 2. subclass is a MagicType.
                return issubclass(subclass.main_cls, cls.main_cls)

        def __repr__(cls):

            name = conditional_repr(cls.main_cls)
            if cls.partial_cls:
                partial = ', '.join(
                    map(
                        conditional_repr,
                        cls.partial_cls
                        if isinstance(cls.partial_cls, tuple)
                        else [cls.partial_cls],
                    ),
                )

                name = '{0}[{1}]'.format(
                    name, partial,
                )

            return repr_return(name)

    return MetaMagicType

def CreateMagicType(generator_cls, MetaMagicType, ABC):

    class MagicType(with_metaclass(MetaMagicType, object)):

        main_cls = ABC
        partial_cls = None

        for attr in [
            '__new__',
            '__init__',
            # iterator.
            '__iter__',
            '__next__',
            # callable.
            '__call__',
        ]:
            if unbound_getattr(generator_cls, attr):
                locals()[attr] = unbound_getattr(generator_cls, attr)

    return MagicType

class MagicTypeGenerator(type):

    def __new__(cls, ABC):
        MetaMagicType = CreateMetaMagicType(cls)
        return CreateMagicType(cls, MetaMagicType, ABC)

def check_type_of_instance(cls, instance):
    return any(
        issubclass(T, cls)
        # handle old style class.
        for T in (instance.__class__, type(instance))
    )

def check_getitem_tuple(type_decl, n):
    # type_decl should be a n-tuple.
    if not isinstance(type_decl, tuple):
        return False
    return len(type_decl) == n

def generate_immutable_abc(supercls, mutable_subclass):

    class ABCImmutableMeta(ABCMeta):

        def __subclasscheck__(cls, subclass):
            if not issubclass(subclass, supercls):
                return False
            return not issubclass(subclass, mutable_subclass)

    class ABCImmutable(with_metaclass(ABCImmutableMeta, object)):
        pass

    # dirty hack to assert issubclass(ABCImmutable, supercls).
    supercls._abc_cache.add(ABCImmutable)

    return ABCImmutable

class SequenceGenerator(MagicTypeGenerator):

    def check_getitem_type_decl(type_decl):
        if type_object(type_decl):
            return True

        if isinstance(type_decl, tuple):
            for T in type_decl:
                if nontype_object(T):
                    return False
            return True
        else:
            return False

    def __instancecheck__(cls, instance):
        if not check_type_of_instance(cls, instance):
            return False

        if not cls.partial_cls:
            return True

        if type_object(cls.partial_cls):
            for e in instance:
                if not isinstance(e, cls.partial_cls):
                    return False
        else:
            if len(cls.partial_cls) != len(instance):
                return False
            for i in range(len(instance)):
                if not isinstance(instance[i], cls.partial_cls[i]):
                    return False

        return True

class SetGenerator(MagicTypeGenerator):

    def check_getitem_type_decl(type_decl):
        return type_object(type_decl)

    def __instancecheck__(cls, instance):
        if not check_type_of_instance(cls, instance):
            return False

        if not cls.partial_cls:
            return True

        if type_object(cls.partial_cls):
            for e in instance:
                if not isinstance(e, cls.partial_cls):
                    return False

        return True

class MappingGenerator(MagicTypeGenerator):

    def check_getitem_type_decl(type_decl):
        if not check_getitem_tuple(type_decl, 2):
            return False

        return type_object(type_decl[0]) and type_object(type_decl[1])

    def __instancecheck__(cls, instance):
        if not check_type_of_instance(cls, instance):
            return False

        if cls.partial_cls:
            key_cls, val_cls = cls.partial_cls
            for key, val in instance.items():
                if not (isinstance(key, key_cls) and isinstance(val, val_cls)):
                    return False
        return True

class IteratorGenerator(MagicTypeGenerator):

    def check_getitem_type_decl(type_decl):
        return type_object(type_decl)

    def __init__(self, iterator):
        if self.partial_cls is None:
            raise MagicTypeError(
                'require T on Iterator[T].'
            )
        if not isinstance(iterator, abc.Iterator):
            raise MagicTypeError(
                'require Iterator.',
                iterator=iterator,
            )

        self.iterator = iterator

    def __iter__(self):
        return self

    def __next__(self):
        element = next(self.iterator)
        if isinstance(element, self.partial_cls):
            return element
        else:
            raise MagicTypeError(
                'type unmatched.',
                element=element,
                type_=self.partial_cls,
            )

    def __instancecheck__(cls, instance):
        return check_type_of_instance(cls, instance)

class IterableGenerator(MagicTypeGenerator):

    def check_getitem_type_decl(type_decl):
        return type_object(type_decl)

    def __init__(self, iterable):
        if self.partial_cls is None:
            raise MagicTypeError(
                'require T on Iterable[T].'
            )
        if not isinstance(iterable, abc.Iterable):
            raise MagicTypeError(
                'require Iterable.',
                iterable=iterable,
            )
        self.iterable = iterable

    def __iter__(self):
        return Iterator[self.partial_cls](
            iter(self.iterable),
        )

    def __instancecheck__(cls, instance):
        return check_type_of_instance(cls, instance)

class CallableGenerator(MagicTypeGenerator):

    def check_getitem_type_decl(type_decl):
        # Callable[[T, ...], T]
        if not check_getitem_tuple(type_decl, 2):
            return False
        if not isinstance(type_decl[0], abc.Iterable) or\
                nontype_object(type_decl[1]):
            return False
        for T in type_decl[0]:
            if nontype_object(T):
                return False

        return True

    def __init__(self, instance):
        # 1. not Callable.
        if not isinstance(instance, self.main_cls):
            raise MagicTypeError(
                'instance should be a Callable.',
                instance=instance,
            )
        # 2. with no specification.
        if not self.partial_cls:
            raise MagicSyntaxError(
                'Callable should be specified to wrap the callable.',
            )

        parameters_types, return_type = self.partial_cls
        self._wrapper = function_constraints(
            *parameters_types,
            # tailing coma is invalid for version < 3.5.
            return_type=return_type
        )(instance)

    def __call__(self, *args, **kwargs):
        return self._wrapper(*args, **kwargs)

    def __instancecheck__(cls, instance):
        if cls.partial_cls or not check_type_of_instance(cls, instance):
            return False
        else:
            # is callable and not Callable[T, ...].
            return True

class AnyMeta(ABCMeta):

    def __instancecheck__(cls, instance):
        return True

    def __subclasscheck__(cls, subclass):
        return type_object(subclass)

class Any(with_metaclass(AnyMeta, object)):
    pass

class UnionGenerator(MagicTypeGenerator):

    def check_getitem_type_decl(type_decl):
        if type_object(type_decl) or not isinstance(type_decl, tuple):
            return False

        for T in type_decl:
            if nontype_object(T):
                return False
        else:
            return True

    def __subclasshook__(cls, subclass):
        return False

    def __instancecheck__(cls, instance):
        if cls.partial_cls is None:
            return False

        for candidate_cls in cls.partial_cls:
            if isinstance(instance, candidate_cls):
                return True
        return False

class OptionalGenerator(MagicTypeGenerator):

    def check_getitem_type_decl(type_decl):
        return type_object(type_decl)

    def __subclasshook__(cls, subclass):
        return False

    def __instancecheck__(cls, instance):
        if cls.partial_cls is None:
            return False

        if instance is None:
            return True
        else:
            return isinstance(instance, cls.partial_cls)

ABCImmutableSequence = generate_immutable_abc(
    abc.Sequence, abc.MutableSequence,
)
ABCImmutableSet = generate_immutable_abc(
    abc.Set, abc.MutableSet,
)
ABCImmutableMapping = generate_immutable_abc(
    abc.Mapping, abc.MutableMapping,
)

Sequence = SequenceGenerator(abc.Sequence)
MutableSequence = SequenceGenerator(abc.MutableSequence)
ImmutableSequence = SequenceGenerator(ABCImmutableSequence)

Set = SetGenerator(abc.Set)
MutableSet = SetGenerator(abc.MutableSet)
ImmutableSet = SetGenerator(ABCImmutableSet)

Mapping = MappingGenerator(abc.Mapping)
MutableMapping = MappingGenerator(abc.MutableMapping)
ImmutableMapping = MappingGenerator(ABCImmutableMapping)

Iterator = IteratorGenerator(abc.Iterator)
Iterable = IterableGenerator(abc.Iterable)

Callable = CallableGenerator(abc.Callable)

Union = UnionGenerator(Any)
Optional = OptionalGenerator(Any)
NoneType = type(None)

from magic_constraints.decorator import function_constraints  # noqa
<|endoftext|>


<commit_before>from __future__ import print_function, absolute_import, division
import sys
import inspect
import socket

import numpy as np
from sklearn.utils import check_random_state
from sklearn.model_selection import ParameterGrid
try:
    from hyperopt import (Trials, tpe, fmin, STATUS_OK, STATUS_RUNNING,
                          STATUS_FAIL)
except ImportError:
    # hyperopt is optional, but required for hyperopt_tpe()
    pass

from .search_space import EnumVariable
from .acquisition_functions import AcquisitionFunction
from .surrogate_models import MaximumLikelihoodGaussianProcess, GaussianProcessKernel

# try:
#     from SALib.sample import sobol_sequence as ss
# except ImportError:
#     ss = None
#     pass

DEFAULT_TIMEOUT = socket._GLOBAL_DEFAULT_TIMEOUT

class BaseStrategy(object):
    short_name = None

    def suggest(self, history, searchspace):
        """
        Parameters
        ----------
        history : list of 3-tuples
            History of past function evaluations. Each element in history
            should be a tuple `(params, score, status)`, where `params` is a
            dict mapping parameter names to values
        searchspace : SearchSpace
            Instance of search_space.SearchSpace
        random_state :i nteger or numpy.RandomState, optional
            The random seed for sampling. If an integer is given, it fixes the
            seed. Defaults to the global numpy random number generator.

        Returns
        -------
        new_params : dict
        """
        raise NotImplementedError()

    @staticmethod
    def is_repeated_suggestion(params, history):
        """
        Parameters
        ----------
        params : dict
            Trial param set
        history : list of 3-tuples
            History of past function evaluations. Each element in history
            should be a tuple `(params, score, status)`, where `params` is a
            dict mapping parameter names to values

        Returns
        -------
        is_repeated_suggestion : bool
        """
        if any(params == hparams and hstatus == 'SUCCEEDED'
               for hparams, hscore, hstatus in history):
            return True
        else:
            return False

# class SobolSearch(BaseStrategy):
#     short_name = 'sobol'
#     _SKIP = int(1e4)
#
#     def __init__(self, length=1000):
#         # TODO length should be n_trials.  But this doesn't seem to be accessible to strategies without major re-write.
#         self.sequence = None
#         self.length = length
#         self.n_dims = 0
#         self.offset = 0
#         self.counter = 0
#
#     def _set_sequence(self):
#         # TODO could get rid of first part of sequence
#         self.sequence = ss.sample(self.length + self._SKIP, self.n_dims)
#
#     def _from_unit_cube(self, result, searchspace):
#         # TODO this should be a method common to both Sobol and GP.
#         # Note that Sobol only deals with float-valued variables, so we have
#         # a transform step on either side, where int and enum valued variables
#         # are transformed before calling gp, and then the result suggested by
#         # Sobol needs to be reverse-transformed.
#         out = {}
#         for gpvalue, var in zip(result, searchspace):
#             out[var.name] = var.point_from_unit(float(gpvalue))
#         return out
#
#     def suggest(self, history, searchspace):
#         if 'SALib' not in sys.modules:
#             raise ImportError('No module named SALib')
#
#         if self.sequence is None:
#             self.n_dims = searchspace.n_dims
#             self.offset = len(history) + self._SKIP
#             self._set_sequence()
#         try:
#             points = self.sequence[self.offset+ self.counter]
#             self.counter += 1
#         except IndexError:
#             raise RuntimeError('Increase sobol sequence length')
#
#         return self._from_unit_cube(points, searchspace)

class RandomSearch(BaseStrategy):
    short_name = 'random'

    def __init__(self, seed=None):
        self.seed = seed

    def suggest(self, history, searchspace):
        """Randomly suggest params from searchspace.
        """
        return searchspace.rvs(self.seed)

class HyperoptTPE(BaseStrategy):
    short_name = 'hyperopt_tpe'

    def __init__(self, seed=None, gamma=0.25, seeds=20):
        self.seed = seed
        self.gamma = gamma
        self.seeds = seeds

    def suggest(self, history, searchspace):
        """
        Suggest params to maximize an objective function based on the
        function evaluation history using a tree of Parzen estimators (TPE),
        as implemented in the hyperopt package.

        Use of this function requires that hyperopt be installed.
        """
        # This function is very odd, because as far as I can tell there's
        # no real documented API for any of the internals of hyperopt. Its
        # execution model is that hyperopt calls your objective function
        # (instead of merely providing you with suggested points, and then
        # you calling the function yourself), and its very tricky (for me)
        # to use the internal hyperopt data structures to get these predictions
        # out directly.

        # so they path we take in this function is to construct a synthetic
        # hyperopt.Trials database which from the `history`, and then call
        # hyoperopt.fmin with a dummy objective function that logs the value
        # used, and then return that value to our client.

        # The form of the hyperopt.Trials database isn't really documented in
        # the code -- most of this comes from reverse engineering it, by
        # running fmin() on a simple function and then inspecting the form of
        # the resulting trials object.
        if 'hyperopt' not in sys.modules:
            raise ImportError('No module named hyperopt')

        random = check_random_state(self.seed)
        hp_searchspace = searchspace.to_hyperopt()

        trials = Trials()
        for i, (params, scores, status) in enumerate(history):
            if status == 'SUCCEEDED':
                # we're doing maximization, hyperopt.fmin() does minimization,
                # so we need to swap the sign
                result = {'loss': -np.mean(scores), 'status': STATUS_OK}
            elif status == 'PENDING':
                result = {'status': STATUS_RUNNING}
            elif status == 'FAILED':
                result = {'status': STATUS_FAIL}
            else:
                raise RuntimeError('unrecognized status: %s' % status)

            # the vals key in the trials dict is basically just the params
            # dict, but enum variables (hyperopt hp.choice() nodes) are
            # different, because the index of the parameter is specified
            # in vals, not the parameter itself.

            vals = {}
            for var in searchspace:
                if isinstance(var, EnumVariable):
                    # get the index in the choices of the parameter, and use
                    # that.
                    matches = [
                        i for i, c in enumerate(var.choices)
                        if c == params[var.name]
                    ]
                    assert len(matches) == 1
                    vals[var.name] = matches
                else:
                    # the other big difference is that all of the param values
                    # are wrapped in length-1 lists.
                    vals[var.name] = [params[var.name]]

            trials.insert_trial_doc({
                'misc': {
                    'cmd': ('domain_attachment', 'FMinIter_Domain'),
                    'idxs': dict((k, [i]) for k in hp_searchspace.keys()),
                    'tid': i,
                    'vals': vals,
                    'workdir': None
                },
                'result': result,
                'tid': i,
                # bunch of fixed fields that hyperopt seems to require
                'owner': None,
                'spec': None,
                'state': 2,
                'book_time': None,
                'exp_key': None,
                'refresh_time': None,
                'version': 0
            })

        trials.refresh()
        chosen_params_container = []

        def suggest(*args, **kwargs):
            return tpe.suggest(*args, **kwargs, gamma=self.gamma, n_startup_jobs=self.seeds)

        def mock_fn(x):
            # http://stackoverflow.com/a/3190783/1079728
            # to get around no nonlocal keywork in python2
            chosen_params_container.append(x)
            return 0

        fmin(fn=mock_fn,
             algo=tpe.suggest,
             space=hp_searchspace,
             trials=trials,
             max_evals=len(trials.trials) + 1,
             **self._hyperopt_fmin_random_kwarg(random))
        chosen_params = chosen_params_container[0]

        return chosen_params

    @staticmethod
    def _hyperopt_fmin_random_kwarg(random):
        if 'rstate' in inspect.getargspec(fmin).args:
            # 0.0.3-dev version uses this argument
            kwargs = {'rstate': random, 'allow_trials_fmin': False}
        elif 'rseed' in inspect.getargspec(fmin).args:
            # 0.0.2 version uses different argument
            kwargs = {'rseed': random.randint(2**32 - 1)}
        return kwargs

class Bayes(BaseStrategy):
    short_name = 'bayes'
    # TODO : n_iter, max_iter should be in acquisition params
    def __init__(self,
                 acquisition=None,
                 surrogate=None,
                 seed=None,
                 seeds=1,
                 max_feval=5E4,
                 max_iter=1E5
                ):
        self.seed = seed
        self.seeds = seeds
        self.max_feval = max_feval
        self.max_iter = max_iter
        # self.n_iter = n_iter
        self.n_dims = None
        if surrogate is None:
            surrogate = 'gp'
        self.surrogate = surrogate
        if kernels is None:
            kernels = [{'name': 'GPy.kern.Matern52', 'params': {'ARD': True},
                              'options': {'independent': False}}]
        self.kernel_params = kernels
        if acquisition is None:
            acquisition = {'name': 'osprey', 'params': {}}
        self.acquisition_params = acquisition

    def _get_data(self, history, searchspace):
        X = []
        Y = []
        V = []
        ignore = []
        for param_dict, scores, status in history:
            # transform points into the GP domain. This invloves bringing
            # int and enum variables to floating point, etc.
            if status == 'FAILED':
                # not sure how to deal with these yet
                continue

            point = searchspace.point_to_unit(param_dict)
            if status == 'SUCCEEDED':
                X.append(point)
                Y.append(np.mean(scores))
                V.append(np.var(scores))

            elif status == 'PENDING':
                ignore.append(point)
            else:
                raise RuntimeError('unrecognized status: %s' % status)

        return (np.array(X).reshape(-1, self.n_dims),
                np.array(Y).reshape(-1, 1), np.array(V).reshape(-1, 1),
                np.array(ignore).reshape(-1, self.n_dims))

    def _from_unit(self, result, searchspace):

        # Note that GP only deals with float-valued variables, so we have
        # a transform step on either side, where int and enum valued variables
        # are transformed before calling gp, and then the result suggested by
        # GP needs to be reverse-transformed.
        out = {}
        for gpvalue, var in zip(result, searchspace):
            out[var.name] = var.point_from_unit(float(gpvalue))
        return out

    def _is_within(self, point, X, tol=1E-2):
        if True in (np.sqrt(((point - X)**2).sum(axis=0)) <= tol):
            return True
        return False

    def suggest(self, history, searchspace, max_tries=5):
        if len(history) < self.seeds:
            return RandomSearch().suggest(history, searchspace)

        self.n_dims = searchspace.n_dims

        X, Y, V, ignore = self._get_data(history, searchspace)
        # TODO make _create_kernel accept optional args.
        # Define and fit model
        if self.surrogate == 'gp':
            kernel = GaussianProcessKernel(self.kernel_params, self.n_dims)
            model = MaximumLikelihoodGaussianProcess(X=X, Y=Y, kernel=kernel.kernel,
                                                     max_feval=self.max_feval)
        else:
            raise NotImplementedError('Surrogate model not recognised.  Please choose from: gp')
        model.fit()

        # Define acquisition function and get best candidate
        af = AcquisitionFunction(surrogate=model,
                                 acquisition_params=self.acquisition_params,
                                 n_dims=self.n_dims,
                                 n_iter=self.n_iter,
                                 max_iter=self.max_iter)
        suggestion = af.get_best_candidate()

        if suggestion in ignore or self._is_within(suggestion, X):
            return RandomSearch().suggest(history, searchspace)

        return self._from_unit(suggestion, searchspace)

class GridSearch(BaseStrategy):
    short_name = 'grid'

    def __init__(self):
        self.param_grid = None
        self.current = -1

    def suggest(self, history, searchspace):
        # Convert searchspace to param_grid
        if self.param_grid is None:
            if not all(isinstance(v, EnumVariable) for v in searchspace):
                raise RuntimeError(
                    "GridSearchStrategy is defined only for all-enum search space"
                )

            self.param_grid = ParameterGrid(
                dict((v.name, v.choices) for v in searchspace))

        # NOTE: there is no way of signaling end of parameters to be searched against
        # so user should pick correctly number of evaluations
        self.current += 1
        return self.param_grid[self.current % len(self.param_grid)]
<commit_msg>Update strategies.py<commit_after>from __future__ import print_function, absolute_import, division
import sys
import inspect
import socket

import numpy as np
from sklearn.utils import check_random_state
from sklearn.model_selection import ParameterGrid
try:
    from hyperopt import (Trials, tpe, fmin, STATUS_OK, STATUS_RUNNING,
                          STATUS_FAIL)
except ImportError:
    # hyperopt is optional, but required for hyperopt_tpe()
    pass

from .search_space import EnumVariable
from .acquisition_functions import AcquisitionFunction
from .surrogate_models import MaximumLikelihoodGaussianProcess, GaussianProcessKernel

# try:
#     from SALib.sample import sobol_sequence as ss
# except ImportError:
#     ss = None
#     pass

DEFAULT_TIMEOUT = socket._GLOBAL_DEFAULT_TIMEOUT

class BaseStrategy(object):
    short_name = None

    def suggest(self, history, searchspace):
        """
        Parameters
        ----------
        history : list of 3-tuples
            History of past function evaluations. Each element in history
            should be a tuple `(params, score, status)`, where `params` is a
            dict mapping parameter names to values
        searchspace : SearchSpace
            Instance of search_space.SearchSpace
        random_state :i nteger or numpy.RandomState, optional
            The random seed for sampling. If an integer is given, it fixes the
            seed. Defaults to the global numpy random number generator.

        Returns
        -------
        new_params : dict
        """
        raise NotImplementedError()

    @staticmethod
    def is_repeated_suggestion(params, history):
        """
        Parameters
        ----------
        params : dict
            Trial param set
        history : list of 3-tuples
            History of past function evaluations. Each element in history
            should be a tuple `(params, score, status)`, where `params` is a
            dict mapping parameter names to values

        Returns
        -------
        is_repeated_suggestion : bool
        """
        if any(params == hparams and hstatus == 'SUCCEEDED'
               for hparams, hscore, hstatus in history):
            return True
        else:
            return False

# class SobolSearch(BaseStrategy):
#     short_name = 'sobol'
#     _SKIP = int(1e4)
#
#     def __init__(self, length=1000):
#         # TODO length should be n_trials.  But this doesn't seem to be accessible to strategies without major re-write.
#         self.sequence = None
#         self.length = length
#         self.n_dims = 0
#         self.offset = 0
#         self.counter = 0
#
#     def _set_sequence(self):
#         # TODO could get rid of first part of sequence
#         self.sequence = ss.sample(self.length + self._SKIP, self.n_dims)
#
#     def _from_unit_cube(self, result, searchspace):
#         # TODO this should be a method common to both Sobol and GP.
#         # Note that Sobol only deals with float-valued variables, so we have
#         # a transform step on either side, where int and enum valued variables
#         # are transformed before calling gp, and then the result suggested by
#         # Sobol needs to be reverse-transformed.
#         out = {}
#         for gpvalue, var in zip(result, searchspace):
#             out[var.name] = var.point_from_unit(float(gpvalue))
#         return out
#
#     def suggest(self, history, searchspace):
#         if 'SALib' not in sys.modules:
#             raise ImportError('No module named SALib')
#
#         if self.sequence is None:
#             self.n_dims = searchspace.n_dims
#             self.offset = len(history) + self._SKIP
#             self._set_sequence()
#         try:
#             points = self.sequence[self.offset+ self.counter]
#             self.counter += 1
#         except IndexError:
#             raise RuntimeError('Increase sobol sequence length')
#
#         return self._from_unit_cube(points, searchspace)

class RandomSearch(BaseStrategy):
    short_name = 'random'

    def __init__(self, seed=None):
        self.seed = seed

    def suggest(self, history, searchspace):
        """Randomly suggest params from searchspace.
        """
        return searchspace.rvs(self.seed)

class HyperoptTPE(BaseStrategy):
    short_name = 'hyperopt_tpe'

    def __init__(self, seed=None, gamma=0.25, seeds=20):
        self.seed = seed
        self.gamma = gamma
        self.seeds = seeds

    def suggest(self, history, searchspace):
        """
        Suggest params to maximize an objective function based on the
        function evaluation history using a tree of Parzen estimators (TPE),
        as implemented in the hyperopt package.

        Use of this function requires that hyperopt be installed.
        """
        # This function is very odd, because as far as I can tell there's
        # no real documented API for any of the internals of hyperopt. Its
        # execution model is that hyperopt calls your objective function
        # (instead of merely providing you with suggested points, and then
        # you calling the function yourself), and its very tricky (for me)
        # to use the internal hyperopt data structures to get these predictions
        # out directly.

        # so they path we take in this function is to construct a synthetic
        # hyperopt.Trials database which from the `history`, and then call
        # hyoperopt.fmin with a dummy objective function that logs the value
        # used, and then return that value to our client.

        # The form of the hyperopt.Trials database isn't really documented in
        # the code -- most of this comes from reverse engineering it, by
        # running fmin() on a simple function and then inspecting the form of
        # the resulting trials object.
        if 'hyperopt' not in sys.modules:
            raise ImportError('No module named hyperopt')

        random = check_random_state(self.seed)
        hp_searchspace = searchspace.to_hyperopt()

        trials = Trials()
        for i, (params, scores, status) in enumerate(history):
            if status == 'SUCCEEDED':
                # we're doing maximization, hyperopt.fmin() does minimization,
                # so we need to swap the sign
                result = {'loss': -np.mean(scores), 'status': STATUS_OK}
            elif status == 'PENDING':
                result = {'status': STATUS_RUNNING}
            elif status == 'FAILED':
                result = {'status': STATUS_FAIL}
            else:
                raise RuntimeError('unrecognized status: %s' % status)

            # the vals key in the trials dict is basically just the params
            # dict, but enum variables (hyperopt hp.choice() nodes) are
            # different, because the index of the parameter is specified
            # in vals, not the parameter itself.

            vals = {}
            for var in searchspace:
                if isinstance(var, EnumVariable):
                    # get the index in the choices of the parameter, and use
                    # that.
                    matches = [
                        i for i, c in enumerate(var.choices)
                        if c == params[var.name]
                    ]
                    assert len(matches) == 1
                    vals[var.name] = matches
                else:
                    # the other big difference is that all of the param values
                    # are wrapped in length-1 lists.
                    vals[var.name] = [params[var.name]]

            trials.insert_trial_doc({
                'misc': {
                    'cmd': ('domain_attachment', 'FMinIter_Domain'),
                    'idxs': dict((k, [i]) for k in hp_searchspace.keys()),
                    'tid': i,
                    'vals': vals,
                    'workdir': None
                },
                'result': result,
                'tid': i,
                # bunch of fixed fields that hyperopt seems to require
                'owner': None,
                'spec': None,
                'state': 2,
                'book_time': None,
                'exp_key': None,
                'refresh_time': None,
                'version': 0
            })

        trials.refresh()
        chosen_params_container = []

        def suggest(*args, **kwargs):
            return tpe.suggest(*args, **kwargs, gamma=self.gamma, n_startup_jobs=self.seeds)

        def mock_fn(x):
            # http://stackoverflow.com/a/3190783/1079728
            # to get around no nonlocal keywork in python2
            chosen_params_container.append(x)
            return 0

        fmin(fn=mock_fn,
             algo=tpe.suggest,
             space=hp_searchspace,
             trials=trials,
             max_evals=len(trials.trials) + 1,
             **self._hyperopt_fmin_random_kwarg(random))
        chosen_params = chosen_params_container[0]

        return chosen_params

    @staticmethod
    def _hyperopt_fmin_random_kwarg(random):
        if 'rstate' in inspect.getargspec(fmin).args:
            # 0.0.3-dev version uses this argument
            kwargs = {'rstate': random, 'allow_trials_fmin': False}
        elif 'rseed' in inspect.getargspec(fmin).args:
            # 0.0.2 version uses different argument
            kwargs = {'rseed': random.randint(2**32 - 1)}
        return kwargs

class Bayes(BaseStrategy):
    short_name = 'bayes'
    # TODO : n_iter, max_iter should be in acquisition params
    def __init__(self,
                 acquisition=None,
                 surrogate=None,
                 kernels=None,
                 seed=None,
                 seeds=1,
                 max_feval=5E4,
                 max_iter=1E5
                ):
        self.seed = seed
        self.seeds = seeds
        self.max_feval = max_feval
        self.max_iter = max_iter
        # self.n_iter = n_iter
        self.n_dims = None
        if surrogate is None:
            surrogate = 'gp'
        self.surrogate = surrogate
        if kernels is None:
            kernels = [{'name': 'GPy.kern.Matern52', 'params': {'ARD': True},
                              'options': {'independent': False}}]
        self.kernel_params = kernels
        if acquisition is None:
            acquisition = {'name': 'osprey', 'params': {}}
        self.acquisition_params = acquisition

    def _get_data(self, history, searchspace):
        X = []
        Y = []
        V = []
        ignore = []
        for param_dict, scores, status in history:
            # transform points into the GP domain. This invloves bringing
            # int and enum variables to floating point, etc.
            if status == 'FAILED':
                # not sure how to deal with these yet
                continue

            point = searchspace.point_to_unit(param_dict)
            if status == 'SUCCEEDED':
                X.append(point)
                Y.append(np.mean(scores))
                V.append(np.var(scores))

            elif status == 'PENDING':
                ignore.append(point)
            else:
                raise RuntimeError('unrecognized status: %s' % status)

        return (np.array(X).reshape(-1, self.n_dims),
                np.array(Y).reshape(-1, 1), np.array(V).reshape(-1, 1),
                np.array(ignore).reshape(-1, self.n_dims))

    def _from_unit(self, result, searchspace):

        # Note that GP only deals with float-valued variables, so we have
        # a transform step on either side, where int and enum valued variables
        # are transformed before calling gp, and then the result suggested by
        # GP needs to be reverse-transformed.
        out = {}
        for gpvalue, var in zip(result, searchspace):
            out[var.name] = var.point_from_unit(float(gpvalue))
        return out

    def _is_within(self, point, X, tol=1E-2):
        if True in (np.sqrt(((point - X)**2).sum(axis=0)) <= tol):
            return True
        return False

    def suggest(self, history, searchspace, max_tries=5):
        if len(history) < self.seeds:
            return RandomSearch().suggest(history, searchspace)

        self.n_dims = searchspace.n_dims

        X, Y, V, ignore = self._get_data(history, searchspace)
        # TODO make _create_kernel accept optional args.
        # Define and fit model
        if self.surrogate == 'gp':
            kernel = GaussianProcessKernel(self.kernel_params, self.n_dims)
            model = MaximumLikelihoodGaussianProcess(X=X, Y=Y, kernel=kernel.kernel,
                                                     max_feval=self.max_feval)
        else:
            raise NotImplementedError('Surrogate model not recognised.  Please choose from: gp')
        model.fit()

        # Define acquisition function and get best candidate
        af = AcquisitionFunction(surrogate=model,
                                 acquisition_params=self.acquisition_params,
                                 n_dims=self.n_dims,
                                 n_iter=self.n_iter,
                                 max_iter=self.max_iter)
        suggestion = af.get_best_candidate()

        if suggestion in ignore or self._is_within(suggestion, X):
            return RandomSearch().suggest(history, searchspace)

        return self._from_unit(suggestion, searchspace)

class GridSearch(BaseStrategy):
    short_name = 'grid'

    def __init__(self):
        self.param_grid = None
        self.current = -1

    def suggest(self, history, searchspace):
        # Convert searchspace to param_grid
        if self.param_grid is None:
            if not all(isinstance(v, EnumVariable) for v in searchspace):
                raise RuntimeError(
                    "GridSearchStrategy is defined only for all-enum search space"
                )

            self.param_grid = ParameterGrid(
                dict((v.name, v.choices) for v in searchspace))

        # NOTE: there is no way of signaling end of parameters to be searched against
        # so user should pick correctly number of evaluations
        self.current += 1
        return self.param_grid[self.current % len(self.param_grid)]
<|endoftext|>


<commit_before>import os

from .tcpabridged import ConnectionTcpAbridged
from ...crypto import AESModeCTR

class ConnectionTcpObfuscated(ConnectionTcpAbridged):
    """
    Encodes the packet just like `ConnectionTcpAbridged`, but encrypts
    every message with a randomly generated key using the
    AES-CTR mode so the packets are harder to discern.
    """
    def __init__(self, ip, port, *, loop, loggers, proxy=None):
        super().__init__(ip, port, loop=loop, loggers=loggers, proxy=proxy)
        self._aes_encrypt = None
        self._aes_decrypt = None

    def _write(self, data):
        self._writer.write(self._aes_encrypt.encrypt(data))

    async def _read(self, n):
        return self._aes_decrypt.encrypt(await self._reader.readexactly(n))

    async def connect(self, timeout=None, ssl=None):
        await super().connect(timeout=timeout, ssl=ssl)

        # Obfuscated messages secrets cannot start with any of these
        keywords = (b'PVrG', b'GET ', b'POST', b'\xee\xee\xee\xee')
        while True:
            random = os.urandom(64)
            if (random[0] != b'\xef' and
                    random[:4] not in keywords and
                    random[4:4] != b'\0\0\0\0'):
                break

        random = bytearray(random)
        random[56] = random[57] = random[58] = random[59] = 0xef
        random_reversed = random[55:7:-1]  # Reversed (8, len=48)

        # Encryption has "continuous buffer" enabled
        encrypt_key = bytes(random[8:40])
        encrypt_iv = bytes(random[40:56])
        decrypt_key = bytes(random_reversed[:32])
        decrypt_iv = bytes(random_reversed[32:48])

        self._aes_encrypt = AESModeCTR(encrypt_key, encrypt_iv)
        self._aes_decrypt = AESModeCTR(decrypt_key, decrypt_iv)

        random[56:64] = self._aes_encrypt.encrypt(bytes(random))[56:64]
        self._writer.write(random)
        await self._writer.drain()
<commit_msg>Fix incorrect check for reserved data prefix in ConnectionTcpObfuscated<commit_after>import os

from .tcpabridged import ConnectionTcpAbridged
from ...crypto import AESModeCTR

class ConnectionTcpObfuscated(ConnectionTcpAbridged):
    """
    Encodes the packet just like `ConnectionTcpAbridged`, but encrypts
    every message with a randomly generated key using the
    AES-CTR mode so the packets are harder to discern.
    """
    def __init__(self, ip, port, *, loop, loggers, proxy=None):
        super().__init__(ip, port, loop=loop, loggers=loggers, proxy=proxy)
        self._aes_encrypt = None
        self._aes_decrypt = None

    def _write(self, data):
        self._writer.write(self._aes_encrypt.encrypt(data))

    async def _read(self, n):
        return self._aes_decrypt.encrypt(await self._reader.readexactly(n))

    async def connect(self, timeout=None, ssl=None):
        await super().connect(timeout=timeout, ssl=ssl)

        # Obfuscated messages secrets cannot start with any of these
        keywords = (b'PVrG', b'GET ', b'POST', b'\xee\xee\xee\xee')
        while True:
            random = os.urandom(64)
            if (random[0] != 0xef and
                    random[:4] not in keywords and
                    random[4:4] != b'\0\0\0\0'):
                break

        random = bytearray(random)
        random[56] = random[57] = random[58] = random[59] = 0xef
        random_reversed = random[55:7:-1]  # Reversed (8, len=48)

        # Encryption has "continuous buffer" enabled
        encrypt_key = bytes(random[8:40])
        encrypt_iv = bytes(random[40:56])
        decrypt_key = bytes(random_reversed[:32])
        decrypt_iv = bytes(random_reversed[32:48])

        self._aes_encrypt = AESModeCTR(encrypt_key, encrypt_iv)
        self._aes_decrypt = AESModeCTR(decrypt_key, decrypt_iv)

        random[56:64] = self._aes_encrypt.encrypt(bytes(random))[56:64]
        self._writer.write(random)
        await self._writer.drain()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import (
    division, absolute_import, print_function, unicode_literals,
)
from builtins import *                  # noqa
from future.builtins.disabled import *  # noqa
from future.utils import with_metaclass

from abc import ABCMeta
# collections.abc dosn't esist in Python 2.x.
import collections as abc

from magic_constraints.utils import (
    type_object, nontype_object,
    conditional_to_bytes, conditional_repr,
)

from magic_constraints.exception import (
    MagicTypeError, MagicIndexError
)

def meta_create_class(prefix, classname, baseclass, generator_cls):

    injected_functions = {}
    for name in dir(generator_cls):
        if not name.startswith(prefix):
            continue
        # get unbound function.
        function = getattr(generator_cls, name)
        if hasattr(function, '__func__'):
            function = function.__func__

        # remove prefix.
        injected_functions[name[len(prefix):]] = function

    return type(
        conditional_to_bytes(classname), (baseclass,), injected_functions,
    )

def create_metaclass(baseclass, generator_cls):
    return meta_create_class(
        '_metaclass_', 'MetaMagicClass', baseclass, generator_cls,
    )

def create_class(baseclass, generator_cls):
    return meta_create_class(
        '_class_', 'MagicClass', baseclass, generator_cls,
    )

def safe_getmethod(cls, name):

    def do_nothing(*args, **kwargs):
        return True

    method = getattr(cls, name, None)
    return method if method else do_nothing

class BasicMagicType(object):
    pass

class BasicMetaMagicType(ABCMeta):

    def __getitem__(cls, type_decl):

        if not safe_getmethod(cls, 'check_getitem_type_decl')(type_decl):
            raise MagicTypeError(
                'invalid type.',
                type_decl=type_decl,
            )

        ret_cls = cls.generator_cls(cls.main_cls)
        ret_cls.partial_cls = type_decl
        return ret_cls

    def __subclasscheck__(cls, subclass):
        if nontype_object(subclass):
            return False

        if not safe_getmethod(cls, 'check_subclass')(subclass):
            return False

        # corner case, subclass isn't MagicType.
        if not issubclass(subclass, BasicMagicType):
            return issubclass(subclass, cls.main_cls)

        # subclass is MagicType.
        if cls.partial_cls or subclass.partial_cls:
            # if subclass has partial_cls, return False.
            return False
        else:
            # 1. subclass is normal type object.
            # 2. subclass is a MagicType.
            return issubclass(subclass.main_cls, cls.main_cls)

    def __instancecheck__(cls, instance):
        return safe_getmethod(cls, 'check_instance')(instance)

    def __repr__(cls):

        name = conditional_repr(cls.main_cls)
        if cls.partial_cls:
            partial = ', '.join(
                map(
                    conditional_repr,
                    cls.partial_cls
                    if isinstance(cls.partial_cls, tuple)
                    else [cls.partial_cls],
                ),
            )
            name = '{0}[{1}]'.format(
                name, partial,
            )

        return conditional_to_bytes(name)

# 1. _metaclass_{name} -> {name} in metaclass.
# 2. _class_{name}     -> {name} in class.
class MagicTypeGenerator(type):

    def __new__(generator_cls, ABC):
        MetaMagicType = create_metaclass(
            BasicMetaMagicType,
            generator_cls,
        )
        MagicType = create_class(
            with_metaclass(MetaMagicType, BasicMagicType),
            generator_cls,
        )

        MagicType.generator_cls = generator_cls
        MagicType.main_cls = ABC
        MagicType.partial_cls = None

        return MagicType

def check_type_of_instance(cls, instance):
    return any(
        issubclass(T, cls)
        # handle old style class.
        for T in (instance.__class__, type(instance))
    )

def check_getitem_tuple(type_decl, n):
    # type_decl should be a n-tuple.
    if not isinstance(type_decl, tuple):
        return False
    return len(type_decl) == n

def generate_immutable_abc(supercls, mutable_subclass):

    class ABCImmutableMeta(ABCMeta):

        def __subclasscheck__(cls, subclass):
            if not issubclass(subclass, supercls):
                return False
            return not issubclass(subclass, mutable_subclass)

    class ABCImmutable(with_metaclass(ABCImmutableMeta, object)):
        pass

    # dirty hack to assert issubclass(ABCImmutable, supercls).
    supercls._abc_cache.add(ABCImmutable)

    return ABCImmutable

class SequenceGenerator(MagicTypeGenerator):

    def _metaclass_check_getitem_type_decl(cls, type_decl):
        if type_object(type_decl):
            return True

        if isinstance(type_decl, tuple):
            for T in type_decl:
                if nontype_object(T):
                    return False
            return True
        else:
            return False

    def _metaclass_check_instance(cls, instance):
        if not check_type_of_instance(cls, instance):
            return False

        if not cls.partial_cls:
            return True

        if type_object(cls.partial_cls):
            for e in instance:
                if not isinstance(e, cls.partial_cls):
                    return False
        else:
            if len(cls.partial_cls) != len(instance):
                return False
            for i in range(len(instance)):
                if not isinstance(instance[i], cls.partial_cls[i]):
                    return False

        return True

class SetGenerator(MagicTypeGenerator):

    def _metaclass_check_getitem_type_decl(cls, type_decl):
        return type_object(type_decl)

    def _metaclass_check_instance(cls, instance):
        if not check_type_of_instance(cls, instance):
            return False

        if not cls.partial_cls:
            return True

        if type_object(cls.partial_cls):
            for e in instance:
                if not isinstance(e, cls.partial_cls):
                    return False

        return True

class MappingGenerator(MagicTypeGenerator):

    def _metaclass_check_getitem_type_decl(cls, type_decl):
        if not check_getitem_tuple(type_decl, 2):
            return False

        return type_object(type_decl[0]) and type_object(type_decl[1])

    def _metaclass_check_instance(cls, instance):
        if not check_type_of_instance(cls, instance):
            return False

        if cls.partial_cls:
            key_cls, val_cls = cls.partial_cls
            for key, val in instance.items():
                if not (isinstance(key, key_cls) and isinstance(val, val_cls)):
                    return False
        return True

class IteratorGenerator(MagicTypeGenerator):

    _class_ITERATOR_CASE_LENGTH = 1
    _class_ITERATOR_CASE_NO_LENGTH = 2

    def _metaclass_check_getitem_type_decl(cls, type_decl):
        # 1. Iterator[T, ...]
        if isinstance(type_decl, tuple):
            for T in type_decl:
                if nontype_object(T):
                    return False
            return True

        # 2. Iterator[T]
        elif type_object(type_decl):
            return True

        else:
            return False

    def _metaclass_check_instance(cls, instance):
        if cls.partial_cls or not check_type_of_instance(cls, instance):
            return False
        else:
            # is Iterator and not Iterator[T, ...].
            return True

    def _class___init__(self, iterator):
        if self.partial_cls is None:
            raise MagicTypeError(
                'Iterator should be specified.'
            )

        if not isinstance(iterator, self.main_cls):
            raise MagicTypeError(
                'require Iterator.',
                iterator=iterator,
            )

        if isinstance(self.partial_cls, tuple):
            # Iterator[T, ...]. Checking on:
            # 1. the number of elements in the iterator.
            # 2. the type of each element.
            self.case = self.ITERATOR_CASE_LENGTH
            self._type_idx = 0
        else:
            # Iterator[T]. Check only the type of element. There's no
            # limitation on the length of iterator.
            self.case = self.ITERATOR_CASE_NO_LENGTH

        self.iterator = iterator

    def _class___iter__(self):
        return self

    def _class___next__(self):
        element = next(self.iterator)

        if self.case == self.ITERATOR_CASE_LENGTH:
            # error 1.
            if self._type_idx >= len(self.partial_cls):
                raise MagicIndexError(
                    'iterator contains more elements declared in type.',
                    type_=self,
                    last_element=element,
                )

            # error 2.
            if not isinstance(element, self.partial_cls[self._type_idx]):
                raise MagicTypeError(
                    'type unmatched.',
                    element=element,
                    type_=self.partial_cls[self._type_idx],
                )

            # good case.
            self._type_idx += 1
            return element

        elif self.case == self.ITERATOR_CASE_NO_LENGTH:
            if not isinstance(element, self.partial_cls):
                raise MagicTypeError(
                    'type unmatched.',
                    element=element,
                    type_=self.partial_cls,
                )

class IterableGenerator(MagicTypeGenerator):

    def _metaclass_check_getitem_type_decl(cls, type_decl):
        return type_object(type_decl)

    def _metaclass_check_instance(cls, instance):
        return check_type_of_instance(cls, instance)

    def _class___init__(self, iterable):
        if self.partial_cls is None:
            raise MagicTypeError(
                'require T on Iterable[T].'
            )
        if not isinstance(iterable, abc.Iterable):
            raise MagicTypeError(
                'require Iterable.',
                iterable=iterable,
            )
        self.iterable = iterable

    def _class___iter__(self):
        return Iterator[self.partial_cls](
            iter(self.iterable),
        )

class CallableGenerator(MagicTypeGenerator):

    def _metaclass_check_getitem_type_decl(cls, type_decl):
        # Callable[[T, ...], T]
        if not check_getitem_tuple(type_decl, 2):
            return False
        if not isinstance(type_decl[0], abc.Iterable) or\
                nontype_object(type_decl[1]):
            return False
        # [T, ...]
        for T in type_decl[0]:
            if nontype_object(T):
                return False

        return True

    def _metaclass_check_instance(cls, instance):
        if cls.partial_cls or not check_type_of_instance(cls, instance):
            return False
        else:
            # is callable and not Callable[T, ...].
            return True

    def _class___new__(self, instance):
        # 1. not Callable.
        if not isinstance(instance, self.main_cls):
            raise MagicTypeError(
                'instance should be a Callable.',
                instance=instance,
            )
        # 2. with no specification.
        if not self.partial_cls:
            raise MagicSyntaxError(
                'Callable should be specified to wrap the callable.',
            )

        parameters_types, return_type = self.partial_cls

        return function_constraints(
            *parameters_types,
            # tailing coma is invalid for version < 3.5.
            return_type=return_type
        )(instance)

class UnionGenerator(MagicTypeGenerator):

    def _metaclass_check_getitem_type_decl(cls, type_decl):
        if type_object(type_decl) or not isinstance(type_decl, tuple):
            return False

        for T in type_decl:
            if nontype_object(T):
                return False
        else:
            return True

    def _metaclass_check_subclass(cls, subclass):
        return False

    def _metaclass_check_instance(cls, instance):
        if cls.partial_cls is None:
            return False

        for candidate_cls in cls.partial_cls:
            if isinstance(instance, candidate_cls):
                return True
        return False

class OptionalGenerator(MagicTypeGenerator):

    def _metaclass_check_getitem_type_decl(cls, type_decl):
        return type_object(type_decl)

    def _metaclass_check_subclass(cls, subclass):
        return False

    def _metaclass_check_instance(cls, instance):
        if cls.partial_cls is None:
            return False

        if instance is None:
            return True
        else:
            return isinstance(instance, cls.partial_cls)

def dummy_class(name):
    return type(conditional_to_bytes(name), (object,), {})

class AnyMeta(ABCMeta):

    def __instancecheck__(cls, instance):
        return True

    def __subclasscheck__(cls, subclass):
        return type_object(subclass)

    def __repr__(self):
        return conditional_to_bytes('Any')

class Any(with_metaclass(AnyMeta, object)):
    pass

ABCImmutableSequence = generate_immutable_abc(
    abc.Sequence, abc.MutableSequence,
)
ABCImmutableSet = generate_immutable_abc(
    abc.Set, abc.MutableSet,
)
ABCImmutableMapping = generate_immutable_abc(
    abc.Mapping, abc.MutableMapping,
)

Sequence = SequenceGenerator(abc.Sequence)
MutableSequence = SequenceGenerator(abc.MutableSequence)
ImmutableSequence = SequenceGenerator(ABCImmutableSequence)

Set = SetGenerator(abc.Set)
MutableSet = SetGenerator(abc.MutableSet)
ImmutableSet = SetGenerator(ABCImmutableSet)

Mapping = MappingGenerator(abc.Mapping)
MutableMapping = MappingGenerator(abc.MutableMapping)
ImmutableMapping = MappingGenerator(ABCImmutableMapping)

Iterator = IteratorGenerator(abc.Iterator)
Iterable = IterableGenerator(abc.Iterable)

Callable = CallableGenerator(abc.Callable)

Union = UnionGenerator(dummy_class('Union'))
Optional = OptionalGenerator(dummy_class('Optional'))
NoneType = type(None)

from magic_constraints.decorator import (
    function_constraints,
)  # noqa
<commit_msg>Small fix.<commit_after># -*- coding: utf-8 -*-
from __future__ import (
    division, absolute_import, print_function, unicode_literals,
)
from builtins import *                  # noqa
from future.builtins.disabled import *  # noqa
from future.utils import with_metaclass

from abc import ABCMeta
# collections.abc dosn't esist in Python 2.x.
import collections as abc

from magic_constraints.utils import (
    type_object, nontype_object,
    conditional_to_bytes, conditional_repr,
)

from magic_constraints.exception import (
    MagicTypeError, MagicIndexError
)

def meta_create_class(prefix, classname, baseclass, generator_cls):

    injected_functions = {}
    for name in dir(generator_cls):
        if not name.startswith(prefix):
            continue
        # get unbound function.
        function = getattr(generator_cls, name)
        if hasattr(function, '__func__'):
            function = function.__func__

        # remove prefix.
        injected_functions[name[len(prefix):]] = function

    return type(
        conditional_to_bytes(classname), (baseclass,), injected_functions,
    )

def create_metaclass(baseclass, generator_cls):
    return meta_create_class(
        '_metaclass_', 'MetaMagicClass', baseclass, generator_cls,
    )

def create_class(baseclass, generator_cls):
    return meta_create_class(
        '_class_', 'MagicClass', baseclass, generator_cls,
    )

def safe_getmethod(cls, name):

    def do_nothing(*args, **kwargs):
        return True

    method = getattr(cls, name, None)
    return method if method else do_nothing

class BasicMagicType(object):
    pass

class BasicMetaMagicType(ABCMeta):

    def __getitem__(cls, type_decl):

        if not safe_getmethod(cls, 'check_getitem_type_decl')(type_decl):
            raise MagicTypeError(
                'invalid type.',
                type_decl=type_decl,
            )

        ret_cls = cls.generator_cls(cls.main_cls)
        ret_cls.partial_cls = type_decl
        return ret_cls

    def __subclasscheck__(cls, subclass):
        if nontype_object(subclass):
            return False

        if not safe_getmethod(cls, 'check_subclass')(subclass):
            return False

        # corner case, subclass isn't MagicType.
        if not issubclass(subclass, BasicMagicType):
            return issubclass(subclass, cls.main_cls)

        # subclass is MagicType.
        if cls.partial_cls or subclass.partial_cls:
            # if subclass has partial_cls, return False.
            return False
        else:
            # 1. subclass is normal type object.
            # 2. subclass is a MagicType.
            return issubclass(subclass.main_cls, cls.main_cls)

    def __instancecheck__(cls, instance):
        return safe_getmethod(cls, 'check_instance')(instance)

    def __repr__(cls):

        name = conditional_repr(cls.main_cls)
        if cls.partial_cls:
            partial = ', '.join(
                map(
                    conditional_repr,
                    cls.partial_cls
                    if isinstance(cls.partial_cls, tuple)
                    else [cls.partial_cls],
                ),
            )
            name = '{0}[{1}]'.format(
                name, partial,
            )

        return conditional_to_bytes(name)

# 1. _metaclass_{name} -> {name} in metaclass.
# 2. _class_{name}     -> {name} in class.
class MagicTypeGenerator(type):

    def __new__(generator_cls, ABC):
        MetaMagicType = create_metaclass(
            BasicMetaMagicType,
            generator_cls,
        )
        MagicType = create_class(
            with_metaclass(MetaMagicType, BasicMagicType),
            generator_cls,
        )

        MagicType.generator_cls = generator_cls
        MagicType.main_cls = ABC
        MagicType.partial_cls = None

        return MagicType

def check_type_of_instance(cls, instance):
    return any(
        issubclass(T, cls)
        # handle old style class.
        for T in (instance.__class__, type(instance))
    )

def check_getitem_tuple(type_decl, n):
    # type_decl should be a n-tuple.
    if not isinstance(type_decl, tuple):
        return False
    return len(type_decl) == n

def generate_immutable_abc(supercls, mutable_subclass):

    class ABCImmutableMeta(ABCMeta):

        def __subclasscheck__(cls, subclass):
            if not issubclass(subclass, supercls):
                return False
            return not issubclass(subclass, mutable_subclass)

    class ABCImmutable(with_metaclass(ABCImmutableMeta, object)):
        pass

    # dirty hack to assert issubclass(ABCImmutable, supercls).
    supercls._abc_cache.add(ABCImmutable)

    return ABCImmutable

class SequenceGenerator(MagicTypeGenerator):

    def _metaclass_check_getitem_type_decl(cls, type_decl):
        if type_object(type_decl):
            return True

        if isinstance(type_decl, tuple):
            for T in type_decl:
                if nontype_object(T):
                    return False
            return True
        else:
            return False

    def _metaclass_check_instance(cls, instance):
        if not check_type_of_instance(cls, instance):
            return False

        if not cls.partial_cls:
            return True

        if type_object(cls.partial_cls):
            for e in instance:
                if not isinstance(e, cls.partial_cls):
                    return False
        else:
            if len(cls.partial_cls) != len(instance):
                return False
            for i in range(len(instance)):
                if not isinstance(instance[i], cls.partial_cls[i]):
                    return False

        return True

class SetGenerator(MagicTypeGenerator):

    def _metaclass_check_getitem_type_decl(cls, type_decl):
        return type_object(type_decl)

    def _metaclass_check_instance(cls, instance):
        if not check_type_of_instance(cls, instance):
            return False

        if not cls.partial_cls:
            return True

        if type_object(cls.partial_cls):
            for e in instance:
                if not isinstance(e, cls.partial_cls):
                    return False

        return True

class MappingGenerator(MagicTypeGenerator):

    def _metaclass_check_getitem_type_decl(cls, type_decl):
        if not check_getitem_tuple(type_decl, 2):
            return False

        return type_object(type_decl[0]) and type_object(type_decl[1])

    def _metaclass_check_instance(cls, instance):
        if not check_type_of_instance(cls, instance):
            return False

        if cls.partial_cls:
            key_cls, val_cls = cls.partial_cls
            for key, val in instance.items():
                if not (isinstance(key, key_cls) and isinstance(val, val_cls)):
                    return False
        return True

class IteratorGenerator(MagicTypeGenerator):

    _class_ITERATOR_CASE_LENGTH = 1
    _class_ITERATOR_CASE_NO_LENGTH = 2

    def _metaclass_check_getitem_type_decl(cls, type_decl):
        # 1. Iterator[T, ...]
        if isinstance(type_decl, tuple):
            for T in type_decl:
                if nontype_object(T):
                    return False
            return True

        # 2. Iterator[T]
        elif type_object(type_decl):
            return True

        else:
            return False

    def _metaclass_check_instance(cls, instance):
        if cls.partial_cls or not check_type_of_instance(cls, instance):
            return False
        else:
            # is Iterator and not Iterator[T, ...].
            return True

    def _class___init__(self, iterator):
        if self.partial_cls is None:
            raise MagicTypeError(
                'Iterator should be specified.'
            )

        if not isinstance(iterator, self.main_cls):
            raise MagicTypeError(
                'require Iterator.',
                iterator=iterator,
            )

        if isinstance(self.partial_cls, tuple):
            # Iterator[T, ...]. Checking on:
            # 1. the number of elements in the iterator.
            # 2. the type of each element.
            self.case = self.ITERATOR_CASE_LENGTH
            self._type_idx = 0
        else:
            # Iterator[T]. Check only the type of element. There's no
            # limitation on the length of iterator.
            self.case = self.ITERATOR_CASE_NO_LENGTH

        self.iterator = iterator

    def _class___iter__(self):
        return self

    def _class___next__(self):
        element = next(self.iterator)

        if self.case == self.ITERATOR_CASE_LENGTH:
            # error 1.
            if self._type_idx >= len(self.partial_cls):
                raise MagicIndexError(
                    'iterator contains more elements declared in type.',
                    type_=self,
                    last_element=element,
                )

            # error 2.
            if not isinstance(element, self.partial_cls[self._type_idx]):
                raise MagicTypeError(
                    'type unmatched.',
                    element=element,
                    type_=self.partial_cls[self._type_idx],
                )

            # good case.
            self._type_idx += 1
            return element

        elif self.case == self.ITERATOR_CASE_NO_LENGTH:
            if not isinstance(element, self.partial_cls):
                raise MagicTypeError(
                    'type unmatched.',
                    element=element,
                    type_=self.partial_cls,
                )

class IterableGenerator(MagicTypeGenerator):

    def _metaclass_check_getitem_type_decl(cls, type_decl):
        return type_object(type_decl)

    def _metaclass_check_instance(cls, instance):
        return check_type_of_instance(cls, instance)

    def _class___init__(self, iterable):
        if self.partial_cls is None:
            raise MagicTypeError(
                'require T on Iterable[T].'
            )
        if not isinstance(iterable, abc.Iterable):
            raise MagicTypeError(
                'require Iterable.',
                iterable=iterable,
            )
        self.iterable = iterable

    def _class___iter__(self):
        return Iterator[self.partial_cls](
            iter(self.iterable),
        )

class CallableGenerator(MagicTypeGenerator):

    def _metaclass_check_getitem_type_decl(cls, type_decl):
        # Callable[[T, ...], T]
        if not check_getitem_tuple(type_decl, 2):
            return False
        if not isinstance(type_decl[0], abc.Iterable) or\
                nontype_object(type_decl[1]):
            return False
        # [T, ...]
        for T in type_decl[0]:
            if nontype_object(T):
                return False

        return True

    def _metaclass_check_instance(cls, instance):
        if cls.partial_cls or not check_type_of_instance(cls, instance):
            return False
        else:
            # is callable and not Callable[T, ...].
            return True

    def _class___new__(cls, instance):
        # 1. not Callable.
        if not isinstance(instance, cls.main_cls):
            raise MagicTypeError(
                'instance should be a Callable.',
                instance=instance,
            )
        # 2. with no specification.
        if not cls.partial_cls:
            raise MagicSyntaxError(
                'Callable should be specified to wrap the callable.',
            )

        parameters_types, return_type = cls.partial_cls

        return function_constraints(
            *parameters_types,
            # tailing coma is invalid for version < 3.5.
            return_type=return_type
        )(instance)

class UnionGenerator(MagicTypeGenerator):

    def _metaclass_check_getitem_type_decl(cls, type_decl):
        if type_object(type_decl) or not isinstance(type_decl, tuple):
            return False

        for T in type_decl:
            if nontype_object(T):
                return False
        else:
            return True

    def _metaclass_check_subclass(cls, subclass):
        return False

    def _metaclass_check_instance(cls, instance):
        if cls.partial_cls is None:
            return False

        for candidate_cls in cls.partial_cls:
            if isinstance(instance, candidate_cls):
                return True
        return False

class OptionalGenerator(MagicTypeGenerator):

    def _metaclass_check_getitem_type_decl(cls, type_decl):
        return type_object(type_decl)

    def _metaclass_check_subclass(cls, subclass):
        return False

    def _metaclass_check_instance(cls, instance):
        if cls.partial_cls is None:
            return False

        if instance is None:
            return True
        else:
            return isinstance(instance, cls.partial_cls)

def dummy_class(name):
    return type(conditional_to_bytes(name), (object,), {})

class AnyMeta(ABCMeta):

    def __instancecheck__(cls, instance):
        return True

    def __subclasscheck__(cls, subclass):
        return type_object(subclass)

    def __repr__(self):
        return conditional_to_bytes('Any')

class Any(with_metaclass(AnyMeta, object)):
    pass

ABCImmutableSequence = generate_immutable_abc(
    abc.Sequence, abc.MutableSequence,
)
ABCImmutableSet = generate_immutable_abc(
    abc.Set, abc.MutableSet,
)
ABCImmutableMapping = generate_immutable_abc(
    abc.Mapping, abc.MutableMapping,
)

Sequence = SequenceGenerator(abc.Sequence)
MutableSequence = SequenceGenerator(abc.MutableSequence)
ImmutableSequence = SequenceGenerator(ABCImmutableSequence)

Set = SetGenerator(abc.Set)
MutableSet = SetGenerator(abc.MutableSet)
ImmutableSet = SetGenerator(ABCImmutableSet)

Mapping = MappingGenerator(abc.Mapping)
MutableMapping = MappingGenerator(abc.MutableMapping)
ImmutableMapping = MappingGenerator(ABCImmutableMapping)

Iterator = IteratorGenerator(abc.Iterator)
Iterable = IterableGenerator(abc.Iterable)

Callable = CallableGenerator(abc.Callable)

Union = UnionGenerator(dummy_class('Union'))
Optional = OptionalGenerator(dummy_class('Optional'))
NoneType = type(None)

from magic_constraints.decorator import (
    function_constraints,
)  # noqa
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# RatTools
#
# Git clones rat-tools and installs the various sub projects
#
# Author P G Jones - 15/10/2012 <p.g.jones@qmul.ac.uk> : First revision
####################################################################################################
import localpackage
import os
import system

class RatTools(localpackage.LocalPackage):
    """ Base class for rat-tools."""
    def __init__(self, name, system, root_dep):
        """ Initialise rat-tools."""
        self._root_dep = root_dep
        super(RatTools, self).__init__(name, system)
    def get_dependencies(self):
        """ Depends on rat-dev and root."""
        return ["rat-dev", self._root_dep]
    def _is_downloaded(self):
        """ Check if downloaded."""
        return os.path.exists(self.get_install_path())
    def _is_installed(self):
        """ RATZDAB wont install on macs."""
        if self._system.get_os_type == system.System.Mac:
            return self._system.file_exists('README.md',os.path.join(self.get_install_path(),'README.md'))
        else:
            return self._system.file_exists('zdab2root',os.path.join(self.get_install_path(),
                                                                     os.path.join('ratzdab','bin')))
    def _download(self):
        """ Git clone rat-dev."""
        self._system.execute_command("git", ["clone", "git@github.com:snoplus/rat-tools.git",
                                             self.get_install_path()],
                                     verbose=True)
    def _install(self):
        """ Install RATZDAB, except on macs."""
        env = {"RATROOT" : self._dependency_paths["rat-dev"],
               "ROOTSYS" : self._dependency_paths[self._root_dep]}
        ratzdab_path = os.path.join(self.get_install_path(), "ratzdab")
        if self._system.get_os_type() == system.System.Mac:
            return
        self._system.execute_command("make", [], ratzdab_path, env)
    def _update(self):
        """ Special updater for rat-tools, just git pull then install again."""
        self._system.execute_command("git", ["pull"], cwd=self.get_install_path(), verbose=True)
        self._install()
<commit_msg>Fixed rattools installer.<commit_after>#!/usr/bin/env python
#
# RatTools
#
# Git clones rat-tools and installs the various sub projects
#
# Author P G Jones - 15/10/2012 <p.g.jones@qmul.ac.uk> : First revision
####################################################################################################
import localpackage
import os
import system

class RatTools(localpackage.LocalPackage):
    """ Base class for rat-tools."""
    def __init__(self, name, system, root_dep):
        """ Initialise rat-tools."""
        self._root_dep = root_dep
        super(RatTools, self).__init__(name, system)
    def get_dependencies(self):
        """ Depends on rat-dev and root."""
        return ["rat-dev", self._root_dep]
    def _is_downloaded(self):
        """ Check if downloaded."""
        return os.path.exists(self.get_install_path())
    def _is_installed(self):
        """ RATZDAB wont install on macs."""
        if self._system.get_os_type == system.System.Mac:
            return self._system.file_exists('README.md',os.path.join(self.get_install_path(),'README.md'))
        else:
            return self._system.file_exists('zdab2root',os.path.join(self.get_install_path(),
                                                                     os.path.join('ratzdab','bin')))
    def _download(self):
        """ Git clone rat-dev."""
        self._system.execute_command("git", ["clone", "git@github.com:snoplus/rat-tools.git",
                                             self.get_install_path()],
                                     verbose=True)
    def _install(self):
        """ Install RATZDAB, except on macs."""
        env = {"RATROOT" : self._dependency_paths["rat-dev"],
               "ROOTSYS" : self._dependency_paths[self._root_dep],
               "LD_LIBRARY_PATH" : os.path.join(self._dependency_paths[self._root_dep], "lib")}
        ratzdab_path = os.path.join(self.get_install_path(), "ratzdab")
        if self._system.get_os_type() == system.System.Mac:
            return
        self._system.execute_command("make", [], ratzdab_path, env)
    def _update(self):
        """ Special updater for rat-tools, just git pull then install again."""
        self._system.execute_command("git", ["pull"], cwd=self.get_install_path(), verbose=True)
        self._install()
<|endoftext|>


<commit_before># Copyright 2013 OpenStack Foundation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import netaddr

from tempest.api.network import base_routers as base
from tempest.common.utils import data_utils
from tempest import config
from tempest import exceptions
from tempest import test

CONF = config.CONF

class RoutersNegativeTest(base.BaseRouterTest):
    _interface = 'json'

    @classmethod
    def resource_setup(cls):
        super(RoutersNegativeTest, cls).resource_setup()
        if not test.is_extension_enabled('router', 'network'):
            msg = "router extension not enabled."
            raise cls.skipException(msg)
        cls.router = cls.create_router(data_utils.rand_name('router-'))
        cls.network = cls.create_network()
        cls.subnet = cls.create_subnet(cls.network)
        cls.tenant_cidr = (CONF.network.tenant_network_cidr
                           if cls._ip_version == 4 else
                           CONF.network.tenant_network_v6_cidr)

    @test.attr(type=['negative', 'smoke'])
    def test_router_add_gateway_invalid_network_returns_404(self):
        self.assertRaises(exceptions.NotFound,
                          self.client.update_router,
                          self.router['id'],
                          external_gateway_info={
                              'network_id': self.router['id']})

    @test.attr(type=['negative', 'smoke'])
    def test_router_add_gateway_net_not_external_returns_400(self):
        alt_network = self.create_network(
            network_name=data_utils.rand_name('router-negative-'))
        sub_cidr = netaddr.IPNetwork(self.tenant_cidr).next()
        self.create_subnet(alt_network, cidr=sub_cidr)
        self.assertRaises(exceptions.BadRequest,
                          self.client.update_router,
                          self.router['id'],
                          external_gateway_info={
                              'network_id': alt_network['id']})

    @test.attr(type=['negative', 'smoke'])
    def test_add_router_interfaces_on_overlapping_subnets_returns_400(self):
        network01 = self.create_network(
            network_name=data_utils.rand_name('router-network01-'))
        network02 = self.create_network(
            network_name=data_utils.rand_name('router-network02-'))
        subnet01 = self.create_subnet(network01)
        subnet02 = self.create_subnet(network02)
        self._add_router_interface_with_subnet_id(self.router['id'],
                                                  subnet01['id'])
        self.assertRaises(exceptions.BadRequest,
                          self._add_router_interface_with_subnet_id,
                          self.router['id'],
                          subnet02['id'])

    @test.attr(type=['negative', 'smoke'])
    def test_router_remove_interface_in_use_returns_409(self):
        self.client.add_router_interface_with_subnet_id(
            self.router['id'], self.subnet['id'])
        self.assertRaises(exceptions.Conflict,
                          self.client.delete_router,
                          self.router['id'])

class RoutersNegativeIpV6Test(RoutersNegativeTest):
    _ip_version = 6
<commit_msg>Routers Negative API Test<commit_after># Copyright 2013 OpenStack Foundation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import netaddr

from tempest.api.network import base_routers as base
from tempest.common.utils import data_utils
from tempest import config
from tempest import exceptions
from tempest import test

CONF = config.CONF

class RoutersNegativeTest(base.BaseRouterTest):
    _interface = 'json'

    @classmethod
    def resource_setup(cls):
        super(RoutersNegativeTest, cls).resource_setup()
        if not test.is_extension_enabled('router', 'network'):
            msg = "router extension not enabled."
            raise cls.skipException(msg)
        cls.router = cls.create_router(data_utils.rand_name('router-'))
        cls.network = cls.create_network()
        cls.subnet = cls.create_subnet(cls.network)
        cls.tenant_cidr = (CONF.network.tenant_network_cidr
                           if cls._ip_version == 4 else
                           CONF.network.tenant_network_v6_cidr)

    @test.attr(type=['negative', 'smoke'])
    def test_router_add_gateway_invalid_network_returns_404(self):
        self.assertRaises(exceptions.NotFound,
                          self.client.update_router,
                          self.router['id'],
                          external_gateway_info={
                              'network_id': self.router['id']})

    @test.attr(type=['negative', 'smoke'])
    def test_router_add_gateway_net_not_external_returns_400(self):
        alt_network = self.create_network(
            network_name=data_utils.rand_name('router-negative-'))
        sub_cidr = netaddr.IPNetwork(self.tenant_cidr).next()
        self.create_subnet(alt_network, cidr=sub_cidr)
        self.assertRaises(exceptions.BadRequest,
                          self.client.update_router,
                          self.router['id'],
                          external_gateway_info={
                              'network_id': alt_network['id']})

    @test.attr(type=['negative', 'smoke'])
    def test_add_router_interfaces_on_overlapping_subnets_returns_400(self):
        network01 = self.create_network(
            network_name=data_utils.rand_name('router-network01-'))
        network02 = self.create_network(
            network_name=data_utils.rand_name('router-network02-'))
        subnet01 = self.create_subnet(network01)
        subnet02 = self.create_subnet(network02)
        self._add_router_interface_with_subnet_id(self.router['id'],
                                                  subnet01['id'])
        self.assertRaises(exceptions.BadRequest,
                          self._add_router_interface_with_subnet_id,
                          self.router['id'],
                          subnet02['id'])

    @test.attr(type=['negative', 'smoke'])
    def test_router_remove_interface_in_use_returns_409(self):
        self.client.add_router_interface_with_subnet_id(
            self.router['id'], self.subnet['id'])
        self.assertRaises(exceptions.Conflict,
                          self.client.delete_router,
                          self.router['id'])

    @test.attr(type=['negative', 'smoke'])
    def test_show_non_existent_router_returns_404(self):
        router = data_utils.rand_name('non_exist_router')
        self.assertRaises(exceptions.NotFound, self.client.show_router,
                          router)

    @test.attr(type=['negative', 'smoke'])
    def test_update_non_existent_router_returns_404(self):
        router = data_utils.rand_name('non_exist_router')
        self.assertRaises(exceptions.NotFound, self.client.update_router,
                          router, name="new_name")

    @test.attr(type=['negative', 'smoke'])
    def test_delete_non_existent_router_returns_404(self):
        router = data_utils.rand_name('non_exist_router')
        self.assertRaises(exceptions.NotFound, self.client.delete_router,
                          router)

class RoutersNegativeIpV6Test(RoutersNegativeTest):
    _ip_version = 6
<|endoftext|>