<commit_before>"""Class Namespaces (internal module).

All of the guts of the class namespace implementation.

"""

# See https://blog.ionelmc.ro/2015/02/09/understanding-python-metaclasses/

import collections.abc
import functools
import itertools
import weakref

from . import ops
from .descriptor_inspector import _DescriptorInspector
from .flags import ENABLE_SET_NAME
from .proxy import _Proxy
from .scope_proxy import _ScopeProxy

_PROXY_INFOS = weakref.WeakKeyDictionary()

def _mro_to_chained(mro, path):
    """Return a chained map of lookups for the given namespace and mro."""
    return collections.ChainMap(*[
        Namespace.get_namespace(cls, path) for cls in
        itertools.takewhile(
            functools.partial(Namespace.no_blocker, path),
            (cls for cls in mro if isinstance(cls, _Namespaceable))) if
        Namespace.namespace_exists(path, cls)])

def _instance_map(ns_proxy):
    """Return a map, possibly chained, of lookups for the given instance."""
    dct, instance, _ = _PROXY_INFOS[ns_proxy]
    if instance is not None:
        if isinstance(instance, _Namespaceable):
            return _mro_to_chained(instance.__mro__, dct.path)
        else:
            return Namespace.get_namespace(instance, dct.path)
    else:
        return {}

def _mro_map(ns_proxy):
    """Return a chained map of lookups for the given owner class."""
    dct, _, owner = _PROXY_INFOS[ns_proxy]
    mro = owner.__mro__
    parent_object = dct.parent_object
    index = mro.index(parent_object)
    mro = mro[index:]
    return _mro_to_chained(mro, dct.path)

def _retarget(ns_proxy):
    """Convert a class lookup to an instance lookup, if needed."""
    dct, instance, owner = _PROXY_INFOS[ns_proxy]
    if instance is None and isinstance(type(owner), _Namespaceable):
        instance, owner = owner, type(owner)
        dct = Namespace.get_namespace(owner, dct.path)
        ns_proxy = _NamespaceProxy(dct, instance, owner)
    return ns_proxy

class _NamespaceProxy(_Proxy):

    """Proxy object for manipulating and querying namespaces."""

    __slots__ = '__weakref__',

    def __init__(self, dct, instance, owner):
        _PROXY_INFOS[self] = dct, instance, owner

    def __dir__(self):
        return collections.ChainMap(_instance_map(self), _mro_map(self))

    def __getattribute__(self, name):
        self = _retarget(self)
        _, instance, owner = _PROXY_INFOS[self]
        instance_map = _instance_map(self)
        mro_map = _mro_map(self)
        instance_value = ops.get(instance_map, name)
        mro_value = ops.get(mro_map, name)
        if ops.is_data(mro_value):
            return mro_value.get(instance, owner)
        elif issubclass(owner, type) and ops.has_get(instance_value):
            return instance_value.get(None, instance)
        elif instance_value is not None:
            return instance_value.object
        elif ops.has_get(mro_value):
            return mro_value.get(instance, owner)
        elif mro_value is not None:
            return mro_value.object
        else:
            raise AttributeError(name)

    def __setattr__(self, name, value):
        self = _retarget(self)
        dct, instance, owner = _PROXY_INFOS[self]
        if instance is None:
            real_map = Namespace.get_namespace(owner, dct.path)
            real_map[name] = value
            return
        mro_map = _mro_map(self)
        target_value = ops.get_data(mro_map, name)
        if target_value is not None:
            # These lines will be called on a data descriptor.
            target_value.set(instance, value)
            return
        instance_map = Namespace.get_namespace(instance, dct.path)
        instance_map[name] = value

    def __delattr__(self, name):
        self = _retarget(self)
        dct, instance, owner = _PROXY_INFOS[self]
        real_map = Namespace.get_namespace(owner, dct.path)
        if instance is None:
            ops.delete(real_map, name)
            return
        value = ops.get_data(real_map, name)
        if value is not None:
            # These lines will be called on a data descriptor.
            value.delete(instance)
            return
        instance_map = Namespace.get_namespace(instance, dct.path)
        ops.delete(instance_map, name)

_NAMESPACE_INFOS = weakref.WeakKeyDictionary()

class Namespace(dict):

    """Namespace."""

    __slots__ = (
        'name', 'scope', 'parent', 'active', 'parent_object', 'needs_setup')

    __namespaces = {}

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        bad_values = tuple(
            value for value in self.values() if
            isinstance(value, (Namespace, _Proxy)))
        if bad_values:
            raise ValueError('Bad values: {}'.format(bad_values))
        self.name = None
        self.scope = None
        self.parent = None
        self.active = False
        self.parent_object = None
        self.needs_setup = False

    @classmethod
    def premake(cls, name, parent):
        """Return an empty namespace with the given name and parent."""
        self = cls()
        self.name = name
        self.parent = parent
        return self

    # Hold up. Do we need a symmetric addon to __delitem__?
    # I forget how this works.
    def __setitem__(self, key, value):
        if (
                self.scope is not None and
                isinstance(value, self.scope.scope_proxy)):
            value = self.scope.proxies[value]
        if isinstance(value, (_ScopeProxy, _NamespaceProxy)):
            raise ValueError('Cannot move scopes between classes.')
        if isinstance(value, Namespace):
            value.push(key, self.scope, self)
            value.add(self.parent_object)
        super().__setitem__(key, value)

    def __enter__(self):
        self.needs_setup = True
        self.activate()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        if self.name is None:
            raise RuntimeError('Namespace must be named.')
        self.deactivate()

    @property
    def path(self):
        """Return the full path of the namespace."""
        if self.name is None or self.parent is None:
            # This line can be hit by Namespace().path.
            raise ValueError
        if isinstance(self.parent, Namespace):
            parent_path = self.parent.path
        else:
            parent_path = ()
        return parent_path + (self.name,)

    @classmethod
    def __get_helper(cls, target, path):
        """Return the namespace for `target` at `path`, create if needed."""
        if isinstance(target, _Namespaceable):
            path_ = target, path
            namespaces = cls.__namespaces
        else:
            path_ = path
            try:
                namespaces = _NAMESPACE_INFOS[target]
            except KeyError:
                namespaces = {}
                _NAMESPACE_INFOS[target] = namespaces
        return path_, namespaces

    @classmethod
    def namespace_exists(cls, path, target):
        """Return whether the given namespace exists."""
        path_, namespaces = cls.__get_helper(target, path)
        return path_ in namespaces

    @classmethod
    def get_namespace(cls, target, path):
        """Return a namespace with given target and path, create if needed."""
        path_, namespaces = cls.__get_helper(target, path)
        try:
            return namespaces[path_]
        except KeyError:
            if len(path) == 1:
                parent = {}
            else:
                parent = cls.get_namespace(target, path[:-1])
            return namespaces.setdefault(path_, cls.premake(path[-1], parent))

    @classmethod
    def no_blocker(cls, path, cls_):
        """Return False if there's a non-Namespace object in the path."""
        try:
            namespace = vars(cls_)
            for name in path:
                namespace = namespace[name]
                if not isinstance(namespace, cls):
                    return False
        except KeyError:
            pass
        return True

    def add(self, target):
        """Add self as a namespace under target."""
        if target is not None:
            path, namespaces = self.__get_helper(target, self.path)
            res = namespaces.setdefault(path, self)
            if res is self:
                self.parent_object = target

    def set_if_none(self, name, value):
        """Set the attribute `name` to `value`, if it's initially None."""
        if getattr(self, name) is None:
            setattr(self, name, value)

    def validate_assignment(self, name, scope):
        """Confirm the instance can be assigned to the given name and scope."""
        if name != self.name:
            raise ValueError('Cannot rename namespace')
        if scope is not self.scope:
            # It should be possible to hit this line by assigning a namespace
            # into another class. It may not be, however.
            raise ValueError('Cannot reuse namespace')

    def validate_parent(self, parent):
        """Confirm that the instance has the correct parent dict."""
        if parent is not self.parent:
            # This line can be hit by assigning a namespace into another
            # namespace.
            raise ValueError('Cannot reparent namespace')

    def push(self, name, scope, parent):
        """Bind self to the given name and scope, and activate."""
        self.set_if_none('name', name)
        self.set_if_none('scope', scope)
        self.set_if_none('parent', parent)
        self.validate_assignment(name, scope)
        self.validate_parent(parent)
        self.scope.namespaces.append(self)
        if self.needs_setup:
            self.activate()

    def activate(self):
        """Take over as the scope for the target."""
        if self.active:
            raise ValueError('Cannot double-activate.')
        if self.scope is not None:
            self.validate_parent(self.scope.dicts[0])
            self.active = True
            self.scope.dicts.insert(0, self)
            self.needs_setup = False

    def deactivate(self):
        """Stop being the scope for the target."""
        if self.scope is not None and self.active:
            self.active = False
            self.scope.dicts.pop(0)

    def __get__(self, instance, owner):
        return _NamespaceProxy(self, instance, owner)

    def __bool__(self):
        return True

class _NamespaceScope(collections.abc.MutableMapping):

    """The class creation namespace for _Namespaceables."""

    __slots__ = 'dicts', 'namespaces', 'proxies', 'scope_proxy'

    def __init__(self, dct):
        self.dicts = [dct]
        self.namespaces = []
        self.proxies = proxies = weakref.WeakKeyDictionary()

        class ScopeProxy(_ScopeProxy):

            """Local version of ScopeProxy for this scope."""

            __slots__ = ()

            def __init__(self, dct):
                super().__init__(dct, proxies)

        self.scope_proxy = ScopeProxy

    # Mapping methods need to know about the dot syntax.
    # Possibly namespaces themselves should know. Would simplify some things.

    def __getitem__(self, key):
        value = collections.ChainMap(*self.dicts)[key]
        if isinstance(value, Namespace):
            value = self.scope_proxy(value)
        return value

    def __setitem__(self, key, value):
        dct = self.dicts[0]
        # We just entered the context successfully.
        if value is dct:
            dct = self.dicts[1]
        if isinstance(value, Namespace):
            value.push(key, self, dct)
        if isinstance(value, self.scope_proxy):
            value = self.proxies[value]
            value.validate_parent(dct)
            value.validate_assignment(key, self)
        if isinstance(value, (_ScopeProxy, _NamespaceProxy)):
            raise ValueError('Cannot move scopes between classes.')
        dct[key] = value

    def __delitem__(self, key):
        del self.dicts[0][key]

    # These functions are incorrect and need to be rewritten.
    def __iter__(self):
        return iter(collections.ChainMap(*self.dicts))

    def __len__(self):
        return len(collections.ChainMap(*self.dicts))

_NAMESPACE_SCOPES = weakref.WeakKeyDictionary()

class _NamespaceBase:

    """Common base class for Namespaceable and its metaclass."""

    __slots__ = ()

    # Note: the dot format of invocation can "escape" self into other objects.
    # This is not intended behavior, and the result of using dots "too deeply"
    # should be considered undefined.
    # I would like it to be an error, if I can figure out how.

    @staticmethod
    def __is_proxy(value):
        if not isinstance(value, _NamespaceProxy):
            # This line can be hit by doing what the error message says.
            raise ValueError('Given a dot attribute that went too deep.')
        return value

    def __getattribute__(self, name):
        parent, is_namespace, name_ = name.rpartition('.')
        if is_namespace:
            self_ = self
            for element in parent.split('.'):
                self_ = self.__is_proxy(getattr(self_, element))
            return getattr(getattr(self, parent), name_)
        return super(_NamespaceBase, type(self)).__getattribute__(self, name)

    def __setattr__(self, name, value):
        parent, is_namespace, name_ = name.rpartition('.')
        if is_namespace:
            setattr(self.__is_proxy(getattr(self, parent)), name_, value)
            return
        super(_NamespaceBase, type(self)).__setattr__(self, name, value)

    def __delattr__(self, name):
        parent, is_namespace, name_ = name.rpartition('.')
        if is_namespace:
            delattr(self.__is_proxy(getattr(self, parent)), name_)
            return
        # This line can be hit by deleting an attribute that isn't a namespace.
        super(_NamespaceBase, type(self)).__delattr__(self, name)

class _Namespaceable(_NamespaceBase, type):

    """Metaclass for classes that can contain namespaces.

    Using the metaclass directly is a bad idea. Use a base class instead.
    """

    @classmethod
    def __prepare__(mcs, name, bases, **kwargs):
        return _NamespaceScope(super().__prepare__(name, bases, **kwargs))

    def __new__(mcs, name, bases, dct, **kwargs):
        cls = super().__new__(mcs, name, bases, dct.dicts[0], **kwargs)
        if _DEFINED and not issubclass(cls, Namespaceable):
            # This line can be hit with class(metaclass=type(Namespaceable)):
            raise ValueError(
                'Cannot create a _Namespaceable that does not inherit from '
                'Namespaceable')
        _NAMESPACE_SCOPES[cls] = dct
        for namespace in dct.namespaces:
            namespace.add(cls)
            if ENABLE_SET_NAME:
                for name, value in namespace.items():
                    wrapped = _DescriptorInspector(value)
                    if wrapped.has_set_name:
                        wrapped.set_name(cls, name)
        return cls

    def __setattr__(cls, name, value):
        if (
                '.' not in name and isinstance(value, Namespace) and
                value.name != name):
            scope = _NAMESPACE_SCOPES[cls]
            value.push(name, scope, scope.dicts[-1])
            value.add(cls)
        super(_Namespaceable, type(cls)).__setattr__(cls, name, value)

_DEFINED = False

class Namespaceable(_NamespaceBase, metaclass=_Namespaceable):

    """Base class for classes that can contain namespaces.

    A note for people extending the functionality:
    The base class for Namespaceable and its metaclass uses a non-standard
    super() invocation in its definitions of several methods. This was the only
    way I could find to mitigate some bugs I encountered with a standard
    invocation. If you override any of methods defined on built-in types, I
    recommend this form for maximal reusability:

    super(class, type(self)).__method__(self, ...)

    This avoids confusing error messages in case self is a subclass of class,
    in addition to being an instance.

    If you're not delegating above Namespaceable, you can probably use the
    standard invocation, unless you bring about the above situation on your own
    types.
    """

_DEFINED = True
<commit_msg>Start hiding the details of scope implementation.<commit_after>"""Class Namespaces (internal module).

All of the guts of the class namespace implementation.

"""

# See https://blog.ionelmc.ro/2015/02/09/understanding-python-metaclasses/

import collections.abc
import functools
import itertools
import weakref

from . import ops
from .descriptor_inspector import _DescriptorInspector
from .flags import ENABLE_SET_NAME
from .proxy import _Proxy
from .scope_proxy import _ScopeProxy

_PROXY_INFOS = weakref.WeakKeyDictionary()

def _mro_to_chained(mro, path):
    """Return a chained map of lookups for the given namespace and mro."""
    return collections.ChainMap(*[
        Namespace.get_namespace(cls, path) for cls in
        itertools.takewhile(
            functools.partial(Namespace.no_blocker, path),
            (cls for cls in mro if isinstance(cls, _Namespaceable))) if
        Namespace.namespace_exists(path, cls)])

def _instance_map(ns_proxy):
    """Return a map, possibly chained, of lookups for the given instance."""
    dct, instance, _ = _PROXY_INFOS[ns_proxy]
    if instance is not None:
        if isinstance(instance, _Namespaceable):
            return _mro_to_chained(instance.__mro__, dct.path)
        else:
            return Namespace.get_namespace(instance, dct.path)
    else:
        return {}

def _mro_map(ns_proxy):
    """Return a chained map of lookups for the given owner class."""
    dct, _, owner = _PROXY_INFOS[ns_proxy]
    mro = owner.__mro__
    parent_object = dct.parent_object
    index = mro.index(parent_object)
    mro = mro[index:]
    return _mro_to_chained(mro, dct.path)

def _retarget(ns_proxy):
    """Convert a class lookup to an instance lookup, if needed."""
    dct, instance, owner = _PROXY_INFOS[ns_proxy]
    if instance is None and isinstance(type(owner), _Namespaceable):
        instance, owner = owner, type(owner)
        dct = Namespace.get_namespace(owner, dct.path)
        ns_proxy = _NamespaceProxy(dct, instance, owner)
    return ns_proxy

class _NamespaceProxy(_Proxy):

    """Proxy object for manipulating and querying namespaces."""

    __slots__ = '__weakref__',

    def __init__(self, dct, instance, owner):
        _PROXY_INFOS[self] = dct, instance, owner

    def __dir__(self):
        return collections.ChainMap(_instance_map(self), _mro_map(self))

    def __getattribute__(self, name):
        self = _retarget(self)
        _, instance, owner = _PROXY_INFOS[self]
        instance_map = _instance_map(self)
        mro_map = _mro_map(self)
        instance_value = ops.get(instance_map, name)
        mro_value = ops.get(mro_map, name)
        if ops.is_data(mro_value):
            return mro_value.get(instance, owner)
        elif issubclass(owner, type) and ops.has_get(instance_value):
            return instance_value.get(None, instance)
        elif instance_value is not None:
            return instance_value.object
        elif ops.has_get(mro_value):
            return mro_value.get(instance, owner)
        elif mro_value is not None:
            return mro_value.object
        else:
            raise AttributeError(name)

    def __setattr__(self, name, value):
        self = _retarget(self)
        dct, instance, owner = _PROXY_INFOS[self]
        if instance is None:
            real_map = Namespace.get_namespace(owner, dct.path)
            real_map[name] = value
            return
        mro_map = _mro_map(self)
        target_value = ops.get_data(mro_map, name)
        if target_value is not None:
            # These lines will be called on a data descriptor.
            target_value.set(instance, value)
            return
        instance_map = Namespace.get_namespace(instance, dct.path)
        instance_map[name] = value

    def __delattr__(self, name):
        self = _retarget(self)
        dct, instance, owner = _PROXY_INFOS[self]
        real_map = Namespace.get_namespace(owner, dct.path)
        if instance is None:
            ops.delete(real_map, name)
            return
        value = ops.get_data(real_map, name)
        if value is not None:
            # These lines will be called on a data descriptor.
            value.delete(instance)
            return
        instance_map = Namespace.get_namespace(instance, dct.path)
        ops.delete(instance_map, name)

_NAMESPACE_INFOS = weakref.WeakKeyDictionary()

class Namespace(dict):

    """Namespace."""

    __slots__ = (
        'name', 'scope', 'parent', 'active', 'parent_object', 'needs_setup')

    __namespaces = {}

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        bad_values = tuple(
            value for value in self.values() if
            isinstance(value, (Namespace, _Proxy)))
        if bad_values:
            raise ValueError('Bad values: {}'.format(bad_values))
        self.name = None
        self.scope = None
        self.parent = None
        self.active = False
        self.parent_object = None
        self.needs_setup = False

    @classmethod
    def premake(cls, name, parent):
        """Return an empty namespace with the given name and parent."""
        self = cls()
        self.name = name
        self.parent = parent
        return self

    # Hold up. Do we need a symmetric addon to __delitem__?
    # I forget how this works.
    def __setitem__(self, key, value):
        if (
                self.scope is not None and
                isinstance(value, self.scope.scope_proxy)):
            value = self.scope.proxies[value]
        if isinstance(value, (_ScopeProxy, _NamespaceProxy)):
            raise ValueError('Cannot move scopes between classes.')
        if isinstance(value, Namespace):
            value.push(key, self.scope, self)
            value.add(self.parent_object)
        super().__setitem__(key, value)

    def __enter__(self):
        self.needs_setup = True
        self.activate()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        if self.name is None:
            raise RuntimeError('Namespace must be named.')
        self.deactivate()

    @property
    def path(self):
        """Return the full path of the namespace."""
        if self.name is None or self.parent is None:
            # This line can be hit by Namespace().path.
            raise ValueError
        if isinstance(self.parent, Namespace):
            parent_path = self.parent.path
        else:
            parent_path = ()
        return parent_path + (self.name,)

    @classmethod
    def __get_helper(cls, target, path):
        """Return the namespace for `target` at `path`, create if needed."""
        if isinstance(target, _Namespaceable):
            path_ = target, path
            namespaces = cls.__namespaces
        else:
            path_ = path
            try:
                namespaces = _NAMESPACE_INFOS[target]
            except KeyError:
                namespaces = {}
                _NAMESPACE_INFOS[target] = namespaces
        return path_, namespaces

    @classmethod
    def namespace_exists(cls, path, target):
        """Return whether the given namespace exists."""
        path_, namespaces = cls.__get_helper(target, path)
        return path_ in namespaces

    @classmethod
    def get_namespace(cls, target, path):
        """Return a namespace with given target and path, create if needed."""
        path_, namespaces = cls.__get_helper(target, path)
        try:
            return namespaces[path_]
        except KeyError:
            if len(path) == 1:
                parent = {}
            else:
                parent = cls.get_namespace(target, path[:-1])
            return namespaces.setdefault(path_, cls.premake(path[-1], parent))

    @classmethod
    def no_blocker(cls, path, cls_):
        """Return False if there's a non-Namespace object in the path."""
        try:
            namespace = vars(cls_)
            for name in path:
                namespace = namespace[name]
                if not isinstance(namespace, cls):
                    return False
        except KeyError:
            pass
        return True

    def add(self, target):
        """Add self as a namespace under target."""
        if target is not None:
            path, namespaces = self.__get_helper(target, self.path)
            res = namespaces.setdefault(path, self)
            if res is self:
                self.parent_object = target

    def set_if_none(self, name, value):
        """Set the attribute `name` to `value`, if it's initially None."""
        if getattr(self, name) is None:
            setattr(self, name, value)

    def validate_assignment(self, name, scope):
        """Confirm the instance can be assigned to the given name and scope."""
        if name != self.name:
            raise ValueError('Cannot rename namespace')
        if scope is not self.scope:
            # It should be possible to hit this line by assigning a namespace
            # into another class. It may not be, however.
            raise ValueError('Cannot reuse namespace')

    def validate_parent(self, parent):
        """Confirm that the instance has the correct parent dict."""
        if parent is not self.parent:
            # This line can be hit by assigning a namespace into another
            # namespace.
            raise ValueError('Cannot reparent namespace')

    def push(self, name, scope, parent):
        """Bind self to the given name and scope, and activate."""
        self.set_if_none('name', name)
        self.set_if_none('scope', scope)
        self.set_if_none('parent', parent)
        self.validate_assignment(name, scope)
        self.validate_parent(parent)
        self.scope.namespaces.append(self)
        if self.needs_setup:
            self.activate()

    def activate(self):
        """Take over as the scope for the target."""
        if self.active:
            raise ValueError('Cannot double-activate.')
        if self.scope is not None:
            self.validate_parent(self.scope.head)
            self.active = True
            self.scope.dicts.insert(0, self)
            self.needs_setup = False

    def deactivate(self):
        """Stop being the scope for the target."""
        if self.scope is not None and self.active:
            self.active = False
            self.scope.dicts.pop(0)

    def __get__(self, instance, owner):
        return _NamespaceProxy(self, instance, owner)

    def __bool__(self):
        return True

class _NamespaceScope(collections.abc.MutableMapping):

    """The class creation namespace for _Namespaceables."""

    __slots__ = 'dicts', 'namespaces', 'proxies', 'scope_proxy'

    def __init__(self, dct):
        self.dicts = [dct]
        self.namespaces = []
        self.proxies = proxies = weakref.WeakKeyDictionary()

        class ScopeProxy(_ScopeProxy):

            """Local version of ScopeProxy for this scope."""

            __slots__ = ()

            def __init__(self, dct):
                super().__init__(dct, proxies)

        self.scope_proxy = ScopeProxy

    # Mapping methods need to know about the dot syntax.
    # Possibly namespaces themselves should know. Would simplify some things.

    @property
    def head(self):
        """The innermost Namespace scope."""
        return self.dicts[0]

    def __getitem__(self, key):
        value = collections.ChainMap(*self.dicts)[key]
        if isinstance(value, Namespace):
            value = self.scope_proxy(value)
        return value

    def __setitem__(self, key, value):
        dct = self.head
        # We just entered the context successfully.
        if value is dct:
            dct = self.dicts[1]
        if isinstance(value, Namespace):
            value.push(key, self, dct)
        if isinstance(value, self.scope_proxy):
            value = self.proxies[value]
            value.validate_parent(dct)
            value.validate_assignment(key, self)
        if isinstance(value, (_ScopeProxy, _NamespaceProxy)):
            raise ValueError('Cannot move scopes between classes.')
        dct[key] = value

    def __delitem__(self, key):
        del self.head[key]

    # These functions are incorrect and need to be rewritten.
    def __iter__(self):
        return iter(collections.ChainMap(*self.dicts))

    def __len__(self):
        return len(collections.ChainMap(*self.dicts))

_NAMESPACE_SCOPES = weakref.WeakKeyDictionary()

class _NamespaceBase:

    """Common base class for Namespaceable and its metaclass."""

    __slots__ = ()

    # Note: the dot format of invocation can "escape" self into other objects.
    # This is not intended behavior, and the result of using dots "too deeply"
    # should be considered undefined.
    # I would like it to be an error, if I can figure out how.

    @staticmethod
    def __is_proxy(value):
        if not isinstance(value, _NamespaceProxy):
            # This line can be hit by doing what the error message says.
            raise ValueError('Given a dot attribute that went too deep.')
        return value

    def __getattribute__(self, name):
        parent, is_namespace, name_ = name.rpartition('.')
        if is_namespace:
            self_ = self
            for element in parent.split('.'):
                self_ = self.__is_proxy(getattr(self_, element))
            return getattr(getattr(self, parent), name_)
        return super(_NamespaceBase, type(self)).__getattribute__(self, name)

    def __setattr__(self, name, value):
        parent, is_namespace, name_ = name.rpartition('.')
        if is_namespace:
            setattr(self.__is_proxy(getattr(self, parent)), name_, value)
            return
        super(_NamespaceBase, type(self)).__setattr__(self, name, value)

    def __delattr__(self, name):
        parent, is_namespace, name_ = name.rpartition('.')
        if is_namespace:
            delattr(self.__is_proxy(getattr(self, parent)), name_)
            return
        # This line can be hit by deleting an attribute that isn't a namespace.
        super(_NamespaceBase, type(self)).__delattr__(self, name)

class _Namespaceable(_NamespaceBase, type):

    """Metaclass for classes that can contain namespaces.

    Using the metaclass directly is a bad idea. Use a base class instead.
    """

    @classmethod
    def __prepare__(mcs, name, bases, **kwargs):
        return _NamespaceScope(super().__prepare__(name, bases, **kwargs))

    def __new__(mcs, name, bases, dct, **kwargs):
        cls = super().__new__(mcs, name, bases, dct.dicts[0], **kwargs)
        if _DEFINED and not issubclass(cls, Namespaceable):
            # This line can be hit with class(metaclass=type(Namespaceable)):
            raise ValueError(
                'Cannot create a _Namespaceable that does not inherit from '
                'Namespaceable')
        _NAMESPACE_SCOPES[cls] = dct
        for namespace in dct.namespaces:
            namespace.add(cls)
            if ENABLE_SET_NAME:
                for name, value in namespace.items():
                    wrapped = _DescriptorInspector(value)
                    if wrapped.has_set_name:
                        wrapped.set_name(cls, name)
        return cls

    def __setattr__(cls, name, value):
        if (
                '.' not in name and isinstance(value, Namespace) and
                value.name != name):
            scope = _NAMESPACE_SCOPES[cls]
            value.push(name, scope, scope.dicts[-1])
            value.add(cls)
        super(_Namespaceable, type(cls)).__setattr__(cls, name, value)

_DEFINED = False

class Namespaceable(_NamespaceBase, metaclass=_Namespaceable):

    """Base class for classes that can contain namespaces.

    A note for people extending the functionality:
    The base class for Namespaceable and its metaclass uses a non-standard
    super() invocation in its definitions of several methods. This was the only
    way I could find to mitigate some bugs I encountered with a standard
    invocation. If you override any of methods defined on built-in types, I
    recommend this form for maximal reusability:

    super(class, type(self)).__method__(self, ...)

    This avoids confusing error messages in case self is a subclass of class,
    in addition to being an instance.

    If you're not delegating above Namespaceable, you can probably use the
    standard invocation, unless you bring about the above situation on your own
    types.
    """

_DEFINED = True
<|endoftext|>


<commit_before>#!/usr/bin/python3

__author__ = 'm_messiah'
import logging
import signal
from requests import Session
import tornado.ioloop
import tornado.web
import tornado.escape

from bot_token import BOT_TOKEN
from commands import CMD

logging.basicConfig(filename="conversation.log", level=logging.INFO)
requests_log = logging.getLogger("requests.packages.urllib3")
requests_log.setLevel(logging.WARNING)

URL = "https://api.telegram.org/bot%s/" % BOT_TOKEN
MyURL = ""  # TODO: add url here, when get valid HTTPS

def not_found(arguments, message):
    return "Command not found. Try /help"

def send_reply(response):
    if 'sticker' in response:
        api.post(URL + "sendSticker", data=response)
    elif 'text' in response:
        api.post(URL + "sendMessage", data=response)

# noinspection PyAbstractClass
class Handler(tornado.web.RequestHandler):
        def post(self):
            try:
                logging.debug("Got request: %s" % self.request.body)
                update = tornado.escape.json_decode(self.request.body)
                message = update['message']
                text = message.get('text')
                if text:
                    logging.info("MESSAGE\t%s\t%s" % (
                        message['from']['id'],
                        text))

                    if text[0] == '/':
                        command, *arguments = text.split(" ", 1)
                        response = CMD.get(command, not_found)(arguments,
                                                               message)

                        logging.info("REPLY\t%s\t%s" % (
                            message['from']['id'],
                            response
                        ))

                        send_reply(response)
                    else:
                        response = CMD["<speech>"](message)
                        logging.info("REPLY\t%s\t%s" % (
                            message['from']['id'],
                            response
                        ))
                        send_reply(response)

                elif message.get("sticker"):
                    send_reply(
                        {'chat_id': message['from']['id'],
                         'text': "Sticker id = %s" % message[
                             "sticker"].get("file_id")
                        })

            except Exception as e:
                logging.warning(str(e))

def signal_term_handler(signal, frame):
    logging.error("Got SIGTERM. Quit.")
    api.get(URL + "setWebhook?url=")
    exit(0)

api = Session()
application = tornado.web.Application([
    (r"/", Handler),
])

if __name__ == '__main__':
    signal.signal(signal.SIGTERM, signal_term_handler)
    try:
        set_hook = api.get(URL + "setWebhook?url=%s" % MyURL)
        if set_hook.status_code != 200:
            logging.error("Can't set hook: %s. Quit." % set_hook.text)
            exit(1)
        application.listen(8888)
        tornado.ioloop.IOLoop.current().start()
    except KeyboardInterrupt:
        signal_term_handler(signal.SIGTERM, None)
<commit_msg>fix command not found<commit_after>#!/usr/bin/python3

__author__ = 'm_messiah'
import logging
import signal
from requests import Session
import tornado.ioloop
import tornado.web
import tornado.escape

from bot_token import BOT_TOKEN
from commands import CMD

logging.basicConfig(filename="conversation.log", level=logging.INFO)
requests_log = logging.getLogger("requests.packages.urllib3")
requests_log.setLevel(logging.WARNING)

URL = "https://api.telegram.org/bot%s/" % BOT_TOKEN
MyURL = ""  # TODO: add url here, when get valid HTTPS

def not_found(arguments, message):
    return {'chat_id': message['from']['id'],
            'text': "Command not found. Try /help"
            }

def send_reply(response):
    if 'sticker' in response:
        api.post(URL + "sendSticker", data=response)
    elif 'text' in response:
        api.post(URL + "sendMessage", data=response)

# noinspection PyAbstractClass
class Handler(tornado.web.RequestHandler):
        def post(self):
            try:
                logging.debug("Got request: %s" % self.request.body)
                update = tornado.escape.json_decode(self.request.body)
                message = update['message']
                text = message.get('text')
                if text:
                    logging.info("MESSAGE\t%s\t%s" % (
                        message['from']['id'],
                        text))

                    if text[0] == '/':
                        command, *arguments = text.split(" ", 1)
                        response = CMD.get(command, not_found)(arguments,
                                                               message)

                        logging.info("REPLY\t%s\t%s" % (
                            message['from']['id'],
                            response
                        ))

                        send_reply(response)
                    else:
                        response = CMD["<speech>"](message)
                        logging.info("REPLY\t%s\t%s" % (
                            message['from']['id'],
                            response
                        ))
                        send_reply(response)

                elif message.get("sticker"):
                    send_reply(
                        {'chat_id': message['from']['id'],
                         'text': "Sticker id = %s" % message[
                             "sticker"].get("file_id")
                        })

            except Exception as e:
                logging.warning(str(e))

def signal_term_handler(signal, frame):
    logging.error("Got SIGTERM. Quit.")
    api.get(URL + "setWebhook?url=")
    exit(0)

api = Session()
application = tornado.web.Application([
    (r"/", Handler),
])

if __name__ == '__main__':
    signal.signal(signal.SIGTERM, signal_term_handler)
    try:
        set_hook = api.get(URL + "setWebhook?url=%s" % MyURL)
        if set_hook.status_code != 200:
            logging.error("Can't set hook: %s. Quit." % set_hook.text)
            exit(1)
        application.listen(8888)
        tornado.ioloop.IOLoop.current().start()
    except KeyboardInterrupt:
        signal_term_handler(signal.SIGTERM, None)
<|endoftext|>


<commit_before>"""Main Controller"""

from tg import expose, redirect, config, validate
from tg.decorators import use_custom_format
from tg.controllers import TGController
from tw.forms import TableForm, TextField, CalendarDatePicker, SingleSelectField, TextArea
from tw.api import WidgetsList

class MovieForm(TableForm):
    # This WidgetsList is just a container
    class fields(WidgetsList):
        title = TextField()
        year = TextField(size=4)
        description = TextArea()

#then, we create an instance of this form
base_movie_form = MovieForm("movie_form", action='create')

class RootController(TGController):
    @expose('genshi:index.html')
    def index(self):
        return {}

    @expose('genshi:genshi_form.html')
    def form(self):
        return dict(form=base_movie_form)

    @expose('json')
    @validate(form=base_movie_form)
    def process_form_errors(self, **kwargs):
        #add error messages to the kwargs dictionary and return it
        kwargs['errors'] = pylons.tmpl_context.form_errors
        return dict(kwargs)

    @expose('genshi:genshi_inherits.html')
    def genshi_inherits(self):
        return {}

    @expose('jinja:tg/test_stack/rendering/templates/jinja_noop.html')
    def jinja_index(self):
        return {}

    @expose('jinja:tg/test_stack/rendering/templates/jinja_inherits.html')
    def jinja_inherits(self):
        return {}

    @expose('chameleon_genshi:index.html')
    def chameleon_genshi_index(self):
        return {}

    @expose('chameleon_genshi:genshi_inherits.html')
    def chameleon_genshi_inherits(self):
        return {}

    @expose('mako:mako_noop.mak')
    def mako_index(self):
        return {}

    @expose('mako:mako_inherits.mak')
    def mako_inherits(self):
        return {}

    @expose('chameleon_genshi:tg.test_stack.rendering.templates.index')
    def chameleon_index_dotted(self):
        return {}

    @expose('genshi:tg.test_stack.rendering.templates.index')
    def index_dotted(self):
        return {}

    @expose('genshi:tg.test_stack.rendering.templates.genshi_inherits')
    def genshi_inherits_dotted(self):
        return {}

    @expose('mako:tg.test_stack.rendering.templates.mako_noop')
    def mako_index_dotted(self):
        return {}

    @expose('mako:tg.test_stack.rendering.templates.mako_inherits_dotted')
    def mako_inherits_dotted(self):
        return {}

    @expose('json', custom_format='json')
    @expose('mako:mako_custom_format.mak', content_type='text/xml', custom_format='xml')
    @expose('genshi:genshi_custom_format.html', content_type='text/html', custom_format='html')
    def custom_format(self, format):
        use_custom_format(self.custom_format, format)
        return dict(format=format, status="ok")
<commit_msg>In the Jinja templating test the templates were specified with additional path components which should not be needed, and in fact the tests only worked for me after removing these.<commit_after>"""Main Controller"""

from tg import expose, redirect, config, validate
from tg.decorators import use_custom_format
from tg.controllers import TGController
from tw.forms import TableForm, TextField, CalendarDatePicker, SingleSelectField, TextArea
from tw.api import WidgetsList

class MovieForm(TableForm):
    # This WidgetsList is just a container
    class fields(WidgetsList):
        title = TextField()
        year = TextField(size=4)
        description = TextArea()

#then, we create an instance of this form
base_movie_form = MovieForm("movie_form", action='create')

class RootController(TGController):
    @expose('genshi:index.html')
    def index(self):
        return {}

    @expose('genshi:genshi_form.html')
    def form(self):
        return dict(form=base_movie_form)

    @expose('json')
    @validate(form=base_movie_form)
    def process_form_errors(self, **kwargs):
        #add error messages to the kwargs dictionary and return it
        kwargs['errors'] = pylons.tmpl_context.form_errors
        return dict(kwargs)

    @expose('genshi:genshi_inherits.html')
    def genshi_inherits(self):
        return {}

    @expose('jinja:jinja_noop.html')
    def jinja_index(self):
        return {}

    @expose('jinja:jinja_inherits.html')
    def jinja_inherits(self):
        return {}

    @expose('chameleon_genshi:index.html')
    def chameleon_genshi_index(self):
        return {}

    @expose('chameleon_genshi:genshi_inherits.html')
    def chameleon_genshi_inherits(self):
        return {}

    @expose('mako:mako_noop.mak')
    def mako_index(self):
        return {}

    @expose('mako:mako_inherits.mak')
    def mako_inherits(self):
        return {}

    @expose('chameleon_genshi:tg.test_stack.rendering.templates.index')
    def chameleon_index_dotted(self):
        return {}

    @expose('genshi:tg.test_stack.rendering.templates.index')
    def index_dotted(self):
        return {}

    @expose('genshi:tg.test_stack.rendering.templates.genshi_inherits')
    def genshi_inherits_dotted(self):
        return {}

    @expose('mako:tg.test_stack.rendering.templates.mako_noop')
    def mako_index_dotted(self):
        return {}

    @expose('mako:tg.test_stack.rendering.templates.mako_inherits_dotted')
    def mako_inherits_dotted(self):
        return {}

    @expose('json', custom_format='json')
    @expose('mako:mako_custom_format.mak', content_type='text/xml', custom_format='xml')
    @expose('genshi:genshi_custom_format.html', content_type='text/html', custom_format='html')
    def custom_format(self, format):
        use_custom_format(self.custom_format, format)
        return dict(format=format, status="ok")
<|endoftext|>


<commit_before>from ekklesia_portal.datamodel import CustomizableText

def customizable_text(request, name):
    lang = request.i18n.get_locale().language
    # TODO: add fallback to default language and/or maybe a default text from the translations.
    customizable_text = request.q(CustomizableText).filter_by(lang=lang, name=name).scalar()
    if customizable_text is None:
        return name
    else:
        return customizable_text.text
<commit_msg>customizable_text helper: use fallback language, add lang arg<commit_after>from ekklesia_portal.datamodel import CustomizableText

def customizable_text(request, name, lang=None):
    if lang is None:
        lang = request.i18n.get_locale().language

    customizable_text = request.q(CustomizableText).filter_by(lang=lang, name=name).scalar()

    if customizable_text is None:
        fallback_language = request.app.settings.app.fallback_language
        customizable_text = request.q(CustomizableText).filter_by(lang=fallback_language, name=name).scalar()

    if customizable_text is None:
        return name
    else:
        return customizable_text.text
<|endoftext|>


<commit_before>from functools import update_wrapper
from inspect import signature, Parameter

from parsl.app.errors import wrap_error
from parsl.app.app import AppBase
from parsl.dataflow.dflow import DataFlowKernelLoader

def remote_side_bash_executor(func, *args, **kwargs):
    """Execute the bash app type function and return the command line string.

    This string is reformatted with the *args, and **kwargs
    from call time.
    """
    import os
    import time
    import subprocess
    import logging
    import parsl.app.errors as pe
    from parsl import set_file_logger

    logbase = "/tmp"
    format_string = "%(asctime)s.%(msecs)03d %(name)s:%(lineno)d [%(levelname)s]  %(message)s"

    # make this name unique per invocation so that each invocation can
    # log to its own file. It would be better to include the task_id here
    # but that is awkward to wire through at the moment as apps do not
    # have access to that execution context.
    t = time.time()

    logname = __name__ + "." + str(t)
    logger = logging.getLogger(logname)
    set_file_logger(filename='{0}/bashexec.{1}.log'.format(logbase, t), name=logname, level=logging.DEBUG, format_string=format_string)

    func_name = func.__name__

    executable = None

    # Try to run the func to compose the commandline
    try:
        # Execute the func to get the commandline
        executable = func(*args, **kwargs)

    except AttributeError as e:
        if executable is not None:
            raise pe.AppBadFormatting("App formatting failed for app '{}' with AttributeError: {}".format(func_name, e))
        else:
            raise pe.BashAppNoReturn("Bash app '{}' did not return a value, or returned None - with this exception: {}".format(func_name, e), None)

    except IndexError as e:
        raise pe.AppBadFormatting("App formatting failed for app '{}' with IndexError: {}".format(func_name, e))
    except Exception as e:
        logger.error("Caught exception during formatting of app '{}': {}".format(func_name, e))
        raise e

    logger.debug("Executable: %s", executable)

    # Updating stdout, stderr if values passed at call time.

    def open_std_fd(fdname):
        # fdname is 'stdout' or 'stderr'
        stdfspec = kwargs.get(fdname)  # spec is str name or tuple (name, mode)
        if stdfspec is None:
            return None
        elif isinstance(stdfspec, str):
            fname = stdfspec
            mode = 'a+'
        elif isinstance(stdfspec, tuple):
            if len(stdfspec) != 2:
                raise pe.BadStdStreamFile("std descriptor %s has incorrect tuple length %s" % (fdname, len(stdfspec)), TypeError('Bad Tuple Length'))
            fname, mode = stdfspec
        else:
            raise pe.BadStdStreamFile("std descriptor %s has unexpected type %s" % (fdname, str(type(stdfspec))), TypeError('Bad Tuple Type'))

        try:
            if os.path.dirname(fname):
                os.makedirs(os.path.dirname(fname), exist_ok=True)
            fd = open(fname, mode)
        except Exception as e:
            raise pe.BadStdStreamFile(fname, e)
        return fd

    std_out = open_std_fd('stdout')
    std_err = open_std_fd('stderr')
    timeout = kwargs.get('walltime')

    if std_err is not None:
        print('--> executable follows <--\n{}\n--> end executable <--'.format(executable), file=std_err, flush=True)

    returncode = None
    try:
        proc = subprocess.Popen(executable, stdout=std_out, stderr=std_err, shell=True, executable='/bin/bash')
        proc.wait(timeout=timeout)
        returncode = proc.returncode

    except subprocess.TimeoutExpired:
        raise pe.AppTimeout("[{}] App exceeded walltime: {}".format(func_name, timeout))

    except Exception as e:
        raise pe.AppException("[{}] App caught exception: {}".format(func_name, proc.returncode), e)

    if returncode != 0:
        raise pe.AppFailure("[{}] App failed with exit code: {}".format(func_name, proc.returncode), proc.returncode)

    # TODO : Add support for globs here

    missing = []
    for outputfile in kwargs.get('outputs', []):
        fpath = outputfile
        if type(outputfile) != str:
            fpath = outputfile.filepath

        if not os.path.exists(fpath):
            missing.extend([outputfile])

    if missing:
        raise pe.MissingOutputs("[{}] Missing outputs".format(func_name), missing)

    return returncode

class BashApp(AppBase):

    def __init__(self, func, data_flow_kernel=None, walltime=60, cache=False, executors='all'):
        super().__init__(func, data_flow_kernel=data_flow_kernel, walltime=60, executors=executors, cache=cache)
        self.kwargs = {}

        # We duplicate the extraction of parameter defaults
        # to self.kwargs to ensure availability at point of
        # command string format. Refer: #349
        sig = signature(func)

        for s in sig.parameters:
            if sig.parameters[s].default != Parameter.empty:
                self.kwargs[s] = sig.parameters[s].default

    def __call__(self, *args, **kwargs):
        """Handle the call to a Bash app.

        Args:
             - Arbitrary

        Kwargs:
             - Arbitrary

        Returns:
                   App_fut

        """
        # Update kwargs in the app definition with ones passed in at calltime
        self.kwargs.update(kwargs)

        if self.data_flow_kernel is None:
            dfk = DataFlowKernelLoader.dfk()
        else:
            dfk = self.data_flow_kernel

        app_fut = dfk.submit(wrap_error(update_wrapper(remote_side_bash_executor, self.func)),
                             self.func, *args,
                             executors=self.executors,
                             fn_hash=self.func_hash,
                             cache=self.cache,
                             **self.kwargs)

        return app_fut
<commit_msg>Update remote_side_bash_executor docstring to modern reality (#1283)<commit_after>from functools import update_wrapper
from inspect import signature, Parameter

from parsl.app.errors import wrap_error
from parsl.app.app import AppBase
from parsl.dataflow.dflow import DataFlowKernelLoader

def remote_side_bash_executor(func, *args, **kwargs):
    """Executes the supplied function with *args and **kwargs to get a
    command-line to run, and then run that command-line using bash.
    """
    import os
    import time
    import subprocess
    import logging
    import parsl.app.errors as pe
    from parsl import set_file_logger

    logbase = "/tmp"
    format_string = "%(asctime)s.%(msecs)03d %(name)s:%(lineno)d [%(levelname)s]  %(message)s"

    # make this name unique per invocation so that each invocation can
    # log to its own file. It would be better to include the task_id here
    # but that is awkward to wire through at the moment as apps do not
    # have access to that execution context.
    t = time.time()

    logname = __name__ + "." + str(t)
    logger = logging.getLogger(logname)
    set_file_logger(filename='{0}/bashexec.{1}.log'.format(logbase, t), name=logname, level=logging.DEBUG, format_string=format_string)

    func_name = func.__name__

    executable = None

    # Try to run the func to compose the commandline
    try:
        # Execute the func to get the commandline
        executable = func(*args, **kwargs)

    except AttributeError as e:
        if executable is not None:
            raise pe.AppBadFormatting("App formatting failed for app '{}' with AttributeError: {}".format(func_name, e))
        else:
            raise pe.BashAppNoReturn("Bash app '{}' did not return a value, or returned None - with this exception: {}".format(func_name, e), None)

    except IndexError as e:
        raise pe.AppBadFormatting("App formatting failed for app '{}' with IndexError: {}".format(func_name, e))
    except Exception as e:
        logger.error("Caught exception during formatting of app '{}': {}".format(func_name, e))
        raise e

    logger.debug("Executable: %s", executable)

    # Updating stdout, stderr if values passed at call time.

    def open_std_fd(fdname):
        # fdname is 'stdout' or 'stderr'
        stdfspec = kwargs.get(fdname)  # spec is str name or tuple (name, mode)
        if stdfspec is None:
            return None
        elif isinstance(stdfspec, str):
            fname = stdfspec
            mode = 'a+'
        elif isinstance(stdfspec, tuple):
            if len(stdfspec) != 2:
                raise pe.BadStdStreamFile("std descriptor %s has incorrect tuple length %s" % (fdname, len(stdfspec)), TypeError('Bad Tuple Length'))
            fname, mode = stdfspec
        else:
            raise pe.BadStdStreamFile("std descriptor %s has unexpected type %s" % (fdname, str(type(stdfspec))), TypeError('Bad Tuple Type'))

        try:
            if os.path.dirname(fname):
                os.makedirs(os.path.dirname(fname), exist_ok=True)
            fd = open(fname, mode)
        except Exception as e:
            raise pe.BadStdStreamFile(fname, e)
        return fd

    std_out = open_std_fd('stdout')
    std_err = open_std_fd('stderr')
    timeout = kwargs.get('walltime')

    if std_err is not None:
        print('--> executable follows <--\n{}\n--> end executable <--'.format(executable), file=std_err, flush=True)

    returncode = None
    try:
        proc = subprocess.Popen(executable, stdout=std_out, stderr=std_err, shell=True, executable='/bin/bash')
        proc.wait(timeout=timeout)
        returncode = proc.returncode

    except subprocess.TimeoutExpired:
        raise pe.AppTimeout("[{}] App exceeded walltime: {}".format(func_name, timeout))

    except Exception as e:
        raise pe.AppException("[{}] App caught exception: {}".format(func_name, proc.returncode), e)

    if returncode != 0:
        raise pe.AppFailure("[{}] App failed with exit code: {}".format(func_name, proc.returncode), proc.returncode)

    # TODO : Add support for globs here

    missing = []
    for outputfile in kwargs.get('outputs', []):
        fpath = outputfile
        if type(outputfile) != str:
            fpath = outputfile.filepath

        if not os.path.exists(fpath):
            missing.extend([outputfile])

    if missing:
        raise pe.MissingOutputs("[{}] Missing outputs".format(func_name), missing)

    return returncode

class BashApp(AppBase):

    def __init__(self, func, data_flow_kernel=None, walltime=60, cache=False, executors='all'):
        super().__init__(func, data_flow_kernel=data_flow_kernel, walltime=60, executors=executors, cache=cache)
        self.kwargs = {}

        # We duplicate the extraction of parameter defaults
        # to self.kwargs to ensure availability at point of
        # command string format. Refer: #349
        sig = signature(func)

        for s in sig.parameters:
            if sig.parameters[s].default != Parameter.empty:
                self.kwargs[s] = sig.parameters[s].default

    def __call__(self, *args, **kwargs):
        """Handle the call to a Bash app.

        Args:
             - Arbitrary

        Kwargs:
             - Arbitrary

        Returns:
                   App_fut

        """
        # Update kwargs in the app definition with ones passed in at calltime
        self.kwargs.update(kwargs)

        if self.data_flow_kernel is None:
            dfk = DataFlowKernelLoader.dfk()
        else:
            dfk = self.data_flow_kernel

        app_fut = dfk.submit(wrap_error(update_wrapper(remote_side_bash_executor, self.func)),
                             self.func, *args,
                             executors=self.executors,
                             fn_hash=self.func_hash,
                             cache=self.cache,
                             **self.kwargs)

        return app_fut
<|endoftext|>


<commit_before>import utilities
import subprocess
import rasterio
import numpy as np
from scipy import stats

def create_continent_ecozone_tiles(tile_id):

    print "Processing:", tile_id

    output_dir = 's3://gfw2-data/climate/carbon_model/fao_ecozones/ecozone_continent/20181001/'
    file_name_base_raw = 'fao_ecozones_continents_raw'
    file_name_base_processed = 'fao_ecozones_continents_processed'

    print "Getting extent of biomass tile"
    ymax, xmin, ymin, xmax = utilities.coords(tile_id)
    print "ymax:", ymax, "; ymin:", ymin, "; xmax", xmax, "; xmin:", xmin

    print "Rasterizing ecozone to extent of biomass tile"

    cont_eco_raw = "{0}_{1}".format(file_name_base_raw, tile_id)

    # utilities.rasterize('fao_ecozones_fra_2000_continents_assigned_dissolved_FINAL_20180906.shp',
    #                                           cont_eco_raw, xmin, ymin, xmax, ymax, '.00025', 'Int16', 'gainEcoCon', '0')

    utilities.upload_final(file_name_base_raw, '{}raw/'.format(output_dir), tile_id)

    # Opens continent-ecozone tile
    with rasterio.open('{}.tif'.format(cont_eco_raw)) as cont_eco_raw_src:

        # Grabs metadata about the tif, like its location/projection/cellsize
        kwargs = cont_eco_raw_src.meta

        # Grabs the windows of the tile (stripes) to iterate over the entire tif without running out of memory
        windows = cont_eco_raw_src.block_windows(1)

        # Updates kwargs for the output dataset.
        # Need to update data type to float 32 so that it can handle fractional gain rates
        kwargs.update(
            driver='GTiff',
            count=1,
            compress='lzw',
            nodata=0
        )

        # Opens the output tile, giving it the arguments of the input tiles
        with rasterio.open('{0}_{1}.tif'.format(file_name_base_processed, tile_id), 'w', **kwargs) as dst:

            # Iterates across the windows (1 pixel strips) of the input tile
            for idx, window in windows:

                # Creates windows for each input raster
                cont_eco_raw = cont_eco_raw_src.read(1, window=window)
                print cont_eco_raw

                non_zeros = np.delete(cont_eco_raw, np.where(cont_eco_raw[0] == 0))
                print non_zeros

                if non_zeros.size < 1:

                    print "All zeros"
                    mode = 0

                else:

                    print "Not all zeros"
                    mode = stats.mode(non_zeros[0])

                print mode

                cont_eco_processed = cont_eco_raw

                cont_eco_processed[cont_eco_processed == 0] = mode

                # Writes the output window to the output
                dst.write_band(1, cont_eco_processed, window=window)

    utilities.upload_final(file_name_base_processed, '{}processed/'.format(output_dir), tile_id)

<commit_msg>Modifying the continent-ecozone tile generation script to assign continent-ecozone to pixels that don't currently have it.<commit_after>import utilities
import subprocess
import rasterio
import numpy as np
from scipy import stats

def create_continent_ecozone_tiles(tile_id):

    print "Processing:", tile_id

    output_dir = 's3://gfw2-data/climate/carbon_model/fao_ecozones/ecozone_continent/20181001/'
    file_name_base_raw = 'fao_ecozones_continents_raw'
    file_name_base_processed = 'fao_ecozones_continents_processed'

    print "Getting extent of biomass tile"
    ymax, xmin, ymin, xmax = utilities.coords(tile_id)
    print "ymax:", ymax, "; ymin:", ymin, "; xmax", xmax, "; xmin:", xmin

    print "Rasterizing ecozone to extent of biomass tile"

    cont_eco_raw = "{0}_{1}".format(file_name_base_raw, tile_id)

    # utilities.rasterize('fao_ecozones_fra_2000_continents_assigned_dissolved_FINAL_20180906.shp',
    #                                           cont_eco_raw, xmin, ymin, xmax, ymax, '.00025', 'Int16', 'gainEcoCon', '0')

    utilities.upload_final(file_name_base_raw, '{}raw/'.format(output_dir), tile_id)

    # Opens continent-ecozone tile
    with rasterio.open('{}.tif'.format(cont_eco_raw)) as cont_eco_raw_src:

        # Grabs metadata about the tif, like its location/projection/cellsize
        kwargs = cont_eco_raw_src.meta

        # Grabs the windows of the tile (stripes) to iterate over the entire tif without running out of memory
        windows = cont_eco_raw_src.block_windows(1)

        # Updates kwargs for the output dataset.
        # Need to update data type to float 32 so that it can handle fractional gain rates
        kwargs.update(
            driver='GTiff',
            count=1,
            compress='lzw',
            nodata=0
        )

        # Opens the output tile, giving it the arguments of the input tiles
        with rasterio.open('{0}_{1}.tif'.format(file_name_base_processed, tile_id), 'w', **kwargs) as dst:

            # Iterates across the windows (1 pixel strips) of the input tile
            for idx, window in windows:

                # Creates windows for each input raster
                cont_eco_raw = cont_eco_raw_src.read(1, window=window)
                print cont_eco_raw

                cont_eco_raw_flat = np.flatten(cont_eco_raw)

                non_zeros = np.delete(cont_eco_raw_flat, np.where(cont_eco_raw_flat[0] == 0))
                print non_zeros

                if non_zeros.size < 1:

                    print "All zeros"
                    mode = 0

                else:

                    print "Not all zeros"
                    mode = stats.mode(non_zeros[0])

                print mode

                cont_eco_processed = cont_eco_raw

                cont_eco_processed[cont_eco_processed == 0] = mode

                # Writes the output window to the output
                dst.write_band(1, cont_eco_processed, window=window)

    utilities.upload_final(file_name_base_processed, '{}processed/'.format(output_dir), tile_id)

<|endoftext|>


<commit_before>from __future__ import absolute_import,unicode_literals

import logging
logger = logging.getLogger(__name__)

def initHostInfo():
    return {
        'format':'pe',
        'platform':'windows',
    }
<commit_msg>hack for old py windows inet_pton and inet_ntop<commit_after>from __future__ import absolute_import,unicode_literals

import ctypes
import socket
import logging

logger = logging.getLogger(__name__)

class sockaddr(ctypes.Structure):
    _fields_ = [
        ("sa_family", ctypes.c_short),
        ("_p1", ctypes.c_ushort),
        ("ipv4", ctypes.c_byte * 4),
        ("ipv6", ctypes.c_byte * 16),
        ("_p2", ctypes.c_ulong)
    ]

if getattr(socket,'inet_pton',None) == None:

    WSAStringToAddressA = ctypes.windll.ws2_32.WSAStringToAddressA

    def inet_pton(fam,text):
        sa = sockaddr()
        sa.sa_family = fam

        size = ctypes.c_int( ctypes.sizeof(sa) )

        saref = ctypes.byref(sa)
        szref = ctypes.byref(size)

        if WSAStringToAddressA(text, fam, None, saref, szref):
            raise socket.error('Invalid Address (fam:%d) %s' % (fam,text))

        return sa.ipv4

    socket.inet_pton = inet_pton

if getattr(socket,'inet_ntop',None) == None:

    WSAAddressToStringA = ctypes.windll.ws2_32.WSAAddressToStringA

    def inet_ntop(fam,byts):
        sa = sockaddr()
        sa.sa_family = fam

        if fam == socket.AF_INET:
            ctypes.memmove(sa.ipv4, byts, 4)

        elif fam == socket.AF_INET6:
            ctypes.memmove(sa.ipv6, byts, 16)

        else:
            raise Exception('Unknown Address Family: %s' % (fam,))

        size = ctypes.c_int(128)
        text = ctypes.create_string_buffer(128)

        szref = ctypes.byref(size)

        if WSAAddressToStringA(sa, ctypes.sizeof(sa), None, text, szref):
            raise socket.error('Invalid Address (fam:%d) %s' % (fam,text))

        return text[:size.value]

    socket.inet_ntop = inet_ntop

def initHostInfo():
    return {
        'format':'pe',
        'platform':'windows',
    }
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
""" This module provides common functions which are usually needed in
all other submodules or interactively.
"""

import collections

from midas.compat import d_itervalues

### Functions that work on MapReduce-style 'key\tvalue' lines ###

def split_key_value(line, sep='\t'):
    return line.strip().split(sep, 1)

def get_key(line, sep='\t'):
    return split_key_value(line, sep)[0]

### Functions that are often needed in interactive use ###

def identity(o):
    return o

def count_by_key(iterable, keyfunc=identity):
    counter = collections.defaultdict(int)
    for item in iterable:
        counter[keyfunc(item)] += 1
    return counter

def collect_by_key(iterable, keyfunc):
    collected = collections.defaultdict(list)
    for item in iterable:
        collected[keyfunc(item)].append(item)
    return collected

def relation_stats(iterable):
    counter = count_by_key(iterable)
    return (sum(d_itervalues(counter)),
            len(counter),
            sum(v for v in d_itervalues(counter) if v > 1),
            sum(1 for v in d_itervalues(counter) if v > 1))

### Functions to query sites and companies ###

def make_p_empty_attr(attr_name):
    def p_empty_attr(o):
        attr = getattr(o, attr_name)
        return attr is None or attr == ''
    return p_empty_attr

### Useful miscelanious stuff ###

def takewhile_common(list1, list2):
    for a, b in zip(list1, list2):
        if a != b:
            break
        yield a

def iter_ts_until_gap(ts, all_dates):
    first_common = all_dates.index(ts[-1])
    return takewhile_common(ts[::-1], all_dates[first_common::-1])
<commit_msg>cleaning up<commit_after># -*- coding: utf-8 -*-
""" This module provides common functions which are usually needed in
all other submodules or interactively.
"""

import collections

from midas.compat import d_itervalues

### Functions that work on MapReduce-style 'key\tvalue' lines ###

def split_key_value(line, sep='\t'):
    return line.strip().split(sep, 1)

def get_key(line, sep='\t'):
    return split_key_value(line, sep)[0]

### Functions that are often needed in interactive use ###

def identity(o):
    return o

def count_by_key(iterable, keyfunc=identity):
    counter = collections.defaultdict(int)
    for item in iterable:
        counter[keyfunc(item)] += 1
    return counter

def collect_by_key(iterable, keyfunc):
    collected = collections.defaultdict(list)
    for item in iterable:
        collected[keyfunc(item)].append(item)
    return collected

def relation_stats(iterable):
    counter = count_by_key(iterable)
    return (sum(d_itervalues(counter)),
            len(counter),
            sum(v for v in d_itervalues(counter) if v > 1),
            sum(1 for v in d_itervalues(counter) if v > 1))

### Functions to query sites and companies ###

def make_p_empty_attr(attr_name):
    def p_empty_attr(o):
        attr = getattr(o, attr_name)
        return attr is None or attr == ''
    return p_empty_attr
<|endoftext|>