<commit_before># Module:   core
# Date:     2nd April 2006
# Author:   James Mills, prologic at shortcircuit dot net dot au

"""
Core of the circuits library containing all of the essentials for a
circuits based application or system. Normal usage of circuits:

>>> from circuits import listener, Manager, Component, Event
"""

import new
import time
from itertools import chain
from threading import Thread
from functools import partial
from collections import deque
from collections import defaultdict
from sys import exc_info as _exc_info
from sys import exc_clear as _exc_clear
from inspect import getargspec, getmembers

class Event(object):
    """Create a new Event Object

    Create a new event object populating it with the given
    list of arguments and keyword arguments.

    :param args: list of arguments for this event
    :type args: list/tuple or iterable
    :param kwargs: keyword arguments for this event
    :type kwargs: dict
    """

    def __init__(self, *args, **kwargs):
        self.args = args
        self.kwargs = kwargs

        self.name = self.__class__.__name__
        self.channel = None
        self.target = None

        self.source = None  # Used by Bridge
        self.ignore = False # Used by Bridge

    def __eq__(self, y):
        """ x.__eq__(y) <==> x==y

        Tests the equality of event self against event y.
        Two events are considered "equal" iif the name,
        channel and target are identical as well as their
        args and kwargs passed.
        """

        attrs = ("name", "args", "kwargs", "channel", "target")
        return all([getattr(self, a) == getattr(y, a) for a in attrs])

    def __repr__(self):
        "x.__repr__() <==> repr(x)"

        if self.channel is not None and self.target is not None:
            channelStr = "%s:%s" % (self.target, self.channel)
        elif self.channel is not None:
            channelStr = self.channel
        else:
            channelStr = ""
        argsStr = ", ".join([("%s" % repr(arg)) for arg in self.args])
        kwargsStr = ", ".join(
                [("%s=%s" % kwarg) for kwarg in self.kwargs.iteritems()])
        return "<%s[%s] (%s, %s)>" % (self.name, channelStr, argsStr, kwargsStr)

    def __getitem__(self, x):
        """x.__getitem__(y) <==> x[y]

        Get and return data from the event object requested by "x".
        If an int is passed to x, the requested argument from self.args
        is returned index by x. If a str is passed to x, the requested
        keyword argument from self.kwargs is returned keyed by x.
        Otherwise a TypeError is raised as nothing else is valid.
        """

        if type(x) == int:
            return self.args[x]
        elif type(x) == str:
            return self.kwargs[x]
        else:
            raise TypeError("Expected int or str, got %r" % type(x))

class Error(Event):
    """Error(type, value, traceback) -> Error Event

    type:      Exception type      -> sys.exc_type
    value:     Exception value     -> sys.exc_value
    traceback: Exception traceback -> sys.exc_traceback
    """

def listener(*args, **kwargs):
    """Creates an Event Handler of a callable object

    Decorator to wrap a callable into an event handler that
    listens on a set of channels defined by args. The type
    of the listener defaults to "listener" and is defined
    by kwargs["type"]. To define a filter, pass type="filter"
    to kwargs. If kwargs["target"] is not None, this event handler
    will be registered and will ignore the channel of it's containing
    Component.

    Examples:

    >>> from circuits.core import listener
    >>> @listener("foo")
    ... def onFOO():
    ...     pass
    >>> @listener("bar", type="filter")
    ... def onBAR():
    ...     pass
    >>> @listener("foo", "bar")
    ... def onFOOBAR():
    ...     pass
    """

    def decorate(f):
        f.type = kwargs.get("type", "listener")
        f.filter = f.type == "filter"
        f.target = kwargs.get("target", None)
        f.channels = args

        _argspec = getargspec(f)
        _args = _argspec[0]
        if _args and _args[0] == "self":
            del _args[0]
        if _args and _args[0] == "event":
            f._passEvent = True
        else:
            f._passEvent = False

        return f
    return decorate

class HandlersType(type):

    def __init__(cls, name, bases, dct):
        super(HandlersType, cls).__init__(name, bases, dct)

        for k, v in dct.iteritems():
            if callable(v) and not (k[0] == "_" or hasattr(v, "type")):
                setattr(cls, k, listener(k, type="listener")(v))

class Manager(object):
    """Creates a new Manager

    Create a new event manager which manages Components and Events.
    """

    def __init__(self, *args, **kwargs):
        "initializes x; see x.__class__.__doc__ for signature"

        self._queue = deque()
        self._handlers = set()
        self._channels = defaultdict(list)

        self._ticks = set()
        self._hidden = set()
        self._components = set()

        self.manager = self

    def __repr__(self):
        q = len(self._queue)
        h = len(self._handlers)
        return "<Manager (q: %d h: %d)>" % (q, h)

    def __len__(self):
        return len(self._queue)

    def __add__(self, y):
        y.register(self.manager)
        return self

    def __iadd__(self, y):
        y.register(self.manager)
        return self

    def __sub__(self, y):
        if y.manager == self:
            y.unregister()
            return self
        else:
            raise TypeError("No registration found for %r" % y)

    def __isub__(self, y):
        if y.manager == self:
            y.unregister()
            return self
        else:
            raise TypeError("No registration found for %r" % y)

    def handlers(self, s):
        if s == "*:*":
            return self._handlers

        if ":" in s:
            target, channel = s.split(":", 1)
        else:
            channel = s
            target = None

        channels = self.channels
        globals = channels["*"]

        if target == "*":
            c = ":%s" % channel
            x = [channels[k] for k in channels if k == channel or k.endswith(c)]
            all = [i for y in x for i in y]
            return chain(globals, all)

        if channel == "*":
            c = "%s:" % target
            x = [channels[k] for k in channels if k.startswith(c) or ":" not in k]
            all = [i for y in x for i in y]
            return chain(globals, all)

        handlers = globals
        if channel in channels:
            handlers = chain(handlers, channels[channel])
        if target and "%s:*" % target in channels:
            handlers = chain(handlers, channels["%s:*" % target])
        if "*:%s" % channel in channels:
            handlers = chain(handlers, channels["*:%s" % channel])
        if target:
            handlers = chain(handlers, channels[s])

        return handlers

    @property
    def components(self):
        return self._components

    @property
    def channels(self):
        return self._channels

    def _add(self, handler, channel=None):
        """E._add(handler, channel) -> None

        Add a new filter or listener to the event manager
        adding it to the given channel. If no channel is
        given, add it to the global channel.
        """

        self._handlers.add(handler)

        if channel is None:
            channel = "*"

        if channel in self.channels:
            if handler not in self.channels[channel]:
                self._channels[channel].append(handler)
                self._channels[channel].sort(key=lambda x: x.type)
        else:
            self._channels[channel] = [handler]

    def _remove(self, handler, channel=None):
        """E._remove(handler, channel=None) -> None

        Remove the given filter or listener from the
        event manager removing it from the given channel.
        if channel is None, remove it from the global
        channel. This will succeed even if the specified
        handler has already been removed.
        """

        if channel is None:
            if handler in self.channels["*"]:
                self._channels["*"].remove(handler)
            keys = self.channels.keys()
        else:
            keys = [channel]

        if handler in self._handlers:
            self._handlers.remove(handler)

        for channel in keys:
            if handler in self.channels[channel]:
                self._channels[channel].remove(handler)

    def push(self, event, channel, target=None):
        """E.push(event, channel, target=None) -> None

        Push the given event onto the given channel.
        This will queue the event up to be processed later
        by flushEvents. If target is given, the event will
        be queued for processing by the component given by
        target.
        """

        if self.manager == self:
            self._queue.append((event, channel, target))
        else:
            self.manager.push(event, channel, target)

    def flush(self):
        """E.flushEvents() -> None

        Flush all events waiting in the queue.
        Any event waiting in the queue will be sent out
        to filters/listeners.
        """

        if self.manager == self:
            q = self._queue
            self._queue = deque()
            while q:
                event, channel, target = q.pop()
                self.send(event, channel, target)
        else:
            self.manager.flush()

    def send(self, event, channel, target=None, errors=False, log=True):
        """E.send(event, channel, target=None, errors=False) -> None

        Send the given event to filters/listeners on the
        channel specified. If target is given, send this
        event to filters/listeners of the given target
        component.
        """

        if self.manager == self:
            event.channel = channel
            event.target = target
            eargs = event.args
            ekwargs = event.kwargs
            if target is not None:
                channel = "%s:%s" % (target, channel)

            r = False
            for handler in self.handlers(channel):
                try:
                    if handler._passEvent:
                        r = partial(handler, event, *eargs, **ekwargs)()
                    else:
                        r = partial(handler, *eargs, **ekwargs)()
                except:
                    if log:
                        self.push(Error(*_exc_info()), "error")
                    if errors:
                        raise
                    else:
                        _exc_clear()
                if r is not None and r and handler.filter:
                    return r
            return r
        else:
            return self.manager.send(event, channel, target, errors, log)

    def start(self, sleep=None):
        group = None
        target = self._run
        name = self.__class__.__name__
        args = (sleep,)
        self.thread = Thread(group, target, name, args)
        self.running = True
        self.thread.setDaemon(True)
        self.thread.start()

    def stop(self):
        self.running = False
        self.thread.join()

    def _run(self, sleep=None):
        while self.running and self.thread.isAlive():
            [f() for f in self._ticks.copy()]
            self.flush()
            if sleep:
                time.sleep(sleep)

    def run(self, sleep=None):
        while True:
            try:
                [f() for f in self._ticks.copy()]
                self.flush()
                if sleep:
                    time.sleep(sleep)
            except (KeyboardInterrupt, SystemExit):
                break

class BaseComponent(Manager):
    """Creates a new Component

    Subclasses of Component define Event Handlers by decorating
    methods by using the listener decorator.

    All listeners found in the Component will automatically be
    picked up when the Component is instantiated.

    :param channel: channel this Component listens on (*default*: ``None``)
    :type channel: str
    """

    channel = None

    def __new__(cls, *args, **kwargs):
        """TODO Work around for Python bug.

        Bug: http://bugs.python.org/issue5322
        """

        return object.__new__(cls)

    def __init__(self, *args, **kwargs):
        "initializes x; see x.__class__.__doc__ for signature"

        super(BaseComponent, self).__init__(*args, **kwargs)

        self.channel = kwargs.get("channel", self.channel)
        self.register(self)

    def __repr__(self):
        name = self.__class__.__name__
        channel = self.channel or ""
        q = len(self._queue)
        h = len(self._handlers)
        return "<%s/%s component (q: %d h: %d)>" % (name, channel, q, h)

    def _registerHidden(self):
        d = 0
        i = 0
        x = self
        stack = []
        done = False
        hidden = set()
        visited = set()
        children = list(x.components)
        while not done:
            if x not in visited:
                p1 = x is not self
                p2 = x.manager is not self
                p3 = x not in self.manager.components
                p4 = x.manager not in hidden
                if p1 and (p2 or p3) and p4:
                    hidden.add(x)
                visited.add(x)
                if x.components:
                    d += 1

            if i < len(children):
                x = children[i]
                i += 1
                if x.components:
                    stack.append((i, children))
                    children = list(x.components)
                    i = 0
            else:
                if stack:
                    i, children = stack.pop()
                    d -= 1
                else:
                    done = True

        for x in hidden:
            x.register(self.manager)

        self.manager._hidden.update(hidden)

    def _getTicks(self):
        ticks = set()
        if hasattr(self, "__tick__"):
            ticks.add(self.__tick__)
        for v in vars(self).itervalues():
            if isinstance(v, Manager):
                if hasattr(v, "__tick__"):
                    ticks.add(v.__tick__)
        for component in self.components:
            if hasattr(component, "__tick__"):
                ticks.add(component.__tick__)
        for component in self._hidden:
            if hasattr(component, "__tick__"):
                ticks.add(component.__tick__)
        return ticks

    def register(self, manager):
        p = lambda x: callable(x) and hasattr(x, "type")
        handlers = [v for k, v in getmembers(self, p)]

        for handler in handlers:
            if handler.channels:
                channels = handler.channels
            else:
                channels = ["*"]

            for channel in channels:
                if self.channel is not None:
                    if handler.target is not None:
                        target = handler.target
                    else:
                        target = self.channel

                    channel = "%s:%s" % (target, channel)

                manager._add(handler, channel)

        self.manager = manager

        if manager is not self:
            self.manager._components.add(self)
            if hasattr(self, "registered"):
                self.registered()
            self._registerHidden()

        self.manager._ticks.update(self._getTicks())

    def unregister(self):
        "Unregister all registered event handlers from the manager."

        for handler in self._handlers.copy():
            self.manager._remove(handler)

        if self in self.manager.components:
            self.manager._components.remove(self)

        self.manager._ticks.difference_update(self._getTicks())

        self.manager = self

        if hasattr(self, "unregistered") and manager is not self:
            self.unregistered()

class Component(BaseComponent):

    __metaclass__ = HandlersType

    def __new__(cls, *args, **kwargs):
        self = BaseComponent.__new__(cls, *args, **kwargs)
        for base in cls.__bases__:
            if issubclass(cls, base):
                for k, v in base.__dict__.iteritems():
                    if callable(v) and hasattr(v, "type"):
                        name = "%s_%s" % (base.__name__, k)
                        method = new.instancemethod(v, self, cls)
                        setattr(self, name, method)
        return self
<commit_msg>core: raise KeyboardInterrupt or SystemExit -always-.<commit_after># Module:   core
# Date:     2nd April 2006
# Author:   James Mills, prologic at shortcircuit dot net dot au

"""
Core of the circuits library containing all of the essentials for a
circuits based application or system. Normal usage of circuits:

>>> from circuits import listener, Manager, Component, Event
"""

import new
import time
from itertools import chain
from threading import Thread
from functools import partial
from collections import deque
from collections import defaultdict
from sys import exc_info as _exc_info
from sys import exc_clear as _exc_clear
from inspect import getargspec, getmembers

class Event(object):
    """Create a new Event Object

    Create a new event object populating it with the given
    list of arguments and keyword arguments.

    :param args: list of arguments for this event
    :type args: list/tuple or iterable
    :param kwargs: keyword arguments for this event
    :type kwargs: dict
    """

    def __init__(self, *args, **kwargs):
        self.args = args
        self.kwargs = kwargs

        self.name = self.__class__.__name__
        self.channel = None
        self.target = None

        self.source = None  # Used by Bridge
        self.ignore = False # Used by Bridge

    def __eq__(self, y):
        """ x.__eq__(y) <==> x==y

        Tests the equality of event self against event y.
        Two events are considered "equal" iif the name,
        channel and target are identical as well as their
        args and kwargs passed.
        """

        attrs = ("name", "args", "kwargs", "channel", "target")
        return all([getattr(self, a) == getattr(y, a) for a in attrs])

    def __repr__(self):
        "x.__repr__() <==> repr(x)"

        if self.channel is not None and self.target is not None:
            channelStr = "%s:%s" % (self.target, self.channel)
        elif self.channel is not None:
            channelStr = self.channel
        else:
            channelStr = ""
        argsStr = ", ".join([("%s" % repr(arg)) for arg in self.args])
        kwargsStr = ", ".join(
                [("%s=%s" % kwarg) for kwarg in self.kwargs.iteritems()])
        return "<%s[%s] (%s, %s)>" % (self.name, channelStr, argsStr, kwargsStr)

    def __getitem__(self, x):
        """x.__getitem__(y) <==> x[y]

        Get and return data from the event object requested by "x".
        If an int is passed to x, the requested argument from self.args
        is returned index by x. If a str is passed to x, the requested
        keyword argument from self.kwargs is returned keyed by x.
        Otherwise a TypeError is raised as nothing else is valid.
        """

        if type(x) == int:
            return self.args[x]
        elif type(x) == str:
            return self.kwargs[x]
        else:
            raise TypeError("Expected int or str, got %r" % type(x))

class Error(Event):
    """Error(type, value, traceback) -> Error Event

    type:      Exception type      -> sys.exc_type
    value:     Exception value     -> sys.exc_value
    traceback: Exception traceback -> sys.exc_traceback
    """

def listener(*args, **kwargs):
    """Creates an Event Handler of a callable object

    Decorator to wrap a callable into an event handler that
    listens on a set of channels defined by args. The type
    of the listener defaults to "listener" and is defined
    by kwargs["type"]. To define a filter, pass type="filter"
    to kwargs. If kwargs["target"] is not None, this event handler
    will be registered and will ignore the channel of it's containing
    Component.

    Examples:

    >>> from circuits.core import listener
    >>> @listener("foo")
    ... def onFOO():
    ...     pass
    >>> @listener("bar", type="filter")
    ... def onBAR():
    ...     pass
    >>> @listener("foo", "bar")
    ... def onFOOBAR():
    ...     pass
    """

    def decorate(f):
        f.type = kwargs.get("type", "listener")
        f.filter = f.type == "filter"
        f.target = kwargs.get("target", None)
        f.channels = args

        _argspec = getargspec(f)
        _args = _argspec[0]
        if _args and _args[0] == "self":
            del _args[0]
        if _args and _args[0] == "event":
            f._passEvent = True
        else:
            f._passEvent = False

        return f
    return decorate

class HandlersType(type):

    def __init__(cls, name, bases, dct):
        super(HandlersType, cls).__init__(name, bases, dct)

        for k, v in dct.iteritems():
            if callable(v) and not (k[0] == "_" or hasattr(v, "type")):
                setattr(cls, k, listener(k, type="listener")(v))

class Manager(object):
    """Creates a new Manager

    Create a new event manager which manages Components and Events.
    """

    def __init__(self, *args, **kwargs):
        "initializes x; see x.__class__.__doc__ for signature"

        self._queue = deque()
        self._handlers = set()
        self._channels = defaultdict(list)

        self._ticks = set()
        self._hidden = set()
        self._components = set()

        self.manager = self

    def __repr__(self):
        q = len(self._queue)
        h = len(self._handlers)
        return "<Manager (q: %d h: %d)>" % (q, h)

    def __len__(self):
        return len(self._queue)

    def __add__(self, y):
        y.register(self.manager)
        return self

    def __iadd__(self, y):
        y.register(self.manager)
        return self

    def __sub__(self, y):
        if y.manager == self:
            y.unregister()
            return self
        else:
            raise TypeError("No registration found for %r" % y)

    def __isub__(self, y):
        if y.manager == self:
            y.unregister()
            return self
        else:
            raise TypeError("No registration found for %r" % y)

    def handlers(self, s):
        if s == "*:*":
            return self._handlers

        if ":" in s:
            target, channel = s.split(":", 1)
        else:
            channel = s
            target = None

        channels = self.channels
        globals = channels["*"]

        if target == "*":
            c = ":%s" % channel
            x = [channels[k] for k in channels if k == channel or k.endswith(c)]
            all = [i for y in x for i in y]
            return chain(globals, all)

        if channel == "*":
            c = "%s:" % target
            x = [channels[k] for k in channels if k.startswith(c) or ":" not in k]
            all = [i for y in x for i in y]
            return chain(globals, all)

        handlers = globals
        if channel in channels:
            handlers = chain(handlers, channels[channel])
        if target and "%s:*" % target in channels:
            handlers = chain(handlers, channels["%s:*" % target])
        if "*:%s" % channel in channels:
            handlers = chain(handlers, channels["*:%s" % channel])
        if target:
            handlers = chain(handlers, channels[s])

        return handlers

    @property
    def components(self):
        return self._components

    @property
    def channels(self):
        return self._channels

    def _add(self, handler, channel=None):
        """E._add(handler, channel) -> None

        Add a new filter or listener to the event manager
        adding it to the given channel. If no channel is
        given, add it to the global channel.
        """

        self._handlers.add(handler)

        if channel is None:
            channel = "*"

        if channel in self.channels:
            if handler not in self.channels[channel]:
                self._channels[channel].append(handler)
                self._channels[channel].sort(key=lambda x: x.type)
        else:
            self._channels[channel] = [handler]

    def _remove(self, handler, channel=None):
        """E._remove(handler, channel=None) -> None

        Remove the given filter or listener from the
        event manager removing it from the given channel.
        if channel is None, remove it from the global
        channel. This will succeed even if the specified
        handler has already been removed.
        """

        if channel is None:
            if handler in self.channels["*"]:
                self._channels["*"].remove(handler)
            keys = self.channels.keys()
        else:
            keys = [channel]

        if handler in self._handlers:
            self._handlers.remove(handler)

        for channel in keys:
            if handler in self.channels[channel]:
                self._channels[channel].remove(handler)

    def push(self, event, channel, target=None):
        """E.push(event, channel, target=None) -> None

        Push the given event onto the given channel.
        This will queue the event up to be processed later
        by flushEvents. If target is given, the event will
        be queued for processing by the component given by
        target.
        """

        if self.manager == self:
            self._queue.append((event, channel, target))
        else:
            self.manager.push(event, channel, target)

    def flush(self):
        """E.flushEvents() -> None

        Flush all events waiting in the queue.
        Any event waiting in the queue will be sent out
        to filters/listeners.
        """

        if self.manager == self:
            q = self._queue
            self._queue = deque()
            while q:
                event, channel, target = q.pop()
                self.send(event, channel, target)
        else:
            self.manager.flush()

    def send(self, event, channel, target=None, errors=False, log=True):
        """E.send(event, channel, target=None, errors=False) -> None

        Send the given event to filters/listeners on the
        channel specified. If target is given, send this
        event to filters/listeners of the given target
        component.
        """

        if self.manager == self:
            event.channel = channel
            event.target = target
            eargs = event.args
            ekwargs = event.kwargs
            if target is not None:
                channel = "%s:%s" % (target, channel)

            r = False
            for handler in self.handlers(channel):
                try:
                    if handler._passEvent:
                        r = partial(handler, event, *eargs, **ekwargs)()
                    else:
                        r = partial(handler, *eargs, **ekwargs)()
                except (KeyboardInterrupt, SystemExit):
                    raise
                except:
                    if log:
                        self.push(Error(*_exc_info()), "error")
                    if errors:
                        raise
                    else:
                        _exc_clear()
                if r is not None and r and handler.filter:
                    return r
            return r
        else:
            return self.manager.send(event, channel, target, errors, log)

    def start(self, sleep=None):
        group = None
        target = self._run
        name = self.__class__.__name__
        args = (sleep,)
        self.thread = Thread(group, target, name, args)
        self.running = True
        self.thread.setDaemon(True)
        self.thread.start()

    def stop(self):
        self.running = False
        self.thread.join()

    def _run(self, sleep=None):
        while self.running and self.thread.isAlive():
            [f() for f in self._ticks.copy()]
            self.flush()
            if sleep:
                time.sleep(sleep)

    def run(self, sleep=None):
        while True:
            try:
                [f() for f in self._ticks.copy()]
                self.flush()
                if sleep:
                    time.sleep(sleep)
            except (KeyboardInterrupt, SystemExit):
                break

class BaseComponent(Manager):
    """Creates a new Component

    Subclasses of Component define Event Handlers by decorating
    methods by using the listener decorator.

    All listeners found in the Component will automatically be
    picked up when the Component is instantiated.

    :param channel: channel this Component listens on (*default*: ``None``)
    :type channel: str
    """

    channel = None

    def __new__(cls, *args, **kwargs):
        """TODO Work around for Python bug.

        Bug: http://bugs.python.org/issue5322
        """

        return object.__new__(cls)

    def __init__(self, *args, **kwargs):
        "initializes x; see x.__class__.__doc__ for signature"

        super(BaseComponent, self).__init__(*args, **kwargs)

        self.channel = kwargs.get("channel", self.channel)
        self.register(self)

    def __repr__(self):
        name = self.__class__.__name__
        channel = self.channel or ""
        q = len(self._queue)
        h = len(self._handlers)
        return "<%s/%s component (q: %d h: %d)>" % (name, channel, q, h)

    def _registerHidden(self):
        d = 0
        i = 0
        x = self
        stack = []
        done = False
        hidden = set()
        visited = set()
        children = list(x.components)
        while not done:
            if x not in visited:
                p1 = x is not self
                p2 = x.manager is not self
                p3 = x not in self.manager.components
                p4 = x.manager not in hidden
                if p1 and (p2 or p3) and p4:
                    hidden.add(x)
                visited.add(x)
                if x.components:
                    d += 1

            if i < len(children):
                x = children[i]
                i += 1
                if x.components:
                    stack.append((i, children))
                    children = list(x.components)
                    i = 0
            else:
                if stack:
                    i, children = stack.pop()
                    d -= 1
                else:
                    done = True

        for x in hidden:
            x.register(self.manager)

        self.manager._hidden.update(hidden)

    def _getTicks(self):
        ticks = set()
        if hasattr(self, "__tick__"):
            ticks.add(self.__tick__)
        for v in vars(self).itervalues():
            if isinstance(v, Manager):
                if hasattr(v, "__tick__"):
                    ticks.add(v.__tick__)
        for component in self.components:
            if hasattr(component, "__tick__"):
                ticks.add(component.__tick__)
        for component in self._hidden:
            if hasattr(component, "__tick__"):
                ticks.add(component.__tick__)
        return ticks

    def register(self, manager):
        p = lambda x: callable(x) and hasattr(x, "type")
        handlers = [v for k, v in getmembers(self, p)]

        for handler in handlers:
            if handler.channels:
                channels = handler.channels
            else:
                channels = ["*"]

            for channel in channels:
                if self.channel is not None:
                    if handler.target is not None:
                        target = handler.target
                    else:
                        target = self.channel

                    channel = "%s:%s" % (target, channel)

                manager._add(handler, channel)

        self.manager = manager

        if manager is not self:
            self.manager._components.add(self)
            if hasattr(self, "registered"):
                self.registered()
            self._registerHidden()

        self.manager._ticks.update(self._getTicks())

    def unregister(self):
        "Unregister all registered event handlers from the manager."

        for handler in self._handlers.copy():
            self.manager._remove(handler)

        if self in self.manager.components:
            self.manager._components.remove(self)

        self.manager._ticks.difference_update(self._getTicks())

        self.manager = self

        if hasattr(self, "unregistered") and manager is not self:
            self.unregistered()

class Component(BaseComponent):

    __metaclass__ = HandlersType

    def __new__(cls, *args, **kwargs):
        self = BaseComponent.__new__(cls, *args, **kwargs)
        for base in cls.__bases__:
            if issubclass(cls, base):
                for k, v in base.__dict__.iteritems():
                    if callable(v) and hasattr(v, "type"):
                        name = "%s_%s" % (base.__name__, k)
                        method = new.instancemethod(v, self, cls)
                        setattr(self, name, method)
        return self
<|endoftext|>


<commit_before>"""
Tests for the pylearn2 autoencoder module.
"""
import numpy as np
import theano
import theano.tensor as tensor
from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams
from pylearn2.autoencoder import Autoencoder

def test_autoencoder_logistic_linear_tied():
    data = np.random.randn(10, 5)
    ae = Autoencoder(5, 7, act_enc='sigmoid', act_dec='linear',
                     tied_weights=True)
    w = ae.weights.get_value()
    ae.hidbias.set_value(np.random.randn(7))
    hb = ae.hidbias.get_value()
    ae.visbias.set_value(np.random.randn(5))
    vb = ae.visbias.get_value()
    d = tensor.matrix()
    result = np.dot(1. / (1 + np.exp(-hb - np.dot(data,  w))), w.T) + vb
    ff = theano.function([d], ae.reconstruct(d))
    assert np.allclose(ff(data), result)

def test_autoencoder_tanh_cos_untied():
    data = np.random.randn(10, 5)
    ae = Autoencoder(5, 7, act_enc='tanh', act_dec='cos',
                     tied_weights=False)
    w = ae.weights.get_value()
    w_prime = ae.w_prime.get_value()
    ae.hidbias.set_value(np.random.randn(7))
    hb = ae.hidbias.get_value()
    ae.visbias.set_value(np.random.randn(5))
    vb = ae.visbias.get_value()
    d = tensor.matrix()
    result = np.cos(np.dot(np.tanh(hb + np.dot(data,  w)), w_prime) + vb)
    ff = theano.function([d], ae.reconstruct(d))
    assert np.allclose(ff(data), result)
<commit_msg>fix test in float32.<commit_after>"""
Tests for the pylearn2 autoencoder module.
"""
import numpy as np
import theano
import theano.tensor as tensor
#Doing this import here have no impact
#from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams
from theano import config

from pylearn2.autoencoder import Autoencoder
from theano.tensor.basic import _allclose #Handle correctly float32 and complex64

def test_autoencoder_logistic_linear_tied():
    data = np.random.randn(10, 5).astype(config.floatX)
    ae = Autoencoder(5, 7, act_enc='sigmoid', act_dec='linear',
                     tied_weights=True)
    w = ae.weights.get_value()
    ae.hidbias.set_value(np.random.randn(7).astype(config.floatX))
    hb = ae.hidbias.get_value()
    ae.visbias.set_value(np.random.randn(5).astype(config.floatX))
    vb = ae.visbias.get_value()
    d = tensor.matrix()
    result = np.dot(1. / (1 + np.exp(-hb - np.dot(data,  w))), w.T) + vb
    ff = theano.function([d], ae.reconstruct(d))
    assert _allclose(ff(data), result)

def test_autoencoder_tanh_cos_untied():
    data = np.random.randn(10, 5).astype(config.floatX)
    ae = Autoencoder(5, 7, act_enc='tanh', act_dec='cos',
                     tied_weights=False)
    w = ae.weights.get_value()
    w_prime = ae.w_prime.get_value()
    ae.hidbias.set_value(np.random.randn(7).astype(config.floatX))
    hb = ae.hidbias.get_value()
    ae.visbias.set_value(np.random.randn(5).astype(config.floatX))
    vb = ae.visbias.get_value()
    d = tensor.matrix()
    result = np.cos(np.dot(np.tanh(hb + np.dot(data,  w)), w_prime) + vb)
    ff = theano.function([d], ae.reconstruct(d))
    assert _allclose(ff(data), result)
<|endoftext|>


<commit_before>import sys
import os.path
from srtmod import srtmod

def main():
    if len(sys.argv) < 5:
        print 'Usage: ' + sys.argv[0] +' <filename> <time amount> <time part>' \
            +' <operation> '
        print 'time part: (S)econds or (M)inutes'
        print 'operation: (/A)dd or (/D)iscount'
    else:
        try:
            file_name = sys.argv[1]
            time_amount = int(sys.argv[2])
            time_part = sys.argv[3]
            if time_part not in ('S', 'M'):
                raise ValueError('Time part should be S or M')
            operation = sys.argv[4]
            if operation not in ('/A', '/D'):
                raise ValueError('Operation should be /A or /D')
            srt = srtmod.SrtMod(file_name, time_amount, time_part, operation)
            if srt.process():
                print 'Subtitle file was created successfully'
                print 'file saved at :' + os.path.splitext(file_name)[0] + '(modified).srt'
        except OverflowError:
            print 'Exception: Invalid time amount for this operation'
        except ValueError as ex:
            print 'Exception: ' + str(ex)

if __name__ == '__main__':
    main()<commit_msg>refactor main file code<commit_after>import sys
import os.path
from srtmod import srtmod

def main():
    if len(sys.argv) < 5:
        print 'Usage: ' + sys.argv[0] +' <file name> <time amount> <time part>' \
            + ' <operation>'
        print 'Time part: (S)econds or (M)inutes'
        print 'Operation: (/A)dd or (/D)iscount'
    else:
        try:
            file_name = sys.argv[1]
            time_amount = int(sys.argv[2])
            time_part = sys.argv[3]
            operation = sys.argv[4]
            if time_part not in ('S', 'M'):
                raise ValueError('Time part should be S or M')
            if operation not in ('/A', '/D'):
                raise ValueError('Operation should be /A or /D')
            srt = srtmod.SrtMod(file_name, time_amount, time_part, operation)
            if srt.process():
                print '\nSubtitle file was created successfully'
                print 'File saved at :' + os.path.splitext(file_name)[0] + '(modified).srt'
            else:
                print '\nSubtitle can not be processed'
        except OverflowError:
            print 'Exception: Invalid time amount for this operation'
        except ValueError as ex:
            print 'Exception: ' + str(ex)

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2015 Objectif Libre
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
#
from gnocchiclient import auth as gauth
from gnocchiclient import client as gclient
from keystoneauth1 import loading as ks_loading
from oslo_config import cfg
from oslo_log import log as logging
from voluptuous import All
from voluptuous import In
from voluptuous import Length
from voluptuous import Required
from voluptuous import Schema

from cloudkitty import collector
from cloudkitty import utils as ck_utils

LOG = logging.getLogger(__name__)

GNOCCHI_COLLECTOR_OPTS = 'gnocchi_collector'
gnocchi_collector_opts = ks_loading.get_auth_common_conf_options()
gcollector_opts = [
    cfg.StrOpt(
        'gnocchi_auth_type',
        default='keystone',
        choices=['keystone', 'basic'],
        help='Gnocchi auth type (keystone or basic). Keystone credentials '
        'can be specified through the "auth_section" parameter',
    ),
    cfg.StrOpt(
        'gnocchi_user',
        default='',
        help='Gnocchi user (for basic auth only)',
    ),
    cfg.StrOpt(
        'gnocchi_endpoint',
        default='',
        help='Gnocchi endpoint (for basic auth only)',
    ),
    cfg.StrOpt(
        'interface',
        default='internalURL',
        help='Endpoint URL type (for keystone auth only)',
    ),
]

cfg.CONF.register_opts(gnocchi_collector_opts, GNOCCHI_COLLECTOR_OPTS)
cfg.CONF.register_opts(gcollector_opts, GNOCCHI_COLLECTOR_OPTS)
ks_loading.register_session_conf_options(
    cfg.CONF,
    GNOCCHI_COLLECTOR_OPTS)
ks_loading.register_auth_conf_options(
    cfg.CONF,
    GNOCCHI_COLLECTOR_OPTS)
CONF = cfg.CONF

GNOCCHI_EXTRA_SCHEMA = {
    Required('extra_args'): {
        Required('resource_type'): All(str, Length(min=1)),
        # Due to Gnocchi model, metric are grouped by resource.
        # This parameter permits to adapt the key of the resource identifier
        Required('resource_key', default='id'): All(str, Length(min=1)),
        # This is needed to allow filtering on the project for the Openstack
        # usecase.
        # NOTE(MCO): maybe be removed in following releases
        Required('scope_key', default='project_id'): All(str, Length(min=1)),
        Required('aggregation_method', default='max'):
            In(['max', 'mean', 'min']),
    },
}

class GnocchiCollector(collector.BaseCollector):

    collector_name = 'gnocchi'

    def __init__(self, transformers, **kwargs):
        super(GnocchiCollector, self).__init__(transformers, **kwargs)

        self.t_gnocchi = self.transformers['GnocchiTransformer']
        self.t_cloudkitty = self.transformers['CloudKittyFormatTransformer']

        adapter_options = {'connect_retries': 3}
        if CONF.gnocchi_collector.gnocchi_auth_type == 'keystone':
            auth_plugin = ks_loading.load_auth_from_conf_options(
                CONF,
                'gnocchi_collector',
            )
            adapter_options['interface'] = CONF.gnocchi_collector.interface
        else:
            auth_plugin = gauth.GnocchiBasicPlugin(
                user=CONF.gnocchi_collector.gnocchi_user,
                endpoint=CONF.gnocchi_collector.gnocchi_endpoint,
            )
        self._conn = gclient.Client(
            '1',
            session_options={'auth': auth_plugin},
            adapter_options=adapter_options,
        )

    @staticmethod
    def check_configuration(conf):
        """Check metrics configuration

        """
        conf = Schema(collector.CONF_BASE_SCHEMA)(conf)
        metric_schema = Schema(collector.METRIC_BASE_SCHEMA).extend(
            GNOCCHI_EXTRA_SCHEMA)

        output = dict()
        for metric_name, metric in conf['metrics'].items():
            output[metric_name] = metric_schema(metric)
            output[metric_name]['groupby'].append(
                output[metric_name]['extra_args']['resource_key']
            )
        return output

    @classmethod
    def get_metadata(cls, resource_name, transformers, conf):
        info = super(GnocchiCollector, cls).get_metadata(resource_name,
                                                         transformers)
        try:
            info["metadata"].extend(
                conf[resource_name]['groupby']
            ).extend(
                conf[resource_name]['metadata']
            )
            info['unit'] = conf[resource_name]['unit']
        except KeyError:
            pass
        return info

    @classmethod
    def gen_filter(cls, cop='=', lop='and', **kwargs):
        """Generate gnocchi filter from kwargs.

        :param cop: Comparison operator.
        :param lop: Logical operator in case of multiple filters.
        """
        q_filter = []
        for kwarg in sorted(kwargs):
            q_filter.append({cop: {kwarg: kwargs[kwarg]}})
        if len(kwargs) > 1:
            return cls.extend_filter(q_filter, lop=lop)
        else:
            return q_filter[0] if len(kwargs) else {}

    @classmethod
    def extend_filter(cls, *args, **kwargs):
        """Extend an existing gnocchi filter with multiple operations.

        :param lop: Logical operator in case of multiple filters.
        """
        lop = kwargs.get('lop', 'and')
        filter_list = []
        for cur_filter in args:
            if isinstance(cur_filter, dict) and cur_filter:
                filter_list.append(cur_filter)
            elif isinstance(cur_filter, list):
                filter_list.extend(cur_filter)
        if len(filter_list) > 1:
            return {lop: filter_list}
        else:
            return filter_list[0] if len(filter_list) else {}

    def _generate_time_filter(self, start, end):
        """Generate timeframe filter.

        :param start: Start of the timeframe.
        :param end: End of the timeframe if needed.
        """
        time_filter = list()
        time_filter.append(self.extend_filter(
            self.gen_filter(ended_at=None),
            self.gen_filter(cop=">=", ended_at=start),
            lop='or'))
        time_filter.append(
            self.gen_filter(cop="<=", started_at=end))
        return time_filter

    def _fetch_resources(self, metric_name, start, end,
                         project_id=None, q_filter=None):
        """Get resources during the timeframe.

        :type metric_name: str
        :param start: Start of the timeframe.
        :param end: End of the timeframe if needed.
        :param project_id: Filter on a specific tenant/project.
        :type project_id: str
        :param q_filter: Append a custom filter.
        :type q_filter: list
        """

        # Get gnocchi specific conf
        extra_args = self.conf[metric_name]['extra_args']
        # Build query
        query_parameters = self._generate_time_filter(start, end)

        resource_type = extra_args['resource_type']

        query_parameters.append(
            self.gen_filter(cop="=", type=resource_type))
        if project_id:
            kwargs = {extra_args['scope_key']: project_id}
            query_parameters.append(self.gen_filter(**kwargs))
        if q_filter:
            query_parameters.append(q_filter)
        resources = self._conn.resource.search(
            resource_type=resource_type,
            query=self.extend_filter(*query_parameters))
        return {res[extra_args['resource_key']]: res for res in resources}

    def _fetch_metric(self, metric_name, start, end,
                      project_id=None, q_filter=None):
        """Get metric during the timeframe.

        :param metric_name: metric name to filter on.
        :type resource_name: str
        :param start: Start of the timeframe.
        :param end: End of the timeframe if needed.
        :param project_id: Filter on a specific tenant/project.
        :type project_id: str
        :param q_filter: Append a custom filter.
        :type q_filter: list
        """

        # Get gnocchi specific conf
        extra_args = self.conf[metric_name]['extra_args']

        # get ressource type
        resource_type = extra_args['resource_type']

        # build search query using ressource type and project_id if provided
        query_parameters = list()
        query_parameters.append(
            self.gen_filter(cop="=", type=resource_type))
        if project_id:
            kwargs = {extra_args['scope_key']: project_id}
            query_parameters.append(self.gen_filter(**kwargs))
        if q_filter:
            query_parameters.append(q_filter)

        # build aggregration operation
        op = ["aggregate", extra_args['aggregation_method'],
              ["metric", metric_name, extra_args['aggregation_method']]]

        # get groupby
        groupby = self.conf[metric_name]['groupby']

        return self._conn.aggregates.fetch(
            op,
            resource_type=resource_type,
            start=ck_utils.ts2dt(start),
            stop=ck_utils.ts2dt(end),
            groupby=groupby,
            search=self.extend_filter(*query_parameters))

    def _format_data(self, metconf, data, resources_info=None):
        """Formats gnocchi data to CK data.

        Returns metadata, groupby and qty

        """
        groupby = data['group']
        # if resource info is provided, add additional
        # metadata as defined in the conf
        metadata = dict()
        if resources_info:
            resource = resources_info[
                groupby[metconf['extra_args']['resource_key']]]
            for i in metconf['metadata']:
                metadata[i] = resource.get(i, '')

        qty = data['measures']['measures']['aggregated'][0][2]
        converted_qty = ck_utils.convert_unit(
            qty, metconf['factor'], metconf['offset'])
        mutated_qty = ck_utils.mutate(converted_qty, metconf['mutate'])
        return metadata, groupby, mutated_qty

    def fetch_all(self, metric_name, start, end,
                  project_id=None, q_filter=None):

        met = self.conf[metric_name]

        data = self._fetch_metric(
            metric_name,
            start,
            end,
            project_id=project_id,
            q_filter=q_filter,
        )

        resources_info = None
        if met['metadata']:
            resources_info = self._fetch_resources(
                metric_name,
                start,
                end,
                project_id=project_id,
                q_filter=q_filter
            )

        formated_resources = list()
        for d in data:
            # Only if aggregates have been found
            if d['measures']['measures']['aggregated']:
                metadata, groupby, qty = self._format_data(
                    met, d, resources_info)
                data = self.t_cloudkitty.format_item(
                    groupby,
                    metadata,
                    met['unit'],
                    qty=qty,
                )
                formated_resources.append(data)
        return formated_resources
<commit_msg>Add multi-region support for gnocchi collector<commit_after># -*- coding: utf-8 -*-
# Copyright 2015 Objectif Libre
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
#
from gnocchiclient import auth as gauth
from gnocchiclient import client as gclient
from keystoneauth1 import loading as ks_loading
from oslo_config import cfg
from oslo_log import log as logging
from voluptuous import All
from voluptuous import In
from voluptuous import Length
from voluptuous import Required
from voluptuous import Schema

from cloudkitty import collector
from cloudkitty import utils as ck_utils

LOG = logging.getLogger(__name__)

GNOCCHI_COLLECTOR_OPTS = 'gnocchi_collector'
gnocchi_collector_opts = ks_loading.get_auth_common_conf_options()
gcollector_opts = [
    cfg.StrOpt(
        'gnocchi_auth_type',
        default='keystone',
        choices=['keystone', 'basic'],
        help='Gnocchi auth type (keystone or basic). Keystone credentials '
        'can be specified through the "auth_section" parameter',
    ),
    cfg.StrOpt(
        'gnocchi_user',
        default='',
        help='Gnocchi user (for basic auth only)',
    ),
    cfg.StrOpt(
        'gnocchi_endpoint',
        default='',
        help='Gnocchi endpoint (for basic auth only)',
    ),
    cfg.StrOpt(
        'interface',
        default='internalURL',
        help='Endpoint URL type (for keystone auth only)',
    ),
    cfg.StrOpt(
        'region_name',
        default='RegionOne',
        help='Region Name'),
]

cfg.CONF.register_opts(gnocchi_collector_opts, GNOCCHI_COLLECTOR_OPTS)
cfg.CONF.register_opts(gcollector_opts, GNOCCHI_COLLECTOR_OPTS)
ks_loading.register_session_conf_options(
    cfg.CONF,
    GNOCCHI_COLLECTOR_OPTS)
ks_loading.register_auth_conf_options(
    cfg.CONF,
    GNOCCHI_COLLECTOR_OPTS)
CONF = cfg.CONF

GNOCCHI_EXTRA_SCHEMA = {
    Required('extra_args'): {
        Required('resource_type'): All(str, Length(min=1)),
        # Due to Gnocchi model, metric are grouped by resource.
        # This parameter permits to adapt the key of the resource identifier
        Required('resource_key', default='id'): All(str, Length(min=1)),
        # This is needed to allow filtering on the project for the Openstack
        # usecase.
        # NOTE(MCO): maybe be removed in following releases
        Required('scope_key', default='project_id'): All(str, Length(min=1)),
        Required('aggregation_method', default='max'):
            In(['max', 'mean', 'min']),
    },
}

class GnocchiCollector(collector.BaseCollector):

    collector_name = 'gnocchi'

    def __init__(self, transformers, **kwargs):
        super(GnocchiCollector, self).__init__(transformers, **kwargs)

        self.t_gnocchi = self.transformers['GnocchiTransformer']
        self.t_cloudkitty = self.transformers['CloudKittyFormatTransformer']

        adapter_options = {'connect_retries': 3}
        if CONF.gnocchi_collector.gnocchi_auth_type == 'keystone':
            auth_plugin = ks_loading.load_auth_from_conf_options(
                CONF,
                'gnocchi_collector',
            )
            adapter_options['interface'] = CONF.gnocchi_collector.interface
        else:
            auth_plugin = gauth.GnocchiBasicPlugin(
                user=CONF.gnocchi_collector.gnocchi_user,
                endpoint=CONF.gnocchi_collector.gnocchi_endpoint,
            )
        adapter_options['region_name'] = CONF.gnocchi_collector.region_name
        self._conn = gclient.Client(
            '1',
            session_options={'auth': auth_plugin},
            adapter_options=adapter_options,
        )

    @staticmethod
    def check_configuration(conf):
        """Check metrics configuration

        """
        conf = Schema(collector.CONF_BASE_SCHEMA)(conf)
        metric_schema = Schema(collector.METRIC_BASE_SCHEMA).extend(
            GNOCCHI_EXTRA_SCHEMA)

        output = dict()
        for metric_name, metric in conf['metrics'].items():
            output[metric_name] = metric_schema(metric)
            output[metric_name]['groupby'].append(
                output[metric_name]['extra_args']['resource_key']
            )
        return output

    @classmethod
    def get_metadata(cls, resource_name, transformers, conf):
        info = super(GnocchiCollector, cls).get_metadata(resource_name,
                                                         transformers)
        try:
            info["metadata"].extend(
                conf[resource_name]['groupby']
            ).extend(
                conf[resource_name]['metadata']
            )
            info['unit'] = conf[resource_name]['unit']
        except KeyError:
            pass
        return info

    @classmethod
    def gen_filter(cls, cop='=', lop='and', **kwargs):
        """Generate gnocchi filter from kwargs.

        :param cop: Comparison operator.
        :param lop: Logical operator in case of multiple filters.
        """
        q_filter = []
        for kwarg in sorted(kwargs):
            q_filter.append({cop: {kwarg: kwargs[kwarg]}})
        if len(kwargs) > 1:
            return cls.extend_filter(q_filter, lop=lop)
        else:
            return q_filter[0] if len(kwargs) else {}

    @classmethod
    def extend_filter(cls, *args, **kwargs):
        """Extend an existing gnocchi filter with multiple operations.

        :param lop: Logical operator in case of multiple filters.
        """
        lop = kwargs.get('lop', 'and')
        filter_list = []
        for cur_filter in args:
            if isinstance(cur_filter, dict) and cur_filter:
                filter_list.append(cur_filter)
            elif isinstance(cur_filter, list):
                filter_list.extend(cur_filter)
        if len(filter_list) > 1:
            return {lop: filter_list}
        else:
            return filter_list[0] if len(filter_list) else {}

    def _generate_time_filter(self, start, end):
        """Generate timeframe filter.

        :param start: Start of the timeframe.
        :param end: End of the timeframe if needed.
        """
        time_filter = list()
        time_filter.append(self.extend_filter(
            self.gen_filter(ended_at=None),
            self.gen_filter(cop=">=", ended_at=start),
            lop='or'))
        time_filter.append(
            self.gen_filter(cop="<=", started_at=end))
        return time_filter

    def _fetch_resources(self, metric_name, start, end,
                         project_id=None, q_filter=None):
        """Get resources during the timeframe.

        :type metric_name: str
        :param start: Start of the timeframe.
        :param end: End of the timeframe if needed.
        :param project_id: Filter on a specific tenant/project.
        :type project_id: str
        :param q_filter: Append a custom filter.
        :type q_filter: list
        """

        # Get gnocchi specific conf
        extra_args = self.conf[metric_name]['extra_args']
        # Build query
        query_parameters = self._generate_time_filter(start, end)

        resource_type = extra_args['resource_type']

        query_parameters.append(
            self.gen_filter(cop="=", type=resource_type))
        if project_id:
            kwargs = {extra_args['scope_key']: project_id}
            query_parameters.append(self.gen_filter(**kwargs))
        if q_filter:
            query_parameters.append(q_filter)
        resources = self._conn.resource.search(
            resource_type=resource_type,
            query=self.extend_filter(*query_parameters))
        return {res[extra_args['resource_key']]: res for res in resources}

    def _fetch_metric(self, metric_name, start, end,
                      project_id=None, q_filter=None):
        """Get metric during the timeframe.

        :param metric_name: metric name to filter on.
        :type resource_name: str
        :param start: Start of the timeframe.
        :param end: End of the timeframe if needed.
        :param project_id: Filter on a specific tenant/project.
        :type project_id: str
        :param q_filter: Append a custom filter.
        :type q_filter: list
        """

        # Get gnocchi specific conf
        extra_args = self.conf[metric_name]['extra_args']

        # get ressource type
        resource_type = extra_args['resource_type']

        # build search query using ressource type and project_id if provided
        query_parameters = list()
        query_parameters.append(
            self.gen_filter(cop="=", type=resource_type))
        if project_id:
            kwargs = {extra_args['scope_key']: project_id}
            query_parameters.append(self.gen_filter(**kwargs))
        if q_filter:
            query_parameters.append(q_filter)

        # build aggregration operation
        op = ["aggregate", extra_args['aggregation_method'],
              ["metric", metric_name, extra_args['aggregation_method']]]

        # get groupby
        groupby = self.conf[metric_name]['groupby']

        return self._conn.aggregates.fetch(
            op,
            resource_type=resource_type,
            start=ck_utils.ts2dt(start),
            stop=ck_utils.ts2dt(end),
            groupby=groupby,
            search=self.extend_filter(*query_parameters))

    def _format_data(self, metconf, data, resources_info=None):
        """Formats gnocchi data to CK data.

        Returns metadata, groupby and qty

        """
        groupby = data['group']
        # if resource info is provided, add additional
        # metadata as defined in the conf
        metadata = dict()
        if resources_info:
            resource = resources_info[
                groupby[metconf['extra_args']['resource_key']]]
            for i in metconf['metadata']:
                metadata[i] = resource.get(i, '')

        qty = data['measures']['measures']['aggregated'][0][2]
        converted_qty = ck_utils.convert_unit(
            qty, metconf['factor'], metconf['offset'])
        mutated_qty = ck_utils.mutate(converted_qty, metconf['mutate'])
        return metadata, groupby, mutated_qty

    def fetch_all(self, metric_name, start, end,
                  project_id=None, q_filter=None):

        met = self.conf[metric_name]

        data = self._fetch_metric(
            metric_name,
            start,
            end,
            project_id=project_id,
            q_filter=q_filter,
        )

        resources_info = None
        if met['metadata']:
            resources_info = self._fetch_resources(
                metric_name,
                start,
                end,
                project_id=project_id,
                q_filter=q_filter
            )

        formated_resources = list()
        for d in data:
            # Only if aggregates have been found
            if d['measures']['measures']['aggregated']:
                metadata, groupby, qty = self._format_data(
                    met, d, resources_info)
                data = self.t_cloudkitty.format_item(
                    groupby,
                    metadata,
                    met['unit'],
                    qty=qty,
                )
                formated_resources.append(data)
        return formated_resources
<|endoftext|>


<commit_before>#
# This file is part of the PyMeasure package.
#
# Copyright (c) 2013-2021 PyMeasure Developers
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#

import logging

log = logging.getLogger()
log.addHandler(logging.NullHandler())

try:
    from IPython import display
except ImportError:
    log.warning("IPython could not be imported")

from .results import unique_filename
from .config import get_config, set_mpl_rcparams
from pymeasure.log import setup_logging, console_log
from pymeasure.experiment import Results, Worker
from .parameters import Measurable
import time, signal
import numpy as np
import tempfile
import gc

def get_array(start, stop, step):
    """Returns a numpy array from start to stop"""
    step = np.sign(stop - start) * abs(step)
    return np.arange(start, stop + step, step)

def get_array_steps(start, stop, numsteps):
    """Returns a numpy array from start to stop in numsteps"""
    return get_array(start, stop, (abs(stop - start) / numsteps))

def get_array_zero(maxval, step):
    """Returns a numpy array from 0 to maxval to -maxval to 0"""
    return np.concatenate((np.arange(0, maxval, step), np.arange(maxval, -maxval, -step),
                           np.arange(-maxval, 0, step)))

def create_filename(title):
    """
    Create a new filename according to the style defined in the config file.
    If no config is specified, create a temporary file.
    """
    config = get_config()
    if 'Filename' in config._sections.keys():
        filename = unique_filename(suffix='_%s' % title, **config._sections['Filename'])
    else:
        filename = tempfile.mktemp()
    return filename

class Experiment(object):
    """ Class which starts logging and creates/runs the results and worker processes.

    .. code-block:: python

        procedure = Procedure()
        experiment = Experiment(title, procedure)
        experiment.start()
        experiment.plot_live('x', 'y', style='.-')

        for a multi-subplot graph:

        import pylab as pl
        ax1 = pl.subplot(121)
        experiment.plot('x','y',ax=ax1)
        ax2 = pl.subplot(122)
        experiment.plot('x','z',ax=ax2)
        experiment.plot_live()

    :var value: The value of the parameter

    :param title: The experiment title
    :param procedure: The procedure object
    :param analyse: Post-analysis function, which takes a pandas dataframe as input and
        returns it with added (analysed) columns. The analysed results are accessible via
        experiment.data, as opposed to experiment.results.data for the 'raw' data.
    :param _data_timeout: Time limit for how long live plotting should wait for datapoints.
    """

    def __init__(self, title, procedure, analyse=(lambda x: x)):
        self.title = title
        self.procedure = procedure
        self.measlist = []
        self.port = 5888
        self.plots = []
        self.figs = []
        self._data = []
        self.analyse = analyse
        self._data_timeout = 10

        config = get_config()
        set_mpl_rcparams(config)
        if 'Logging' in config._sections.keys():
            self.scribe = setup_logging(log, **config._sections['Logging'])
        else:
            self.scribe = console_log(log)
        self.scribe.start()

        self.filename = create_filename(self.title)
        log.info("Using data file: %s" % self.filename)

        self.results = Results(self.procedure, self.filename)
        log.info("Set up Results")

        self.worker = Worker(self.results, self.scribe.queue, logging.DEBUG)
        log.info("Create worker")

    def start(self):
        """Start the worker"""
        log.info("Starting worker...")
        self.worker.start()

    @property
    def data(self):
        """Data property which returns analysed data, if an analyse function
        is defined, otherwise returns the raw data."""
        self._data = self.analyse(self.results.data.copy())
        return self._data

    def wait_for_data(self):
        """Wait for the data attribute to fill with datapoints."""
        t = time.time()
        while self.data.empty:
            time.sleep(.1)
            if (time.time() - t) > self._data_timeout:
                log.warning('Timeout, no data received for liveplot')
                return False
        return True

    def plot_live(self, *args, **kwargs):
        """Live plotting loop for jupyter notebook, which automatically updates
        (an) in-line matplotlib graph(s). Will create a new plot as specified by input
        arguments, or will update (an) existing plot(s)."""
        if self.wait_for_data():
            if not (self.plots):
                self.plot(*args, **kwargs)
            while not self.worker.should_stop():
                self.update_plot()
            display.clear_output(wait=True)
            if self.worker.is_alive():
                self.worker.terminate()
            self.scribe.stop()

    def plot(self, *args, **kwargs):
        """Plot the results from the experiment.data pandas dataframe. Store the
        plots in a plots list attribute."""
        if self.wait_for_data():
            kwargs['title'] = self.title
            ax = self.data.plot(*args, **kwargs)
            self.plots.append({'type': 'plot', 'args': args, 'kwargs': kwargs, 'ax': ax})
            if ax.get_figure() not in self.figs:
                self.figs.append(ax.get_figure())
            self._user_interrupt = False

    def clear_plot(self):
        """Clear the figures and plot lists."""
        for fig in self.figs:
            fig.clf()
            pl.close()
        self.figs = []
        self.plots = []
        gc.collect()

    def update_plot(self):
        """Update the plots in the plots list with new data from the experiment.data
        pandas dataframe."""
        try:
            tasks = []
            self.data
            for plot in self.plots:
                ax = plot['ax']
                if plot['type'] == 'plot':
                    x, y = plot['args'][0], plot['args'][1]
                    if type(y) == str:
                        y = [y]
                    for yname, line in zip(y, ax.lines):
                        self.update_line(ax, line, x, yname)
                if plot['type'] == 'pcolor':
                    x, y, z = plot['x'], plot['y'], plot['z']
                    update_pcolor(ax, x, y, z)

            display.clear_output(wait=True)
            display.display(*self.figs)
            time.sleep(0.1)
        except KeyboardInterrupt:
            display.clear_output(wait=True)
            display.display(*self.figs)
            self._user_interrupt = True

    def pcolor(self, xname, yname, zname, *args, **kwargs):
        """Plot the results from the experiment.data pandas dataframe in a pcolor graph.
        Store the plots in a plots list attribute."""
        title = self.title
        x, y, z = self._data[xname], self._data[yname], self._data[zname]
        shape = (len(y.unique()), len(x.unique()))
        diff = shape[0] * shape[1] - len(z)
        Z = np.concatenate((z.values, np.zeros(diff))).reshape(shape)
        df = pd.DataFrame(Z, index=y.unique(), columns=x.unique())
        ax = sns.heatmap(df)
        pl.title(title)
        pl.xlabel(xname)
        pl.ylabel(yname)
        ax.invert_yaxis()
        pl.plt.show()
        self.plots.append(
            {'type': 'pcolor', 'x': xname, 'y': yname, 'z': zname, 'args': args, 'kwargs': kwargs,
             'ax': ax})
        if ax.get_figure() not in self.figs:
            self.figs.append(ax.get_figure())

    def update_pcolor(self, ax, xname, yname, zname):
        """Update a pcolor graph with new data."""
        x, y, z = self._data[xname], self._data[yname], self._data[zname]
        shape = (len(y.unique()), len(x.unique()))
        diff = shape[0] * shape[1] - len(z)
        Z = np.concatenate((z.values, np.zeros(diff))).reshape(shape)
        df = pd.DataFrame(Z, index=y.unique(), columns=x.unique())
        cbar_ax = ax.get_figure().axes[1]
        sns.heatmap(df, ax=ax, cbar_ax=cbar_ax)
        ax.set_xlabel(xname)
        ax.set_ylabel(yname)
        ax.invert_yaxis()

    def update_line(self, ax, hl, xname, yname):
        """Update a line in a matplotlib graph with new data."""
        del hl._xorig, hl._yorig
        hl.set_xdata(self._data[xname])
        hl.set_ydata(self._data[yname])
        ax.relim()
        ax.autoscale()
        gc.collect()

    def __del__(self):
        self.scribe.stop()
        if self.worker.is_alive():
            self.worker.recorder_queue.put(None)
            self.worker.monitor_queue.put(None)
            self.worker.stop()
<commit_msg>Ensure plots are cleared from the Experiment class<commit_after>#
# This file is part of the PyMeasure package.
#
# Copyright (c) 2013-2021 PyMeasure Developers
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#

import logging

log = logging.getLogger()
log.addHandler(logging.NullHandler())

try:
    from IPython import display
except ImportError:
    log.warning("IPython could not be imported")

from .results import unique_filename
from .config import get_config, set_mpl_rcparams
from pymeasure.log import setup_logging, console_log
from pymeasure.experiment import Results, Worker
from .parameters import Measurable
import time, signal
import numpy as np
import tempfile
import gc

def get_array(start, stop, step):
    """Returns a numpy array from start to stop"""
    step = np.sign(stop - start) * abs(step)
    return np.arange(start, stop + step, step)

def get_array_steps(start, stop, numsteps):
    """Returns a numpy array from start to stop in numsteps"""
    return get_array(start, stop, (abs(stop - start) / numsteps))

def get_array_zero(maxval, step):
    """Returns a numpy array from 0 to maxval to -maxval to 0"""
    return np.concatenate((np.arange(0, maxval, step), np.arange(maxval, -maxval, -step),
                           np.arange(-maxval, 0, step)))

def create_filename(title):
    """
    Create a new filename according to the style defined in the config file.
    If no config is specified, create a temporary file.
    """
    config = get_config()
    if 'Filename' in config._sections.keys():
        filename = unique_filename(suffix='_%s' % title, **config._sections['Filename'])
    else:
        filename = tempfile.mktemp()
    return filename

class Experiment(object):
    """ Class which starts logging and creates/runs the results and worker processes.

    .. code-block:: python

        procedure = Procedure()
        experiment = Experiment(title, procedure)
        experiment.start()
        experiment.plot_live('x', 'y', style='.-')

        for a multi-subplot graph:

        import pylab as pl
        ax1 = pl.subplot(121)
        experiment.plot('x','y',ax=ax1)
        ax2 = pl.subplot(122)
        experiment.plot('x','z',ax=ax2)
        experiment.plot_live()

    :var value: The value of the parameter

    :param title: The experiment title
    :param procedure: The procedure object
    :param analyse: Post-analysis function, which takes a pandas dataframe as input and
        returns it with added (analysed) columns. The analysed results are accessible via
        experiment.data, as opposed to experiment.results.data for the 'raw' data.
    :param _data_timeout: Time limit for how long live plotting should wait for datapoints.
    """

    def __init__(self, title, procedure, analyse=(lambda x: x)):
        self.title = title
        self.procedure = procedure
        self.measlist = []
        self.port = 5888
        self.plots = []
        self.figs = []
        self._data = []
        self.analyse = analyse
        self._data_timeout = 10

        config = get_config()
        set_mpl_rcparams(config)
        if 'Logging' in config._sections.keys():
            self.scribe = setup_logging(log, **config._sections['Logging'])
        else:
            self.scribe = console_log(log)
        self.scribe.start()

        self.filename = create_filename(self.title)
        log.info("Using data file: %s" % self.filename)

        self.results = Results(self.procedure, self.filename)
        log.info("Set up Results")

        self.worker = Worker(self.results, self.scribe.queue, logging.DEBUG)
        log.info("Create worker")

    def start(self):
        """Start the worker"""
        log.info("Starting worker...")
        self.worker.start()

    @property
    def data(self):
        """Data property which returns analysed data, if an analyse function
        is defined, otherwise returns the raw data."""
        self._data = self.analyse(self.results.data.copy())
        return self._data

    def wait_for_data(self):
        """Wait for the data attribute to fill with datapoints."""
        t = time.time()
        while self.data.empty:
            time.sleep(.1)
            if (time.time() - t) > self._data_timeout:
                log.warning('Timeout, no data received for liveplot')
                return False
        return True

    def plot_live(self, *args, **kwargs):
        """Live plotting loop for jupyter notebook, which automatically updates
        (an) in-line matplotlib graph(s). Will create a new plot as specified by input
        arguments, or will update (an) existing plot(s)."""
        if self.wait_for_data():
            if not (self.plots):
                self.plot(*args, **kwargs)
            while not self.worker.should_stop():
                self.update_plot()
            display.clear_output(wait=True)
            if self.worker.is_alive():
                self.worker.terminate()
            self.scribe.stop()

    def plot(self, *args, **kwargs):
        """Plot the results from the experiment.data pandas dataframe. Store the
        plots in a plots list attribute."""
        if self.wait_for_data():
            kwargs['title'] = self.title
            ax = self.data.plot(*args, **kwargs)
            self.plots.append({'type': 'plot', 'args': args, 'kwargs': kwargs, 'ax': ax})
            if ax.get_figure() not in self.figs:
                self.figs.append(ax.get_figure())
            self._user_interrupt = False

    def clear_plot(self):
        """Clear the figures and plot lists."""
        for fig in self.figs:
            fig.clf()
        for pl in self.plots:
            pl.close()
        self.figs = []
        self.plots = []
        gc.collect()

    def update_plot(self):
        """Update the plots in the plots list with new data from the experiment.data
        pandas dataframe."""
        try:
            tasks = []
            self.data
            for plot in self.plots:
                ax = plot['ax']
                if plot['type'] == 'plot':
                    x, y = plot['args'][0], plot['args'][1]
                    if type(y) == str:
                        y = [y]
                    for yname, line in zip(y, ax.lines):
                        self.update_line(ax, line, x, yname)
                if plot['type'] == 'pcolor':
                    x, y, z = plot['x'], plot['y'], plot['z']
                    update_pcolor(ax, x, y, z)

            display.clear_output(wait=True)
            display.display(*self.figs)
            time.sleep(0.1)
        except KeyboardInterrupt:
            display.clear_output(wait=True)
            display.display(*self.figs)
            self._user_interrupt = True

    def pcolor(self, xname, yname, zname, *args, **kwargs):
        """Plot the results from the experiment.data pandas dataframe in a pcolor graph.
        Store the plots in a plots list attribute."""
        title = self.title
        x, y, z = self._data[xname], self._data[yname], self._data[zname]
        shape = (len(y.unique()), len(x.unique()))
        diff = shape[0] * shape[1] - len(z)
        Z = np.concatenate((z.values, np.zeros(diff))).reshape(shape)
        df = pd.DataFrame(Z, index=y.unique(), columns=x.unique())
        ax = sns.heatmap(df)
        pl.title(title)
        pl.xlabel(xname)
        pl.ylabel(yname)
        ax.invert_yaxis()
        pl.plt.show()
        self.plots.append(
            {'type': 'pcolor', 'x': xname, 'y': yname, 'z': zname, 'args': args, 'kwargs': kwargs,
             'ax': ax})
        if ax.get_figure() not in self.figs:
            self.figs.append(ax.get_figure())

    def update_pcolor(self, ax, xname, yname, zname):
        """Update a pcolor graph with new data."""
        x, y, z = self._data[xname], self._data[yname], self._data[zname]
        shape = (len(y.unique()), len(x.unique()))
        diff = shape[0] * shape[1] - len(z)
        Z = np.concatenate((z.values, np.zeros(diff))).reshape(shape)
        df = pd.DataFrame(Z, index=y.unique(), columns=x.unique())
        cbar_ax = ax.get_figure().axes[1]
        sns.heatmap(df, ax=ax, cbar_ax=cbar_ax)
        ax.set_xlabel(xname)
        ax.set_ylabel(yname)
        ax.invert_yaxis()

    def update_line(self, ax, hl, xname, yname):
        """Update a line in a matplotlib graph with new data."""
        del hl._xorig, hl._yorig
        hl.set_xdata(self._data[xname])
        hl.set_ydata(self._data[yname])
        ax.relim()
        ax.autoscale()
        gc.collect()

    def __del__(self):
        self.scribe.stop()
        if self.worker.is_alive():
            self.worker.recorder_queue.put(None)
            self.worker.monitor_queue.put(None)
            self.worker.stop()
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
main.py -- Udacity conference server-side Python App Engine
    HTTP controller handlers for memcache & task queue access

$Id$

created by wesc on 2014 may 24

"""

__author__ = 'wesc+api@google.com (Wesley Chun)'

import webapp2
from google.appengine.api import app_identity
from google.appengine.api import mail
from conference import ConferenceApi

class SetAnnouncementHandler(webapp2.RequestHandler):
    def get(self):
        """Set Announcement in Memcache."""
        ConferenceApi._cacheAnnouncement()
        self.response.set_status(204)

class SendConfirmationEmailHandler(webapp2.RequestHandler):
    def post(self):
        """Send email confirming Conference creation."""
        mail.send_mail(
            'noreply@%s.appspotmail.com' % (
                app_identity.get_application_id()),     # from
            self.request.get('email'),                  # to
            'You created a new Conference!',            # subj
            'Hi, you have created a following '         # body
            'conference:\r\n\r\n%s' % self.request.get(
                'conferenceInfo')
        )

class setFeaturedSpeakerHandler(webapp2.RequestHandler):
    def post(self):
        """Set featured speaker in memcache"""
        ConferenceApi.cacheFeaturedSpeaker(
            self.request.get('speaker_name'), self.request.get('session_list'))
        self.response.set_status(204)

app = webapp2.WSGIApplication([
    ('/crons/set_announcement', SetAnnouncementHandler),
    ('/tasks/send_confirmation_email', SendConfirmationEmailHandler),
    ('/tasks/set_featured_speaker', setFeaturedSpeaker),
], debug=True)
<commit_msg>fixed typo<commit_after>#!/usr/bin/env python

"""
main.py -- Udacity conference server-side Python App Engine
    HTTP controller handlers for memcache & task queue access

$Id$

created by wesc on 2014 may 24

"""

__author__ = 'wesc+api@google.com (Wesley Chun)'

import webapp2
from google.appengine.api import app_identity
from google.appengine.api import mail
from conference import ConferenceApi

class SetAnnouncementHandler(webapp2.RequestHandler):
    def get(self):
        """Set Announcement in Memcache."""
        ConferenceApi._cacheAnnouncement()
        self.response.set_status(204)

class SendConfirmationEmailHandler(webapp2.RequestHandler):
    def post(self):
        """Send email confirming Conference creation."""
        mail.send_mail(
            'noreply@%s.appspotmail.com' % (
                app_identity.get_application_id()),     # from
            self.request.get('email'),                  # to
            'You created a new Conference!',            # subj
            'Hi, you have created a following '         # body
            'conference:\r\n\r\n%s' % self.request.get(
                'conferenceInfo')
        )

class setFeaturedSpeakerHandler(webapp2.RequestHandler):
    def post(self):
        """Set featured speaker in memcache"""
        ConferenceApi.cacheFeaturedSpeaker(
            self.request.get('speaker_name'), self.request.get('session_list'))
        self.response.set_status(204)

app = webapp2.WSGIApplication([
    ('/crons/set_announcement', SetAnnouncementHandler),
    ('/tasks/send_confirmation_email', SendConfirmationEmailHandler),
    ('/tasks/set_featured_speaker', setFeaturedSpeakerHandler),
], debug=True)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import cluster_labeling.cluster_count_table as cct
import cluster_labeling.cluster_totalcount_table as ctct

from numpy import *

def bin_to_int(idx):
    return int(idx, 2)

def get_N_UC(cluster_id, parent_cluster_id, version, word):
    count_table = cct.ClusterReviewCount
    totalcount_table = ctct.ClusterReviewTotalCount

    N = zeros((2, 2))

    N[1, 1] = count_table.get_value(cluster_id, version, word)
    N[1, 0] = count_table.get_value(parent_cluster_id, version, word) - N[1, 1]
    N[0, 1] = totalcount_table.get_value(cluster_id, version) - N[1, 1]
    N[0, 0] = totalcount_table.get_value(parent_cluster_id, version) - N[0, 1] - N[1, 0] - N[1, 1]

    return N

def P_UC_to_P_C(P_UC):
    P_C = sum(P_UC, 0)
    return P_C

def P_UC_to_P_U(P_UC):
    P_U = sum(P_UC, 1)
    return P_U

def compute_scores_for_cluster(cluster_id, parent_cluster_id,
                               numchildren, version,
                               score_fn, score_table):
    words = cct.ClusterReviewCount\
            .get_words_for_cluster(cluster_id, version)
    scores = dict(map(lambda word:
                      (word,
                       score_fn(cluster_id, parent_cluster_id, version, word)),
                      words))

    score_table.add_values_from(cluster_id, parent_cluster_id,
                                numchildren, version, scores)

import cluster_labeling.optemo_django_models as optemo
def compute_all_scores(version, score_fn, score_table):
    # Recursively compute score for each word in each cluster and
    # store it in the score table.
    root_children = optemo.CameraCluster.get_root_children(version)

    clusters_todo = []
    clusters_todo.extend(root_children)

    compute_scores_for_cluster(0, -1, root_children.count(), version,
                               score_fn, score_table)

    while len(clusters_todo) > 0:
        curr_cluster = clusters_todo.pop()
        curr_cluster_children = curr_cluster.get_children()
        clusters_todo.extend(curr_cluster_children)

        compute_scores_for_cluster\
            (curr_cluster.id, curr_cluster.parent_id,
             curr_cluster_children.count(), version,
             score_fn, score_table)

<commit_msg>Word scores for cluster 0 are no longer computed, because it has no parent<commit_after>#!/usr/bin/env python
import cluster_labeling.cluster_count_table as cct
import cluster_labeling.cluster_totalcount_table as ctct

from numpy import *

def bin_to_int(idx):
    return int(idx, 2)

def get_N_UC(cluster_id, parent_cluster_id, version, word):
    count_table = cct.ClusterReviewCount
    totalcount_table = ctct.ClusterReviewTotalCount

    N = zeros((2, 2))

    N[1, 1] = count_table.get_value(cluster_id, version, word)
    N[1, 0] = count_table.get_value(parent_cluster_id, version, word) - N[1, 1]
    N[0, 1] = totalcount_table.get_value(cluster_id, version) - N[1, 1]
    N[0, 0] = totalcount_table.get_value(parent_cluster_id, version) - N[0, 1] - N[1, 0] - N[1, 1]

    return N

def P_UC_to_P_C(P_UC):
    P_C = sum(P_UC, 0)
    return P_C

def P_UC_to_P_U(P_UC):
    P_U = sum(P_UC, 1)
    return P_U

def compute_scores_for_cluster(cluster_id, parent_cluster_id,
                               numchildren, version,
                               score_fn, score_table):
    words = cct.ClusterReviewCount\
            .get_words_for_cluster(cluster_id, version)
    scores = dict(map(lambda word:
                      (word,
                       score_fn(cluster_id, parent_cluster_id, version, word)),
                      words))

    score_table.add_values_from(cluster_id, parent_cluster_id,
                                numchildren, version, scores)

import cluster_labeling.optemo_django_models as optemo
def compute_all_scores(version, score_fn, score_table):
    # Recursively compute score for each word in each cluster and
    # store it in the score table.
    root_children = optemo.CameraCluster.get_root_children(version)

    clusters_todo = []
    clusters_todo.extend(root_children)

    while len(clusters_todo) > 0:
        curr_cluster = clusters_todo.pop()
        curr_cluster_children = curr_cluster.get_children()
        clusters_todo.extend(curr_cluster_children)

        compute_scores_for_cluster\
            (curr_cluster.id, curr_cluster.parent_id,
             curr_cluster_children.count(), version,
             score_fn, score_table)

<|endoftext|>