<commit_before>#
# Kiwi: a Framework and Enhanced Widgets for Python
#
# Copyright (C) 2002-2005 Async Open Source
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
# USA
#
# Author(s): Christian Reis <kiko@async.com.br>
#            Lorenzo Gil Sanchez <lgs@sicem.biz>
#            Gustavo Rahal <gustavo@async.com.br>
#            Johan Dahlin <jdahlin@async.com.br>
#

"""This module defines the Proxy class, which is a facility that can be used
to keep the state of a model object synchronized with a View.
"""

import gobject
import gtk

from kiwi import ValueUnset
from kiwi.accessor import kgetattr, ksetattr, clear_attr_cache
from kiwi.decorators import deprecated
from kiwi.interfaces import IProxyWidget, IValidatableProxyWidget
from kiwi.log import Logger

class ProxyError(Exception):
    pass

log = Logger('proxy')

def block_widget(widget):
    """Blocks the signal handler of the 'content-changed' signal on widget"""
    connection_id = widget.get_data('content-changed-id')
    if connection_id:
        widget.handler_block(connection_id)

def unblock_widget(widget):
    """Unblocks the signal handler of the 'content-changed' signal on widget"""
    connection_id = widget.get_data('content-changed-id')
    if connection_id:
        widget.handler_unblock(connection_id)

class Proxy:
    """ A Proxy is a class that 'attaches' an instance to an interface's
    widgets, and transparently manipulates that instance's attributes as
    the user alters the content of the widgets.

    The Proxy takes the widget list and detects what widgets are to be
    attached to the model by looking if it is a KiwiWidget and if it
    has the model-attribute set.
    """

    def __init__(self, view, model=None, widgets=()):
        """
        @param view:    view attched to the slave
        @type  view:    a L{kiwi.ui.views.BaseView} subclass
        @param model:   model attached to proxy
        @param widgets: the widget names
        @type  widgets: list of strings
        """
        self._view = view
        self._model = model
        self._model_attributes = {}

        for widget_name in widgets:
            widget = getattr(self._view, widget_name, None)
            if widget is None:
                raise AttributeError("The widget %s was not found in the "
                                     "view %s" % (
                    widget_name, self._view.__class__.__name__))

            self._setup_widget(widget_name, widget)

    # Private API

    def _reset_widget(self, attribute, widget):
        if self._model is None:
            # if we have no model, leave value unset so we pick up
            # the widget default below.
            value = ValueUnset
        else:
            # if we have a model, grab its value to update the widgets
            self._register_proxy_in_model(attribute)
            value = kgetattr(self._model, attribute, ValueUnset)

        self.update(attribute, value, block=True)

        # The initial value of the model is set, at this point
        # do a read, it'll trigger a validation for widgets who
        # supports it.
        if not IValidatableProxyWidget.providedBy(widget):
            return

        widget.validate(force=True)

    def _setup_widget(self, widget_name, widget):
        if not IProxyWidget.providedBy(widget):
            raise ProxyError("The widget %s (%r), in view %s is not "
                             "a kiwi widget and cannot be added to a proxy"
                             % (widget_name, widget,
                                self._view.__class__.__name__))

        data_type = widget.get_property('data-type')
        if data_type is None:
            raise ProxyError("The kiwi widget %s (%r) in view %s should "
                             "have a data type set" % (
                widget_name, widget, self._view.__class__.__name__))

        attribute = widget.get_property('model-attribute')
        if not attribute:
            raise ProxyError(
                "The widget %s (%s) in view %s is a kiwi widget but does "
                "not have a model attribute set so it will not be "
                "associated with the model" % (
                widget_name, widget, self._view.__class__.__name__))

        # Do a isinstance here instead of in the callback,
        # as an optimization, it'll never change in runtime anyway
        connection_id = widget.connect(
            'content-changed',
            self._on_widget__content_changed,
            attribute,
            IValidatableProxyWidget.providedBy(widget))
        widget.set_data('content-changed-id', connection_id)

        connection_id = widget.connect(
            'notify::visible',
            self._on_widget__notify)
        widget.set_data('notify-visible-id', connection_id)

        connection_id = widget.connect(
            'notify::sensitive',
            self._on_widget__notify)
        widget.set_data('notify-sensitive-id', connection_id)

        model_attributes = self._model_attributes
        # save this widget in our map
        if (attribute in model_attributes and
            # RadioButtons are allowed several times
            not gobject.type_is_a(widget, 'GtkRadioButton')):
            old_widget = model_attributes[attribute]
            raise KeyError("The widget %s (%r) in view %s is already in "
                           "the proxy, defined by widget %s (%r)" % (
                widget_name, widget, self._view.__class__.__name__,
                old_widget.name, old_widget))

        model_attributes[attribute] = widget
        self._reset_widget(attribute, widget)

    def _register_proxy_in_model(self, attribute):
        model = self._model
        if not hasattr(model, "register_proxy_for_attribute"):
            return
        try:
            model.register_proxy_for_attribute(attribute, self)
        except AttributeError:
            msg = ("Failed to run register_proxy() on Model %s "
                   "(that was supplied to  %s. \n"
                   "(Hint: if this model also inherits from ZODB's "
                   "Persistent class, this problem occurs if you haven't "
                   "set __setstate__() up correctly.  __setstate__() "
                   "should call Model.__init__() (and "
                   "Persistent.__setstate__() of course) to rereset "
                   "things properly.)")
            raise TypeError(msg % (model, self))

    def _unregister_proxy_in_model(self):
        if self._model and hasattr(self._model, "unregister_proxy"):
            self._model.unregister_proxy(self)

    # Callbacks

    def _on_widget__content_changed(self, widget, attribute, validate):
        """This is called as soon as the content of one of the widget
        changes, the widgets tries fairly hard to not emit when it's not
        neccessary"""

        # skip updates for model if there is none, right?
        if self._model is None:
            return

        if validate:
            value = widget.validate()
        else:
            value = widget.read()

        log('%s.%s = %r' % (self._model.__class__.__name__,
                            attribute, value))

        # only update the model if the data is correct
        if value is ValueUnset:
            return

        model = self._model
        # XXX: one day we might want to queue and unique updates?
        if hasattr(model, "block_proxy"):
            model.block_proxy(self)
            ksetattr(model, attribute, value)
            model.unblock_proxy(self)
        else:
            ksetattr(model, attribute, value)

        # Call global update hook
        self.proxy_updated(widget, attribute, value)

    # notify::sensitive and notify::visible are connected here
    def _on_widget__notify(self, widget, pspec):
        widget.emit('validation-changed', widget.is_valid())

    # Properties

    def _get_model(self):
        return self._model
    model = property(_get_model)

    # Public API

    def proxy_updated(self, widget, attribute, value):
        """ This is a hook that is called whenever the proxy updates the
        model. Implement it in the inherited class to perform actions that
        should be done each time the user changes something in the interface.
        This hook by default does nothing.
        @param widget:
        @param attribute:
        @param value:
        """

    def update_many(self, attributes, value=ValueUnset, block=False):
        """
        Like L{update} but takes a sequence of attributes

        @param attributes: sequence of attributes to update
        @param value: see L{update}
        @param block: see L{update}
        """

        for attribute in attributes:
            self.update(attribute, value, block)

    def update(self, attribute, value=ValueUnset, block=False):
        """ Generic frontend function to update the contentss of a widget based
        on its model attribute name using the internal update functions.

        @param attribute: the name of the attribute whose widget we wish to
          updated.  If accessing a radiobutton, specify its group
          name.
        @param value: specifies the value to set in the widget. If
          unspecified, it defaults to the current model's value
          (through an accessor, if it exists, or getattr).
        @param block: defines if we are to block cascading proxy updates
          triggered by this update. You should use block if you are
          calling update on *the same attribute that is currently
          being updated*.
          This means if you have hooked to a signal of the widget
          associated to that attribute, and you call update() for
          the *same attribute*, use block=True. And pray. 8). If
          block is set to False, the normal update mechanism will
          occur (the model being updated in the end, hopefully).
        """

        if value is ValueUnset:
        # We want to obtain a value from our model
            if self._model is None:
                # We really want to avoid trying to update our UI if our
                # model doesn't exist yet and no value was provided.
                # update() is also called by user code, but it should be
                # safe to return here since you shouldn't need to code
                # around the lack of a model in your callbacks if you
                # can help it.
                value = None
            else:
                value = kgetattr(self._model, attribute, ValueUnset)

        widget = self._model_attributes.get(attribute, None)

        if widget is None:
            raise AttributeError("Called update for `%s', which isn't "
                                 "attached to the proxy %s. Valid "
                                 "attributes are: %s (you may have "
                                 "forgetten to add `:' to the name in "
                                 "the widgets list)"
                                 % (attribute, self,
                                    self._model_attributes.keys()))

        # The type of value should match the data-type property. The two
        # exceptions to this rule are ValueUnset and None
        if not (value is ValueUnset or value is None):
            data_type = widget.get_property('data-type')
            value_type = type(value)
            if not isinstance(value, data_type):
                raise TypeError(
                    "attribute %s of model %r requires a value of "
                    "type %s, not %s" % (
                    attribute, self._model,
                    data_type.__name__,
                    value_type.__name__))

        if block:
            block_widget(widget)
            self._view.handler_block(widget)
            widget.update(value)
            self._view.handler_unblock(widget)
            unblock_widget(widget)
        else:
            widget.update(value)
        return True

    def set_model(self, model, relax_type=False):
        """
        Updates the model instance of the proxy.
        Allows a proxy interface to change model without the need to destroy
        and recreate the UI (which would cause flashing, at least)

        @param model:
        @param relax_type:
        """
        if self._model is not None and model is not None:
            if (not relax_type and
                type(model) != type(self._model) and
                not isinstance(model, self._model.__class__)):
                raise TypeError("model has wrong type %s, expected %s"
                                % (type(model), type(self._model)))

        # the following isn't strictly necessary, but it currently works
        # around a bug with reused ids in the attribute cache and also
        # makes a lot of sense for most applications (don't want a huge
        # eternal cache pointing to models that you're not using anyway)
        clear_attr_cache()

        # unregister previous proxy
        self._unregister_proxy_in_model()

        self._model = model

        for attribute, widget in self._model_attributes.items():
            self._reset_widget(attribute, widget)

    def add_widget(self, name, widget):
        """
        Adds a new widget to the proxy

        @param name: name of the widget
        @param widget: widget, must be a gtk.Widget subclass
        """
        if name in self._model_attributes:
            raise TypeError("there is already a widget called %s" % name)

        if not isinstance(widget, gtk.Widget):
            raise TypeError("%r must be a gtk.Widget subclass" % widget)

        self._setup_widget(name, widget)

    def remove_widget(self, name):
        """
        Removes a widget from the proxy

        @param name: the name of the widget to remove
        """
        if not name in self._model_attributes:
            raise TypeError("there is no widget called %s" % name)

        widget = self._model_attributes.pop(name)
        if IValidatableProxyWidget.providedBy(widget):
            for data_name in ('content-changed-id',
                              'notify-visible-id',
                              'notify-sensitive-id'):
                widget.disconnect(widget.get_data(data_name))

    # Backwards compatibility

    def new_model(self, model, relax_type=False):
        self.set_model(model)
    new_model = deprecated('set_model', log)(new_model)

<commit_msg>Only connect to notify::sensitive & notify::visible for validatable widgets<commit_after>#
# Kiwi: a Framework and Enhanced Widgets for Python
#
# Copyright (C) 2002-2005 Async Open Source
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
# USA
#
# Author(s): Christian Reis <kiko@async.com.br>
#            Lorenzo Gil Sanchez <lgs@sicem.biz>
#            Gustavo Rahal <gustavo@async.com.br>
#            Johan Dahlin <jdahlin@async.com.br>
#

"""This module defines the Proxy class, which is a facility that can be used
to keep the state of a model object synchronized with a View.
"""

import gobject
import gtk

from kiwi import ValueUnset
from kiwi.accessor import kgetattr, ksetattr, clear_attr_cache
from kiwi.decorators import deprecated
from kiwi.interfaces import IProxyWidget, IValidatableProxyWidget
from kiwi.log import Logger

class ProxyError(Exception):
    pass

log = Logger('proxy')

def block_widget(widget):
    """Blocks the signal handler of the 'content-changed' signal on widget"""
    connection_id = widget.get_data('content-changed-id')
    if connection_id:
        widget.handler_block(connection_id)

def unblock_widget(widget):
    """Unblocks the signal handler of the 'content-changed' signal on widget"""
    connection_id = widget.get_data('content-changed-id')
    if connection_id:
        widget.handler_unblock(connection_id)

class Proxy:
    """ A Proxy is a class that 'attaches' an instance to an interface's
    widgets, and transparently manipulates that instance's attributes as
    the user alters the content of the widgets.

    The Proxy takes the widget list and detects what widgets are to be
    attached to the model by looking if it is a KiwiWidget and if it
    has the model-attribute set.
    """

    def __init__(self, view, model=None, widgets=()):
        """
        @param view:    view attched to the slave
        @type  view:    a L{kiwi.ui.views.BaseView} subclass
        @param model:   model attached to proxy
        @param widgets: the widget names
        @type  widgets: list of strings
        """
        self._view = view
        self._model = model
        self._model_attributes = {}

        for widget_name in widgets:
            widget = getattr(self._view, widget_name, None)
            if widget is None:
                raise AttributeError("The widget %s was not found in the "
                                     "view %s" % (
                    widget_name, self._view.__class__.__name__))

            self._setup_widget(widget_name, widget)

    # Private API

    def _reset_widget(self, attribute, widget):
        if self._model is None:
            # if we have no model, leave value unset so we pick up
            # the widget default below.
            value = ValueUnset
        else:
            # if we have a model, grab its value to update the widgets
            self._register_proxy_in_model(attribute)
            value = kgetattr(self._model, attribute, ValueUnset)

        self.update(attribute, value, block=True)

        # The initial value of the model is set, at this point
        # do a read, it'll trigger a validation for widgets who
        # supports it.
        if not IValidatableProxyWidget.providedBy(widget):
            return

        widget.validate(force=True)

    def _setup_widget(self, widget_name, widget):
        if not IProxyWidget.providedBy(widget):
            raise ProxyError("The widget %s (%r), in view %s is not "
                             "a kiwi widget and cannot be added to a proxy"
                             % (widget_name, widget,
                                self._view.__class__.__name__))

        data_type = widget.get_property('data-type')
        if data_type is None:
            raise ProxyError("The kiwi widget %s (%r) in view %s should "
                             "have a data type set" % (
                widget_name, widget, self._view.__class__.__name__))

        attribute = widget.get_property('model-attribute')
        if not attribute:
            raise ProxyError(
                "The widget %s (%s) in view %s is a kiwi widget but does "
                "not have a model attribute set so it will not be "
                "associated with the model" % (
                widget_name, widget, self._view.__class__.__name__))

        # Do a isinstance here instead of in the callback,
        # as an optimization, it'll never change in runtime anyway
        connection_id = widget.connect(
            'content-changed',
            self._on_widget__content_changed,
            attribute,
            IValidatableProxyWidget.providedBy(widget))
        widget.set_data('content-changed-id', connection_id)

        if IValidatableProxyWidget.providedBy(widget):
            connection_id = widget.connect(
                'notify::visible',
                self._on_widget__notify)
            widget.set_data('notify-visible-id', connection_id)

            connection_id = widget.connect(
                'notify::sensitive',
                self._on_widget__notify)
            widget.set_data('notify-sensitive-id', connection_id)

        model_attributes = self._model_attributes
        # save this widget in our map
        if (attribute in model_attributes and
            # RadioButtons are allowed several times
            not gobject.type_is_a(widget, 'GtkRadioButton')):
            old_widget = model_attributes[attribute]
            raise KeyError("The widget %s (%r) in view %s is already in "
                           "the proxy, defined by widget %s (%r)" % (
                widget_name, widget, self._view.__class__.__name__,
                old_widget.name, old_widget))

        model_attributes[attribute] = widget
        self._reset_widget(attribute, widget)

    def _register_proxy_in_model(self, attribute):
        model = self._model
        if not hasattr(model, "register_proxy_for_attribute"):
            return
        try:
            model.register_proxy_for_attribute(attribute, self)
        except AttributeError:
            msg = ("Failed to run register_proxy() on Model %s "
                   "(that was supplied to  %s. \n"
                   "(Hint: if this model also inherits from ZODB's "
                   "Persistent class, this problem occurs if you haven't "
                   "set __setstate__() up correctly.  __setstate__() "
                   "should call Model.__init__() (and "
                   "Persistent.__setstate__() of course) to rereset "
                   "things properly.)")
            raise TypeError(msg % (model, self))

    def _unregister_proxy_in_model(self):
        if self._model and hasattr(self._model, "unregister_proxy"):
            self._model.unregister_proxy(self)

    # Callbacks

    def _on_widget__content_changed(self, widget, attribute, validate):
        """This is called as soon as the content of one of the widget
        changes, the widgets tries fairly hard to not emit when it's not
        neccessary"""

        # skip updates for model if there is none, right?
        if self._model is None:
            return

        if validate:
            value = widget.validate()
        else:
            value = widget.read()

        log('%s.%s = %r' % (self._model.__class__.__name__,
                            attribute, value))

        # only update the model if the data is correct
        if value is ValueUnset:
            return

        model = self._model
        # XXX: one day we might want to queue and unique updates?
        if hasattr(model, "block_proxy"):
            model.block_proxy(self)
            ksetattr(model, attribute, value)
            model.unblock_proxy(self)
        else:
            ksetattr(model, attribute, value)

        # Call global update hook
        self.proxy_updated(widget, attribute, value)

    # notify::sensitive and notify::visible are connected here
    def _on_widget__notify(self, widget, pspec):
        widget.emit('validation-changed', widget.is_valid())

    # Properties

    def _get_model(self):
        return self._model
    model = property(_get_model)

    # Public API

    def proxy_updated(self, widget, attribute, value):
        """ This is a hook that is called whenever the proxy updates the
        model. Implement it in the inherited class to perform actions that
        should be done each time the user changes something in the interface.
        This hook by default does nothing.
        @param widget:
        @param attribute:
        @param value:
        """

    def update_many(self, attributes, value=ValueUnset, block=False):
        """
        Like L{update} but takes a sequence of attributes

        @param attributes: sequence of attributes to update
        @param value: see L{update}
        @param block: see L{update}
        """

        for attribute in attributes:
            self.update(attribute, value, block)

    def update(self, attribute, value=ValueUnset, block=False):
        """ Generic frontend function to update the contentss of a widget based
        on its model attribute name using the internal update functions.

        @param attribute: the name of the attribute whose widget we wish to
          updated.  If accessing a radiobutton, specify its group
          name.
        @param value: specifies the value to set in the widget. If
          unspecified, it defaults to the current model's value
          (through an accessor, if it exists, or getattr).
        @param block: defines if we are to block cascading proxy updates
          triggered by this update. You should use block if you are
          calling update on *the same attribute that is currently
          being updated*.
          This means if you have hooked to a signal of the widget
          associated to that attribute, and you call update() for
          the *same attribute*, use block=True. And pray. 8). If
          block is set to False, the normal update mechanism will
          occur (the model being updated in the end, hopefully).
        """

        if value is ValueUnset:
        # We want to obtain a value from our model
            if self._model is None:
                # We really want to avoid trying to update our UI if our
                # model doesn't exist yet and no value was provided.
                # update() is also called by user code, but it should be
                # safe to return here since you shouldn't need to code
                # around the lack of a model in your callbacks if you
                # can help it.
                value = None
            else:
                value = kgetattr(self._model, attribute, ValueUnset)

        widget = self._model_attributes.get(attribute, None)

        if widget is None:
            raise AttributeError("Called update for `%s', which isn't "
                                 "attached to the proxy %s. Valid "
                                 "attributes are: %s (you may have "
                                 "forgetten to add `:' to the name in "
                                 "the widgets list)"
                                 % (attribute, self,
                                    self._model_attributes.keys()))

        # The type of value should match the data-type property. The two
        # exceptions to this rule are ValueUnset and None
        if not (value is ValueUnset or value is None):
            data_type = widget.get_property('data-type')
            value_type = type(value)
            if not isinstance(value, data_type):
                raise TypeError(
                    "attribute %s of model %r requires a value of "
                    "type %s, not %s" % (
                    attribute, self._model,
                    data_type.__name__,
                    value_type.__name__))

        if block:
            block_widget(widget)
            self._view.handler_block(widget)
            widget.update(value)
            self._view.handler_unblock(widget)
            unblock_widget(widget)
        else:
            widget.update(value)
        return True

    def set_model(self, model, relax_type=False):
        """
        Updates the model instance of the proxy.
        Allows a proxy interface to change model without the need to destroy
        and recreate the UI (which would cause flashing, at least)

        @param model:
        @param relax_type:
        """
        if self._model is not None and model is not None:
            if (not relax_type and
                type(model) != type(self._model) and
                not isinstance(model, self._model.__class__)):
                raise TypeError("model has wrong type %s, expected %s"
                                % (type(model), type(self._model)))

        # the following isn't strictly necessary, but it currently works
        # around a bug with reused ids in the attribute cache and also
        # makes a lot of sense for most applications (don't want a huge
        # eternal cache pointing to models that you're not using anyway)
        clear_attr_cache()

        # unregister previous proxy
        self._unregister_proxy_in_model()

        self._model = model

        for attribute, widget in self._model_attributes.items():
            self._reset_widget(attribute, widget)

    def add_widget(self, name, widget):
        """
        Adds a new widget to the proxy

        @param name: name of the widget
        @param widget: widget, must be a gtk.Widget subclass
        """
        if name in self._model_attributes:
            raise TypeError("there is already a widget called %s" % name)

        if not isinstance(widget, gtk.Widget):
            raise TypeError("%r must be a gtk.Widget subclass" % widget)

        self._setup_widget(name, widget)

    def remove_widget(self, name):
        """
        Removes a widget from the proxy

        @param name: the name of the widget to remove
        """
        if not name in self._model_attributes:
            raise TypeError("there is no widget called %s" % name)

        widget = self._model_attributes.pop(name)
        widget.disconnect(widget.get_data('content-changed-id'))

        if IValidatableProxyWidget.providedBy(widget):
            for data_name in ('notify-visible-id',
                              'notify-sensitive-id'):
                widget.disconnect(widget.get_data(data_name))

    # Backwards compatibility

    def new_model(self, model, relax_type=False):
        self.set_model(model)
    new_model = deprecated('set_model', log)(new_model)

<|endoftext|>


<commit_before>#!/usr/bin/python
#
# Copyright (c) 2012 Mikkel Schubert <MSchubert@snm.ku.dk>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
import os
import re
import random

import pypeline.common.fileutils as fileutils

from pypeline.node import CommandNode
from pypeline.atomiccmd import AtomicCmd

class RAxMLReduceNode(CommandNode):
    def __init__(self, inalignment, outalignment, inpartitions, outpartitions, dependencies = ()):
        call = ["raxmlHPC",
                "-f", "c",
                "-m", "GTRGAMMA", # Model required, but not used
                "-n", "GTRGAMMA",
                "-w", "%(TEMP_DIR)s",
                "-s", "%(TEMP_IN_ALIGNMENT)s",
                "-q", "%(TEMP_IN_PARTITIONS)s"]

        self._kwargs = {"IN_ALIGNMENT"        : inalignment,
                        "IN_PARTITIONS"       : inpartitions,

                        "TEMP_IN_ALIGNMENT"   : "RAXML_alignment",
                        "TEMP_IN_PARTITIONS"  : "RAXML_partitions",
                        "TEMP_OUT_INFO"       : "RAxML_info.GTRGAMMA",

                        "OUT_ALIGNMENT"       : outalignment,
                        "OUT_PARTITIONS"      : outpartitions}

        CommandNode.__init__(self,
                             command      = AtomicCmd(call, **self._kwargs),
                             description  = "<RAxMLReduce: '%s' -> '%s'>" \
                                     % (inalignment, outalignment),
                             dependencies = dependencies)

    def _setup(self, config, temp):
        for key in ("IN_ALIGNMENT", "IN_PARTITIONS"):
            source      = self._kwargs[key]
            destination = os.path.join(temp, self._kwargs["TEMP_" + key])

            fileutils.copy_file(source, destination)

        CommandNode._setup(self, config, temp)

    def _teardown(self, config, temp):
        for postfix in ("ALIGNMENT", "PARTITIONS"):
            filenames = [self._kwargs["TEMP_IN_" + postfix],
                         self._kwargs["TEMP_IN_" + postfix] + ".reduced",
                         self._kwargs["OUT_" + postfix]]

            for (source, destination) in zip(filenames, filenames[1:]):
                source      = fileutils.reroot_path(temp, source)
                destination = fileutils.reroot_path(temp, destination)

                if not os.path.exists(destination):
                    fileutils.move_file(source, destination)
                elif source != destination:
                    os.remove(source)

        CommandNode._teardown(self, config, temp)

class RAxMLRapidBSNode(CommandNode):
    def __init__(self, infile, partitions, destination, model = "GTRGAMMAI", threads = 1, dependencies = ()):
        """
        Arguments:
        infile      -- An alignment file in a format readable by RAxML.
        partitions  -- A set of partitions in a format readable by RAxML.
        destination -- A template string used to construct final filenames. Should consist
                       of a full path, including a single '%s', which is replaced with the
                       variable part of RAxML output files (e.g. 'info', 'bestTree', ...).
                       Example destination: '/disk/project/SN013420.RAxML.%s'
                       Example output:      '/disk/project/SN013420.RAxML.bestTree'
        model       -- DNA or Amino acid substitution model to use.
        threads     -- Number of threads to use for each RAxML instance."""

        self._symlinks = [infile, partitions]
        self._template = os.path.basename(destination)

        call = ["raxmlHPC-PTHREADS" if (threads > 1) else "raxmlHPC",
                "-f", "a",
                "-m", model,
                "-n", "RapidBS",
                "-s", "%(TEMP_OUT_ALN)s",
                "-q", "%(TEMP_OUT_PART)s",
                "-w", "%(TEMP_DIR)s",
                "-x", int(random.random() * 2**31 - 1),
                "-p", int(random.random() * 2**31 - 1),
                "-N", "autoMRE"]
        if threads > 1:
            call.extend(("-T", threads))

        command = AtomicCmd(call,
                            IN_ALIGNMENT    = infile,
                            IN_PARTITIONS   = partitions,

                            OUT_INFO        = destination % "info",
                            OUT_BESTTREE    = destination % "bestTree",
                            OUT_BOOTSTRAP   = destination % "bootstrap",
                            OUT_BIPART      = destination % "bipartitions",
                            OUT_BIPARTLABEL = destination % "bipartitionsBranchLabels",

                            # RAxML may write reduced alignment. These are not saved. If
                            # needed, they may be generated upfront using RAxMLReduceNode.
                            TEMP_OUT_ALN    = os.path.basename(infile),
                            TEMP_OUT_PART   = os.path.basename(partitions),
                            TEMP_OUT_R_ALN  = os.path.basename(infile) + ".reduced",
                            TEMP_OUT_R_PART = os.path.basename(partitions) + ".reduced")

        CommandNode.__init__(self,
                             command      = command,
                             description  = "<RAxMLRapidBS (%i threads): '%s' -> '%s'>" \
                                     % (threads, infile, destination),
                             threads      = threads,
                             dependencies = dependencies)

    def _setup(self, config, temp):
        CommandNode._setup(self, config, temp)

        # Required to avoid the creation of files outside the temp folder
        for filename in self._symlinks:
            source      = os.path.abspath(filename)
            destination = os.path.join(temp, os.path.basename(filename))

            os.symlink(source, destination)

    def _teardown(self, config, temp):
        for filename in os.listdir(temp):
            match = re.match("RAxML_(.*).RapidBS", filename)
            if match:
                source      = os.path.join(temp, filename)
                destination = os.path.join(temp, self._template % match.groups())

                fileutils.move_file(source, destination)

        CommandNode._teardown(self, config, temp)
<commit_msg>Adding additional options to RAxML node.<commit_after>#!/usr/bin/python
#
# Copyright (c) 2012 Mikkel Schubert <MSchubert@snm.ku.dk>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
import os
import re
import random

import pypeline.common.fileutils as fileutils

from pypeline.node import CommandNode
from pypeline.atomiccmd import AtomicCmd

class RAxMLReduceNode(CommandNode):
    def __init__(self, inalignment, outalignment, inpartitions, outpartitions, dependencies = ()):
        call = ["raxmlHPC",
                "-f", "c",
                "-m", "GTRGAMMA", # Model required, but not used
                "-n", "GTRGAMMA",
                "-w", "%(TEMP_DIR)s",
                "-s", "%(TEMP_IN_ALIGNMENT)s",
                "-q", "%(TEMP_IN_PARTITIONS)s"]

        self._kwargs = {"IN_ALIGNMENT"        : inalignment,
                        "IN_PARTITIONS"       : inpartitions,

                        "TEMP_IN_ALIGNMENT"   : "RAXML_alignment",
                        "TEMP_IN_PARTITIONS"  : "RAXML_partitions",
                        "TEMP_OUT_INFO"       : "RAxML_info.GTRGAMMA",

                        "OUT_ALIGNMENT"       : outalignment,
                        "OUT_PARTITIONS"      : outpartitions}

        CommandNode.__init__(self,
                             command      = AtomicCmd(call, **self._kwargs),
                             description  = "<RAxMLReduce: '%s' -> '%s'>" \
                                     % (inalignment, outalignment),
                             dependencies = dependencies)

    def _setup(self, config, temp):
        for key in ("IN_ALIGNMENT", "IN_PARTITIONS"):
            source      = self._kwargs[key]
            destination = os.path.join(temp, self._kwargs["TEMP_" + key])

            fileutils.copy_file(source, destination)

        CommandNode._setup(self, config, temp)

    def _teardown(self, config, temp):
        for postfix in ("ALIGNMENT", "PARTITIONS"):
            filenames = [self._kwargs["TEMP_IN_" + postfix],
                         self._kwargs["TEMP_IN_" + postfix] + ".reduced",
                         self._kwargs["OUT_" + postfix]]

            for (source, destination) in zip(filenames, filenames[1:]):
                source      = fileutils.reroot_path(temp, source)
                destination = fileutils.reroot_path(temp, destination)

                if not os.path.exists(destination):
                    fileutils.move_file(source, destination)
                elif source != destination:
                    os.remove(source)

        CommandNode._teardown(self, config, temp)

class RAxMLRapidBSNode(CommandNode):
    def __init__(self, infile, partitions, destination, model = "GTRGAMMAI", replicates = "autoMRE", bs_branchln = False, threads = 1, dependencies = ()):
        """
        Arguments:
        infile      -- An alignment file in a format readable by RAxML.
        partitions  -- A set of partitions in a format readable by RAxML.
        destination -- A template string used to construct final filenames. Should consist
                       of a full path, including a single '%s', which is replaced with the
                       variable part of RAxML output files (e.g. 'info', 'bestTree', ...).
                       Example destination: '/disk/project/SN013420.RAxML.%s'
                       Example output:      '/disk/project/SN013420.RAxML.bestTree'
        model       -- DNA or Amino acid substitution model to use.
        replicates  -- Number of bootstrap replicates. Defaults to automatic selection.
        bs_branchln -- Calculate branch lengths for bootstrap trees.
        threads     -- Number of threads to use for each RAxML instance."""

        self._symlinks = [infile, partitions]
        self._template = os.path.basename(destination)

        call = ["raxmlHPC-PTHREADS" if (threads > 1) else "raxmlHPC",
                "-f", "a",
                "-m", model,
                "-n", "RapidBS",
                "-s", "%(TEMP_OUT_ALN)s",
                "-q", "%(TEMP_OUT_PART)s",
                "-w", "%(TEMP_DIR)s",
                "-x", int(random.random() * 2**31 - 1),
                "-p", int(random.random() * 2**31 - 1),
                "-N", replicates]
        if bs_branchln:
            call.append("-k")
        if threads > 1:
            call.extend(("-T", threads))

        command = AtomicCmd(call,
                            IN_ALIGNMENT    = infile,
                            IN_PARTITIONS   = partitions,

                            OUT_INFO        = destination % "info",
                            OUT_BESTTREE    = destination % "bestTree",
                            OUT_BOOTSTRAP   = destination % "bootstrap",
                            OUT_BIPART      = destination % "bipartitions",
                            OUT_BIPARTLABEL = destination % "bipartitionsBranchLabels",

                            # RAxML may write reduced alignment. These are not saved. If
                            # needed, they may be generated upfront using RAxMLReduceNode.
                            TEMP_OUT_ALN    = os.path.basename(infile),
                            TEMP_OUT_PART   = os.path.basename(partitions),
                            TEMP_OUT_R_ALN  = os.path.basename(infile) + ".reduced",
                            TEMP_OUT_R_PART = os.path.basename(partitions) + ".reduced")

        CommandNode.__init__(self,
                             command      = command,
                             description  = "<RAxMLRapidBS (%i threads): '%s' -> '%s'>" \
                                     % (threads, infile, destination),
                             threads      = threads,
                             dependencies = dependencies)

    def _setup(self, config, temp):
        CommandNode._setup(self, config, temp)

        # Required to avoid the creation of files outside the temp folder
        for filename in self._symlinks:
            source      = os.path.abspath(filename)
            destination = os.path.join(temp, os.path.basename(filename))

            os.symlink(source, destination)

    def _teardown(self, config, temp):
        for filename in os.listdir(temp):
            match = re.match("RAxML_(.*).RapidBS", filename)
            if match:
                source      = os.path.join(temp, filename)
                destination = os.path.join(temp, self._template % match.groups())

                fileutils.move_file(source, destination)

        CommandNode._teardown(self, config, temp)
<|endoftext|>


<commit_before>import sys
import datetime
import urllib2

from lxml import etree
from urllib2 import urlopen

from django.utils import timezone
from django.core.management.base import NoArgsCommand

from bulk_update.helper import bulk_update

from api.models import EyeHistory
from api.models import PopularHistory
from api.models import PopularHistoryInfo
from api.utils import humanize_time

from accounts.models import UserProfile

from common.helpers import queryset_iterator
from common.helpers import queryset_iterator_chunkify

from eyebrowse.log import logger

CHUNK_SIZE = 10000

class Command(NoArgsCommand):
    help = 'Updates popular history whoo!'

    def handle(self, **options):
        self.log('Beginning update')
        self._reset_values()
        self._populate_popular_history()
        self._calculate_scores()
        self.log('Update complete.')

    def log(self, msg):
        msg = 'update_popular_history:::%s\n' % msg
        logger.info(msg)
        self.stdout.write(msg)

    def _log_updates(self, i, total_updates, function):
        self.log(
            "Completed %d/%d updates. [%s]" % (i, total_updates, function))

    def _reset_values(self):
        self.log('resetting values')

        # set total time ago (sum of time ago for all eyebrowse visits to this page in last 10 weeks) to 0
        # set total time spent (sum of time spend for all eyebrowse visits to
        # this page in last 10 weeks) to 0
        PopularHistory.objects.update(total_time_ago=0, total_time_spent=0)

    def _create_pop(self, e, url):
        self.log("_create_pop, %s" % e.title)
        p, _ = PopularHistoryInfo.objects.get_or_create(url=url,
                                                        domain=e.domain,
                                                        favicon_url=e.favicon_url,
                                                        title=e.title)

        try:
            try:
                conn = urlopen(e.url)
            except urllib2.HTTPError:
                p.save()
                return p
            f = conn.read()
        except:
            p.save()
            return p
        try:
            tree = etree.HTML(f)
            m = tree.xpath("//meta[@property='og:image']")
            if m:
                p.img_url = m[0].get('content', '')
            else:
                n = tree.xpath("//meta[@property='twitter:image']")
                if n:
                    p.img_url = n[0].get('content', '')

            m = tree.xpath("//meta[@property='og:description']")
            if m:
                p.description = m[0].get('content', '')
            else:
                n = tree.xpath("//meta[@property='twitter:description']")
                if n:
                    p.description = n[0].get('content', '')
        except:
            pass

        p.save()

        return p

    def _populate_popular_history(self):
        self.log('_populate_popular_history')

        month_ago = datetime.datetime.now() - datetime.timedelta(weeks=10)
        timezone.make_aware(month_ago, timezone.get_current_timezone())

        # get all eyehistory events from 10 weeks ago to today
        eye_history = EyeHistory.objects.filter(
            start_time__gt=month_ago).select_related()

        i = 0  # in case we try to log but there is no eye_history
        total_updates = eye_history.count()
        for i, e in enumerate(queryset_iterator(eye_history)):
            url = e.url
            url = url[:min(255, len(url))]

            # popularhistoryinfo stores general information about this page
            # such as description, title, domain, image, etc.
            p = PopularHistoryInfo.objects.filter(url=url)
            if not p.exists():
                # try to extract description, title, etc from the page and
                # create a popularhistoryinfo item from it
                p = self._create_pop(e, url)
            else:
                p = p[0]

            # each popularhistory is associated with a popularhistoryitem
            # popularhistory contains scoring info and is tied either to
            # no user (so scoring for the firehose) or to a particular user
            # (so scoring for their following feed)
            pop_items = PopularHistory.objects.filter(
                popular_history=p, user=None).prefetch_related('eye_hists')
            if pop_items.count() == 0:
                total_pop = PopularHistory.objects.create(
                    popular_history=p, user=None)
            elif pop_items.count() > 1:
                total_pop = pop_items[0]
                for i in pop_items[1:]:
                    i.delete()
            else:
                total_pop = pop_items[0]
            self._add_users_and_messages(total_pop, e)

            # for each of the users that are following the person
            # in this eyehistory, we add this eyehistory to the
            # the popularhistory item for that user
            follow_users = UserProfile.objects.filter(
                follows=e.user.profile).select_related()

            # do this outside of the loop so we can use an iterator
            user_pop, _ = PopularHistory.objects.get_or_create(
                popular_history=p, user=e.user)
            self._add_users_and_messages(user_pop, e)

            for user_prof in queryset_iterator(follow_users):
                user_pop, _ = PopularHistory.objects.get_or_create(
                    popular_history=p, user=user_prof.user)
                self._add_users_and_messages(user_pop, e)

            if i != 0 and i % CHUNK_SIZE == 0:
                self._log_updates(i, total_updates, 'popular_history')

        self._log_updates(i, total_updates, 'popular_history')
        self._delete_old()

        # we're interested in including one's own visits to the score in
        # one's own feed, but don't want to include in list of users
        popular_history = PopularHistory.objects.filter(
            user__isnull=False).prefetch_related(
                'visitors').select_related()

        for p in queryset_iterator(popular_history):
            if p.visitors.count() == 1:
                if p.visitors.all()[0] == p.user:
                    p.delete()

        # remove eyehistories that are from over 10 weeks ago
        # if everything gets removed then delete the popularhistory
        # though this shouldn't happen (see above)
        popular_history = PopularHistory.objects.all().prefetch_related(
            'eye_hists')
        for i in queryset_iterator(popular_history):
            i.eye_hists.remove(*i.eye_hists.filter(start_time__lt=month_ago))

        # if there are any popularhistoryinfo items now that have
        # no corresponding popularhistory objects (because they've been
        # deleted for being stale presumably) then delete
        PopularHistoryInfo.objects.filter(popularhistory=None).delete()

    def _add_users_and_messages(self, popular_history_item, e):
        # we add every eyehistory to the firehose popularhistory counterpoint
        # and also keep track of the users and messages to that page
        if not popular_history_item.eye_hists.filter(pk=e.id).exists():
            popular_history_item.eye_hists.add(e)
            popular_history_item.visitors.add(e.user)

            popular_history_item.messages.add(*e.eyehistorymessage_set.all())

        # we increment the total time spent and total time ago
        popular_history_item.total_time_spent += e.total_time

        time_diff = timezone.now() - e.end_time
        popular_history_item.total_time_ago += int(
            round(time_diff.total_seconds() / 3600.0))

        popular_history_item.save()

    def _delete_old(self):
        # if there are any popularhistory items that still have
        # total_time_ago or total_time_spent as 0 then that means
        # that no eyehistories in the preceeding loop
        # had been to that page. This means the popularhistory is
        # stale (all visits are from over 10 weeks ago) so we delete
        self.log('deleting total pop')
        p = PopularHistory.objects.filter(total_time_ago=0)
        self.log('count %s' % p.count())
        p.delete()

        self.log('deleting total spent should be 0')
        p = PopularHistory.objects.filter(total_time_spent=0)
        self.log('count %s' % p.count())
        p.delete()

    def _calculate_scores(self):
        self.log('_calculate_scores')
        # we should have lists of eyehistories, list of users,
        # list of messages, total time ago, and total time spent
        # populated for each popularhistory. Now we calculate
        # scores based on these things.
        popular_history = PopularHistory.objects.all().prefetch_related(
            'eye_hists', 'messages', 'visitors')
        total_updates = popular_history.count()
        i = 0  # in case we try to log but there is no eye_history
        for i, p in enumerate(queryset_iterator(popular_history)):
            try:
                eye_hist_count = p.eye_hists.count()
                if eye_hist_count == 0:
                    p.delete()
                    continue

                # avg time ago is total time ago / number of eyehistories
                time = p.total_time_ago / float(eye_hist_count)
                p.avg_time_ago = datetime.datetime.now(
                ) - datetime.timedelta(hours=time)

                # avg time spent is total time spent / eyehistories
                time_spent = p.total_time_spent / \
                    float(eye_hist_count)
                p.humanize_avg_time = humanize_time(
                    datetime.timedelta(milliseconds=time_spent))

                # num comment score gives score based on num
                # comments with a time decay factor
                num_comments = p.messages.count()
                comment_score = float(num_comments * 40.0) / \
                    float(
                        ((
                            float(p.total_time_ago) + 1.0) /
                            float(eye_hist_count)) ** 1.2)
                p.num_comment_score = comment_score

                # num visitors score gives score based on num
                # visitors with a time decay factor
                num_vistors = p.visitors.count()
                visitor_score = float((num_vistors - 1.0) * 50.0) / \
                    float(
                        ((
                            float(p.total_time_ago) + 1.0) /
                            float(eye_hist_count)) ** 1.2)
                p.unique_visitor_score = visitor_score

                # num time score gives score based on avg time
                # spent with a time decay factor
                num_time = float(p.total_time_spent) / \
                    float(eye_hist_count)

                time_score = float((num_time ** .8) / 1000.0) / \
                    float(
                        ((
                            float(p.total_time_ago) + 1.0) /
                            float(eye_hist_count)) ** 1.5)

                time_score_2 = float((num_time - 5000) / 1000.0) / \
                    float(
                        ((
                            float(p.total_time_ago) + 1.0) /
                            float(eye_hist_count)) ** 1)
                p.avg_time_spent_score = time_score_2

                # top score combines all the scores together
                p.top_score = float(comment_score + visitor_score + time_score)

                if i != 0 and i % CHUNK_SIZE == 0:
                    self._log_updates(i, total_updates, 'calculate_scores')
            except Exception, e:
                self.log(e)
                continue

        for i, batch in enumerate(queryset_iterator_chunkify(popular_history, chunksize=CHUNK_SIZE)):
            self._log_updates(i, total_updates, 'calculate_scores:::save_objs')
            bulk_update(batch, batch_size=CHUNK_SIZE)
        self._log_updates(i, total_updates, 'calculate_scores')
<commit_msg>Fix bug with not updating values<commit_after>import sys
import datetime
import urllib2

from lxml import etree
from urllib2 import urlopen

from django.utils import timezone
from django.core.management.base import NoArgsCommand

from bulk_update.helper import bulk_update

from api.models import EyeHistory
from api.models import PopularHistory
from api.models import PopularHistoryInfo
from api.utils import humanize_time

from accounts.models import UserProfile

from common.helpers import queryset_iterator
from common.helpers import queryset_iterator_chunkify

from eyebrowse.log import logger

CHUNK_SIZE = 10000

class Command(NoArgsCommand):
    help = 'Updates popular history whoo!'

    def handle(self, **options):
        self.log('Beginning update')
        self._reset_values()
        self._populate_popular_history()
        self._calculate_scores()
        self.log('Update complete.')

    def log(self, msg):
        msg = 'update_popular_history:::%s\n' % msg
        logger.info(msg)
        self.stdout.write(msg)

    def _log_updates(self, i, total_updates, function):
        self.log(
            "Completed %d/%d updates. [%s]" % (i, total_updates, function))

    def _reset_values(self):
        self.log('resetting values')

        # set total time ago (sum of time ago for all eyebrowse visits to this page in last 10 weeks) to 0
        # set total time spent (sum of time spend for all eyebrowse visits to
        # this page in last 10 weeks) to 0
        PopularHistory.objects.update(total_time_ago=0, total_time_spent=0)

    def _create_pop(self, e, url):
        self.log("_create_pop, %s" % e.title)
        p, _ = PopularHistoryInfo.objects.get_or_create(url=url,
                                                        domain=e.domain,
                                                        favicon_url=e.favicon_url,
                                                        title=e.title)

        try:
            try:
                conn = urlopen(e.url)
            except urllib2.HTTPError:
                p.save()
                return p
            f = conn.read()
        except:
            p.save()
            return p
        try:
            tree = etree.HTML(f)
            m = tree.xpath("//meta[@property='og:image']")
            if m:
                p.img_url = m[0].get('content', '')
            else:
                n = tree.xpath("//meta[@property='twitter:image']")
                if n:
                    p.img_url = n[0].get('content', '')

            m = tree.xpath("//meta[@property='og:description']")
            if m:
                p.description = m[0].get('content', '')
            else:
                n = tree.xpath("//meta[@property='twitter:description']")
                if n:
                    p.description = n[0].get('content', '')
        except:
            pass

        p.save()

        return p

    def _populate_popular_history(self):
        self.log('_populate_popular_history')

        month_ago = datetime.datetime.now() - datetime.timedelta(weeks=10)
        timezone.make_aware(month_ago, timezone.get_current_timezone())

        # get all eyehistory events from 10 weeks ago to today
        eye_history = EyeHistory.objects.filter(
            start_time__gt=month_ago).select_related()

        i = 0  # in case we try to log but there is no eye_history
        total_updates = eye_history.count()
        for i, e in enumerate(queryset_iterator(eye_history)):
            url = e.url
            url = url[:min(255, len(url))]

            # popularhistoryinfo stores general information about this page
            # such as description, title, domain, image, etc.
            p = PopularHistoryInfo.objects.filter(url=url)
            if not p.exists():
                # try to extract description, title, etc from the page and
                # create a popularhistoryinfo item from it
                p = self._create_pop(e, url)
            else:
                p = p[0]

            # each popularhistory is associated with a popularhistoryitem
            # popularhistory contains scoring info and is tied either to
            # no user (so scoring for the firehose) or to a particular user
            # (so scoring for their following feed)
            pop_items = PopularHistory.objects.filter(
                popular_history=p, user=None).prefetch_related('eye_hists')
            if pop_items.count() == 0:
                total_pop = PopularHistory.objects.create(
                    popular_history=p, user=None)
            elif pop_items.count() > 1:
                total_pop = pop_items[0]
                for i in pop_items[1:]:
                    i.delete()
            else:
                total_pop = pop_items[0]
            self._add_users_and_messages(total_pop, e)

            # for each of the users that are following the person
            # in this eyehistory, we add this eyehistory to the
            # the popularhistory item for that user
            follow_users = UserProfile.objects.filter(
                follows=e.user.profile).select_related()

            # do this outside of the loop so we can use an iterator
            user_pop, _ = PopularHistory.objects.get_or_create(
                popular_history=p, user=e.user)
            self._add_users_and_messages(user_pop, e)

            for user_prof in queryset_iterator(follow_users):
                user_pop, _ = PopularHistory.objects.get_or_create(
                    popular_history=p, user=user_prof.user)
                self._add_users_and_messages(user_pop, e)

            if i != 0 and i % CHUNK_SIZE == 0:
                self._log_updates(i, total_updates, 'popular_history')

        self._log_updates(i, total_updates, 'popular_history')
        self._delete_old()

        # we're interested in including one's own visits to the score in
        # one's own feed, but don't want to include in list of users
        popular_history = PopularHistory.objects.filter(
            user__isnull=False).prefetch_related(
                'visitors').select_related()

        for p in queryset_iterator(popular_history):
            if p.visitors.count() == 1:
                if p.visitors.all()[0] == p.user:
                    p.delete()

        # remove eyehistories that are from over 10 weeks ago
        # if everything gets removed then delete the popularhistory
        # though this shouldn't happen (see above)
        popular_history = PopularHistory.objects.all().prefetch_related(
            'eye_hists')
        for i in queryset_iterator(popular_history):
            i.eye_hists.remove(*i.eye_hists.filter(start_time__lt=month_ago))

        # if there are any popularhistoryinfo items now that have
        # no corresponding popularhistory objects (because they've been
        # deleted for being stale presumably) then delete
        PopularHistoryInfo.objects.filter(popularhistory=None).delete()

    def _add_users_and_messages(self, popular_history_item, e):
        # we add every eyehistory to the firehose popularhistory counterpoint
        # and also keep track of the users and messages to that page
        if not popular_history_item.eye_hists.filter(pk=e.id).exists():
            popular_history_item.eye_hists.add(e)
            popular_history_item.visitors.add(e.user)

            popular_history_item.messages.add(*e.eyehistorymessage_set.all())

        # we increment the total time spent and total time ago
        popular_history_item.total_time_spent += e.total_time

        time_diff = timezone.now() - e.end_time
        popular_history_item.total_time_ago += int(
            round(time_diff.total_seconds() / 3600.0))

        popular_history_item.save()

    def _delete_old(self):
        # if there are any popularhistory items that still have
        # total_time_ago or total_time_spent as 0 then that means
        # that no eyehistories in the preceeding loop
        # had been to that page. This means the popularhistory is
        # stale (all visits are from over 10 weeks ago) so we delete
        self.log('deleting total pop')
        p = PopularHistory.objects.filter(total_time_ago=0)
        self.log('count %s' % p.count())
        p.delete()

        self.log('deleting total spent should be 0')
        p = PopularHistory.objects.filter(total_time_spent=0)
        self.log('count %s' % p.count())
        p.delete()

    def _calculate_scores(self):
        self.log('_calculate_scores')
        # we should have lists of eyehistories, list of users,
        # list of messages, total time ago, and total time spent
        # populated for each popularhistory. Now we calculate
        # scores based on these things.
        popular_history = PopularHistory.objects.all().prefetch_related(
            'eye_hists', 'messages', 'visitors')
        total_updates = popular_history.count()
        i = 0  # in case we try to log but there is no eye_history
        for i, p in enumerate(queryset_iterator(popular_history)):
            try:
                eye_hist_count = p.eye_hists.count()
                if eye_hist_count == 0:
                    p.delete()
                    continue

                # avg time ago is total time ago / number of eyehistories
                time = p.total_time_ago / float(eye_hist_count)
                p.avg_time_ago = datetime.datetime.now(
                ) - datetime.timedelta(hours=time)

                # avg time spent is total time spent / eyehistories
                time_spent = p.total_time_spent / \
                    float(eye_hist_count)
                p.humanize_avg_time = humanize_time(
                    datetime.timedelta(milliseconds=time_spent))

                # num comment score gives score based on num
                # comments with a time decay factor
                num_comments = p.messages.count()
                comment_score = float(num_comments * 40.0) / \
                    float(
                        ((
                            float(p.total_time_ago) + 1.0) /
                            float(eye_hist_count)) ** 1.2)
                p.num_comment_score = comment_score

                # num visitors score gives score based on num
                # visitors with a time decay factor
                num_vistors = p.visitors.count()
                visitor_score = float((num_vistors - 1.0) * 50.0) / \
                    float(
                        ((
                            float(p.total_time_ago) + 1.0) /
                            float(eye_hist_count)) ** 1.2)
                p.unique_visitor_score = visitor_score

                # num time score gives score based on avg time
                # spent with a time decay factor
                num_time = float(p.total_time_spent) / \
                    float(eye_hist_count)

                time_score = float((num_time ** .8) / 1000.0) / \
                    float(
                        ((
                            float(p.total_time_ago) + 1.0) /
                            float(eye_hist_count)) ** 1.5)

                time_score_2 = float((num_time - 5000) / 1000.0) / \
                    float(
                        ((
                            float(p.total_time_ago) + 1.0) /
                            float(eye_hist_count)) ** 1)
                p.avg_time_spent_score = time_score_2

                # top score combines all the scores together
                p.top_score = float(comment_score + visitor_score + time_score)

                if i != 0 and i % CHUNK_SIZE == 0:
                    self._log_updates(i, total_updates, 'calculate_scores')
            except Exception, e:
                self.log(e)
                continue

        bulk_update(popular_history, batch_size=CHUNK_SIZE)
        self._log_updates(i, total_updates, 'calculate_scores')
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# vim:set ts=4 sts=4 sw=4 et:
"""MC Helpers.
"""
from __future__ import absolute_import, print_function, division

import numpy as np

from km3pipe import Module
from km3pipe.mc import pdg2name
from km3pipe.math import zenith, azimuth

__author__ = "Michael Moser and Tamas Gal and Moritz Lotze"

NEUTRINOS = {
    'nu_e',
    'anu_e',
    'nu_mu',
    'anu_mu',
    'nu_tau',
    'anu_tau',
}

class McTruth(Module):
    """Extract MC info of 1st MC track.

    Parameters
    ----------
    most_energetic_primary: bool, default=True
    """

    def configure(self):
        self.most_energetic_primary = bool(self.get('most_energetic_primary')
                                           ) or True

    @classmethod
    def t2f(cls, row):
        return pdg2name(row['type'])

    @classmethod
    def is_nu(cls, flavor):
        return flavor in NEUTRINOS

    def process(self, blob):
        mc = blob['McTracks'].conv_to('pandas')
        if self.most_energetic_primary:
            mc.sort_values('energy', ascending=False, inplace=True)
        mc = mc.head(1)
        mc['zenith'] = zenith(mc[['dir_x', 'dir_y', 'dir_z']])
        mc['azimuth'] = azimuth(mc[['dir_x', 'dir_y', 'dir_z']])
        flavor = mc.apply(self.t2f, axis=1)
        mc['is_neutrino'] = flavor.apply(self.is_nu)
        blob['McTruth'] = mc
        return blob

def convert_mctime(hits_times_jte, evt_timestamp_in_ns, evt_mc_time):
    """
    Function that converts JTE times of hits to MC times.

    Parameters
    ----------
    hits_times_jte : np.ndarray
        Hits array with JTE times.
    evt_timestamp_in_ns : int
        Total timestamp of the event in nanoseconds.
    evt_mc_time : int
        Mc time of the event in nanoseconds.

    Returns
    -------
    ndarray
        Converted hits array with MC times.
    """
    hits_times_jte = np.atleast_1d(hits_times_jte)
    hits_times_mc = hits_times_jte + evt_timestamp_in_ns - evt_mc_time
    return hits_times_mc

class MCTimeCorrector(Module):
    """
    Module that converts JTE hit times to MC times.

    Parameters
    ----------
    hits_key : str, optional
        Name of the Hits to convert (default: 'Hits').
    event_info_key : str, optional
        Name of the EventInfo to store this in (default: 'EventInfo').
    """

    def configure(self):
        self.hits_key = self.get('hits_key', default='Hits')
        self.event_info_key = self.get('event_info_key', default='EventInfo')
        self.convert_time = np.frompyfunc(convert_mctime, 3, 1)

    def process(self, blob):
        event_info = blob[self.event_info_key]
        hits = blob[self.hits_key]
        hits['time'] = self.convert_time(
            hits.time, event_info.timestamp_in_ns, event_info.mc_time
        )
        blob[self.hits_key] = hits
        return blob
<commit_msg>FIX convert timestamp to ns<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# vim:set ts=4 sts=4 sw=4 et:
"""MC Helpers.
"""
from __future__ import absolute_import, print_function, division

import numpy as np

from km3pipe import Module
from km3pipe.mc import pdg2name
from km3pipe.math import zenith, azimuth

__author__ = "Michael Moser and Tamas Gal and Moritz Lotze"

NEUTRINOS = {
    'nu_e',
    'anu_e',
    'nu_mu',
    'anu_mu',
    'nu_tau',
    'anu_tau',
}

class McTruth(Module):
    """Extract MC info of 1st MC track.

    Parameters
    ----------
    most_energetic_primary: bool, default=True
    """

    def configure(self):
        self.most_energetic_primary = bool(self.get('most_energetic_primary')
                                           ) or True

    @classmethod
    def t2f(cls, row):
        return pdg2name(row['type'])

    @classmethod
    def is_nu(cls, flavor):
        return flavor in NEUTRINOS

    def process(self, blob):
        mc = blob['McTracks'].conv_to('pandas')
        if self.most_energetic_primary:
            mc.sort_values('energy', ascending=False, inplace=True)
        mc = mc.head(1)
        mc['zenith'] = zenith(mc[['dir_x', 'dir_y', 'dir_z']])
        mc['azimuth'] = azimuth(mc[['dir_x', 'dir_y', 'dir_z']])
        flavor = mc.apply(self.t2f, axis=1)
        mc['is_neutrino'] = flavor.apply(self.is_nu)
        blob['McTruth'] = mc
        return blob

def convert_mctime(hits_times_jte, evt_timestamp_in_ns, evt_mc_time):
    """
    Function that converts JTE times of hits to MC times.

    Parameters
    ----------
    hits_times_jte : np.ndarray
        Hits array with JTE times.
    evt_timestamp_in_ns : int
        Total timestamp of the event in nanoseconds.
    evt_mc_time : int
        Mc time of the event in nanoseconds.

    Returns
    -------
    ndarray
        Converted hits array with MC times.
    """
    hits_times_jte = np.atleast_1d(hits_times_jte)
    hits_times_mc = hits_times_jte + evt_timestamp_in_ns - evt_mc_time
    return hits_times_mc

class MCTimeCorrector(Module):
    """
    Module that converts JTE hit times to MC times.

    Parameters
    ----------
    hits_key : str, optional
        Name of the Hits to convert (default: 'Hits').
    event_info_key : str, optional
        Name of the EventInfo to store this in (default: 'EventInfo').
    """

    def configure(self):
        self.hits_key = self.get('hits_key', default='Hits')
        self.event_info_key = self.get('event_info_key', default='EventInfo')
        self.convert_time = np.frompyfunc(convert_mctime, 3, 1)

    def process(self, blob):
        event_info = blob[self.event_info_key]
        hits = blob[self.hits_key]
        timestamp_in_ns = event_info.timestamp * 1e9 + event_info.nanoseconds
        hits['time'] = self.convert_time(
            hits.time, timestamp_in_ns, event_info.mc_time
        )
        blob[self.hits_key] = hits
        return blob
<|endoftext|>


<commit_before>import sys
import os.path
import argparse

import gdbm as db

from mailbox import Maildir, MaildirMessage
from collections import Counter

from .config import IniConfig

def get_maildir(maildir, cache):
    try:
        return cache[maildir]
    except KeyError:
        pass

    result = cache[maildir] = Maildir(
        os.path.expanduser(maildir), factory=None, create=True)

    result.name = os.path.basename(maildir)

    return result

def get_state(config, account, cache):
    try:
        return cache[account]
    except KeyError:
        pass

    result = cache[account] = db.open(
        os.path.join(os.path.expanduser(config.state_dir), account + '.state'), 'c')

    return result

def store_message(maildir, state, uid, message, flags):
    msg = MaildirMessage(message)
    if '\\Seen' in flags:
        msg.add_flag('S')

    if uid in state:
        key, flags = state[uid].split('\n')
        if flags != msg.get_flags():
            oldmessage = maildir[key]
            oldmessage.set_flags(msg.get_flags())
            maildir[key] = oldmessage
            maildir.flush()
    else:
        state[uid] = maildir.add(msg) + '\n' + msg.get_flags()
        state.sync()
        maildir.flush()

def sync_local(maildir, state, account):
    uid = state.firstkey()
    maxuid = 0
    changes = {'seen':[], 'trash':[]}
    while uid is not None:
        key, flags = state[uid].split('\n')
        flags = set(flags)
        maxuid = max(int(uid), maxuid)

        try:
            message = maildir[key]
        except KeyError:
            changes['trash'].append(uid)
        else:
            mflags = set(message.get_flags())

            if 'S' in mflags and 'S' not in flags:
                changes['seen'].append(uid)

        uid = state.nextkey(uid)

    return maxuid, changes

def sync(config):
    state_cache = {}
    maildir_cache = {}
    for s in config.sync_list:
        account = config.accounts[s.account]
        maildir = get_maildir(s.maildir, maildir_cache)
        state = get_state(config, s.account, state_cache)
        maxuid, changes = sync_local(maildir, state, account)
        folder = account.folders[s.folder]

        folder.apply_changes(changes, state, s.trash)

        messages = folder.fetch(config.fetch_last, maxuid)
        for m in messages:
            store_message(maildir, state, m['uid'], m['body'], m['flags'])

def check(config):
    maildir_cache = {}
    for s in config.sync_list:
        get_maildir(s.maildir, maildir_cache)

    result = Counter()
    for maildir in maildir_cache.values():
        for message in maildir:
            if 'S' not in message.get_flags():
                result[maildir.name] += 1

    for k, v in result.iteritems():
        print '{}\t{}'.format(k, v)

    return result

def show_folders(config):
    for account, box in config.accounts.iteritems():
        print '{}: {}'.format(account, list(box.folders))

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-C', '--config', dest='config',
        default=os.path.expanduser('~/.config/norlessrc'))

    parser.add_argument('-c', '--check', dest='check', action='store_true')
    parser.add_argument('-s', '--show-folders', dest='show_folders', action='store_true')

    args = parser.parse_args()

    config = IniConfig(args.config)

    if args.show_folders:
        show_folders(config)
    else:
        sync(config)
        if args.check:
            if not check(config):
                sys.exit(1)

<commit_msg>keep state for every imap folder<commit_after>import sys
import os.path
import argparse

import gdbm as db

from mailbox import Maildir, MaildirMessage
from collections import Counter

from .config import IniConfig

def get_maildir(maildir, cache):
    try:
        return cache[maildir]
    except KeyError:
        pass

    result = cache[maildir] = Maildir(
        os.path.expanduser(maildir), factory=None, create=True)

    result.name = os.path.basename(maildir)

    return result

def get_state(config, sync, cache):
    key = sync.account, sync.folder
    try:
        return cache[key]
    except KeyError:
        pass

    result = cache[key] = db.open(
        os.path.join(os.path.expanduser(config.state_dir), '{}.{}'.format(*key)), 'c')

    return result

def store_message(maildir, state, uid, message, flags):
    msg = MaildirMessage(message)
    if '\\Seen' in flags:
        msg.add_flag('S')

    if uid in state:
        key, flags = state[uid].split('\n')
        if flags != msg.get_flags():
            oldmessage = maildir[key]
            oldmessage.set_flags(msg.get_flags())
            maildir[key] = oldmessage
            maildir.flush()
    else:
        state[uid] = maildir.add(msg) + '\n' + msg.get_flags()
        state.sync()
        maildir.flush()

def sync_local(maildir, state, account):
    uid = state.firstkey()
    maxuid = 0
    changes = {'seen':[], 'trash':[]}
    while uid is not None:
        key, flags = state[uid].split('\n')
        flags = set(flags)
        maxuid = max(int(uid), maxuid)

        try:
            message = maildir[key]
        except KeyError:
            changes['trash'].append(uid)
        else:
            mflags = set(message.get_flags())

            if 'S' in mflags and 'S' not in flags:
                changes['seen'].append(uid)

        uid = state.nextkey(uid)

    return maxuid, changes

def sync(config):
    state_cache = {}
    maildir_cache = {}
    for s in config.sync_list:
        account = config.accounts[s.account]
        maildir = get_maildir(s.maildir, maildir_cache)
        state = get_state(config, s, state_cache)
        maxuid, changes = sync_local(maildir, state, account)
        folder = account.folders[s.folder]

        folder.apply_changes(changes, state, s.trash)

        messages = folder.fetch(config.fetch_last, maxuid)
        for m in messages:
            store_message(maildir, state, m['uid'], m['body'], m['flags'])

def check(config):
    maildir_cache = {}
    for s in config.sync_list:
        get_maildir(s.maildir, maildir_cache)

    result = Counter()
    for maildir in maildir_cache.values():
        for message in maildir:
            if 'S' not in message.get_flags():
                result[maildir.name] += 1

    for k, v in result.iteritems():
        print '{}\t{}'.format(k, v)

    return result

def show_folders(config):
    for account, box in config.accounts.iteritems():
        print '{}: {}'.format(account, list(box.folders))

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-C', '--config', dest='config',
        default=os.path.expanduser('~/.config/norlessrc'))

    parser.add_argument('-c', '--check', dest='check', action='store_true')
    parser.add_argument('-s', '--show-folders', dest='show_folders', action='store_true')

    args = parser.parse_args()

    config = IniConfig(args.config)

    if args.show_folders:
        show_folders(config)
    else:
        sync(config)
        if args.check:
            if not check(config):
                sys.exit(1)

<|endoftext|>


<commit_before>__version__ = "0.5.0"
<commit_msg>bump version<commit_after>__version__ = "0.5.1"
<|endoftext|>


<commit_before>import ast
from importlib import import_module

from .lexer import Token, tokenise

def kompile(src):
    '''
    Creates a new class based on the supplied template, and returnsit.

    class Template(object):
        def __call__(self, context):
            return ''.join(str(x) for x in self._root(context))

        def _root(self, context):
            yield ''
            yield ...
            yield from self.head(context)

    Blocks create new methods, and add a 'yield from self.{block}(context)' to
    the current function

    '''

    state = {
        'bases': ['object'],
        'methods': [],
        'stream': tokenise(src),
        'tags': {},
    }

    # Define the __call__ method
    func = ast.FunctionDef(
        name='__call__',
        args=ast.arguments(
            args=[
                ast.arg(arg='self', annotation=None),
                ast.arg(arg='context', annotation=None),
            ],
            vararg=None,
            kwonlyargs=[],
            kwarg=None,
            defaults=[],
            kw_defaults=[],
        ),
        body=[
            ast.Return(
                value=ast.Call(
                    func=ast.Attribute(
                        value=ast.Str(s=''),
                        attr='join',
                        ctx=ast.Load()
                    ),
                    args=[
                        ast.GeneratorExp(
                            elt=ast.Call(
                                func=ast.Name(id='str', ctx=ast.Load()),
                                args=[
                                    ast.Name(id='x', ctx=ast.Load()),
                                ],
                                keywords=[], starargs=None, kwargs=None
                            ),
                            generators=[
                                ast.comprehension(
                                    target=ast.Name(id='x', ctx=ast.Store()),
                                    iter=ast.Call(
                                        func=ast.Attribute(
                                            value=ast.Name(id='self', ctx=ast.Load()),
                                            attr='_root',
                                            ctx=ast.Load()
                                        ),
                                        args=[
                                            ast.Name(id='context', ctx=ast.Load()),
                                        ], keywords=[], starargs=None, kwargs=None
                                    ),
                                    ifs=[]
                                ),
                            ]
                        ),
                    ],
                    keywords=[], starargs=None, kwargs=None
                )
            ),
        ],
        decorator_list=[],
    )

    state['methods'].append(func)

    state['methods'].append(build_method(state, '_root'))

    # define the class
    klass = ast.ClassDef(
        name='Template',
        bases=[
            ast.Name(id=base, ctx=ast.Load())
            for base in reversed(state['bases'])
        ],
        body=state['methods'],
        keywords=[],
        starargs=None,
        kwargs=None,
        decorator_list=[]
    )

    # Wrap it in a module
    inst = ast.Module(body=[klass])

    ast.fix_missing_locations(inst)

    # Compile code to create class
    code = compile(inst, filename='<compiler>', mode='exec')

    # Execute it and return the instance
    g = {}
    eval(code, g)

    return g['Template']

def build_method(state, name, endnode=None):

    # Define the body of the function
    body = [
        # Include a blank to ensure it's never empty
        ast.Expr(value=ast.Yield(value=ast.Str(s=''))),
    ]

    # Create the root method
    func = ast.FunctionDef(
        name=name,
        args=ast.arguments(
            args=[
                ast.arg(arg='self', annotation=None),
                ast.arg(arg='context', annotation=None),
            ],
            vararg=None,
            kwonlyargs=[],
            kwarg=None,
            defaults=[],
            kw_defaults=[],
        ),
        body=body,
        decorator_list=[],
    )

    # Parse Nodes
    # They need the parser, and state
    body.extend(parse_node(state, endnode))

    return func

class VarVisitor(ast.NodeTransformer):
    def visit_Name(self, node):
        return ast.Subscript(
            value=ast.Name(id='context', ctx=ast.Load()),
            slice=ast.Index(value=ast.Str(s=node.id)),
            ctx=ast.Load(),
        )

def parse_node(state, endnode=None):
    for mode, token in state['stream']:
        if mode == Token.load:
            load_library(state, token)
            continue
        elif mode == Token.text:
            node = ast.Yield(value=ast.Str(s=token))
        elif mode == Token.var:
            code = ast.parse(token, mode='eval')
            VarVisitor().visit(code)
            node = ast.Yield(value=code.body)
        elif mode == Token.block:
            bits = token.strip().split(' ', 1)
            tag_name = bits.pop(0).strip()
            if tag_name == endnode:
                return
            func = state['tags'][tag_name]
            node = func(state, *bits)
        else:
            # Must be a comment
            continue

        yield ast.Expr(value=node)

def load_library(state, path):
    '''
    Load a template library into the state.
    '''
    module = import_module(path)
    state['tags'].update(module.register.tags)
<commit_msg>Class-ify parsing<commit_after>import ast
from importlib import import_module

from .lexer import TokenType, tokenise

class VarVisitor(ast.NodeTransformer):
    def visit_Name(self, node):
        return ast.Subscript(
            value=ast.Name(id='context', ctx=ast.Load()),
            slice=ast.Index(value=ast.Str(s=node.id)),
            ctx=ast.Load(),
        )

class Parser:
    def __init__(self, src):
        self.stream = tokenise(src)
        self.bases = ['object']
        self.methods = []
        self.tags = {}

    def load_library(self, path):
        '''
        Load a template library into the state.
        '''
        module = import_module(path)
        self.tags.update(module.register.tags)

    def build_method(self, name, endnodes=None):

        # Define the body of the function
        body = [
            # Include a blank to ensure it's never empty
            ast.Expr(value=ast.Yield(value=ast.Str(s=''))),
        ]

        # Create the root method
        func = ast.FunctionDef(
            name=name,
            args=ast.arguments(
                args=[
                    ast.arg(arg='self', annotation=None),
                    ast.arg(arg='context', annotation=None),
                ],
                vararg=None,
                kwonlyargs=[],
                kwarg=None,
                defaults=[],
                kw_defaults=[],
            ),
            body=body,
            decorator_list=[],
        )

        body.extend(self.parse_node(endnodes))

        self.methods.append(func)

        return func

    def parse_node(self, endnodes=None):
        for token in self.stream:
            if token.mode == TokenType.load:
                self.load_library(token.token)
                continue
            elif token.mode == TokenType.text:
                node = ast.Yield(value=ast.Str(s=token.token))
            elif token.mode == TokenType.var:
                code = ast.parse(token.token, mode='eval')
                VarVisitor().visit(code)
                node = ast.Yield(value=code.body)
            elif token.mode == TokenType.block:
                bits = token.token.strip().split(' ', 1)
                tag_name = bits.pop(0).strip()
                if endnodes and tag_name in endnodes:
                    return token
                func = self.tags[tag_name]
                node = func(self, *bits)
            else:
                # Must be a comment
                continue

            yield ast.Expr(value=node, lineno=token.lineno)

    def build_class(self):
        return ast.ClassDef(
            name='Template',
            bases=[
                ast.Name(id=base, ctx=ast.Load())
                for base in reversed(self.bases)
            ],
            body=self.methods,
            keywords=[],
            starargs=None,
            kwargs=None,
            decorator_list=[]
        )

def kompile(src):
    '''
    Creates a new class based on the supplied template, and returnsit.

    class Template(object):
        def __call__(self, context):
            return ''.join(str(x) for x in self._root(context))

        def _root(self, context):
            yield ''
            yield ...
            yield from self.head(context)

    Blocks create new methods, and add a 'yield from self.{block}(context)' to
    the current function

    '''

    parser = Parser(src)
    parser.load_library('knights.k_tags')

    # Define the __call__ method
    func = ast.FunctionDef(
        name='__call__',
        args=ast.arguments(
            args=[
                ast.arg(arg='self', annotation=None),
                ast.arg(arg='context', annotation=None),
            ],
            vararg=None,
            kwonlyargs=[],
            kwarg=None,
            defaults=[],
            kw_defaults=[],
        ),
        body=[
            ast.Return(
                value=ast.Call(
                    func=ast.Attribute(
                        value=ast.Str(s=''),
                        attr='join',
                        ctx=ast.Load()
                    ),
                    args=[
                        ast.GeneratorExp(
                            elt=ast.Call(
                                func=ast.Name(id='str', ctx=ast.Load()),
                                args=[
                                    ast.Name(id='x', ctx=ast.Load()),
                                ],
                                keywords=[], starargs=None, kwargs=None
                            ),
                            generators=[
                                ast.comprehension(
                                    target=ast.Name(id='x', ctx=ast.Store()),
                                    iter=ast.Call(
                                        func=ast.Attribute(
                                            value=ast.Name(id='self', ctx=ast.Load()),
                                            attr='_root',
                                            ctx=ast.Load()
                                        ),
                                        args=[
                                            ast.Name(id='context', ctx=ast.Load()),
                                        ], keywords=[], starargs=None, kwargs=None
                                    ),
                                    ifs=[]
                                ),
                            ]
                        ),
                    ],
                    keywords=[], starargs=None, kwargs=None
                )
            ),
        ],
        decorator_list=[],
    )

    parser.methods.append(func)
    parser.build_method('_root')

    klass = parser.build_class()

    # Wrap it in a module
    inst = ast.Module(body=[klass])

    ast.fix_missing_locations(inst)

    # Compile code to create class
    code = compile(inst, filename='<compiler>', mode='exec')

    # Execute it and return the instance
    g = {}
    eval(code, g)

    return g['Template']
<|endoftext|>


<commit_before>VERSION = "0.4"

# August 9, 2010 -- ss -- made a long version to use in generated files
# TODO: make a switch to control generation of "long banners" in generated
#       files.  I like to have this info when i go back to a system after a
#       while but don't want to have to hand-code globs of stuff into every
#       template.  The switch will control how much information is automatically
#       inserted at the top of every file.
LONG_VERSION = "Kokki version %s : http://github.com/samuel/kokki" % VERSION

def version():
    return VERSION

def long_version():
    return LONG_VERSION
<commit_msg>Version now 0.4.1<commit_after>VERSION = "0.4.1"

# August 9, 2010 -- ss -- made a long version to use in generated files
# TODO: make a switch to control generation of "long banners" in generated
#       files.  I like to have this info when i go back to a system after a
#       while but don't want to have to hand-code globs of stuff into every
#       template.  The switch will control how much information is automatically
#       inserted at the top of every file.
LONG_VERSION = "Kokki version %s : http://github.com/samuel/kokki" % VERSION

def version():
    return VERSION

def long_version():
    return LONG_VERSION
<|endoftext|>