<commit_before>#
# ------------------------------------------------------------
# Copyright (c) All rights reserved
# SiLab, Institute of Physics, University of Bonn
# ------------------------------------------------------------
#

import logging
from copy import deepcopy
import collections
import array

from basil.utils.BitLogic import BitLogic
from basil.HL.HardwareLayer import HardwareLayer

# description attributes
read_only = ['read_only', 'read-only', 'readonly', 'ro']
write_only = ['write_only', 'write-only', 'writeonly', 'wo']
is_byte_array = ['byte_array', 'byte-array', 'bytearray']

class RegisterHardwareLayer(HardwareLayer, dict):
    '''Register Hardware Layer.

    Implementation of advanced register operations.

    Example:
    _registers = {'RESET': {'descr': {'addr': 0, 'size': 8, 'properties': ['writeonly']}},       <-- 8-bit reset register, writeonly
                  'LOST_DATA_COUNTER': {'descr': {'addr': 0, 'size': 8, 'properties': ['ro']}},  <-- 8-bit data register, 'ro' equivalent to 'readonly'
                  'ENABLE': {'descr': {'addr': 1, 'size': 1, 'offset': 0}},                      <-- 1-bit register
                  'ENABLE_EXTERN': {'descr': {'addr': 1, 'size': 1, 'offset': 1}},               <-- 1-bit register
                  'EN_ARMING': {'descr': {'addr': 1, 'size': 1, 'offset': 2}},                   <-- 1-bit register
                  'EN_WRITE_TIMESTAMP': {'descr': {'addr': 1, 'size': 1, 'offset': 3}},          <-- 1-bit register
                  'EVENT_COUNTER': {'descr': {'addr': 2, 'size': 32, 'properties': ['ro']}}      <-- 32-bit register, 'ro' equivalent to 'readonly'
    _require_version = '==3'  <-- or use '<=', '>=', ... accordingly
    '''
    _registers = {}
    _require_version = None

    def __init__(self, intf, conf):
        super(RegisterHardwareLayer, self).__init__(intf, conf)
        # require interface and base address
        self._intf = intf
        self._base_addr = conf['base_addr']
        for reg in self._registers.iterkeys():
            self.add_property(reg)
            dict.__setitem__(self, reg, None)  # set values, but not writing to the interface

    def init(self):
        # reset on initialization
#         self.RESET = 0
        if 'VERSION' in self._registers:
            version = str(self.VERSION)
        else:
            version = None
        logging.debug("Initializing %s from module %s (Version: %s)" % (self.__class__.__name__, self.__class__.__module__, version if 'VERSION' in self._registers else 'n/a'))
        if self._require_version and not eval(version + self._require_version):
            raise Exception("FPGA module %s does not satisfy version requirements (read: %s, require: %s)" % (self.__class__.__module__, version, self._require_version.strip()))
        for reg, value in self._registers.iteritems():
            if reg in self._init:
                self[reg] = self._init[reg]
            elif 'default' in value and not ('properties' in value['descr'] and [i for i in read_only if i in value['descr']['properties']]):
                self[reg] = value['default']
            else:  # do nothing here, no value to write
                pass
#                 self[reg] = None

    def set_value(self, value, addr, size, offset, **kwargs):
        '''Writing a value of any arbitrary size (max. unsigned int 64) and offset to a register

        Parameters
        ----------
        value : int, str
            The register value (int, long, bit string) to be written.
        addr : int
            The register address.
        size : int
            Bit size/length of the value to be written to the register.
        offset : int
            Offset of the value to be written to the register (in number of bits).

        Returns
        -------
        nothing
        '''
        if not size and isinstance(value, (int, long)):
            raise ValueError('Size must be greater than zero')
        if isinstance(value, (int, long)) and value.bit_length() > size:
            raise ValueError('Value is too big for given size')
        elif isinstance(value, basestring) and size and len(value) != size:
            raise ValueError('Bit string does not match to the given size')
        div, mod = divmod(size + offset, 8)
        if mod:
            div += 1
        ret = self._intf.read(self._base_addr + addr, size=div)
        reg = BitLogic()
        reg.frombytes(ret.tostring())
        if isinstance(value, (int, long)):
            reg[size + offset - 1:offset] = BitLogic.from_value(value, size=size)
        elif isinstance(value, basestring):
            reg[size + offset - 1:offset] = BitLogic(value)
        else:
            raise ValueError('Type not supported: %s' % type(value))
        self._intf.write(self._base_addr + addr, data=array.array('B', reg.tobytes()))

    def get_value(self, addr, size, offset, **kwargs):
        '''Reading a value of any arbitrary size (max. unsigned int 64) and offset from a register

        Parameters
        ----------
        addr : int
            The register address.
        size : int
            Bit size/length of the value.
        offset : int
            Offset of the value to be written to the register (in number of bits).

        Returns
        -------
        reg : int
            Register value.
        '''
        div, mod = divmod(size + offset, 8)
        if mod:
            div += 1
        ret = self._intf.read(self._base_addr + addr, size=div)
        reg = BitLogic()
        reg.frombytes(ret.tostring())
        return reg[size + offset - 1:offset].tovalue()

    def set_bytes(self, data, addr, **kwargs):
        '''Writing bytes of any arbitrary size

        Parameters
        ----------
        data : iterable
            The data (byte array) to be written.
        addr : int
            The register address.

        Returns
        -------
        nothing
        '''
        self._intf.write(self._conf['base_addr'] + addr, data)

    def get_bytes(self, addr, size, **kwargs):
        '''Reading bytes of any arbitrary size

        Parameters
        ----------.
        addr : int
            The register address.
        size : int
            Byte length of the value.

        Returns
        -------
        data : iterable
            Byte array.
        '''
        return self._intf.read(self._conf['base_addr'] + addr, size)

    def set_configuration(self, conf):
        if conf:
            for reg, value in conf.iteritems():
                self[reg] = value

    def get_configuration(self):
        conf = {}
        for reg in self._registers.iterkeys():
            descr = self._registers[reg]['descr']
            if not ('properties' in descr and [i for i in write_only if i in descr['properties']]) and not ('properties' in descr and [i for i in read_only if i in descr['properties']]):
                conf[reg] = self[reg]
        return conf

    def add_property(self, attribute):
        # create local setter and getter with a particular attribute name
#         getter = lambda self: self._get(attribute)
#         setter = lambda self, value: self._set(attribute, value)
        # Workaround: obviously dynamic properties catch exceptions
        # Print error message and return None

        def getter(self):
            try:
                return self._get(attribute)
            except Exception, e:
                logging.error(e)
                return None

        def setter(self, value):
            try:
                return self._set(attribute, value)
            except Exception, e:
                logging.error(e)
                return None
        # construct property attribute and add it to the class
        setattr(self.__class__, attribute, property(fget=getter, fset=setter, doc=attribute + ' register'))

    def set_default(self):
        for reg, value in self._registers.iteritems():
            if 'default' in value and not ('properties' in value['descr'] and [i for i in read_only if i in self._registers[reg]['descr']['properties']]):
                self._set(reg, value['default'])

    def _get(self, reg):
        descr = deepcopy(self._registers[reg]['descr'])
        if 'properties' in descr and [i for i in write_only if i in descr['properties']]:
            # allows a lazy-style of programming
            if 'default' in self._registers[reg]:
                self._set(reg, self._registers[reg]['default'])
            else:
                self._set(reg, 0)
            # raise error when doing read on write-only register
#             raise IOError('Register is write-only')
            # return None to prevent misuse
            return None
        else:
            if 'properties' in descr and [i for i in is_byte_array if i in descr['properties']]:
                ret_val = self.get_bytes(**descr)
                ret_val = array.array('B', ret_val).tolist()
#                 curr_val = dict.__getitem__(self, reg)
            else:
                descr.setdefault('offset', 0)
                ret_val = self.get_value(**descr)
                curr_val = dict.__getitem__(self, reg)
#                 curr_val = self.setdefault(reg, None)
                if curr_val is not None and 'properties' in descr and not [i for i in read_only if i in descr['properties']] and curr_val != ret_val:
                    raise ValueError('Read value was not expected: read: %s, expected: %s' % (str(ret_val), str(curr_val)))
            return ret_val

    def _set(self, reg, value):
        descr = deepcopy(self._registers[reg]['descr'])
        if 'properties' in descr and [i for i in read_only if i in descr['properties']]:
            raise IOError('Register is read-only')
        if 'properties' in descr and [i for i in is_byte_array if i in descr['properties']]:
            if not isinstance(value, collections.Iterable):
                raise ValueError('For array byte_register iterable object is needed')
            value = array.array('B', value).tolist()
            self.set_bytes(value, **descr)
            dict.__setitem__(self, reg, value)
        else:
            descr.setdefault('offset', 0)
            try:
                self.set_value(value, **descr)
            except ValueError:
                raise
            else:
                dict.__setitem__(self, reg, value if isinstance(value, (int, long)) else int(value, base=2))

    def __getitem__(self, name):
        return self._get(name)

    def __setitem__(self, name, value):
        return self._set(name, value)

    def __getattr__(self, name):
        '''called only on last resort if there are no attributes in the instance that match the name
        '''
        if name.isupper() and name not in self._registers:
            raise ValueError('%s register does not exist in %s' % (name, self.__class__))

        def method(*args, **kwargs):
            nsplit = name.split('_', 1)
            if len(nsplit) == 2 and nsplit[0] == 'set' and len(args) == 1 and not kwargs:
                self[nsplit[1]] = args[0]  # returns None
            elif len(nsplit) == 2 and nsplit[0] == 'get' and not args and not kwargs:
                return self[nsplit[1]]
            else:
                raise AttributeError("%r object has no attribute %r" % (self.__class__, name))
        return method

    def __setattr__(self, name, value):
        if name.isupper() and name not in self._registers:
            raise ValueError('%s register does not exist in %s' % (name, self.__class__))
        return dict.__setattr__(self, name, value)
<commit_msg>ENH: remove checks, rely on BitLogic<commit_after>#
# ------------------------------------------------------------
# Copyright (c) All rights reserved
# SiLab, Institute of Physics, University of Bonn
# ------------------------------------------------------------
#

import logging
from copy import deepcopy
import collections
import array

from basil.utils.BitLogic import BitLogic
from basil.HL.HardwareLayer import HardwareLayer

# description attributes
read_only = ['read_only', 'read-only', 'readonly', 'ro']
write_only = ['write_only', 'write-only', 'writeonly', 'wo']
is_byte_array = ['byte_array', 'byte-array', 'bytearray']

class RegisterHardwareLayer(HardwareLayer, dict):
    '''Register Hardware Layer.

    Implementation of advanced register operations.

    Example:
    _registers = {'RESET': {'descr': {'addr': 0, 'size': 8, 'properties': ['writeonly']}},       <-- 8-bit reset register, writeonly
                  'LOST_DATA_COUNTER': {'descr': {'addr': 0, 'size': 8, 'properties': ['ro']}},  <-- 8-bit data register, 'ro' equivalent to 'readonly'
                  'ENABLE': {'descr': {'addr': 1, 'size': 1, 'offset': 0}},                      <-- 1-bit register
                  'ENABLE_EXTERN': {'descr': {'addr': 1, 'size': 1, 'offset': 1}},               <-- 1-bit register
                  'EN_ARMING': {'descr': {'addr': 1, 'size': 1, 'offset': 2}},                   <-- 1-bit register
                  'EN_WRITE_TIMESTAMP': {'descr': {'addr': 1, 'size': 1, 'offset': 3}},          <-- 1-bit register
                  'EVENT_COUNTER': {'descr': {'addr': 2, 'size': 32, 'properties': ['ro']}}      <-- 32-bit register, 'ro' equivalent to 'readonly'
    _require_version = '==3'  <-- or use '<=', '>=', ... accordingly
    '''
    _registers = {}
    _require_version = None

    def __init__(self, intf, conf):
        super(RegisterHardwareLayer, self).__init__(intf, conf)
        # require interface and base address
        self._intf = intf
        self._base_addr = conf['base_addr']
        for reg in self._registers.iterkeys():
            self.add_property(reg)
            dict.__setitem__(self, reg, None)  # set values, but not writing to the interface

    def init(self):
        # reset on initialization
#         self.RESET = 0
        if 'VERSION' in self._registers:
            version = str(self.VERSION)
        else:
            version = None
        logging.debug("Initializing %s from module %s (Version: %s)" % (self.__class__.__name__, self.__class__.__module__, version if 'VERSION' in self._registers else 'n/a'))
        if self._require_version and not eval(version + self._require_version):
            raise Exception("FPGA module %s does not satisfy version requirements (read: %s, require: %s)" % (self.__class__.__module__, version, self._require_version.strip()))
        for reg, value in self._registers.iteritems():
            if reg in self._init:
                self[reg] = self._init[reg]
            elif 'default' in value and not ('properties' in value['descr'] and [i for i in read_only if i in value['descr']['properties']]):
                self[reg] = value['default']
            else:  # do nothing here, no value to write
                pass
#                 self[reg] = None

    def set_value(self, value, addr, size, offset, **kwargs):
        '''Writing a value of any arbitrary size (max. unsigned int 64) and offset to a register

        Parameters
        ----------
        value : int, str
            The register value (int, long, bit string) to be written.
        addr : int
            The register address.
        size : int
            Bit size/length of the value to be written to the register.
        offset : int
            Offset of the value to be written to the register (in number of bits).

        Returns
        -------
        nothing
        '''
        div, mod = divmod(size + offset, 8)
        if mod:
            div += 1
        ret = self._intf.read(self._base_addr + addr, size=div)
        reg = BitLogic()
        reg.frombytes(ret.tostring())
        reg[size + offset - 1:offset] = value
        self._intf.write(self._base_addr + addr, data=array.array('B', reg.tobytes()))

    def get_value(self, addr, size, offset, **kwargs):
        '''Reading a value of any arbitrary size (max. unsigned int 64) and offset from a register

        Parameters
        ----------
        addr : int
            The register address.
        size : int
            Bit size/length of the value.
        offset : int
            Offset of the value to be written to the register (in number of bits).

        Returns
        -------
        reg : int
            Register value.
        '''
        div, mod = divmod(size + offset, 8)
        if mod:
            div += 1
        ret = self._intf.read(self._base_addr + addr, size=div)
        reg = BitLogic()
        reg.frombytes(ret.tostring())
        return reg[size + offset - 1:offset].tovalue()

    def set_bytes(self, data, addr, **kwargs):
        '''Writing bytes of any arbitrary size

        Parameters
        ----------
        data : iterable
            The data (byte array) to be written.
        addr : int
            The register address.

        Returns
        -------
        nothing
        '''
        self._intf.write(self._conf['base_addr'] + addr, data)

    def get_bytes(self, addr, size, **kwargs):
        '''Reading bytes of any arbitrary size

        Parameters
        ----------.
        addr : int
            The register address.
        size : int
            Byte length of the value.

        Returns
        -------
        data : iterable
            Byte array.
        '''
        return self._intf.read(self._conf['base_addr'] + addr, size)

    def set_configuration(self, conf):
        if conf:
            for reg, value in conf.iteritems():
                self[reg] = value

    def get_configuration(self):
        conf = {}
        for reg in self._registers.iterkeys():
            descr = self._registers[reg]['descr']
            if not ('properties' in descr and [i for i in write_only if i in descr['properties']]) and not ('properties' in descr and [i for i in read_only if i in descr['properties']]):
                conf[reg] = self[reg]
        return conf

    def add_property(self, attribute):
        # create local setter and getter with a particular attribute name
#         getter = lambda self: self._get(attribute)
#         setter = lambda self, value: self._set(attribute, value)
        # Workaround: obviously dynamic properties catch exceptions
        # Print error message and return None

        def getter(self):
            try:
                return self._get(attribute)
            except Exception, e:
                logging.error(e)
                return None

        def setter(self, value):
            try:
                return self._set(attribute, value)
            except Exception, e:
                logging.error(e)
                return None
        # construct property attribute and add it to the class
        setattr(self.__class__, attribute, property(fget=getter, fset=setter, doc=attribute + ' register'))

    def set_default(self):
        for reg, value in self._registers.iteritems():
            if 'default' in value and not ('properties' in value['descr'] and [i for i in read_only if i in self._registers[reg]['descr']['properties']]):
                self._set(reg, value['default'])

    def _get(self, reg):
        descr = deepcopy(self._registers[reg]['descr'])
        if 'properties' in descr and [i for i in write_only if i in descr['properties']]:
            # allows a lazy-style of programming
            if 'default' in self._registers[reg]:
                self._set(reg, self._registers[reg]['default'])
            else:
                self._set(reg, 0)
            # raise error when doing read on write-only register
#             raise IOError('Register is write-only')
            # return None to prevent misuse
            return None
        else:
            if 'properties' in descr and [i for i in is_byte_array if i in descr['properties']]:
                ret_val = self.get_bytes(**descr)
                ret_val = array.array('B', ret_val).tolist()
#                 curr_val = dict.__getitem__(self, reg)
            else:
                descr.setdefault('offset', 0)
                ret_val = self.get_value(**descr)
                curr_val = dict.__getitem__(self, reg)
#                 curr_val = self.setdefault(reg, None)
                if curr_val is not None and 'properties' in descr and not [i for i in read_only if i in descr['properties']] and curr_val != ret_val:
                    raise ValueError('Read value was not expected: read: %s, expected: %s' % (str(ret_val), str(curr_val)))
            return ret_val

    def _set(self, reg, value):
        descr = deepcopy(self._registers[reg]['descr'])
        if 'properties' in descr and [i for i in read_only if i in descr['properties']]:
            raise IOError('Register is read-only')
        if 'properties' in descr and [i for i in is_byte_array if i in descr['properties']]:
            if not isinstance(value, collections.Iterable):
                raise ValueError('For array byte_register iterable object is needed')
            value = array.array('B', value).tolist()
            self.set_bytes(value, **descr)
            dict.__setitem__(self, reg, value)
        else:
            descr.setdefault('offset', 0)
            try:
                self.set_value(value, **descr)
            except ValueError:
                raise
            else:
                dict.__setitem__(self, reg, value if isinstance(value, (int, long)) else int(value, base=2))

    def __getitem__(self, name):
        return self._get(name)

    def __setitem__(self, name, value):
        return self._set(name, value)

    def __getattr__(self, name):
        '''called only on last resort if there are no attributes in the instance that match the name
        '''
        if name.isupper() and name not in self._registers:
            raise ValueError('%s register does not exist in %s' % (name, self.__class__))

        def method(*args, **kwargs):
            nsplit = name.split('_', 1)
            if len(nsplit) == 2 and nsplit[0] == 'set' and len(args) == 1 and not kwargs:
                self[nsplit[1]] = args[0]  # returns None
            elif len(nsplit) == 2 and nsplit[0] == 'get' and not args and not kwargs:
                return self[nsplit[1]]
            else:
                raise AttributeError("%r object has no attribute %r" % (self.__class__, name))
        return method

    def __setattr__(self, name, value):
        if name.isupper() and name not in self._registers:
            raise ValueError('%s register does not exist in %s' % (name, self.__class__))
        return dict.__setattr__(self, name, value)
<|endoftext|>


<commit_before>"""Base Widget class.  Allows user to create widgets in the backend that render
in the IPython notebook frontend.
"""
#-----------------------------------------------------------------------------
# Copyright (c) 2013, the IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
from copy import copy
from glob import glob
import uuid
import sys
import os
import inspect
import types

import IPython
from IPython.kernel.comm import Comm
from IPython.config import LoggingConfigurable
from IPython.utils.traitlets import Unicode, Dict, List, Instance, Bool
from IPython.display import Javascript, display
from IPython.utils.py3compat import string_types

#-----------------------------------------------------------------------------
# Classes
#-----------------------------------------------------------------------------

class Widget(LoggingConfigurable):

    # Shared declarations (Class level)
    widget_construction_callback = None

    keys = ['view_name']

    def on_widget_constructed(callback):
        """Class method, registers a callback to be called when a widget is
        constructed.  The callback must have the following signature:
        callback(widget)"""
        Widget.widget_construction_callback = callback

    def _call_widget_constructed(widget):
        """Class method, called when a widget is constructed."""
        if Widget.widget_construction_callback is not None and callable(Widget.widget_construction_callback):
            Widget.widget_construction_callback(widget)

    # Public declarations (Instance level)
    target_name = Unicode('widget', help="""Name of the backbone model
        registered in the frontend to create and sync this widget with.""")
    view_name = Unicode(help="""Default view registered in the frontend
        to use to represent the widget.""")

    # Private/protected declarations
    # todo: change this to a context manager
    _property_lock = (None, None) # Last updated (key, value) from the front-end.  Prevents echo.
    _displayed = False
    _comm = Instance('IPython.kernel.comm.Comm')

    def __init__(self, **kwargs):
        """Public constructor
        """
        self._display_callbacks = []
        self._msg_callbacks = []
        super(Widget, self).__init__(**kwargs)

        self.on_trait_change(self._handle_property_changed, self.keys)
        Widget._call_widget_constructed(self)

    def __del__(self):
        """Object disposal"""
        self.close()

    def close(self):
        """Close method.  Closes the widget which closes the underlying comm.
        When the comm is closed, all of the widget views are automatically
        removed from the frontend."""
        self._close_communication()

    @property
    def comm(self):
        if self._comm is None:
            self._open_communication()
        return self._comm

    @property
    def model_id(self):
        return self.comm.comm_id

    # Event handlers
    def _handle_msg(self, msg):
        """Called when a msg is recieved from the frontend"""
        data = msg['content']['data']
        method = data['method']

        # Handle backbone sync methods CREATE, PATCH, and UPDATE all in one.
        if method == 'backbone' and 'sync_data' in data:
            sync_data = data['sync_data']
            self._handle_recieve_state(sync_data) # handles all methods

        # Handle a custom msg from the front-end
        elif method == 'custom':
            if 'custom_content' in data:
                self._handle_custom_msg(data['custom_content'])

    def _handle_custom_msg(self, content):
        """Called when a custom msg is recieved."""
        for handler in self._msg_callbacks:
            if callable(handler):
                argspec = inspect.getargspec(handler)
                nargs = len(argspec[0])

                # Bound methods have an additional 'self' argument
                if isinstance(handler, types.MethodType):
                    nargs -= 1

                # Call the callback
                if nargs == 1:
                    handler(content)
                elif nargs == 2:
                    handler(self, content)
                else:
                    raise TypeError('Widget msg callback must ' \
                        'accept 1 or 2 arguments, not %d.' % nargs)

    def _handle_recieve_state(self, sync_data):
        """Called when a state is recieved from the frontend."""
        for name in self.keys:
            if name in sync_data:
                try:
                    self._property_lock = (name, sync_data[name])
                    setattr(self, name, sync_data[name])
                finally:
                    self._property_lock = (None, None)

    def _handle_property_changed(self, name, old, new):
        """Called when a property has been changed."""
        # Make sure this isn't information that the front-end just sent us.
        if self._property_lock[0] != name and self._property_lock[1] != new:
            # Send new state to frontend
            self.send_state(key=name)

    def _handle_displayed(self, **kwargs):
        """Called when a view has been displayed for this widget instance

        Parameters
        ----------
        [view_name]: unicode (optional kwarg)
            Name of the view that was displayed."""
        for handler in self._display_callbacks:
            if callable(handler):
                argspec = inspect.getargspec(handler)
                nargs = len(argspec[0])

                # Bound methods have an additional 'self' argument
                if isinstance(handler, types.MethodType):
                    nargs -= 1

                # Call the callback
                if nargs == 0:
                    handler()
                elif nargs == 1:
                    handler(self)
                elif nargs == 2:
                    handler(self, kwargs.get('view_name', None))
                else:
                    handler(self, **kwargs)

    # Public methods
    def send_state(self, key=None):
        """Sends the widget state, or a piece of it, to the frontend.

        Parameters
        ----------
        key : unicode (optional)
            A single property's name to sync with the frontend.
        """
        self._send({"method": "update",
            "state": self.get_state()})

    def get_state(self, key=None):
        """Gets the widget state, or a piece of it.

        Parameters
        ----------
        key : unicode (optional)
            A single property's name to get.
        """
        state = {}

        # If a key is provided, just send the state of that key.
        if key is None:
            keys = self.keys[:]
        else:
            keys = [key]
        for k in keys:
            value = getattr(self, k)

            # a more elegant solution to encoding Widgets would be
            # to tap into the JSON encoder and teach it how to deal
            # with Widget objects, or maybe just teach the JSON
            # encoder to look for a _repr_json property before giving
            # up encoding
            if isinstance(value, Widget):
                value = value.model_id
            elif isinstance(value, list) and len(value)>0 and isinstance(value[0], Widget):
                # assume all elements of the list are widgets
                value = [i.model_id for i in value]
            state[k] = value
        return state

    def send(self, content):
        """Sends a custom msg to the widget model in the front-end.

        Parameters
        ----------
        content : dict
            Content of the message to send.
        """
        self._send({"method": "custom",
            "custom_content": content})

    def on_msg(self, callback, remove=False):
        """Register a callback for when a custom msg is recieved from the front-end

        Parameters
        ----------
        callback: method handler
            Can have a signature of:
            - callback(content)
            - callback(sender, content)
        remove: bool
            True if the callback should be unregistered."""
        if remove and callback in self._msg_callbacks:
            self._msg_callbacks.remove(callback)
        elif not remove and not callback in self._msg_callbacks:
            self._msg_callbacks.append(callback)

    def on_displayed(self, callback, remove=False):
        """Register a callback to be called when the widget has been displayed

        Parameters
        ----------
        callback: method handler
            Can have a signature of:
            - callback()
            - callback(sender)
            - callback(sender, view_name)
            - callback(sender, **kwargs)
              kwargs from display call passed through without modification.
        remove: bool
            True if the callback should be unregistered."""
        if remove and callback in self._display_callbacks:
            self._display_callbacks.remove(callback)
        elif not remove and not callback in self._display_callbacks:
            self._display_callbacks.append(callback)

    # Support methods
    def _repr_widget_(self, **kwargs):
        """Function that is called when `IPython.display.display` is called on
        the widget.

        Parameters
        ----------
        view_name: unicode (optional)
            View to display in the frontend.  Overrides view_name."""

        # Show view.
        self._send({"method": "display", "view_name": view_name})
        self._displayed = True
        self._handle_displayed(**kwargs)

    def _open_communication(self):
        """Opens a communication with the front-end."""
        # Create a comm.
        if self._comm is None:
            self._comm = Comm(target_name=self.target_name)
            self._comm.on_msg(self._handle_msg)
            self._comm.on_close(self._close_communication)

        # first update
        self.send_state()

    def _close_communication(self):
        """Closes a communication with the front-end."""
        if self._comm is not None:
            try:
                self._comm.close()
            finally:
                self._comm = None

    def _send(self, msg):
        """Sends a message to the model in the front-end"""
        if self._comm is not None:
            self._comm.send(msg)
            return True
        else:
            return False

class DOMWidget(Widget):
    visible = Bool(True, help="Whether or not the widget is visible.")

    # Private/protected declarations
    _css = Dict() # Internal CSS property dict

    keys = ['visible', '_css'] + Widget.keys

    def get_css(self, key, selector=""):
        """Get a CSS property of the widget.  Note, this function does not
        actually request the CSS from the front-end;  Only properties that have
        been set with set_css can be read.

        Parameters
        ----------
        key: unicode
            CSS key
        selector: unicode (optional)
            JQuery selector used when the CSS key/value was set.
        """
        if selector in self._css and key in self._css[selector]:
            return self._css[selector][key]
        else:
            return None

    def set_css(self, *args, **kwargs):
        """Set one or more CSS properties of the widget (shared among all of the
        views).  This function has two signatures:
        - set_css(css_dict, [selector=''])
        - set_css(key, value, [selector=''])

        Parameters
        ----------
        css_dict : dict
            CSS key/value pairs to apply
        key: unicode
            CSS key
        value
            CSS value
        selector: unicode (optional)
            JQuery selector to use to apply the CSS key/value.
        """
        selector = kwargs.get('selector', '')

        # Signature 1: set_css(css_dict, [selector=''])
        if len(args) == 1:
            if isinstance(args[0], dict):
                for (key, value) in args[0].items():
                    self.set_css(key, value, selector=selector)
            else:
                raise Exception('css_dict must be a dict.')

        # Signature 2: set_css(key, value, [selector=''])
        elif len(args) == 2 or len(args) == 3:

            # Selector can be a positional arg if it's the 3rd value
            if len(args) == 3:
                selector = args[2]
            if selector not in self._css:
                self._css[selector] = {}

            # Only update the property if it has changed.
            key = args[0]
            value = args[1]
            if not (key in self._css[selector] and value in self._css[selector][key]):
                self._css[selector][key] = value
                self.send_state('_css') # Send new state to client.
        else:
            raise Exception('set_css only accepts 1-3 arguments')

    def add_class(self, class_names, selector=""):
        """Add class[es] to a DOM element

        Parameters
        ----------
        class_names: unicode or list
            Class name(s) to add to the DOM element(s).
        selector: unicode (optional)
            JQuery selector to select the DOM element(s) that the class(es) will
            be added to.
        """
        class_list = class_names
        if isinstance(list, class_list):
            class_list = ' '.join(class_list)

        self.send({"msg_type": "add_class",
            "class_list": class_list,
            "selector": selector})

    def remove_class(self, class_names, selector=""):
        """Remove class[es] from a DOM element

        Parameters
        ----------
        class_names: unicode or list
            Class name(s) to remove from  the DOM element(s).
        selector: unicode (optional)
            JQuery selector to select the DOM element(s) that the class(es) will
            be removed from.
        """
        class_list = class_names
        if isinstance(list, class_list):
            class_list = ' '.join(class_list)

        self.send({"msg_type": "remove_class",
            "class_list": class_list,
            "selector": selector})
<commit_msg>Fixed _send so it can open a comm if needed. It no longer returns anything.<commit_after>"""Base Widget class.  Allows user to create widgets in the backend that render
in the IPython notebook frontend.
"""
#-----------------------------------------------------------------------------
# Copyright (c) 2013, the IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------
from copy import copy
from glob import glob
import uuid
import sys
import os
import inspect
import types

import IPython
from IPython.kernel.comm import Comm
from IPython.config import LoggingConfigurable
from IPython.utils.traitlets import Unicode, Dict, List, Instance, Bool
from IPython.display import Javascript, display
from IPython.utils.py3compat import string_types

#-----------------------------------------------------------------------------
# Classes
#-----------------------------------------------------------------------------

class Widget(LoggingConfigurable):

    # Shared declarations (Class level)
    widget_construction_callback = None

    keys = ['view_name']

    def on_widget_constructed(callback):
        """Class method, registers a callback to be called when a widget is
        constructed.  The callback must have the following signature:
        callback(widget)"""
        Widget.widget_construction_callback = callback

    def _call_widget_constructed(widget):
        """Class method, called when a widget is constructed."""
        if Widget.widget_construction_callback is not None and callable(Widget.widget_construction_callback):
            Widget.widget_construction_callback(widget)

    # Public declarations (Instance level)
    target_name = Unicode('widget', help="""Name of the backbone model
        registered in the frontend to create and sync this widget with.""")
    view_name = Unicode(help="""Default view registered in the frontend
        to use to represent the widget.""")

    # Private/protected declarations
    # todo: change this to a context manager
    _property_lock = (None, None) # Last updated (key, value) from the front-end.  Prevents echo.
    _displayed = False
    _comm = Instance('IPython.kernel.comm.Comm')

    def __init__(self, **kwargs):
        """Public constructor
        """
        self._display_callbacks = []
        self._msg_callbacks = []
        super(Widget, self).__init__(**kwargs)

        self.on_trait_change(self._handle_property_changed, self.keys)
        Widget._call_widget_constructed(self)

    def __del__(self):
        """Object disposal"""
        self.close()

    def close(self):
        """Close method.  Closes the widget which closes the underlying comm.
        When the comm is closed, all of the widget views are automatically
        removed from the frontend."""
        self._close_communication()

    @property
    def comm(self):
        if self._comm is None:
            self._open_communication()
        return self._comm

    @property
    def model_id(self):
        return self.comm.comm_id

    # Event handlers
    def _handle_msg(self, msg):
        """Called when a msg is recieved from the frontend"""
        data = msg['content']['data']
        method = data['method']

        # Handle backbone sync methods CREATE, PATCH, and UPDATE all in one.
        if method == 'backbone' and 'sync_data' in data:
            sync_data = data['sync_data']
            self._handle_recieve_state(sync_data) # handles all methods

        # Handle a custom msg from the front-end
        elif method == 'custom':
            if 'custom_content' in data:
                self._handle_custom_msg(data['custom_content'])

    def _handle_custom_msg(self, content):
        """Called when a custom msg is recieved."""
        for handler in self._msg_callbacks:
            if callable(handler):
                argspec = inspect.getargspec(handler)
                nargs = len(argspec[0])

                # Bound methods have an additional 'self' argument
                if isinstance(handler, types.MethodType):
                    nargs -= 1

                # Call the callback
                if nargs == 1:
                    handler(content)
                elif nargs == 2:
                    handler(self, content)
                else:
                    raise TypeError('Widget msg callback must ' \
                        'accept 1 or 2 arguments, not %d.' % nargs)

    def _handle_recieve_state(self, sync_data):
        """Called when a state is recieved from the frontend."""
        for name in self.keys:
            if name in sync_data:
                try:
                    self._property_lock = (name, sync_data[name])
                    setattr(self, name, sync_data[name])
                finally:
                    self._property_lock = (None, None)

    def _handle_property_changed(self, name, old, new):
        """Called when a property has been changed."""
        # Make sure this isn't information that the front-end just sent us.
        if self._property_lock[0] != name and self._property_lock[1] != new:
            # Send new state to frontend
            self.send_state(key=name)

    def _handle_displayed(self, **kwargs):
        """Called when a view has been displayed for this widget instance

        Parameters
        ----------
        [view_name]: unicode (optional kwarg)
            Name of the view that was displayed."""
        for handler in self._display_callbacks:
            if callable(handler):
                argspec = inspect.getargspec(handler)
                nargs = len(argspec[0])

                # Bound methods have an additional 'self' argument
                if isinstance(handler, types.MethodType):
                    nargs -= 1

                # Call the callback
                if nargs == 0:
                    handler()
                elif nargs == 1:
                    handler(self)
                elif nargs == 2:
                    handler(self, kwargs.get('view_name', None))
                else:
                    handler(self, **kwargs)

    # Public methods
    def send_state(self, key=None):
        """Sends the widget state, or a piece of it, to the frontend.

        Parameters
        ----------
        key : unicode (optional)
            A single property's name to sync with the frontend.
        """
        self._send({"method": "update",
            "state": self.get_state()})

    def get_state(self, key=None):
        """Gets the widget state, or a piece of it.

        Parameters
        ----------
        key : unicode (optional)
            A single property's name to get.
        """
        state = {}

        # If a key is provided, just send the state of that key.
        if key is None:
            keys = self.keys[:]
        else:
            keys = [key]
        for k in keys:
            value = getattr(self, k)

            # a more elegant solution to encoding Widgets would be
            # to tap into the JSON encoder and teach it how to deal
            # with Widget objects, or maybe just teach the JSON
            # encoder to look for a _repr_json property before giving
            # up encoding
            if isinstance(value, Widget):
                value = value.model_id
            elif isinstance(value, list) and len(value)>0 and isinstance(value[0], Widget):
                # assume all elements of the list are widgets
                value = [i.model_id for i in value]
            state[k] = value
        return state

    def send(self, content):
        """Sends a custom msg to the widget model in the front-end.

        Parameters
        ----------
        content : dict
            Content of the message to send.
        """
        self._send({"method": "custom",
            "custom_content": content})

    def on_msg(self, callback, remove=False):
        """Register a callback for when a custom msg is recieved from the front-end

        Parameters
        ----------
        callback: method handler
            Can have a signature of:
            - callback(content)
            - callback(sender, content)
        remove: bool
            True if the callback should be unregistered."""
        if remove and callback in self._msg_callbacks:
            self._msg_callbacks.remove(callback)
        elif not remove and not callback in self._msg_callbacks:
            self._msg_callbacks.append(callback)

    def on_displayed(self, callback, remove=False):
        """Register a callback to be called when the widget has been displayed

        Parameters
        ----------
        callback: method handler
            Can have a signature of:
            - callback()
            - callback(sender)
            - callback(sender, view_name)
            - callback(sender, **kwargs)
              kwargs from display call passed through without modification.
        remove: bool
            True if the callback should be unregistered."""
        if remove and callback in self._display_callbacks:
            self._display_callbacks.remove(callback)
        elif not remove and not callback in self._display_callbacks:
            self._display_callbacks.append(callback)

    # Support methods
    def _repr_widget_(self, **kwargs):
        """Function that is called when `IPython.display.display` is called on
        the widget.

        Parameters
        ----------
        view_name: unicode (optional)
            View to display in the frontend.  Overrides view_name."""

        # Show view.  By sending a display message, the comm is opened and the
        # initial state is sent.
        self._send({"method": "display", "view_name": view_name})
        self._displayed = True
        self._handle_displayed(**kwargs)

    def _open_communication(self):
        """Opens a communication with the front-end."""
        # Create a comm.
        if self._comm is None:
            self._comm = Comm(target_name=self.target_name)
            self._comm.on_msg(self._handle_msg)
            self._comm.on_close(self._close_communication)

        # first update
        self.send_state()

    def _close_communication(self):
        """Closes a communication with the front-end."""
        if self._comm is not None:
            try:
                self._comm.close()
            finally:
                self._comm = None

    def _send(self, msg):
        """Sends a message to the model in the front-end"""
        self.comm.send(msg)

class DOMWidget(Widget):
    visible = Bool(True, help="Whether or not the widget is visible.")

    # Private/protected declarations
    _css = Dict() # Internal CSS property dict

    keys = ['visible', '_css'] + Widget.keys

    def get_css(self, key, selector=""):
        """Get a CSS property of the widget.  Note, this function does not
        actually request the CSS from the front-end;  Only properties that have
        been set with set_css can be read.

        Parameters
        ----------
        key: unicode
            CSS key
        selector: unicode (optional)
            JQuery selector used when the CSS key/value was set.
        """
        if selector in self._css and key in self._css[selector]:
            return self._css[selector][key]
        else:
            return None

    def set_css(self, *args, **kwargs):
        """Set one or more CSS properties of the widget (shared among all of the
        views).  This function has two signatures:
        - set_css(css_dict, [selector=''])
        - set_css(key, value, [selector=''])

        Parameters
        ----------
        css_dict : dict
            CSS key/value pairs to apply
        key: unicode
            CSS key
        value
            CSS value
        selector: unicode (optional)
            JQuery selector to use to apply the CSS key/value.
        """
        selector = kwargs.get('selector', '')

        # Signature 1: set_css(css_dict, [selector=''])
        if len(args) == 1:
            if isinstance(args[0], dict):
                for (key, value) in args[0].items():
                    self.set_css(key, value, selector=selector)
            else:
                raise Exception('css_dict must be a dict.')

        # Signature 2: set_css(key, value, [selector=''])
        elif len(args) == 2 or len(args) == 3:

            # Selector can be a positional arg if it's the 3rd value
            if len(args) == 3:
                selector = args[2]
            if selector not in self._css:
                self._css[selector] = {}

            # Only update the property if it has changed.
            key = args[0]
            value = args[1]
            if not (key in self._css[selector] and value in self._css[selector][key]):
                self._css[selector][key] = value
                self.send_state('_css') # Send new state to client.
        else:
            raise Exception('set_css only accepts 1-3 arguments')

    def add_class(self, class_names, selector=""):
        """Add class[es] to a DOM element

        Parameters
        ----------
        class_names: unicode or list
            Class name(s) to add to the DOM element(s).
        selector: unicode (optional)
            JQuery selector to select the DOM element(s) that the class(es) will
            be added to.
        """
        class_list = class_names
        if isinstance(list, class_list):
            class_list = ' '.join(class_list)

        self.send({"msg_type": "add_class",
            "class_list": class_list,
            "selector": selector})

    def remove_class(self, class_names, selector=""):
        """Remove class[es] from a DOM element

        Parameters
        ----------
        class_names: unicode or list
            Class name(s) to remove from  the DOM element(s).
        selector: unicode (optional)
            JQuery selector to select the DOM element(s) that the class(es) will
            be removed from.
        """
        class_list = class_names
        if isinstance(list, class_list):
            class_list = ' '.join(class_list)

        self.send({"msg_type": "remove_class",
            "class_list": class_list,
            "selector": selector})
<|endoftext|>


<commit_before>
import socket
socket.setdefaulttimeout(2)

from apps.rss_feeds.models import Feed, Story, FeedUpdateHistory
# from apps.rss_feeds.models import FeedXML
from django.core.cache import cache
from apps.reader.models import UserSubscription
from apps.rss_feeds.importer import PageImporter
from utils import feedparser
from django.db.models import Q
from django.db import IntegrityError
from utils.story_functions import pre_process_story
from utils import log as logging
import sys
import time
import datetime
import traceback
import multiprocessing
import urllib2
import xml.sax

# Refresh feed code adapted from Feedjack.
# http://feedjack.googlecode.com

VERSION = '0.4'
URL = 'http://www.newsblur.com/'
USER_AGENT = 'NewsBlur Fetcher %s - %s' % (VERSION, URL)
SLOWFEED_WARNING = 10
ENTRY_NEW, ENTRY_UPDATED, ENTRY_SAME, ENTRY_ERR = range(4)
FEED_OK, FEED_SAME, FEED_ERRPARSE, FEED_ERRHTTP, FEED_ERREXC = range(5)

socket.setdefaulttimeout(30)

def mtime(ttime):
    """ datetime auxiliar function.
    """
    return datetime.datetime.fromtimestamp(time.mktime(ttime))

class FetchFeed:
    def __init__(self, feed, options):
        self.feed = feed
        self.options = options
        self.fpf = None

    def fetch(self):
        """ Downloads and parses a feed.
        """
        identity = self.get_identity()
        log_msg = u'%2s ---> Fetching %s (%d)' % (identity,
                                                 self.feed.feed_title,
                                                 self.feed.id)
        logging.debug(log_msg)

        # Check if feed still needs to be updated
        feed = Feed.objects.get(pk=self.feed.pk)
        if feed.last_update > datetime.datetime.now() and not self.options.get('force'):
            log_msg = u'        ---> Already fetched %s (%d)' % (self.feed.feed_title,
                                                                 self.feed.id)
            logging.debug(log_msg)
            feed.save_feed_history(303, "Already fetched")
            return FEED_SAME, None

        modified = self.feed.last_modified.utctimetuple()[:7] if self.feed.last_modified else None
        self.fpf = feedparser.parse(self.feed.feed_address,
                                    agent=USER_AGENT,
                                    etag=self.feed.etag,
                                    modified=modified)

        return FEED_OK, self.fpf

    def get_identity(self):
        identity = "X"

        current_process = multiprocessing.current_process()
        if current_process._identity:
            identity = current_process._identity[0]

        return identity

class ProcessFeed:
    def __init__(self, feed, fpf, options):
        self.feed = feed
        self.options = options
        self.fpf = fpf
        self.lock = multiprocessing.Lock()

    def process(self):
        """ Downloads and parses a feed.
        """

        ret_values = {
            ENTRY_NEW:0,
            ENTRY_UPDATED:0,
            ENTRY_SAME:0,
            ENTRY_ERR:0}

        logging.debug(u' ---> [%d] Processing %s' % (self.feed.id,
                                               self.feed.feed_title))

        if hasattr(self.fpf, 'status'):
            if self.options['verbose']:
                logging.debug(u'[%d] HTTP status %d: %s' % (self.feed.id,
                                                     self.fpf.status,
                                                     self.feed.feed_address))
            if self.fpf.status == 304:
                # this means the feed has not changed
                if self.options['verbose']:
                    logging.debug('[%d] Feed has not changed since ' \
                           'last check: %s' % (self.feed.id,
                                               self.feed.feed_address))
                self.feed.save()
                self.feed.save_feed_history(304, "Not modified")
                return FEED_SAME, ret_values

            if self.fpf.status >= 400:
                # http error, ignore
                logging.debug('[%d] !HTTP_ERROR! %d: %s' % (self.feed.id,
                                                     self.fpf.status,
                                                     self.feed.feed_address))
                self.feed.save()
                self.feed.save_feed_history(self.fpf.status, "HTTP Error")
                return FEED_ERRHTTP, ret_values

        if self.fpf.bozo and isinstance(self.fpf.bozo_exception, feedparser.NonXMLContentType):
            logging.debug("   ---> Non-xml feed: %s." % self.feed)
            if not self.fpf.entries:
                self.feed.save_feed_history(502, 'Non-xml feed', self.fpf.bozo_exception)
                return FEED_ERRPARSE, ret_values
        elif self.fpf.bozo and isinstance(self.fpf.bozo_exception, xml.sax._exceptions.SAXException):
            if not self.fpf.entries:
                self.feed.save_feed_history(503, 'SAX Exception', self.fpf.bozo_exception)
                return FEED_ERRPARSE, ret_values

        # the feed has changed (or it is the first time we parse it)
        # saving the etag and last_modified fields
        self.feed.etag = self.fpf.get('etag')
        # some times this is None (it never should) *sigh*
        if self.feed.etag is None:
            self.feed.etag = ''

        try:
            self.feed.last_modified = mtime(self.fpf.modified)
        except:
            pass

        self.feed.feed_title = self.fpf.feed.get('title', self.feed.feed_title)
        self.feed.feed_tagline = self.fpf.feed.get('tagline', self.feed.feed_tagline)
        self.feed.feed_link = self.fpf.feed.get('link', self.feed.feed_link)
        self.feed.last_update = datetime.datetime.now()

        guids = []
        for entry in self.fpf.entries:
            if entry.get('id', ''):
                guids.append(entry.get('id', ''))
            elif entry.title:
                guids.append(entry.title)
            elif entry.link:
                guids.append(entry.link)

        self.lock.acquire()
        try:
            self.feed.save()
        finally:
            self.lock.release()

        # Compare new stories to existing stories, adding and updating
        start_date = datetime.datetime.now()
        end_date = datetime.datetime.now()
        story_guids = []
        for entry in self.fpf.entries:
            story = pre_process_story(entry)
            if story.get('published') < start_date:
                start_date = story.get('published')
            if story.get('published') > end_date:
                end_date = story.get('published')
            story_guids.append(story.get('guid') or story.get('link'))
        existing_stories = Story.objects.filter(
            story_date__gte=start_date,
            story_date__lte=end_date,
            story_feed=self.feed
        ).order_by('-story_date')
        ret_values = self.feed.add_update_stories(self.fpf.entries, existing_stories)

        self.feed.count_subscribers(lock=self.lock)
        self.feed.count_stories(lock=self.lock)
        self.feed.save_popular_authors(lock=self.lock)
        self.feed.save_popular_tags(lock=self.lock)
        self.feed.save_feed_history(200, "OK")

        return FEED_OK, ret_values

class Dispatcher:
    def __init__(self, options, num_threads):
        self.options = options
        self.entry_stats = {
            ENTRY_NEW:0,
            ENTRY_UPDATED:0,
            ENTRY_SAME:0,
            ENTRY_ERR:0}
        self.feed_stats = {
            FEED_OK:0,
            FEED_SAME:0,
            FEED_ERRPARSE:0,
            FEED_ERRHTTP:0,
            FEED_ERREXC:0}
        self.entry_trans = {
            ENTRY_NEW:'new',
            ENTRY_UPDATED:'updated',
            ENTRY_SAME:'same',
            ENTRY_ERR:'error'}
        self.feed_trans = {
            FEED_OK:'ok',
            FEED_SAME:'unchanged',
            FEED_ERRPARSE:'cant_parse',
            FEED_ERRHTTP:'http_error',
            FEED_ERREXC:'exception'}
        self.entry_keys = sorted(self.entry_trans.keys())
        self.feed_keys = sorted(self.feed_trans.keys())
        self.num_threads = num_threads
        self.time_start = datetime.datetime.now()
        self.workers = []

    def process_feed_wrapper(self, feed_queue):
        """ wrapper for ProcessFeed
        """
        # Close the DB so the connection can be re-opened on a per-process basis
        from django.db import connection
        connection.close()

        current_process = multiprocessing.current_process()

        identity = "X"
        if current_process._identity:
            identity = current_process._identity[0]
        for feed in feed_queue:
            ret_entries = {
                ENTRY_NEW: 0,
                ENTRY_UPDATED: 0,
                ENTRY_SAME: 0,
                ENTRY_ERR: 0
            }
            start_time = datetime.datetime.now()

            feed.set_next_scheduled_update()

            ### Uncomment to test feed fetcher
            # from random import randint
            # if randint(0,10) < 10:
            #     continue

            try:
                ffeed = FetchFeed(feed, self.options)
                ret_feed, fetched_feed = ffeed.fetch()

                delta = datetime.datetime.now() - start_time

                if fetched_feed and ret_feed == FEED_OK:
                    pfeed = ProcessFeed(feed, fetched_feed, self.options)
                    ret_feed, ret_entries = pfeed.process()

                    if ret_entries.get(ENTRY_NEW):
                        user_subs = UserSubscription.objects.filter(feed=feed)
                        for sub in user_subs:
                            cache.delete('usersub:%s' % sub.user_id)
                            sub.calculate_feed_scores(silent=True)
                    if ret_entries.get(ENTRY_NEW) or ret_entries.get(ENTRY_UPDATED):
                        feed.get_stories(force=True)

                if (fetched_feed and
                    feed.feed_link and
                    (ret_feed == FEED_OK or
                     (ret_feed == FEED_SAME and feed.stories_last_month > 10))):
                    page_importer = PageImporter(feed.feed_link, feed)
                    page_importer.fetch_page()
            except KeyboardInterrupt:
                break
            except urllib2.HTTPError, e:
                feed.save_feed_history(e.code, e.msg, e.fp.read())
            except Exception, e:
                logging.debug('[%d] ! -------------------------' % (feed.id,))
                tb = traceback.format_exc()
                logging.debug(tb)
                logging.debug('[%d] ! -------------------------' % (feed.id,))
                ret_feed = FEED_ERREXC
                feed.save_feed_history(500, "Error", tb)

            if not delta:
                delta = datetime.datetime.now() - start_time
            if delta.seconds > SLOWFEED_WARNING:
                comment = u' (SLOW FEED!)'
            else:
                comment = u''

            feed.last_load_time = max(1, delta.seconds)
            feed.fetched_once = True
            try:
                feed.save()
            except IntegrityError:
                logging.debug(" ---> IntegrityError on feed: %s - %s" % (feed, feed.feed_address,))

            done_msg = (u'%2s ---> Processed %s (%d) in %s\n        ---> [%s] [%s]%s' % (
                identity, feed.feed_title, feed.id, unicode(delta),
                u' '.join(u'%s=%d' % (self.entry_trans[key],
                          ret_entries[key]) for key in self.entry_keys),
                self.feed_trans[ret_feed],
                comment))
            logging.debug(done_msg)

            self.feed_stats[ret_feed] += 1
            for key, val in ret_entries.items():
                self.entry_stats[key] += val

        time_taken = datetime.datetime.now() - self.time_start
        history = FeedUpdateHistory(
            number_of_feeds=len(feed_queue),
            seconds_taken=time_taken.seconds
        )
        history.save()
        if not self.options['single_threaded']:
            logging.debug("---> DONE WITH PROCESS: %s" % current_process.name)
            sys.exit()

    def add_jobs(self, feeds_queue, feeds_count=1):
        """ adds a feed processing job to the pool
        """
        self.feeds_queue = feeds_queue
        self.feeds_count = feeds_count

    def run_jobs(self):
        if self.options['single_threaded']:
            self.process_feed_wrapper(self.feeds_queue[0])
        else:
            for i in range(self.num_threads):
                feed_queue = self.feeds_queue[i]
                self.workers.append(multiprocessing.Process(target=self.process_feed_wrapper,
                                                            args=(feed_queue,)))
                # worker.setName("Thread #%s" % (i+1))
                # worker.setDaemon(True)
            for i in range(self.num_threads):
                self.workers[i].start()

    def poll(self):
        """ polls the active threads
        """
        if not self.options['single_threaded']:
            for i in range(self.num_threads):
                self.workers[i].join()
            done = (u'* DONE in %s\n* Feeds: %s\n* Entries: %s' % (
                    unicode(datetime.datetime.now() - self.time_start),
                    u' '.join(u'%s=%d' % (self.feed_trans[key],
                              self.feed_stats[key])
                              for key in self.feed_keys),
                    u' '.join(u'%s=%d' % (self.entry_trans[key],
                              self.entry_stats[key])
                              for key in self.entry_keys)
                    ))
            logging.debug(done)
            return

                <commit_msg>Backwards. Just use the GUIDs of stories. This may lead to dupe-stories when the GUID changes. Ugh.<commit_after>
import socket
socket.setdefaulttimeout(2)

from apps.rss_feeds.models import Feed, Story, FeedUpdateHistory
# from apps.rss_feeds.models import FeedXML
from django.core.cache import cache
from apps.reader.models import UserSubscription
from apps.rss_feeds.importer import PageImporter
from utils import feedparser
from django.db.models import Q
from django.db import IntegrityError
from utils.story_functions import pre_process_story
from utils import log as logging
import sys
import time
import datetime
import traceback
import multiprocessing
import urllib2
import xml.sax

# Refresh feed code adapted from Feedjack.
# http://feedjack.googlecode.com

VERSION = '0.4'
URL = 'http://www.newsblur.com/'
USER_AGENT = 'NewsBlur Fetcher %s - %s' % (VERSION, URL)
SLOWFEED_WARNING = 10
ENTRY_NEW, ENTRY_UPDATED, ENTRY_SAME, ENTRY_ERR = range(4)
FEED_OK, FEED_SAME, FEED_ERRPARSE, FEED_ERRHTTP, FEED_ERREXC = range(5)

socket.setdefaulttimeout(30)

def mtime(ttime):
    """ datetime auxiliar function.
    """
    return datetime.datetime.fromtimestamp(time.mktime(ttime))

class FetchFeed:
    def __init__(self, feed, options):
        self.feed = feed
        self.options = options
        self.fpf = None

    def fetch(self):
        """ Downloads and parses a feed.
        """
        identity = self.get_identity()
        log_msg = u'%2s ---> Fetching %s (%d)' % (identity,
                                                 self.feed.feed_title,
                                                 self.feed.id)
        logging.debug(log_msg)

        # Check if feed still needs to be updated
        feed = Feed.objects.get(pk=self.feed.pk)
        if feed.last_update > datetime.datetime.now() and not self.options.get('force'):
            log_msg = u'        ---> Already fetched %s (%d)' % (self.feed.feed_title,
                                                                 self.feed.id)
            logging.debug(log_msg)
            feed.save_feed_history(303, "Already fetched")
            return FEED_SAME, None

        modified = self.feed.last_modified.utctimetuple()[:7] if self.feed.last_modified else None
        self.fpf = feedparser.parse(self.feed.feed_address,
                                    agent=USER_AGENT,
                                    etag=self.feed.etag,
                                    modified=modified)

        return FEED_OK, self.fpf

    def get_identity(self):
        identity = "X"

        current_process = multiprocessing.current_process()
        if current_process._identity:
            identity = current_process._identity[0]

        return identity

class ProcessFeed:
    def __init__(self, feed, fpf, options):
        self.feed = feed
        self.options = options
        self.fpf = fpf
        self.lock = multiprocessing.Lock()

    def process(self):
        """ Downloads and parses a feed.
        """

        ret_values = {
            ENTRY_NEW:0,
            ENTRY_UPDATED:0,
            ENTRY_SAME:0,
            ENTRY_ERR:0}

        logging.debug(u' ---> [%d] Processing %s' % (self.feed.id,
                                               self.feed.feed_title))

        if hasattr(self.fpf, 'status'):
            if self.options['verbose']:
                logging.debug(u'[%d] HTTP status %d: %s' % (self.feed.id,
                                                     self.fpf.status,
                                                     self.feed.feed_address))
            if self.fpf.status == 304:
                # this means the feed has not changed
                if self.options['verbose']:
                    logging.debug('[%d] Feed has not changed since ' \
                           'last check: %s' % (self.feed.id,
                                               self.feed.feed_address))
                self.feed.save()
                self.feed.save_feed_history(304, "Not modified")
                return FEED_SAME, ret_values

            if self.fpf.status >= 400:
                # http error, ignore
                logging.debug('[%d] !HTTP_ERROR! %d: %s' % (self.feed.id,
                                                     self.fpf.status,
                                                     self.feed.feed_address))
                self.feed.save()
                self.feed.save_feed_history(self.fpf.status, "HTTP Error")
                return FEED_ERRHTTP, ret_values

        if self.fpf.bozo and isinstance(self.fpf.bozo_exception, feedparser.NonXMLContentType):
            logging.debug("   ---> Non-xml feed: %s." % self.feed)
            if not self.fpf.entries:
                self.feed.save_feed_history(502, 'Non-xml feed', self.fpf.bozo_exception)
                return FEED_ERRPARSE, ret_values
        elif self.fpf.bozo and isinstance(self.fpf.bozo_exception, xml.sax._exceptions.SAXException):
            if not self.fpf.entries:
                self.feed.save_feed_history(503, 'SAX Exception', self.fpf.bozo_exception)
                return FEED_ERRPARSE, ret_values

        # the feed has changed (or it is the first time we parse it)
        # saving the etag and last_modified fields
        self.feed.etag = self.fpf.get('etag')
        # some times this is None (it never should) *sigh*
        if self.feed.etag is None:
            self.feed.etag = ''

        try:
            self.feed.last_modified = mtime(self.fpf.modified)
        except:
            pass

        self.feed.feed_title = self.fpf.feed.get('title', self.feed.feed_title)
        self.feed.feed_tagline = self.fpf.feed.get('tagline', self.feed.feed_tagline)
        self.feed.feed_link = self.fpf.feed.get('link', self.feed.feed_link)
        self.feed.last_update = datetime.datetime.now()

        guids = []
        for entry in self.fpf.entries:
            if entry.get('id', ''):
                guids.append(entry.get('id', ''))
            elif entry.title:
                guids.append(entry.title)
            elif entry.link:
                guids.append(entry.link)

        self.lock.acquire()
        try:
            self.feed.save()
        finally:
            self.lock.release()

        # Compare new stories to existing stories, adding and updating
        start_date = datetime.datetime.now()
        end_date = datetime.datetime.now()
        story_guids = []
        for entry in self.fpf.entries:
            story = pre_process_story(entry)
            if story.get('published') < start_date:
                start_date = story.get('published')
            if story.get('published') > end_date:
                end_date = story.get('published')
            story_guids.append(story.get('guid') or story.get('link'))
        existing_stories = Story.objects.filter(
            story_guid__in=story_guids,
            story_feed=self.feed
        ).order_by('-story_date')
        ret_values = self.feed.add_update_stories(self.fpf.entries, existing_stories)

        self.feed.count_subscribers(lock=self.lock)
        self.feed.count_stories(lock=self.lock)
        self.feed.save_popular_authors(lock=self.lock)
        self.feed.save_popular_tags(lock=self.lock)
        self.feed.save_feed_history(200, "OK")

        return FEED_OK, ret_values

class Dispatcher:
    def __init__(self, options, num_threads):
        self.options = options
        self.entry_stats = {
            ENTRY_NEW:0,
            ENTRY_UPDATED:0,
            ENTRY_SAME:0,
            ENTRY_ERR:0}
        self.feed_stats = {
            FEED_OK:0,
            FEED_SAME:0,
            FEED_ERRPARSE:0,
            FEED_ERRHTTP:0,
            FEED_ERREXC:0}
        self.entry_trans = {
            ENTRY_NEW:'new',
            ENTRY_UPDATED:'updated',
            ENTRY_SAME:'same',
            ENTRY_ERR:'error'}
        self.feed_trans = {
            FEED_OK:'ok',
            FEED_SAME:'unchanged',
            FEED_ERRPARSE:'cant_parse',
            FEED_ERRHTTP:'http_error',
            FEED_ERREXC:'exception'}
        self.entry_keys = sorted(self.entry_trans.keys())
        self.feed_keys = sorted(self.feed_trans.keys())
        self.num_threads = num_threads
        self.time_start = datetime.datetime.now()
        self.workers = []

    def process_feed_wrapper(self, feed_queue):
        """ wrapper for ProcessFeed
        """
        # Close the DB so the connection can be re-opened on a per-process basis
        from django.db import connection
        connection.close()

        current_process = multiprocessing.current_process()

        identity = "X"
        if current_process._identity:
            identity = current_process._identity[0]
        for feed in feed_queue:
            ret_entries = {
                ENTRY_NEW: 0,
                ENTRY_UPDATED: 0,
                ENTRY_SAME: 0,
                ENTRY_ERR: 0
            }
            start_time = datetime.datetime.now()

            feed.set_next_scheduled_update()

            ### Uncomment to test feed fetcher
            # from random import randint
            # if randint(0,10) < 10:
            #     continue

            try:
                ffeed = FetchFeed(feed, self.options)
                ret_feed, fetched_feed = ffeed.fetch()

                delta = datetime.datetime.now() - start_time

                if fetched_feed and ret_feed == FEED_OK:
                    pfeed = ProcessFeed(feed, fetched_feed, self.options)
                    ret_feed, ret_entries = pfeed.process()

                    if ret_entries.get(ENTRY_NEW):
                        user_subs = UserSubscription.objects.filter(feed=feed)
                        for sub in user_subs:
                            cache.delete('usersub:%s' % sub.user_id)
                            sub.calculate_feed_scores(silent=True)
                    if ret_entries.get(ENTRY_NEW) or ret_entries.get(ENTRY_UPDATED):
                        feed.get_stories(force=True)

                if (fetched_feed and
                    feed.feed_link and
                    (ret_feed == FEED_OK or
                     (ret_feed == FEED_SAME and feed.stories_last_month > 10))):
                    page_importer = PageImporter(feed.feed_link, feed)
                    page_importer.fetch_page()
            except KeyboardInterrupt:
                break
            except urllib2.HTTPError, e:
                feed.save_feed_history(e.code, e.msg, e.fp.read())
            except Exception, e:
                logging.debug('[%d] ! -------------------------' % (feed.id,))
                tb = traceback.format_exc()
                logging.debug(tb)
                logging.debug('[%d] ! -------------------------' % (feed.id,))
                ret_feed = FEED_ERREXC
                feed.save_feed_history(500, "Error", tb)

            if not delta:
                delta = datetime.datetime.now() - start_time
            if delta.seconds > SLOWFEED_WARNING:
                comment = u' (SLOW FEED!)'
            else:
                comment = u''

            feed.last_load_time = max(1, delta.seconds)
            feed.fetched_once = True
            try:
                feed.save()
            except IntegrityError:
                logging.debug(" ---> IntegrityError on feed: %s - %s" % (feed, feed.feed_address,))

            done_msg = (u'%2s ---> Processed %s (%d) in %s\n        ---> [%s] [%s]%s' % (
                identity, feed.feed_title, feed.id, unicode(delta),
                u' '.join(u'%s=%d' % (self.entry_trans[key],
                          ret_entries[key]) for key in self.entry_keys),
                self.feed_trans[ret_feed],
                comment))
            logging.debug(done_msg)

            self.feed_stats[ret_feed] += 1
            for key, val in ret_entries.items():
                self.entry_stats[key] += val

        time_taken = datetime.datetime.now() - self.time_start
        history = FeedUpdateHistory(
            number_of_feeds=len(feed_queue),
            seconds_taken=time_taken.seconds
        )
        history.save()
        if not self.options['single_threaded']:
            logging.debug("---> DONE WITH PROCESS: %s" % current_process.name)
            sys.exit()

    def add_jobs(self, feeds_queue, feeds_count=1):
        """ adds a feed processing job to the pool
        """
        self.feeds_queue = feeds_queue
        self.feeds_count = feeds_count

    def run_jobs(self):
        if self.options['single_threaded']:
            self.process_feed_wrapper(self.feeds_queue[0])
        else:
            for i in range(self.num_threads):
                feed_queue = self.feeds_queue[i]
                self.workers.append(multiprocessing.Process(target=self.process_feed_wrapper,
                                                            args=(feed_queue,)))
                # worker.setName("Thread #%s" % (i+1))
                # worker.setDaemon(True)
            for i in range(self.num_threads):
                self.workers[i].start()

    def poll(self):
        """ polls the active threads
        """
        if not self.options['single_threaded']:
            for i in range(self.num_threads):
                self.workers[i].join()
            done = (u'* DONE in %s\n* Feeds: %s\n* Entries: %s' % (
                    unicode(datetime.datetime.now() - self.time_start),
                    u' '.join(u'%s=%d' % (self.feed_trans[key],
                              self.feed_stats[key])
                              for key in self.feed_keys),
                    u' '.join(u'%s=%d' % (self.entry_trans[key],
                              self.entry_stats[key])
                              for key in self.entry_keys)
                    ))
            logging.debug(done)
            return

                <|endoftext|>


<commit_before>import datetime

from iris.models import Topic
from teams.models import Team
from teams.templatetags.team_tags import iscoordinatorofteam
from wakawaka.models import Revision, WikiPage

class TopicAccessBackend(object):

    supports_object_permissions = True
    supports_anonymous_user = True

    def authenticate(self, username, password):
        return None

    def has_perm(self, user_obj, perm, obj=None):
        #
        if perm == 'iris.add_topic':
            # Authenticated users can add topics.
            if user_obj.is_authenticated():
                return True
            # Default: cannot add a topic.
            return False
        #
        if perm == 'iris.join_topic':
            # Default: cannot join a topic.
            return False
        #
        if perm == 'iris.view_topic' and isinstance(obj, Topic):
            if user_obj.is_authenticated():
                # Find teams joined to the topic.
                # If the user is a member of any of the teams in the topic,
                # or none of the teams joined to the topic are private,
                # allow viewing.
                teams_joined = obj.participants_of_type(Team)
                any_private = False
                any_member = False
                for participant in teams_joined:
                    team = participant.content
                    if team.is_private:
                        any_private = True
                    if team.user_is_member(user_obj):
                        any_member = True
                        break
                if any_member or not any_private:
                    return True
            # Default: cannot view any topic.
            return False
        #
        if perm == 'iris.add_to_topic':
            if user_obj.is_authenticated():
                # Find teams joined to the topic.
                # If the user is a member of any of the teams in the topic,
                # allow adding.
                teams_joined = obj.participants_of_type(Team)
                for participant in teams_joined:
                    team = participant.content
                    if team.user_is_member(user_obj):
                        return True
            # Default: cannot add any item.
            return False

class WikiAccessBackend(object):

    supports_object_permissions = True
    supports_anonymous_user = True

    def authenticate(self, username, password):
        return None

    def has_perm(self, user_obj, perm, obj=None):
        def has_permission_for_group(group):
            return (
                # Page has a group associated.
                group is not None
                and (
                    # User is coordinator of the group, or
                    iscoordinatorofteam(user_obj, group)
                    or (
                        # User is member and either the group is private
                        # or the user has been a member for more than 4 days.
                        group.user_is_member(user_obj)
                        and (
                            group.is_private
                            or (user_obj.date_joined < datetime.datetime.now() - datetime.timedelta(days=4))
                        )
                    )
                )
            )
        if isinstance(obj, WikiPage):
            # Allow coordinators to change wiki pages in public groups.
            # Allow any member to change wiki pages in private groups.
            if perm == 'wakawaka.change_wikipage' or perm == 'wakawaka.change_revision':
                return obj.group and has_permission_for_group(obj.group)
            # Allow coordinators to reset edit locks on wiki pages in public groups.
            # Allow any member to reset edit locks in private groups.
            if perm == 'wakawaka.reset_lock':
                return obj.group and has_permission_for_group(obj.group)
            # Allow coordinators to delete wiki pages in public groups.
            # Allow any member to delete wiki pages in private groups.
            if perm == 'wakawaka.delete_wikipage' or perm == 'wakawaka.delete_revision':
                return obj.group and has_permission_for_group(obj.group)
        elif isinstance(obj, Team):
            # Allow coordinators to add wiki pages in public groups.
            # Allow any member to add wiki pages in private groups.
            if perm == 'wakawaka.add_wikipage' or perm == 'wakawaka.add_revision':
                return has_permission_for_group(obj)

# @@@ replace this with use of per-object restrictions

from groups.templatetags.group_tags import ismemberofgroup
from teams.templatetags.team_tags import iscoordinatorofteam

def event_edit(request, event):
    group = request.group
    user = request.user
    if iscoordinatorofteam(user, group):
        return True
    if getattr(group, 'is_private', False) and group.user_is_member(user):
        return True
    return False

event_delete = event_edit
<commit_msg>wakawaka.reset_lock is a team-level permission, not a page-level permission<commit_after>import datetime

from iris.models import Topic
from teams.models import Team
from teams.templatetags.team_tags import iscoordinatorofteam
from wakawaka.models import Revision, WikiPage

class TopicAccessBackend(object):

    supports_object_permissions = True
    supports_anonymous_user = True

    def authenticate(self, username, password):
        return None

    def has_perm(self, user_obj, perm, obj=None):
        #
        if perm == 'iris.add_topic':
            # Authenticated users can add topics.
            if user_obj.is_authenticated():
                return True
            # Default: cannot add a topic.
            return False
        #
        if perm == 'iris.join_topic':
            # Default: cannot join a topic.
            return False
        #
        if perm == 'iris.view_topic' and isinstance(obj, Topic):
            if user_obj.is_authenticated():
                # Find teams joined to the topic.
                # If the user is a member of any of the teams in the topic,
                # or none of the teams joined to the topic are private,
                # allow viewing.
                teams_joined = obj.participants_of_type(Team)
                any_private = False
                any_member = False
                for participant in teams_joined:
                    team = participant.content
                    if team.is_private:
                        any_private = True
                    if team.user_is_member(user_obj):
                        any_member = True
                        break
                if any_member or not any_private:
                    return True
            # Default: cannot view any topic.
            return False
        #
        if perm == 'iris.add_to_topic':
            if user_obj.is_authenticated():
                # Find teams joined to the topic.
                # If the user is a member of any of the teams in the topic,
                # allow adding.
                teams_joined = obj.participants_of_type(Team)
                for participant in teams_joined:
                    team = participant.content
                    if team.user_is_member(user_obj):
                        return True
            # Default: cannot add any item.
            return False

class WikiAccessBackend(object):

    supports_object_permissions = True
    supports_anonymous_user = True

    def authenticate(self, username, password):
        return None

    def has_perm(self, user_obj, perm, obj=None):
        def has_permission_for_group(group):
            return (
                # Page has a group associated.
                group is not None
                and (
                    # User is coordinator of the group, or
                    iscoordinatorofteam(user_obj, group)
                    or (
                        # User is member and either the group is private
                        # or the user has been a member for more than 4 days.
                        group.user_is_member(user_obj)
                        and (
                            group.is_private
                            or (user_obj.date_joined < datetime.datetime.now() - datetime.timedelta(days=4))
                        )
                    )
                )
            )
        if isinstance(obj, WikiPage):
            # Allow coordinators to change wiki pages in public groups.
            # Allow any member to change wiki pages in private groups.
            if perm == 'wakawaka.change_wikipage' or perm == 'wakawaka.change_revision':
                return obj.group and has_permission_for_group(obj.group)
            # Allow coordinators to delete wiki pages in public groups.
            # Allow any member to delete wiki pages in private groups.
            if perm == 'wakawaka.delete_wikipage' or perm == 'wakawaka.delete_revision':
                return obj.group and has_permission_for_group(obj.group)
        elif isinstance(obj, Team):
            # Allow coordinators to add wiki pages in public groups.
            # Allow any member to add wiki pages in private groups.
            if perm == 'wakawaka.add_wikipage' or perm == 'wakawaka.add_revision':
                return has_permission_for_group(obj)
            # Allow coordinators to reset edit locks on wiki pages in public groups.
            # Allow any member to reset edit locks in private groups.
            if perm == 'wakawaka.reset_lock':
                return has_permission_for_group(obj)

# @@@ replace this with use of per-object restrictions

from groups.templatetags.group_tags import ismemberofgroup
from teams.templatetags.team_tags import iscoordinatorofteam

def event_edit(request, event):
    group = request.group
    user = request.user
    if iscoordinatorofteam(user, group):
        return True
    if getattr(group, 'is_private', False) and group.user_is_member(user):
        return True
    return False

event_delete = event_edit
<|endoftext|>