<commit_before># Package:  components
# Date:     11th April 2010
# Author:   James Mills, prologic at shortcircuit dot net dot au
"""
This module defines the BaseComponent and its subclass Component.
"""

from circuits.core.events import Event

from types import MethodType
from collections import Callable
from inspect import getmembers, isclass

from .manager import Manager
from .utils import flatten, findroot
from .events import Registered, Unregistered
from .handlers import handler, HandlerMetaClass

def check_singleton(x, y):
    """Return True if x contains a singleton that is already a member of y"""

    singletons = filter(lambda i: getattr(i, "singleton", False), flatten(x))

    for component in singletons:
        singleton = getattr(component, "singleton", False)
        if isclass(singleton) and issubclass(singleton, Manager):
            if any([isinstance(c, singleton) for c in flatten(findroot(y))]):
                return True
        elif singleton:
            if any([type(component) in c for c in flatten(findroot(y))]):
                return True

    return False

class SingletonError(Exception):
    """Raised if a Component with the `singleton` class attribute is True.
    """

class PrepareUnregister(Event):
    """
    This event is fired when a component is about to be unregistered
    from the component tree. Unregistering a component actually
    detaches the complete subtree that the unregistered component
    is the root of. Components that need to know if they
    are removed from the main tree (e.g. because they maintain
    relationships to other components in the tree) handle this
    event, check if the component being unregistered is one
    of their ancestors and act accordingly.

    :param component: the component that will be unregistered
    :type  type: :class:`~.BaseComponent`
    """

    complete = True

    def __init__(self, *args, **kwargs):
        super(PrepareUnregister, self).__init__(*args, **kwargs)

    def in_subtree(self, component):
        """
        Convenience method that checks if the given *component*
        is in the subtree that is about to be detached.
        """
        while True:
            if component == self.args[0]:
                return True
            if component == component.root:
                return False
            component = component.parent

class BaseComponent(Manager):
    """
    This is the base class for all components in a circuits based application.
    Components can (and should, except for root components) be registered
    with a parent component.

    BaseComponents can declare methods as event handlers using the
    handler decoration (see :func:`circuits.core.handlers.handler`). The
    handlers are invoked for matching events from the
    component's channel (specified as the component's ``channel`` attribute).

    BaseComponents inherit from :class:`circuits.core.manager.Manager`.
    This provides components with the
    :func:`circuits.core.manager.Manager.fireEvent` method that can
    be used to fire events as the result of some computation.

    Apart from the ``fireEvent()`` method, the Manager nature is important
    for root components that are started or run.

    :ivar channel: a component can be associated with a specific channel
        by setting this attribute. This should either be done by
        specifying a class attribute *channel* in the derived class or by
        passing a keyword parameter *channel="..."* to *__init__*.
        If specified, the component's handlers receive events on the
        specified channel only, and events fired by the component will
        be sent on the specified channel (this behavior may be overridden,
        see :class:`~circuits.core.events.Event`, :meth:`~.fireEvent` and
        :func:`~circuits.core.handlers.handler`). By default, the channel
        attribute is set to "*", meaning that events are fired on all
        channels and received from all channels.
    """

    channel = "*"
    singleton = False

    def __new__(cls, *args, **kwargs):
        self = super(BaseComponent, cls).__new__(cls)

        handlers = dict(
            [(k, v) for k, v in cls.__dict__.items()
                if getattr(v, "handler", False)]
        )

        overridden = lambda x: x in handlers and handlers[x].override

        for base in cls.__bases__:
            if issubclass(cls, base):
                for k, v in list(base.__dict__.items()):
                    p1 = isinstance(v, Callable)
                    p2 = getattr(v, "handler", False)
                    p3 = overridden(k)
                    if p1 and p2 and not p3:
                        name = "%s_%s" % (base.__name__, k)
                        method = MethodType(v, self)
                        setattr(self, name, method)

        return self

    def __init__(self, *args, **kwargs):
        "initializes x; see x.__class__.__doc__ for signature"

        super(BaseComponent, self).__init__(*args, **kwargs)

        self.channel = kwargs.get("channel", self.channel) or "*"

        for k, v in getmembers(self):
            if getattr(v, "handler", False) is True:
                self.addHandler(v)
            if v is not self and isinstance(v, BaseComponent) \
                    and v not in ('parent', 'root'):
                v.register(self)

        if hasattr(self, "init") and callable(self.init):
            self.init(*args, **kwargs)

    def register(self, parent):
        """
        Inserts this component in the component tree as a child
        of the given *parent* node.

        :param parent: the parent component after registration has completed.
        :type parent: :class:`~.manager.Manager`

        This method fires a :class:`~.events.Registered` event to inform
        other components in the tree about the new member.
        """
        if check_singleton(self, parent):
            raise SingletonError(self)

        self.parent = parent
        self.root = parent.root

        if parent is not self:
            parent.registerChild(self)
            self.fire(Registered(self, self.parent))

        self._updateRoot(parent.root)

        return self

    @handler('unregister')
    def _on_unregister(self, component):
        if component is not self:
            return
        return self.unregister()

    def unregister(self):
        """
        Removes this component from the component tree.

        Removing a component from the component tree is a two stage process.
        First, the component is marked as to be removed, which prevents it
        from receiving further events, and a
        :class:`~.components.PrepareUnregister` event is fired. This
        allows other components to e.g. release references to the component
        to be removed before it is actually removed from the component tree.

        After the processing of the ``PrepareUnregister`` event has completed,
        the component is removed from the tree and an
        :class:`~.events.Unregistered` event is fired.
        """

        if self.unregister_pending or self.parent == self:
            return self

        # tick shouldn't be called anymore, although component is still in tree
        self._unregister_pending = True
        self.root._cache.clear()

        # Give components a chance to prepare for unregister
        evt = PrepareUnregister(self)
        evt.complete_channels = (self,)
        self.fire(evt)

        return self

    @property
    def unregister_pending(self):
        return getattr(self, "_unregister_pending", False)

    @handler("prepare_unregister_complete")
    def _on_prepare_unregister_complete(self, e, value):
        # Remove component from tree now
        delattr(self, "_unregister_pending")
        self.fire(Unregistered(self, self.parent))

        if self.parent is not self:
            self.parent.unregisterChild(self)
            self.parent = self

        self._updateRoot(self)
        return self

    def _updateRoot(self, root):
        self.root = root
        for c in self.components:
            c._updateRoot(root)

    @classmethod
    def handles(cls, *names):
        """Returns True if all names are valid event handlers"""

        handlers = dict(
            [(k, v) for k, v in cls.__dict__.items()
                if getattr(v, "handler", False)]
        )

        return all(name in handlers for name in names)

Component = HandlerMetaClass("Component", (BaseComponent,), {})
"""
If you use Component instead of BaseComponent as base class for your own
component class, then all methods that are not marked as private
(i.e: start with an underscore) are automatically decorated as handlers.

The methods are invoked for all events from the component's channel
where the event's name matches the method's name.
"""
<commit_msg>Implemented .handlers() and re-implemented .handles(). [Delivers #41573841]<commit_after># Package:  components
# Date:     11th April 2010
# Author:   James Mills, prologic at shortcircuit dot net dot au
"""
This module defines the BaseComponent and its subclass Component.
"""

from types import MethodType
from operator import itemgetter
from collections import Callable
from inspect import getmembers, isclass

from .manager import Manager
from .utils import flatten, findroot
from .events import Event, Registered, Unregistered
from .handlers import handler, HandlerMetaClass

def check_singleton(x, y):
    """Return True if x contains a singleton that is already a member of y"""

    singletons = filter(lambda i: getattr(i, "singleton", False), flatten(x))

    for component in singletons:
        singleton = getattr(component, "singleton", False)
        if isclass(singleton) and issubclass(singleton, Manager):
            if any([isinstance(c, singleton) for c in flatten(findroot(y))]):
                return True
        elif singleton:
            if any([type(component) in c for c in flatten(findroot(y))]):
                return True

    return False

class SingletonError(Exception):
    """Raised if a Component with the `singleton` class attribute is True.
    """

class PrepareUnregister(Event):
    """
    This event is fired when a component is about to be unregistered
    from the component tree. Unregistering a component actually
    detaches the complete subtree that the unregistered component
    is the root of. Components that need to know if they
    are removed from the main tree (e.g. because they maintain
    relationships to other components in the tree) handle this
    event, check if the component being unregistered is one
    of their ancestors and act accordingly.

    :param component: the component that will be unregistered
    :type  type: :class:`~.BaseComponent`
    """

    complete = True

    def __init__(self, *args, **kwargs):
        super(PrepareUnregister, self).__init__(*args, **kwargs)

    def in_subtree(self, component):
        """
        Convenience method that checks if the given *component*
        is in the subtree that is about to be detached.
        """
        while True:
            if component == self.args[0]:
                return True
            if component == component.root:
                return False
            component = component.parent

class BaseComponent(Manager):
    """
    This is the base class for all components in a circuits based application.
    Components can (and should, except for root components) be registered
    with a parent component.

    BaseComponents can declare methods as event handlers using the
    handler decoration (see :func:`circuits.core.handlers.handler`). The
    handlers are invoked for matching events from the
    component's channel (specified as the component's ``channel`` attribute).

    BaseComponents inherit from :class:`circuits.core.manager.Manager`.
    This provides components with the
    :func:`circuits.core.manager.Manager.fireEvent` method that can
    be used to fire events as the result of some computation.

    Apart from the ``fireEvent()`` method, the Manager nature is important
    for root components that are started or run.

    :ivar channel: a component can be associated with a specific channel
        by setting this attribute. This should either be done by
        specifying a class attribute *channel* in the derived class or by
        passing a keyword parameter *channel="..."* to *__init__*.
        If specified, the component's handlers receive events on the
        specified channel only, and events fired by the component will
        be sent on the specified channel (this behavior may be overridden,
        see :class:`~circuits.core.events.Event`, :meth:`~.fireEvent` and
        :func:`~circuits.core.handlers.handler`). By default, the channel
        attribute is set to "*", meaning that events are fired on all
        channels and received from all channels.
    """

    channel = "*"
    singleton = False

    def __new__(cls, *args, **kwargs):
        self = super(BaseComponent, cls).__new__(cls)

        handlers = dict(
            [(k, v) for k, v in cls.__dict__.items()
                if getattr(v, "handler", False)]
        )

        overridden = lambda x: x in handlers and handlers[x].override

        for base in cls.__bases__:
            if issubclass(cls, base):
                for k, v in list(base.__dict__.items()):
                    p1 = isinstance(v, Callable)
                    p2 = getattr(v, "handler", False)
                    p3 = overridden(k)
                    if p1 and p2 and not p3:
                        name = "%s_%s" % (base.__name__, k)
                        method = MethodType(v, self)
                        setattr(self, name, method)

        return self

    def __init__(self, *args, **kwargs):
        "initializes x; see x.__class__.__doc__ for signature"

        super(BaseComponent, self).__init__(*args, **kwargs)

        self.channel = kwargs.get("channel", self.channel) or "*"

        for k, v in getmembers(self):
            if getattr(v, "handler", False) is True:
                self.addHandler(v)
            if v is not self and isinstance(v, BaseComponent) \
                    and v not in ('parent', 'root'):
                v.register(self)

        if hasattr(self, "init") and callable(self.init):
            self.init(*args, **kwargs)

    def register(self, parent):
        """
        Inserts this component in the component tree as a child
        of the given *parent* node.

        :param parent: the parent component after registration has completed.
        :type parent: :class:`~.manager.Manager`

        This method fires a :class:`~.events.Registered` event to inform
        other components in the tree about the new member.
        """
        if check_singleton(self, parent):
            raise SingletonError(self)

        self.parent = parent
        self.root = parent.root

        if parent is not self:
            parent.registerChild(self)
            self.fire(Registered(self, self.parent))

        self._updateRoot(parent.root)

        return self

    @handler('unregister')
    def _on_unregister(self, component):
        if component is not self:
            return
        return self.unregister()

    def unregister(self):
        """
        Removes this component from the component tree.

        Removing a component from the component tree is a two stage process.
        First, the component is marked as to be removed, which prevents it
        from receiving further events, and a
        :class:`~.components.PrepareUnregister` event is fired. This
        allows other components to e.g. release references to the component
        to be removed before it is actually removed from the component tree.

        After the processing of the ``PrepareUnregister`` event has completed,
        the component is removed from the tree and an
        :class:`~.events.Unregistered` event is fired.
        """

        if self.unregister_pending or self.parent == self:
            return self

        # tick shouldn't be called anymore, although component is still in tree
        self._unregister_pending = True
        self.root._cache.clear()

        # Give components a chance to prepare for unregister
        evt = PrepareUnregister(self)
        evt.complete_channels = (self,)
        self.fire(evt)

        return self

    @property
    def unregister_pending(self):
        return getattr(self, "_unregister_pending", False)

    @handler("prepare_unregister_complete")
    def _on_prepare_unregister_complete(self, e, value):
        # Remove component from tree now
        delattr(self, "_unregister_pending")
        self.fire(Unregistered(self, self.parent))

        if self.parent is not self:
            self.parent.unregisterChild(self)
            self.parent = self

        self._updateRoot(self)
        return self

    def _updateRoot(self, root):
        self.root = root
        for c in self.components:
            c._updateRoot(root)

    @classmethod
    def handlers(cls):
        """Returns a list of all event handlers for this Component"""

        return map(
            itemgetter(0),
            filter(
                lambda (k, v): hasattr(v, "handler"),
                cls.__dict__.items()
            )
        )

    @classmethod
    def handles(cls, *names):
        """Returns True if all names are event handlers of this Component"""

        return all(name in cls.handlers() for name in names)

Component = HandlerMetaClass("Component", (BaseComponent,), {})
"""
If you use Component instead of BaseComponent as base class for your own
component class, then all methods that are not marked as private
(i.e: start with an underscore) are automatically decorated as handlers.

The methods are invoked for all events from the component's channel
where the event's name matches the method's name.
"""
<|endoftext|>


<commit_before>'''
Harvester for OAI-PMH interfaces.
'''
#pylint: disable-msg=E1101,E0611,F0401
import logging
import json
import unicodedata
import string
import urllib2
import urllib
import datetime

from ckan.model import Session, Package, Group
from ckan import model

from ckanext.harvest.harvesters.base import HarvesterBase
from ckan.lib.munge import  munge_tag
from ckanext.harvest.model import HarvestObject, HarvestJob
from ckan.model.authz import setup_default_user_roles
from ckan.controllers.storage import BUCKET, get_ofs
from ckan.lib import helpers as h
from pylons import config

import oaipmh.client
from oaipmh.metadata import MetadataRegistry, oai_dc_reader
from oaipmh.error import NoSetHierarchyError

log = logging.getLogger(__name__)

class OAIPMHHarvester(HarvesterBase):
    '''
    OAI-PMH Harvester for ckanext-harvester.
    '''

    config = None
    incremental = None

    def _set_config(self, config_str):
        '''Set the configuration string.
        '''
        if config_str:
            self.config = json.loads(config_str)
        else:
            self.config = {}

    def info(self):
        '''
        Return information about this harvester.
        '''
        return {
                'name': 'OAI-PMH',
                'title': 'OAI-PMH',
                'description': 'A server which has a OAI-PMH interface available.'
                }

    def gather_stage(self, harvest_job):
        '''
        The gather stage will recieve a HarvestJob object and will be
        responsible for:
            - gathering all the necessary objects to fetch on a later.
              stage (e.g. for a CSW server, perform a GetRecords request)
            - creating the necessary HarvestObjects in the database, specifying
              the guid and a reference to its source and job.
            - creating and storing any suitable HarvestGatherErrors that may
              occur.
            - returning a list with all the ids of the created HarvestObjects.

        :param harvest_job: HarvestJob object
        :returns: A list of HarvestObject ids
        '''
        self._set_config(harvest_job.source.config)
        from_ = None
        previous_job = Session.query(HarvestJob) \
                .filter(HarvestJob.source==harvest_job.source) \
                .filter(HarvestJob.gather_finished!=None) \
                .filter(HarvestJob.id!=harvest_job.id) \
                .order_by(HarvestJob.gather_finished.desc()) \
                .limit(1).first()
        if previous_job:
            self.incremental = True
            from_ = previous_job.gather_started
        sets = []
        harvest_objs = []
        registry = MetadataRegistry()
        registry.registerReader('oai_dc', oai_dc_reader)
        client = oaipmh.client.Client(harvest_job.source.url, registry)
        try:
            identifier = client.identify()
        except urllib2.URLError:
            self._save_gather_error('Could not gather anything from %s!' %
                                    harvest_job.source.url, harvest_job)
            return None
        domain = identifier.repositoryName()
        group = Group.by_name(domain)
        if not group:
            group = Group(name=domain, description=domain)
        query = self.config['query'] if 'query' in self.config else ''
        try:
            for set in client.listSets():
                identifier, name, _ = set
                if 'query' in self.config:
                    if query in name:
                        sets.append((identifier, name))
                else:
                    sets.append((identifier, name))
        except NoSetHierarchyError:
            sets.append(('1', 'Default'))
            self._save_gather_error('Could not fetch sets!', harvest_job)

        for set_id, set_name in sets:
            harvest_obj = HarvestObject(job=harvest_job)
            harvest_obj.content = json.dumps(
                                             {
                                              'set': set_id, \
                                              'set_name': set_name, \
                                              'domain': domain,
                                              'incremental': from_.strftime('%Y-%m-%dT%H:%M:%S') if from_ else None,
                                              }
                                             )
            harvest_obj.save()
            harvest_objs.append(harvest_obj.id)
        model.repo.commit()
        return harvest_objs

    def fetch_stage(self, harvest_object):
        '''
        The fetch stage will receive a HarvestObject object and will be
        responsible for:
            - getting the contents of the remote object (e.g. for a CSW server,
              perform a GetRecordById request).
            - saving the content in the provided HarvestObject.
            - creating and storing any suitable HarvestObjectErrors that may
              occur.
            - returning True if everything went as expected, False otherwise.

        :param harvest_object: HarvestObject object
        :returns: True if everything went right, False if errors were found
        '''
        sets = json.loads(harvest_object.content)
        registry = MetadataRegistry()
        registry.registerReader('oai_dc', oai_dc_reader)
        client = oaipmh.client.Client(harvest_object.job.source.url, registry)
        records = []
        recs = []
        from_ = datetime.datetime.strptime(sets['incremental'], '%Y-%m-%dT%H:%M:%S') if sets['incremental'] else None
        try:
            recs = client.listRecords(metadataPrefix='oai_dc', set=sets['set'],
                                      from_=from_)
        except Exception, e:
            self._save_object_error('%r' % e, harvest_object)
            pass
        for rec in recs:
            header, metadata, _ = rec
            if metadata:
                records.append((header.identifier(), metadata.getMap(), None))
        if len(records):
            sets['records'] = records
            harvest_object.content = json.dumps(sets)
        else:
            self._save_object_error('Could not find any records for set %s!' %
                                    sets['set'], harvest_object)
            return False
        return True

    def import_stage(self, harvest_object):
        '''
        The import stage will receive a HarvestObject object and will be
        responsible for:
            - performing any necessary action with the fetched object (e.g
              create a CKAN package).
              Note: if this stage creates or updates a package, a reference
              to the package must be added to the HarvestObject.
              Additionally, the HarvestObject must be flagged as current.
            - creating the HarvestObject - Package relation (if necessary)
            - creating and storing any suitable HarvestObjectErrors that may
              occur.
            - returning True if everything went as expected, False otherwise.

        :param harvest_object: HarvestObject object
        :returns: True if everything went right, False if errors were found
        '''
        model.repo.new_revision()
        master_data = json.loads(harvest_object.content)
        domain = master_data['domain']
        group = Group.get(domain)
        if not group:
            group = Group(name=domain, description=domain)
        if 'records' in master_data:
            records = master_data['records']
            set_name = master_data['set_name']
            for rec in records:
                identifier, metadata, _ = rec
                if metadata:
                    title = metadata['title'][0] if len(metadata['title']) else identifier
                    description = metadata['description'][0]\
                                if len(metadata['description']) else ''
                    name = None
                    for ident in metadata['identifier']:
                        if ident.startswith('http://'):
                            continue
                        elif ident.startswith('URN'):
                            name = ident
                            break
                        if not name:
                            name = ident
                            break
                    if not name:
                        name = identifier
                    pkg = Package.get(name)
                    if not pkg:
                        pkg = Package(name=name, title=title, id=identifier)
                    extras = {}
                    lastidx = 0
                    for met in metadata.items():
                        key, value = met
                        if len(value) > 0:
                            if key == 'subject' or key == 'type':
                                for tag in value:
                                    if tag:
                                        for tagi in tag.split(','):
                                            tagi = tagi.strip()
                                            tagi = munge_tag(tagi[:100])
                                            tag_obj = model.Tag.by_name(tagi)
                                            if not tag_obj:
                                                tag_obj = model.Tag(name=tagi)
                                            if tag_obj:
                                                pkgtag = model.PackageTag(
                                                                      tag=tag_obj,
                                                                      package=pkg)
                                                Session.add(tag_obj)
                                                Session.add(pkgtag)
                            elif key == 'creator' or key == 'contributor':
                                for auth in value:
                                    extras['organization_%d' % lastidx] = ""
                                    extras['author_%d' % lastidx] = auth
                                    lastidx += 1
                            elif key != 'title':
                                extras[key] = ' '.join(value)
                    pkg.title = title
                    pkg.notes = description
                    extras['lastmod'] = extras['date']
                    pkg.extras = extras
                    pkg.url = \
                    "%s?verb=GetRecord&identifier=%s&metadataPrefix=oai_dc"\
                                % (harvest_object.job.source.url, identifier)
                    pkg.save()
                    ofs = get_ofs()
                    nowstr = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%f')
                    label = "%s/%s.xml" % (\
                        nowstr,
                        identifier)
                    try:
                        f = urllib2.urlopen(pkg.url)
                        ofs.put_stream(BUCKET, label, f, {})
                        fileurl = config.get('ckan.site_url') + h.url_for('storage_file', label=label)
                        pkg.add_resource(url=fileurl, description="Original metadata record",
                                 format="xml", size=len(f.read()))
                    except urllib2.HTTPError:
                        self._save_object_error('Could not get original metadata record!',
                                                harvest_object, stage='Import')
                    harvest_object.package_id = pkg.id
                    harvest_object.current = True
                    harvest_object.save()
                    Session.add(harvest_object)
                    setup_default_user_roles(pkg)
                    title = metadata['title'][0] if len(metadata['title'])\
                                                    else ''
                    description = metadata['description'][0]\
                                    if len(metadata['description']) else ''
                    url = ''
                    for ids in metadata['identifier']:
                        if ids.startswith('http://'):
                            url = ids
                    if url != '':
                        pkg.add_resource(url,
                                         description=description,
                                         name=title,
                                         format='html' if url.startswith('http://') else '')
                    group.add_package_by_name(pkg.name)
                    subg_name = "%s - %s" % (domain, set_name)
                    subgroup = Group.by_name(subg_name)
                    if not subgroup:
                        subgroup = Group(name=subg_name, description=subg_name)
                    subgroup.add_package_by_name(pkg.name)
                    Session.add(group)
                    Session.add(subgroup)
                    setup_default_user_roles(group)
                    setup_default_user_roles(subgroup)
            model.repo.commit()
        else:
            self._save_object_error('Could not receive any objects from fetch!'
                                    , harvest_object, stage='Import')
            return False
        return True
<commit_msg>Return quote_plus.<commit_after>'''
Harvester for OAI-PMH interfaces.
'''
#pylint: disable-msg=E1101,E0611,F0401
import logging
import json
import unicodedata
import string
import urllib2
import urllib
import datetime

from ckan.model import Session, Package, Group
from ckan import model

from ckanext.harvest.harvesters.base import HarvesterBase
from ckan.lib.munge import  munge_tag
from ckanext.harvest.model import HarvestObject, HarvestJob
from ckan.model.authz import setup_default_user_roles
from ckan.controllers.storage import BUCKET, get_ofs
from ckan.lib import helpers as h
from pylons import config

import oaipmh.client
from oaipmh.metadata import MetadataRegistry, oai_dc_reader
from oaipmh.error import NoSetHierarchyError

log = logging.getLogger(__name__)

class OAIPMHHarvester(HarvesterBase):
    '''
    OAI-PMH Harvester for ckanext-harvester.
    '''

    config = None
    incremental = None

    def _set_config(self, config_str):
        '''Set the configuration string.
        '''
        if config_str:
            self.config = json.loads(config_str)
        else:
            self.config = {}

    def info(self):
        '''
        Return information about this harvester.
        '''
        return {
                'name': 'OAI-PMH',
                'title': 'OAI-PMH',
                'description': 'A server which has a OAI-PMH interface available.'
                }

    def gather_stage(self, harvest_job):
        '''
        The gather stage will recieve a HarvestJob object and will be
        responsible for:
            - gathering all the necessary objects to fetch on a later.
              stage (e.g. for a CSW server, perform a GetRecords request)
            - creating the necessary HarvestObjects in the database, specifying
              the guid and a reference to its source and job.
            - creating and storing any suitable HarvestGatherErrors that may
              occur.
            - returning a list with all the ids of the created HarvestObjects.

        :param harvest_job: HarvestJob object
        :returns: A list of HarvestObject ids
        '''
        self._set_config(harvest_job.source.config)
        from_ = None
        previous_job = Session.query(HarvestJob) \
                .filter(HarvestJob.source==harvest_job.source) \
                .filter(HarvestJob.gather_finished!=None) \
                .filter(HarvestJob.id!=harvest_job.id) \
                .order_by(HarvestJob.gather_finished.desc()) \
                .limit(1).first()
        if previous_job:
            self.incremental = True
            from_ = previous_job.gather_started
        sets = []
        harvest_objs = []
        registry = MetadataRegistry()
        registry.registerReader('oai_dc', oai_dc_reader)
        client = oaipmh.client.Client(harvest_job.source.url, registry)
        try:
            identifier = client.identify()
        except urllib2.URLError:
            self._save_gather_error('Could not gather anything from %s!' %
                                    harvest_job.source.url, harvest_job)
            return None
        domain = identifier.repositoryName()
        group = Group.by_name(domain)
        if not group:
            group = Group(name=domain, description=domain)
        query = self.config['query'] if 'query' in self.config else ''
        try:
            for set in client.listSets():
                identifier, name, _ = set
                if 'query' in self.config:
                    if query in name:
                        sets.append((identifier, name))
                else:
                    sets.append((identifier, name))
        except NoSetHierarchyError:
            sets.append(('1', 'Default'))
            self._save_gather_error('Could not fetch sets!', harvest_job)

        for set_id, set_name in sets:
            harvest_obj = HarvestObject(job=harvest_job)
            harvest_obj.content = json.dumps(
                                             {
                                              'set': set_id, \
                                              'set_name': set_name, \
                                              'domain': domain,
                                              'incremental': from_.strftime('%Y-%m-%dT%H:%M:%S') if from_ else None,
                                              }
                                             )
            harvest_obj.save()
            harvest_objs.append(harvest_obj.id)
        model.repo.commit()
        return harvest_objs

    def fetch_stage(self, harvest_object):
        '''
        The fetch stage will receive a HarvestObject object and will be
        responsible for:
            - getting the contents of the remote object (e.g. for a CSW server,
              perform a GetRecordById request).
            - saving the content in the provided HarvestObject.
            - creating and storing any suitable HarvestObjectErrors that may
              occur.
            - returning True if everything went as expected, False otherwise.

        :param harvest_object: HarvestObject object
        :returns: True if everything went right, False if errors were found
        '''
        sets = json.loads(harvest_object.content)
        registry = MetadataRegistry()
        registry.registerReader('oai_dc', oai_dc_reader)
        client = oaipmh.client.Client(harvest_object.job.source.url, registry)
        records = []
        recs = []
        from_ = datetime.datetime.strptime(sets['incremental'], '%Y-%m-%dT%H:%M:%S') if sets['incremental'] else None
        try:
            recs = client.listRecords(metadataPrefix='oai_dc', set=sets['set'],
                                      from_=from_)
        except Exception, e:
            self._save_object_error('%r' % e, harvest_object)
            pass
        for rec in recs:
            header, metadata, _ = rec
            if metadata:
                records.append((header.identifier(), metadata.getMap(), None))
        if len(records):
            sets['records'] = records
            harvest_object.content = json.dumps(sets)
        else:
            self._save_object_error('Could not find any records for set %s!' %
                                    sets['set'], harvest_object)
            return False
        return True

    def import_stage(self, harvest_object):
        '''
        The import stage will receive a HarvestObject object and will be
        responsible for:
            - performing any necessary action with the fetched object (e.g
              create a CKAN package).
              Note: if this stage creates or updates a package, a reference
              to the package must be added to the HarvestObject.
              Additionally, the HarvestObject must be flagged as current.
            - creating the HarvestObject - Package relation (if necessary)
            - creating and storing any suitable HarvestObjectErrors that may
              occur.
            - returning True if everything went as expected, False otherwise.

        :param harvest_object: HarvestObject object
        :returns: True if everything went right, False if errors were found
        '''
        model.repo.new_revision()
        master_data = json.loads(harvest_object.content)
        domain = master_data['domain']
        group = Group.get(domain)
        if not group:
            group = Group(name=domain, description=domain)
        if 'records' in master_data:
            records = master_data['records']
            set_name = master_data['set_name']
            for rec in records:
                identifier, metadata, _ = rec
                if metadata:
                    title = metadata['title'][0] if len(metadata['title']) else identifier
                    description = metadata['description'][0]\
                                if len(metadata['description']) else ''
                    name = None
                    for ident in metadata['identifier']:
                        if ident.startswith('http://'):
                            continue
                        elif ident.startswith('URN'):
                            name = ident
                            break
                        if not name:
                            name = ident
                            break
                    if not name:
                        name = urllib2.quote_plus(identifier)
                    pkg = Package.get(name)
                    if not pkg:
                        pkg = Package(name=name, title=title, id=identifier)
                    extras = {}
                    lastidx = 0
                    for met in metadata.items():
                        key, value = met
                        if len(value) > 0:
                            if key == 'subject' or key == 'type':
                                for tag in value:
                                    if tag:
                                        for tagi in tag.split(','):
                                            tagi = tagi.strip()
                                            tagi = munge_tag(tagi[:100])
                                            tag_obj = model.Tag.by_name(tagi)
                                            if not tag_obj:
                                                tag_obj = model.Tag(name=tagi)
                                            if tag_obj:
                                                pkgtag = model.PackageTag(
                                                                      tag=tag_obj,
                                                                      package=pkg)
                                                Session.add(tag_obj)
                                                Session.add(pkgtag)
                            elif key == 'creator' or key == 'contributor':
                                for auth in value:
                                    extras['organization_%d' % lastidx] = ""
                                    extras['author_%d' % lastidx] = auth
                                    lastidx += 1
                            elif key != 'title':
                                extras[key] = ' '.join(value)
                    pkg.title = title
                    pkg.notes = description
                    extras['lastmod'] = extras['date']
                    pkg.extras = extras
                    pkg.url = \
                    "%s?verb=GetRecord&identifier=%s&metadataPrefix=oai_dc"\
                                % (harvest_object.job.source.url, identifier)
                    pkg.save()
                    ofs = get_ofs()
                    nowstr = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%f')
                    label = "%s/%s.xml" % (\
                        nowstr,
                        identifier)
                    try:
                        f = urllib2.urlopen(pkg.url)
                        ofs.put_stream(BUCKET, label, f, {})
                        fileurl = config.get('ckan.site_url') + h.url_for('storage_file', label=label)
                        pkg.add_resource(url=fileurl, description="Original metadata record",
                                 format="xml", size=len(f.read()))
                    except urllib2.HTTPError:
                        self._save_object_error('Could not get original metadata record!',
                                                harvest_object, stage='Import')
                    harvest_object.package_id = pkg.id
                    harvest_object.current = True
                    harvest_object.save()
                    Session.add(harvest_object)
                    setup_default_user_roles(pkg)
                    title = metadata['title'][0] if len(metadata['title'])\
                                                    else ''
                    description = metadata['description'][0]\
                                    if len(metadata['description']) else ''
                    url = ''
                    for ids in metadata['identifier']:
                        if ids.startswith('http://'):
                            url = ids
                    if url != '':
                        pkg.add_resource(url,
                                         description=description,
                                         name=title,
                                         format='html' if url.startswith('http://') else '')
                    group.add_package_by_name(pkg.name)
                    subg_name = "%s - %s" % (domain, set_name)
                    subgroup = Group.by_name(subg_name)
                    if not subgroup:
                        subgroup = Group(name=subg_name, description=subg_name)
                    subgroup.add_package_by_name(pkg.name)
                    Session.add(group)
                    Session.add(subgroup)
                    setup_default_user_roles(group)
                    setup_default_user_roles(subgroup)
            model.repo.commit()
        else:
            self._save_object_error('Could not receive any objects from fetch!'
                                    , harvest_object, stage='Import')
            return False
        return True
<|endoftext|>


<commit_before>import struct
import elftools
import logging

from .elf import ELF
from .. import register_backend
from ...errors import CLEError, CLECompatibilityError

l = logging.getLogger(name=__name__)

# TODO: yall know struct.unpack_from exists, right? maybe even bitstream?

class CoreNote(object):
    """
    This class is used when parsing the NOTES section of a core file.
    """
    n_type_lookup = {
            1: 'NT_PRSTATUS',
            2: 'NT_PRFPREG',
            3: 'NT_PRPSINFO',
            4: 'NT_TASKSTRUCT',
            6: 'NT_AUXV',
            0x53494749: 'NT_SIGINFO',
            0x46494c45: 'NT_FILE',
            0x46e62b7f: 'NT_PRXFPREG'
            }

    def __init__(self, n_type, name, desc):
        self.n_type = n_type
        if n_type in CoreNote.n_type_lookup:
            self.n_type = CoreNote.n_type_lookup[n_type]
        self.name = name
        self.desc = desc

    def __repr__(self):
        return "<Note %s %s %#x>" % (self.name, self.n_type, len(self.desc))

class ELFCore(ELF):
    """
    Loader class for ELF core files.
    """
    is_default = True # Tell CLE to automatically consider using the ELFCore backend

    def __init__(self, binary, **kwargs):
        super(ELFCore, self).__init__(binary, **kwargs)

        self.notes = []
        self.prstatuses = []

        self.__extract_note_info()

    @staticmethod
    def is_compatible(stream):
        stream.seek(0)
        identstring = stream.read(0x1000)
        stream.seek(0)
        if identstring.startswith(b'\x7fELF'):
            if elftools.elf.elffile.ELFFile(stream).header['e_type'] == 'ET_CORE':
                return True
            return False
        return False

    @property
    def threads(self):
        return list(range(len(self.prstatuses)))

    def thread_registers(self, thread=None):
        if thread is None:
            thread = 0
        return self.prstatuses[thread]['registers']

    def __extract_note_info(self):
        """
        All meaningful information about the process's state at crashtime is stored in the note segment.
        """
        for seg_readelf in self.reader.iter_segments():
            if seg_readelf.header.p_type == 'PT_NOTE':
                self.__parse_notes(seg_readelf)
                break
        else:
            l.warning("Could not find note segment, cannot initialize registers")

    def __parse_notes(self, seg):
        """
        This exists, because note parsing in elftools is not good.
        """

        blob = seg.data()

        note_pos = 0
        while note_pos < len(blob):
            name_sz, desc_sz, n_type = struct.unpack("<3I", blob[note_pos:note_pos+12])
            name_sz_rounded = (((name_sz + (4 - 1)) // 4) * 4)
            desc_sz_rounded = (((desc_sz + (4 - 1)) // 4) * 4)
            # description size + the rounded name size + header size
            n_size = desc_sz_rounded + name_sz_rounded + 12

            # name_sz includes the null byte
            name = blob[note_pos+12:note_pos+12+name_sz-1]
            desc = blob[note_pos+12+name_sz_rounded:note_pos+12+name_sz_rounded+desc_sz]

            self.notes.append(CoreNote(n_type, name, desc))
            note_pos += n_size

        # prstatus
        self.prstatuses = [self.__parse_prstatus(x) for x in self.notes if x.n_type == 'NT_PRSTATUS']

    def __parse_prstatus(self, prstatus):
        """
        Parse out the prstatus, accumulating the general purpose register values. Supports AMD64, X86, ARM, and AARCH64
        at the moment.

        :param prstatus: a note object of type NT_PRSTATUS.
        """

        # TODO: support all architectures angr supports

        result = {}
        result['si_signo'], result['si_code'], result['si_errno'] = struct.unpack("<3I", prstatus.desc[:12])

        # this field is a short, but it's padded to an int
        result['pr_cursig'] = struct.unpack("<I", prstatus.desc[12:16])[0]

        arch_bytes = self.arch.bytes
        if arch_bytes == 4:
            fmt = "I"
        elif arch_bytes == 8:
            fmt = "Q"
        else:
            raise CLEError("Architecture must have a bitwidth of either 64 or 32")

        result['pr_sigpend'], result['pr_sighold'] = struct.unpack("<" + (fmt * 2), prstatus.desc[16:16+(2*arch_bytes)])

        attrs = struct.unpack("<IIII", prstatus.desc[16+(2*arch_bytes):16+(2*arch_bytes)+(4*4)])
        result['pr_pid'], result['pr_ppid'], result['pr_pgrp'], result['pr_sid'] = attrs

        # parse out the 4 timevals
        pos = 16+(2*arch_bytes)+(4*4)
        usec = struct.unpack("<" + fmt, prstatus.desc[pos:pos+arch_bytes])[0] * 1000
        result['pr_utime_usec'] = struct.unpack("<" + fmt, prstatus.desc[pos+arch_bytes:pos+arch_bytes*2])[0] + usec

        pos += arch_bytes * 2
        usec = struct.unpack("<" + fmt, prstatus.desc[pos:pos+arch_bytes])[0] * 1000
        result['pr_stime_usec'] = struct.unpack("<" + fmt, prstatus.desc[pos+arch_bytes:pos+arch_bytes*2])[0] + usec

        pos += arch_bytes * 2
        usec = struct.unpack("<" + fmt, prstatus.desc[pos:pos+arch_bytes])[0] * 1000
        result['pr_cutime_usec'] = struct.unpack("<" + fmt, prstatus.desc[pos+arch_bytes:pos+arch_bytes*2])[0] + usec

        pos += arch_bytes * 2
        usec = struct.unpack("<" + fmt, prstatus.desc[pos:pos+arch_bytes])[0] * 1000
        result['pr_cstime_usec'] = struct.unpack("<" + fmt, prstatus.desc[pos+arch_bytes:pos+arch_bytes*2])[0] + usec

        pos += arch_bytes * 2

        # parse out general purpose registers
        if self.arch.name == 'AMD64':
            # register names as they appear in dump
            rnames = ['r15', 'r14', 'r13', 'r12', 'rbp', 'rbx', 'r11', 'r10', 'r9', 'r8', 'rax', 'rcx',
                    'rdx', 'rsi', 'rdi', 'xxx', 'rip', 'cs', 'eflags', 'rsp', 'ss', 'xxx', 'xxx', 'ds', 'es',
                    'fs', 'gs']
            nreg = 27
        elif self.arch.name == 'X86':
            rnames = ['ebx', 'ecx', 'edx', 'esi', 'edi', 'ebp', 'eax', 'ds', 'es', 'fs', 'gs', 'xxx', 'eip',
                    'cs', 'eflags', 'esp', 'ss']
            nreg = 17
        elif self.arch.name == 'ARMHF' or self.arch.name == 'ARMEL':
            rnames = ['r0', 'r1', 'r2', 'r3', 'r4', 'r5', 'r6', 'r7', 'r8', 'r9', 'r10', 'r11', 'r12', 'r13',
                    'r14', 'r15', 'xxx', 'xxx']
            nreg = 18
        elif self.arch.name == 'AARCH64':
            rnames =  ['x%d' % i for i in range(32)]
            rnames.append('pc')
            rnames.append('xxx')
            nreg = 34
        elif self.arch.name == 'MIPS32':
            rnames = ['xxx', 'xxx', 'xxx', 'xxx', 'xxx', 'xxx',
                    'zero', 'at', 'v0', 'v1', 'a0', 'a1', 'a2', 'a3',
                    't0', 't1', 't2', 't3', 't4', 't5', 't6', 't7',
                    's0', 's1', 's2', 's3', 's4', 's5', 's6', 's7',
                    't8', 't9', 'k0', 'k1', 'gp', 'sp', 's8', 'ra',
                    'lo', 'hi', 'pc', 'bad', 'sr', 'status', 'cause']
            nreg = 45
        else:
            raise CLECompatibilityError("Architecture '%s' unsupported by ELFCore" % self.arch.name)

        regvals = []
        for idx in range(pos, pos+nreg*arch_bytes, arch_bytes):
            regvals.append(struct.unpack("<" + fmt, prstatus.desc[idx:idx+arch_bytes])[0])
        result['registers'] = dict(zip(rnames, regvals))
        del result['registers']['xxx']

        pos += nreg * arch_bytes
        result['pr_fpvalid'] = struct.unpack("<I", prstatus.desc[pos:pos+4])[0]

register_backend('elfcore', ELFCore)
<commit_msg>add missing return statement<commit_after>import struct
import elftools
import logging

from .elf import ELF
from .. import register_backend
from ...errors import CLEError, CLECompatibilityError

l = logging.getLogger(name=__name__)

# TODO: yall know struct.unpack_from exists, right? maybe even bitstream?

class CoreNote(object):
    """
    This class is used when parsing the NOTES section of a core file.
    """
    n_type_lookup = {
            1: 'NT_PRSTATUS',
            2: 'NT_PRFPREG',
            3: 'NT_PRPSINFO',
            4: 'NT_TASKSTRUCT',
            6: 'NT_AUXV',
            0x53494749: 'NT_SIGINFO',
            0x46494c45: 'NT_FILE',
            0x46e62b7f: 'NT_PRXFPREG'
            }

    def __init__(self, n_type, name, desc):
        self.n_type = n_type
        if n_type in CoreNote.n_type_lookup:
            self.n_type = CoreNote.n_type_lookup[n_type]
        self.name = name
        self.desc = desc

    def __repr__(self):
        return "<Note %s %s %#x>" % (self.name, self.n_type, len(self.desc))

class ELFCore(ELF):
    """
    Loader class for ELF core files.
    """
    is_default = True # Tell CLE to automatically consider using the ELFCore backend

    def __init__(self, binary, **kwargs):
        super(ELFCore, self).__init__(binary, **kwargs)

        self.notes = []
        self.prstatuses = []

        self.__extract_note_info()

    @staticmethod
    def is_compatible(stream):
        stream.seek(0)
        identstring = stream.read(0x1000)
        stream.seek(0)
        if identstring.startswith(b'\x7fELF'):
            if elftools.elf.elffile.ELFFile(stream).header['e_type'] == 'ET_CORE':
                return True
            return False
        return False

    @property
    def threads(self):
        return list(range(len(self.prstatuses)))

    def thread_registers(self, thread=None):
        if thread is None:
            thread = 0
        return self.prstatuses[thread]['registers']

    def __extract_note_info(self):
        """
        All meaningful information about the process's state at crashtime is stored in the note segment.
        """
        for seg_readelf in self.reader.iter_segments():
            if seg_readelf.header.p_type == 'PT_NOTE':
                self.__parse_notes(seg_readelf)
                break
        else:
            l.warning("Could not find note segment, cannot initialize registers")

    def __parse_notes(self, seg):
        """
        This exists, because note parsing in elftools is not good.
        """

        blob = seg.data()

        note_pos = 0
        while note_pos < len(blob):
            name_sz, desc_sz, n_type = struct.unpack("<3I", blob[note_pos:note_pos+12])
            name_sz_rounded = (((name_sz + (4 - 1)) // 4) * 4)
            desc_sz_rounded = (((desc_sz + (4 - 1)) // 4) * 4)
            # description size + the rounded name size + header size
            n_size = desc_sz_rounded + name_sz_rounded + 12

            # name_sz includes the null byte
            name = blob[note_pos+12:note_pos+12+name_sz-1]
            desc = blob[note_pos+12+name_sz_rounded:note_pos+12+name_sz_rounded+desc_sz]

            self.notes.append(CoreNote(n_type, name, desc))
            note_pos += n_size

        # prstatus
        self.prstatuses = [self.__parse_prstatus(x) for x in self.notes if x.n_type == 'NT_PRSTATUS']

    def __parse_prstatus(self, prstatus):
        """
        Parse out the prstatus, accumulating the general purpose register values. Supports AMD64, X86, ARM, and AARCH64
        at the moment.

        :param prstatus: a note object of type NT_PRSTATUS.
        """

        # TODO: support all architectures angr supports

        result = {}
        result['si_signo'], result['si_code'], result['si_errno'] = struct.unpack("<3I", prstatus.desc[:12])

        # this field is a short, but it's padded to an int
        result['pr_cursig'] = struct.unpack("<I", prstatus.desc[12:16])[0]

        arch_bytes = self.arch.bytes
        if arch_bytes == 4:
            fmt = "I"
        elif arch_bytes == 8:
            fmt = "Q"
        else:
            raise CLEError("Architecture must have a bitwidth of either 64 or 32")

        result['pr_sigpend'], result['pr_sighold'] = struct.unpack("<" + (fmt * 2), prstatus.desc[16:16+(2*arch_bytes)])

        attrs = struct.unpack("<IIII", prstatus.desc[16+(2*arch_bytes):16+(2*arch_bytes)+(4*4)])
        result['pr_pid'], result['pr_ppid'], result['pr_pgrp'], result['pr_sid'] = attrs

        # parse out the 4 timevals
        pos = 16+(2*arch_bytes)+(4*4)
        usec = struct.unpack("<" + fmt, prstatus.desc[pos:pos+arch_bytes])[0] * 1000
        result['pr_utime_usec'] = struct.unpack("<" + fmt, prstatus.desc[pos+arch_bytes:pos+arch_bytes*2])[0] + usec

        pos += arch_bytes * 2
        usec = struct.unpack("<" + fmt, prstatus.desc[pos:pos+arch_bytes])[0] * 1000
        result['pr_stime_usec'] = struct.unpack("<" + fmt, prstatus.desc[pos+arch_bytes:pos+arch_bytes*2])[0] + usec

        pos += arch_bytes * 2
        usec = struct.unpack("<" + fmt, prstatus.desc[pos:pos+arch_bytes])[0] * 1000
        result['pr_cutime_usec'] = struct.unpack("<" + fmt, prstatus.desc[pos+arch_bytes:pos+arch_bytes*2])[0] + usec

        pos += arch_bytes * 2
        usec = struct.unpack("<" + fmt, prstatus.desc[pos:pos+arch_bytes])[0] * 1000
        result['pr_cstime_usec'] = struct.unpack("<" + fmt, prstatus.desc[pos+arch_bytes:pos+arch_bytes*2])[0] + usec

        pos += arch_bytes * 2

        # parse out general purpose registers
        if self.arch.name == 'AMD64':
            # register names as they appear in dump
            rnames = ['r15', 'r14', 'r13', 'r12', 'rbp', 'rbx', 'r11', 'r10', 'r9', 'r8', 'rax', 'rcx',
                    'rdx', 'rsi', 'rdi', 'xxx', 'rip', 'cs', 'eflags', 'rsp', 'ss', 'xxx', 'xxx', 'ds', 'es',
                    'fs', 'gs']
            nreg = 27
        elif self.arch.name == 'X86':
            rnames = ['ebx', 'ecx', 'edx', 'esi', 'edi', 'ebp', 'eax', 'ds', 'es', 'fs', 'gs', 'xxx', 'eip',
                    'cs', 'eflags', 'esp', 'ss']
            nreg = 17
        elif self.arch.name == 'ARMHF' or self.arch.name == 'ARMEL':
            rnames = ['r0', 'r1', 'r2', 'r3', 'r4', 'r5', 'r6', 'r7', 'r8', 'r9', 'r10', 'r11', 'r12', 'r13',
                    'r14', 'r15', 'xxx', 'xxx']
            nreg = 18
        elif self.arch.name == 'AARCH64':
            rnames =  ['x%d' % i for i in range(32)]
            rnames.append('pc')
            rnames.append('xxx')
            nreg = 34
        elif self.arch.name == 'MIPS32':
            rnames = ['xxx', 'xxx', 'xxx', 'xxx', 'xxx', 'xxx',
                    'zero', 'at', 'v0', 'v1', 'a0', 'a1', 'a2', 'a3',
                    't0', 't1', 't2', 't3', 't4', 't5', 't6', 't7',
                    's0', 's1', 's2', 's3', 's4', 's5', 's6', 's7',
                    't8', 't9', 'k0', 'k1', 'gp', 'sp', 's8', 'ra',
                    'lo', 'hi', 'pc', 'bad', 'sr', 'status', 'cause']
            nreg = 45
        else:
            raise CLECompatibilityError("Architecture '%s' unsupported by ELFCore" % self.arch.name)

        regvals = []
        for idx in range(pos, pos+nreg*arch_bytes, arch_bytes):
            regvals.append(struct.unpack("<" + fmt, prstatus.desc[idx:idx+arch_bytes])[0])
        result['registers'] = dict(zip(rnames, regvals))
        del result['registers']['xxx']

        pos += nreg * arch_bytes
        result['pr_fpvalid'] = struct.unpack("<I", prstatus.desc[pos:pos+4])[0]
        return result

register_backend('elfcore', ELFCore)
<|endoftext|>


<commit_before>import logging

from django.core.management.base import BaseCommand

from ...utils.annotation import annotate_content
from ...utils.channel_import import FutureSchemaError
from ...utils.channel_import import import_channel_from_local_db
from ...utils.channel_import import InvalidSchemaVersionError
from ...utils.channels import get_channel_ids_for_content_database_dir
from kolibri.core.content.models import ChannelMetadata
from kolibri.core.content.utils.paths import get_content_database_dir_path

logger = logging.getLogger(__name__)

class Command(BaseCommand):
    """
    This command will try to fix the content tables in the database,
    reviewing the availability of all the existing files, to recover
    content that's not visible in Kolibri.
    """

    help = "Scan content and databases in Kolibri folder and updates the database to show if available"

    def handle(self, *args, **options):
        storage_channel_ids = get_channel_ids_for_content_database_dir(
            get_content_database_dir_path()
        )
        database_channel_ids = list(
            ChannelMetadata.objects.all().values_list("id", flat=True)
        )
        all_channel_ids = set(storage_channel_ids + database_channel_ids)
        for channel_id in all_channel_ids:
            if channel_id not in database_channel_ids:
                try:
                    import_channel_from_local_db(channel_id)
                    annotate_content(channel_id)
                except (InvalidSchemaVersionError, FutureSchemaError):
                    logger.warning(
                        "Tried to import channel {channel_id}, but database file was incompatible".format(
                            channel_id=channel_id
                        )
                    )
            else:
                annotate_content(channel_id)
<commit_msg>Propagate corrupt database handling to scanforcontent command.<commit_after>import logging

from django.core.management.base import BaseCommand
from sqlalchemy.exc import DatabaseError

from ...utils.annotation import annotate_content
from ...utils.channel_import import FutureSchemaError
from ...utils.channel_import import import_channel_from_local_db
from ...utils.channel_import import InvalidSchemaVersionError
from ...utils.channels import get_channel_ids_for_content_database_dir
from kolibri.core.content.models import ChannelMetadata
from kolibri.core.content.utils.paths import get_content_database_dir_path

logger = logging.getLogger(__name__)

class Command(BaseCommand):
    """
    This command will try to fix the content tables in the database,
    reviewing the availability of all the existing files, to recover
    content that's not visible in Kolibri.
    """

    help = "Scan content and databases in Kolibri folder and updates the database to show if available"

    def handle(self, *args, **options):
        storage_channel_ids = get_channel_ids_for_content_database_dir(
            get_content_database_dir_path()
        )
        database_channel_ids = list(
            ChannelMetadata.objects.all().values_list("id", flat=True)
        )
        all_channel_ids = set(storage_channel_ids + database_channel_ids)
        for channel_id in all_channel_ids:
            if channel_id not in database_channel_ids:
                try:
                    import_channel_from_local_db(channel_id)
                    annotate_content(channel_id)
                except (InvalidSchemaVersionError, FutureSchemaError):
                    logger.warning(
                        "Tried to import channel {channel_id}, but database file was incompatible".format(
                            channel_id=channel_id
                        )
                    )
                except DatabaseError:
                    logger.warning(
                        "Tried to import channel {channel_id}, but database file was corrupted.".format(
                            channel_id=channel_id
                        )
                    )
            else:
                annotate_content(channel_id)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Addons modules by CLEARCORP S.A.
#    Copyright (C) 2009-TODAY CLEARCORP S.A. (<http://clearcorp.co.cr>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
"""
Parser for Davivienda format files
"""
import re
from datetime import datetime
from dateutil import parser
from pprint import PrettyPrinter
from copy import copy
from openerp.osv import osv, fields
from openerp.tools.translate import _

class DaviviendaParser( object ):

    #Define the header for the extract to import.
    '''
     ** Kwargs parameter is used for a dynamic list of parameters.
        The wizard imported extracts used in all parsers and not all parsers have all the necessary information in your file,
        so get information from the wizard and passed by the ** kwargs.
        Then in the parses that are needed, are extracted from the ** kwargs and if needed,
        the parser still works the same way without this parameter.

        The rest of the methods must receive this parameter. (As the method that parse the header and the lines).

        If you need a new parameter, you specify its name and value, using the ** kwargs is a dictionary,
        extract its value, with the respective key
    '''
    def statement_record ( self, rec, **kwargs):
        lines = []
        line_dict = {}

        line_dict = {
            'transref': '', # _transmission_number
            'account_number': '', #_account_number
            'statementnr':'', # statement_number
            'startingbalance': 0.0, #_opening_balance
            'currencycode': '', #currencycode
            'endingbalance': 0.0, #_closing_balance
            'bookingdate': '', #moving_date
            'amount': 0.0,
            'id': '',
        }

        #Separe the file in statements
        list_split = rec.split('\n')
        #Obtain the first line to know the account number
        fist_line = list_split[1]
        first_line_split = fist_line.split(';')

        account_number_wizard = kwargs['account_number']#from wizard
        account_number_file = first_line_split[11]#from file.

        #if the account_number in the file match with the account selected in the wizard, return True
        if account_number_file.find(account_number_wizard) > -1:
            #currency_code (local_currency in the stament) extracted from account_number object from the wizard.
            #account_number (local_account) extracted from account_number object from the wizard.
            #date_to_str and date_from_str are the dates in wizard, both are strings
            #the parameters come from davivienda_format in parser class.
            line_dict['account_number'] = kwargs['account_number']

            line_dict['currencycode'] = kwargs['local_currency']

            line_dict['statementnr'] = kwargs['date_from_str'] + ' - '+ kwargs['date_to_str'] + 'Extracto Davivienda ' + line_dict['account_number'] #Interval time of the file.

            startingbalance = endingbalance = 0.0

            #transmission_number (Date when done the import)
            date_obj= datetime.now()
            line_dict['transref'] = date_obj.strftime("%d-%m-%Y %H:%M:%S")
            #bookingdate
            line_dict['bookingdate'] = date_obj.strftime("%d-%m-%Y %H:%M:%S")

            #with the first line compute the initial_balance
            fist_line = list_split[1]
            first_line_split = fist_line.split(';')
            startingbalance = float(first_line_split[5].replace(",","")) + float(first_line_split[3].replace(",","")) - float(first_line_split[4].replace(",",""))
            line_dict['startingbalance'] =  str(startingbalance)

            #the ending_balance is the balance of the last line.
            last_position = (len(list_split) - 1)
            last_line = list_split[last_position]
            #last line can be blanck, find the last line with data.
            if last_line == "":
                while True:
                    last_position -= 1
                    last_line = list_split[last_position]
                    if len(last_line) > 0 and last_line != "" and last_line != '\r':
                        break
            last_line_split = last_line.split(';')
            endingbalance += float(last_line_split[5].replace(",",""))
            line_dict['endingbalance'] =  str(endingbalance)

            line_dict['amount'] = str(startingbalance + endingbalance)
            line_dict['id'] = kwargs['date_from_str'] + ' - '+ kwargs['date_to_str'] + ' Extracto Davivienda ' + line_dict['account_number']

            return line_dict

        else:
            raise osv.except_osv(_('Error'),
                        _('Error en la importación! La cuenta especificada en el archivo no coincide con la seleccionada en el asistente de importacion'))

    '''
    Parse all the lines in the file. Once the header is parser, the next step are the lines.
    '''
    def statement_lines ( self, rec ):
        parser = DaviviendaParser()
        mapping = {
            'execution_date' : '',
            'effective_date' : '',
            'transfer_type' : '',
            'reference' : '',
            'message' : '',
            'name' : '',
            'transferred_amount': '',
            'creditmarker': '',
        }

        lines = []
        line_dict = {}
        currencycode = ''

        list_split = rec.split('\n')
        entrada = False

        start = 1
        end = (len(list_split) - 1)
        last_line = list_split[end]
        #last line can be blanck, find the last line with data.
        if last_line == "":
            while True:
                end -= 1
                last_line = list_split[end]
                if last_line is not "":
                    break

        sub_list = list_split [start:end]
        for sub in sub_list:
            line = sub.split(';')
            #effective_date
            date_str = line[0].replace("/","-")
            date= datetime.strptime(date_str, "%d-%m-%Y")
            mapping['effective_date'] = date #fecha_contable.
            #execution_date
            mapping['execution_date'] = date #fecha_movimiento

            mapping['transfer_type'] = 'NTRF'
            mapping['reference'] = line[2] #Ref 1
            mapping['message'] = line[1] #Description
            mapping['name'] = line[1] #Description
            mapping['id'] = line[1] #Description

            if line[10] == 'C':
                credit = float(line[4].replace(",",""))
                mapping['transferred_amount'] = credit
                mapping['creditmarker'] = 'C'

            else:
                #In this case, the debit is negative.
                debit = float(line[3].replace(",",""))
                mapping['transferred_amount'] =  -1 * debit

            lines.append(copy(mapping))

        return lines

    #clear special characters in a row.
    def clean_special_characters(self, text_celd):
        special_characters = {'\r':''}

        for i, j in special_characters.iteritems():
            text = text_celd.replace(i, j)

        #remove all the blank space.
        return re.sub(r'\s', '', text)

    """
    ** Kwargs parameter is used for a dynamic list of parameters.
        The wizard imported extracts used in all parsers and not all parsers have all the necessary information in your file,
        so get information from the wizard and passed by the ** kwargs.
        Then in the parses that are needed, are extracted from the ** kwargs and if needed,
        the parser still works the same way without this parameter.

        The rest of the methods must receive this parameter. (As the method that parse the header and the lines).

        If you need a new parameter, you specify its name and value, using the ** kwargs is a dictionary,
        extract its value, with the respective key
    """

    def parse_stamenent_record( self, rec, **kwargs): #parse the header.

        matchdict = dict()

        #Set the header for the stament.
        matchdict = self.statement_record(rec, **kwargs);

        # Remove members set to None
        #matchdict = dict( [( k, v ) for k, v in matchdict.iteritems() if v] )

        matchkeys = set( matchdict.keys() )
        needstrip = set( [ 'transref', 'account_number', 'statementnr', 'currencycode', 'endingbalance', 'bookingdate'] )

        for field in matchkeys & needstrip:
            matchdict[field] = matchdict[field].strip()

        # Convert to float. Comma is decimal separator
        needsfloat = set( ["startingbalance", "endingbalance", "amount"] )
        for field in matchkeys & needsfloat:
            matchdict[field] = float( matchdict[field].replace( ',', '.' ) )

        # Convert date fields
        needdate = set( ["bookingdate", "effective_date", "execution_date"] )

        for field in matchkeys & needdate:
            datestring = matchdict[field]
            date_obj= datetime.strptime(datestring, "%d-%m-%Y %H:%M:%S")
            matchdict[field] = date_obj

        return matchdict

    #call the method that parse the header and the statements.
    def parse( self, cr, data ):
        records = []
        # Some records are multiline
        for line in data:
            if len(line) <= 1:
                continue
            if line[0] == ':' and len(line) > 1:
                records.append(line)
            else:
                records[-1] = '\n'.join([records[-1], line])

        output = []

        for rec in records:
            #parse_stamenent_record parse the header and the statements
            output.append( self.parse_stamenent_record( rec ) )

        return output

<commit_msg>[FIX] l10n_cr_account_banking_cr_davivienda: Fixed missing line importing DaVivienda's bank statements file<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Addons modules by CLEARCORP S.A.
#    Copyright (C) 2009-TODAY CLEARCORP S.A. (<http://clearcorp.co.cr>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
"""
Parser for Davivienda format files
"""
import re
from datetime import datetime
from dateutil import parser
from pprint import PrettyPrinter
from copy import copy
from openerp.osv import osv, fields
from openerp.tools.translate import _

class DaviviendaParser( object ):

    #Define the header for the extract to import.
    '''
     ** Kwargs parameter is used for a dynamic list of parameters.
        The wizard imported extracts used in all parsers and not all parsers have all the necessary information in your file,
        so get information from the wizard and passed by the ** kwargs.
        Then in the parses that are needed, are extracted from the ** kwargs and if needed,
        the parser still works the same way without this parameter.

        The rest of the methods must receive this parameter. (As the method that parse the header and the lines).

        If you need a new parameter, you specify its name and value, using the ** kwargs is a dictionary,
        extract its value, with the respective key
    '''
    def statement_record ( self, rec, **kwargs):
        lines = []
        line_dict = {}

        line_dict = {
            'transref': '', # _transmission_number
            'account_number': '', #_account_number
            'statementnr':'', # statement_number
            'startingbalance': 0.0, #_opening_balance
            'currencycode': '', #currencycode
            'endingbalance': 0.0, #_closing_balance
            'bookingdate': '', #moving_date
            'amount': 0.0,
            'id': '',
        }

        #Separe the file in statements
        list_split = rec.split('\n')
        #Obtain the first line to know the account number
        fist_line = list_split[1]
        first_line_split = fist_line.split(';')

        account_number_wizard = kwargs['account_number']#from wizard
        account_number_file = first_line_split[11]#from file.

        #if the account_number in the file match with the account selected in the wizard, return True
        if account_number_file.find(account_number_wizard) > -1:
            #currency_code (local_currency in the stament) extracted from account_number object from the wizard.
            #account_number (local_account) extracted from account_number object from the wizard.
            #date_to_str and date_from_str are the dates in wizard, both are strings
            #the parameters come from davivienda_format in parser class.
            line_dict['account_number'] = kwargs['account_number']

            line_dict['currencycode'] = kwargs['local_currency']

            line_dict['statementnr'] = kwargs['date_from_str'] + ' - '+ kwargs['date_to_str'] + 'Extracto Davivienda ' + line_dict['account_number'] #Interval time of the file.

            startingbalance = endingbalance = 0.0

            #transmission_number (Date when done the import)
            date_obj= datetime.now()
            line_dict['transref'] = date_obj.strftime("%d-%m-%Y %H:%M:%S")
            #bookingdate
            line_dict['bookingdate'] = date_obj.strftime("%d-%m-%Y %H:%M:%S")

            #with the first line compute the initial_balance
            fist_line = list_split[1]
            first_line_split = fist_line.split(';')
            startingbalance = float(first_line_split[5].replace(",","")) + float(first_line_split[3].replace(",","")) - float(first_line_split[4].replace(",",""))
            line_dict['startingbalance'] =  str(startingbalance)

            #the ending_balance is the balance of the last line.
            last_position = (len(list_split) - 1)
            last_line = list_split[last_position]
            #last line can be blanck, find the last line with data.
            if last_line == "":
                while True:
                    last_position -= 1
                    last_line = list_split[last_position]
                    if len(last_line) > 0 and last_line != "" and last_line != '\r':
                        break
            last_line_split = last_line.split(';')
            endingbalance += float(last_line_split[5].replace(",",""))
            line_dict['endingbalance'] =  str(endingbalance)

            line_dict['amount'] = str(startingbalance + endingbalance)
            line_dict['id'] = kwargs['date_from_str'] + ' - '+ kwargs['date_to_str'] + ' Extracto Davivienda ' + line_dict['account_number']

            return line_dict

        else:
            raise osv.except_osv(_('Error'),
                        _('Error en la importación! La cuenta especificada en el archivo no coincide con la seleccionada en el asistente de importacion'))

    '''
    Parse all the lines in the file. Once the header is parser, the next step are the lines.
    '''
    def statement_lines ( self, rec ):
        parser = DaviviendaParser()
        mapping = {
            'execution_date' : '',
            'effective_date' : '',
            'transfer_type' : '',
            'reference' : '',
            'message' : '',
            'name' : '',
            'transferred_amount': '',
            'creditmarker': '',
        }

        lines = []
        line_dict = {}
        currencycode = ''

        list_split = rec.split('\n')
        entrada = False

        start = 1
        end = (len(list_split) - 1)
        last_line = list_split[end]
        #last line can be blanck, find the last line with data.
        if last_line == "":
            while True:
                last_line = list_split[end]
                if last_line is not "":
                    break
                end -= 1
        sub_list = list_split [start:end+1]
        for sub in sub_list:
            line = sub.split(';')
            #effective_date
            date_str = line[0].replace("/","-")
            date= datetime.strptime(date_str, "%d-%m-%Y")
            mapping['effective_date'] = date #fecha_contable.
            #execution_date
            mapping['execution_date'] = date #fecha_movimiento

            mapping['transfer_type'] = 'NTRF'
            mapping['reference'] = line[2] #Ref 1
            mapping['message'] = line[1] #Description
            mapping['name'] = line[1] #Description
            mapping['id'] = line[1] #Description

            if line[10] == 'C':
                credit = float(line[4].replace(",",""))
                mapping['transferred_amount'] = credit
                mapping['creditmarker'] = 'C'

            else:
                #In this case, the debit is negative.
                debit = float(line[3].replace(",",""))
                mapping['transferred_amount'] =  -1 * debit

            lines.append(copy(mapping))

        return lines

    #clear special characters in a row.
    def clean_special_characters(self, text_celd):
        special_characters = {'\r':''}

        for i, j in special_characters.iteritems():
            text = text_celd.replace(i, j)

        #remove all the blank space.
        return re.sub(r'\s', '', text)

    """
    ** Kwargs parameter is used for a dynamic list of parameters.
        The wizard imported extracts used in all parsers and not all parsers have all the necessary information in your file,
        so get information from the wizard and passed by the ** kwargs.
        Then in the parses that are needed, are extracted from the ** kwargs and if needed,
        the parser still works the same way without this parameter.

        The rest of the methods must receive this parameter. (As the method that parse the header and the lines).

        If you need a new parameter, you specify its name and value, using the ** kwargs is a dictionary,
        extract its value, with the respective key
    """

    def parse_stamenent_record( self, rec, **kwargs): #parse the header.

        matchdict = dict()

        #Set the header for the stament.
        matchdict = self.statement_record(rec, **kwargs);

        # Remove members set to None
        #matchdict = dict( [( k, v ) for k, v in matchdict.iteritems() if v] )

        matchkeys = set( matchdict.keys() )
        needstrip = set( [ 'transref', 'account_number', 'statementnr', 'currencycode', 'endingbalance', 'bookingdate'] )

        for field in matchkeys & needstrip:
            matchdict[field] = matchdict[field].strip()

        # Convert to float. Comma is decimal separator
        needsfloat = set( ["startingbalance", "endingbalance", "amount"] )
        for field in matchkeys & needsfloat:
            matchdict[field] = float( matchdict[field].replace( ',', '.' ) )

        # Convert date fields
        needdate = set( ["bookingdate", "effective_date", "execution_date"] )

        for field in matchkeys & needdate:
            datestring = matchdict[field]
            date_obj= datetime.strptime(datestring, "%d-%m-%Y %H:%M:%S")
            matchdict[field] = date_obj

        return matchdict

    #call the method that parse the header and the statements.
    def parse( self, cr, data ):
        records = []
        # Some records are multiline
        for line in data:
            if len(line) <= 1:
                continue
            if line[0] == ':' and len(line) > 1:
                records.append(line)
            else:
                records[-1] = '\n'.join([records[-1], line])

        output = []

        for rec in records:
            #parse_stamenent_record parse the header and the statements
            output.append( self.parse_stamenent_record( rec ) )

        return output

<|endoftext|>