<commit_before>from django.contrib.contenttypes.fields import GenericRelation
from rdflib import Graph, BNode
from rdflib.collection import Collection
from rdflib.namespace import Namespace, NamespaceManager, DC, DCTERMS, RDF, RDFS
from rdflib.plugin import register
from rdflib.plugins.serializers.rdfxml import PrettyXMLSerializer, XMLLANG, OWL_NS, XMLBASE
from rdflib.plugins.serializers.xmlwriter import XMLWriter
from rdflib.serializer import Serializer
from rdflib.term import Literal, URIRef
from rdflib.py3compat import b
from rdflib.util import first

HSTERMS = Namespace("http://hydroshare.org/terms/")
RDFS1 = Namespace("http://www.w3.org/2000/01/rdf-schema#")

NAMESPACE_MANAGER = NamespaceManager(Graph())
NAMESPACE_MANAGER.bind('hsterms', HSTERMS, override=False)
NAMESPACE_MANAGER.bind("rdfs1", RDFS1, override=False)
NAMESPACE_MANAGER.bind('dc', DC, override=False)
NAMESPACE_MANAGER.bind('dcterms', DCTERMS, override=False)

class RDF_MetaData_Mixin(object):

    def rdf_subject(self):
        raise NotImplementedError("RDF_Metadata_Mixin implementations must implement rdf_subject")

    def ingest_metadata(self, graph):
        subject = self.rdf_subject()

        for s, p, o in graph.triples((subject, HSTERMS.extendedMetadata, None)):
            key = graph.value(subject=o, predicate=HSTERMS.key).value
            value = graph.value(subject=o, predicate=HSTERMS.value).value
            self.resource.extra_metadata[key] = value

        for field in self.__class__._meta.fields:
            if field.name in ['id', 'object_id', 'content_type', 'extra_metadata', 'is_dirty']:
                continue
            field_value = graph.value(subject=subject, predicate=getattr(HSTERMS, field.name))
            if field_value:
                setattr(self, field.name, field_value)

        generic_relations = list(filter(lambda f: isinstance(f, GenericRelation), type(self)._meta.virtual_fields))
        for generic_relation in generic_relations:
            generic_relation.related_model.ingest_rdf(graph, self)
        self.save()

    def get_rdf_graph(self):
        graph = Graph()
        graph.namespace_manager = NAMESPACE_MANAGER

        subject = self.rdf_subject()

        # add any key/value metadata items
        if len(self.resource.extra_metadata) > 0:
            extendedMetadata = BNode()
            graph.add((subject, HSTERMS.extendedMetadata, extendedMetadata))
            for key, value in list(self.resource.extra_metadata.items()):
                graph.add((extendedMetadata, HSTERMS.key, Literal(key)))
                graph.add((extendedMetadata, HSTERMS.value, Literal(value)))

        for field in self.__class__._meta.fields:
            if field.name in ['id', 'object_id', 'content_type', 'extra_metadata', 'is_dirty']:
                continue
            graph.add((subject, getattr(HSTERMS, field.name), Literal(field.value_from_object(self))))

        generic_relations = list(filter(lambda f: isinstance(f, GenericRelation), type(self)._meta.virtual_fields))
        for generic_relation in generic_relations:
            for f in getattr(self, getattr(generic_relation, 'name', None), None).all():
                f.rdf_triples(subject, graph)

        from .hydroshare import current_site_url
        TYPE_SUBJECT = getattr(Namespace("{}/terms/".format(current_site_url())), self.resource.__class__.__name__)
        graph.add((TYPE_SUBJECT, RDFS1.label, Literal(self.resource._meta.verbose_name)))
        graph.add((TYPE_SUBJECT, RDFS1.isDefinedBy, URIRef(HSTERMS)))
        return graph

    def get_xml(self, pretty_print=True, include_format_elements=True):
        """Generates ORI+RDF xml for this aggregation metadata"""

        # get the xml root element and the xml element to which contains all other elements
        g = self.get_rdf_graph()
        return g.serialize(format='hydro-xml').decode()

class RDF_Term_MixIn(object):
    """Provides methods for serializing a django model into and rdflib triples and deserializing from an rdflib Graph
     back to a django model.  This mixin is designed for django models that are generic relations of another django
     model which represents a group of metadata.

     Without any configuration, the rdf_triples(subject) method will add all fields on the model except
     for 'id', 'object_id', 'content_type' will be serialized as rdf triples.  If rdf_term is defined, it will be
     used as the predicate.  If rdf_term is not defined, one will be defined using the class name and the
     HSTERMS namespace.

     When a model requires an rdf-xml format that the default configuration will not provide, you may overwrite
     the rdf_triples method and the ingest_rdf method.  If you overwrite one, you must overwrite both or this mixin
     will not work correctly.
     """

    ignored_fields = ['id', 'object_id', 'content_type']

    def rdf_triples(self, subject, graph):
        """Default implementation that parses by convention."""
        term = self.rdf_term if hasattr(self, 'rdf_term') else getattr(HSTERMS, self.__class__.__name__)
        metadata_node = BNode()
        graph.add((subject, term, metadata_node))
        for field in self.__class__._meta.fields:
            if self.ignored_fields and field.name in self.ignored_fields:
                continue
            field_term_attr = field.name + '_rdf_term'
            if hasattr(self, field_term_attr):
                field_term = getattr(self, field_term_attr)
            else:
                field_term = getattr(HSTERMS, field.name)
            field_value = getattr(self, field.name)
            # urls should be a URIRef term, all others should be a Literal term
            if field_value and field_value != 'None':
                if isinstance(field_value, str) and field_value.startswith('http'):
                    field_value = URIRef(field_value)
                else:
                    field_value = Literal(field_value)
                graph.add((metadata_node, field_term, field_value))

    @classmethod
    def ingest_rdf(cls, graph, content_object):
        """Default implementation that ingests by convention"""
        term = cls.rdf_term if hasattr(cls, 'rdf_term') else getattr(HSTERMS, cls.__name__)
        value_dict = {}
        subject = content_object.rdf_subject()
        metadata_nodes = graph.objects(subject=subject, predicate=term)
        for metadata_node in metadata_nodes:
            for field in cls._meta.fields:
                if cls.ignored_fields and field.name in cls.ignored_fields:
                    continue
                field_term_attr = field.name + '_rdf_term'
                if hasattr(cls, field_term_attr):
                    field_term = getattr(cls, field_term_attr)
                else:
                    field_term = getattr(HSTERMS, field.name)
                val = graph.value(metadata_node, field_term)
                if val:
                    value_dict[field.name] = val.value if isinstance(val, Literal) else str(val)
            if value_dict:
                cls.create(content_object=content_object, **value_dict)

def rdf_terms(class_term, **field_terms):
    def decorator(obj):
        if class_term:
            obj.rdf_term = class_term
        for k, v in field_terms.items():
            if not hasattr(obj, k):
                raise Exception("field {} not found".format(k))
            setattr(obj, k + '_rdf_term', v)
        return obj
    return decorator

class HydroPrettyXMLSerializer(Serializer):
    """Same as PrettyXMLSerializer but with stripped out node ids"""

    def __init__(self, store, max_depth=3):
        super(PrettyXMLSerializer, self).__init__(store)
        self.forceRDFAbout = set()

    def serialize(self, stream, base=None, encoding=None, **args):
        self.__serialized = {}
        store = self.store
        self.base = base
        self.max_depth = args.get("max_depth", 3)
        assert self.max_depth > 0, "max_depth must be greater than 0"

        self.nm = nm = store.namespace_manager
        self.writer = writer = XMLWriter(stream, nm, encoding)
        namespaces = {}

        possible = set(store.predicates()).union(
            store.objects(None, RDF.type))

        for predicate in possible:
            prefix, namespace, local = nm.compute_qname(predicate)
            namespaces[prefix] = namespace

        namespaces["rdf"] = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"

        writer.push(RDF.RDF)

        if "xml_base" in args:
            writer.attribute(XMLBASE, args["xml_base"])

        writer.namespaces(iter(list(namespaces.items())))

        # Write out subjects that can not be inline
        for subject in store.subjects():
            if (None, None, subject) in store:
                if (subject, None, subject) in store:
                    self.subject(subject, 1)
            else:
                self.subject(subject, 1)

        # write out anything that has not yet been reached
        # write out BNodes last (to ensure they can be inlined where possible)
        bnodes = set()

        for subject in store.subjects():
            if isinstance(subject, BNode):
                bnodes.add(subject)
                continue
            self.subject(subject, 1)

        # now serialize only those BNodes that have not been serialized yet
        for bnode in bnodes:
            if bnode not in self.__serialized:
                self.subject(subject, 1)

        writer.pop(RDF.RDF)
        stream.write(b("\n"))

        # Set to None so that the memory can get garbage collected.
        self.__serialized = None

    def subject(self, subject, depth=1):
        store = self.store
        writer = self.writer

        if subject in self.forceRDFAbout:
            writer.push(RDF.Description)
            writer.attribute(RDF.about, self.relativize(subject))
            writer.pop(RDF.Description)
            self.forceRDFAbout.remove(subject)

        elif not subject in self.__serialized:
            self.__serialized[subject] = 1
            type = first(store.objects(subject, RDF.type))

            try:
                self.nm.qname(type)
            except:
                type = None

            element = type or RDF.Description
            writer.push(element)

            if isinstance(subject, BNode):
                def subj_as_obj_more_than(ceil):
                    return True
                    # more_than(store.triples((None, None, subject)), ceil)

            else:
                writer.attribute(RDF.about, self.relativize(subject))

            if (subject, None, None) in store:
                for predicate, object in store.predicate_objects(subject):
                    if not (predicate == RDF.type and object == type):
                        self.predicate(predicate, object, depth + 1)

            writer.pop(element)

        elif subject in self.forceRDFAbout:
            writer.push(RDF.Description)
            writer.attribute(RDF.about, self.relativize(subject))
            writer.pop(RDF.Description)
            self.forceRDFAbout.remove(subject)

    def predicate(self, predicate, object, depth=1):
        writer = self.writer
        store = self.store
        writer.push(predicate)

        if isinstance(object, Literal):
            if object.language:
                writer.attribute(XMLLANG, object.language)

            if object.datatype:
                writer.attribute(RDF.datatype, object.datatype)

            writer.text(object)

        elif object in self.__serialized or not (object, None, None) in store:

            if isinstance(object, BNode):
                pass
            else:
                writer.attribute(RDF.resource, self.relativize(object))

        else:
            if first(store.objects(object, RDF.first)):  # may not have type
                                                         # RDF.List

                self.__serialized[object] = 1

                # Warn that any assertions on object other than
                # RDF.first and RDF.rest are ignored... including RDF.List
                import warnings
                warnings.warn(
                    "Assertions on %s other than RDF.first " % repr(object) +
                    "and RDF.rest are ignored ... including RDF.List",
                    UserWarning, stacklevel=2)
                writer.attribute(RDF.parseType, "Collection")

                col = Collection(store, object)

                for item in col:

                    if isinstance(item, URIRef):
                        self.forceRDFAbout.add(item)
                    self.subject(item)

                    if not isinstance(item, URIRef):
                        self.__serialized[item] = 1
            else:
                if first(store.triples_choices(
                    (object, RDF.type, [OWL_NS.Class, RDFS.Class]))) \
                        and isinstance(object, URIRef):
                    writer.attribute(RDF.resource, self.relativize(object))

                elif depth <= self.max_depth:
                    self.subject(object, depth + 1)

                elif isinstance(object, BNode):

                    if not object in self.__serialized \
                            and (object, None, None) in store \
                            and len(list(store.subjects(object=object))) == 1:
                        # inline blank nodes if they haven't been serialized yet
                        # and are only referenced once (regardless of depth)
                        self.subject(object, depth + 1)

                else:
                    writer.attribute(RDF.resource, self.relativize(object))

        writer.pop(predicate)

register(
    'hydro-xml', Serializer,
    'hs_core.hs_rdf', 'HydroPrettyXMLSerializer')
<commit_msg>Lookup super of hydropretty<commit_after>from django.contrib.contenttypes.fields import GenericRelation
from rdflib import Graph, BNode
from rdflib.collection import Collection
from rdflib.namespace import Namespace, NamespaceManager, DC, DCTERMS, RDF, RDFS
from rdflib.plugin import register
from rdflib.plugins.serializers.rdfxml import PrettyXMLSerializer, XMLLANG, OWL_NS, XMLBASE
from rdflib.plugins.serializers.xmlwriter import XMLWriter
from rdflib.serializer import Serializer
from rdflib.term import Literal, URIRef
from rdflib.py3compat import b
from rdflib.util import first

HSTERMS = Namespace("http://hydroshare.org/terms/")
RDFS1 = Namespace("http://www.w3.org/2000/01/rdf-schema#")

NAMESPACE_MANAGER = NamespaceManager(Graph())
NAMESPACE_MANAGER.bind('hsterms', HSTERMS, override=False)
NAMESPACE_MANAGER.bind("rdfs1", RDFS1, override=False)
NAMESPACE_MANAGER.bind('dc', DC, override=False)
NAMESPACE_MANAGER.bind('dcterms', DCTERMS, override=False)

class RDF_MetaData_Mixin(object):

    def rdf_subject(self):
        raise NotImplementedError("RDF_Metadata_Mixin implementations must implement rdf_subject")

    def ingest_metadata(self, graph):
        subject = self.rdf_subject()

        for s, p, o in graph.triples((subject, HSTERMS.extendedMetadata, None)):
            key = graph.value(subject=o, predicate=HSTERMS.key).value
            value = graph.value(subject=o, predicate=HSTERMS.value).value
            self.resource.extra_metadata[key] = value

        for field in self.__class__._meta.fields:
            if field.name in ['id', 'object_id', 'content_type', 'extra_metadata', 'is_dirty']:
                continue
            field_value = graph.value(subject=subject, predicate=getattr(HSTERMS, field.name))
            if field_value:
                setattr(self, field.name, field_value)

        generic_relations = list(filter(lambda f: isinstance(f, GenericRelation), type(self)._meta.virtual_fields))
        for generic_relation in generic_relations:
            generic_relation.related_model.ingest_rdf(graph, self)
        self.save()

    def get_rdf_graph(self):
        graph = Graph()
        graph.namespace_manager = NAMESPACE_MANAGER

        subject = self.rdf_subject()

        # add any key/value metadata items
        if len(self.resource.extra_metadata) > 0:
            extendedMetadata = BNode()
            graph.add((subject, HSTERMS.extendedMetadata, extendedMetadata))
            for key, value in list(self.resource.extra_metadata.items()):
                graph.add((extendedMetadata, HSTERMS.key, Literal(key)))
                graph.add((extendedMetadata, HSTERMS.value, Literal(value)))

        for field in self.__class__._meta.fields:
            if field.name in ['id', 'object_id', 'content_type', 'extra_metadata', 'is_dirty']:
                continue
            graph.add((subject, getattr(HSTERMS, field.name), Literal(field.value_from_object(self))))

        generic_relations = list(filter(lambda f: isinstance(f, GenericRelation), type(self)._meta.virtual_fields))
        for generic_relation in generic_relations:
            for f in getattr(self, getattr(generic_relation, 'name', None), None).all():
                f.rdf_triples(subject, graph)

        from .hydroshare import current_site_url
        TYPE_SUBJECT = getattr(Namespace("{}/terms/".format(current_site_url())), self.resource.__class__.__name__)
        graph.add((TYPE_SUBJECT, RDFS1.label, Literal(self.resource._meta.verbose_name)))
        graph.add((TYPE_SUBJECT, RDFS1.isDefinedBy, URIRef(HSTERMS)))
        return graph

    def get_xml(self, pretty_print=True, include_format_elements=True):
        """Generates ORI+RDF xml for this aggregation metadata"""

        # get the xml root element and the xml element to which contains all other elements
        g = self.get_rdf_graph()
        return g.serialize(format='hydro-xml').decode()

class RDF_Term_MixIn(object):
    """Provides methods for serializing a django model into and rdflib triples and deserializing from an rdflib Graph
     back to a django model.  This mixin is designed for django models that are generic relations of another django
     model which represents a group of metadata.

     Without any configuration, the rdf_triples(subject) method will add all fields on the model except
     for 'id', 'object_id', 'content_type' will be serialized as rdf triples.  If rdf_term is defined, it will be
     used as the predicate.  If rdf_term is not defined, one will be defined using the class name and the
     HSTERMS namespace.

     When a model requires an rdf-xml format that the default configuration will not provide, you may overwrite
     the rdf_triples method and the ingest_rdf method.  If you overwrite one, you must overwrite both or this mixin
     will not work correctly.
     """

    ignored_fields = ['id', 'object_id', 'content_type']

    def rdf_triples(self, subject, graph):
        """Default implementation that parses by convention."""
        term = self.rdf_term if hasattr(self, 'rdf_term') else getattr(HSTERMS, self.__class__.__name__)
        metadata_node = BNode()
        graph.add((subject, term, metadata_node))
        for field in self.__class__._meta.fields:
            if self.ignored_fields and field.name in self.ignored_fields:
                continue
            field_term_attr = field.name + '_rdf_term'
            if hasattr(self, field_term_attr):
                field_term = getattr(self, field_term_attr)
            else:
                field_term = getattr(HSTERMS, field.name)
            field_value = getattr(self, field.name)
            # urls should be a URIRef term, all others should be a Literal term
            if field_value and field_value != 'None':
                if isinstance(field_value, str) and field_value.startswith('http'):
                    field_value = URIRef(field_value)
                else:
                    field_value = Literal(field_value)
                graph.add((metadata_node, field_term, field_value))

    @classmethod
    def ingest_rdf(cls, graph, content_object):
        """Default implementation that ingests by convention"""
        term = cls.rdf_term if hasattr(cls, 'rdf_term') else getattr(HSTERMS, cls.__name__)
        value_dict = {}
        subject = content_object.rdf_subject()
        metadata_nodes = graph.objects(subject=subject, predicate=term)
        for metadata_node in metadata_nodes:
            for field in cls._meta.fields:
                if cls.ignored_fields and field.name in cls.ignored_fields:
                    continue
                field_term_attr = field.name + '_rdf_term'
                if hasattr(cls, field_term_attr):
                    field_term = getattr(cls, field_term_attr)
                else:
                    field_term = getattr(HSTERMS, field.name)
                val = graph.value(metadata_node, field_term)
                if val:
                    value_dict[field.name] = val.value if isinstance(val, Literal) else str(val)
            if value_dict:
                cls.create(content_object=content_object, **value_dict)

def rdf_terms(class_term, **field_terms):
    def decorator(obj):
        if class_term:
            obj.rdf_term = class_term
        for k, v in field_terms.items():
            if not hasattr(obj, k):
                raise Exception("field {} not found".format(k))
            setattr(obj, k + '_rdf_term', v)
        return obj
    return decorator

class HydroPrettyXMLSerializer(Serializer):
    """Same as PrettyXMLSerializer but with stripped out node ids"""

    def __init__(self, store, max_depth=3):
        super(HydroPrettyXMLSerializer, self).__init__(store)
        self.forceRDFAbout = set()

    def serialize(self, stream, base=None, encoding=None, **args):
        self.__serialized = {}
        store = self.store
        self.base = base
        self.max_depth = args.get("max_depth", 3)
        assert self.max_depth > 0, "max_depth must be greater than 0"

        self.nm = nm = store.namespace_manager
        self.writer = writer = XMLWriter(stream, nm, encoding)
        namespaces = {}

        possible = set(store.predicates()).union(
            store.objects(None, RDF.type))

        for predicate in possible:
            prefix, namespace, local = nm.compute_qname(predicate)
            namespaces[prefix] = namespace

        namespaces["rdf"] = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"

        writer.push(RDF.RDF)

        if "xml_base" in args:
            writer.attribute(XMLBASE, args["xml_base"])

        writer.namespaces(iter(list(namespaces.items())))

        # Write out subjects that can not be inline
        for subject in store.subjects():
            if (None, None, subject) in store:
                if (subject, None, subject) in store:
                    self.subject(subject, 1)
            else:
                self.subject(subject, 1)

        # write out anything that has not yet been reached
        # write out BNodes last (to ensure they can be inlined where possible)
        bnodes = set()

        for subject in store.subjects():
            if isinstance(subject, BNode):
                bnodes.add(subject)
                continue
            self.subject(subject, 1)

        # now serialize only those BNodes that have not been serialized yet
        for bnode in bnodes:
            if bnode not in self.__serialized:
                self.subject(subject, 1)

        writer.pop(RDF.RDF)
        stream.write(b("\n"))

        # Set to None so that the memory can get garbage collected.
        self.__serialized = None

    def subject(self, subject, depth=1):
        store = self.store
        writer = self.writer

        if subject in self.forceRDFAbout:
            writer.push(RDF.Description)
            writer.attribute(RDF.about, self.relativize(subject))
            writer.pop(RDF.Description)
            self.forceRDFAbout.remove(subject)

        elif not subject in self.__serialized:
            self.__serialized[subject] = 1
            type = first(store.objects(subject, RDF.type))

            try:
                self.nm.qname(type)
            except:
                type = None

            element = type or RDF.Description
            writer.push(element)

            if isinstance(subject, BNode):
                def subj_as_obj_more_than(ceil):
                    return True
                    # more_than(store.triples((None, None, subject)), ceil)

            else:
                writer.attribute(RDF.about, self.relativize(subject))

            if (subject, None, None) in store:
                for predicate, object in store.predicate_objects(subject):
                    if not (predicate == RDF.type and object == type):
                        self.predicate(predicate, object, depth + 1)

            writer.pop(element)

        elif subject in self.forceRDFAbout:
            writer.push(RDF.Description)
            writer.attribute(RDF.about, self.relativize(subject))
            writer.pop(RDF.Description)
            self.forceRDFAbout.remove(subject)

    def predicate(self, predicate, object, depth=1):
        writer = self.writer
        store = self.store
        writer.push(predicate)

        if isinstance(object, Literal):
            if object.language:
                writer.attribute(XMLLANG, object.language)

            if object.datatype:
                writer.attribute(RDF.datatype, object.datatype)

            writer.text(object)

        elif object in self.__serialized or not (object, None, None) in store:

            if isinstance(object, BNode):
                pass
            else:
                writer.attribute(RDF.resource, self.relativize(object))

        else:
            if first(store.objects(object, RDF.first)):  # may not have type
                                                         # RDF.List

                self.__serialized[object] = 1

                # Warn that any assertions on object other than
                # RDF.first and RDF.rest are ignored... including RDF.List
                import warnings
                warnings.warn(
                    "Assertions on %s other than RDF.first " % repr(object) +
                    "and RDF.rest are ignored ... including RDF.List",
                    UserWarning, stacklevel=2)
                writer.attribute(RDF.parseType, "Collection")

                col = Collection(store, object)

                for item in col:

                    if isinstance(item, URIRef):
                        self.forceRDFAbout.add(item)
                    self.subject(item)

                    if not isinstance(item, URIRef):
                        self.__serialized[item] = 1
            else:
                if first(store.triples_choices(
                    (object, RDF.type, [OWL_NS.Class, RDFS.Class]))) \
                        and isinstance(object, URIRef):
                    writer.attribute(RDF.resource, self.relativize(object))

                elif depth <= self.max_depth:
                    self.subject(object, depth + 1)

                elif isinstance(object, BNode):

                    if not object in self.__serialized \
                            and (object, None, None) in store \
                            and len(list(store.subjects(object=object))) == 1:
                        # inline blank nodes if they haven't been serialized yet
                        # and are only referenced once (regardless of depth)
                        self.subject(object, depth + 1)

                else:
                    writer.attribute(RDF.resource, self.relativize(object))

        writer.pop(predicate)

register(
    'hydro-xml', Serializer,
    'hs_core.hs_rdf', 'HydroPrettyXMLSerializer')
<|endoftext|>


<commit_before>import sys
import os
import sqlite3
import re
import logging
import time

from util import Identity
import apsw_connection

import apsw
from twisted.enterprise import adbapi
from twisted.internet import reactor

import thread

class CreateTableException(Exception):
    pass

class SqlConnections(object):
    """
    A pool of connections to a sql backend.
    Will be called either "sql_backend_writer" or "sql_backend_reader"
    by the data manager.
    """

    def __init__(self, filename, data_manager, archive_db_dir):
        self.data_manager = data_manager
        self.tables = {}
        self.connections = {}

        self.archive_db_dir = archive_db_dir
        self.main_thread = thread.get_ident()
        self.open_new_db(filename)

    def open_new_db(self, filename):
        self.filename = filename
        ##adbapi.noisy = True  # and add log.startLogging(sys.stdout) somewhere
        adbapi.connectionFactory = apsw_connection.ApswConnection
        self.dbpool = adbapi.ConnectionPool('apsw_connection', filename) ##, check_same_thread=False)
        self.dbpool.connectionFactory = apsw_connection.ApswConnection

    def __del__(self):
        # This hack deals with when the condition where the last
        # reference to the sqlconnections object is held by a callback
        # and so the dtor is called from within a adbapi thread in the
        # thread pool.  If we are in one of these threads and call
        # dbpool.close it will throw an exception saying it can't join
        # the current thread - leaving a connection.  This may be a
        # non-issue and it only occurs with the first db created
        # (generation == 0) but all the same, detect when we are in a
        # different thread then the one we were contructed in and
        # schedule a call to shutdown (which must be static since this
        # is the dtor!) for later.

        if self.main_thread == thread.get_ident():
            SqlConnections.shutdown(self.dbpool, self.filename, self.archive_db_dir)
        else:
            reactor.callLater(0.1, SqlConnections.shutdown, self.dbpool, self.filename, self.archive_db_dir)

    @staticmethod
    def shutdown(dbpool, filename, archive_db_dir):
        """
        Facilitates cleaning up filesystem objects after the last
        user of the connection pool goes away.
        """

        try:
            dbpool.close()
        except Exception as e:
            logging.info("SqlConnections.__del__: dbpool.close failed: %s", e)

        if filename != ':memory:':

            if archive_db_dir:
                logging.info("SqlConnections: Archiving db file '%s' to '%s'", filename, archive_db_dir)
                try:
                    os.renames(filename, archive_db_dir)
                except Exception as e:
                    logging.info("SqlConnections: ERROR: Archive failed! %s", e)
            else:
                logging.info("SqlConnections: Removing db file '%s'", filename)
                try:
                    os.remove(filename)
                except Exception as e:
                    pass  # ignore these
                    ##logging.info("SqlConnections: ERROR: Could not remove db file! %s", e)

    def do_select(self, txn, sql):

        try:
            retval = txn.execute(sql)

        except Exception as e:
            return { "retval": 1, "error_message" : e }

        # Find the max column with for each column.

        max_widths = {}

        # Get the column names.

        column_names = []

        try:
            for column_index, column_name in enumerate(txn.getdescription()):

                if not max_widths.has_key(str(column_index)) or max_widths[str(column_index)] < len(column_name[0]):
                    max_widths[str(column_index)] = len(column_name[0])

                column_names.append(column_name[0])

        except apsw.ExecutionCompleteError as e:
            # Ugh, getdescription fails if the query succeeds but returns no rows!  This is the message:
            #    "Can't get description for statements that have completed execution"

            # For now return a zero row empty table:

            data = { "column_names" : [ "no_data" ], "rows" : [ ] }

            return { "retval" : 0, "error_message" : '', "data" : data, "max_widths" : { "0" : 7 } }

        # Save the data here:

        saved_rows = []
        nr_rows = 0

        # For each row.

        for row in txn.fetchall():

            saved_row = []

            # For each column.

            for column_index, cell in enumerate(row):

                # Convert all to strings and compute the length.

                if cell == None:
                    cell = ''
                else:
                    cell = str(cell)

                length = len(cell)

                if not max_widths.has_key(str(column_index)) or max_widths[str(column_index)] < length:
                    max_widths[str(column_index)] = length

                # Save the cell to the row for output later.

                saved_row.append(cell)

            # Save the row to the rows for output later.

            saved_rows.append(saved_row)

        data = { "column_names" : column_names,
                 "rows" : saved_rows }

        return { "retval" : 0, "error_message" : '', "data" : data, "max_widths" : max_widths }

    def get_schema(self, txn, table):

        schema = []

        txn.execute("pragma table_info( %s );" % table)

        for row in txn.fetchall():
            schema.append( [ str(row[1]), str(row[2]) ] )

        return schema

    def do_sql(self, txn, sql):
        """ Execute generic sql """

        result = None

        result = txn.execute(sql)

        return result

    def do_desc(self, txn, tablename):

        schema = self.get_schema(txn, tablename)

        if schema == None or len(schema) <= 0:
            return (1, "Table %s not found" % (tablename,), None)

        table = { "column_names" : [ "column_name", "column_type" ],
                  "rows" : schema }

        return (0, '', table)

    def do_update_table(self, txn, table, identity):

        tablename = table["tablename"]

        # Use only the first row for now.
        bind_values = [ x for x in table["rows"][0] ]

        # Make <column_name1>=?, <column_name2>=?, ...
        set_pairs = "=?, ".join( table["column_names"] ) + "=?"

        sql = "UPDATE %s SET %s WHERE identity=?" % (tablename, set_pairs)

        bind_values.append(identity)

        txn.execute(sql, bind_values)

        return txn.rowcount

    def do_insert_table(self, txn, table):

        tablename = table["tablename"]
        column_str = ", ".join( table["column_names"] )
        bind_str = ", ".join( [ "?" for x in table["column_names"] ] )

        sql = ''

        rowcount = 0

        for row_index, row in enumerate(table["rows"]):

            sql = "INSERT INTO %s ( %s ) VALUES ( %s )" % (tablename, column_str, bind_str)

            txn.execute(sql, row )

            rowcount += 1

        return rowcount

    def async_insert_info_table(self, db_generation_number, generation_start_epoch, generation_duration_s):
        """
        Adds a status table (qasino_server_info) to the database in each generation.
        """

        table = { "tablename" : "qasino_server_info",
                  "column_names" : [ "generation_number", "generation_duration_s", "generation_start_epoch" ],
                  "column_types" : [ "int", "int", "int" ],
                  "rows" : [ [ str(db_generation_number), generation_duration_s, generation_start_epoch ] ]
                  }

        return self.async_add_table_data(table, Identity.get_identity())

    def async_insert_tables_table(self):
        """
        Adds a table (qasino_server_tables) to the database with per table info.
        """

        rows = []

        for tablename, table_data in self.tables.items():

            rows.append( [ tablename,
                           str(table_data["nr_rows"]),
                           str(table_data["updates"]),
                           table_data["last_update_epoch"] ] )

        # the chicken or the egg - how do we add ourselves?

        rows.append( [ "qasino_server_tables",
                       len(rows) + 1,
                       1,
                       time.time() ] )

        table = { "tablename" : "qasino_server_tables",
                  "column_names" : [ "tablename", "nr_rows", "nr_updates", "last_update_epoch" ],
                  "column_types" : [ "varchar", "int", "int", "int" ],
                  "rows" : rows
                }

        return self.async_add_table_data(table, Identity.get_identity())

    def async_insert_connections_table(self):
        """
        Adds a table (qasino_server_connections) to the database with per table info.
        """

        rows = []

        for connection, connection_data in self.connections.items():

            rows.append( [ connection, # identity
                           str(len(connection_data["tables"])),
                           connection_data["last_update_epoch"] ] )

        table = { "tablename" : "qasino_server_connections",
                  "column_names" : [ "identity", "nr_tables", "last_update_epoch" ],
                  "column_types" : [ "varchar", "int", "int" ],
                  "rows" : rows
                }

        return self.async_add_table_data(table, Identity.get_identity())

    def async_add_table_data(self, table, identity, **kwargs):
        """
        Initiate a adbapi async interaction to add a table to the backend, returns the deferred obj.
        """
        return self.dbpool.runInteraction(self.add_table_data, table, identity, **kwargs)

    def add_table_data(self, txn, table, identity, persist=False, update=False):
        """
        Add a table to the backend if it doesn't exist and insert the data.
        This is executed using adbapi in a thread pool.
        """

        tablename = table["tablename"]

        # Check if we need to save the table in case of persistence.

        self.data_manager.check_save_table(tablename, table, identity, persist, update)

        # Now check if we've already added the table.

        try:
            if not update and self.connections[identity]["tables"][tablename] > 0:
                logging.info("SqlConnections: '%s' has already sent an update for table '%s'", identity, tablename)
                return
            else:
                logging.info("SqlConnections: Update for table '%s' from '%s'", tablename, identity)
        except KeyError:
            # Ignore
            pass

        # Do we need to create the table in the db?

        did_create = False

        try:
            # Wrap the create, alter tables and inserts or updates in
            # a transaction... All or none please.

            txn.execute("BEGIN;")

            # Use this to detect if the table already exists - we
            # could just not do this and use the create table to
            # detect if a table is already there or not...

            schema = self.get_schema(txn, tablename)

            # Do we need to create the table?

            if schema == None or len(schema) <= 0:

                create_sql = "CREATE TABLE %s ( \n" % tablename

                columns = []

                for column_name, column_type in zip(table["column_names"], table["column_types"]):
                    columns.append( "%s %s DEFAULT NULL" % (column_name, column_type) )

                create_sql += ",\n".join( columns )

                create_sql += ")"

                logging.info("SqlConnections: Creating table '%s' from '%s'", tablename, identity)

                try:
                    result = self.do_sql(txn, create_sql)
                except Exception as e:
                    errorstr = str(e)

                    # Is this a "table already created" error?  Then its a race condition between checking if a table exists and actually creating it.
                    if "already exists" in errorstr:
                        # Don't need to log this as it is fairly common.
                        ##logging.info("SqlConnections: Warning: Create table race condition '%s' for %s: '%s' ... retrying", tablename, identity, errorstr)
                        raise CreateTableException("Create table race condition '%s' for %s: '%s'" % (tablename, identity, errorstr,))

                    else:
                        # Otherwise some kind of sql error, pass the exception up
                        raise e

                if result != None:
                    did_create = True

            else:
                logging.info("SqlConnections: Adding to table '%s' from '%s'", tablename, identity)

            # If we were not first and didn't create the table we need to check for merge.

            if not did_create:
                self.data_manager.table_merger.merge_table(txn, table, schema, self)

            # Now update the table - call do_update if this is an update type and we didn't create the initial table.

            if update and not did_create:
                rowcount = self.do_update_table(txn, table, identity)
            else:
                rowcount = self.do_insert_table(txn, table)

            txn.execute("COMMIT;")

        except apsw.BusyError as e:
            # This should only happen if we exceed the busy timeout.
            logging.info("SqlConnections: Warning: Lock contention for adding table '%s': %s for %s", tablename, str(e), identity)

            txn.execute("ROLLBACK;")

            # Place this request back on the event queue for a retry.
            self.async_add_table_data(table, identity, persist=persist, update=update)

            return 0

        except CreateTableException as e:
            # Message when raised

            txn.execute("ROLLBACK;")

            # Place this request back on the event queue for a retry.
            self.async_add_table_data(table, identity, persist=persist, update=update)

            return 0

        except Exception as e:
            logging.info("SqlConnections: ERROR: Failed to add table '%s': %s", tablename, e)

            txn.execute("ROLLBACK;")

            return 0

        now = time.time()

        nr_rows = len(table["rows"])

        # Keep track of how many updates a table has received.

        if tablename not in self.tables:
            self.tables[tablename] = { "updates" : 1,
                                       "nr_rows" : rowcount,
                                       "last_update_epoch" : now }
        else:
            if not update:
                self.tables[tablename]["nr_rows"] += rowcount

            self.tables[tablename]["updates"] += 1
            self.tables[tablename]["last_update_epoch"] = now

        # Keep track of which "identities" have added to a table.

        if identity not in self.connections:

            self.connections[identity] = { 'tables' : { tablename : rowcount },
                                           'last_update_epoch' : now }
        else:
            self.connections[identity]["last_update_epoch"] = now

            if tablename not in self.connections[identity]["tables"]:
                self.connections[identity]["tables"][tablename] = rowcount
            elif not update:
                self.connections[identity]["tables"][tablename] += rowcount

        return 1

<commit_msg>Tweak to zero rows return.<commit_after>import sys
import os
import sqlite3
import re
import logging
import time

from util import Identity
import apsw_connection

import apsw
from twisted.enterprise import adbapi
from twisted.internet import reactor

import thread

class CreateTableException(Exception):
    pass

class SqlConnections(object):
    """
    A pool of connections to a sql backend.
    Will be called either "sql_backend_writer" or "sql_backend_reader"
    by the data manager.
    """

    def __init__(self, filename, data_manager, archive_db_dir):
        self.data_manager = data_manager
        self.tables = {}
        self.connections = {}

        self.archive_db_dir = archive_db_dir
        self.main_thread = thread.get_ident()
        self.open_new_db(filename)

    def open_new_db(self, filename):
        self.filename = filename
        ##adbapi.noisy = True  # and add log.startLogging(sys.stdout) somewhere
        adbapi.connectionFactory = apsw_connection.ApswConnection
        self.dbpool = adbapi.ConnectionPool('apsw_connection', filename) ##, check_same_thread=False)
        self.dbpool.connectionFactory = apsw_connection.ApswConnection

    def __del__(self):
        # This hack deals with when the condition where the last
        # reference to the sqlconnections object is held by a callback
        # and so the dtor is called from within a adbapi thread in the
        # thread pool.  If we are in one of these threads and call
        # dbpool.close it will throw an exception saying it can't join
        # the current thread - leaving a connection.  This may be a
        # non-issue and it only occurs with the first db created
        # (generation == 0) but all the same, detect when we are in a
        # different thread then the one we were contructed in and
        # schedule a call to shutdown (which must be static since this
        # is the dtor!) for later.

        if self.main_thread == thread.get_ident():
            SqlConnections.shutdown(self.dbpool, self.filename, self.archive_db_dir)
        else:
            reactor.callLater(0.1, SqlConnections.shutdown, self.dbpool, self.filename, self.archive_db_dir)

    @staticmethod
    def shutdown(dbpool, filename, archive_db_dir):
        """
        Facilitates cleaning up filesystem objects after the last
        user of the connection pool goes away.
        """

        try:
            dbpool.close()
        except Exception as e:
            logging.info("SqlConnections.__del__: dbpool.close failed: %s", e)

        if filename != ':memory:':

            if archive_db_dir:
                logging.info("SqlConnections: Archiving db file '%s' to '%s'", filename, archive_db_dir)
                try:
                    os.renames(filename, archive_db_dir)
                except Exception as e:
                    logging.info("SqlConnections: ERROR: Archive failed! %s", e)
            else:
                logging.info("SqlConnections: Removing db file '%s'", filename)
                try:
                    os.remove(filename)
                except Exception as e:
                    pass  # ignore these
                    ##logging.info("SqlConnections: ERROR: Could not remove db file! %s", e)

    def do_select(self, txn, sql):

        try:
            retval = txn.execute(sql)

        except Exception as e:
            return { "retval": 1, "error_message" : e }

        # Find the max column with for each column.

        max_widths = {}

        # Get the column names.

        column_names = []

        try:
            for column_index, column_name in enumerate(txn.getdescription()):

                if not max_widths.has_key(str(column_index)) or max_widths[str(column_index)] < len(column_name[0]):
                    max_widths[str(column_index)] = len(column_name[0])

                column_names.append(column_name[0])

        except apsw.ExecutionCompleteError as e:
            # Ugh, getdescription fails if the query succeeds but returns no rows!  This is the message:
            #    "Can't get description for statements that have completed execution"

            # For now return a zero row empty table:

            data = { "column_names" : [ "query returned zero rows" ], "rows" : [ ] }

            return { "retval" : 0, "error_message" : '', "data" : data, "max_widths" : { "0" : 24 } }

        # Save the data here:

        saved_rows = []
        nr_rows = 0

        # For each row.

        for row in txn.fetchall():

            saved_row = []

            # For each column.

            for column_index, cell in enumerate(row):

                # Convert all to strings and compute the length.

                if cell == None:
                    cell = ''
                else:
                    cell = str(cell)

                length = len(cell)

                if not max_widths.has_key(str(column_index)) or max_widths[str(column_index)] < length:
                    max_widths[str(column_index)] = length

                # Save the cell to the row for output later.

                saved_row.append(cell)

            # Save the row to the rows for output later.

            saved_rows.append(saved_row)

        data = { "column_names" : column_names,
                 "rows" : saved_rows }

        return { "retval" : 0, "error_message" : '', "data" : data, "max_widths" : max_widths }

    def get_schema(self, txn, table):

        schema = []

        txn.execute("pragma table_info( %s );" % table)

        for row in txn.fetchall():
            schema.append( [ str(row[1]), str(row[2]) ] )

        return schema

    def do_sql(self, txn, sql):
        """ Execute generic sql """

        result = None

        result = txn.execute(sql)

        return result

    def do_desc(self, txn, tablename):

        schema = self.get_schema(txn, tablename)

        if schema == None or len(schema) <= 0:
            return (1, "Table %s not found" % (tablename,), None)

        table = { "column_names" : [ "column_name", "column_type" ],
                  "rows" : schema }

        return (0, '', table)

    def do_update_table(self, txn, table, identity):

        tablename = table["tablename"]

        # Use only the first row for now.
        bind_values = [ x for x in table["rows"][0] ]

        # Make <column_name1>=?, <column_name2>=?, ...
        set_pairs = "=?, ".join( table["column_names"] ) + "=?"

        sql = "UPDATE %s SET %s WHERE identity=?" % (tablename, set_pairs)

        bind_values.append(identity)

        txn.execute(sql, bind_values)

        return txn.rowcount

    def do_insert_table(self, txn, table):

        tablename = table["tablename"]
        column_str = ", ".join( table["column_names"] )
        bind_str = ", ".join( [ "?" for x in table["column_names"] ] )

        sql = ''

        rowcount = 0

        for row_index, row in enumerate(table["rows"]):

            sql = "INSERT INTO %s ( %s ) VALUES ( %s )" % (tablename, column_str, bind_str)

            txn.execute(sql, row )

            rowcount += 1

        return rowcount

    def async_insert_info_table(self, db_generation_number, generation_start_epoch, generation_duration_s):
        """
        Adds a status table (qasino_server_info) to the database in each generation.
        """

        table = { "tablename" : "qasino_server_info",
                  "column_names" : [ "generation_number", "generation_duration_s", "generation_start_epoch" ],
                  "column_types" : [ "int", "int", "int" ],
                  "rows" : [ [ str(db_generation_number), generation_duration_s, generation_start_epoch ] ]
                  }

        return self.async_add_table_data(table, Identity.get_identity())

    def async_insert_tables_table(self):
        """
        Adds a table (qasino_server_tables) to the database with per table info.
        """

        rows = []

        for tablename, table_data in self.tables.items():

            rows.append( [ tablename,
                           str(table_data["nr_rows"]),
                           str(table_data["updates"]),
                           table_data["last_update_epoch"] ] )

        # the chicken or the egg - how do we add ourselves?

        rows.append( [ "qasino_server_tables",
                       len(rows) + 1,
                       1,
                       time.time() ] )

        table = { "tablename" : "qasino_server_tables",
                  "column_names" : [ "tablename", "nr_rows", "nr_updates", "last_update_epoch" ],
                  "column_types" : [ "varchar", "int", "int", "int" ],
                  "rows" : rows
                }

        return self.async_add_table_data(table, Identity.get_identity())

    def async_insert_connections_table(self):
        """
        Adds a table (qasino_server_connections) to the database with per table info.
        """

        rows = []

        for connection, connection_data in self.connections.items():

            rows.append( [ connection, # identity
                           str(len(connection_data["tables"])),
                           connection_data["last_update_epoch"] ] )

        table = { "tablename" : "qasino_server_connections",
                  "column_names" : [ "identity", "nr_tables", "last_update_epoch" ],
                  "column_types" : [ "varchar", "int", "int" ],
                  "rows" : rows
                }

        return self.async_add_table_data(table, Identity.get_identity())

    def async_add_table_data(self, table, identity, **kwargs):
        """
        Initiate a adbapi async interaction to add a table to the backend, returns the deferred obj.
        """
        return self.dbpool.runInteraction(self.add_table_data, table, identity, **kwargs)

    def add_table_data(self, txn, table, identity, persist=False, update=False):
        """
        Add a table to the backend if it doesn't exist and insert the data.
        This is executed using adbapi in a thread pool.
        """

        tablename = table["tablename"]

        # Check if we need to save the table in case of persistence.

        self.data_manager.check_save_table(tablename, table, identity, persist, update)

        # Now check if we've already added the table.

        try:
            if not update and self.connections[identity]["tables"][tablename] > 0:
                logging.info("SqlConnections: '%s' has already sent an update for table '%s'", identity, tablename)
                return
            else:
                logging.info("SqlConnections: Update for table '%s' from '%s'", tablename, identity)
        except KeyError:
            # Ignore
            pass

        # Do we need to create the table in the db?

        did_create = False

        try:
            # Wrap the create, alter tables and inserts or updates in
            # a transaction... All or none please.

            txn.execute("BEGIN;")

            # Use this to detect if the table already exists - we
            # could just not do this and use the create table to
            # detect if a table is already there or not...

            schema = self.get_schema(txn, tablename)

            # Do we need to create the table?

            if schema == None or len(schema) <= 0:

                create_sql = "CREATE TABLE %s ( \n" % tablename

                columns = []

                for column_name, column_type in zip(table["column_names"], table["column_types"]):
                    columns.append( "%s %s DEFAULT NULL" % (column_name, column_type) )

                create_sql += ",\n".join( columns )

                create_sql += ")"

                logging.info("SqlConnections: Creating table '%s' from '%s'", tablename, identity)

                try:
                    result = self.do_sql(txn, create_sql)
                except Exception as e:
                    errorstr = str(e)

                    # Is this a "table already created" error?  Then its a race condition between checking if a table exists and actually creating it.
                    if "already exists" in errorstr:
                        # Don't need to log this as it is fairly common.
                        ##logging.info("SqlConnections: Warning: Create table race condition '%s' for %s: '%s' ... retrying", tablename, identity, errorstr)
                        raise CreateTableException("Create table race condition '%s' for %s: '%s'" % (tablename, identity, errorstr,))

                    else:
                        # Otherwise some kind of sql error, pass the exception up
                        raise e

                if result != None:
                    did_create = True

            else:
                logging.info("SqlConnections: Adding to table '%s' from '%s'", tablename, identity)

            # If we were not first and didn't create the table we need to check for merge.

            if not did_create:
                self.data_manager.table_merger.merge_table(txn, table, schema, self)

            # Now update the table - call do_update if this is an update type and we didn't create the initial table.

            if update and not did_create:
                rowcount = self.do_update_table(txn, table, identity)
            else:
                rowcount = self.do_insert_table(txn, table)

            txn.execute("COMMIT;")

        except apsw.BusyError as e:
            # This should only happen if we exceed the busy timeout.
            logging.info("SqlConnections: Warning: Lock contention for adding table '%s': %s for %s", tablename, str(e), identity)

            txn.execute("ROLLBACK;")

            # Place this request back on the event queue for a retry.
            self.async_add_table_data(table, identity, persist=persist, update=update)

            return 0

        except CreateTableException as e:
            # Message when raised

            txn.execute("ROLLBACK;")

            # Place this request back on the event queue for a retry.
            self.async_add_table_data(table, identity, persist=persist, update=update)

            return 0

        except Exception as e:
            logging.info("SqlConnections: ERROR: Failed to add table '%s': %s", tablename, e)

            txn.execute("ROLLBACK;")

            return 0

        now = time.time()

        nr_rows = len(table["rows"])

        # Keep track of how many updates a table has received.

        if tablename not in self.tables:
            self.tables[tablename] = { "updates" : 1,
                                       "nr_rows" : rowcount,
                                       "last_update_epoch" : now }
        else:
            if not update:
                self.tables[tablename]["nr_rows"] += rowcount

            self.tables[tablename]["updates"] += 1
            self.tables[tablename]["last_update_epoch"] = now

        # Keep track of which "identities" have added to a table.

        if identity not in self.connections:

            self.connections[identity] = { 'tables' : { tablename : rowcount },
                                           'last_update_epoch' : now }
        else:
            self.connections[identity]["last_update_epoch"] = now

            if tablename not in self.connections[identity]["tables"]:
                self.connections[identity]["tables"][tablename] = rowcount
            elif not update:
                self.connections[identity]["tables"][tablename] += rowcount

        return 1

<|endoftext|>


<commit_before>"""
Copyright (c) 2016 Jet Propulsion Laboratory,
California Institute of Technology.  All rights reserved
"""
import sys
import os
import traceback
import logging
from cStringIO import StringIO
from datetime import datetime
from multiprocessing import Pool, Manager

import matplotlib.dates as mdates
import matplotlib.pyplot as plt
import numpy as np
import itertools
from webservice.SparkAlg import SparkAlg
from webservice.NexusHandler import NexusHandler, nexus_handler, DEFAULT_PARAMETERS_SPEC
from nexustiles.nexustiles import NexusTileService
from scipy import stats

from webservice import Filtering as filt
from webservice.webmodel import NexusResults, NexusProcessingException, NoDataException
from pyspark import SparkContext,SparkConf

SENTINEL = 'STOP'

@nexus_handler
class TimeSeriesHandlerImpl(NexusHandler):
    name = "Time Series Spark"
    path = "/timeSeriesSpark"
    description = "Computes a time series plot between one or more datasets given an arbitrary geographical area and time range"
    params = DEFAULT_PARAMETERS_SPEC
    singleton = True

    def __init__(self):
        NexusHandler.__init__(self, skipCassandra=True)
        self.log = logging.getLogger(__name__)

    def calc(self, computeOptions, **args):
        """

        :param computeOptions: StatsComputeOptions
        :param args: dict
        :return:
        """

        ds = computeOptions.get_dataset()

        if type(ds) != list and type(ds) != tuple:
            ds = (ds,)

        resultsRaw = []

        for shortName in ds:
            results, meta = self.getTimeSeriesStatsForBoxSingleDataSet(computeOptions.get_min_lat(),
                                                                       computeOptions.get_max_lat(),
                                                                       computeOptions.get_min_lon(),
                                                                       computeOptions.get_max_lon(),
                                                                       shortName,
                                                                       computeOptions.get_start_time(),
                                                                       computeOptions.get_end_time(),
                                                                       computeOptions.get_apply_seasonal_cycle_filter(),
                                                                       computeOptions.get_apply_low_pass_filter())
            resultsRaw.append([results, meta])

        results = self._mergeResults(resultsRaw)

        if len(ds) == 2:
            stats = self.calculateComparisonStats(results, suffix="")
            if computeOptions.get_apply_seasonal_cycle_filter():
                s = self.calculateComparisonStats(results, suffix="Seasonal")
                stats = self._mergeDicts(stats, s)
            if computeOptions.get_apply_low_pass_filter():
                s = self.calculateComparisonStats(results, suffix="LowPass")
                stats = self._mergeDicts(stats, s)
            if computeOptions.get_apply_seasonal_cycle_filter() and computeOptions.get_apply_low_pass_filter():
                s = self.calculateComparisonStats(results, suffix="SeasonalLowPass")
                stats = self._mergeDicts(stats, s)
        else:
            stats = {}

        meta = []
        for singleRes in resultsRaw:
            meta.append(singleRes[1])

        res = TimeSeriesResults(results=results, meta=meta, stats=stats,
                                computeOptions=computeOptions)
        return res

    def getTimeSeriesStatsForBoxSingleDataSet(self, min_lat, max_lat,
                                              min_lon, max_lon, ds,
                                              start_time=0, end_time=-1,
                                              applySeasonalFilter=False,
                                              applyLowPass=False):

        daysinrange = self._tile_service.find_days_in_range_asc(min_lat,
                                                                max_lat,
                                                                min_lon,
                                                                max_lon, ds,
                                                                start_time,
                                                                end_time)

        if len(daysinrange) == 0:
            raise NoDataException(reason="No data found for selected timeframe")

        cwd = os.getcwd()

        # Configure Spark
        sp_conf = SparkConf()
        sp_conf.setAppName("Spark Time Avg Map")
        sp_conf.set("spark.executorEnv.HOME",
                    os.path.join(os.getenv('HOME'), 'spark_exec_home'))
        sp_conf.set("spark.executorEnv.PYTHONPATH", cwd)
        #sp_conf.set("spark.executor.memoryOverhead", "4g")
        sp_conf.set("spark.executor.memory", "4g")

        #num_parts = 1
        #num_parts = 16
        #num_parts = 32
        #num_parts = 64
        num_parts = 128
        #num_execs = 1
        #num_execs = 16
        #num_execs = 32
        num_execs = 64
        cores_per_exec = 1
        sp_conf.setMaster("yarn-client")
        #sp_conf.setMaster("local[16]")
        #sp_conf.setMaster("local[1]")
        sp_conf.set("spark.executor.instances", num_execs)
        sp_conf.set("spark.executor.cores", cores_per_exec)

        #print sp_conf.getAll()
        sc = SparkContext(conf=sp_conf)

        nexus_tiles_spark = [(min_lat, max_lat, min_lon, max_lon, ds,
                              list(daysinrange_part), cwd)
                             for daysinrange_part
                             in np.array_split(daysinrange, num_parts)]

        #for tile in nexus_tiles_spark:
        #    print tile

        # Launch Spark computations
        rdd = sc.parallelize(nexus_tiles_spark,num_parts)
        results = rdd.map(TimeSeriesCalculator.calc_average_on_day).collect()
        #
        results = list(itertools.chain.from_iterable(results))
        results = sorted(results, key=lambda entry: entry["time"])

        #filt.applyAllFiltersOnField(results, 'mean', applySeasonal=applySeasonalFilter, applyLowPass=applyLowPass)
        #filt.applyAllFiltersOnField(results, 'max', applySeasonal=applySeasonalFilter, applyLowPass=applyLowPass)
        #filt.applyAllFiltersOnField(results, 'min', applySeasonal=applySeasonalFilter, applyLowPass=applyLowPass)

        return results, {}

    def calculateComparisonStats(self, results, suffix=""):

        xy = [[], []]

        for item in results:
            if len(item) == 2:
                xy[item[0]["ds"]].append(item[0]["mean%s" % suffix])
                xy[item[1]["ds"]].append(item[1]["mean%s" % suffix])

        slope, intercept, r_value, p_value, std_err = stats.linregress(xy[0], xy[1])
        comparisonStats = {
            "slope%s" % suffix: slope,
            "intercept%s" % suffix: intercept,
            "r%s" % suffix: r_value,
            "p%s" % suffix: p_value,
            "err%s" % suffix: std_err
        }

        return comparisonStats

class TimeSeriesResults(NexusResults):
    LINE_PLOT = "line"
    SCATTER_PLOT = "scatter"

    __SERIES_COLORS = ['red', 'blue']

    def __init__(self, results=None, meta=None, stats=None, computeOptions=None):
        NexusResults.__init__(self, results=results, meta=meta, stats=stats, computeOptions=computeOptions)

    def toImage(self):

        type = self.computeOptions().get_plot_type()

        if type == TimeSeriesResults.LINE_PLOT or type == "default":
            return self.createLinePlot()
        elif type == TimeSeriesResults.SCATTER_PLOT:
            return self.createScatterPlot()
        else:
            raise Exception("Invalid or unsupported time series plot specified")

    def createScatterPlot(self):
        timeSeries = []
        series0 = []
        series1 = []

        res = self.results()
        meta = self.meta()

        plotSeries = self.computeOptions().get_plot_series() if self.computeOptions is not None else None
        if plotSeries is None:
            plotSeries = "mean"

        for m in res:
            if len(m) == 2:
                timeSeries.append(datetime.fromtimestamp(m[0]["time"] / 1000))
                series0.append(m[0][plotSeries])
                series1.append(m[1][plotSeries])

        title = ', '.join(set([m['title'] for m in meta]))
        sources = ', '.join(set([m['source'] for m in meta]))
        dateRange = "%s - %s" % (timeSeries[0].strftime('%b %Y'), timeSeries[-1].strftime('%b %Y'))

        fig, ax = plt.subplots()
        fig.set_size_inches(11.0, 8.5)
        ax.scatter(series0, series1, alpha=0.5)
        ax.set_xlabel(meta[0]['units'])
        ax.set_ylabel(meta[1]['units'])
        ax.set_title("%s\n%s\n%s" % (title, sources, dateRange))

        par = np.polyfit(series0, series1, 1, full=True)
        slope = par[0][0]
        intercept = par[0][1]
        xl = [min(series0), max(series0)]
        yl = [slope * xx + intercept for xx in xl]
        plt.plot(xl, yl, '-r')

        # r = self.stats()["r"]
        # plt.text(0.5, 0.5, "r = foo")

        ax.grid(True)
        fig.tight_layout()

        sio = StringIO()
        plt.savefig(sio, format='png')
        return sio.getvalue()

    def createLinePlot(self):
        nseries = len(self.meta())
        res = self.results()
        meta = self.meta()

        timeSeries = [datetime.fromtimestamp(m[0]["time"] / 1000) for m in res]

        means = [[np.nan] * len(res) for n in range(0, nseries)]

        plotSeries = self.computeOptions().get_plot_series() if self.computeOptions is not None else None
        if plotSeries is None:
            plotSeries = "mean"

        for n in range(0, len(res)):
            timeSlot = res[n]
            for seriesValues in timeSlot:
                means[seriesValues['ds']][n] = seriesValues[plotSeries]

        x = timeSeries

        fig, axMain = plt.subplots()
        fig.set_size_inches(11.0, 8.5)
        fig.autofmt_xdate()

        title = ', '.join(set([m['title'] for m in meta]))
        sources = ', '.join(set([m['source'] for m in meta]))
        dateRange = "%s - %s" % (timeSeries[0].strftime('%b %Y'), timeSeries[-1].strftime('%b %Y'))

        axMain.set_title("%s\n%s\n%s" % (title, sources, dateRange))
        axMain.set_xlabel('Date')
        axMain.grid(True)
        axMain.xaxis.set_major_locator(mdates.YearLocator())
        axMain.xaxis.set_major_formatter(mdates.DateFormatter('%b %Y'))
        axMain.xaxis.set_minor_locator(mdates.MonthLocator())
        axMain.format_xdata = mdates.DateFormatter('%Y-%m-%d')

        plots = []

        for n in range(0, nseries):
            if n == 0:
                ax = axMain
            else:
                ax = ax.twinx()

            plots += ax.plot(x, means[n], color=self.__SERIES_COLORS[n], zorder=10, linewidth=3, label=meta[n]['title'])
            ax.set_ylabel(meta[n]['units'])

        labs = [l.get_label() for l in plots]
        axMain.legend(plots, labs, loc=0)

        sio = StringIO()
        plt.savefig(sio, format='png')
        return sio.getvalue()

class TimeSeriesCalculator(SparkAlg):
    def __init__(self):
        self.__tile_service = NexusTileService()

    @staticmethod
    def calc_average_on_day(tile_in_spark):
        (min_lat, max_lat, min_lon, max_lon, dataset,
         timestamps, cwd) = tile_in_spark
        os.chdir(cwd)
        start_time = timestamps[0]
        end_time = timestamps[-1]
        tile_service = NexusTileService()
        #ds1_nexus_tiles = \
        #    tile_service.get_tiles_bounded_by_box_at_time(min_lat, max_lat,
        #                                                  min_lon, max_lon,
        #                                                  dataset,
        #                                                  timeinseconds)
        #ds1_nexus_tiles = \
        #    tile_service.get_tiles_bounded_by_box(min_lat, max_lat,
        #                                          min_lon, max_lon,
        #                                          dataset,
        #                                          timeinseconds,
        #                                          timeinseconds)
        ds1_nexus_tiles = TimeSeriesCalculator.query_by_parts(tile_service,
                                                              min_lat,
                                                              max_lat,
                                                              min_lon,
                                                              max_lon,
                                                              dataset,
                                                              timestamps[0],
                                                              timestamps[-1])

        # debug code
        #for tile in ds1_nexus_tiles:
            #print 'tile shape: ', tile.data.shape
            #print 'tile data shape: ', tile.data.data.shape
            #print 'tile mask shape: ', tile.data.mask.shape
            #print 'tile: ', tile.data

        stats_arr = []
        for timeinseconds in timestamps:
            tile_data_agg = np.ma.array([tile.data.flatten() \
                                             for tile in ds1_nexus_tiles \
                                             if (tile.times[0] == timeinseconds)])
            if (len(tile_data_agg) == 0) or tile_data_agg.mask.all():
                data_min = 0.
                data_max = 0.
                daily_mean = 0.
                data_count = 0
                data_std = 0.
            else:
                data_min = np.ma.min(tile_data_agg)
                data_max = np.ma.max(tile_data_agg)
                daily_mean = np.ma.mean(tile_data_agg).item()
                data_count = np.ma.count(tile_data_agg).item()
                data_std = np.ma.std(tile_data_agg)

        # Return Stats by day
            stat = {
                'min': data_min,
                'max': data_max,
                'mean': daily_mean,
                'cnt': data_count,
                'std': data_std,
                'time': int(timeinseconds)
                }
            stats_arr.append(stat)
        return stats_arr

def pool_worker(work_queue, done_queue):
    try:
        calculator = TimeSeriesCalculator()

        for work in iter(work_queue.get, SENTINEL):
            scifunction = work[0]
            args = work[1:]
            result = calculator.__getattribute__(scifunction)(*args)
            done_queue.put(result)
    except Exception as e:
        e_str = traceback.format_exc(e)
        done_queue.put({'error': e_str})
<commit_msg>Do not call item on value returned by np.ma.count since it should already be an int type.<commit_after>"""
Copyright (c) 2016 Jet Propulsion Laboratory,
California Institute of Technology.  All rights reserved
"""
import sys
import os
import traceback
import logging
from cStringIO import StringIO
from datetime import datetime
from multiprocessing import Pool, Manager

import matplotlib.dates as mdates
import matplotlib.pyplot as plt
import numpy as np
import itertools
from webservice.SparkAlg import SparkAlg
from webservice.NexusHandler import NexusHandler, nexus_handler, DEFAULT_PARAMETERS_SPEC
from nexustiles.nexustiles import NexusTileService
from scipy import stats

from webservice import Filtering as filt
from webservice.webmodel import NexusResults, NexusProcessingException, NoDataException
from pyspark import SparkContext,SparkConf

SENTINEL = 'STOP'

@nexus_handler
class TimeSeriesHandlerImpl(NexusHandler):
    name = "Time Series Spark"
    path = "/timeSeriesSpark"
    description = "Computes a time series plot between one or more datasets given an arbitrary geographical area and time range"
    params = DEFAULT_PARAMETERS_SPEC
    singleton = True

    def __init__(self):
        NexusHandler.__init__(self, skipCassandra=True)
        self.log = logging.getLogger(__name__)

    def calc(self, computeOptions, **args):
        """

        :param computeOptions: StatsComputeOptions
        :param args: dict
        :return:
        """

        ds = computeOptions.get_dataset()

        if type(ds) != list and type(ds) != tuple:
            ds = (ds,)

        resultsRaw = []

        for shortName in ds:
            results, meta = self.getTimeSeriesStatsForBoxSingleDataSet(computeOptions.get_min_lat(),
                                                                       computeOptions.get_max_lat(),
                                                                       computeOptions.get_min_lon(),
                                                                       computeOptions.get_max_lon(),
                                                                       shortName,
                                                                       computeOptions.get_start_time(),
                                                                       computeOptions.get_end_time(),
                                                                       computeOptions.get_apply_seasonal_cycle_filter(),
                                                                       computeOptions.get_apply_low_pass_filter())
            resultsRaw.append([results, meta])

        results = self._mergeResults(resultsRaw)

        if len(ds) == 2:
            stats = self.calculateComparisonStats(results, suffix="")
            if computeOptions.get_apply_seasonal_cycle_filter():
                s = self.calculateComparisonStats(results, suffix="Seasonal")
                stats = self._mergeDicts(stats, s)
            if computeOptions.get_apply_low_pass_filter():
                s = self.calculateComparisonStats(results, suffix="LowPass")
                stats = self._mergeDicts(stats, s)
            if computeOptions.get_apply_seasonal_cycle_filter() and computeOptions.get_apply_low_pass_filter():
                s = self.calculateComparisonStats(results, suffix="SeasonalLowPass")
                stats = self._mergeDicts(stats, s)
        else:
            stats = {}

        meta = []
        for singleRes in resultsRaw:
            meta.append(singleRes[1])

        res = TimeSeriesResults(results=results, meta=meta, stats=stats,
                                computeOptions=computeOptions)
        return res

    def getTimeSeriesStatsForBoxSingleDataSet(self, min_lat, max_lat,
                                              min_lon, max_lon, ds,
                                              start_time=0, end_time=-1,
                                              applySeasonalFilter=False,
                                              applyLowPass=False):

        daysinrange = self._tile_service.find_days_in_range_asc(min_lat,
                                                                max_lat,
                                                                min_lon,
                                                                max_lon, ds,
                                                                start_time,
                                                                end_time)

        if len(daysinrange) == 0:
            raise NoDataException(reason="No data found for selected timeframe")

        cwd = os.getcwd()

        # Configure Spark
        sp_conf = SparkConf()
        sp_conf.setAppName("Spark Time Avg Map")
        sp_conf.set("spark.executorEnv.HOME",
                    os.path.join(os.getenv('HOME'), 'spark_exec_home'))
        sp_conf.set("spark.executorEnv.PYTHONPATH", cwd)
        #sp_conf.set("spark.executor.memoryOverhead", "4g")
        sp_conf.set("spark.executor.memory", "4g")

        #num_parts = 1
        #num_parts = 16
        #num_parts = 32
        #num_parts = 64
        num_parts = 128
        #num_execs = 1
        #num_execs = 16
        #num_execs = 32
        num_execs = 64
        cores_per_exec = 1
        sp_conf.setMaster("yarn-client")
        #sp_conf.setMaster("local[16]")
        #sp_conf.setMaster("local[1]")
        sp_conf.set("spark.executor.instances", num_execs)
        sp_conf.set("spark.executor.cores", cores_per_exec)

        #print sp_conf.getAll()
        sc = SparkContext(conf=sp_conf)

        nexus_tiles_spark = [(min_lat, max_lat, min_lon, max_lon, ds,
                              list(daysinrange_part), cwd)
                             for daysinrange_part
                             in np.array_split(daysinrange, num_parts)]

        #for tile in nexus_tiles_spark:
        #    print tile

        # Launch Spark computations
        rdd = sc.parallelize(nexus_tiles_spark,num_parts)
        results = rdd.map(TimeSeriesCalculator.calc_average_on_day).collect()
        #
        results = list(itertools.chain.from_iterable(results))
        results = sorted(results, key=lambda entry: entry["time"])

        #filt.applyAllFiltersOnField(results, 'mean', applySeasonal=applySeasonalFilter, applyLowPass=applyLowPass)
        #filt.applyAllFiltersOnField(results, 'max', applySeasonal=applySeasonalFilter, applyLowPass=applyLowPass)
        #filt.applyAllFiltersOnField(results, 'min', applySeasonal=applySeasonalFilter, applyLowPass=applyLowPass)

        return results, {}

    def calculateComparisonStats(self, results, suffix=""):

        xy = [[], []]

        for item in results:
            if len(item) == 2:
                xy[item[0]["ds"]].append(item[0]["mean%s" % suffix])
                xy[item[1]["ds"]].append(item[1]["mean%s" % suffix])

        slope, intercept, r_value, p_value, std_err = stats.linregress(xy[0], xy[1])
        comparisonStats = {
            "slope%s" % suffix: slope,
            "intercept%s" % suffix: intercept,
            "r%s" % suffix: r_value,
            "p%s" % suffix: p_value,
            "err%s" % suffix: std_err
        }

        return comparisonStats

class TimeSeriesResults(NexusResults):
    LINE_PLOT = "line"
    SCATTER_PLOT = "scatter"

    __SERIES_COLORS = ['red', 'blue']

    def __init__(self, results=None, meta=None, stats=None, computeOptions=None):
        NexusResults.__init__(self, results=results, meta=meta, stats=stats, computeOptions=computeOptions)

    def toImage(self):

        type = self.computeOptions().get_plot_type()

        if type == TimeSeriesResults.LINE_PLOT or type == "default":
            return self.createLinePlot()
        elif type == TimeSeriesResults.SCATTER_PLOT:
            return self.createScatterPlot()
        else:
            raise Exception("Invalid or unsupported time series plot specified")

    def createScatterPlot(self):
        timeSeries = []
        series0 = []
        series1 = []

        res = self.results()
        meta = self.meta()

        plotSeries = self.computeOptions().get_plot_series() if self.computeOptions is not None else None
        if plotSeries is None:
            plotSeries = "mean"

        for m in res:
            if len(m) == 2:
                timeSeries.append(datetime.fromtimestamp(m[0]["time"] / 1000))
                series0.append(m[0][plotSeries])
                series1.append(m[1][plotSeries])

        title = ', '.join(set([m['title'] for m in meta]))
        sources = ', '.join(set([m['source'] for m in meta]))
        dateRange = "%s - %s" % (timeSeries[0].strftime('%b %Y'), timeSeries[-1].strftime('%b %Y'))

        fig, ax = plt.subplots()
        fig.set_size_inches(11.0, 8.5)
        ax.scatter(series0, series1, alpha=0.5)
        ax.set_xlabel(meta[0]['units'])
        ax.set_ylabel(meta[1]['units'])
        ax.set_title("%s\n%s\n%s" % (title, sources, dateRange))

        par = np.polyfit(series0, series1, 1, full=True)
        slope = par[0][0]
        intercept = par[0][1]
        xl = [min(series0), max(series0)]
        yl = [slope * xx + intercept for xx in xl]
        plt.plot(xl, yl, '-r')

        # r = self.stats()["r"]
        # plt.text(0.5, 0.5, "r = foo")

        ax.grid(True)
        fig.tight_layout()

        sio = StringIO()
        plt.savefig(sio, format='png')
        return sio.getvalue()

    def createLinePlot(self):
        nseries = len(self.meta())
        res = self.results()
        meta = self.meta()

        timeSeries = [datetime.fromtimestamp(m[0]["time"] / 1000) for m in res]

        means = [[np.nan] * len(res) for n in range(0, nseries)]

        plotSeries = self.computeOptions().get_plot_series() if self.computeOptions is not None else None
        if plotSeries is None:
            plotSeries = "mean"

        for n in range(0, len(res)):
            timeSlot = res[n]
            for seriesValues in timeSlot:
                means[seriesValues['ds']][n] = seriesValues[plotSeries]

        x = timeSeries

        fig, axMain = plt.subplots()
        fig.set_size_inches(11.0, 8.5)
        fig.autofmt_xdate()

        title = ', '.join(set([m['title'] for m in meta]))
        sources = ', '.join(set([m['source'] for m in meta]))
        dateRange = "%s - %s" % (timeSeries[0].strftime('%b %Y'), timeSeries[-1].strftime('%b %Y'))

        axMain.set_title("%s\n%s\n%s" % (title, sources, dateRange))
        axMain.set_xlabel('Date')
        axMain.grid(True)
        axMain.xaxis.set_major_locator(mdates.YearLocator())
        axMain.xaxis.set_major_formatter(mdates.DateFormatter('%b %Y'))
        axMain.xaxis.set_minor_locator(mdates.MonthLocator())
        axMain.format_xdata = mdates.DateFormatter('%Y-%m-%d')

        plots = []

        for n in range(0, nseries):
            if n == 0:
                ax = axMain
            else:
                ax = ax.twinx()

            plots += ax.plot(x, means[n], color=self.__SERIES_COLORS[n], zorder=10, linewidth=3, label=meta[n]['title'])
            ax.set_ylabel(meta[n]['units'])

        labs = [l.get_label() for l in plots]
        axMain.legend(plots, labs, loc=0)

        sio = StringIO()
        plt.savefig(sio, format='png')
        return sio.getvalue()

class TimeSeriesCalculator(SparkAlg):
    def __init__(self):
        self.__tile_service = NexusTileService()

    @staticmethod
    def calc_average_on_day(tile_in_spark):
        (min_lat, max_lat, min_lon, max_lon, dataset,
         timestamps, cwd) = tile_in_spark
        os.chdir(cwd)
        start_time = timestamps[0]
        end_time = timestamps[-1]
        tile_service = NexusTileService()
        #ds1_nexus_tiles = \
        #    tile_service.get_tiles_bounded_by_box_at_time(min_lat, max_lat,
        #                                                  min_lon, max_lon,
        #                                                  dataset,
        #                                                  timeinseconds)
        #ds1_nexus_tiles = \
        #    tile_service.get_tiles_bounded_by_box(min_lat, max_lat,
        #                                          min_lon, max_lon,
        #                                          dataset,
        #                                          timeinseconds,
        #                                          timeinseconds)
        ds1_nexus_tiles = TimeSeriesCalculator.query_by_parts(tile_service,
                                                              min_lat,
                                                              max_lat,
                                                              min_lon,
                                                              max_lon,
                                                              dataset,
                                                              timestamps[0],
                                                              timestamps[-1])

        # debug code
        #for tile in ds1_nexus_tiles:
            #print 'tile shape: ', tile.data.shape
            #print 'tile data shape: ', tile.data.data.shape
            #print 'tile mask shape: ', tile.data.mask.shape
            #print 'tile: ', tile.data

        stats_arr = []
        for timeinseconds in timestamps:
            tile_data_agg = np.ma.array([tile.data.flatten() \
                                             for tile in ds1_nexus_tiles \
                                             if (tile.times[0] == timeinseconds)])
            if (len(tile_data_agg) == 0) or tile_data_agg.mask.all():
                data_min = 0.
                data_max = 0.
                daily_mean = 0.
                data_count = 0
                data_std = 0.
            else:
                data_min = np.ma.min(tile_data_agg)
                data_max = np.ma.max(tile_data_agg)
                daily_mean = np.ma.mean(tile_data_agg).item()
                data_count = np.ma.count(tile_data_agg)
                data_std = np.ma.std(tile_data_agg)

        # Return Stats by day
            stat = {
                'min': data_min,
                'max': data_max,
                'mean': daily_mean,
                'cnt': data_count,
                'std': data_std,
                'time': int(timeinseconds)
                }
            stats_arr.append(stat)
        return stats_arr

def pool_worker(work_queue, done_queue):
    try:
        calculator = TimeSeriesCalculator()

        for work in iter(work_queue.get, SENTINEL):
            scifunction = work[0]
            args = work[1:]
            result = calculator.__getattribute__(scifunction)(*args)
            done_queue.put(result)
    except Exception as e:
        e_str = traceback.format_exc(e)
        done_queue.put({'error': e_str})
<|endoftext|>