<commit_before>"""Implementation of the XBlock facility.

This code is in the Runtime layer, because it is authored once by edX
and used by all runtimes.

"""
import copy
import functools
import inspect
try:
    import simplesjson as json
except ImportError:
    import json

from collections import namedtuple
from webob import Response

from .plugin import Plugin

class BlockScope(object):
    USAGE, DEFINITION, TYPE, ALL = xrange(4)

class Scope(namedtuple('ScopeBase', 'user block')):
    pass

class Sentinel(object):
    """
    Class for implementing sentinel objects (only equal to themselves).
    """
    def __init__(self, name):
        """
        `name` is the name used to identify the sentinel (which will
            be displayed as the __repr__) of the sentinel.
        """
        self.name = name

    def __repr__(self):
        return self.name

Scope.content = Scope(user=False, block=BlockScope.DEFINITION)
Scope.settings = Scope(user=False, block=BlockScope.USAGE)
Scope.user_state = Scope(user=True, block=BlockScope.USAGE)
Scope.preferences = Scope(user=True, block=BlockScope.TYPE)
Scope.user_info = Scope(user=True, block=BlockScope.ALL)
Scope.children = Sentinel('Scope.children')
Scope.parent = Sentinel('Scope.parent')

# define a placeholder ('nil') value to indicate when nothing has been stored in
# the cache.  ("None" may be a valid value in the cache, so we cannot use it.)
NO_CACHE_VALUE = object()

class ModelType(object):
    """
    A field class that can be used as a class attribute to define what data the class will want
    to refer to.

    When the class is instantiated, it will be available as an instance attribute of the same
    name, by proxying through to self._model_data on the containing object.

    Parameters:
      help : documentation of field class, suitable for presenting in a GUI (defaults to None)
      default : static value to default to if not otherwise specified (defaults to None)
      scope : the scope in which this field class is used (defaults to Scope.content)
      computed_default : provides the ability to specify a function for computing a default value (defaults to None)
      display_name : the display name for the field class, suitable for presenting in a GUI (defaults to name of class)
      values : for field classes with a finite number of valid values, provides the ability to specify the set of
               valid values. This can be specified as either a static return value, or a function that generates
               the valid values.
    """

    def __init__(self, help=None, default=None, scope=Scope.content, computed_default=None, display_name=None,
                 values=None):
        self._name = "unknown"
        self.help = help
        self._default = default
        self.computed_default = computed_default
        self.scope = scope
        self._display_name = display_name
        self._values = values

    @property
    def default(self):
        return self._default

    @property
    def name(self):
        return self._name

    @property
    def values(self):
        """
        Returns the valid values for this class.

        If this field class does not define a finite number of valid values, this method will return None.
        """
        if self._values is None:
            return None
        elif callable(self._values):
            return self._values()
        else:
            return self._values

    @property
    def display_name(self):
        """
        Returns the display name for this class, suitable for use in a GUI.

        If no display name has been set, returns the name of the class.
        """
        return self._display_name if self._display_name is not None else self.name

    def _get_cached_value(self, instance):
        """
        Return a value from the instance's cache, or a marker value if either the cache
        doesn't exist or the value is not found in the cache.
        """
        return getattr(instance, '_model_data_cache', {}).get(self.name, NO_CACHE_VALUE)

    def _set_cached_value(self, instance, value):
        """Store a value in the instance's cache, creating the cache if necessary."""
        if not hasattr(instance, '_model_data_cache'):
            instance._model_data_cache = {}
        instance._model_data_cache[self.name] = value

    def _del_cached_value(self, instance):
        """Remove a value from the instance's cache, if the cache exists."""
        if hasattr(instance, '_model_data_cache') and self.name in instance._model_data_cache:
            del instance._model_data_cache[self.name]

    def _use_computed_default(self):
        """Returns boolean indicating if a default value can/should be computed"""
        # Check self._default to see if a default was provided (and not
        # self.default, since that property may be redefined).
        return self._default is None and self.computed_default is not None

    def __get__(self, instance, owner):
        if instance is None:
            return self

        value = self._get_cached_value(instance)
        if value is NO_CACHE_VALUE:
            try:
                value = self.from_json(instance._model_data[self.name])
                self._set_cached_value(instance, value)
            except KeyError:
                # Computed defaults are not stored in cache, but are
                # always recomputed.  Regular defaults are always copied,
                # in case the provided default value is mutable (e.g list or
                # dict).  Regular defaults are also cached.
                if self._use_computed_default():
                    value = self.computed_default(instance)
                else:
                    value = copy.deepcopy(self.default)
                    self._set_cached_value(instance, value)

        return value

    def __set__(self, instance, value):
        # Update both the persistent store and the cache:
        instance._model_data[self.name] = self.to_json(value)
        self._set_cached_value(instance, value)

    def __delete__(self, instance):
        # Try to perform the deletion on the model_data, and accept
        # that it's okay if the key is not present.  (It may never
        # have been persisted at all.)
        try:
            del instance._model_data[self.name]
        except KeyError:
            pass

        # Since we know that the model_data no longer contains the value, we can
        # avoid the possible database lookup that a future get() call would
        # entail by setting the cached value now to its default value.
        # However, we should not store cached values for computed defaults, so
        # we just clear the cache in that case.
        if self._use_computed_default():
            self._del_cached_value(instance)
        else:
            self._set_cached_value(instance, copy.deepcopy(self.default))

    def __repr__(self):
        return "<{0.__class__.__name__} {0._name}>".format(self)

    def to_json(self, value):
        """
        Return value in the form of nested lists and dictionaries (suitable
        for passing to json.dumps).

        This is called during field writes to convert the native python
        type to the value stored in the database
        """
        return value

    def from_json(self, value):
        """
        Return value as a native full featured python type (the inverse of to_json)

        Called during field reads to convert the stored value into a full featured python
        object
        """
        return value

    def read_from(self, model):
        """
        Retrieve the value for this field from the specified model object
        """
        return self.__get__(model, model.__class__)

    def read_json(self, model):
        """
        Retrieve the serialized value for this field from the specified model object
        """
        return self.to_json(self.read_from(model))

    def write_to(self, model, value):
        """
        Set the value for this field to value on the supplied model object
        """
        self.__set__(model, value)

    def delete_from(self, model):
        """
        Delete the value for this field from the supplied model object
        """
        self.__delete__(model)

    def __hash__(self):
        return hash(self.name)

    def __eq__(self, other):
        return self.name == other.name

class Integer(ModelType):
    pass

class Float(ModelType):
    pass

class Boolean(ModelType):
    pass

class Object(ModelType):
    @property
    def default(self):
        if self._default is None:
            return {}
        else:
            return self._default

class List(ModelType):
    @property
    def default(self):
        if self._default is None:
            return []
        else:
            return self._default

class String(ModelType):
    pass

class Any(ModelType):
    pass

class ModelMetaclass(type):
    """
    A metaclass to be used for classes that want to use ModelTypes as class attributes
    to define data access.

    All class attributes that are ModelTypes will be added to the 'fields' attribute on
    the instance.

    Additionally, any namespaces registered in the `xblock.namespace` will be added to
    the instance.
    """
    def __new__(cls, name, bases, attrs):
        fields = set()
        for n, v in attrs.items() + sum([inspect.getmembers(base) for base in bases], []):
            if isinstance(v, ModelType):
                v._name = n
                fields.add(v)

        attrs['fields'] = list(fields)
        return super(ModelMetaclass, cls).__new__(cls, name, bases, attrs)

class NamespacesMetaclass(type):
    """
    A metaclass to be used for classes that want to include namespaced fields in their
    instances.

    Any namespaces registered in the `xblock.namespace` will be added to
    the instance
    """
    def __new__(cls, name, bases, attrs):
        namespaces = []
        for ns_name, namespace in Namespace.load_classes():
            if issubclass(namespace, Namespace):
                attrs[ns_name] = NamespaceDescriptor(namespace)
                namespaces.append(ns_name)
        attrs['namespaces'] = namespaces

        return super(NamespacesMetaclass, cls).__new__(cls, name, bases, attrs)

class ChildrenModelMetaclass(type):
    """
    A ModelMetaclass that transforms the attribute `has_children = True`
    into a List field with an empty scope.
    """
    def __new__(cls, name, bases, attrs):
        if (attrs.get('has_children', False) or
            any(getattr(base, 'has_children', False) for base in bases)):
            attrs['children'] = List(help='The ids of the children of this XBlock', scope=Scope.children)
        else:
            attrs['has_children'] = False

        return super(ChildrenModelMetaclass, cls).__new__(cls, name, bases, attrs)

class NamespaceDescriptor(object):
    def __init__(self, namespace):
        self._namespace = namespace

    def __get__(self, instance, owner):
        return self._namespace(instance)

class Namespace(Plugin):
    """
    A baseclass that sets up machinery for ModelType fields that makes those fields be called
    with the container as the field instance
    """
    __metaclass__ = ModelMetaclass

    entry_point = 'xblock.namespace'

    def __init__(self, container):
        self._container = container

    def __getattribute__(self, name):
        container = super(Namespace, self).__getattribute__('_container')
        namespace_attr = getattr(type(self), name, None)

        if namespace_attr is None or not isinstance(namespace_attr, ModelType):
            return super(Namespace, self).__getattribute__(name)

        return namespace_attr.__get__(container, type(container))

    def __setattr__(self, name, value):
        try:
            container = super(Namespace, self).__getattribute__('_container')
        except AttributeError:
            super(Namespace, self).__setattr__(name, value)
            return

        namespace_attr = getattr(type(self), name, None)

        if namespace_attr is None or not isinstance(namespace_attr, ModelType):
            return super(Namespace, self).__setattr__(name, value)

        return namespace_attr.__set__(container, value)

    def __delattr__(self, name):
        container = super(Namespace, self).__getattribute__('_container')
        namespace_attr = getattr(type(self), name, None)

        if namespace_attr is None or not isinstance(namespace_attr, ModelType):
            return super(Namespace, self).__detattr__(name)

        return namespace_attr.__delete__(container)

class TagCombiningMetaclass(type):
    def __new__(cls, name, bases, attrs):
        class_tags = set([])
        # Collect the tags from all base classes.
        for base in bases:
            try:
                class_tags.update(base._class_tags)
            except AttributeError:
                # Base classes may have no ._class_tags, that's ok.
                pass
        attrs['_class_tags'] = class_tags
        return super(TagCombiningMetaclass, cls).__new__(cls, name, bases, attrs)

class XBlockMetaclass(
    ChildrenModelMetaclass,
    NamespacesMetaclass,
    ModelMetaclass,
    TagCombiningMetaclass,
    ):
    pass

# -- Base Block

class XBlock(Plugin):
    """Base class for XBlocks.

    Derive from this class to create a new kind of XBlock.  There are no
    required methods, but you will probably need at least one view.

    Don't provide the ``__init__`` method when deriving from this class.

    """

    __metaclass__ = XBlockMetaclass

    entry_point = 'xblock.v1'

    parent = Object(help='The id of the parent of this XBlock', default=None, scope=Scope.parent)
    name = String(help="Short name for the block", scope=Scope.settings)
    tags = List(help="Tags for this block", scope=Scope.settings)

    _class_tags = set()

    @classmethod
    def json_handler(cls, fn):
        """Wrap a handler to consume and produce JSON.

        Rather than a Request object, the method will now be passed the
        JSON-decoded body of the request.  Any data returned by the function
        will be JSON-encoded and returned as the response.

        """
        @functools.wraps(fn)
        def wrapper(self, request):
            request_json = json.loads(request.body)
            response_json = json.dumps(fn(self, request_json))
            return Response(response_json, content_type='application/json')
        return wrapper

    @classmethod
    def tag(cls, tags):
        """Add the words in `tags` as class tags to this class."""
        def dec(cls):
            # Add in this class's tags
            cls._class_tags.update(tags.replace(",", " ").split())
            return cls
        return dec

    @classmethod
    def load_tagged_classes(cls, tag):
        """Produce a sequence of all XBlock classes tagged with `tag`."""
        for name, class_ in cls.load_classes():
            if tag in class_._class_tags:
                yield name, class_

    @classmethod
    def preprocess_input(cls, node, usage_factory):
        """The class can adjust a parsed Usage tree."""
        return node

    @classmethod
    def postprocess_input(cls, node, usage_factory):
        """The class can adjust a parsed Usage tree."""
        return node

    def __init__(self, runtime, model_data):
        """

        `runtime` is an instance of :class:`xblock.core.Runtime`. Use it to
        access the environment.  It is available in XBlock code as
        ``self.runtime``.

        `model_data` is a dictionary-like interface to runtime storage.
        XBlock uses it to implement your storage fields.

        """
        self.runtime = runtime
        self._model_data = model_data

    def __repr__(self):
        attrs = []
        for field in self.fields:
            value = getattr(self, field.name)
            if isinstance(value, basestring):
                value = value.strip()
                if len(value) > 40:
                    value = value[:37] + "..."
            attrs.append(" %s=%r" % (field.name, value))
        return "<%s @%04X%s>" % (
            self.__class__.__name__,
            id(self) % 0xFFFF,
            ','.join(attrs)
        )
<commit_msg>Code review feedback.<commit_after>"""Implementation of the XBlock facility.

This code is in the Runtime layer, because it is authored once by edX
and used by all runtimes.

"""
import copy
import functools
import inspect
try:
    import simplesjson as json
except ImportError:
    import json

from collections import namedtuple
from webob import Response

from .plugin import Plugin

class BlockScope(object):
    USAGE, DEFINITION, TYPE, ALL = xrange(4)

class Scope(namedtuple('ScopeBase', 'user block')):
    pass

class Sentinel(object):
    """
    Class for implementing sentinel objects (only equal to themselves).
    """
    def __init__(self, name):
        """
        `name` is the name used to identify the sentinel (which will
            be displayed as the __repr__) of the sentinel.
        """
        self.name = name

    def __repr__(self):
        return self.name

Scope.content = Scope(user=False, block=BlockScope.DEFINITION)
Scope.settings = Scope(user=False, block=BlockScope.USAGE)
Scope.user_state = Scope(user=True, block=BlockScope.USAGE)
Scope.preferences = Scope(user=True, block=BlockScope.TYPE)
Scope.user_info = Scope(user=True, block=BlockScope.ALL)
Scope.children = Sentinel('Scope.children')
Scope.parent = Sentinel('Scope.parent')

# define a placeholder ('nil') value to indicate when nothing has been stored in
# the cache.  ("None" may be a valid value in the cache, so we cannot use it.)
NO_CACHE_VALUE = object()

class ModelType(object):
    """
    A field class that can be used as a class attribute to define what data the class will want
    to refer to.

    When the class is instantiated, it will be available as an instance attribute of the same
    name, by proxying through to self._model_data on the containing object.

    Parameters:
      `help` : documentation of field class, suitable for presenting in a GUI (defaults to None)
      `default` : static value to default to if not otherwise specified (defaults to None)
      `scope` : the scope in which this field class is used (defaults to Scope.content)
      `computed_default` : provides the ability to specify a function for computing a default value (defaults to None)
      `display_name` : the display name for the field class, suitable for presenting in a GUI (defaults to name of class)
      `values` : for field classes with a finite number of valid values, provides the ability to specify the set of
               valid values. This can be specified as either a static return value, or a function that generates
               the valid values.
    """

    def __init__(self, help=None, default=None, scope=Scope.content, computed_default=None, display_name=None,
                 values=None):
        self._name = "unknown"
        self.help = help
        self._default = default
        self.computed_default = computed_default
        self.scope = scope
        self._display_name = display_name
        self._values = values

    @property
    def default(self):
        return self._default

    @property
    def name(self):
        return self._name

    @property
    def values(self):
        """
        Returns the valid values for this class.

        If this field class does not define a finite number of valid values, this method will return None.
        """
        if callable(self._values):
            return self._values()
        else:
            return self._values

    @property
    def display_name(self):
        """
        Returns the display name for this class, suitable for use in a GUI.

        If no display name has been set, returns the name of the class.
        """
        return self._display_name if self._display_name is not None else self.name

    def _get_cached_value(self, instance):
        """
        Return a value from the instance's cache, or a marker value if either the cache
        doesn't exist or the value is not found in the cache.
        """
        return getattr(instance, '_model_data_cache', {}).get(self.name, NO_CACHE_VALUE)

    def _set_cached_value(self, instance, value):
        """Store a value in the instance's cache, creating the cache if necessary."""
        if not hasattr(instance, '_model_data_cache'):
            instance._model_data_cache = {}
        instance._model_data_cache[self.name] = value

    def _del_cached_value(self, instance):
        """Remove a value from the instance's cache, if the cache exists."""
        if hasattr(instance, '_model_data_cache') and self.name in instance._model_data_cache:
            del instance._model_data_cache[self.name]

    def _use_computed_default(self):
        """Returns boolean indicating if a default value can/should be computed"""
        # Check self._default to see if a default was provided (and not
        # self.default, since that property may be redefined).
        return self._default is None and self.computed_default is not None

    def __get__(self, instance, owner):
        if instance is None:
            return self

        value = self._get_cached_value(instance)
        if value is NO_CACHE_VALUE:
            try:
                value = self.from_json(instance._model_data[self.name])
                self._set_cached_value(instance, value)
            except KeyError:
                # Computed defaults are not stored in cache, but are
                # always recomputed.  Regular defaults are always copied,
                # in case the provided default value is mutable (e.g list or
                # dict).  Regular defaults are also cached.
                if self._use_computed_default():
                    value = self.computed_default(instance)
                else:
                    value = copy.deepcopy(self.default)
                    self._set_cached_value(instance, value)

        return value

    def __set__(self, instance, value):
        # Update both the persistent store and the cache:
        instance._model_data[self.name] = self.to_json(value)
        self._set_cached_value(instance, value)

    def __delete__(self, instance):
        # Try to perform the deletion on the model_data, and accept
        # that it's okay if the key is not present.  (It may never
        # have been persisted at all.)
        try:
            del instance._model_data[self.name]
        except KeyError:
            pass

        # Since we know that the model_data no longer contains the value, we can
        # avoid the possible database lookup that a future get() call would
        # entail by setting the cached value now to its default value.
        # However, we should not store cached values for computed defaults, so
        # we just clear the cache in that case.
        if self._use_computed_default():
            self._del_cached_value(instance)
        else:
            self._set_cached_value(instance, copy.deepcopy(self.default))

    def __repr__(self):
        return "<{0.__class__.__name__} {0._name}>".format(self)

    def to_json(self, value):
        """
        Return value in the form of nested lists and dictionaries (suitable
        for passing to json.dumps).

        This is called during field writes to convert the native python
        type to the value stored in the database
        """
        return value

    def from_json(self, value):
        """
        Return value as a native full featured python type (the inverse of to_json)

        Called during field reads to convert the stored value into a full featured python
        object
        """
        return value

    def read_from(self, model):
        """
        Retrieve the value for this field from the specified model object
        """
        return self.__get__(model, model.__class__)

    def read_json(self, model):
        """
        Retrieve the serialized value for this field from the specified model object
        """
        return self.to_json(self.read_from(model))

    def write_to(self, model, value):
        """
        Set the value for this field to value on the supplied model object
        """
        self.__set__(model, value)

    def delete_from(self, model):
        """
        Delete the value for this field from the supplied model object
        """
        self.__delete__(model)

    def __hash__(self):
        return hash(self.name)

    def __eq__(self, other):
        return self.name == other.name

class Integer(ModelType):
    pass

class Float(ModelType):
    pass

class Boolean(ModelType):
    pass

class Object(ModelType):
    @property
    def default(self):
        if self._default is None:
            return {}
        else:
            return self._default

class List(ModelType):
    @property
    def default(self):
        if self._default is None:
            return []
        else:
            return self._default

class String(ModelType):
    pass

class Any(ModelType):
    pass

class ModelMetaclass(type):
    """
    A metaclass to be used for classes that want to use ModelTypes as class attributes
    to define data access.

    All class attributes that are ModelTypes will be added to the 'fields' attribute on
    the instance.

    Additionally, any namespaces registered in the `xblock.namespace` will be added to
    the instance.
    """
    def __new__(cls, name, bases, attrs):
        fields = set()
        for n, v in attrs.items() + sum([inspect.getmembers(base) for base in bases], []):
            if isinstance(v, ModelType):
                v._name = n
                fields.add(v)

        attrs['fields'] = list(fields)
        return super(ModelMetaclass, cls).__new__(cls, name, bases, attrs)

class NamespacesMetaclass(type):
    """
    A metaclass to be used for classes that want to include namespaced fields in their
    instances.

    Any namespaces registered in the `xblock.namespace` will be added to
    the instance
    """
    def __new__(cls, name, bases, attrs):
        namespaces = []
        for ns_name, namespace in Namespace.load_classes():
            if issubclass(namespace, Namespace):
                attrs[ns_name] = NamespaceDescriptor(namespace)
                namespaces.append(ns_name)
        attrs['namespaces'] = namespaces

        return super(NamespacesMetaclass, cls).__new__(cls, name, bases, attrs)

class ChildrenModelMetaclass(type):
    """
    A ModelMetaclass that transforms the attribute `has_children = True`
    into a List field with an empty scope.
    """
    def __new__(cls, name, bases, attrs):
        if (attrs.get('has_children', False) or
            any(getattr(base, 'has_children', False) for base in bases)):
            attrs['children'] = List(help='The ids of the children of this XBlock', scope=Scope.children)
        else:
            attrs['has_children'] = False

        return super(ChildrenModelMetaclass, cls).__new__(cls, name, bases, attrs)

class NamespaceDescriptor(object):
    def __init__(self, namespace):
        self._namespace = namespace

    def __get__(self, instance, owner):
        return self._namespace(instance)

class Namespace(Plugin):
    """
    A baseclass that sets up machinery for ModelType fields that makes those fields be called
    with the container as the field instance
    """
    __metaclass__ = ModelMetaclass

    entry_point = 'xblock.namespace'

    def __init__(self, container):
        self._container = container

    def __getattribute__(self, name):
        container = super(Namespace, self).__getattribute__('_container')
        namespace_attr = getattr(type(self), name, None)

        if namespace_attr is None or not isinstance(namespace_attr, ModelType):
            return super(Namespace, self).__getattribute__(name)

        return namespace_attr.__get__(container, type(container))

    def __setattr__(self, name, value):
        try:
            container = super(Namespace, self).__getattribute__('_container')
        except AttributeError:
            super(Namespace, self).__setattr__(name, value)
            return

        namespace_attr = getattr(type(self), name, None)

        if namespace_attr is None or not isinstance(namespace_attr, ModelType):
            return super(Namespace, self).__setattr__(name, value)

        return namespace_attr.__set__(container, value)

    def __delattr__(self, name):
        container = super(Namespace, self).__getattribute__('_container')
        namespace_attr = getattr(type(self), name, None)

        if namespace_attr is None or not isinstance(namespace_attr, ModelType):
            return super(Namespace, self).__detattr__(name)

        return namespace_attr.__delete__(container)

class TagCombiningMetaclass(type):
    def __new__(cls, name, bases, attrs):
        class_tags = set([])
        # Collect the tags from all base classes.
        for base in bases:
            try:
                class_tags.update(base._class_tags)
            except AttributeError:
                # Base classes may have no ._class_tags, that's ok.
                pass
        attrs['_class_tags'] = class_tags
        return super(TagCombiningMetaclass, cls).__new__(cls, name, bases, attrs)

class XBlockMetaclass(
    ChildrenModelMetaclass,
    NamespacesMetaclass,
    ModelMetaclass,
    TagCombiningMetaclass,
    ):
    pass

# -- Base Block

class XBlock(Plugin):
    """Base class for XBlocks.

    Derive from this class to create a new kind of XBlock.  There are no
    required methods, but you will probably need at least one view.

    Don't provide the ``__init__`` method when deriving from this class.

    """

    __metaclass__ = XBlockMetaclass

    entry_point = 'xblock.v1'

    parent = Object(help='The id of the parent of this XBlock', default=None, scope=Scope.parent)
    name = String(help="Short name for the block", scope=Scope.settings)
    tags = List(help="Tags for this block", scope=Scope.settings)

    _class_tags = set()

    @classmethod
    def json_handler(cls, fn):
        """Wrap a handler to consume and produce JSON.

        Rather than a Request object, the method will now be passed the
        JSON-decoded body of the request.  Any data returned by the function
        will be JSON-encoded and returned as the response.

        """
        @functools.wraps(fn)
        def wrapper(self, request):
            request_json = json.loads(request.body)
            response_json = json.dumps(fn(self, request_json))
            return Response(response_json, content_type='application/json')
        return wrapper

    @classmethod
    def tag(cls, tags):
        """Add the words in `tags` as class tags to this class."""
        def dec(cls):
            # Add in this class's tags
            cls._class_tags.update(tags.replace(",", " ").split())
            return cls
        return dec

    @classmethod
    def load_tagged_classes(cls, tag):
        """Produce a sequence of all XBlock classes tagged with `tag`."""
        for name, class_ in cls.load_classes():
            if tag in class_._class_tags:
                yield name, class_

    @classmethod
    def preprocess_input(cls, node, usage_factory):
        """The class can adjust a parsed Usage tree."""
        return node

    @classmethod
    def postprocess_input(cls, node, usage_factory):
        """The class can adjust a parsed Usage tree."""
        return node

    def __init__(self, runtime, model_data):
        """

        `runtime` is an instance of :class:`xblock.core.Runtime`. Use it to
        access the environment.  It is available in XBlock code as
        ``self.runtime``.

        `model_data` is a dictionary-like interface to runtime storage.
        XBlock uses it to implement your storage fields.

        """
        self.runtime = runtime
        self._model_data = model_data

    def __repr__(self):
        attrs = []
        for field in self.fields:
            value = getattr(self, field.name)
            if isinstance(value, basestring):
                value = value.strip()
                if len(value) > 40:
                    value = value[:37] + "..."
            attrs.append(" %s=%r" % (field.name, value))
        return "<%s @%04X%s>" % (
            self.__class__.__name__,
            id(self) % 0xFFFF,
            ','.join(attrs)
        )
<|endoftext|>


<commit_before>import datetime
import unittest

from iso8601 import ParseError

from apel.parsers import BlahParser
from apel.db.records.record import InvalidRecordException

class ParserBlahTest(unittest.TestCase):
    '''
    Test case for LSF parser
    '''

    def setUp(self):
        self.parser = BlahParser('testSite', 'testHost')

    def test_parse(self):
        line1 = ('"timestamp=2012-05-20 23:59:47" '
                +'"userDN=/O=GermanGrid/OU=UniWuppertal/CN=Torsten Harenberg" '
                +'"userFQAN=/atlas/Role=production/Capability=NULL" '
                +'"ceID=cream-2-fzk.gridka.de:8443/cream-pbs-atlasXL" '
                +'"jobID=CREAM410741480" "lrmsID=9575064.lrms1" "localUser=11999"')

        line1_values = {"TimeStamp": datetime.datetime(2012, 5, 20, 23, 59, 47),
                        "GlobalUserName":"/O=GermanGrid/OU=UniWuppertal/CN=Torsten Harenberg",
                        "FQAN": "/atlas/Role=production/Capability=NULL",
                        "CE": "cream-2-fzk.gridka.de:8443/cream-pbs-atlasXL",
                        "GlobalJobId": "CREAM410741480",
                        "LrmsId": "9575064.lrms1",
                        }

        cases = {}
        cases[line1] = line1_values

        for line in cases.keys():

            record = self.parser.parse(line)
            cont = record._record_content

            # Keys presence in record
            self.assertTrue(cont.has_key("TimeStamp"))
            self.assertTrue(cont.has_key("GlobalUserName"))
            self.assertTrue(cont.has_key("FQAN"))
            self.assertTrue(cont.has_key("CE"))
            self.assertTrue(cont.has_key("GlobalJobId"))
            self.assertTrue(cont.has_key("LrmsId"))

            for key in cases[line].keys():
                self.assertEqual(cont[key], cases[line][key], "%s != %s for key %s" % (cont[key], cases[line][key], key))

    def test_invalid_timestamp(self):
        '''
        Test if parser raises exception for invalid timestamp
        '''

        line_invalidtimestamp = ('"timestamp=2012-05-20A23:59:47" '
                +'"userDN=/O=GermanGrid/OU=UniWuppertal/CN=Torsten Harenberg" '
                +'"userFQAN=/atlas/Role=production/Capability=NULL" '
                +'"ceID=cream-2-fzk.gridka.de:8443/cream-pbs-atlasXL" '
                +'"jobID=CREAM410741480" "lrmsID=9575064.lrms1" "localUser=11999"')
        # Should raise an exception - we have 'A' between date and time
        try:
            # iso8601 >= 0.1.9 version of test (now stricter in what it accepts)
            self.assertRaises(ParseError, self.parser.parse, line_invalidtimestamp)
        except:
            # iso8601 <= 0.1.8 version of test (should be deprecated)
            self.assertRaises(InvalidRecordException, self.parser.parse, line_invalidtimestamp)

    def test_invalid_record_line(self):
        line_invalid = ('"timestamp=2012-05-20 23:59:47" '
                +'"userDN=/O=GermanGrid/OU=UniWuppertal/CN=Torsten Harenberg" '
                +'"userFQAN=/atlas&Role=production/Capability=NULL" '
                +'"ceID=cream-2-fzk.gridka.de:8443/cream-pbs-atlasXL" '
                +'"jobID=CREAM410741480"&sd"lrmsID=9575064.lrms1" "localUser=11999"')
        self.assertRaises(ValueError, self.parser.parse, line_invalid)

    def test_multiple_fqans(self):
        """The parser should take the first FQAN to be the primary FQAN."""
        lines = (
            '"timestamp=2014-05-18 00:00:58" "userDN=/C=CA/O=Grid/OU=triumf.ca/'
            'CN=Asoka De Silva GC1" "userFQAN=/atlas/Role=pilot/Capability=NULL'
            '" "userFQAN=/atlas/Role=NULL/Capability=NULL" "userFQAN=/atlas/ca/'
            'Role=NULL/Capability=NULL" "userFQAN=/atlas/lcg1/Role=NULL/Capabil'
            'ity=NULL" "ceID=ce1.triumf.ca:8443/cream-pbs-atlas" "jobID=CREAM66'
            '3276716" "lrmsID=15876368.ce1.triumf.ca" "localUser=41200" "client'
            'ID=cream_663276716"',
            '"timestamp=2014-05-18 00:03:00" "userDN=/DC=ch/DC=cern/OU=Organic '
            'Units/OU=Users/CN=atlpilo2/CN=531497/CN=Robot: ATLAS Pilot2" "user'
            'FQAN=/atlas/Role=pilot/Capability=NULL" "userFQAN=/atlas/Role=NULL'
            '/Capability=NULL" "userFQAN=/atlas/lcg1/Role=NULL/Capability=NULL"'
            ' "userFQAN=/atlas/usatlas/Role=NULL/Capability=NULL" "ceID=ce1.tri'
            'umf.ca:8443/cream-pbs-atlas" "jobID=CREAM503347888" "lrmsID=158764'
            '80.ce1.triumf.ca" "localUser=41200" "clientID=cream_503347888"',
        )
        values = (
            (datetime.datetime(2014, 5, 18, 00, 00, 58),
             '/C=CA/O=Grid/OU=triumf.ca/CN=Asoka De Silva GC1',
             '/atlas/Role=pilot/Capability=NULL',  # primary FQAN is first one
             'atlas',
             '/atlas',
             'Role=pilot',
             'ce1.triumf.ca:8443/cream-pbs-atlas',
             'CREAM663276716',
             '15876368.ce1.triumf.ca',
             datetime.datetime(2014, 5, 17, 00, 00, 58),
             datetime.datetime(2014, 6, 15, 00, 00, 58),
             0
             ),
            (datetime.datetime(2014, 5, 18, 00, 03, 00),
             '/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=atlpilo2/CN=531497/CN'
                '=Robot: ATLAS Pilot2',
             '/atlas/Role=pilot/Capability=NULL',  # primary FQAN is first one
             'atlas',
             '/atlas',
             'Role=pilot',
             'ce1.triumf.ca:8443/cream-pbs-atlas',
             'CREAM503347888',
             '15876480.ce1.triumf.ca',
             datetime.datetime(2014, 5, 17, 00, 03, 00),
             datetime.datetime(2014, 6, 15, 00, 03, 00),
             0
             ),
        )

        fields = ('TimeStamp', 'GlobalUserName', 'FQAN', 'VO', 'VOGroup',
                  'VORole', 'CE', 'GlobalJobId', 'LrmsId', 'ValidFrom',
                  'ValidUntil', 'Processed')

        cases = {}
        for line, value in zip(lines, values):
            cases[line] = dict(zip(fields, value))

        for line in cases.keys():
            record = self.parser.parse(line)
            cont = record._record_content

            # Check that 'Site' has been set
            self.assertEqual(cont['Site'], 'testSite')

            for key in cases[line].keys():
                # Check all fields are present
                self.assertTrue(key in cont, "Key '%s' not in record." % key)
                # Check values are correct
                self.assertEqual(cont[key], cases[line][key],
                                 "'%s' != '%s' for key '%s'" %
                                 (cont[key], cases[line][key], key))

if __name__ == '__main__':
    unittest.main()
<commit_msg>Remove trailing whitespace from test/test_blah.py<commit_after>import datetime
import unittest

from iso8601 import ParseError

from apel.parsers import BlahParser
from apel.db.records.record import InvalidRecordException

class ParserBlahTest(unittest.TestCase):
    '''
    Test case for LSF parser
    '''

    def setUp(self):
        self.parser = BlahParser('testSite', 'testHost')

    def test_parse(self):
        line1 = ('"timestamp=2012-05-20 23:59:47" '
                +'"userDN=/O=GermanGrid/OU=UniWuppertal/CN=Torsten Harenberg" '
                +'"userFQAN=/atlas/Role=production/Capability=NULL" '
                +'"ceID=cream-2-fzk.gridka.de:8443/cream-pbs-atlasXL" '
                +'"jobID=CREAM410741480" "lrmsID=9575064.lrms1" "localUser=11999"')

        line1_values = {"TimeStamp": datetime.datetime(2012, 5, 20, 23, 59, 47),
                        "GlobalUserName":"/O=GermanGrid/OU=UniWuppertal/CN=Torsten Harenberg",
                        "FQAN": "/atlas/Role=production/Capability=NULL",
                        "CE": "cream-2-fzk.gridka.de:8443/cream-pbs-atlasXL",
                        "GlobalJobId": "CREAM410741480",
                        "LrmsId": "9575064.lrms1",
                        }

        cases = {}
        cases[line1] = line1_values

        for line in cases.keys():

            record = self.parser.parse(line)
            cont = record._record_content

            # Keys presence in record
            self.assertTrue(cont.has_key("TimeStamp"))
            self.assertTrue(cont.has_key("GlobalUserName"))
            self.assertTrue(cont.has_key("FQAN"))
            self.assertTrue(cont.has_key("CE"))
            self.assertTrue(cont.has_key("GlobalJobId"))
            self.assertTrue(cont.has_key("LrmsId"))

            for key in cases[line].keys():
                self.assertEqual(cont[key], cases[line][key], "%s != %s for key %s" % (cont[key], cases[line][key], key))

    def test_invalid_timestamp(self):
        '''
        Test if parser raises exception for invalid timestamp
        '''

        line_invalidtimestamp = ('"timestamp=2012-05-20A23:59:47" '
                +'"userDN=/O=GermanGrid/OU=UniWuppertal/CN=Torsten Harenberg" '
                +'"userFQAN=/atlas/Role=production/Capability=NULL" '
                +'"ceID=cream-2-fzk.gridka.de:8443/cream-pbs-atlasXL" '
                +'"jobID=CREAM410741480" "lrmsID=9575064.lrms1" "localUser=11999"')
        # Should raise an exception - we have 'A' between date and time
        try:
            # iso8601 >= 0.1.9 version of test (now stricter in what it accepts)
            self.assertRaises(ParseError, self.parser.parse, line_invalidtimestamp)
        except:
            # iso8601 <= 0.1.8 version of test (should be deprecated)
            self.assertRaises(InvalidRecordException, self.parser.parse, line_invalidtimestamp)

    def test_invalid_record_line(self):
        line_invalid = ('"timestamp=2012-05-20 23:59:47" '
                +'"userDN=/O=GermanGrid/OU=UniWuppertal/CN=Torsten Harenberg" '
                +'"userFQAN=/atlas&Role=production/Capability=NULL" '
                +'"ceID=cream-2-fzk.gridka.de:8443/cream-pbs-atlasXL" '
                +'"jobID=CREAM410741480"&sd"lrmsID=9575064.lrms1" "localUser=11999"')
        self.assertRaises(ValueError, self.parser.parse, line_invalid)

    def test_multiple_fqans(self):
        """The parser should take the first FQAN to be the primary FQAN."""
        lines = (
            '"timestamp=2014-05-18 00:00:58" "userDN=/C=CA/O=Grid/OU=triumf.ca/'
            'CN=Asoka De Silva GC1" "userFQAN=/atlas/Role=pilot/Capability=NULL'
            '" "userFQAN=/atlas/Role=NULL/Capability=NULL" "userFQAN=/atlas/ca/'
            'Role=NULL/Capability=NULL" "userFQAN=/atlas/lcg1/Role=NULL/Capabil'
            'ity=NULL" "ceID=ce1.triumf.ca:8443/cream-pbs-atlas" "jobID=CREAM66'
            '3276716" "lrmsID=15876368.ce1.triumf.ca" "localUser=41200" "client'
            'ID=cream_663276716"',
            '"timestamp=2014-05-18 00:03:00" "userDN=/DC=ch/DC=cern/OU=Organic '
            'Units/OU=Users/CN=atlpilo2/CN=531497/CN=Robot: ATLAS Pilot2" "user'
            'FQAN=/atlas/Role=pilot/Capability=NULL" "userFQAN=/atlas/Role=NULL'
            '/Capability=NULL" "userFQAN=/atlas/lcg1/Role=NULL/Capability=NULL"'
            ' "userFQAN=/atlas/usatlas/Role=NULL/Capability=NULL" "ceID=ce1.tri'
            'umf.ca:8443/cream-pbs-atlas" "jobID=CREAM503347888" "lrmsID=158764'
            '80.ce1.triumf.ca" "localUser=41200" "clientID=cream_503347888"',
        )
        values = (
            (datetime.datetime(2014, 5, 18, 00, 00, 58),
             '/C=CA/O=Grid/OU=triumf.ca/CN=Asoka De Silva GC1',
             '/atlas/Role=pilot/Capability=NULL',  # primary FQAN is first one
             'atlas',
             '/atlas',
             'Role=pilot',
             'ce1.triumf.ca:8443/cream-pbs-atlas',
             'CREAM663276716',
             '15876368.ce1.triumf.ca',
             datetime.datetime(2014, 5, 17, 00, 00, 58),
             datetime.datetime(2014, 6, 15, 00, 00, 58),
             0
             ),
            (datetime.datetime(2014, 5, 18, 00, 03, 00),
             '/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=atlpilo2/CN=531497/CN'
                '=Robot: ATLAS Pilot2',
             '/atlas/Role=pilot/Capability=NULL',  # primary FQAN is first one
             'atlas',
             '/atlas',
             'Role=pilot',
             'ce1.triumf.ca:8443/cream-pbs-atlas',
             'CREAM503347888',
             '15876480.ce1.triumf.ca',
             datetime.datetime(2014, 5, 17, 00, 03, 00),
             datetime.datetime(2014, 6, 15, 00, 03, 00),
             0
             ),
        )

        fields = ('TimeStamp', 'GlobalUserName', 'FQAN', 'VO', 'VOGroup',
                  'VORole', 'CE', 'GlobalJobId', 'LrmsId', 'ValidFrom',
                  'ValidUntil', 'Processed')

        cases = {}
        for line, value in zip(lines, values):
            cases[line] = dict(zip(fields, value))

        for line in cases.keys():
            record = self.parser.parse(line)
            cont = record._record_content

            # Check that 'Site' has been set
            self.assertEqual(cont['Site'], 'testSite')

            for key in cases[line].keys():
                # Check all fields are present
                self.assertTrue(key in cont, "Key '%s' not in record." % key)
                # Check values are correct
                self.assertEqual(cont[key], cases[line][key],
                                 "'%s' != '%s' for key '%s'" %
                                 (cont[key], cases[line][key], key))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import numbers
from types import GeneratorType
from fiona.collection import Collection
import netCDF4
import numpy
from shapely.geometry.base import BaseGeometry
import six

class ParameterNotValidError(ValueError):
    """Indicates that a value is not valid for the parameter type"""

class ParameterBase(type):
    """Parameter metaclass, used to register parameter classes for lookup by name."""

    _parameters_by_id = {}

    def __new__(mcs, name, bases, attrs):
        new_class = super(ParameterBase, mcs).__new__(mcs, name, bases, attrs)

        name = getattr(new_class, 'id', None)
        if name:
            mcs._parameters_by_id[new_class.id] = new_class

        setattr(new_class, '_parameters_by_id', mcs._parameters_by_id)

        return new_class

class Parameter(six.with_metaclass(ParameterBase)):
    """
    Base parameter (input, output, or uniform) class for a task or workflow. Extended to implement specific parameter
    types.
    """

    id = None  # A unique name for this parameter, used for serialization

    def __init__(self, name, required=True):
        """
        :param name: The parameter name. This will be used as a keyword argument to the task's `execute()` method.
        :param required: When True, input validation will fail if no value is provided.
        """

        self.name = name
        self.required = required

    @classmethod
    def by_id(cls, param_id):
        return cls._parameters_by_id.get(param_id)

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        return value

    def serialize_args(self):
        """Returns (args, kwargs) to be used when deserializing this parameter."""

        return (self.name,), {'required': True}

    @staticmethod
    def deserialize_args(args, kwargs):
        """Returns deserialized forms of (args, kwargs) to be used when re-constructing this parameter."""

        return args, kwargs

class ParameterCollection(object):
    """Manages a collection of parameter types and values"""

    def __init__(self, parameters):
        """
        :param parameters: A list of `Parameter` objects
        """

        self.parameters = parameters
        self.by_name = {p.name: p for p in self.parameters}
        self.values = {}

    def __setitem__(self, key, value):
        self.values[key] = self.by_name[key].clean(value)

    def __getitem__(self, item):
        return self.values.get(item)

    @property
    def is_complete(self):
        """Do all required parameters have values?"""

        return all(p.name in self.values for p in self.parameters if p.required)

    def format_args(self):
        """Returns dictionary containing values for all named parameters."""

        return self.values

class AnyParameter(Parameter):
    """Accepts any value"""

    id = 'any'

class MultiParameter(Parameter):
    """
    Accepts a value matching one of several defined types.
    E.g., `MultiParameter([StringParameter(''), DictParameter('')])` will accept both strings and dictionaries.
    """

    id = 'multiple'

    def __init__(self, types, *args, **kwargs):
        """
        :param types: A list of `Parameter` instances to accept.
        """

        super(MultiParameter, self).__init__(*args, **kwargs)

        self.types = types

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        for parameter_obj in self.types:
            try:
                return parameter_obj.clean(value)
            except ParameterNotValidError:
                continue

        raise ParameterNotValidError

    def serialize_args(self):
        """Returns (args, kwargs) to be used when deserializing this parameter."""

        args, kwargs = super(MultiParameter, self).serialize_args()
        args.insert(0, [[t.id, t.serialize_args()] for t in self.types])

        return args, kwargs

    @staticmethod
    def deserialize_args(args, kwargs):
        args, kwargs = super(MultiParameter, MultiParameter).deserialize_args(args[1:], kwargs)

        types = []
        for type_id, type_args in args[0]:
            type_cls = Parameter.by_id(type_id)
            type_args, type_kwargs = type_cls.deserialize_args(*type_args)
            types.append(type_cls(*type_args, **type_kwargs))

        args.insert(0, types)

        return args, kwargs

class ListParameter(Parameter):
    """
    Accepts a list of a given parameter type. This parameter will accept either a list-type object (list, tuple, set)
    or a generator object. If it receives a generator object, `clean()` will also return a generator object. Otherwise,
    it will return a list.
    """

    id = 'list'

    def __init__(self, param_type, *args, **kwargs):
        """
        :param param_type: A `Parameter` instance.
        """

        super(ListParameter, self).__init__(*args, **kwargs)

        self.param_type = param_type

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        if not isinstance(value, (list, tuple, set, GeneratorType)):
            value = [value]

        gen = (self.param_type.clean(x) for x in value)

        if isinstance(value, GeneratorType):
            return gen
        else:
            return list(gen)

    def serialize_args(self):
        """Returns (args, kwargs) to be used when deserializing this parameter."""

        args, kwargs = super(ListParameter, self).serialize_args()
        args.insert(0, [self.param_type.id, self.param_type.serialize_args()])

    @staticmethod
    def deserialize_args(args, kwargs):
        type_id, type_args = args[0]
        args, kwargs = super(ListParameter, ListParameter).deserialize_args(args[1:], kwargs)

        type_cls = Parameter.by_id(type_id)
        type_args, type_kwargs = type_cls.deserialize_args(*type_args)
        args.insert(0, type_cls(*type_args, **type_kwargs))

        return args, kwargs

class StringParameter(Parameter):
    """Accepts a string. Will convert numbers to string."""

    id = 'string'

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        if isinstance(value, six.string_types):
            return value
        elif isinstance(value, numbers.Number):
            return str(value)

        raise ParameterNotValidError

class NumberParameter(Parameter):
    """Accepts a number. Will convert numeric strings."""

    id = 'number'

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        if isinstance(value, numbers.Number):
            return value
        elif isinstance(value, six.string_types):
            try:
                value = float(value)
                return int(value) if value.is_integer() else value
            except ValueError:
                raise ParameterNotValidError

        raise ParameterNotValidError

class IntParameter(NumberParameter):
    """Accepts an integer value. Will convert strings, other numbers."""

    id = 'int'

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        return int(super(IntParameter, self).clean(value))

class FloatParameter(NumberParameter):
    """Accepts a float value. Will convert strings, other numbers."""

    id = 'float'

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        return float(super(FloatParameter, self).clean(value))

class DictParameter(Parameter):
    """Accepts a dictionary"""

    id = 'dict'

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        if isinstance(value, dict):
            return value

        raise ParameterNotValidError

class NdArrayParameter(Parameter):
    """Accepts a numpy array or Python list"""

    id = 'array'

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        if isinstance(value, numpy.ndarray):
            return value
        elif isinstance(value, (list, tuple)):
            return numpy.array(value)

        raise ParameterNotValidError

class FeatureParameter(Parameter):
    """Accepts a single feature (as a Shapely geometry object)"""

    id = 'feature'

    # Todo: geometry type checking

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        if isinstance(value, BaseGeometry):
            return value

        raise ParameterNotValidError

class FeatureCollectionParameter(FeatureParameter):
    """Accepts a list of features (as Shapely geometry objects)"""

    id = 'feature_collection'

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        if isinstance(value, (list, tuple)):
            return [super(FeatureCollectionParameter, self).clean(x) for x in value]

        raise ParameterNotValidError

class RasterDatasetParameter(Parameter):
    """Accepts a NetCDF `Dataset` object"""

    id = 'raster_dataset'

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        if isinstance(value, netCDF4.Dataset):
            return value

        raise ParameterNotValidError

class FeatureDatasetParameter(Parameter):
    """Accepts a Fiona `Collection` object"""

    id = 'feature_dataset'

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        if isinstance(value, Collection):
            return value

        raise ParameterNotValidError
<commit_msg>Add raster and registered dataset parameters.<commit_after>import numbers
import os
from types import GeneratorType

from django.core.exceptions import ObjectDoesNotExist
from fiona.collection import Collection
import netCDF4
import numpy
from shapely.geometry.base import BaseGeometry
import six

from ncdjango.geoprocessing.data import Raster
from ncdjango.models import Service, SERVICE_DATA_ROOT

class ParameterNotValidError(ValueError):
    """Indicates that a value is not valid for the parameter type"""

class ParameterBase(type):
    """Parameter metaclass, used to register parameter classes for lookup by name."""

    _parameters_by_id = {}

    def __new__(mcs, name, bases, attrs):
        new_class = super(ParameterBase, mcs).__new__(mcs, name, bases, attrs)

        name = getattr(new_class, 'id', None)
        if name:
            mcs._parameters_by_id[new_class.id] = new_class

        setattr(new_class, '_parameters_by_id', mcs._parameters_by_id)

        return new_class

class Parameter(six.with_metaclass(ParameterBase)):
    """
    Base parameter (input, output, or uniform) class for a task or workflow. Extended to implement specific parameter
    types.
    """

    id = None  # A unique name for this parameter, used for serialization

    def __init__(self, name, required=True):
        """
        :param name: The parameter name. This will be used as a keyword argument to the task's `execute()` method.
        :param required: When True, input validation will fail if no value is provided.
        """

        self.name = name
        self.required = required

    @classmethod
    def by_id(cls, param_id):
        return cls._parameters_by_id.get(param_id)

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        return value

    def serialize_args(self):
        """Returns (args, kwargs) to be used when deserializing this parameter."""

        return (self.name,), {'required': True}

    @staticmethod
    def deserialize_args(args, kwargs):
        """Returns deserialized forms of (args, kwargs) to be used when re-constructing this parameter."""

        return args, kwargs

class ParameterCollection(object):
    """Manages a collection of parameter types and values"""

    def __init__(self, parameters):
        """
        :param parameters: A list of `Parameter` objects
        """

        self.parameters = parameters
        self.by_name = {p.name: p for p in self.parameters}
        self.values = {}

    def __setitem__(self, key, value):
        self.values[key] = self.by_name[key].clean(value)

    def __getitem__(self, item):
        return self.values.get(item)

    @property
    def is_complete(self):
        """Do all required parameters have values?"""

        return all(p.name in self.values for p in self.parameters if p.required)

    def format_args(self):
        """Returns dictionary containing values for all named parameters."""

        return self.values

class AnyParameter(Parameter):
    """Accepts any value"""

    id = 'any'

class MultiParameter(Parameter):
    """
    Accepts a value matching one of several defined types.
    E.g., `MultiParameter([StringParameter(''), DictParameter('')])` will accept both strings and dictionaries.
    """

    id = 'multiple'

    def __init__(self, types, *args, **kwargs):
        """
        :param types: A list of `Parameter` instances to accept.
        """

        super(MultiParameter, self).__init__(*args, **kwargs)

        self.types = types

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        for parameter_obj in self.types:
            try:
                return parameter_obj.clean(value)
            except ParameterNotValidError:
                continue

        raise ParameterNotValidError

    def serialize_args(self):
        """Returns (args, kwargs) to be used when deserializing this parameter."""

        args, kwargs = super(MultiParameter, self).serialize_args()
        args.insert(0, [[t.id, t.serialize_args()] for t in self.types])

        return args, kwargs

    @staticmethod
    def deserialize_args(args, kwargs):
        args, kwargs = super(MultiParameter, MultiParameter).deserialize_args(args[1:], kwargs)

        types = []
        for type_id, type_args in args[0]:
            type_cls = Parameter.by_id(type_id)
            type_args, type_kwargs = type_cls.deserialize_args(*type_args)
            types.append(type_cls(*type_args, **type_kwargs))

        args.insert(0, types)

        return args, kwargs

class ListParameter(Parameter):
    """
    Accepts a list of a given parameter type. This parameter will accept either a list-type object (list, tuple, set)
    or a generator object. If it receives a generator object, `clean()` will also return a generator object. Otherwise,
    it will return a list.
    """

    id = 'list'

    def __init__(self, param_type, *args, **kwargs):
        """
        :param param_type: A `Parameter` instance.
        """

        super(ListParameter, self).__init__(*args, **kwargs)

        self.param_type = param_type

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        if not isinstance(value, (list, tuple, set, GeneratorType)):
            value = [value]

        gen = (self.param_type.clean(x) for x in value)

        if isinstance(value, GeneratorType):
            return gen
        else:
            return list(gen)

    def serialize_args(self):
        """Returns (args, kwargs) to be used when deserializing this parameter."""

        args, kwargs = super(ListParameter, self).serialize_args()
        args.insert(0, [self.param_type.id, self.param_type.serialize_args()])

    @staticmethod
    def deserialize_args(args, kwargs):
        type_id, type_args = args[0]
        args, kwargs = super(ListParameter, ListParameter).deserialize_args(args[1:], kwargs)

        type_cls = Parameter.by_id(type_id)
        type_args, type_kwargs = type_cls.deserialize_args(*type_args)
        args.insert(0, type_cls(*type_args, **type_kwargs))

        return args, kwargs

class StringParameter(Parameter):
    """Accepts a string. Will convert numbers to string."""

    id = 'string'

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        if isinstance(value, six.string_types):
            return value
        elif isinstance(value, numbers.Number):
            return str(value)

        raise ParameterNotValidError

class NumberParameter(Parameter):
    """Accepts a number. Will convert numeric strings."""

    id = 'number'

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        if isinstance(value, numbers.Number):
            return value
        elif isinstance(value, six.string_types):
            try:
                value = float(value)
                return int(value) if value.is_integer() else value
            except ValueError:
                raise ParameterNotValidError

        raise ParameterNotValidError

class IntParameter(NumberParameter):
    """Accepts an integer value. Will convert strings, other numbers."""

    id = 'int'

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        return int(super(IntParameter, self).clean(value))

class FloatParameter(NumberParameter):
    """Accepts a float value. Will convert strings, other numbers."""

    id = 'float'

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        return float(super(FloatParameter, self).clean(value))

class DictParameter(Parameter):
    """Accepts a dictionary"""

    id = 'dict'

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        if isinstance(value, (dict, list)):
            return value

        raise ParameterNotValidError

class NdArrayParameter(Parameter):
    """Accepts a numpy array or Python list"""

    id = 'array'

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        if isinstance(value, numpy.ndarray):
            return value
        elif isinstance(value, (list, tuple)):
            return numpy.array(value)

        raise ParameterNotValidError

class RasterParameter(Parameter):
    """Accepts a `Raster` instance"""

    id = 'raster'

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        if isinstance(value, Raster):
            return value

        raise ParameterNotValidError

class FeatureParameter(Parameter):
    """Accepts a single feature (as a Shapely geometry object)"""

    id = 'feature'

    # Todo: geometry type checking

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        if isinstance(value, BaseGeometry):
            return value

        raise ParameterNotValidError

class FeatureCollectionParameter(FeatureParameter):
    """Accepts a list of features (as Shapely geometry objects)"""

    id = 'feature_collection'

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        if isinstance(value, (list, tuple)):
            return [super(FeatureCollectionParameter, self).clean(x) for x in value]

        raise ParameterNotValidError

class RasterDatasetParameter(Parameter):
    """Accepts a NetCDF `Dataset` object"""

    id = 'raster_dataset'

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        if isinstance(value, netCDF4.Dataset):
            return value

        raise ParameterNotValidError

class FeatureDatasetParameter(Parameter):
    """Accepts a Fiona `Collection` object"""

    id = 'feature_dataset'

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        if isinstance(value, Collection):
            return value

        raise ParameterNotValidError

class RegisteredDatasetParameter(Parameter):
    """Used by the web API to map registered datasets to inputs"""

    def clean(self, value):
        """Cleans and returns the given value, or raises a ParameterNotValidError exception"""

        if not isinstance(value, six.string_types):
            raise ParameterNotValidError

        try:
            source, value = value.split('://', 1)
        except ValueError:
            raise ParameterNotValidError

        if source == 'service':
            if ':' in value:
                service_name, variable_name = value.split(':', 1)
            else:
                service_name = value
                variable_name = None

            try:
                service = Service.objects.get(name=service_name)
            except ObjectDoesNotExist:
                raise ParameterNotValidError("Service '{}' not found".format(service_name))

            path = os.path.join(SERVICE_DATA_ROOT, service.data_path)
            dataset = netCDF4.Dataset(path, 'r')

            if variable_name:
                try:
                    variable = service.variable_set.all().get(variable=variable_name)
                except ObjectDoesNotExist:
                    raise ParameterNotValidError("Variable '{}' not found".format(variable_name))

                return Raster(
                    dataset.variables[variable.variable][:], variable.full_extent, variable.x_dimension,
                    variable.y_dimension
                )  # Todo: determine y_increasing
            else:
                return dataset
        else:
            raise ParameterNotValidError('Invalid source: {}'.format(source))
<|endoftext|>


<commit_before>from sqlshare_rest.models import FileUpload
from sqlshare_rest.parser import Parser
from sqlshare_rest.util.db import get_backend
from sqlshare_rest.dao.dataset import create_dataset_from_query
from time import sleep
from sqlshare_rest.util.queue_triggers import trigger_upload_queue_processing
from sqlshare_rest.util.queue_triggers import UPLOAD_QUEUE_PORT_NUMBER
from sqlshare_rest.logger import getLogger
import atexit

import socket
from threading import Thread

import six

if six.PY2:
    from Queue import Queue
elif six.PY3:
    from queue import Queue

def process_dataset_queue(thread_count=0, run_once=True, verbose=False):
    q = Queue()

    def worker():
        logger = getLogger(__name__)
        """
        Get a file upload object from the queue, and turn it into a dataset.
        """
        backend = get_backend()
        keep_looping = True
        while keep_looping:
            oldest = q.get()
            msg = "Processing file upload: %s" % oldest.pk
            logger.info(msg)
            if verbose:
                print(msg)
            user = oldest.owner
            backend = get_backend()
            try:
                p = Parser()
                p.delimiter(oldest.delimiter)
                p.has_header_row(oldest.has_column_header)

                file_path = oldest.user_file.path
                handle = open(file_path, "rt")
                handle.seek(0)
                p.parse(handle)

                name = oldest.dataset_name
                table_name = backend.create_table_from_parser(name,
                                                              p,
                                                              oldest,
                                                              user)

                dataset_sql = backend.get_view_sql_for_dataset(table_name,
                                                               user)
                dataset = create_dataset_from_query(user.username,
                                                    oldest.dataset_name,
                                                    dataset_sql)

                dataset.description = oldest.dataset_description
                dataset.is_public = oldest.dataset_is_public
                dataset.save()
                oldest.dataset = dataset
                oldest.dataset_created = True
                oldest.save()
            except Exception as ex:
                msg = "Error on %s: %s" % (oldest.pk, str(ex))
                logger.error(msg)
                if verbose:
                    print(msg)
                oldest.has_error = True
                oldest.error = str(ex)
                oldest.save()
            finally:
                backend.close_user_connection(user)

            q.task_done()
            msg = "Finished file upload %s." % oldest.pk
            logger.info(msg)
            if verbose:
                print(msg)
            if run_once:
                keep_looping = False

    def periodic_check():
        """
        Every 5 seconds, do a check for new files.  Just in case something
        needs processing, but didn't call trigger_processing() itself.
        """
        while True:
            sleep(5)
            if verbose:
                print("Triggering periodic processing.")
            trigger_upload_queue_processing()

    filtered = get_initial_filter_list()

    if run_once:
        try:
            oldest = filtered.order_by('id')[:1].get()
        except FileUpload.DoesNotExist:
            return
        q.put(oldest)
        worker()
    else:
        # Track the oldest query, so we only select ones newer that
        newest_pk = 0
        for i in range(thread_count):
            t = Thread(target=worker)
            t.setDaemon(True)
            t.start()

        # Start with any queries already in the queue:
        for upload in filtered:
            if upload.pk > newest_pk:
                newest_pk = upload.pk
            if verbose:
                print("Adding file upload ID %s to the queue." % upload.pk)
            q.put(upload)

        # Just in case things get off the rails - maybe a connection to the
        # server gets blocked? - periodically trigger a check for new queries
        kicker = Thread(target=periodic_check)
        kicker.setDaemon(True)
        kicker.start()

        # Start the socket server for getting notifications of new queries
        server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        # Make it so we can run the server right away after killing it
        server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        server.bind(('localhost', UPLOAD_QUEUE_PORT_NUMBER))

        # Make sure we close our socket when we're killed.
        def close_socket():
            server.close()

        atexit.register(close_socket)

        server.listen(5)
        while True:
            (clientsocket, address) = server.accept()
            # We don't actually have a protocol to speak...
            clientsocket.close()

            uploads = FileUpload.objects.filter(dataset_created=False,
                                                is_finalized=True,
                                                has_error=False,
                                                pk__gt=newest_pk)
            for upload in uploads:
                if upload.pk > newest_pk:
                    newest_pk = upload.pk
                if verbose:
                    print("Adding upload ID %s to the queue." % upload.pk)
                q.put(upload)

    q.join()

def get_initial_filter_list():
    return FileUpload.objects.filter(dataset_created=False,
                                     has_error=False,
                                     is_finalized=True)
<commit_msg>Catch an error on the dataset queue<commit_after>from sqlshare_rest.models import FileUpload
from sqlshare_rest.parser import Parser
from sqlshare_rest.util.db import get_backend
from sqlshare_rest.dao.dataset import create_dataset_from_query
from time import sleep
from sqlshare_rest.util.queue_triggers import trigger_upload_queue_processing
from sqlshare_rest.util.queue_triggers import UPLOAD_QUEUE_PORT_NUMBER
from sqlshare_rest.logger import getLogger
from django.db.utils import DatabaseError
import atexit

import socket
from threading import Thread

import six

if six.PY2:
    from Queue import Queue
elif six.PY3:
    from queue import Queue

def process_dataset_queue(thread_count=0, run_once=True, verbose=False):
    q = Queue()

    def worker():
        logger = getLogger(__name__)
        """
        Get a file upload object from the queue, and turn it into a dataset.
        """
        backend = get_backend()
        keep_looping = True
        while keep_looping:
            oldest = q.get()
            msg = "Processing file upload: %s" % oldest.pk
            logger.info(msg)
            if verbose:
                print(msg)
            user = oldest.owner
            backend = get_backend()
            try:
                p = Parser()
                p.delimiter(oldest.delimiter)
                p.has_header_row(oldest.has_column_header)

                file_path = oldest.user_file.path
                handle = open(file_path, "rt")
                handle.seek(0)
                p.parse(handle)

                name = oldest.dataset_name
                table_name = backend.create_table_from_parser(name,
                                                              p,
                                                              oldest,
                                                              user)

                dataset_sql = backend.get_view_sql_for_dataset(table_name,
                                                               user)
                dataset = create_dataset_from_query(user.username,
                                                    oldest.dataset_name,
                                                    dataset_sql)

                dataset.description = oldest.dataset_description
                dataset.is_public = oldest.dataset_is_public
                dataset.save()
                oldest.dataset = dataset
                oldest.dataset_created = True
                oldest.save()
            except Exception as ex:
                msg = "Error on %s: %s" % (oldest.pk, str(ex))
                logger.error(msg)
                if verbose:
                    print(msg)
                oldest.has_error = True
                oldest.error = str(ex)
                oldest.save()
            finally:
                backend.close_user_connection(user)

            q.task_done()
            msg = "Finished file upload %s." % oldest.pk
            logger.info(msg)
            if verbose:
                print(msg)
            if run_once:
                keep_looping = False

    def periodic_check():
        """
        Every 5 seconds, do a check for new files.  Just in case something
        needs processing, but didn't call trigger_processing() itself.
        """
        while True:
            sleep(5)
            if verbose:
                print("Triggering periodic processing.")
            trigger_upload_queue_processing()

    filtered = get_initial_filter_list()

    if run_once:
        try:
            oldest = filtered.order_by('id')[:1].get()
        except FileUpload.DoesNotExist:
            return
        q.put(oldest)
        worker()
    else:
        # Track the oldest query, so we only select ones newer that
        newest_pk = 0
        for i in range(thread_count):
            t = Thread(target=worker)
            t.setDaemon(True)
            t.start()

        # Start with any queries already in the queue:
        for upload in filtered:
            if upload.pk > newest_pk:
                newest_pk = upload.pk
            if verbose:
                print("Adding file upload ID %s to the queue." % upload.pk)
            q.put(upload)

        # Just in case things get off the rails - maybe a connection to the
        # server gets blocked? - periodically trigger a check for new queries
        kicker = Thread(target=periodic_check)
        kicker.setDaemon(True)
        kicker.start()

        # Start the socket server for getting notifications of new queries
        server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        # Make it so we can run the server right away after killing it
        server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        server.bind(('localhost', UPLOAD_QUEUE_PORT_NUMBER))

        # Make sure we close our socket when we're killed.
        def close_socket():
            server.close()

        atexit.register(close_socket)

        server.listen(5)
        while True:
            (clientsocket, address) = server.accept()
            # We don't actually have a protocol to speak...
            clientsocket.close()

            try:
                uploads = FileUpload.objects.filter(dataset_created=False,
                                                    is_finalized=True,
                                                    has_error=False,
                                                    pk__gt=newest_pk)
                for upload in uploads:
                    if upload.pk > newest_pk:
                        newest_pk = upload.pk
                    if verbose:
                        print("Adding upload ID %s to the queue." % upload.pk)
                    q.put(upload)
            except DatabaseError as ex:
                ex_str = str(ex)
                # If there's just, say, a network glitch, carry on.
                # If it's anything else, re-raise the error.
                if str_ex.find("Read from the server failed") < 0:
                    raise

    q.join()

def get_initial_filter_list():
    return FileUpload.objects.filter(dataset_created=False,
                                     has_error=False,
                                     is_finalized=True)
<|endoftext|>


<commit_before>#pip install neo4j-driver wordcloud matplotlib pillow image
from neo4j.v1 import GraphDatabase, basic_auth

import csv
import matplotlib
import wordcloud
import random

def grey_color_func(word, font_size, position, orientation, random_state=None, **kwargs):
	return "hsl(%d, 100%%, 100%%)" % random.randint(120, 160)

def get_most_common_attributes(db_driver, n_attributes):

	print 'Querying database for the %d most common attributes' % n_attributes
	attr_types = []
	with db_driver.session() as session:
		results = session.run("MATCH (s:Sample)-[:hasAttribute]->(:Attribute)-->(t:AttributeType) "
		                      "WITH t, COUNT(s) AS usage_count "
		                      "RETURN t.name AS type, usage_count "
		                      "ORDER BY usage_count DESC "
		                      "LIMIT %d" % n_attributes)
		for result in results:
			attr_types.append((result["type"],result["usage_count"]))
	return attr_types

def generate_spreadsheet(db_driver):

	print "generating spreadsheet of most common attribute types and values"
	with open("attr_common.csv", "w") as outfile:
		csvout = csv.writer(outfile)

		common = get_most_common_attributes(db_driver, 100)
		for attr in common:
			print "{} ({})".format(attr[0], attr[1])

			row = ["{} ({})".format(attr[0], attr[1])]

			with db_driver.session() as session2:
				cypher = "MATCH (s:Sample)-->(a:Attribute)-->(t:AttributeType{name:'"+attr[0]+"'}), (a:Attribute)-->(v:AttributeValue) RETURN v.name AS value, COUNT(s) AS usage_count ORDER BY usage_count DESC LIMIT 10"
				#print cypher
				results2 = session2.run(cypher)
				for result2 in results2:
					row.append("{} ({})".format(result2["value"], result2["usage_count"]))
					print "{} ({})".format(result2["value"], result2["usage_count"])

			csvout.writerow(row)

	print "generated spreadsheet of most common attribute types and values"

def generate_wordcloud(db_driver):

	print "generating wordcloud of most common attribute types and values"
	freq = []

	common = get_most_common_attributes(db_driver, 1000)
	for attr in common:
		i = 0
		freq.append((attr[0], attr[1]))
		if i < 10:
			i += 1
			print "generating wordcloud of values of", attr[0]
			freq2 = []
			with db_driver.session() as session2:
				cypher = \
					"MATCH (s:Sample)-->(a:Attribute)-->(t:AttributeType{name:'"+attr[0]+"'}), (a:Attribute)-->(v:AttributeValue) RETURN v.name AS value, COUNT(s) AS usage_count ORDER BY usage_count DESC LIMIT 1000"
				results2 = session2.run(cypher)
				for result2 in results2:
					freq2.append((result2["value"], result2["usage_count"]))
			wc = wordcloud.WordCloud(width=640, height=512, scale=2.0, max_words=1000).generate_from_frequencies(freq2)
			wc.recolor(color_func=grey_color_func, random_state=3)
			wc.to_file("word_clouds/cloud-values-{}.png".format(attr[0]))
			print "generated wordcloud of values of", attr[0]
		wc = wordcloud.WordCloud(width=640, height=512, scale=2.0, max_words=1000).generate_from_frequencies(freq)
		wc.recolor(color_func=grey_color_func, random_state=3)
		wc.to_file("word_clouds/cloud-types.png")
	print "generated wordcloud of most common attribute types and values"

def attribute_values_mapped(db_driver):

	print "generating the list with the percentage of attributes values mapped to ontology term"
	with open("perc_attr_mapped.csv", "w") as outfile:
		csvout = csv.writer(outfile)

		common_attrs = get_most_common_attributes(db_driver, 100)
		for attr in common_attrs:
			cypher = "MATCH (s:Sample)-->(a:Attribute{type:'"+attr[0]+"'}) " \
					 "RETURN COUNT(s) AS samples, " \
					 "COUNT(a.iri) AS mapped "
			with db_driver.session() as session:
				result = session.run(cypher)
				for record in result:
					print attr, record["samples"], record["mapped"]
					row = [str(attr), round(float(record["mapped"])*100/record["samples"], 2)]
					print row
					csvout.writerow(row)
				# print "%s: Ratio=%.2f" % (attr, float(record["mapped"])/record["samples"])

def attribute_value_coverage(db_driver):

	print "generating coverage stats"
	prop = 0.75
	maxcount = 100

	common_attrs = get_most_common_attributes(db_driver, 100)
	for attr in common_attrs:
		cypher = "MATCH (s:Sample)--(a:Attribute)--(t:AttributeType{name:'"+attr[0]+"'}), (a)--(v:AttributeValue) RETURN v.name, count(s) AS count_s ORDER BY count(s) DESC"
		#print cypher
		with db_driver.session() as session:
			result = session.run(cypher)
			running_total = 0
			i = 0
			for record in result:
				i += 1
				running_total += record["count_s"]
				#print attr[1], float(attr[1])*prop, running_total, record["count_s"]
				if running_total > float(attr[1])*prop:
					print "for type",attr[0],"the top",i,"values cover",int(prop*100.0),"% of samples"
					break
				if i >= maxcount:
					print "for type",attr[0],"the top",maxcount,"values do not cover",int(prop*100.0),"% of samples"

if __name__ == "__main__":

	driver = GraphDatabase.driver("bolt://localhost", auth=basic_auth("neo4j", "password"))

	# spreadsheet of most common attribute types and values
	#generate_spreadsheet(driver)

	# wordcloud of most common attribute types and values
	#generate_wordcloud(driver)

	# Percentage of attribute values mapped to ontology for each attribute type
	#attribute_values_mapped(driver)

	# top N values to cover P proportion of samples
	attribute_value_coverage(driver)
<commit_msg>Add number of attribute values for each attribute type<commit_after># pip install neo4j-driver wordcloud matplotlib pillow image
from neo4j.v1 import GraphDatabase, basic_auth

import argparse
import csv
import wordcloud
import random
import numpy as np

def grey_color_func(word, font_size, position, orientation, random_state=None, **kwargs):
	return "hsl(%d, 100%%, 100%%)" % random.randint(120, 160)

def get_most_common_attributes(db_driver, n_attributes):
	print 'Querying database for the %d most common attributes' % n_attributes
	attr_types = []
	with db_driver.session() as session:
		results = session.run("MATCH (s:Sample)-[:hasAttribute]->(:Attribute)-->(t:AttributeType) "
		                      "WITH t, COUNT(s) AS usage_count "
		                      "RETURN t.name AS type, usage_count "
		                      "ORDER BY usage_count DESC "
		                      "LIMIT %d" % n_attributes)
		for result in results:
			attr_types.append((result["type"], result["usage_count"]))
	return attr_types

def generate_spreadsheet(db_driver):
	print "generating spreadsheet of most common attribute types and values"
	with open("attr_common.csv", "w") as outfile:
		csvout = csv.writer(outfile)

		common = get_most_common_attributes(db_driver, 100)
		for attr in common:
			print "{} ({})".format(attr[0], attr[1])

			row = ["{} ({})".format(attr[0], attr[1])]

			with db_driver.session() as session2:
				cypher = "MATCH (s:Sample)-->(a:Attribute)-->(t:AttributeType{name:'"+attr[0]+"'}), (a:Attribute)-->(v:AttributeValue) RETURN v.name AS value, COUNT(s) AS usage_count ORDER BY usage_count DESC LIMIT 10"
				#print cypher
				print cypher
				results2 = session2.run(cypher)
				for result2 in results2:
					row.append("{} ({})".format(result2["value"], result2["usage_count"]))
					print "{} ({})".format(result2["value"], result2["usage_count"])

			csvout.writerow(row)

	print "generated spreadsheet of most common attribute types and values"

def generate_wordcloud(db_driver):
	print "generating wordcloud of most common attribute types and values"
	freq = []

	common = get_most_common_attributes(db_driver, 1000)
	for attr in common:
		i = 0
		freq.append((attr[0], attr[1]))
		if i < 10:
			i += 1
			print "generating wordcloud of values of", attr[0]
			freq2 = []
			with db_driver.session() as session2:
				cypher = \
					"MATCH (s:Sample)-->(a:Attribute)-->(t:AttributeType{name:'"+attr[0]+"'}), (a:Attribute)-->(v:AttributeValue) RETURN v.name AS value, COUNT(s) AS usage_count ORDER BY usage_count DESC LIMIT 1000"
				results2 = session2.run(cypher)
				for result2 in results2:
					freq2.append((result2["value"], result2["usage_count"]))
			wc = wordcloud.WordCloud(width=640, height=512, scale=2.0, max_words=1000).generate_from_frequencies(freq2)
			wc.recolor(color_func=grey_color_func, random_state=3)
			wc.to_file("word_clouds/cloud-values-{}.png".format(attr[0]))
			print "generated wordcloud of values of", attr[0]
		wc = wordcloud.WordCloud(width=640, height=512, scale=2.0, max_words=1000).generate_from_frequencies(freq)
		wc.recolor(color_func=grey_color_func, random_state=3)
		wc.to_file("word_clouds/cloud-types.png")
	print "generated wordcloud of most common attribute types and values"

def attribute_values_mapped(db_driver):
	print "generating the list with the percentage of attributes values mapped to ontology term"
	with open("perc_attr_mapped.csv", "w") as outfile:
		csvout = csv.writer(outfile)

		common_attrs = get_most_common_attributes(db_driver, 200)
		for attr in common_attrs:
			cypher = "MATCH (s:Sample)-->(a:Attribute{type:'" + attr[0] + "'}) " \
			                                                              "RETURN COUNT(s) AS samples, " \
			                                                              "COUNT(a.iri) AS mapped "
			with db_driver.session() as session:
				result = session.run(cypher)
				for record in result:
					print attr, record["samples"], record["mapped"]
					row = [str(attr[0]), attr[1], round(float(record["mapped"]) * 100 / record["samples"], 2)]
					print row
					csvout.writerow(row)
				# print "%s: Ratio=%.2f" % (attr, float(record["mapped"])/record["samples"])

def attribute_value_coverage(db_driver):

	print "generating coverage stats"
	prop = 0.75
	maxcount = 100

	common_attrs = get_most_common_attributes(db_driver, 100)
	for attr in common_attrs:
		cypher = "MATCH (s:Sample)--(a:Attribute)--(t:AttributeType{name:'"+attr[0]+"'}), (a)--(v:AttributeValue) RETURN v.name, count(s) AS count_s ORDER BY count(s) DESC"
		#print cypher
		with db_driver.session() as session:
			result = session.run(cypher)
			running_total = 0
			i = 0
			for record in result:
				i += 1
				running_total += record["count_s"]
				#print attr[1], float(attr[1])*prop, running_total, record["count_s"]
				if running_total > float(attr[1])*prop:
					print "for type",attr[0],"the top",i,"values cover",int(prop*100.0),"% of samples"
					break
				if i >= maxcount:
					print "for type",attr[0],"the top",maxcount,"values do not cover",int(prop*100.0),"% of samples"

def number_of_attributes_ditribution(db_driver, show_graph):
	print "generating the spreadsheet with Number of attributes frequencies among samples"
	with open("num_attributes_distribution.csv", "w") as outfile:
		csvout = csv.writer(outfile)
		csvout.writerow(["attr_count", "samples_count"])
		print "Attr_count, Samples_count"
		cypher = "MATCH (s:Sample)-[:hasAttribute]->(:Attribute)-[:hasType]->(at:AttributeType) \
					  WITH s, COUNT(DISTINCT at) AS attr_count \
					  RETURN attr_count, COUNT(s) as samples_count \
					  ORDER BY attr_count DESC LIMIT 100"
		n_attr = []
		n_samples = []
		with db_driver.session() as session:
			results = session.run(cypher)
			for record in results:
				n_attr.append(record["attr_count"])
				n_samples.append(record["samples_count"])
				print "%s, %d" % (record["attr_count"], record["samples_count"])
				csvout.writerow([record["attr_count"], record["samples_count"]])
		if show_graph:
			plt.bar(n_attr, n_samples)
			plt.gca().set_yscale("log")
			plt.gca().set_xlabel("Number of attributes per sample")
			plt.gca().set_ylabel("Number of samples")
			plt.show()

def number_of_values_per_type(db_driver, show_graph):
	print "generating spreadsheet with number of values for each attribute type"
	with open("num_values_distribution.csv", "w") as fileout:
		csvout = csv.writer(fileout)

		cypher = "MATCH (s:Sample)-->(a:Attribute)-->(at:AttributeType) \
                  WITH at.name AS attr_type, COUNT(DISTINCT a.value) AS n_values, COUNT(s) AS n_samples \
                  RETURN attr_type, n_values, n_samples, toFloat(n_values)/toFloat(n_samples) AS ratio \
                  ORDER BY ratio \
                  LIMIT 100"
		print "%s, %s, %s, %s" % ("Attribute type", "Number of values", "Number of samples", "Ratio")
		csvout.writerow(["Attribute type", "Number of values", "Number of samples", "Ratio"])
		values = []
		with db_driver.session() as session:
			results = session.run(cypher)
			for record in results:
				record_tuple = (record["attr_type"],
				                record["n_values"],
				                record["n_samples"],
				                record["ratio"])
				print "%s, %d, %d, %.2f" % record_tuple
				values.append(record_tuple)
				csvout.writerow([x for x in record_tuple])
		attr_types, n_values, n_samples, ratios = map(list, zip(*values))
		counts = np.bincount(n_values)
		stats = {"mean": np.mean(n_values), "median": np.median(n_values), "mode": np.argmax(counts)}

		print "Mean: %d" % (stats["mean"])
		print "Median: %d" % (stats["median"])
		print "Mode: %d" % (stats["mode"])

		if show_graph:
			fig = plt.figure()
			ax1 = fig.add_subplot(211)
			ax1.bar(np.arange(len(n_values)), n_values)
			ax1.set_yscale("log")
			ax1.set_xticks(np.arange(len(n_values)))
			ax1.set_ylabel("Number of attribute values")

			ax2 = fig.add_subplot(212)
			ax2.bar(np.arange(len(n_values)), ratios)
			ax2.set_xticks(np.arange(len(n_values)))
			ax2.set_xticklabels(attr_types, rotation="vertical")
			ax2.set_xlabel("Attribute types")
			ax2.set_ylabel("Values diversity within samples")
			plt.show()

def main_func(db_driver, graph):

	# spreadsheet of most common attribute types and values
	# generate_spreadsheet(driver)

	# wordcloud of most common attribute types and values
	# generate_wordcloud(driver)

	# Percentage of attribute values mapped to ontology for each attribute type
	# attribute_values_mapped(driver)

	# Number of attributes per sample
	# number_of_attributes_ditribution(driver, show_graph)

	# Number of attribute values per attribute type
	number_of_values_per_type(db_driver, graph)

	# top N values to cover P proportion of samples
	attribute_value_coverage(driver)

if __name__ == "__main__":
	parser = argparse.ArgumentParser()
	parser.add_argument('--graph', '-g', type=bool, default=False)
	args = parser.parse_args()

	driver = GraphDatabase.driver("bolt://localhost", auth=basic_auth("neo4j", "password"))
	if args.graph:
		import matplotlib.pyplot as plt

	main_func(driver, args.graph)
<|endoftext|>