<commit_before># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
import hashlib

import mongoengine as me

from st2common.models.db import stormbase
from st2common.fields import ComplexDateTimeField
from st2common.util import date as date_utils
from st2common.constants.types import ResourceType

from st2common.models.db import MongoDBAccess

__all__ = [
    'TraceDB',
    'TraceComponentDB'
]

class TraceComponentDB(me.EmbeddedDocument):
    """
    """
    object_id = me.StringField()
    ref = me.StringField(default='')
    updated_at = ComplexDateTimeField(
        default=date_utils.get_datetime_utc_now,
        help_text='The timestamp when the TraceComponent was included.')
    caused_by = me.DictField(help_text='Causal component.')

    def __str__(self):
        return 'TraceComponentDB@(object_id:{}, updated_at:{})'.format(
            self.object_id, self.updated_at)

class TraceDB(stormbase.StormFoundationDB, stormbase.UIDFieldMixin):
    """
    Trace is a collection of all TriggerInstances, Rules and ActionExecutions
    that represent an activity which begins with the introduction of a
    TriggerInstance or request of an ActionExecution and ends with the
    completion of an ActionExecution. Given the closed feedback look sort of
    nature of StackStorm this implies a Trace can comprise of multiple
    TriggerInstances, Rules and ActionExecutions.

    :param trace_tag: A user specified reference to the trace.

    :param trigger_instances: TriggerInstances associated with this trace.

    :param rules: Rules associated with this trace.

    :param action_executions: ActionExecutions associated with this trace.
    """

    RESOURCE_TYPE = ResourceType.TRACE

    trace_tag = me.StringField(required=True,
                               help_text='A user specified reference to the trace.')
    trigger_instances = me.ListField(field=me.EmbeddedDocumentField(TraceComponentDB),
                                     required=False,
                                     help_text='Associated TriggerInstances.')
    rules = me.ListField(field=me.EmbeddedDocumentField(TraceComponentDB),
                         required=False,
                         help_text='Associated Rules.')
    action_executions = me.ListField(field=me.EmbeddedDocumentField(TraceComponentDB),
                                     required=False,
                                     help_text='Associated ActionExecutions.')
    start_timestamp = ComplexDateTimeField(default=date_utils.get_datetime_utc_now,
                                           help_text='The timestamp when the Trace was created.')

    meta = {
        'indexes': [
            {'fields': ['trace_tag']},
            {'fields': ['start_timestamp']},
            {'fields': ['action_executions.object_id']},
            {'fields': ['trigger_instances.object_id']},
            {'fields': ['rules.object_id']},
            {'fields': ['-start_timestamp', 'trace_tag']},
        ]
    }

    def __init__(self, *args, **values):
        super(TraceDB, self).__init__(*args, **values)
        self.uid = self.get_uid()

    def get_uid(self):
        parts = []
        parts.append(self.RESOURCE_TYPE)

        componenets_hash = hashlib.md5()
        componenets_hash.update(str(self.trace_tag).encode())
        componenets_hash.update(str(self.trigger_instances).encode())
        componenets_hash.update(str(self.rules).encode())
        componenets_hash.update(str(self.action_executions).encode())
        componenets_hash.update(str(self.start_timestamp).encode())

        parts.append(componenets_hash.hexdigest())

        uid = self.UID_SEPARATOR.join(parts)
        return uid

# specialized access objects
trace_access = MongoDBAccess(TraceDB)

MODELS = [TraceDB]
<commit_msg>Fix typo.<commit_after># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
import hashlib

import mongoengine as me

from st2common.models.db import stormbase
from st2common.fields import ComplexDateTimeField
from st2common.util import date as date_utils
from st2common.constants.types import ResourceType

from st2common.models.db import MongoDBAccess

__all__ = [
    'TraceDB',
    'TraceComponentDB'
]

class TraceComponentDB(me.EmbeddedDocument):
    """
    """
    object_id = me.StringField()
    ref = me.StringField(default='')
    updated_at = ComplexDateTimeField(
        default=date_utils.get_datetime_utc_now,
        help_text='The timestamp when the TraceComponent was included.')
    caused_by = me.DictField(help_text='Causal component.')

    def __str__(self):
        return 'TraceComponentDB@(object_id:{}, updated_at:{})'.format(
            self.object_id, self.updated_at)

class TraceDB(stormbase.StormFoundationDB, stormbase.UIDFieldMixin):
    """
    Trace is a collection of all TriggerInstances, Rules and ActionExecutions
    that represent an activity which begins with the introduction of a
    TriggerInstance or request of an ActionExecution and ends with the
    completion of an ActionExecution. Given the closed feedback look sort of
    nature of StackStorm this implies a Trace can comprise of multiple
    TriggerInstances, Rules and ActionExecutions.

    :param trace_tag: A user specified reference to the trace.

    :param trigger_instances: TriggerInstances associated with this trace.

    :param rules: Rules associated with this trace.

    :param action_executions: ActionExecutions associated with this trace.
    """

    RESOURCE_TYPE = ResourceType.TRACE

    trace_tag = me.StringField(required=True,
                               help_text='A user specified reference to the trace.')
    trigger_instances = me.ListField(field=me.EmbeddedDocumentField(TraceComponentDB),
                                     required=False,
                                     help_text='Associated TriggerInstances.')
    rules = me.ListField(field=me.EmbeddedDocumentField(TraceComponentDB),
                         required=False,
                         help_text='Associated Rules.')
    action_executions = me.ListField(field=me.EmbeddedDocumentField(TraceComponentDB),
                                     required=False,
                                     help_text='Associated ActionExecutions.')
    start_timestamp = ComplexDateTimeField(default=date_utils.get_datetime_utc_now,
                                           help_text='The timestamp when the Trace was created.')

    meta = {
        'indexes': [
            {'fields': ['trace_tag']},
            {'fields': ['start_timestamp']},
            {'fields': ['action_executions.object_id']},
            {'fields': ['trigger_instances.object_id']},
            {'fields': ['rules.object_id']},
            {'fields': ['-start_timestamp', 'trace_tag']},
        ]
    }

    def __init__(self, *args, **values):
        super(TraceDB, self).__init__(*args, **values)
        self.uid = self.get_uid()

    def get_uid(self):
        parts = []
        parts.append(self.RESOURCE_TYPE)

        components_hash = hashlib.md5()
        components_hash.update(str(self.trace_tag).encode())
        components_hash.update(str(self.trigger_instances).encode())
        components_hash.update(str(self.rules).encode())
        components_hash.update(str(self.action_executions).encode())
        components_hash.update(str(self.start_timestamp).encode())

        parts.append(components_hash.hexdigest())

        uid = self.UID_SEPARATOR.join(parts)
        return uid

# specialized access objects
trace_access = MongoDBAccess(TraceDB)

MODELS = [TraceDB]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Nodular's NodeMixin and Node models are the base classes for all content
objects.
"""

import os.path
import weakref
from datetime import datetime
from collections import MutableMapping
from werkzeug import cached_property
from sqlalchemy import event
from sqlalchemy.orm import validates, mapper
from sqlalchemy.orm.collections import InstrumentedList
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy.ext.hybrid import hybrid_property
from coaster import newid, parse_isoformat
from coaster.sqlalchemy import TimestampMixin, PermissionMixin, BaseScopedNameMixin
from .db import db

__all__ = ['Node', 'NodeAlias', 'NodeMixin']

# Adapted from
# https://bitbucket.org/sqlalchemy/sqlalchemy/src/0d2e6fb5410e/examples/dynamic_dict/dynamic_dict.py?at=default
class ProxyDict(MutableMapping):
    def __init__(self, parent, collection_name, childclass, keyname):
        self.parent = weakref.proxy(parent)
        self.collection_name = collection_name
        self.childclass = childclass
        self.keyname = keyname

        collection = self.collection
        if isinstance(collection, InstrumentedList):
            self.islist = True
        else:
            self.islist = False

    @property
    def collection(self):
        return getattr(self.parent, self.collection_name)

    def keys(self):
        if self.islist:
            return [getattr(x, self.keyname) for x in self.collection]
        else:
            descriptor = getattr(self.childclass, self.keyname)
            return [x[0] for x in self.collection.values(descriptor)]

    def __getitem__(self, key):
        if self.islist:
            try:
                return (i for i in self.collection if getattr(i, self.keyname) == key).next()
            except StopIteration:
                raise KeyError(key)
        else:
            item = self.collection.filter_by(**{self.keyname: key}).first()
            if item is not None:
                return item
            else:
                raise KeyError(key)

    def get(self, key, default=None):
        if self.islist:
            try:
                return self[key]
            except KeyError:
                return default
        else:
            return self.collection.filter_by(**{self.keyname: key}).first() or default

    def __setitem__(self, key, value):
        try:
            existing = self[key]
            self.collection.remove(existing)
        except KeyError:
            pass
        setattr(value, self.keyname, key)
        self.collection.append(value)

    def __delitem__(self, key):
        existing = self[key]
        self.collection.remove(existing)

    def __contains__(self, key):
        if self.islist:
            default = []
            return self.get(key, default) is not default
        else:
            return self.collection.filter_by(**{self.keyname: key}).count() > 0

    def __iter__(self):
        return iter(self.keys())

    def __len__(self):
        if self.islist:
            return len(self.collection)
        else:
            return self.collection.count()

class Node(BaseScopedNameMixin, db.Model):
    """
    Base class for all content objects.
    """
    __tablename__ = u'node'
    #: Full path to this node for URL traversal
    _path = db.Column('path', db.Unicode(1000), unique=True, nullable=False, default=u'')
    #: Id of the node across sites (staging, production, etc) for import/export
    buid = db.Column(db.Unicode(22), unique=True, default=newid, nullable=False)
    user_id = db.Column(db.Integer, db.ForeignKey('user.id'), nullable=True)
    #: User who made this node, empty for auto-generated nodes
    user = db.relationship('User')
    _parent_id = db.Column('parent_id', db.Integer, db.ForeignKey('node.id', ondelete='CASCADE'),
                          nullable=True)
    parent = db.relationship('Node', remote_side='Node.id',
        backref=db.backref('_nodes', order_by='Node.name',
            cascade='all, delete-orphan',
            lazy='dynamic', passive_deletes=True))
    #: Publication date (defaults to creation date)
    published_at = db.Column(db.DateTime, default=datetime.utcnow, nullable=False)
    #: Type of node, for polymorphic identity
    type = db.Column('type', db.Unicode(30))
    __table_args__ = (db.UniqueConstraint('name', 'parent_id'),)
    __mapper_args__ = {'polymorphic_on': type, 'polymorphic_identity': u'node'}

    def __repr__(self):
        return u'<Node /%s "%s">' % (self.path, self.title)

    @validates('name')
    def _validate_name(self, key, value):
        try:
            assert value is not None
            assert u'/' not in value
            value = value.strip()
            assert value != u''
            return value
        except AssertionError:
            raise ValueError(value)

    @hybrid_property
    def parent_id(self):
        """Container for this node (used mainly to enforce uniqueness of 'name')"""
        return self._parent_id

    @hybrid_property
    def path(self):
        """Path to this node for URL traversal"""
        return self._path

    def _update_path(self, newparent=None, newname=None):
        if not newparent and not self.parent:
            self._path = u'/'  # We're root. Our name is irrelevant
        else:
            path = os.path.join((newparent or self.parent).path, (newname or self.name))
            if len(path) > 1000:
                raise ValueError("Path is too long")
            self._path = path
        for child in self._nodes:
            child._update_path()

    @cached_property
    def nodes(self):
        """Dictionary of all sub-nodes."""
        return ProxyDict(self, '_nodes', Node, 'name')

    @cached_property
    def aliases(self):
        """Dictionary of all aliases for renamed nodes."""
        return ProxyDict(self, '_aliases', NodeAlias, 'name')

    def rename(self, newname):
        """
        Rename node and make an alias linking from the old name.
        This method does not check if the new name is available.
        """
        alias = NodeAlias.query.get((self.parent_id, self.name))
        if alias is None:
            alias = NodeAlias(parent=self.parent, name=self.name, node=self)
            #db.session.add(alias)
        else:
            alias.node = self
        self.name = newname

    def as_json(self):
        return {
            'buid': self.buid,
            'name': self.name,
            'title': self.title,
            'path': self.path,
            'created_at': self.created_at.isoformat() + 'Z',
            'updated_at': self.updated_at.isoformat() + 'Z',
            'published_at': self.published_at.isoformat() + 'Z',
            'userid': self.user.userid if self.user else None,
            'type': self.type,
        }

    def import_from(self, data):
        self.uuid = data['uuid']
        self.name = data['name']
        self.title = data['title']
        self.author = data.get('author')
        self.published_at = parse_isoformat(data['published_at'])
        self.properties = data['properties']

    def import_from_internal(self, data):
        # Only required for nodes that keep internal references to other nodes
        pass

def _node_parent_listener(target, value, oldvalue, initiator):
    """Listen for Node.parent being modified and update path"""
    if value != oldvalue and value is not None:
        target._update_path(newparent=value)
    return value

def _node_name_listener(target, value, oldvalue, initiator):
    """Listen for Node.name being modified and update path"""
    if value != oldvalue:
        target._update_path(newname=value)
    return value

@event.listens_for(mapper, "mapper_configured")
def _node_mapper_listener(mapper, class_):
    if issubclass(class_, Node):
        event.listen(class_.parent, 'set', _node_parent_listener, retval=True)
        event.listen(class_.name, 'set', _node_name_listener, retval=True)

class NodeAlias(TimestampMixin, db.Model):
    """
    When a node is renamed, it gets an alias connecting the old name to the new.
    NodeAlias makes it possible for users to rename nodes without breaking links.
    """
    #: Container id for this alias
    parent_id = db.Column(None, db.ForeignKey('node.id', ondelete='CASCADE'),
        nullable=False, primary_key=True)
    #: Container node
    parent = db.relationship(Node, primaryjoin=parent_id == Node.id,
        backref=db.backref('_aliases', lazy='dynamic',
            order_by='NodeAlias.name', cascade='all, delete-orphan'))
    #: The aliased name
    name = db.Column(db.Unicode(250), nullable=False, primary_key=True)
    #: Node id this name redirects to
    node_id = db.Column(None, db.ForeignKey('node.id'), nullable=False)
    #: Node this name redirects to
    node = db.relationship(Node, primaryjoin=node_id == Node.id)

class NodeMixin(TimestampMixin, PermissionMixin):
    """
    NodeMixin provides functionality for content objects to connect to the
    Node base table. NodeMixin and Node should be used together::

        class MyContentType(NodeMixin, Node):
            __tablename__ = 'my_content_type'
            my_column = db.Column(...)

    NodeMixin will use ``__tablename__`` as the node :attr:`~Node.type` identifier
    and will autogenerate a ``__title__`` attribute. This title is used in the UI
    when the user adds a new node.
    """
    @declared_attr
    def id(cls):
        """Link back to node"""
        return db.Column(None, db.ForeignKey('node.id', ondelete='CASCADE'),
                         primary_key=True, nullable=False)

    @declared_attr
    def __mapper_args__(cls):
        """Use the table name as the polymorphic identifier"""
        return {'polymorphic_identity': cls.__tablename__}

    @declared_attr
    def __title__(cls):
        """Create a title for the type from the class name"""
        return cls.__tablename__.replace('_', ' ').title()
<commit_msg>Bring back lazy='dynamic' with a workaround.<commit_after># -*- coding: utf-8 -*-

"""
Nodular's NodeMixin and Node models are the base classes for all content
objects.
"""

import os.path
import weakref
from datetime import datetime
from collections import MutableMapping
from werkzeug import cached_property
from sqlalchemy import event
from sqlalchemy.orm import validates, mapper
from sqlalchemy.orm.collections import InstrumentedList
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy.ext.hybrid import hybrid_property
from coaster import newid, parse_isoformat
from coaster.sqlalchemy import TimestampMixin, PermissionMixin, BaseScopedNameMixin
from .db import db

__all__ = ['Node', 'NodeAlias', 'NodeMixin', 'ProxyDict']

# Adapted from
# https://bitbucket.org/sqlalchemy/sqlalchemy/src/0d2e6fb5410e/examples/dynamic_dict/dynamic_dict.py?at=default
class ProxyDict(MutableMapping):
    """
    Proxies a dictionary on relationship. This is intended for use with
    ``lazy='dynamic'`` relationships, but can also be used with regular
    InstrumentedList relationships.

    ProxyDict is used for :attr:`Node.nodes` and :attr:`Node.aliases`.

    :param parent: The instance in which this dictionary exists.
    :param collection_name: The relationship that is being proxied.
    :param childclass: The model referred to in the relationship.
    :param keyname: Attribute in childclass that will be the dictionary key.
    :param parentkey: Attribute in childclass that refers back to this parent.
    """
    def __init__(self, parent, collection_name, childclass, keyname, parentkey):
        self.parent = weakref.ref(parent)
        self.collection_name = collection_name
        self.childclass = childclass
        self.keyname = keyname
        self.parentkey = parentkey

        collection = self.collection
        if isinstance(collection, InstrumentedList):
            self.islist = True
        else:
            self.islist = False

    @property
    def collection(self):
        return getattr(self.parent(), self.collection_name)

    def keys(self):
        if self.islist:
            return [getattr(x, self.keyname) for x in self.collection]
        else:
            descriptor = getattr(self.childclass, self.keyname)
            return [x[0] for x in self.collection.values(descriptor)]

    def __getitem__(self, key):
        if self.islist:
            try:
                return (i for i in self.collection if getattr(i, self.keyname) == key).next()
            except StopIteration:
                raise KeyError(key)
        else:
            item = self.collection.filter_by(**{self.keyname: key}).first()
            if item is not None:
                return item
            else:
                raise KeyError(key)

    def get(self, key, default=None):
        if self.islist:
            try:
                return self[key]
            except KeyError:
                return default
        else:
            return self.collection.filter_by(**{self.keyname: key}).first() or default

    def __setitem__(self, key, value):
        try:
            existing = self[key]
            self.collection.remove(existing)
        except KeyError:
            pass
        setattr(value, self.keyname, key)
        if self.islist:
            self.collection.append(value)
        else:
            setattr(value, self.parentkey, self.parent())

    def __delitem__(self, key):
        existing = self[key]
        self.collection.remove(existing)

    def __contains__(self, key):
        if self.islist:
            default = []
            return self.get(key, default) is not default
        else:
            return self.collection.filter_by(**{self.keyname: key}).count() > 0

    def __iter__(self):
        return iter(self.keys())

    def __len__(self):
        if self.islist:
            return len(self.collection)
        else:
            return self.collection.count()

class Node(BaseScopedNameMixin, db.Model):
    """
    Base class for all content objects.
    """
    __tablename__ = u'node'
    #: Full path to this node for URL traversal
    _path = db.Column('path', db.Unicode(1000), unique=True, nullable=False, default=u'')
    #: Id of the node across sites (staging, production, etc) for import/export
    buid = db.Column(db.Unicode(22), unique=True, default=newid, nullable=False)
    user_id = db.Column(db.Integer, db.ForeignKey('user.id'), nullable=True)
    #: User who made this node, empty for auto-generated nodes
    user = db.relationship('User')
    _parent_id = db.Column('parent_id', db.Integer, db.ForeignKey('node.id', ondelete='CASCADE'),
                          nullable=True)
    parent = db.relationship('Node', remote_side='Node.id',
        backref=db.backref('_nodes', order_by='Node.name',
            cascade='all, delete-orphan',
            lazy='dynamic', passive_deletes=True))
    #: Publication date (defaults to creation date)
    published_at = db.Column(db.DateTime, default=datetime.utcnow, nullable=False)
    #: Type of node, for polymorphic identity
    type = db.Column('type', db.Unicode(30))
    __table_args__ = (db.UniqueConstraint('name', 'parent_id'),)
    __mapper_args__ = {'polymorphic_on': type, 'polymorphic_identity': u'node'}

    def __repr__(self):
        return u'<Node /%s "%s">' % (self.path, self.title)

    @validates('name')
    def _validate_name(self, key, value):
        try:
            assert value is not None
            assert u'/' not in value
            value = value.strip()
            assert value != u''
            return value
        except AssertionError:
            raise ValueError(value)

    @hybrid_property
    def parent_id(self):
        """Container for this node (used mainly to enforce uniqueness of 'name')"""
        return self._parent_id

    @hybrid_property
    def path(self):
        """Path to this node for URL traversal"""
        return self._path

    def _update_path(self, newparent=None, newname=None):
        if not newparent and not self.parent:
            self._path = u'/'  # We're root. Our name is irrelevant
        else:
            path = os.path.join((newparent or self.parent).path, (newname or self.name))
            if len(path) > 1000:
                raise ValueError("Path is too long")
            self._path = path
        for child in self._nodes:
            child._update_path()

    @cached_property
    def nodes(self):
        """Dictionary of all sub-nodes."""
        return ProxyDict(self, '_nodes', Node, 'name', 'parent')

    @cached_property
    def aliases(self):
        """Dictionary of all aliases for renamed nodes."""
        return ProxyDict(self, '_aliases', NodeAlias, 'name', 'parent')

    def rename(self, newname):
        """
        Rename node and make an alias linking from the old name.
        This method does not check if the new name is available.
        """
        alias = NodeAlias.query.get((self.parent_id, self.name))
        if alias is None:
            alias = NodeAlias(parent=self.parent, name=self.name, node=self)
            #db.session.add(alias)
        else:
            alias.node = self
        self.name = newname

    def as_json(self):
        return {
            'buid': self.buid,
            'name': self.name,
            'title': self.title,
            'path': self.path,
            'created_at': self.created_at.isoformat() + 'Z',
            'updated_at': self.updated_at.isoformat() + 'Z',
            'published_at': self.published_at.isoformat() + 'Z',
            'userid': self.user.userid if self.user else None,
            'type': self.type,
        }

    def import_from(self, data):
        self.uuid = data['uuid']
        self.name = data['name']
        self.title = data['title']
        self.author = data.get('author')
        self.published_at = parse_isoformat(data['published_at'])
        self.properties = data['properties']

    def import_from_internal(self, data):
        # Only required for nodes that keep internal references to other nodes
        pass

def _node_parent_listener(target, value, oldvalue, initiator):
    """Listen for Node.parent being modified and update path"""
    if value != oldvalue and value is not None:
        target._update_path(newparent=value)
    return value

def _node_name_listener(target, value, oldvalue, initiator):
    """Listen for Node.name being modified and update path"""
    if value != oldvalue:
        target._update_path(newname=value)
    return value

@event.listens_for(mapper, "mapper_configured")
def _node_mapper_listener(mapper, class_):
    if issubclass(class_, Node):
        event.listen(class_.parent, 'set', _node_parent_listener, retval=True)
        event.listen(class_.name, 'set', _node_name_listener, retval=True)

class NodeAlias(TimestampMixin, db.Model):
    """
    When a node is renamed, it gets an alias connecting the old name to the new.
    NodeAlias makes it possible for users to rename nodes without breaking links.
    """
    #: Container id for this alias
    parent_id = db.Column(None, db.ForeignKey('node.id', ondelete='CASCADE'),
        nullable=False, primary_key=True)
    #: Container node
    parent = db.relationship(Node, primaryjoin=parent_id == Node.id,
        backref=db.backref('_aliases', lazy='dynamic',
            order_by='NodeAlias.name', cascade='all, delete-orphan'))
    #: The aliased name
    name = db.Column(db.Unicode(250), nullable=False, primary_key=True)
    #: Node id this name redirects to
    node_id = db.Column(None, db.ForeignKey('node.id'), nullable=False)
    #: Node this name redirects to
    node = db.relationship(Node, primaryjoin=node_id == Node.id)

class NodeMixin(TimestampMixin, PermissionMixin):
    """
    NodeMixin provides functionality for content objects to connect to the
    Node base table. NodeMixin and Node should be used together::

        class MyContentType(NodeMixin, Node):
            __tablename__ = 'my_content_type'
            my_column = db.Column(...)

    NodeMixin will use ``__tablename__`` as the node :attr:`~Node.type` identifier
    and will autogenerate a ``__title__`` attribute. This title is used in the UI
    when the user adds a new node.
    """
    @declared_attr
    def id(cls):
        """Link back to node"""
        return db.Column(None, db.ForeignKey('node.id', ondelete='CASCADE'),
                         primary_key=True, nullable=False)

    @declared_attr
    def __mapper_args__(cls):
        """Use the table name as the polymorphic identifier"""
        return {'polymorphic_identity': cls.__tablename__}

    @declared_attr
    def __title__(cls):
        """Create a title for the type from the class name"""
        return cls.__tablename__.replace('_', ' ').title()
<|endoftext|>


<commit_before>import arkInit
arkInit.init()

import coren

class Database(object):
	def __init__(self, apiRoot):
		self.apiRoot = apiRoot
		self.coren = coren.Coren(apiRoot)

	def create(self, entityType, data):
		return self.coren.create(entityType, data)

	def find(self, entityType):
		return self.coren.find(entityType)

	def update(self, entityType, data):
		return self.coren.update(entityType, data)

	def remove(self, entityType):
		return self.coren.remove(entityType)

	def execute(self, queryParams, queryOptions):
		return self.coren.execute(queryParams, queryOptions)
<commit_msg>made data optional<commit_after>import arkInit
arkInit.init()

import coren

class Database(object):
	def __init__(self, apiRoot):
		self.apiRoot = apiRoot
		self.coren = coren.Coren(apiRoot)

	def create(self, entityType, data):
		return self.coren.create(entityType, data)

	def find(self, entityType):
		return self.coren.find(entityType)

	def update(self, entityType, data=None):
		return self.coren.update(entityType, data)

	def remove(self, entityType):
		return self.coren.remove(entityType)

	def execute(self, queryParams, queryOptions):
		return self.coren.execute(queryParams, queryOptions)
<|endoftext|>


<commit_before>import datetime
import decimal
import uuid
from inspect import isclass

from marshmallow import fields, missing, Schema, validate
from marshmallow.class_registry import get_class
from marshmallow.decorators import post_dump

from .compat import (
    text_type,
    binary_type,
    basestring,
    dot_data_backwards_compatible,
    list_inner,
    INCLUDE,
    EXCLUDE,
    RAISE,
)
from .exceptions import UnsupportedValueError
from .validation import handle_length, handle_one_of, handle_range

__all__ = ("JSONSchema",)

TYPE_MAP = {
    dict: {"type": "object"},
    list: {"type": "array"},
    datetime.time: {"type": "string", "format": "time"},
    datetime.timedelta: {
        # TODO explore using 'range'?
        "type": "string"
    },
    datetime.datetime: {"type": "string", "format": "date-time"},
    datetime.date: {"type": "string", "format": "date"},
    uuid.UUID: {"type": "string", "format": "uuid"},
    text_type: {"type": "string"},
    binary_type: {"type": "string"},
    decimal.Decimal: {"type": "number", "format": "decimal"},
    set: {"type": "array"},
    tuple: {"type": "array"},
    float: {"type": "number", "format": "float"},
    int: {"type": "number", "format": "integer"},
    bool: {"type": "boolean"},
}

FIELD_VALIDATORS = {
    validate.Length: handle_length,
    validate.OneOf: handle_one_of,
    validate.Range: handle_range,
}

def _resolve_additional_properties(cls):
    meta = cls.Meta

    additional_properties = getattr(meta, "additional_properties", None)
    if additional_properties is not None:
        if additional_properties in (True, False):
            return additional_properties
        else:
            raise UnsupportedValueError(
                "`additional_properties` must be either True or False"
            )

    unknown = getattr(meta, "unknown", None)
    if unknown is None:
        return False
    elif unknown in (RAISE, EXCLUDE):
        return False
    elif unknown == INCLUDE:
        return True
    else:
        raise UnsupportedValueError("Unknown value %s for `unknown`" % unknown)

class JSONSchema(Schema):
    """Converts to JSONSchema as defined by http://json-schema.org/."""

    properties = fields.Method("get_properties")
    type = fields.Constant("object")
    required = fields.Method("get_required")

    def __init__(self, *args, **kwargs):
        """Setup internal cache of nested fields, to prevent recursion."""
        self._nested_schema_classes = {}
        self.nested = kwargs.pop("nested", False)
        super(JSONSchema, self).__init__(*args, **kwargs)

    def _get_default_mapping(self, obj):
        """Return default mapping if there are no special needs."""
        mapping = {v: k for k, v in obj.TYPE_MAPPING.items()}
        mapping.update(
            {
                fields.Email: text_type,
                fields.Dict: dict,
                fields.Url: text_type,
                fields.List: list,
                fields.DateTime: datetime.datetime,
                fields.Nested: "_from_nested_schema",
                fields.Number: decimal.Decimal,
            }
        )
        return mapping

    def get_properties(self, obj):
        """Fill out properties field."""
        properties = {}

        for field_name, field in sorted(obj.fields.items()):
            schema = self._get_schema_for_field(obj, field)
            properties[field.metadata.get("name") or field.name] = schema

        return properties

    def get_required(self, obj):
        """Fill out required field."""
        required = []

        for field_name, field in sorted(obj.fields.items()):
            if field.required:
                required.append(field.name)

        return required or missing

    def _from_python_type(self, obj, field, pytype):
        """Get schema definition from python type."""
        json_schema = {"title": field.attribute or field.name}

        for key, val in TYPE_MAP[pytype].items():
            json_schema[key] = val

        if field.dump_only:
            json_schema["readonly"] = True

        if field.default is not missing:
            json_schema["default"] = field.default

        # NOTE: doubled up to maintain backwards compatibility
        metadata = field.metadata.get("metadata", {})
        metadata.update(field.metadata)

        for md_key, md_val in metadata.items():
            if md_key in ("metadata", "name"):
                continue
            json_schema[md_key] = md_val

        if isinstance(field, fields.List):
            json_schema["items"] = self._get_schema_for_field(obj, list_inner(field))
        return json_schema

    def _get_pytype(self, field, mapping):
        """Get pytype based on field subclass"""
        for map_class, pytype in mapping.items():
            if issubclass(field.__class__, map_class):
                return pytype
        raise UnsupportedValueError("unsupported field type %s" % field)

    def _get_schema_for_field(self, obj, field):
        """Get schema and validators for field."""
        mapping = self._get_default_mapping(obj)
        if hasattr(field, "_jsonschema_type_mapping"):
            schema = field._jsonschema_type_mapping()
        elif "_jsonschema_type_mapping" in field.metadata:
            schema = field.metadata["_jsonschema_type_mapping"]
        else:
            pytype = self._get_pytype(field, mapping)
            if isinstance(pytype, basestring):
                schema = getattr(self, pytype)(obj, field)
            else:
                schema = self._from_python_type(obj, field, pytype)
        # Apply any and all validators that field may have
        for validator in field.validators:
            if validator.__class__ in FIELD_VALIDATORS:
                schema = FIELD_VALIDATORS[validator.__class__](
                    schema, field, validator, obj
                )
        return schema

    def _from_nested_schema(self, obj, field):
        """Support nested field."""
        if isinstance(field.nested, basestring):
            nested = get_class(field.nested)
        else:
            nested = field.nested

        if isclass(nested) and issubclass(nested, Schema):
            name = nested.__name__
            only = field.only
            exclude = field.exclude
            nested_cls = nested
            nested_instance = nested(only=only, exclude=exclude)
        else:
            nested_cls = nested.__class__
            name = nested_cls.__name__
            nested_instance = nested

        outer_name = obj.__class__.__name__
        # If this is not a schema we've seen, and it's not this schema (checking this for recursive schemas),
        # put it in our list of schema defs
        if name not in self._nested_schema_classes and name != outer_name:
            wrapped_nested = self.__class__(nested=True)
            wrapped_dumped = dot_data_backwards_compatible(
                wrapped_nested.dump(nested_instance)
            )

            wrapped_dumped["additionalProperties"] = _resolve_additional_properties(
                nested_cls
            )

            self._nested_schema_classes[name] = wrapped_dumped

            self._nested_schema_classes.update(wrapped_nested._nested_schema_classes)

        # and the schema is just a reference to the def
        schema = {"type": "object", "$ref": "#/definitions/{}".format(name)}

        # NOTE: doubled up to maintain backwards compatibility
        metadata = field.metadata.get("metadata", {})
        metadata.update(field.metadata)

        for md_key, md_val in metadata.items():
            if md_key in ("metadata", "name"):
                continue
            schema[md_key] = md_val

        if field.many:
            schema = {
                "type": "array" if field.required else ["array", "null"],
                "items": schema,
            }

        return schema

    def dump(self, obj, **kwargs):
        """Take obj for later use: using class name to namespace definition."""
        self.obj = obj
        return super(JSONSchema, self).dump(obj, **kwargs)

    @post_dump
    def wrap(self, data, **_):
        """Wrap this with the root schema definitions."""
        if self.nested:  # no need to wrap, will be in outer defs
            return data

        cls = self.obj.__class__
        name = cls.__name__

        data["additionalProperties"] = _resolve_additional_properties(cls)

        self._nested_schema_classes[name] = data
        root = {
            "$schema": "http://json-schema.org/draft-07/schema#",
            "definitions": self._nested_schema_classes,
            "$ref": "#/definitions/{name}".format(name=name),
        }
        return root
<commit_msg>Use pairs instead of mapping for marshmallow type subclass check.<commit_after>import datetime
import decimal
import uuid
from inspect import isclass

from marshmallow import fields, missing, Schema, validate
from marshmallow.class_registry import get_class
from marshmallow.decorators import post_dump

from .compat import (
    text_type,
    binary_type,
    basestring,
    dot_data_backwards_compatible,
    list_inner,
    INCLUDE,
    EXCLUDE,
    RAISE,
)
from .exceptions import UnsupportedValueError
from .validation import handle_length, handle_one_of, handle_range

__all__ = ("JSONSchema",)

PY_TO_JSON_TYPES_MAP = {
    dict: {"type": "object"},
    list: {"type": "array"},
    datetime.time: {"type": "string", "format": "time"},
    datetime.timedelta: {
        # TODO explore using 'range'?
        "type": "string"
    },
    datetime.datetime: {"type": "string", "format": "date-time"},
    datetime.date: {"type": "string", "format": "date"},
    uuid.UUID: {"type": "string", "format": "uuid"},
    text_type: {"type": "string"},
    binary_type: {"type": "string"},
    decimal.Decimal: {"type": "number", "format": "decimal"},
    set: {"type": "array"},
    tuple: {"type": "array"},
    float: {"type": "number", "format": "float"},
    int: {"type": "number", "format": "integer"},
    bool: {"type": "boolean"},
}

# We use these pairs to get proper python type from marshmallow type.
# We can't use mapping as earlier Python versions might shuffle dict contents
# and then `fields.Number` might end up before `fields.Integer`.
# As we perform sequential subclass check to determine proper Python type,
# we can't let that happen.
MARSHMALLOW_TO_PY_TYPES_PAIRS = (
    # This part of a mapping is carefully selected from marshmallow source code,
    # see marshmallow.BaseSchema.TYPE_MAPPING.
    (fields.String, text_type),
    (fields.DateTime, datetime.datetime),
    (fields.Float, float),
    (fields.Raw, text_type),
    (fields.Boolean, bool),
    (fields.Integer, int),
    (fields.UUID, uuid.UUID),
    (fields.Time, datetime.time),
    (fields.Date, datetime.date),
    (fields.TimeDelta, datetime.timedelta),
    (fields.Decimal, decimal.Decimal),
    # These are some mappings that generally make sense for the rest
    # of marshmallow fields.
    (fields.Email, text_type),
    (fields.Dict, dict),
    (fields.Url, text_type),
    (fields.List, list),
    (fields.Number, decimal.Decimal),
    # This one is here just for completeness sake and to check for
    # unknown marshmallow fields more cleanly.
    (fields.Nested, dict),
)

FIELD_VALIDATORS = {
    validate.Length: handle_length,
    validate.OneOf: handle_one_of,
    validate.Range: handle_range,
}

def _resolve_additional_properties(cls):
    meta = cls.Meta

    additional_properties = getattr(meta, "additional_properties", None)
    if additional_properties is not None:
        if additional_properties in (True, False):
            return additional_properties
        else:
            raise UnsupportedValueError(
                "`additional_properties` must be either True or False"
            )

    unknown = getattr(meta, "unknown", None)
    if unknown is None:
        return False
    elif unknown in (RAISE, EXCLUDE):
        return False
    elif unknown == INCLUDE:
        return True
    else:
        raise UnsupportedValueError("Unknown value %s for `unknown`" % unknown)

class JSONSchema(Schema):
    """Converts to JSONSchema as defined by http://json-schema.org/."""

    properties = fields.Method("get_properties")
    type = fields.Constant("object")
    required = fields.Method("get_required")

    def __init__(self, *args, **kwargs):
        """Setup internal cache of nested fields, to prevent recursion."""
        self._nested_schema_classes = {}
        self.nested = kwargs.pop("nested", False)
        super(JSONSchema, self).__init__(*args, **kwargs)

    def get_properties(self, obj):
        """Fill out properties field."""
        properties = {}

        for field_name, field in sorted(obj.fields.items()):
            schema = self._get_schema_for_field(obj, field)
            properties[field.metadata.get("name") or field.name] = schema

        return properties

    def get_required(self, obj):
        """Fill out required field."""
        required = []

        for field_name, field in sorted(obj.fields.items()):
            if field.required:
                required.append(field.name)

        return required or missing

    def _from_python_type(self, obj, field, pytype):
        """Get schema definition from python type."""
        json_schema = {"title": field.attribute or field.name}

        for key, val in PY_TO_JSON_TYPES_MAP[pytype].items():
            json_schema[key] = val

        if field.dump_only:
            json_schema["readonly"] = True

        if field.default is not missing:
            json_schema["default"] = field.default

        # NOTE: doubled up to maintain backwards compatibility
        metadata = field.metadata.get("metadata", {})
        metadata.update(field.metadata)

        for md_key, md_val in metadata.items():
            if md_key in ("metadata", "name"):
                continue
            json_schema[md_key] = md_val

        if isinstance(field, fields.List):
            json_schema["items"] = self._get_schema_for_field(obj, list_inner(field))
        return json_schema

    def _get_python_type(self, field):
        """Get python type based on field subclass"""
        for map_class, pytype in MARSHMALLOW_TO_PY_TYPES_PAIRS:
            if issubclass(field.__class__, map_class):
                return pytype

        raise UnsupportedValueError("unsupported field type %s" % field)

    def _get_schema_for_field(self, obj, field):
        """Get schema and validators for field."""
        if hasattr(field, "_jsonschema_type_mapping"):
            schema = field._jsonschema_type_mapping()
        elif "_jsonschema_type_mapping" in field.metadata:
            schema = field.metadata["_jsonschema_type_mapping"]
        else:
            pytype = self._get_python_type(field)
            if isinstance(field, fields.Nested):
                # Special treatment for nested fields.
                schema = self._from_nested_schema(obj, field)
            else:
                schema = self._from_python_type(obj, field, pytype)
        # Apply any and all validators that field may have
        for validator in field.validators:
            if validator.__class__ in FIELD_VALIDATORS:
                schema = FIELD_VALIDATORS[validator.__class__](
                    schema, field, validator, obj
                )
        return schema

    def _from_nested_schema(self, obj, field):
        """Support nested field."""
        if isinstance(field.nested, basestring):
            nested = get_class(field.nested)
        else:
            nested = field.nested

        if isclass(nested) and issubclass(nested, Schema):
            name = nested.__name__
            only = field.only
            exclude = field.exclude
            nested_cls = nested
            nested_instance = nested(only=only, exclude=exclude)
        else:
            nested_cls = nested.__class__
            name = nested_cls.__name__
            nested_instance = nested

        outer_name = obj.__class__.__name__
        # If this is not a schema we've seen, and it's not this schema (checking this for recursive schemas),
        # put it in our list of schema defs
        if name not in self._nested_schema_classes and name != outer_name:
            wrapped_nested = self.__class__(nested=True)
            wrapped_dumped = dot_data_backwards_compatible(
                wrapped_nested.dump(nested_instance)
            )

            wrapped_dumped["additionalProperties"] = _resolve_additional_properties(
                nested_cls
            )

            self._nested_schema_classes[name] = wrapped_dumped

            self._nested_schema_classes.update(wrapped_nested._nested_schema_classes)

        # and the schema is just a reference to the def
        schema = {"type": "object", "$ref": "#/definitions/{}".format(name)}

        # NOTE: doubled up to maintain backwards compatibility
        metadata = field.metadata.get("metadata", {})
        metadata.update(field.metadata)

        for md_key, md_val in metadata.items():
            if md_key in ("metadata", "name"):
                continue
            schema[md_key] = md_val

        if field.many:
            schema = {
                "type": "array" if field.required else ["array", "null"],
                "items": schema,
            }

        return schema

    def dump(self, obj, **kwargs):
        """Take obj for later use: using class name to namespace definition."""
        self.obj = obj
        return super(JSONSchema, self).dump(obj, **kwargs)

    @post_dump
    def wrap(self, data, **_):
        """Wrap this with the root schema definitions."""
        if self.nested:  # no need to wrap, will be in outer defs
            return data

        cls = self.obj.__class__
        name = cls.__name__

        data["additionalProperties"] = _resolve_additional_properties(cls)

        self._nested_schema_classes[name] = data
        root = {
            "$schema": "http://json-schema.org/draft-07/schema#",
            "definitions": self._nested_schema_classes,
            "$ref": "#/definitions/{name}".format(name=name),
        }
        return root
<|endoftext|>


<commit_before>from txmongo.database import Database
from twisted.internet.defer import (
    inlineCallbacks, Deferred, DeferredList, returnValue, maybeDeferred)
from txmongo import filter as qf
from pymongo.errors import DuplicateKeyError

from ..builder import BaseBuilder
from ..document import DocumentImplementation
from ..data_proxy import DataProxy, missing
from ..data_objects import Reference
from ..exceptions import NotCreatedError, UpdateError, DeleteError, ValidationError
from ..fields import ReferenceField, ListField, EmbeddedField
from ..query_mapper import map_query

from .tools import cook_find_filter

class TxMongoDocument(DocumentImplementation):

    __slots__ = ()

    opts = DocumentImplementation.opts

    @inlineCallbacks
    def reload(self):
        """
        Retrieve and replace document's data by the ones in database.

        Raises :class:`umongo.exceptions.NotCreatedError` if the document
        doesn't exist in database.
        """
        if not self.is_created:
            raise NotCreatedError("Document doesn't exists in database")
        ret = yield self.collection.find_one(self.pk)
        if ret is None:
            raise NotCreatedError("Document doesn't exists in database")
        self._data = DataProxy(self.schema)
        self._data.from_mongo(ret)

    @inlineCallbacks
    def commit(self, io_validate_all=False, conditions=None):
        """
        Commit the document in database.
        If the document doesn't already exist it will be inserted, otherwise
        it will be updated.

        :param io_validate_all:
        :param conditions: only perform commit if matching record in db
            satisfies condition(s) (e.g. version number).
            Raises :class:`umongo.exceptions.UpdateError` if the
            conditions are not satisfied.
         :return: A :class:`pymongo.results.UpdateResult` or
            :class:`pymongo.results.InsertOneResult` depending of the operation.
       """
        try:
            if self.is_created:
                if self.is_modified():
                    query = conditions or {}
                    query['_id'] = self.pk
                    # pre_update can provide additional query filter and/or
                    # modify the fields' values
                    additional_filter = yield maybeDeferred(self.pre_update)
                    if additional_filter:
                        query.update(map_query(additional_filter, self.schema.fields))
                    yield self.io_validate(validate_all=io_validate_all)
                    payload = self._data.to_mongo(update=True)
                    ret = yield self.collection.update_one(query, payload)
                    if ret.matched_count != 1:
                        raise UpdateError(ret.raw_result)
                    yield maybeDeferred(self.post_update, ret)
                else:
                    ret = None
            elif conditions:
                raise RuntimeError('Document must already exist in database to use `conditions`.')
            else:
                yield maybeDeferred(self.pre_insert)
                yield self.io_validate(validate_all=io_validate_all)
                payload = self._data.to_mongo(update=False)
                ret = yield self.collection.insert_one(payload)
                # TODO: check ret ?
                self._data.set_by_mongo_name('_id', ret.inserted_id)
                self.is_created = True
                yield maybeDeferred(self.post_insert, ret)
        except DuplicateKeyError as exc:
            # Need to dig into error message to find faulting index
            errmsg = exc.details['errmsg']
            for index in self.opts.indexes:
                if ('.$%s' % index.document['name'] in errmsg or
                        ' %s ' % index.document['name'] in errmsg):
                    keys = index.document['key'].keys()
                    if len(keys) == 1:
                        msg = self.schema.fields[keys[0]].error_messages['unique']
                        raise ValidationError({keys[0]: msg})
                    else:
                        fields = self.schema.fields
                        # Compound index (sort value to make testing easier)
                        keys = sorted(keys)
                        raise ValidationError({k: fields[k].error_messages[
                            'unique_compound'].format(fields=keys) for k in keys})
        self._data.clear_modified()
        return ret

    @inlineCallbacks
    def delete(self):
        """
        Remove the document from database.

        Raises :class:`umongo.exceptions.NotCreatedError` if the document
        is not created (i.e. ``doc.is_created`` is False)
        Raises :class:`umongo.exceptions.DeleteError` if the document
        doesn't exist in database.

        :return: A :class:`pymongo.results.DeleteResult`
        """
        if not self.is_created:
            raise NotCreatedError("Document doesn't exists in database")
        yield maybeDeferred(self.pre_delete)
        ret = yield self.collection.delete_one({'_id': self.pk})
        if ret.deleted_count != 1:
            raise DeleteError(ret.raw_result)
        self.is_created = False
        yield maybeDeferred(self.post_delete, ret)
        return ret

    def io_validate(self, validate_all=False):
        """
        Run the io_validators of the document's fields.

        :param validate_all: If False only run the io_validators of the
            fields that have been modified.
        """
        if validate_all:
            return _io_validate_data_proxy(self.schema, self._data)
        else:
            return _io_validate_data_proxy(
                self.schema, self._data, partial=self._data.get_modified_fields())

    @classmethod
    @inlineCallbacks
    def find_one(cls, spec=None, *args, **kwargs):
        """
        Find a single document in database.
        """
        # In txmongo, `spec` is for filtering and `filter` is for sorting
        spec = cook_find_filter(cls, spec)
        ret = yield cls.collection.find_one(*args, spec=spec, **kwargs)
        if ret is not None:
            ret = cls.build_from_mongo(ret, use_cls=True)
        return ret

    @classmethod
    @inlineCallbacks
    def find(cls, spec=None, *args, **kwargs):
        """
        Find a list document in database.

        Returns a cursor that provide Documents.
        """
        # In txmongo, `spec` is for filtering and `filter` is for sorting
        spec = cook_find_filter(cls, spec)
        raw_cursor_or_list = yield cls.collection.find(*args, spec=spec, **kwargs)
        if isinstance(raw_cursor_or_list, tuple):

            def wrap_raw_results(result):
                cursor = result[1]
                if cursor is not None:
                    cursor.addCallback(wrap_raw_results)
                return ([cls.build_from_mongo(e, use_cls=True) for e in result[0]], cursor)

            return wrap_raw_results(raw_cursor_or_list)
        else:
            return [cls.build_from_mongo(e, use_cls=True) for e in raw_cursor_or_list]

    @classmethod
    def count(cls, spec=None, **kwargs):
        """
        Get the number of documents in this collection.
        """
        # In txmongo, `spec` is for filtering and `filter` is for sorting
        spec = cook_find_filter(cls, spec)
        return cls.collection.count(spec=spec, **kwargs)

    @classmethod
    @inlineCallbacks
    def ensure_indexes(cls):
        """
        Check&create if needed the Document's indexes in database
        """
        for index in cls.opts.indexes:
            kwargs = index.document.copy()
            keys = kwargs.pop('key')
            index = qf.sort([(k, d) for k, d in keys.items()])
            yield cls.collection.create_index(index, **kwargs)

def _errback_factory(errors, field=None):

    def errback(err):
        if isinstance(err.value, ValidationError):
            if field:
                errors[field] = err.value.messages
            else:
                errors.extend(err.value.messages)
        else:
            raise err.value

    return errback

# Run multiple validators and collect all errors in one
@inlineCallbacks
def _run_validators(validators, field, value):
    errors = []
    defers = []
    for validator in validators:
        defer = validator(field, value)
        assert isinstance(defer, Deferred), 'io_validate functions must return a Deferred'
        defer.addErrback(_errback_factory(errors))
        defers.append(defer)
    yield DeferredList(defers)
    if errors:
        raise ValidationError(errors)

@inlineCallbacks
def _io_validate_data_proxy(schema, data_proxy, partial=None):
    errors = {}
    defers = []
    for name, field in schema.fields.items():
        if partial and name not in partial:
            continue
        data_name = field.attribute or name
        value = data_proxy._data[data_name]
        try:
            # Also look for required
            field._validate_missing(value)
            if value is not missing:
                if field.io_validate:
                    defer = _run_validators(field.io_validate, field, value)
                    defer.addErrback(_errback_factory(errors, name))
                    defers.append(defer)
        except ValidationError as ve:
            errors[name] = ve.messages
    yield DeferredList(defers)
    if errors:
        raise ValidationError(errors)

def _reference_io_validate(field, value):
    return value.fetch(no_data=True)

@inlineCallbacks
def _list_io_validate(field, value):
    validators = field.container.io_validate
    if not validators or not value:
        return
    errors = {}
    defers = []
    for i, e in enumerate(value):
        defer = _run_validators(validators, field.container, e)
        defer.addErrback(_errback_factory(errors, i))
        defers.append(defer)
    yield DeferredList(defers)
    if errors:
        raise ValidationError(errors)

def _embedded_document_io_validate(field, value):
    return _io_validate_data_proxy(value.schema, value._data)

class TxMongoReference(Reference):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._document = None

    @inlineCallbacks
    def fetch(self, no_data=False):
        if not self._document:
            if self.pk is None:
                raise ReferenceError('Cannot retrieve a None Reference')
            self._document = yield self.document_cls.find_one(self.pk)
            if not self._document:
                raise ValidationError(self.error_messages['not_found'].format(
                    document=self.document_cls.__name__))
        returnValue(self._document)

class TxMongoBuilder(BaseBuilder):

    BASE_DOCUMENT_CLS = TxMongoDocument

    @staticmethod
    def is_compatible_with(db):
        return isinstance(db, Database)

    def _patch_field(self, field):
        super()._patch_field(field)

        validators = field.io_validate
        if not validators:
            field.io_validate = []
        else:
            if hasattr(validators, '__iter__'):
                validators = list(validators)
            else:
                validators = [validators]
            field.io_validate = validators
        if isinstance(field, ListField):
            field.io_validate.append(_list_io_validate)
        if isinstance(field, ReferenceField):
            field.io_validate.append(_reference_io_validate)
            field.reference_cls = TxMongoReference
        if isinstance(field, EmbeddedField):
            field.io_validate.append(_embedded_document_io_validate)
<commit_msg>Return mongo result for DeleteError and UpdateError in txmongo<commit_after>from txmongo.database import Database
from twisted.internet.defer import (
    inlineCallbacks, Deferred, DeferredList, returnValue, maybeDeferred)
from txmongo import filter as qf
from pymongo.errors import DuplicateKeyError

from ..builder import BaseBuilder
from ..document import DocumentImplementation
from ..data_proxy import DataProxy, missing
from ..data_objects import Reference
from ..exceptions import NotCreatedError, UpdateError, DeleteError, ValidationError
from ..fields import ReferenceField, ListField, EmbeddedField
from ..query_mapper import map_query

from .tools import cook_find_filter

class TxMongoDocument(DocumentImplementation):

    __slots__ = ()

    opts = DocumentImplementation.opts

    @inlineCallbacks
    def reload(self):
        """
        Retrieve and replace document's data by the ones in database.

        Raises :class:`umongo.exceptions.NotCreatedError` if the document
        doesn't exist in database.
        """
        if not self.is_created:
            raise NotCreatedError("Document doesn't exists in database")
        ret = yield self.collection.find_one(self.pk)
        if ret is None:
            raise NotCreatedError("Document doesn't exists in database")
        self._data = DataProxy(self.schema)
        self._data.from_mongo(ret)

    @inlineCallbacks
    def commit(self, io_validate_all=False, conditions=None):
        """
        Commit the document in database.
        If the document doesn't already exist it will be inserted, otherwise
        it will be updated.

        :param io_validate_all:
        :param conditions: only perform commit if matching record in db
            satisfies condition(s) (e.g. version number).
            Raises :class:`umongo.exceptions.UpdateError` if the
            conditions are not satisfied.
         :return: A :class:`pymongo.results.UpdateResult` or
            :class:`pymongo.results.InsertOneResult` depending of the operation.
       """
        try:
            if self.is_created:
                if self.is_modified():
                    query = conditions or {}
                    query['_id'] = self.pk
                    # pre_update can provide additional query filter and/or
                    # modify the fields' values
                    additional_filter = yield maybeDeferred(self.pre_update)
                    if additional_filter:
                        query.update(map_query(additional_filter, self.schema.fields))
                    yield self.io_validate(validate_all=io_validate_all)
                    payload = self._data.to_mongo(update=True)
                    ret = yield self.collection.update_one(query, payload)
                    if ret.matched_count != 1:
                        raise UpdateError(ret)
                    yield maybeDeferred(self.post_update, ret)
                else:
                    ret = None
            elif conditions:
                raise RuntimeError('Document must already exist in database to use `conditions`.')
            else:
                yield maybeDeferred(self.pre_insert)
                yield self.io_validate(validate_all=io_validate_all)
                payload = self._data.to_mongo(update=False)
                ret = yield self.collection.insert_one(payload)
                # TODO: check ret ?
                self._data.set_by_mongo_name('_id', ret.inserted_id)
                self.is_created = True
                yield maybeDeferred(self.post_insert, ret)
        except DuplicateKeyError as exc:
            # Need to dig into error message to find faulting index
            errmsg = exc.details['errmsg']
            for index in self.opts.indexes:
                if ('.$%s' % index.document['name'] in errmsg or
                        ' %s ' % index.document['name'] in errmsg):
                    keys = index.document['key'].keys()
                    if len(keys) == 1:
                        msg = self.schema.fields[keys[0]].error_messages['unique']
                        raise ValidationError({keys[0]: msg})
                    else:
                        fields = self.schema.fields
                        # Compound index (sort value to make testing easier)
                        keys = sorted(keys)
                        raise ValidationError({k: fields[k].error_messages[
                            'unique_compound'].format(fields=keys) for k in keys})
        self._data.clear_modified()
        return ret

    @inlineCallbacks
    def delete(self):
        """
        Remove the document from database.

        Raises :class:`umongo.exceptions.NotCreatedError` if the document
        is not created (i.e. ``doc.is_created`` is False)
        Raises :class:`umongo.exceptions.DeleteError` if the document
        doesn't exist in database.

        :return: A :class:`pymongo.results.DeleteResult`
        """
        if not self.is_created:
            raise NotCreatedError("Document doesn't exists in database")
        yield maybeDeferred(self.pre_delete)
        ret = yield self.collection.delete_one({'_id': self.pk})
        if ret.deleted_count != 1:
            raise DeleteError(ret)
        self.is_created = False
        yield maybeDeferred(self.post_delete, ret)
        return ret

    def io_validate(self, validate_all=False):
        """
        Run the io_validators of the document's fields.

        :param validate_all: If False only run the io_validators of the
            fields that have been modified.
        """
        if validate_all:
            return _io_validate_data_proxy(self.schema, self._data)
        else:
            return _io_validate_data_proxy(
                self.schema, self._data, partial=self._data.get_modified_fields())

    @classmethod
    @inlineCallbacks
    def find_one(cls, spec=None, *args, **kwargs):
        """
        Find a single document in database.
        """
        # In txmongo, `spec` is for filtering and `filter` is for sorting
        spec = cook_find_filter(cls, spec)
        ret = yield cls.collection.find_one(*args, spec=spec, **kwargs)
        if ret is not None:
            ret = cls.build_from_mongo(ret, use_cls=True)
        return ret

    @classmethod
    @inlineCallbacks
    def find(cls, spec=None, *args, **kwargs):
        """
        Find a list document in database.

        Returns a cursor that provide Documents.
        """
        # In txmongo, `spec` is for filtering and `filter` is for sorting
        spec = cook_find_filter(cls, spec)
        raw_cursor_or_list = yield cls.collection.find(*args, spec=spec, **kwargs)
        if isinstance(raw_cursor_or_list, tuple):

            def wrap_raw_results(result):
                cursor = result[1]
                if cursor is not None:
                    cursor.addCallback(wrap_raw_results)
                return ([cls.build_from_mongo(e, use_cls=True) for e in result[0]], cursor)

            return wrap_raw_results(raw_cursor_or_list)
        else:
            return [cls.build_from_mongo(e, use_cls=True) for e in raw_cursor_or_list]

    @classmethod
    def count(cls, spec=None, **kwargs):
        """
        Get the number of documents in this collection.
        """
        # In txmongo, `spec` is for filtering and `filter` is for sorting
        spec = cook_find_filter(cls, spec)
        return cls.collection.count(spec=spec, **kwargs)

    @classmethod
    @inlineCallbacks
    def ensure_indexes(cls):
        """
        Check&create if needed the Document's indexes in database
        """
        for index in cls.opts.indexes:
            kwargs = index.document.copy()
            keys = kwargs.pop('key')
            index = qf.sort([(k, d) for k, d in keys.items()])
            yield cls.collection.create_index(index, **kwargs)

def _errback_factory(errors, field=None):

    def errback(err):
        if isinstance(err.value, ValidationError):
            if field:
                errors[field] = err.value.messages
            else:
                errors.extend(err.value.messages)
        else:
            raise err.value

    return errback

# Run multiple validators and collect all errors in one
@inlineCallbacks
def _run_validators(validators, field, value):
    errors = []
    defers = []
    for validator in validators:
        defer = validator(field, value)
        assert isinstance(defer, Deferred), 'io_validate functions must return a Deferred'
        defer.addErrback(_errback_factory(errors))
        defers.append(defer)
    yield DeferredList(defers)
    if errors:
        raise ValidationError(errors)

@inlineCallbacks
def _io_validate_data_proxy(schema, data_proxy, partial=None):
    errors = {}
    defers = []
    for name, field in schema.fields.items():
        if partial and name not in partial:
            continue
        data_name = field.attribute or name
        value = data_proxy._data[data_name]
        try:
            # Also look for required
            field._validate_missing(value)
            if value is not missing:
                if field.io_validate:
                    defer = _run_validators(field.io_validate, field, value)
                    defer.addErrback(_errback_factory(errors, name))
                    defers.append(defer)
        except ValidationError as ve:
            errors[name] = ve.messages
    yield DeferredList(defers)
    if errors:
        raise ValidationError(errors)

def _reference_io_validate(field, value):
    return value.fetch(no_data=True)

@inlineCallbacks
def _list_io_validate(field, value):
    validators = field.container.io_validate
    if not validators or not value:
        return
    errors = {}
    defers = []
    for i, e in enumerate(value):
        defer = _run_validators(validators, field.container, e)
        defer.addErrback(_errback_factory(errors, i))
        defers.append(defer)
    yield DeferredList(defers)
    if errors:
        raise ValidationError(errors)

def _embedded_document_io_validate(field, value):
    return _io_validate_data_proxy(value.schema, value._data)

class TxMongoReference(Reference):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._document = None

    @inlineCallbacks
    def fetch(self, no_data=False):
        if not self._document:
            if self.pk is None:
                raise ReferenceError('Cannot retrieve a None Reference')
            self._document = yield self.document_cls.find_one(self.pk)
            if not self._document:
                raise ValidationError(self.error_messages['not_found'].format(
                    document=self.document_cls.__name__))
        returnValue(self._document)

class TxMongoBuilder(BaseBuilder):

    BASE_DOCUMENT_CLS = TxMongoDocument

    @staticmethod
    def is_compatible_with(db):
        return isinstance(db, Database)

    def _patch_field(self, field):
        super()._patch_field(field)

        validators = field.io_validate
        if not validators:
            field.io_validate = []
        else:
            if hasattr(validators, '__iter__'):
                validators = list(validators)
            else:
                validators = [validators]
            field.io_validate = validators
        if isinstance(field, ListField):
            field.io_validate.append(_list_io_validate)
        if isinstance(field, ReferenceField):
            field.io_validate.append(_reference_io_validate)
            field.reference_cls = TxMongoReference
        if isinstance(field, EmbeddedField):
            field.io_validate.append(_embedded_document_io_validate)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import argparse
import subprocess as sp
import os
import uuid
import sys

HERE = os.path.realpath(os.path.dirname(__file__))
ENV_NAME = 'lcdb-workflows-%s-env' % os.environ['USER']
REQUIREMENTS = os.path.join(HERE, 'requirements.txt')
BLUE = '\033[94m'
GREEN = '\033[92m'
YELLOW = '\033[93m'
RED = '\033[91m'
ENDC = '\033[0m'

ap = argparse.ArgumentParser()

ap.add_argument('repo', help='Path to lcdb-workflows directory to test')

ap.add_argument(
    '--build-env', action='store_true',
    help='Build environment "%s"' % ENV_NAME)

ap.add_argument(
    '--no-test', action='store_true',
    help="Make the temp bash script, but don't actually run the test. "
    "Useful in combination with --build-env")

ap.add_argument(
    '--clean', action='store_true',
    help="Delete the test data directory and re-download data.")

ap.add_argument(
    '--sbatch', action='store_true',
    help='Print the sbatch command required to run the tests')

ap.add_argument(
    '--threads',
    type=int,
    help='''Threads to run. If --sbatch is specified, then request this many
    --cpus-per-task; if $SLURM_CPUS_PER_TASK exists (that is, you're on an
    interactive node) use that; otherwise assume running on a node with this
    many cores.  Snakemake will be run with the -j argument set to this many
    threads.''')

ap.add_argument(
    '--mem',
    default='32g',
    help='Memory to request. Only has an effect if --sbatch also specified.')

ap.add_argument(
    '--workflow',
    default='test',
    help='Which workflow to test')

args = ap.parse_args()

if args.config:
    CONFIG = args.config

else:
    CONFIG = os.path.join(os.path.dirname(args.workflow), 'config.yaml')

# If no threads supplied and we're on an interactive node, use as many CPUs as
# are allocated.
if args.threads is None:
    try:
        threads = os.environ['SLURM_CPUS_PER_TASK']
    except KeyError:
        threads = 1
else:
    threads = args.threads

def env_exists(name):
    """
    See if the environment already exists by checking output of `conda env
    list`.
    """
    return any(
        map(
            lambda x: name in x,
            sp.check_output(
                ['conda', 'env', 'list'], universal_newlines=True
            ).splitlines()
        )
    )

def create_env(name):
    """
    Remove environment if it exists and build a new one.
    """
    if env_exists(name):
        print(
            YELLOW
            + 'conda env '
            + name + ' exists, removing and rebuilding based on '
            + requirements(repo)
            +  ENDC)
        sp.check_call([
            'conda', 'remove', '-n', name, '--all'])
    sp.check_call([
        'conda', 'create', '-n', name, '--file', requirements(repo), '-c', 'bioconda', '-c', 'r', 'python=3', '-y'])

if args.build_env:
    create_env(ENV_NAME)

DIRECTORY = os.path.dirname(args.workflow)
SNAKEMAKE = "snakemake --directory {DIRECTORY} -s {args.workflow}".format(**locals())

if args.clean:
    CLEAN = SNAKEMAKE + ' clean'
else:
    CLEAN = ""

# This will be saved as a temp file and either run or printed out in the sbatch
# command.
script = """\
#!/bin/bash
set -eo pipefail
source activate {ENV_NAME}
{CLEAN}
{SNAKEMAKE} -j {threads} -p -r -T --verbose --configfile {CONFIG}
"""
script_name = 'lcdb-workflows-submit-%s.sh' % (str(uuid.uuid4()).split('-')[0])
with open(script_name, 'w') as fout:
    fout.write(script.format(**locals()))
print(BLUE + "Wrote " + script_name  + ENDC)

if args.sbatch:
    cmd = [
        'sbatch',
        '--mem', args.mem,
        '--cpus-per-task', str(threads),
        '--mail-type=END,FAIL',
        script_name]
    sp.check_call(cmd)
    sys.exit(0)

    sp.check_call(['bash', script_name])
<commit_msg>add new args to run_test and call lcdb-submit if needed<commit_after>#!/usr/bin/env python

import argparse
import subprocess as sp
import os
import uuid
import sys

HERE = os.path.realpath(os.path.dirname(__file__))
ENV_NAME = 'lcdb-workflows-%s-env' % os.environ['USER']
REQUIREMENTS = os.path.join(HERE, 'requirements.txt')
BLUE = '\033[94m'
GREEN = '\033[92m'
YELLOW = '\033[93m'
RED = '\033[91m'
ENDC = '\033[0m'

ap = argparse.ArgumentParser()

ap.add_argument(
    'workflow',
    default='test/Snakefile',
    help='Which workflow to test (path to snakefile)')

ap.add_argument(
    '--build-env', action='store_true',
    help='Build environment "%s"' % ENV_NAME)

ap.add_argument(
    '--no-test', action='store_true',
    help="Make the temp bash script, but don't actually run the test. "
    "Useful in combination with --build-env")

ap.add_argument(
    '--clean', action='store_true',
    help="Run the 'clean' rule of the snakefile before calling it a second time")

ap.add_argument(
    '--sbatch', action='store_true',
    help='Print the sbatch command required to run the tests')

ap.add_argument(
    '--threads',
    type=int,
    help='''Threads to run. If --sbatch is specified, then request this many
    --cpus-per-task; if $SLURM_CPUS_PER_TASK exists (that is, you're on an
    interactive node) use that; otherwise assume running on a node with this
    many cores.  Snakemake will be run with the -j argument set to this many
    threads. If --cluster is specified, this has no effect (the cluster
    submission command limits the number of jobs to 999)''')

ap.add_argument(
    '--mem',
    default='32g',
    help='Memory to request. Only has an effect if --sbatch also specified.')

ap.add_argument(
    '--cluster',
    action='store_true',
    help='Instead of submitting a single job to the cluster, build a submit '
    'script and cluster config and submit that to the cluster')

ap.add_argument(
    '--config',
    help='''Specify a default config file. The default is to use the
    config.yaml file in the same directory as the specified workflow.''')

args = ap.parse_args()

if args.config:
    CONFIG = args.config

else:
    CONFIG = os.path.join(os.path.dirname(args.workflow), 'config.yaml')

# If no threads supplied and we're on an interactive node, use as many CPUs as
# are allocated.
if args.threads is None:
    try:
        threads = os.environ['SLURM_CPUS_PER_TASK']
    except KeyError:
        threads = 1
else:
    threads = args.threads

def env_exists(name):
    """
    See if the environment already exists by checking output of `conda env
    list`.
    """
    return any(
        map(
            lambda x: name in x,
            sp.check_output(
                ['conda', 'env', 'list'], universal_newlines=True
            ).splitlines()
        )
    )

def create_env(name):
    """
    Remove environment if it exists and build a new one.
    """
    if env_exists(name):
        print(
            YELLOW
            + 'conda env '
            + name + ' exists, removing and rebuilding based on '
            + requirements(repo)
            +  ENDC)
        sp.check_call([
            'conda', 'remove', '-n', name, '--all'])
    sp.check_call([
        'conda', 'create', '-n', name, '--file', requirements(repo), '-c', 'bioconda', '-c', 'r', 'python=3', '-y'])

if args.build_env:
    create_env(ENV_NAME)

DIRECTORY = os.path.dirname(args.workflow)
SNAKEMAKE = "snakemake --directory {DIRECTORY} -s {args.workflow}".format(**locals())

if args.clean:
    CLEAN = SNAKEMAKE + ' clean'
else:
    CLEAN = ""

# This will be saved as a temp file and either run or printed out in the sbatch
# command.
script = """\
#!/bin/bash
set -eo pipefail
source activate {ENV_NAME}
{CLEAN}
{SNAKEMAKE} -j {threads} -p -r -T --verbose --configfile {CONFIG}
"""
script_name = 'lcdb-workflows-submit-%s.sh' % (str(uuid.uuid4()).split('-')[0])
with open(script_name, 'w') as fout:
    fout.write(script.format(**locals()))
print(BLUE + "Wrote " + script_name  + ENDC)

if args.sbatch:
    cmd = [
        'sbatch',
        '--mem', args.mem,
        '--cpus-per-task', str(threads),
        '--mail-type=END,FAIL',
        script_name]
    sp.check_call(cmd)
    sys.exit(0)

elif args.cluster:
    # run the separate submit script so that it can do the config parsing and
    # cluster-config.yaml building work
    cmd = [
        'python', 'lcdb/lcdb-submit.py',
        '--config', CONFIG,
        '--workflow', args.workflow,
        '--cluster-config', 'test-cluster-config.yaml',
        '--output', script_name,
        '--pre-block', CLEAN
    ]
    sp.check_call(cmd)

if not args.no_test:
    print(BLUE + "Running " + script_name + ENDC)
    sp.check_call(['bash', script_name])
<|endoftext|>