<commit_before>from rest_framework import serializers as ser
from rest_framework import exceptions
from modularodm.exceptions import ValidationValueError
from framework.exceptions import PermissionsError
from website.exceptions import NodeStateError

from website.models import Node
from api.base.serializers import LinksField, RelationshipField, DevOnly, JSONAPIRelationshipSerializer
from api.base.serializers import JSONAPISerializer, IDField, TypeField, relationship_diff
from api.base.exceptions import InvalidModelValueError, RelationshipPostMakesNoChanges
from api.base.utils import absolute_reverse, get_user_auth
from api.nodes.serializers import NodeLinksSerializer

class CollectionSerializer(JSONAPISerializer):
    filterable_fields = frozenset([
        'title',
        'date_created',
        'date_modified',
    ])

    id = IDField(source='_id', read_only=True)
    type = TypeField()

    title = ser.CharField(required=True)
    date_created = ser.DateTimeField(read_only=True)
    date_modified = ser.DateTimeField(read_only=True)
    bookmarks = ser.BooleanField(read_only=False, default=False, source='is_bookmark_collection')

    links = LinksField({})

    node_links = DevOnly(RelationshipField(
        related_view='collections:node-pointers',
        related_view_kwargs={'collection_id': '<pk>'},
        related_meta={'count': 'get_node_links_count'}
    ))

    # TODO: Add a self link to this when it's available
    linked_nodes = DevOnly(RelationshipField(
        related_view='collections:linked-nodes',
        related_view_kwargs={'collection_id': '<pk>'},
        related_meta={'count': 'get_node_links_count'},
        self_view='collections:collection-node-pointer-relationship',
        self_view_kwargs={'collection_id': '<pk>'}
    ))

    class Meta:
        type_ = 'collections'

    def get_absolute_url(self, obj):
        return absolute_reverse('collections:collection-detail', kwargs={'collection_id': obj._id})

    def get_node_links_count(self, obj):
        return len(obj.nodes_pointer)

    def create(self, validated_data):
        node = Node(**validated_data)
        node.is_collection = True
        node.category = ''
        try:
            node.save()
        except ValidationValueError as e:
            raise InvalidModelValueError(detail=e.message)
        except NodeStateError:
            raise ser.ValidationError('Each user cannot have more than one Bookmark collection.')
        return node

    def update(self, node, validated_data):
        """Update instance with the validated data. Requires
        the request to be in the serializer context.
        """
        assert isinstance(node, Node), 'collection must be a Node'
        auth = get_user_auth(self.context['request'])

        if validated_data:
            try:
                node.update(validated_data, auth=auth)
            except ValidationValueError as e:
                raise InvalidModelValueError(detail=e.message)
            except PermissionsError:
                raise exceptions.PermissionDenied

        return node

class CollectionDetailSerializer(CollectionSerializer):
    """
    Overrides CollectionSerializer to make id required.
    """
    id = IDField(source='_id', required=True)

class CollectionNodeLinkSerializer(NodeLinksSerializer):
    def get_absolute_url(self, obj):
        node_id = self.context['request'].parser_context['kwargs']['collection_id']
        return absolute_reverse(
            'collections:node-pointer-detail',
            kwargs={
                'collection_id': node_id,
                'node_link_id': obj._id
            }
        )

class LinkedNode(JSONAPIRelationshipSerializer):
    id = ser.CharField(source='node._id', required=False, allow_null=True)
    class Meta:
        type_ = 'linked_nodes'

class CollectionLinkedNodesRelationshipSerializer(ser.Serializer):

    data = ser.ListField(child=LinkedNode())
    links = LinksField({'self': 'get_self_url',
                        'html': 'get_related_url'})

    def get_self_url(self, obj):
        return obj['self'].linked_nodes_self_url

    def get_related_url(self, obj):
        return obj['self'].linked_nodes_related_url

    class Meta:
        type_ = 'linked_nodes'

    def get_pointers_to_add_remove(self, pointers, new_pointers):
        diff = relationship_diff(
            current_items={pointer.node._id: pointer for pointer in pointers},
            new_items={val['node']['_id']: val for val in new_pointers}
        )

        nodes_to_add = []
        for node_id in diff['add']:
            node = Node.load(node_id)
            if not node:
                raise exceptions.NotFound(detail='Node with id "{}" was not found'.format(node_id))
            nodes_to_add.append(node)

        return nodes_to_add, diff['remove'].values()

    def make_instance_obj(self, obj):
        # Convenience method to format instance based on view's get_object
        return {'data': [
            pointer for pointer in
            obj.nodes_pointer
            if not pointer.node.is_deleted and not pointer.node.is_collection
        ], 'self': obj}

    def update(self, instance, validated_data):
        collection = instance['self']
        auth = get_user_auth(self.context['request'])

        add, remove = self.get_pointers_to_add_remove(pointers=instance['data'], new_pointers=validated_data['data'])

        for pointer in remove:
            collection.rm_pointer(pointer, auth)
        for node in add:
            collection.add_pointer(node, auth)

        return self.make_instance_obj(collection)

    def create(self, validated_data):
        instance = self.context['view'].get_object()
        auth = get_user_auth(self.context['request'])
        collection = instance['self']

        add, remove = self.get_pointers_to_add_remove(pointers=instance['data'], new_pointers=validated_data['data'])

        if not len(add):
            raise RelationshipPostMakesNoChanges

        for node in add:
            collection.add_pointer(node, auth)

        return self.make_instance_obj(collection)
<commit_msg>Remove DevOnly from necessary collections fields<commit_after>from rest_framework import serializers as ser
from rest_framework import exceptions
from modularodm.exceptions import ValidationValueError
from framework.exceptions import PermissionsError
from website.exceptions import NodeStateError

from website.models import Node
from api.base.serializers import LinksField, RelationshipField, JSONAPIRelationshipSerializer
from api.base.serializers import JSONAPISerializer, IDField, TypeField, relationship_diff
from api.base.exceptions import InvalidModelValueError, RelationshipPostMakesNoChanges
from api.base.utils import absolute_reverse, get_user_auth
from api.nodes.serializers import NodeLinksSerializer

class CollectionSerializer(JSONAPISerializer):
    filterable_fields = frozenset([
        'title',
        'date_created',
        'date_modified',
    ])

    id = IDField(source='_id', read_only=True)
    type = TypeField()

    title = ser.CharField(required=True)
    date_created = ser.DateTimeField(read_only=True)
    date_modified = ser.DateTimeField(read_only=True)
    bookmarks = ser.BooleanField(read_only=False, default=False, source='is_bookmark_collection')

    links = LinksField({})

    node_links = RelationshipField(
        related_view='collections:node-pointers',
        related_view_kwargs={'collection_id': '<pk>'},
        related_meta={'count': 'get_node_links_count'}
    )

    # TODO: Add a self link to this when it's available
    linked_nodes = RelationshipField(
        related_view='collections:linked-nodes',
        related_view_kwargs={'collection_id': '<pk>'},
        related_meta={'count': 'get_node_links_count'},
        self_view='collections:collection-node-pointer-relationship',
        self_view_kwargs={'collection_id': '<pk>'}
    )

    class Meta:
        type_ = 'collections'

    def get_absolute_url(self, obj):
        return absolute_reverse('collections:collection-detail', kwargs={'collection_id': obj._id})

    def get_node_links_count(self, obj):
        return len(obj.nodes_pointer)

    def create(self, validated_data):
        node = Node(**validated_data)
        node.is_collection = True
        node.category = ''
        try:
            node.save()
        except ValidationValueError as e:
            raise InvalidModelValueError(detail=e.message)
        except NodeStateError:
            raise ser.ValidationError('Each user cannot have more than one Bookmark collection.')
        return node

    def update(self, node, validated_data):
        """Update instance with the validated data. Requires
        the request to be in the serializer context.
        """
        assert isinstance(node, Node), 'collection must be a Node'
        auth = get_user_auth(self.context['request'])

        if validated_data:
            try:
                node.update(validated_data, auth=auth)
            except ValidationValueError as e:
                raise InvalidModelValueError(detail=e.message)
            except PermissionsError:
                raise exceptions.PermissionDenied

        return node

class CollectionDetailSerializer(CollectionSerializer):
    """
    Overrides CollectionSerializer to make id required.
    """
    id = IDField(source='_id', required=True)

class CollectionNodeLinkSerializer(NodeLinksSerializer):
    def get_absolute_url(self, obj):
        node_id = self.context['request'].parser_context['kwargs']['collection_id']
        return absolute_reverse(
            'collections:node-pointer-detail',
            kwargs={
                'collection_id': node_id,
                'node_link_id': obj._id
            }
        )

class LinkedNode(JSONAPIRelationshipSerializer):
    id = ser.CharField(source='node._id', required=False, allow_null=True)
    class Meta:
        type_ = 'linked_nodes'

class CollectionLinkedNodesRelationshipSerializer(ser.Serializer):

    data = ser.ListField(child=LinkedNode())
    links = LinksField({'self': 'get_self_url',
                        'html': 'get_related_url'})

    def get_self_url(self, obj):
        return obj['self'].linked_nodes_self_url

    def get_related_url(self, obj):
        return obj['self'].linked_nodes_related_url

    class Meta:
        type_ = 'linked_nodes'

    def get_pointers_to_add_remove(self, pointers, new_pointers):
        diff = relationship_diff(
            current_items={pointer.node._id: pointer for pointer in pointers},
            new_items={val['node']['_id']: val for val in new_pointers}
        )

        nodes_to_add = []
        for node_id in diff['add']:
            node = Node.load(node_id)
            if not node:
                raise exceptions.NotFound(detail='Node with id "{}" was not found'.format(node_id))
            nodes_to_add.append(node)

        return nodes_to_add, diff['remove'].values()

    def make_instance_obj(self, obj):
        # Convenience method to format instance based on view's get_object
        return {'data': [
            pointer for pointer in
            obj.nodes_pointer
            if not pointer.node.is_deleted and not pointer.node.is_collection
        ], 'self': obj}

    def update(self, instance, validated_data):
        collection = instance['self']
        auth = get_user_auth(self.context['request'])

        add, remove = self.get_pointers_to_add_remove(pointers=instance['data'], new_pointers=validated_data['data'])

        for pointer in remove:
            collection.rm_pointer(pointer, auth)
        for node in add:
            collection.add_pointer(node, auth)

        return self.make_instance_obj(collection)

    def create(self, validated_data):
        instance = self.context['view'].get_object()
        auth = get_user_auth(self.context['request'])
        collection = instance['self']

        add, remove = self.get_pointers_to_add_remove(pointers=instance['data'], new_pointers=validated_data['data'])

        if not len(add):
            raise RelationshipPostMakesNoChanges

        for node in add:
            collection.add_pointer(node, auth)

        return self.make_instance_obj(collection)
<|endoftext|>


<commit_before># Copyright 2012 10gen, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Test Motor, an asynchronous driver for MongoDB and Tornado."""

import datetime
import functools
import threading
import time
import unittest

from pymongo.errors import OperationFailure
from tornado import ioloop, gen

import motor
from test import host, port, MotorTest, async_test_engine, AssertRaises

class MotorTailTest(MotorTest):
    def setUp(self):
        super(MotorTailTest, self).setUp()
        self.sync_db.capped.drop()
        # autoIndexId catches test bugs that try to insert duplicate _id's
        self.sync_db.create_collection(
            'capped', capped=True, size=1000, autoIndexId=True)

        self.sync_db.uncapped.drop()
        self.sync_db.uncapped.insert({})

    def test_tail_callback(self):
        test_db = self.motor_client(host, port).pymongo_test
        capped = test_db.capped
        self.check_required_callback(capped.find().tail)

    def start_insertion_thread(self, pauses):
        """A thread that gradually inserts documents into a capped collection
        """
        def add_docs():
            i = 0
            for pause in pauses:
                if pause == 'drop':
                    self.sync_db.capped.drop()
                else:
                    time.sleep(pause)
                    self.sync_db.capped.insert({'_id': i})
                    i += 1

        t = threading.Thread(target=add_docs)
        t.start()
        return t

    # Used by test_tail, test_tail_drop_collection, etc.
    def each(self, results, n_expected, callback, result, error):
        if error:
            results.append(type(error))
        elif result:
            results.append(result)
            if len(results) != n_expected:
                # Continue
                return

        # Cancel iteration
        callback()
        return False

    # Need at least one longish pause to ensure tail() recovers when cursor
    # times out and returns None
    tail_pauses = (
        1, 0, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0.1, 0.1, 0, 0)

    @async_test_engine(timeout_sec=sum(tail_pauses) + 30)
    def test_tail(self, done):
        t = self.start_insertion_thread(self.tail_pauses)
        results = []
        each = functools.partial(
            self.each, results, len(self.tail_pauses),
            (yield gen.Callback('done')))

        test_db = self.motor_client(host, port).pymongo_test
        capped = test_db.capped

        # Note we do *not* pass tailable or await_data to find(), the
        # convenience method handles it for us.
        capped.find().tail(each)
        yield gen.Wait('done')
        self.assertEqual(
            results,
            [{'_id': i} for i in range(len(self.tail_pauses))])

        t.join()
        yield gen.Task(self.wait_for_cursors)
        done()

    @async_test_engine(timeout_sec=30)
    def test_tail_empty(self, done):
        pauses = (0, 1)
        results = []
        each = functools.partial(
            self.each, results, len(pauses),
            (yield gen.Callback('done')))

        test_db = self.motor_client(host, port).pymongo_test
        capped = test_db.capped

        capped.find().tail(each)
        loop = ioloop.IOLoop.instance()
        # Tail empty collection for a while before inserting
        yield gen.Task(loop.add_timeout, datetime.timedelta(seconds=2))
        t = self.start_insertion_thread(pauses)
        yield gen.Wait('done')
        self.assertEqual(
            results,
            [{'_id': i} for i in range(len(pauses))])

        t.join()
        yield gen.Task(self.wait_for_cursors)
        done()

    drop_collection_pauses = (0, 0, 5, 'drop', 1, 0, 0)

    @async_test_engine(timeout_sec=30)
    def test_tail_drop_collection(self, done):
        # Ensure tail() throws error when its collection is dropped
        results = []
        each = functools.partial(
            self.each, results, len(self.drop_collection_pauses),
            (yield gen.Callback('done')))

        test_db = self.motor_client(host, port).pymongo_test
        capped = test_db.capped
        capped.find().tail(each)
        t = self.start_insertion_thread(self.drop_collection_pauses)
        yield gen.Wait('done')

        # Don't assume that the first 3 results before the drop will be
        # recorded -- dropping a collection kills the cursor even if not
        # fully iterated.
        self.assertTrue(OperationFailure in results)
        self.assertFalse('cancelled' in results)
        t.join()
        yield gen.Task(self.wait_for_cursors)
        done()

    @async_test_engine()
    def test_tail_uncapped_collection(self, done):
        test_db = self.motor_client(host, port).pymongo_test
        uncapped = test_db.uncapped
        yield AssertRaises(OperationFailure, uncapped.find().tail)
        done()

    @async_test_engine(timeout_sec=30)
    def test_tail_nonempty_collection(self, done):
        self.sync_db.capped.insert([{'_id': -2}, {'_id': -1}])

        pauses = (0, 0, 1, 0, 0)
        t = self.start_insertion_thread(pauses)

        results = []
        each = functools.partial(
            self.each, results, len(pauses) + 2, (yield gen.Callback('done')))

        test_db = self.motor_client(host, port).pymongo_test
        capped = test_db.capped
        capped.find().tail(each)
        yield gen.Wait('done')
        self.assertEqual([{'_id': i} for i in range(-2, len(pauses))], results)

        t.join()
        yield gen.Task(self.wait_for_cursors)
        done()

    @async_test_engine(timeout_sec=10)
    def test_tail_close(self, done):
        # Make sure closing a cursor stops iteration immediately
        pauses = (0, 0, 1, 0, 0)
        t = self.start_insertion_thread(pauses)

        results = []

        test_db = self.motor_client(host, port).pymongo_test
        cursor = test_db.capped.find()

        def each(result, error):
            if error:
                results.append(type(error))
            elif result:
                results.append(result)
                if len(results) == 3:
                    cursor.close()

        cursor.tail(each)
        yield gen.Task(ioloop.IOLoop.instance().add_timeout, time.time() + 2)
        t.join()
        self.assertEqual([{'_id': i} for i in range(3)], results)
        yield gen.Task(self.wait_for_cursors)
        done()

    @async_test_engine(timeout_sec=30)
    def test_tail_gen(self, done):
        pauses = (1, 0.5, 1, 0, 0)
        t = self.start_insertion_thread(pauses)

        loop = ioloop.IOLoop.instance()
        results = []

        test_db = self.motor_client(host, port).pymongo_test
        capped = test_db.capped
        cursor = capped.find(tailable=True, await_data=True)
        while len(results) < len(pauses):
            if not cursor.alive:
                # While collection is empty, tailable cursor dies immediately
                yield gen.Task(loop.add_timeout, time.time() + 0.1)
                cursor = capped.find(tailable=True, await_data=True)

            if (yield cursor.fetch_next):
                results.append(cursor.next_object())

        t.join()
        self.assertEqual([{'_id': i} for i in range(len(pauses))], results)
        yield motor.Op(cursor.close)
        yield gen.Task(self.wait_for_cursors)
        done()

if __name__ == '__main__':
    unittest.main()
<commit_msg>Try to avoid test_tail_drop_collection timeouts<commit_after># Copyright 2012 10gen, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Test Motor, an asynchronous driver for MongoDB and Tornado."""

import datetime
import functools
import threading
import time
import unittest

from pymongo.errors import OperationFailure
from tornado import ioloop, gen

import motor
from test import host, port, MotorTest, async_test_engine, AssertRaises

class MotorTailTest(MotorTest):
    def setUp(self):
        super(MotorTailTest, self).setUp()
        self.sync_db.capped.drop()
        # autoIndexId catches test bugs that try to insert duplicate _id's
        self.sync_db.create_collection(
            'capped', capped=True, size=1000, autoIndexId=True)

        self.sync_db.uncapped.drop()
        self.sync_db.uncapped.insert({})

    def test_tail_callback(self):
        test_db = self.motor_client(host, port).pymongo_test
        capped = test_db.capped
        self.check_required_callback(capped.find().tail)

    def start_insertion_thread(self, pauses):
        """A thread that gradually inserts documents into a capped collection
        """
        def add_docs():
            i = 0
            for pause in pauses:
                if pause == 'drop':
                    self.sync_db.capped.drop()
                else:
                    time.sleep(pause)
                    self.sync_db.capped.insert({'_id': i})
                    i += 1

        t = threading.Thread(target=add_docs)
        t.start()
        return t

    # Used by test_tail, test_tail_drop_collection, etc.
    def each(self, results, n_expected, callback, result, error):
        if error:
            results.append(type(error))
        elif result:
            results.append(result)
            if len(results) != n_expected:
                # Continue
                return

        # Cancel iteration
        callback()
        return False

    # Need at least one longish pause to ensure tail() recovers when cursor
    # times out and returns None
    tail_pauses = (
        1, 0, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0.1, 0.1, 0, 0)

    @async_test_engine(timeout_sec=sum(tail_pauses) + 30)
    def test_tail(self, done):
        t = self.start_insertion_thread(self.tail_pauses)
        results = []
        each = functools.partial(
            self.each, results, len(self.tail_pauses),
            (yield gen.Callback('done')))

        test_db = self.motor_client(host, port).pymongo_test
        capped = test_db.capped

        # Note we do *not* pass tailable or await_data to find(), the
        # convenience method handles it for us.
        capped.find().tail(each)
        yield gen.Wait('done')
        self.assertEqual(
            results,
            [{'_id': i} for i in range(len(self.tail_pauses))])

        t.join()
        yield gen.Task(self.wait_for_cursors)
        done()

    @async_test_engine(timeout_sec=30)
    def test_tail_empty(self, done):
        pauses = (0, 1)
        results = []
        each = functools.partial(
            self.each, results, len(pauses),
            (yield gen.Callback('done')))

        test_db = self.motor_client(host, port).pymongo_test
        capped = test_db.capped

        capped.find().tail(each)
        loop = ioloop.IOLoop.instance()
        # Tail empty collection for a while before inserting
        yield gen.Task(loop.add_timeout, datetime.timedelta(seconds=2))
        t = self.start_insertion_thread(pauses)
        yield gen.Wait('done')
        self.assertEqual(
            results,
            [{'_id': i} for i in range(len(pauses))])

        t.join()
        yield gen.Task(self.wait_for_cursors)
        done()

    drop_collection_pauses = (0, 0, 15, 'drop', 1, 0, 0)

    @async_test_engine(timeout_sec=30)
    def test_tail_drop_collection(self, done):
        # Ensure tail() throws error when its collection is dropped
        results = []
        each = functools.partial(
            self.each, results, len(self.drop_collection_pauses),
            (yield gen.Callback('done')))

        test_db = self.motor_client(host, port).pymongo_test
        capped = test_db.capped
        capped.find().tail(each)
        t = self.start_insertion_thread(self.drop_collection_pauses)
        yield gen.Wait('done')

        # Don't assume that the first 3 results before the drop will be
        # recorded -- dropping a collection kills the cursor even if not
        # fully iterated.
        self.assertTrue(OperationFailure in results)
        self.assertFalse('cancelled' in results)
        t.join()
        yield gen.Task(self.wait_for_cursors)
        done()

    @async_test_engine()
    def test_tail_uncapped_collection(self, done):
        test_db = self.motor_client(host, port).pymongo_test
        uncapped = test_db.uncapped
        yield AssertRaises(OperationFailure, uncapped.find().tail)
        done()

    @async_test_engine(timeout_sec=30)
    def test_tail_nonempty_collection(self, done):
        self.sync_db.capped.insert([{'_id': -2}, {'_id': -1}])

        pauses = (0, 0, 1, 0, 0)
        t = self.start_insertion_thread(pauses)

        results = []
        each = functools.partial(
            self.each, results, len(pauses) + 2, (yield gen.Callback('done')))

        test_db = self.motor_client(host, port).pymongo_test
        capped = test_db.capped
        capped.find().tail(each)
        yield gen.Wait('done')
        self.assertEqual([{'_id': i} for i in range(-2, len(pauses))], results)

        t.join()
        yield gen.Task(self.wait_for_cursors)
        done()

    @async_test_engine(timeout_sec=10)
    def test_tail_close(self, done):
        # Make sure closing a cursor stops iteration immediately
        pauses = (0, 0, 1, 0, 0)
        t = self.start_insertion_thread(pauses)

        results = []

        test_db = self.motor_client(host, port).pymongo_test
        cursor = test_db.capped.find()

        def each(result, error):
            if error:
                results.append(type(error))
            elif result:
                results.append(result)
                if len(results) == 3:
                    cursor.close()

        cursor.tail(each)
        yield gen.Task(ioloop.IOLoop.instance().add_timeout, time.time() + 2)
        t.join()
        self.assertEqual([{'_id': i} for i in range(3)], results)
        yield gen.Task(self.wait_for_cursors)
        done()

    @async_test_engine(timeout_sec=30)
    def test_tail_gen(self, done):
        pauses = (1, 0.5, 1, 0, 0)
        t = self.start_insertion_thread(pauses)

        loop = ioloop.IOLoop.instance()
        results = []

        test_db = self.motor_client(host, port).pymongo_test
        capped = test_db.capped
        cursor = capped.find(tailable=True, await_data=True)
        while len(results) < len(pauses):
            if not cursor.alive:
                # While collection is empty, tailable cursor dies immediately
                yield gen.Task(loop.add_timeout, time.time() + 0.1)
                cursor = capped.find(tailable=True, await_data=True)

            if (yield cursor.fetch_next):
                results.append(cursor.next_object())

        t.join()
        self.assertEqual([{'_id': i} for i in range(len(pauses))], results)
        yield motor.Op(cursor.close)
        yield gen.Task(self.wait_for_cursors)
        done()

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import sys
import unittest
import logging

sys.path.append('..')
sys.path.append('../lib')
import parse
from patXML import *

basedir = os.path.dirname(__file__)
testdir = os.path.join(basedir, './fixtures/xml/')
testfileone = 'ipg120327.one.xml'
testfiletwo = 'ipg120327.two.xml'
xmlclasses = [AssigneeXML, CitationXML, ClassXML, InventorXML, \
              PatentXML, PatdescXML, LawyerXML, ScirefXML, UsreldocXML]

class TestParseFile(unittest.TestCase):

    def setUp(self):
        pass

    def test_parse_file_one(self):
        parsed_output = parse.parse_file(testdir+testfileone)
        self.assertTrue(isinstance(parsed_output, list))
        self.assertTrue(len(parsed_output) == 1)
        self.assertTrue(isinstance(parsed_output[0], str))

    def test_parallel_parse_one(self):
        filelist = [testdir+testfileone]
        parsed_output = parse.parallel_parse(filelist)
        self.assertTrue(isinstance(parsed_output,list))
        self.assertTrue(len(parsed_output) == 1)
        self.assertTrue(isinstance(parsed_output[0],list))
        self.assertTrue(len(parsed_output[0]) == 1)
        self.assertTrue(isinstance(parsed_output[0][0], str))

    def test_parse_file_two(self):
        parsed_output = parse.parse_file(testdir+testfiletwo)
        self.assertTrue(isinstance(parsed_output, list))
        self.assertTrue(len(parsed_output) == 2)
        self.assertTrue(isinstance(parsed_output[0], str))
        self.assertTrue(isinstance(parsed_output[1], str))

    def test_parallel_parse_two(self):
        filelist = [testdir+testfiletwo]
        parsed_output = parse.parallel_parse(filelist)
        self.assertTrue(isinstance(parsed_output,list))
        self.assertTrue(len(parsed_output) == 1)
        self.assertTrue(isinstance(parsed_output[0],list))
        self.assertTrue(len(parsed_output[0]) == 2)
        self.assertTrue(isinstance(parsed_output[0][0], str))
        self.assertTrue(isinstance(parsed_output[0][1], str))

    def test_use_parallel_parse_one(self):
        filelist = [testdir+testfileone]
        parsed_output = parse.parallel_parse(filelist)
        parsed_xml = [xmlclass(parsed_output[0][0]) for xmlclass in xmlclasses]
        self.assertTrue(len(parsed_xml) == len(xmlclasses))
        self.assertTrue(all(parsed_xml))

    def test_use_parallel_parse_two(self):
        filelist = [testdir+testfiletwo]
        parsed_output = parse.parallel_parse(filelist)
        parsed_xml = []
        for us_patent_grant in parsed_output[0]:
            self.assertTrue(isinstance(us_patent_grant, str))
            for xmlclass in xmlclasses:
                parsed_xml.append(xmlclass(us_patent_grant))
        self.assertTrue(len(parsed_xml) == 2 * len(xmlclasses))
        self.assertTrue(all(parsed_xml))

    def test_list_files(self):
        patentroot = '.'
        testdir = [os.path.join(basedir, './fixtures/unittest/fixtures')]
        xmlregex = r'2012_\d.xml'
        files = parse.list_files(testdir, patentroot, xmlregex)
        self.assertTrue(isinstance(files, list))
        self.assertTrue(len(files) == 9)
        self.assertTrue(all(filter(lambda x: isinstance(x, str), files)))
        self.assertTrue(all(map(lambda x: os.path.exists(x), files)))

if __name__ == '__main__':
    unittest.main()
<commit_msg>add sanity check to ensure we parse out XML strings<commit_after>#!/usr/bin/env python

import os
import sys
import unittest
import logging

sys.path.append('..')
sys.path.append('../lib')
import parse
from patXML import *

basedir = os.path.dirname(__file__)
testdir = os.path.join(basedir, './fixtures/xml/')
testfileone = 'ipg120327.one.xml'
testfiletwo = 'ipg120327.two.xml'
regex = re.compile(r"""
 ([<][?]xml[ ]version.*?[>]       #all XML starts with ?xml
.*?
[<][/]us[-]patent[-]grant[>])    #and here is the end tag
""", re.I+re.S+re.X)
xmlclasses = [AssigneeXML, CitationXML, ClassXML, InventorXML, \
              PatentXML, PatdescXML, LawyerXML, ScirefXML, UsreldocXML]

class TestParseFile(unittest.TestCase):

    def setUp(self):
        pass

    def test_parse_file_one(self):
        parsed_output = parse.parse_file(testdir+testfileone)
        self.assertTrue(isinstance(parsed_output, list))
        self.assertTrue(len(parsed_output) == 1)
        self.assertTrue(isinstance(parsed_output[0], str))
        self.assertTrue(regex.match(parsed_output[0]))

    def test_parallel_parse_one(self):
        filelist = [testdir+testfileone]
        parsed_output = parse.parallel_parse(filelist)
        self.assertTrue(isinstance(parsed_output,list))
        self.assertTrue(len(parsed_output) == 1)
        self.assertTrue(isinstance(parsed_output[0],list))
        self.assertTrue(len(parsed_output[0]) == 1)
        self.assertTrue(isinstance(parsed_output[0][0], str))
        self.assertTrue(regex.match(parsed_output[0]))

    def test_parse_file_two(self):
        parsed_output = parse.parse_file(testdir+testfiletwo)
        self.assertTrue(isinstance(parsed_output, list))
        self.assertTrue(len(parsed_output) == 2)
        self.assertTrue(isinstance(parsed_output[0], str))
        self.assertTrue(isinstance(parsed_output[1], str))
        self.assertTrue(regex.match(parsed_output[0]))
        self.assertTrue(regex.match(parsed_output[1]))

    def test_parallel_parse_two(self):
        filelist = [testdir+testfiletwo]
        parsed_output = parse.parallel_parse(filelist)
        self.assertTrue(isinstance(parsed_output,list))
        self.assertTrue(len(parsed_output) == 1)
        self.assertTrue(isinstance(parsed_output[0],list))
        self.assertTrue(len(parsed_output[0]) == 2)
        self.assertTrue(isinstance(parsed_output[0][0], str))
        self.assertTrue(isinstance(parsed_output[0][1], str))
        self.assertTrue(regex.match(parsed_output[0]))
        self.assertTrue(regex.match(parsed_output[1]))

    def test_use_parallel_parse_one(self):
        filelist = [testdir+testfileone]
        parsed_output = parse.parallel_parse(filelist)
        parsed_xml = [xmlclass(parsed_output[0][0]) for xmlclass in xmlclasses]
        self.assertTrue(len(parsed_xml) == len(xmlclasses))
        self.assertTrue(all(parsed_xml))

    def test_use_parallel_parse_two(self):
        filelist = [testdir+testfiletwo]
        parsed_output = parse.parallel_parse(filelist)
        parsed_xml = []
        for us_patent_grant in parsed_output[0]:
            self.assertTrue(isinstance(us_patent_grant, str))
            for xmlclass in xmlclasses:
                parsed_xml.append(xmlclass(us_patent_grant))
        self.assertTrue(len(parsed_xml) == 2 * len(xmlclasses))
        self.assertTrue(all(parsed_xml))

    def test_list_files(self):
        patentroot = '.'
        testdir = [os.path.join(basedir, './fixtures/unittest/fixtures')]
        xmlregex = r'2012_\d.xml'
        files = parse.list_files(testdir, patentroot, xmlregex)
        self.assertTrue(isinstance(files, list))
        self.assertTrue(len(files) == 9)
        self.assertTrue(all(filter(lambda x: isinstance(x, str), files)))
        self.assertTrue(all(map(lambda x: os.path.exists(x), files)))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import collections
import itertools
import numpy as np
import pandas as pd
from scipy import sparse, ndimage as ndi
import networkx as nx
import numba

## CSGraph and Numba-based implementation

# adapted from github.com/janelia-flyem/gala
def smallest_int_dtype(number, signed=False, min_dtype=np.int8):
    if number < 0:
        signed = True
    if not signed:
        if number <= np.iinfo(np.uint8).max:
            dtype = np.uint8
        if number <= np.iinfo(np.uint16).max:
            dtype = np.uint16
        if number <= np.iinfo(np.uint32).max:
            dtype = np.uint32
        if number <= np.iinfo(np.uint64).max:
            dtype = np.uint64
    else:
        if np.iinfo(np.int8).min <= number <= np.iinfo(np.int8).max:
            dtype = np.int8
        if np.iinfo(np.int16).min <= number <= np.iinfo(np.int16).max:
            dtype = np.int16
        if np.iinfo(np.int32).min <= number <= np.iinfo(np.int32).max:
            dtype = np.int32
        if np.iinfo(np.int64).min <= number <= np.iinfo(np.int64).max:
            dtype = np.int64
    if np.iinfo(dtype).max < np.iinfo(min_dtype).max:
        dtype = min_dtype
    return dtype

# adapted from github.com/janelia-flyem/gala
def pad(ar, vals, axes=None):
    if ar.size == 0:
        return ar
    if axes is None:
        axes = list(range(ar.ndim))
    if not isinstance(vals, collections.Iterable):
        vals = [vals]
    if not isinstance(axes, collections.Iterable):
        axes = [axes]
    p = len(vals)
    newshape = np.array(ar.shape)
    for ax in axes:
        newshape[ax] += 2*p
    vals = np.reshape(vals, (p,) + (1,) * (ar.ndim-1))
    new_dtype = ar.dtype
    if np.issubdtype(new_dtype, np.integer):
        maxval = max([np.max(vals), np.max(ar)])
        minval = min([np.min(vals), np.min(ar)])
        signed = (minval < 0)
        maxval = max(abs(minval), maxval)
        new_dtype = smallest_int_dtype(maxval, signed=signed,
                                       min_dtype=new_dtype)
    ar2 = np.empty(newshape, dtype=new_dtype)
    center = np.ones(newshape, dtype=bool)
    for ax in axes:
        ar2.swapaxes(0, ax)[p-1::-1,...] = vals
        ar2.swapaxes(0, ax)[-p:,...] = vals
        center.swapaxes(0, ax)[p-1::-1,...] = False
        center.swapaxes(0, ax)[-p:,...] = False
    ar2[center] = ar.ravel()
    return ar2

def raveled_steps_to_neighbors(shape, connectivity=1, order='C',
                               return_distances=True):
    if order == 'C':
        dims = shape[-1:0:-1]
    else:
        dims = shape[:-1]
    stepsizes = np.cumprod((1,) + dims)[::-1]
    steps = [stepsizes, -stepsizes]
    distances = [1] * 2 * stepsizes.size
    for nhops in range(2, connectivity + 1):
        prod = np.array(list(itertools.product(*[[1, -1]] * nhops)))
        multisteps = np.array(list(itertools.combinations(stepsizes, nhops))).T
        steps.append((prod @ multisteps).ravel())
        distances.extend([np.sqrt(nhops)] * steps[-1].size)
    if return_distances:
        return np.concatenate(steps).astype(int), np.array(distances)
    else:
        return np.concatenate(steps).astype(int)

@numba.jit(nopython=True, cache=True, nogil=True)
def write_pixel_graph(image, steps, distances, row, col, data):
    """Step over `image` to build a graph of nonzero pixel neighbors.

    Parameters
    ----------
    image : int array
        The input image.
    steps : int array, shape (N,)
        The raveled index steps to find a pixel's neighbors in `image`.
    distances : float array, shape (N,)
        The euclidean distance from a pixel to its corresponding
        neighbor in `steps`.
    row : int array
        Output array to be filled with the "center" pixel IDs.
    col : int array
        Output array to be filled with the "neighbor" pixel IDs.
    data : float array
        Output array to be filled with the distances from center to
        neighbor pixels.

    Notes
    -----
    No size or bounds checking is performed. Users should ensure that
    - No index in `indices` falls on any edge of `image` (or the
      neighbor computation will fail or segfault).
    - The `steps` and `distances` arrays have the same shape.
    - The `row`, `col`, `data` are long enough to hold all of the
      edges.
    """
    image = image.ravel()
    n_neighbors = steps.size
    start_idx = steps.size
    end_idx = image.size + np.min(steps)
    k = 0
    for i in range(start_idx, end_idx + 1):
        if image[i] != 0:
            for j in range(n_neighbors):
                n = steps[j] + i
                if image[n] != 0:
                    row[k] = image[i]
                    col[k] = image[n]
                    data[k] = distances[j]
                    k += 1

def skeleton_to_csgraph(skel):
    """Convert a skeleton image of thin lines to a graph of neighbor pixels.

    Parameters
    ----------
    skel : array
        An input image in which every nonzero pixel is considered part of
        the skeleton, and links between pixels are determined by a full
        n-dimensional neighborhood.

    Returns
    -------
    graph : sparse.csr_matrix
        A graph of shape (Nnz + 1, Nnz + 1), where Nnz is the number of
        nonzero pixels in `skel`. The value graph[i, j] is the distance
        between adjacent pixels i and j. In a 2D image, that would be
        1 for immediately adjacent pixels and sqrt(2) for diagonally
        adjacent ones.
    pixel_indices : array of int
        An array of shape (Nnz + 1,), mapping indices in `graph` to
        raveled indices in `degree_image` or `skel`.
    degree_image : array of int, same shape as skel
        An image where each pixel value contains the degree of its
        corresponding node in `graph`. This is useful to classify nodes.
    """
    skel = skel.astype(bool)  # ensure we have a bool image
                              # since we later use it for bool indexing
    ndim = skel.ndim
    pixel_indices = np.concatenate(([0], np.flatnonzero(skel)))
    skelint = np.zeros(skel.shape, int)
    skelint.ravel()[pixel_indices] = np.arange(pixel_indices.size + 1)
    skelint = pad(skelint, 0)

    degree_kernel = np.ones((3,) * ndim)
    degree_kernel.ravel()[3**ndim // 2] = 0  # remove centre pix
    degree_image = ndi.convolve(skel.astype(int), degree_kernel,
                                mode='constant')
    num_edges = np.sum(degree_image)  # *2, which is how many we need to store
    row, col = np.zeros(num_edges, dtype=int), np.zeros(num_edges, dtype=int)
    data = np.zeros(num_edges, dtype=float)
    steps, distances = raveled_steps_to_neighbors(skelint.shape, ndim)
    write_pixel_graph(skelint, steps, distances, row, col, data)
    graph = sparse.coo_matrix((data, (row, col))).tocsr()
    return graph, pixel_indices, degree_image

def _expand_path_csr(graph, source, step, visited, degrees):
        d = graph[source, step]
        while degrees[step] == 2 and not visited[step]:
            loc = graph.indptr[source]
            n1, n2 = graph.indices[loc:loc+2]
            nextstep = n1 if n1 != source else n2
            source, step = step, nextstep
            d += graph[source, step]
            visited[source] = True
        return source, d, degrees[step]

def branch_statistics_csr(graph, pixel_indices, degree_image):
    """Compute the length and type of each branch in a skeleton graph.

    Parameters
    ----------
    graph : sparse.csr_matrix, shape (N, N)
        A skeleton graph.
    pixel_indices : array of int, shape (N,)
        A map from rows/cols of `graph` to image coordinates.
    degree_image : array of int, shape (P, Q, ...)
        The image corresponding to the skeleton, where each value is
        its degree in `graph`.

    Returns
    -------
    branches : array of float, shape (N, 4)
        An array containing branch endpoint IDs, length, and branch type.
        The types are:
        - tip-tip (0)
        - tip-junction (1)
        - junction-junction (2)
    """
    degree_image = degree_image.ravel()
    degrees = degree_image[pixel_indices]
    visited = np.zeros(pixel_indices.shape, dtype=bool)
    endpoints = (degrees != 2)
    num_paths = np.sum(degrees[endpoints])
    result = np.zeros((num_paths, 4), dtype=float)
    num_results = 0
    num_cycles = 0
    for node in range(1, graph.shape[0]):
        pixel = pixel_indices[node]
        if degree_image[pixel] == 2 and not visited[node]:
            visited[node] = True
            loc = graph.indptr[node]
            left, right = graph.indices[loc:loc+2]
            id0, d0, deg0 = _expand_path_csr(graph, node, left,
                                             visited, degrees)
            id1, d1, deg1 = _expand_path_csr(graph, node, right,
                                             visited, degrees)
            kind = 2  # default: junction-to-junction
            if deg0 == 1 and deg1 == 1:  # tip-tip
                kind = 0
            elif deg0 == 1 or deg1 == 1:  # tip-junction, tip-path impossible
                kind = 1
            elif deg0 == 2:  # must be a cycle
                num_cycles += 1
                continue
            result[num_results, :] = id0, id1, d0 + d1, kind
            num_results += 1
    return result[:num_results]

## NetworkX-based implementation

def _add_skeleton_edges(values, graph, distances):
    values = values.astype(int)
    center_idx = len(values) // 2
    center = values[center_idx]
    if center == 0:
        return 0
    count = 0.
    for value, distance in zip(values, distances):
        if value != center and value != 0:
            graph.add_edge(center, value, weight=distance)
            count += 1
    node = graph.node[center]
    if count <= 1:
        node['type'] = 'tip'
    elif count == 2:
        node['type'] = 'path'
    else:
        node['type'] = 'junction'
    return count

def _neighbor_distances(ndim):
    center = np.ones((3,) * ndim, dtype=bool)
    center.ravel()[center.size//2] = False
    out = ndi.distance_transform_edt(center).ravel()
    return out

def skeleton_to_nx(skel):  # to do: add pixel spacing
    g = nx.Graph()
    distances = _neighbor_distances(skel.ndim)
    skelint = np.zeros(skel.shape, int)
    num_nodes = np.sum(skel)
    g.add_nodes_from(range(1, num_nodes + 1))
    skelint[skel] = np.arange(1, num_nodes + 1)
    counts = ndi.generic_filter(skelint, function=_add_skeleton_edges,
                                size=3, mode='constant', cval=0,
                                extra_arguments=(g, distances))
    return g, counts, skelint

def branch_statistics(g):
    """Compute the length and type of each branch in a skeleton graph.

    Parameters
    ----------
    g : nx.Graph
        A skeleton graph. Its nodes must be marked as 'tip', 'path', or
        'junction'.

    Returns
    -------
    branches : array of float, shape (N, 4, 2)
        An array containing branch endpoint IDs, length, and branch type.
        The types are:
        - tip-tip (0)
        - tip-junction (1)
        - junction-junction (2)
    """
    visited = np.zeros(max(g) + 1, dtype=bool)
    type_dict = {'tiptip': 0, 'tipjunction': 1, 'junctiontip': 1,
                 'junctionjunction': 2, 'pathpath': 3}
    result = []
    for node, data in g.nodes_iter(data=True):
        if data['type'] == 'path' and not visited[node]:
            # we expand the path in either direction
            visited[node] = True
            left, right = g.neighbors(node)
            id0, d0, kind0 = _expand_path(g, node, left, visited)
            id1, d1, kind1 = _expand_path(g, node, right, visited)
            result.append([id0, id1, d0 + d1, type_dict[kind0 + kind1]])
    return np.array(result)

def _expand_path(g, source, step, visited):
    d = g[source][step]['weight']
    while g.node[step]['type'] == 'path' and not visited[step]:
        n1, n2 = g.neighbors(step)
        nextstep = n1 if n1 != source else n2
        source, step = step, nextstep
        d += g[source][step]['weight']
        visited[source] = True
    return step, d, g.node[step]['type']

def summarise(skelimage):
    ndim = skelimage.ndim
    g, counts, skelimage_labeled = skeleton_to_nx(skelimage)
    coords = np.nonzero(skelimage)
    ids = skelimage_labeled[coords]
    sorted_coords = np.transpose(coords)[np.argsort(ids)]
    tables = []
    for i, cc in enumerate(nx.connected_component_subgraphs(g)):
        stats = branch_statistics(cc)
        if stats.size == 0:
            continue
        coords0 = sorted_coords[stats[:, 0].astype(int) - 1]
        coords1 = sorted_coords[stats[:, 1].astype(int) - 1]
        distances = np.sqrt(np.sum((coords0 - coords1)**2, axis=1))
        skeleton_id = np.full(distances.shape, i, dtype=float)
        tables.append(np.column_stack((skeleton_id, stats,
                                       coords0, coords1, distances)))
    columns = (['skeleton-id', 'node-id-0', 'node-id-1', 'branch-distance',
                'branch-type'] +
               ['coord-0-%i' % i for i in range(ndim)] +
               ['coord-1-%i' % i for i in range(ndim)] +
               ['euclidean-distance'])
    column_types = [int, int, int, float, int] + 2*ndim*[int] + [float]
    arr = np.row_stack(tables).T
    data_dict = {col: dat.astype(dtype)
                 for col, dat, dtype in zip(columns, arr, column_types)}
    df = pd.DataFrame(data_dict)
    return df
<commit_msg>Bug fix: check step neighbors, not source; indent<commit_after>import collections
import itertools
import numpy as np
import pandas as pd
from scipy import sparse, ndimage as ndi
import networkx as nx
import numba

## CSGraph and Numba-based implementation

# adapted from github.com/janelia-flyem/gala
def smallest_int_dtype(number, signed=False, min_dtype=np.int8):
    if number < 0:
        signed = True
    if not signed:
        if number <= np.iinfo(np.uint8).max:
            dtype = np.uint8
        if number <= np.iinfo(np.uint16).max:
            dtype = np.uint16
        if number <= np.iinfo(np.uint32).max:
            dtype = np.uint32
        if number <= np.iinfo(np.uint64).max:
            dtype = np.uint64
    else:
        if np.iinfo(np.int8).min <= number <= np.iinfo(np.int8).max:
            dtype = np.int8
        if np.iinfo(np.int16).min <= number <= np.iinfo(np.int16).max:
            dtype = np.int16
        if np.iinfo(np.int32).min <= number <= np.iinfo(np.int32).max:
            dtype = np.int32
        if np.iinfo(np.int64).min <= number <= np.iinfo(np.int64).max:
            dtype = np.int64
    if np.iinfo(dtype).max < np.iinfo(min_dtype).max:
        dtype = min_dtype
    return dtype

# adapted from github.com/janelia-flyem/gala
def pad(ar, vals, axes=None):
    if ar.size == 0:
        return ar
    if axes is None:
        axes = list(range(ar.ndim))
    if not isinstance(vals, collections.Iterable):
        vals = [vals]
    if not isinstance(axes, collections.Iterable):
        axes = [axes]
    p = len(vals)
    newshape = np.array(ar.shape)
    for ax in axes:
        newshape[ax] += 2*p
    vals = np.reshape(vals, (p,) + (1,) * (ar.ndim-1))
    new_dtype = ar.dtype
    if np.issubdtype(new_dtype, np.integer):
        maxval = max([np.max(vals), np.max(ar)])
        minval = min([np.min(vals), np.min(ar)])
        signed = (minval < 0)
        maxval = max(abs(minval), maxval)
        new_dtype = smallest_int_dtype(maxval, signed=signed,
                                       min_dtype=new_dtype)
    ar2 = np.empty(newshape, dtype=new_dtype)
    center = np.ones(newshape, dtype=bool)
    for ax in axes:
        ar2.swapaxes(0, ax)[p-1::-1,...] = vals
        ar2.swapaxes(0, ax)[-p:,...] = vals
        center.swapaxes(0, ax)[p-1::-1,...] = False
        center.swapaxes(0, ax)[-p:,...] = False
    ar2[center] = ar.ravel()
    return ar2

def raveled_steps_to_neighbors(shape, connectivity=1, order='C',
                               return_distances=True):
    if order == 'C':
        dims = shape[-1:0:-1]
    else:
        dims = shape[:-1]
    stepsizes = np.cumprod((1,) + dims)[::-1]
    steps = [stepsizes, -stepsizes]
    distances = [1] * 2 * stepsizes.size
    for nhops in range(2, connectivity + 1):
        prod = np.array(list(itertools.product(*[[1, -1]] * nhops)))
        multisteps = np.array(list(itertools.combinations(stepsizes, nhops))).T
        steps.append((prod @ multisteps).ravel())
        distances.extend([np.sqrt(nhops)] * steps[-1].size)
    if return_distances:
        return np.concatenate(steps).astype(int), np.array(distances)
    else:
        return np.concatenate(steps).astype(int)

@numba.jit(nopython=True, cache=True, nogil=True)
def write_pixel_graph(image, steps, distances, row, col, data):
    """Step over `image` to build a graph of nonzero pixel neighbors.

    Parameters
    ----------
    image : int array
        The input image.
    steps : int array, shape (N,)
        The raveled index steps to find a pixel's neighbors in `image`.
    distances : float array, shape (N,)
        The euclidean distance from a pixel to its corresponding
        neighbor in `steps`.
    row : int array
        Output array to be filled with the "center" pixel IDs.
    col : int array
        Output array to be filled with the "neighbor" pixel IDs.
    data : float array
        Output array to be filled with the distances from center to
        neighbor pixels.

    Notes
    -----
    No size or bounds checking is performed. Users should ensure that
    - No index in `indices` falls on any edge of `image` (or the
      neighbor computation will fail or segfault).
    - The `steps` and `distances` arrays have the same shape.
    - The `row`, `col`, `data` are long enough to hold all of the
      edges.
    """
    image = image.ravel()
    n_neighbors = steps.size
    start_idx = steps.size
    end_idx = image.size + np.min(steps)
    k = 0
    for i in range(start_idx, end_idx + 1):
        if image[i] != 0:
            for j in range(n_neighbors):
                n = steps[j] + i
                if image[n] != 0:
                    row[k] = image[i]
                    col[k] = image[n]
                    data[k] = distances[j]
                    k += 1

def skeleton_to_csgraph(skel):
    """Convert a skeleton image of thin lines to a graph of neighbor pixels.

    Parameters
    ----------
    skel : array
        An input image in which every nonzero pixel is considered part of
        the skeleton, and links between pixels are determined by a full
        n-dimensional neighborhood.

    Returns
    -------
    graph : sparse.csr_matrix
        A graph of shape (Nnz + 1, Nnz + 1), where Nnz is the number of
        nonzero pixels in `skel`. The value graph[i, j] is the distance
        between adjacent pixels i and j. In a 2D image, that would be
        1 for immediately adjacent pixels and sqrt(2) for diagonally
        adjacent ones.
    pixel_indices : array of int
        An array of shape (Nnz + 1,), mapping indices in `graph` to
        raveled indices in `degree_image` or `skel`.
    degree_image : array of int, same shape as skel
        An image where each pixel value contains the degree of its
        corresponding node in `graph`. This is useful to classify nodes.
    """
    skel = skel.astype(bool)  # ensure we have a bool image
                              # since we later use it for bool indexing
    ndim = skel.ndim
    pixel_indices = np.concatenate(([0], np.flatnonzero(skel)))
    skelint = np.zeros(skel.shape, int)
    skelint.ravel()[pixel_indices] = np.arange(pixel_indices.size + 1)
    skelint = pad(skelint, 0)

    degree_kernel = np.ones((3,) * ndim)
    degree_kernel.ravel()[3**ndim // 2] = 0  # remove centre pix
    degree_image = ndi.convolve(skel.astype(int), degree_kernel,
                                mode='constant')
    num_edges = np.sum(degree_image)  # *2, which is how many we need to store
    row, col = np.zeros(num_edges, dtype=int), np.zeros(num_edges, dtype=int)
    data = np.zeros(num_edges, dtype=float)
    steps, distances = raveled_steps_to_neighbors(skelint.shape, ndim)
    write_pixel_graph(skelint, steps, distances, row, col, data)
    graph = sparse.coo_matrix((data, (row, col))).tocsr()
    return graph, pixel_indices, degree_image

def _expand_path_csr(graph, source, step, visited, degrees):
    d = graph[source, step]
    while degrees[step] == 2 and not visited[step]:
        loc = graph.indptr[step]
        n1, n2 = graph.indices[loc:loc+2]
        nextstep = n1 if n1 != source else n2
        source, step = step, nextstep
        d += graph[source, step]
        visited[source] = True
    return step, d, degrees[step]

def branch_statistics_csr(graph, pixel_indices, degree_image):
    """Compute the length and type of each branch in a skeleton graph.

    Parameters
    ----------
    graph : sparse.csr_matrix, shape (N, N)
        A skeleton graph.
    pixel_indices : array of int, shape (N,)
        A map from rows/cols of `graph` to image coordinates.
    degree_image : array of int, shape (P, Q, ...)
        The image corresponding to the skeleton, where each value is
        its degree in `graph`.

    Returns
    -------
    branches : array of float, shape (N, 4)
        An array containing branch endpoint IDs, length, and branch type.
        The types are:
        - tip-tip (0)
        - tip-junction (1)
        - junction-junction (2)
    """
    degree_image = degree_image.ravel()
    degrees = degree_image[pixel_indices]
    visited = np.zeros(pixel_indices.shape, dtype=bool)
    endpoints = (degrees != 2)
    num_paths = np.sum(degrees[endpoints])
    result = np.zeros((num_paths, 4), dtype=float)
    num_results = 0
    num_cycles = 0
    for node in range(1, graph.shape[0]):
        pixel = pixel_indices[node]
        if degree_image[pixel] == 2 and not visited[node]:
            visited[node] = True
            loc = graph.indptr[node]
            left, right = graph.indices[loc:loc+2]
            id0, d0, deg0 = _expand_path_csr(graph, node, left,
                                             visited, degrees)
            id1, d1, deg1 = _expand_path_csr(graph, node, right,
                                             visited, degrees)
            kind = 2  # default: junction-to-junction
            if deg0 == 1 and deg1 == 1:  # tip-tip
                kind = 0
            elif deg0 == 1 or deg1 == 1:  # tip-junction, tip-path impossible
                kind = 1
            elif deg0 == 2:  # must be a cycle
                num_cycles += 1
                continue
            result[num_results, :] = id0, id1, d0 + d1, kind
            num_results += 1
    return result[:num_results]

## NetworkX-based implementation

def _add_skeleton_edges(values, graph, distances):
    values = values.astype(int)
    center_idx = len(values) // 2
    center = values[center_idx]
    if center == 0:
        return 0
    count = 0.
    for value, distance in zip(values, distances):
        if value != center and value != 0:
            graph.add_edge(center, value, weight=distance)
            count += 1
    node = graph.node[center]
    if count <= 1:
        node['type'] = 'tip'
    elif count == 2:
        node['type'] = 'path'
    else:
        node['type'] = 'junction'
    return count

def _neighbor_distances(ndim):
    center = np.ones((3,) * ndim, dtype=bool)
    center.ravel()[center.size//2] = False
    out = ndi.distance_transform_edt(center).ravel()
    return out

def skeleton_to_nx(skel):  # to do: add pixel spacing
    g = nx.Graph()
    distances = _neighbor_distances(skel.ndim)
    skelint = np.zeros(skel.shape, int)
    num_nodes = np.sum(skel)
    g.add_nodes_from(range(1, num_nodes + 1))
    skelint[skel] = np.arange(1, num_nodes + 1)
    counts = ndi.generic_filter(skelint, function=_add_skeleton_edges,
                                size=3, mode='constant', cval=0,
                                extra_arguments=(g, distances))
    return g, counts, skelint

def branch_statistics(g):
    """Compute the length and type of each branch in a skeleton graph.

    Parameters
    ----------
    g : nx.Graph
        A skeleton graph. Its nodes must be marked as 'tip', 'path', or
        'junction'.

    Returns
    -------
    branches : array of float, shape (N, 4, 2)
        An array containing branch endpoint IDs, length, and branch type.
        The types are:
        - tip-tip (0)
        - tip-junction (1)
        - junction-junction (2)
    """
    visited = np.zeros(max(g) + 1, dtype=bool)
    type_dict = {'tiptip': 0, 'tipjunction': 1, 'junctiontip': 1,
                 'junctionjunction': 2, 'pathpath': 3}
    result = []
    for node, data in g.nodes_iter(data=True):
        if data['type'] == 'path' and not visited[node]:
            # we expand the path in either direction
            visited[node] = True
            left, right = g.neighbors(node)
            id0, d0, kind0 = _expand_path(g, node, left, visited)
            id1, d1, kind1 = _expand_path(g, node, right, visited)
            result.append([id0, id1, d0 + d1, type_dict[kind0 + kind1]])
    return np.array(result)

def _expand_path(g, source, step, visited):
    d = g[source][step]['weight']
    while g.node[step]['type'] == 'path' and not visited[step]:
        n1, n2 = g.neighbors(step)
        nextstep = n1 if n1 != source else n2
        source, step = step, nextstep
        d += g[source][step]['weight']
        visited[source] = True
    return step, d, g.node[step]['type']

def summarise(skelimage):
    ndim = skelimage.ndim
    g, counts, skelimage_labeled = skeleton_to_nx(skelimage)
    coords = np.nonzero(skelimage)
    ids = skelimage_labeled[coords]
    sorted_coords = np.transpose(coords)[np.argsort(ids)]
    tables = []
    for i, cc in enumerate(nx.connected_component_subgraphs(g)):
        stats = branch_statistics(cc)
        if stats.size == 0:
            continue
        coords0 = sorted_coords[stats[:, 0].astype(int) - 1]
        coords1 = sorted_coords[stats[:, 1].astype(int) - 1]
        distances = np.sqrt(np.sum((coords0 - coords1)**2, axis=1))
        skeleton_id = np.full(distances.shape, i, dtype=float)
        tables.append(np.column_stack((skeleton_id, stats,
                                       coords0, coords1, distances)))
    columns = (['skeleton-id', 'node-id-0', 'node-id-1', 'branch-distance',
                'branch-type'] +
               ['coord-0-%i' % i for i in range(ndim)] +
               ['coord-1-%i' % i for i in range(ndim)] +
               ['euclidean-distance'])
    column_types = [int, int, int, float, int] + 2*ndim*[int] + [float]
    arr = np.row_stack(tables).T
    data_dict = {col: dat.astype(dtype)
                 for col, dat, dtype in zip(columns, arr, column_types)}
    df = pd.DataFrame(data_dict)
    return df
<|endoftext|>


<commit_before># ######################################################################
# Copyright (c) 2014, Brookhaven Science Associates, Brookhaven        #
# National Laboratory. All rights reserved.                            #
#                                                                      #
# Developed at the NSLS-II, Brookhaven National Laboratory             #
# Developed by Sameera K. Abeykoon, April 2014                         #
#                                                                      #
# Redistribution and use in source and binary forms, with or without   #
# modification, are permitted provided that the following conditions   #
# are met:                                                             #
#                                                                      #
# * Redistributions of source code must retain the above copyright     #
#   notice, this list of conditions and the following disclaimer.      #
#                                                                      #
# * Redistributions in binary form must reproduce the above copyright  #
#   notice this list of conditions and the following disclaimer in     #
#   the documentation and/or other materials provided with the         #
#   distribution.                                                      #
#                                                                      #
# * Neither the name of the Brookhaven Science Associates, Brookhaven  #
#   National Laboratory nor the names of its contributors may be used  #
#   to endorse or promote products derived from this software without  #
#   specific prior written permission.                                 #
#                                                                      #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS  #
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT    #
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS    #
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE       #
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,           #
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES   #
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR   #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)   #
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,  #
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OTHERWISE) ARISING   #
# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE   #
# POSSIBILITY OF SUCH DAMAGE.                                          #
########################################################################

from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import six
import logging
from collections import deque
import numpy as np

import matplotlib.pyplot as plt

"""
    Plotting tools for X-Ray Speckle Visibility Spectroscopy
"""

def mean_intensity_plotter(ax, mean_intensity_sets, index_list,
                           title="Mean intensities",
                           xlabel="Frames", ylabel="Mean Intensity"):
    """
    This will plot mean intensities for ROIS' of the labeled array
    for different image sets.

    Parameters
    ----------
    ax : Axes

    num_rois : int

    mean_intensity_sets : list

    index_list : list

    title : str, optional
        title of the plot

    x_label : str, optional
        x axis label

    y_label : str, optional
        y axis label

    """
    if np.all(map(lambda x: x == index_list[0], index_list)):
        ax[len(index_list[0])-1].set_xlabel(xlabel)
        for i in range(len(index_list[0])):
            ax[i].set_ylabel(ylabel)
            ax[i].set_title(title+" for ROI " + str(i+1))
            total = 0
            first = 0
            x = [len(elm) for elm in mean_intensity_sets]
            x_val=np.arange(sum(x))
            for j in range(len(mean_intensity_sets)):
                total += x[j]
                ax[i].plot(x_val[first:total], mean_intensity_sets[j][:, i],
                         label=str(j+1)+" image_set")
                first = total
        plt.legend()
    else:
        raise ValueError("Labels list for the image sets are different")

def combine_intensity_plotter(ax, combine_intensity, title="Mean intensities",
                           xlabel="Frames", ylabel="Mean Intensity"):
    """
    This will plot the combine intensities for all image sets

    Parameters
    ----------
    ax : Axes

    num_rois : int

    mean_intensity_sets : list

    index_list : list

    title : str, optional
        title of the plot

    x_label : str, optional
        x axis label

    y_label : str, optional
        y axis label
    """
    for i in range(combin_intensity.shape[1]):
        ax.set_ylabel(ylabel)
        ax_set_xlabel(xlabel)
        ax.set_title(title)
        ax.plot(combine_intensity[:, i], label=str(j)+" ROI")
    plt.legend()

def cirular_average_plotter(ax1, ax2, image_data, ring_averages, bin_centers,
                            i_title="Image Data", c_title="Circular Average",
                            cmap="gray", vmin=None, vmax=None, marker='o',
                            line_color='blue', xlabel="Bin Centers",
                            ylabel="Ring Average", left=0.1, width = 0.65,
                            bottom= 0.1, height=0.65):
    plot_axes = [left, bottom, width, height]

    ax1.imshow(imge_data, cmap=cmap, vmin=vmin, vmax=vmax)
    ax1.set_title(i_title)

    ax2.semilogy(bin_centers, ring_averages, c=line_color, marker=maker)
    ax2.set_title(c_title)
    ax2.set_xlabel(xlabel)
    ax2.set_ylabel(ylabel)
<commit_msg>ENH: added more plotting tools for XSVS analysis, image, circular_average, roi_pixel_plotter<commit_after># ######################################################################
# Copyright (c) 2014, Brookhaven Science Associates, Brookhaven        #
# National Laboratory. All rights reserved.                            #
#                                                                      #
# Developed at the NSLS-II, Brookhaven National Laboratory             #
# Developed by Sameera K. Abeykoon, April 2014                         #
#                                                                      #
# Redistribution and use in source and binary forms, with or without   #
# modification, are permitted provided that the following conditions   #
# are met:                                                             #
#                                                                      #
# * Redistributions of source code must retain the above copyright     #
#   notice, this list of conditions and the following disclaimer.      #
#                                                                      #
# * Redistributions in binary form must reproduce the above copyright  #
#   notice this list of conditions and the following disclaimer in     #
#   the documentation and/or other materials provided with the         #
#   distribution.                                                      #
#                                                                      #
# * Neither the name of the Brookhaven Science Associates, Brookhaven  #
#   National Laboratory nor the names of its contributors may be used  #
#   to endorse or promote products derived from this software without  #
#   specific prior written permission.                                 #
#                                                                      #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS  #
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT    #
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS    #
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE       #
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,           #
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES   #
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR   #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)   #
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,  #
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OTHERWISE) ARISING   #
# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE   #
# POSSIBILITY OF SUCH DAMAGE.                                          #
########################################################################

from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import six
import logging
from collections import deque
import numpy as np

import matplotlib.pyplot as plt

"""
    Plotting tools for X-Ray Speckle Visibility Spectroscopy(XSVS)
"""

def mean_intensity_plotter(ax, mean_intensity_sets, index_list,
                           title="Mean intensities",
                           xlabel="Frames", ylabel="Mean Intensity"):
    """
    This will plot mean intensities for ROIS' of the labeled array
    for different image sets.

    Parameters
    ----------
     ax : Axes
        The `Axes` object to add the artist tool

    mean_intensity_sets : list
        Average intensities of each ROI as a list
        shape len(images_sets)

    index_list : list
        labels list for each image set

    title : str, optional
        title of the plot

    x_label : str, optional
        x axis label

    y_label : str, optional
        y axis label

    """
    if np.all(map(lambda x: x == index_list[0], index_list)):
        ax[len(index_list[0])-1].set_xlabel(xlabel)
        for i in range(len(index_list[0])):
            ax[i].set_ylabel(ylabel)
            ax[i].set_title(title+" for ROI " + str(i+1))
            total = 0
            first = 0
            x = [len(elm) for elm in mean_intensity_sets]
            x_val = np.arange(sum(x))
            for j in range(len(mean_intensity_sets)):
                total += x[j]
                ax[i].plot(x_val[first:total], mean_intensity_sets[j][:, i],
                           label=str(j+1)+" image_set")
                first = total
        plt.legend()
    else:
        raise ValueError("Labels list for the image sets are different")

def combine_intensity_plotter(ax, combine_intensity,
                              title="Mean Intensities - All Image Sets",
                              xlabel="Frames", ylabel="Mean Intensity"):
    """
    This will plot the combine intensities for all image sets

    Parameters
    ----------
    ax : Axes
        The `Axes` object to add the artist tool

    combine_intensity : array
        combine intensities for each ROI from image data sets

    title : str, optional
        title of the plot

    x_label : str, optional
        x axis label

    y_label : str, optional
        y axis label
    """
    num_rois = combine_intensity.shape[1]
    for i in range(num_rois):
        ax.set_ylabel(ylabel)
        ax_set_xlabel(xlabel)
        ax.set_title(title)
        ax.plot(combine_intensity[:, i], label=str(j)+" ROI")
    plt.legend()

def circular_average_plotter(ax1, ax2, image_data, ring_averages, bin_centers,
                             i_title="Image Data", c_title="Circular Average",
                             cmap="gray", vmin=None, vmax=None, marker='o',
                             line_color='blue', xlabel="Bin Centers",
                             ylabel="Ring Average"):
    """
    Parameters
    ----------
    ax1 : Axes
        The `Axes` object to add the artist tool

    ax2 : Axes
        The `Axes` object to add the artist tool

    image_data : array

    ring_averages : array

    bin_centers: array

    i_title : str, optional
        title for the image data

    c_title : str, optional
        title for the circular avergae of image data

    cmap : str, optional
        colormap for image data

    vmin : float, optional
        arguments specify the color limits

    vmax : float, optional
        arguments specify the color limits

    marker : str, optional
        line marker

    line_color : str, optional
        line color

    x_label : str, optional
        x axis label circular average plot

    y_label : str, optional
        y axis label circular average plot
    """
    ax1.imshow(imge_data, cmap=cmap, vmin=vmin, vmax=vmax)
    ax1.set_title(i_title)

    ax2.semilogy(bin_centers, ring_averages, c=line_color, marker=maker)
    ax2.set_title(c_title)
    ax2.set_xlabel(xlabel)
    ax2.set_ylabel(ylabel)

def roi_kymograph_plotter(ax, kymograph_data, title="ROI Kymograph",
                          fig_size=(8, 10), xlabel="pixel list",
                          ylabel="Frames", cmap='gist_earth'):
    """
    This will plot graphical representation of pixels variation over time
    for required ROI.

    Parameters
    ----------
    ax : Axes
        The `Axes` object to add the artist tool

    kymograph_data : array
        data for graphical representation of pixels variation over time
        for required ROI

    title : str, optional
        title of the plot

    fig_size : tuple, optional
        figure size

    x_label : str, optional
        x axis label of the plot

    y_label : str, optional
        y axis label

    cmap : str, optional
        colormap for the data

    """
    ax.set_xlabel(xlabel)
    ax.set_ylabel(ylabel)
    ax.set_title(title)
    ax.imshow(kymograph_data, cmap=cmap)

def roi_pixel_plotter(axes, roi_pixel_data, title='', xlabel='pixel list',
                      ylabel='Intensity'):
    """
    ROI pixel plotter
    Parameters
    ----------
    ax :

    mean_intensity : array

    title : str, optional

    fig_size : tuple, optional

    x_label : str, optional

    y_label : str, optional

    """
    num_rois = roi_pixel_data.shape[1]
    for i in range(num_rois):
        axes[i].plot(roi_pixel_data[:, i])
        axes[i].set_xlabel(xlabel)
        axes[i].set_ylabel(ylabel)
        axes[i].set_title(title)
    plt.legend()
<|endoftext|>


<commit_before># The resources we'll be editing
# NOTE: These should ordinarily be amended to reflect the project...
resources = {}
resources["stress"] = 0.1
resources["building"] = True
resources["money"] = 10000
resources["tests passing"] = False
resources["integration tests passing"] = False
resources["user acceptance tests passing"] = False
resources["successful deployment"] = False
<commit_msg>Fixed broken resources<commit_after># The resources we'll be editing
# NOTE: These should ordinarily be amended to reflect the initial state of the project...
resources = {}
resources["stress"] = 0.1
resources["building"] = True
resources["money"] = 10000
resources["tests passing"] = False
resources["integration tests passing"] = False
resources["user acceptance tests passing"] = False
resources["successful deployment"] = False
resources["tickets"] = 0
resources["branches"] = 1<|endoftext|>


<commit_before>from neomodel.properties import (IntegerProperty, DateTimeProperty,
    DateProperty, StringProperty, JSONProperty)
from neomodel.exception import InflateError, DeflateError
from neomodel import StructuredNode
from pytz import timezone
from datetime import datetime, date

class FooBar(object):
    pass

def test_string_property_w_choice():
    class TestChoices(StructuredNode):
        SEXES = (('M', 'Male'), ('F', 'Female'))
        sex = StringProperty(required=True, choices=SEXES)

    try:
        TestChoices(sex='Z').save()
    except DeflateError as e:
        assert True
        assert str(e).index('choice')
    else:
        assert False

    node = TestChoices(sex='M').save()
    assert node.get_sex_display() == 'Male'

def test_deflate_inflate():
    prop = IntegerProperty(required=True)
    prop.name = 'age'
    prop.owner = FooBar

    try:
        prop.inflate("six")
    except InflateError as e:
        assert True
        assert str(e).index('inflate property')
    else:
        assert False

    try:
        prop.deflate("six")
    except DeflateError as e:
        assert True
        assert str(e).index('deflate property')
    else:
        assert False

def test_datetimes_timezones():
    prop = DateTimeProperty()
    prop.name = 'foo'
    prop.owner = FooBar
    t = datetime.utcnow()
    gr = timezone('Europe/Athens')
    gb = timezone('Europe/London')
    dt1 = gr.localize(t)
    dt2 = gb.localize(t)
    time1 = prop.inflate(prop.deflate(dt1))
    time2 = prop.inflate(prop.deflate(dt2))
    assert time1.utctimetuple() == dt1.utctimetuple()
    assert time1.utctimetuple() < time2.utctimetuple()
    assert time1.tzname() == 'UTC'

def test_date():
    prop = DateProperty()
    prop.name = 'foo'
    prop.owner = FooBar
    somedate = date(2012, 12, 15)
    assert prop.deflate(somedate) == '2012-12-15'
    assert prop.inflate('2012-12-15') == somedate

def test_datetime_exceptions():
    prop = DateTimeProperty()
    prop.name = 'created'
    prop.owner = FooBar
    faulty = 'dgdsg'

    try:
        prop.inflate(faulty)
    except InflateError as e:
        assert True
        assert str(e).index('inflate property')
    else:
        assert False

    try:
        prop.deflate(faulty)
    except DeflateError as e:
        assert True
        assert str(e).index('deflate property')
    else:
        assert False

def test_date_exceptions():
    prop = DateProperty()
    prop.name = 'date'
    prop.owner = FooBar
    faulty = '2012-14-13'

    try:
        prop.inflate(faulty)
    except InflateError as e:
        assert True
        assert str(e).index('inflate property')
    else:
        assert False

    try:
        prop.deflate(faulty)
    except DeflateError as e:
        assert True
        assert str(e).index('deflate property')
    else:
        assert False

def test_json():
    prop = JSONProperty()
    prop.name = 'json'
    prop.owner = FooBar

    value = {'test': [1, 2, 3]}

    assert prop.deflate(value) == '{"test": [1, 2, 3]}'
    assert prop.inflate('{"test": [1, 2, 3]}') == value

def test_default_value():
    class DefaultTestValue(StructuredNode):
        name_xx = StringProperty(default='jim', index=True)

    a = DefaultTestValue()
    assert a.name_xx == 'jim'
    a.save()
    return
    b = DefaultTestValue.index.get(name='jim')
    assert b.name == 'jim'

    c = DefaultTestValue(name=None)
    assert c.name == 'jim'

def test_default_value_callable():
    def uid_generator():
        return 'xx'

    class DefaultTestValueTwo(StructuredNode):
        uid = StringProperty(default=uid_generator, index=True)

    a = DefaultTestValueTwo().save()
    assert a.uid == 'xx'

def test_default_valude_callable_type():
    # check our object gets converted to str without serializing and reload
    def factory():
        class Foo(object):
            def __str__(self):
                return "123"
        return Foo()

    class DefaultTestValueThree(StructuredNode):
        uid = StringProperty(default=factory, index=True)

    x = DefaultTestValueThree()
    assert x.uid == '123'
    x.save()
    assert x.uid == '123'
    x.refresh()
    assert x.uid == '123'
<commit_msg>Test case for independent property name in database<commit_after>from neomodel.properties import (IntegerProperty, DateTimeProperty,
    DateProperty, StringProperty, JSONProperty)
from neomodel.exception import InflateError, DeflateError
from neomodel import StructuredNode, db
from pytz import timezone
from datetime import datetime, date

class FooBar(object):
    pass

def test_string_property_w_choice():
    class TestChoices(StructuredNode):
        SEXES = (('M', 'Male'), ('F', 'Female'))
        sex = StringProperty(required=True, choices=SEXES)

    try:
        TestChoices(sex='Z').save()
    except DeflateError as e:
        assert True
        assert str(e).index('choice')
    else:
        assert False

    node = TestChoices(sex='M').save()
    assert node.get_sex_display() == 'Male'

def test_deflate_inflate():
    prop = IntegerProperty(required=True)
    prop.name = 'age'
    prop.owner = FooBar

    try:
        prop.inflate("six")
    except InflateError as e:
        assert True
        assert str(e).index('inflate property')
    else:
        assert False

    try:
        prop.deflate("six")
    except DeflateError as e:
        assert True
        assert str(e).index('deflate property')
    else:
        assert False

def test_datetimes_timezones():
    prop = DateTimeProperty()
    prop.name = 'foo'
    prop.owner = FooBar
    t = datetime.utcnow()
    gr = timezone('Europe/Athens')
    gb = timezone('Europe/London')
    dt1 = gr.localize(t)
    dt2 = gb.localize(t)
    time1 = prop.inflate(prop.deflate(dt1))
    time2 = prop.inflate(prop.deflate(dt2))
    assert time1.utctimetuple() == dt1.utctimetuple()
    assert time1.utctimetuple() < time2.utctimetuple()
    assert time1.tzname() == 'UTC'

def test_date():
    prop = DateProperty()
    prop.name = 'foo'
    prop.owner = FooBar
    somedate = date(2012, 12, 15)
    assert prop.deflate(somedate) == '2012-12-15'
    assert prop.inflate('2012-12-15') == somedate

def test_datetime_exceptions():
    prop = DateTimeProperty()
    prop.name = 'created'
    prop.owner = FooBar
    faulty = 'dgdsg'

    try:
        prop.inflate(faulty)
    except InflateError as e:
        assert True
        assert str(e).index('inflate property')
    else:
        assert False

    try:
        prop.deflate(faulty)
    except DeflateError as e:
        assert True
        assert str(e).index('deflate property')
    else:
        assert False

def test_date_exceptions():
    prop = DateProperty()
    prop.name = 'date'
    prop.owner = FooBar
    faulty = '2012-14-13'

    try:
        prop.inflate(faulty)
    except InflateError as e:
        assert True
        assert str(e).index('inflate property')
    else:
        assert False

    try:
        prop.deflate(faulty)
    except DeflateError as e:
        assert True
        assert str(e).index('deflate property')
    else:
        assert False

def test_json():
    prop = JSONProperty()
    prop.name = 'json'
    prop.owner = FooBar

    value = {'test': [1, 2, 3]}

    assert prop.deflate(value) == '{"test": [1, 2, 3]}'
    assert prop.inflate('{"test": [1, 2, 3]}') == value

def test_default_value():
    class DefaultTestValue(StructuredNode):
        name_xx = StringProperty(default='jim', index=True)

    a = DefaultTestValue()
    assert a.name_xx == 'jim'
    a.save()
    return
    b = DefaultTestValue.index.get(name='jim')
    assert b.name == 'jim'

    c = DefaultTestValue(name=None)
    assert c.name == 'jim'

def test_default_value_callable():
    def uid_generator():
        return 'xx'

    class DefaultTestValueTwo(StructuredNode):
        uid = StringProperty(default=uid_generator, index=True)

    a = DefaultTestValueTwo().save()
    assert a.uid == 'xx'

def test_default_valude_callable_type():
    # check our object gets converted to str without serializing and reload
    def factory():
        class Foo(object):
            def __str__(self):
                return "123"
        return Foo()

    class DefaultTestValueThree(StructuredNode):
        uid = StringProperty(default=factory, index=True)

    x = DefaultTestValueThree()
    assert x.uid == '123'
    x.save()
    assert x.uid == '123'
    x.refresh()
    assert x.uid == '123'

def test_independent_property_name():
    class TestNode(StructuredNode):
        name_ = StringProperty(db_property="name")
    x = TestNode()
    x.name_ = "jim"
    x.save()

    # check database property name on low level
    results, meta = db.cypher_query("MATCH (n:TestNode) RETURN n")
    assert results[0][0].properties['name'] == "jim"

    assert not results[0][0].properties.has_key('name_')
    assert not hasattr(x, 'name')
    assert hasattr(x, 'name_')
    assert TestNode.nodes.filter(name_="jim").all()[0].name_ == x.name_
    assert TestNode.nodes.get(name_="jim").name_ == x.name_

    # delete node afterwards
    x.delete()
<|endoftext|>


<commit_before>from functools import lru_cache

import sublime
import sublime_plugin

__all__ = (
    "ProjectFileChanges",
)

MYPY = False
if MYPY:
    from typing import Callable, Dict, Optional, TypeVar
    T = TypeVar("T")

class GitSavvySettings:
    def __init__(self, window=None):
        # type: (sublime.Window) -> None
        self._window = window or sublime.active_window()
        self._global_settings = get_global_settings()

    def get(self, key, default=None):
        try:
            return get_project_settings(self._window)[key]
        except KeyError:
            return self._global_settings.get(key, default)

    def set(self, key, value):
        self._global_settings.set(key, value)

CHANGE_COUNT = 0

class ProjectFileChanges(sublime_plugin.EventListener):
    def on_post_save(self, view):
        # type: (sublime.View) -> None
        global CHANGE_COUNT
        file_path = view.file_name()
        if file_path and file_path.endswith(".sublime-project"):
            CHANGE_COUNT += 1

def get_project_settings(window):
    # type: (sublime.Window) -> Dict
    global CHANGE_COUNT
    return _get_project_settings(window.id(), CHANGE_COUNT)

@lru_cache(maxsize=16)
def _get_project_settings(wid, _counter):
    # type: (sublime.WindowId, int) -> Dict
    window = sublime.Window(wid)
    project_data = window.project_data()
    if not project_data:
        return {}
    return project_data.get("settings", {}).get("GitSavvy", {})

@lru_cache(maxsize=1)
def get_global_settings():
    return GlobalSettings("GitSavvy.sublime-settings")

class GlobalSettings:
    def __init__(self, name):
        self._settings = s = sublime.load_settings(name)
        s.clear_on_change(name)
        s.add_on_change(name, self._on_update)
        self._cache = {}

    def get(self, name, default=None):
        try:
            return self._cache[name]
        except KeyError:
            self._cache[name] = current_value = self._settings.get(name, default)
            return current_value

    def set(self, name, value):
        self._settings.set(name, value)

    def _on_update(self):
        self._cache.clear()

class SettingsMixin:
    _savvy_settings = None

    @property
    def savvy_settings(self):
        if not self._savvy_settings:
            window = (
                maybe(lambda: self.window)  # type: ignore[attr-defined]
                or maybe(lambda: self.view.window())  # type: ignore[attr-defined]
                or sublime.active_window()
            )
            self._savvy_settings = GitSavvySettings(window)
        return self._savvy_settings

def maybe(fn):
    # type: (Callable[[], T]) -> Optional[T]
    try:
        return fn()
    except Exception:
        return None
<commit_msg>Extract `some_window()` method<commit_after>from functools import lru_cache

import sublime
import sublime_plugin

__all__ = (
    "ProjectFileChanges",
)

MYPY = False
if MYPY:
    from typing import Callable, Dict, Optional, TypeVar
    T = TypeVar("T")

class GitSavvySettings:
    def __init__(self, window=None):
        # type: (sublime.Window) -> None
        self._window = window or sublime.active_window()
        self._global_settings = get_global_settings()

    def get(self, key, default=None):
        try:
            return get_project_settings(self._window)[key]
        except KeyError:
            return self._global_settings.get(key, default)

    def set(self, key, value):
        self._global_settings.set(key, value)

CHANGE_COUNT = 0

class ProjectFileChanges(sublime_plugin.EventListener):
    def on_post_save(self, view):
        # type: (sublime.View) -> None
        global CHANGE_COUNT
        file_path = view.file_name()
        if file_path and file_path.endswith(".sublime-project"):
            CHANGE_COUNT += 1

def get_project_settings(window):
    # type: (sublime.Window) -> Dict
    global CHANGE_COUNT
    return _get_project_settings(window.id(), CHANGE_COUNT)

@lru_cache(maxsize=16)
def _get_project_settings(wid, _counter):
    # type: (sublime.WindowId, int) -> Dict
    window = sublime.Window(wid)
    project_data = window.project_data()
    if not project_data:
        return {}
    return project_data.get("settings", {}).get("GitSavvy", {})

@lru_cache(maxsize=1)
def get_global_settings():
    return GlobalSettings("GitSavvy.sublime-settings")

class GlobalSettings:
    def __init__(self, name):
        self._settings = s = sublime.load_settings(name)
        s.clear_on_change(name)
        s.add_on_change(name, self._on_update)
        self._cache = {}

    def get(self, name, default=None):
        try:
            return self._cache[name]
        except KeyError:
            self._cache[name] = current_value = self._settings.get(name, default)
            return current_value

    def set(self, name, value):
        self._settings.set(name, value)

    def _on_update(self):
        self._cache.clear()

class SettingsMixin:
    _savvy_settings = None

    @property
    def savvy_settings(self):
        if not self._savvy_settings:
            window = self.some_window()
            self._savvy_settings = GitSavvySettings(window)
        return self._savvy_settings

    def some_window(self):
        # type: () -> sublime.Window
        return (
            maybe(lambda: self.window)  # type: ignore[attr-defined]
            or maybe(lambda: self.view.window())  # type: ignore[attr-defined]
            or sublime.active_window()
        )

def maybe(fn):
    # type: (Callable[[], T]) -> Optional[T]
    try:
        return fn()
    except Exception:
        return None
<|endoftext|>