<commit_before># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import copy
import bson
import datetime

from st2tests.fixtures import history as fixture
from st2tests import DbTestCase
from st2common.util import isotime
from st2common.persistence.history import ActionExecutionHistory
from st2common.models.api.history import ActionExecutionHistoryAPI

class TestActionExecutionHistoryModel(DbTestCase):

    def setUp(self):
        super(TestActionExecutionHistoryModel, self).setUp()

        # Fake history record for action executions triggered by workflow runner.
        self.fake_history_subtasks = [
            {
                'id': str(bson.ObjectId()),
                'action': copy.deepcopy(fixture.ARTIFACTS['actions']['local']),
                'runner': copy.deepcopy(fixture.ARTIFACTS['runners']['run-local']),
                'execution': copy.deepcopy(fixture.ARTIFACTS['executions']['task1']),
            },
            {
                'id': str(bson.ObjectId()),
                'action': copy.deepcopy(fixture.ARTIFACTS['actions']['local']),
                'runner': copy.deepcopy(fixture.ARTIFACTS['runners']['run-local']),
                'execution': copy.deepcopy(fixture.ARTIFACTS['executions']['task2']),
            }
        ]

        # Fake history record for a workflow action execution triggered by rule.
        self.fake_history_workflow = {
            'id': str(bson.ObjectId()),
            'trigger': copy.deepcopy(fixture.ARTIFACTS['trigger']),
            'trigger_type': copy.deepcopy(fixture.ARTIFACTS['trigger_type']),
            'trigger_instance': copy.deepcopy(fixture.ARTIFACTS['trigger_instance']),
            'rule': copy.deepcopy(fixture.ARTIFACTS['rule']),
            'action': copy.deepcopy(fixture.ARTIFACTS['actions']['chain']),
            'runner': copy.deepcopy(fixture.ARTIFACTS['runners']['action-chain']),
            'execution': copy.deepcopy(fixture.ARTIFACTS['executions']['workflow']),
            'children': [task['id'] for task in self.fake_history_subtasks]
        }

        # Assign parent to the history records for the subtasks.
        for task in self.fake_history_subtasks:
            task['parent'] = self.fake_history_workflow['id']

    def test_model_complete(self):

        # Create API object.
        obj = ActionExecutionHistoryAPI(**copy.deepcopy(self.fake_history_workflow))
        self.assertDictEqual(obj.trigger, self.fake_history_workflow['trigger'])
        self.assertDictEqual(obj.trigger_type, self.fake_history_workflow['trigger_type'])
        self.assertDictEqual(obj.trigger_instance, self.fake_history_workflow['trigger_instance'])
        self.assertDictEqual(obj.rule, self.fake_history_workflow['rule'])
        self.assertDictEqual(obj.action, self.fake_history_workflow['action'])
        self.assertDictEqual(obj.runner, self.fake_history_workflow['runner'])
        self.assertDictEqual(obj.execution, self.fake_history_workflow['execution'])
        self.assertIsNone(getattr(obj, 'parent', None))
        self.assertListEqual(obj.children, self.fake_history_workflow['children'])

        # Convert API object to DB model.
        model = ActionExecutionHistoryAPI.to_model(obj)
        self.assertEqual(str(model.id), obj.id)
        self.assertDictEqual(model.trigger, self.fake_history_workflow['trigger'])
        self.assertDictEqual(model.trigger_type, self.fake_history_workflow['trigger_type'])
        self.assertDictEqual(model.trigger_instance, self.fake_history_workflow['trigger_instance'])
        self.assertDictEqual(model.rule, self.fake_history_workflow['rule'])
        self.assertDictEqual(model.action, self.fake_history_workflow['action'])
        self.assertDictEqual(model.runner, self.fake_history_workflow['runner'])
        doc = copy.deepcopy(self.fake_history_workflow['execution'])
        doc['start_timestamp'] = isotime.parse(doc['start_timestamp'])
        self.assertDictEqual(model.execution, doc)
        self.assertIsNone(getattr(model, 'parent', None))
        self.assertListEqual(model.children, self.fake_history_workflow['children'])

        # Convert DB model to API object.
        obj = ActionExecutionHistoryAPI.from_model(model)
        self.assertEqual(str(model.id), obj.id)
        self.assertDictEqual(obj.trigger, self.fake_history_workflow['trigger'])
        self.assertDictEqual(obj.trigger_type, self.fake_history_workflow['trigger_type'])
        self.assertDictEqual(obj.trigger_instance, self.fake_history_workflow['trigger_instance'])
        self.assertDictEqual(obj.rule, self.fake_history_workflow['rule'])
        self.assertDictEqual(obj.action, self.fake_history_workflow['action'])
        self.assertDictEqual(obj.runner, self.fake_history_workflow['runner'])
        self.assertDictEqual(obj.execution, self.fake_history_workflow['execution'])
        self.assertIsNone(getattr(obj, 'parent', None))
        self.assertListEqual(obj.children, self.fake_history_workflow['children'])

    def test_crud_complete(self):
        # Create the DB record.
        obj = ActionExecutionHistoryAPI(**copy.deepcopy(self.fake_history_workflow))
        ActionExecutionHistory.add_or_update(ActionExecutionHistoryAPI.to_model(obj))
        model = ActionExecutionHistory.get_by_id(obj.id)
        self.assertEqual(str(model.id), obj.id)
        self.assertDictEqual(model.trigger, self.fake_history_workflow['trigger'])
        self.assertDictEqual(model.trigger_type, self.fake_history_workflow['trigger_type'])
        self.assertDictEqual(model.trigger_instance, self.fake_history_workflow['trigger_instance'])
        self.assertDictEqual(model.rule, self.fake_history_workflow['rule'])
        self.assertDictEqual(model.action, self.fake_history_workflow['action'])
        self.assertDictEqual(model.runner, self.fake_history_workflow['runner'])
        doc = copy.deepcopy(self.fake_history_workflow['execution'])
        doc['start_timestamp'] = isotime.parse(doc['start_timestamp'])
        self.assertDictEqual(model.execution, doc)
        self.assertIsNone(getattr(model, 'parent', None))
        self.assertListEqual(model.children, self.fake_history_workflow['children'])

        # Update the DB record.
        children = [str(bson.ObjectId()), str(bson.ObjectId())]
        model.children = children
        ActionExecutionHistory.add_or_update(model)
        model = ActionExecutionHistory.get_by_id(obj.id)
        self.assertListEqual(model.children, children)

        # Delete the DB record.
        ActionExecutionHistory.delete(model)
        self.assertRaises(ValueError, ActionExecutionHistory.get_by_id, obj.id)

    def test_model_partial(self):
        # Create API object.
        obj = ActionExecutionHistoryAPI(**copy.deepcopy(self.fake_history_subtasks[0]))
        self.assertIsNone(getattr(obj, 'trigger', None))
        self.assertIsNone(getattr(obj, 'trigger_type', None))
        self.assertIsNone(getattr(obj, 'trigger_instance', None))
        self.assertIsNone(getattr(obj, 'rule', None))
        self.assertDictEqual(obj.action, self.fake_history_subtasks[0]['action'])
        self.assertDictEqual(obj.runner, self.fake_history_subtasks[0]['runner'])
        self.assertDictEqual(obj.execution, self.fake_history_subtasks[0]['execution'])
        self.assertEqual(obj.parent, self.fake_history_subtasks[0]['parent'])
        self.assertIsNone(getattr(obj, 'children', None))

        # Convert API object to DB model.
        model = ActionExecutionHistoryAPI.to_model(obj)
        self.assertEqual(str(model.id), obj.id)
        self.assertDictEqual(model.trigger, {})
        self.assertDictEqual(model.trigger_type, {})
        self.assertDictEqual(model.trigger_instance, {})
        self.assertDictEqual(model.rule, {})
        self.assertDictEqual(model.action, self.fake_history_subtasks[0]['action'])
        self.assertDictEqual(model.runner, self.fake_history_subtasks[0]['runner'])
        doc = copy.deepcopy(self.fake_history_subtasks[0]['execution'])
        doc['start_timestamp'] = isotime.parse(doc['start_timestamp'])
        self.assertDictEqual(model.execution, doc)
        self.assertEqual(model.parent, self.fake_history_subtasks[0]['parent'])
        self.assertListEqual(model.children, [])

        # Convert DB model to API object.
        obj = ActionExecutionHistoryAPI.from_model(model)
        self.assertEqual(str(model.id), obj.id)
        self.assertIsNone(getattr(obj, 'trigger', None))
        self.assertIsNone(getattr(obj, 'trigger_type', None))
        self.assertIsNone(getattr(obj, 'trigger_instance', None))
        self.assertIsNone(getattr(obj, 'rule', None))
        self.assertDictEqual(obj.action, self.fake_history_subtasks[0]['action'])
        self.assertDictEqual(obj.runner, self.fake_history_subtasks[0]['runner'])
        self.assertDictEqual(obj.execution, self.fake_history_subtasks[0]['execution'])
        self.assertEqual(obj.parent, self.fake_history_subtasks[0]['parent'])
        self.assertIsNone(getattr(obj, 'children', None))

    def test_crud_partial(self):
        # Create the DB record.
        obj = ActionExecutionHistoryAPI(**copy.deepcopy(self.fake_history_subtasks[0]))
        ActionExecutionHistory.add_or_update(ActionExecutionHistoryAPI.to_model(obj))
        model = ActionExecutionHistory.get_by_id(obj.id)
        self.assertEqual(str(model.id), obj.id)
        self.assertDictEqual(model.trigger, {})
        self.assertDictEqual(model.trigger_type, {})
        self.assertDictEqual(model.trigger_instance, {})
        self.assertDictEqual(model.rule, {})
        self.assertDictEqual(model.action, self.fake_history_subtasks[0]['action'])
        self.assertDictEqual(model.runner, self.fake_history_subtasks[0]['runner'])
        doc = copy.deepcopy(self.fake_history_subtasks[0]['execution'])
        doc['start_timestamp'] = isotime.parse(doc['start_timestamp'])
        self.assertDictEqual(model.execution, doc)
        self.assertEqual(model.parent, self.fake_history_subtasks[0]['parent'])
        self.assertListEqual(model.children, [])

        # Update the DB record.
        children = [str(bson.ObjectId()), str(bson.ObjectId())]
        model.children = children
        ActionExecutionHistory.add_or_update(model)
        model = ActionExecutionHistory.get_by_id(obj.id)
        self.assertListEqual(model.children, children)

        # Delete the DB record.
        ActionExecutionHistory.delete(model)
        self.assertRaises(ValueError, ActionExecutionHistory.get_by_id, obj.id)

    def test_datetime_range(self):
        base = isotime.add_utc_tz(datetime.datetime(2014, 12, 25, 0, 0, 0))
        for i in range(60):
            timestamp = base + datetime.timedelta(seconds=i)
            doc = copy.deepcopy(self.fake_history_subtasks[0])
            doc['id'] = str(bson.ObjectId())
            doc['execution']['start_timestamp'] = isotime.format(timestamp)
            obj = ActionExecutionHistoryAPI(**doc)
            ActionExecutionHistory.add_or_update(ActionExecutionHistoryAPI.to_model(obj))

        dt_range = '2014-12-25T00:00:10Z..2014-12-25T00:00:19Z'
        objs = ActionExecutionHistory.query(execution__start_timestamp=dt_range)
        self.assertEqual(len(objs), 10)

        dt_range = '2014-12-25T00:00:19Z..2014-12-25T00:00:10Z'
        objs = ActionExecutionHistory.query(execution__start_timestamp=dt_range)
        self.assertEqual(len(objs), 10)

    def test_sort_by_start_timestamp(self):
        base = isotime.add_utc_tz(datetime.datetime(2014, 12, 25, 0, 0, 0))
        for i in range(60):
            timestamp = base + datetime.timedelta(seconds=i)
            doc = copy.deepcopy(self.fake_history_subtasks[0])
            doc['id'] = str(bson.ObjectId())
            doc['execution']['start_timestamp'] = isotime.format(timestamp)
            obj = ActionExecutionHistoryAPI(**doc)
            ActionExecutionHistory.add_or_update(ActionExecutionHistoryAPI.to_model(obj))

        dt_range = '2014-12-25T00:00:10Z..2014-12-25T00:00:19Z'
        objs = ActionExecutionHistory.query(execution__start_timestamp=dt_range,
                                            order_by=['execution__start_timestamp'])
        self.assertLess(objs[0].execution['start_timestamp'], objs[9].execution['start_timestamp'])

        dt_range = '2014-12-25T00:00:19Z..2014-12-25T00:00:10Z'
        objs = ActionExecutionHistory.query(execution__start_timestamp=dt_range,
                                            order_by=['-execution__start_timestamp'])
        self.assertLess(objs[9].execution['start_timestamp'], objs[0].execution['start_timestamp'])
<commit_msg>Update affected tests.<commit_after># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import copy
import bson
import datetime

from st2tests.fixtures import history as fixture
from st2tests import DbTestCase
from st2common.util import isotime
from st2common.persistence.history import ActionExecutionHistory
from st2common.models.api.history import ActionExecutionHistoryAPI

class TestActionExecutionHistoryModel(DbTestCase):

    def setUp(self):
        super(TestActionExecutionHistoryModel, self).setUp()

        # Fake history record for action executions triggered by workflow runner.
        self.fake_history_subtasks = [
            {
                'id': str(bson.ObjectId()),
                'action': copy.deepcopy(fixture.ARTIFACTS['actions']['local']),
                'runner': copy.deepcopy(fixture.ARTIFACTS['runners']['run-local']),
                'execution': copy.deepcopy(fixture.ARTIFACTS['executions']['task1']),
            },
            {
                'id': str(bson.ObjectId()),
                'action': copy.deepcopy(fixture.ARTIFACTS['actions']['local']),
                'runner': copy.deepcopy(fixture.ARTIFACTS['runners']['run-local']),
                'execution': copy.deepcopy(fixture.ARTIFACTS['executions']['task2']),
            }
        ]

        # Fake history record for a workflow action execution triggered by rule.
        self.fake_history_workflow = {
            'id': str(bson.ObjectId()),
            'trigger': copy.deepcopy(fixture.ARTIFACTS['trigger']),
            'trigger_type': copy.deepcopy(fixture.ARTIFACTS['trigger_type']),
            'trigger_instance': copy.deepcopy(fixture.ARTIFACTS['trigger_instance']),
            'rule': copy.deepcopy(fixture.ARTIFACTS['rule']),
            'action': copy.deepcopy(fixture.ARTIFACTS['actions']['chain']),
            'runner': copy.deepcopy(fixture.ARTIFACTS['runners']['action-chain']),
            'execution': copy.deepcopy(fixture.ARTIFACTS['executions']['workflow']),
            'children': [task['id'] for task in self.fake_history_subtasks]
        }

        # Assign parent to the history records for the subtasks.
        for task in self.fake_history_subtasks:
            task['parent'] = self.fake_history_workflow['id']

    def test_model_complete(self):

        # Create API object.
        obj = ActionExecutionHistoryAPI(**copy.deepcopy(self.fake_history_workflow))
        self.assertDictEqual(obj.trigger, self.fake_history_workflow['trigger'])
        self.assertDictEqual(obj.trigger_type, self.fake_history_workflow['trigger_type'])
        self.assertDictEqual(obj.trigger_instance, self.fake_history_workflow['trigger_instance'])
        self.assertDictEqual(obj.rule, self.fake_history_workflow['rule'])
        self.assertDictEqual(obj.action, self.fake_history_workflow['action'])
        self.assertDictEqual(obj.runner, self.fake_history_workflow['runner'])
        self.assertDictEqual(obj.execution, self.fake_history_workflow['execution'])
        self.assertIsNone(getattr(obj, 'parent', None))
        self.assertListEqual(obj.children, self.fake_history_workflow['children'])

        # Convert API object to DB model.
        model = ActionExecutionHistoryAPI.to_model(obj)
        self.assertEqual(str(model.id), obj.id)
        self.assertDictEqual(model.trigger, self.fake_history_workflow['trigger'])
        self.assertDictEqual(model.trigger_type, self.fake_history_workflow['trigger_type'])
        self.assertDictEqual(model.trigger_instance, self.fake_history_workflow['trigger_instance'])
        self.assertDictEqual(model.rule, self.fake_history_workflow['rule'])
        self.assertDictEqual(model.action, self.fake_history_workflow['action'])
        self.assertDictEqual(model.runner, self.fake_history_workflow['runner'])
        doc = copy.deepcopy(self.fake_history_workflow['execution'])
        doc['start_timestamp'] = isotime.parse(doc['start_timestamp'])
        doc['end_timestamp'] = isotime.parse(doc['end_timestamp'])
        self.assertDictEqual(model.execution, doc)
        self.assertIsNone(getattr(model, 'parent', None))
        self.assertListEqual(model.children, self.fake_history_workflow['children'])

        # Convert DB model to API object.
        obj = ActionExecutionHistoryAPI.from_model(model)
        self.assertEqual(str(model.id), obj.id)
        self.assertDictEqual(obj.trigger, self.fake_history_workflow['trigger'])
        self.assertDictEqual(obj.trigger_type, self.fake_history_workflow['trigger_type'])
        self.assertDictEqual(obj.trigger_instance, self.fake_history_workflow['trigger_instance'])
        self.assertDictEqual(obj.rule, self.fake_history_workflow['rule'])
        self.assertDictEqual(obj.action, self.fake_history_workflow['action'])
        self.assertDictEqual(obj.runner, self.fake_history_workflow['runner'])
        self.assertDictEqual(obj.execution, self.fake_history_workflow['execution'])
        self.assertIsNone(getattr(obj, 'parent', None))
        self.assertListEqual(obj.children, self.fake_history_workflow['children'])

    def test_crud_complete(self):
        # Create the DB record.
        obj = ActionExecutionHistoryAPI(**copy.deepcopy(self.fake_history_workflow))
        ActionExecutionHistory.add_or_update(ActionExecutionHistoryAPI.to_model(obj))
        model = ActionExecutionHistory.get_by_id(obj.id)
        self.assertEqual(str(model.id), obj.id)
        self.assertDictEqual(model.trigger, self.fake_history_workflow['trigger'])
        self.assertDictEqual(model.trigger_type, self.fake_history_workflow['trigger_type'])
        self.assertDictEqual(model.trigger_instance, self.fake_history_workflow['trigger_instance'])
        self.assertDictEqual(model.rule, self.fake_history_workflow['rule'])
        self.assertDictEqual(model.action, self.fake_history_workflow['action'])
        self.assertDictEqual(model.runner, self.fake_history_workflow['runner'])
        doc = copy.deepcopy(self.fake_history_workflow['execution'])
        doc['start_timestamp'] = isotime.parse(doc['start_timestamp'])
        doc['end_timestamp'] = isotime.parse(doc['end_timestamp'])
        self.assertDictEqual(model.execution, doc)
        self.assertIsNone(getattr(model, 'parent', None))
        self.assertListEqual(model.children, self.fake_history_workflow['children'])

        # Update the DB record.
        children = [str(bson.ObjectId()), str(bson.ObjectId())]
        model.children = children
        ActionExecutionHistory.add_or_update(model)
        model = ActionExecutionHistory.get_by_id(obj.id)
        self.assertListEqual(model.children, children)

        # Delete the DB record.
        ActionExecutionHistory.delete(model)
        self.assertRaises(ValueError, ActionExecutionHistory.get_by_id, obj.id)

    def test_model_partial(self):
        # Create API object.
        obj = ActionExecutionHistoryAPI(**copy.deepcopy(self.fake_history_subtasks[0]))
        self.assertIsNone(getattr(obj, 'trigger', None))
        self.assertIsNone(getattr(obj, 'trigger_type', None))
        self.assertIsNone(getattr(obj, 'trigger_instance', None))
        self.assertIsNone(getattr(obj, 'rule', None))
        self.assertDictEqual(obj.action, self.fake_history_subtasks[0]['action'])
        self.assertDictEqual(obj.runner, self.fake_history_subtasks[0]['runner'])
        self.assertDictEqual(obj.execution, self.fake_history_subtasks[0]['execution'])
        self.assertEqual(obj.parent, self.fake_history_subtasks[0]['parent'])
        self.assertIsNone(getattr(obj, 'children', None))

        # Convert API object to DB model.
        model = ActionExecutionHistoryAPI.to_model(obj)
        self.assertEqual(str(model.id), obj.id)
        self.assertDictEqual(model.trigger, {})
        self.assertDictEqual(model.trigger_type, {})
        self.assertDictEqual(model.trigger_instance, {})
        self.assertDictEqual(model.rule, {})
        self.assertDictEqual(model.action, self.fake_history_subtasks[0]['action'])
        self.assertDictEqual(model.runner, self.fake_history_subtasks[0]['runner'])
        doc = copy.deepcopy(self.fake_history_subtasks[0]['execution'])
        doc['start_timestamp'] = isotime.parse(doc['start_timestamp'])
        doc['end_timestamp'] = isotime.parse(doc['end_timestamp'])
        self.assertDictEqual(model.execution, doc)
        self.assertEqual(model.parent, self.fake_history_subtasks[0]['parent'])
        self.assertListEqual(model.children, [])

        # Convert DB model to API object.
        obj = ActionExecutionHistoryAPI.from_model(model)
        self.assertEqual(str(model.id), obj.id)
        self.assertIsNone(getattr(obj, 'trigger', None))
        self.assertIsNone(getattr(obj, 'trigger_type', None))
        self.assertIsNone(getattr(obj, 'trigger_instance', None))
        self.assertIsNone(getattr(obj, 'rule', None))
        self.assertDictEqual(obj.action, self.fake_history_subtasks[0]['action'])
        self.assertDictEqual(obj.runner, self.fake_history_subtasks[0]['runner'])
        self.assertDictEqual(obj.execution, self.fake_history_subtasks[0]['execution'])
        self.assertEqual(obj.parent, self.fake_history_subtasks[0]['parent'])
        self.assertIsNone(getattr(obj, 'children', None))

    def test_crud_partial(self):
        # Create the DB record.
        obj = ActionExecutionHistoryAPI(**copy.deepcopy(self.fake_history_subtasks[0]))
        ActionExecutionHistory.add_or_update(ActionExecutionHistoryAPI.to_model(obj))
        model = ActionExecutionHistory.get_by_id(obj.id)
        self.assertEqual(str(model.id), obj.id)
        self.assertDictEqual(model.trigger, {})
        self.assertDictEqual(model.trigger_type, {})
        self.assertDictEqual(model.trigger_instance, {})
        self.assertDictEqual(model.rule, {})
        self.assertDictEqual(model.action, self.fake_history_subtasks[0]['action'])
        self.assertDictEqual(model.runner, self.fake_history_subtasks[0]['runner'])
        doc = copy.deepcopy(self.fake_history_subtasks[0]['execution'])
        doc['start_timestamp'] = isotime.parse(doc['start_timestamp'])
        doc['end_timestamp'] = isotime.parse(doc['end_timestamp'])
        self.assertDictEqual(model.execution, doc)
        self.assertEqual(model.parent, self.fake_history_subtasks[0]['parent'])
        self.assertListEqual(model.children, [])

        # Update the DB record.
        children = [str(bson.ObjectId()), str(bson.ObjectId())]
        model.children = children
        ActionExecutionHistory.add_or_update(model)
        model = ActionExecutionHistory.get_by_id(obj.id)
        self.assertListEqual(model.children, children)

        # Delete the DB record.
        ActionExecutionHistory.delete(model)
        self.assertRaises(ValueError, ActionExecutionHistory.get_by_id, obj.id)

    def test_datetime_range(self):
        base = isotime.add_utc_tz(datetime.datetime(2014, 12, 25, 0, 0, 0))
        for i in range(60):
            timestamp = base + datetime.timedelta(seconds=i)
            doc = copy.deepcopy(self.fake_history_subtasks[0])
            doc['id'] = str(bson.ObjectId())
            doc['execution']['start_timestamp'] = isotime.format(timestamp)
            obj = ActionExecutionHistoryAPI(**doc)
            ActionExecutionHistory.add_or_update(ActionExecutionHistoryAPI.to_model(obj))

        dt_range = '2014-12-25T00:00:10Z..2014-12-25T00:00:19Z'
        objs = ActionExecutionHistory.query(execution__start_timestamp=dt_range)
        self.assertEqual(len(objs), 10)

        dt_range = '2014-12-25T00:00:19Z..2014-12-25T00:00:10Z'
        objs = ActionExecutionHistory.query(execution__start_timestamp=dt_range)
        self.assertEqual(len(objs), 10)

    def test_sort_by_start_timestamp(self):
        base = isotime.add_utc_tz(datetime.datetime(2014, 12, 25, 0, 0, 0))
        for i in range(60):
            timestamp = base + datetime.timedelta(seconds=i)
            doc = copy.deepcopy(self.fake_history_subtasks[0])
            doc['id'] = str(bson.ObjectId())
            doc['execution']['start_timestamp'] = isotime.format(timestamp)
            obj = ActionExecutionHistoryAPI(**doc)
            ActionExecutionHistory.add_or_update(ActionExecutionHistoryAPI.to_model(obj))

        dt_range = '2014-12-25T00:00:10Z..2014-12-25T00:00:19Z'
        objs = ActionExecutionHistory.query(execution__start_timestamp=dt_range,
                                            order_by=['execution__start_timestamp'])
        self.assertLess(objs[0].execution['start_timestamp'], objs[9].execution['start_timestamp'])

        dt_range = '2014-12-25T00:00:19Z..2014-12-25T00:00:10Z'
        objs = ActionExecutionHistory.query(execution__start_timestamp=dt_range,
                                            order_by=['-execution__start_timestamp'])
        self.assertLess(objs[9].execution['start_timestamp'], objs[0].execution['start_timestamp'])
<|endoftext|>


<commit_before># Copyright (c) 2008-2016 Szczepan Faber, Serhiy Oplakanets, Herr Kaste
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from . import invocation
from . import verification

from .utils import get_function_host
from .mocking import Mock
from .mock_registry import mock_registry
from .verification import VerificationError

class ArgumentError(Exception):
    pass

def _multiple_arguments_in_use(*args):
    return len(filter(lambda x: x, args)) > 1

def _invalid_argument(value):
    return (value is not None and value < 1) or value == 0

def _invalid_between(between):
    if between is not None:
        start, end = between
        if start > end or start < 0:
            return True
    return False

def _get_wanted_verification(
        times=None, atleast=None, atmost=None, between=None):
    if times is not None and times < 0:
        raise ArgumentError("'times' argument has invalid value.\n"
                            "It should be at least 0. You wanted to set it to:"
                            " %i" % times)
    if _multiple_arguments_in_use(atleast, atmost, between):
        raise ArgumentError(
            "You can set only one of the arguments: 'atleast', "
            "'atmost' or 'between'.""")
    if _invalid_argument(atleast):
        raise ArgumentError("'atleast' argument has invalid value.\n"
                            "It should be at least 1.  You wanted to set it "
                            "to: %i" % atleast)
    if _invalid_argument(atmost):
        raise ArgumentError("'atmost' argument has invalid value.\n"
                            "It should be at least 1.  You wanted to set it "
                            "to: %i""" % atmost)
    if _invalid_between(between):
        raise ArgumentError("""'between' argument has invalid value.
            It should consist of positive values with second number not greater
            than first e.g. [1, 4] or [0, 3] or [2, 2]
            You wanted to set it to: %s""" % between)

    if atleast:
        return verification.AtLeast(atleast)
    elif atmost:
        return verification.AtMost(atmost)
    elif between:
        return verification.Between(*between)
    elif times is not None:
        return verification.Times(times)

def _get_mock(obj, strict=True):
    theMock = mock_registry.mock_for(obj)
    if theMock is None:
        theMock = Mock(obj, strict=strict, spec=obj)
        mock_registry.register(obj, theMock)
    return theMock

def verify(obj, times=1, atleast=None, atmost=None, between=None,
           inorder=False):

    verification_fn = _get_wanted_verification(
        times=times, atleast=atleast, atmost=atmost, between=between)
    if inorder:
        verification_fn = verification.InOrder(verification_fn)

    # FIXME?: Catch error if obj is neither a Mock nor a known stubbed obj
    theMock = _get_mock(obj)

    class Verify(object):
        def __getattr__(self, method_name):
            return invocation.VerifiableInvocation(
                theMock, method_name, verification_fn)

    return Verify()

def when(obj, strict=True):
    """Central interface to stub functions on a given ``obj``

    ``obj`` should be a module, a class or an instance of a class; it can be
    a Dummy you created with ``mock``. ``when`` exposes a fluent interface
    where you configure a stub in three steps::

        when(<obj>).<method_name>(<args>).thenReturn(<value>)

    Compared to simple *patching*, stubbing in mockito requires you to specify
    conrete ``args`` for which the stub will answer with a concrete
    ``<value>``. All invocations that do not match this specific call signature
    will be rejected. They usually throw at call time.

    Stubbing in mockito's sense thus means not only to get rid of unwanted
    side effects, but effectively to turn function calls into constants.

    E.g.::

        # Given ``dog`` is an instance of a ``Dog``
        when(dog).bark('Grrr').thenReturn('Wuff')
        when(dog).bark('Miau').thenRaise(TypeError())

        # With this configuration set up:
        assert dog.bark('Grrr') == 'Wuff'
        dog.bark('Miau')  # will throw TypeError
        dog.bark('Wuff')  # will throw unwanted interaction

    Stubbing can effectively be used as monkeypatching; usage shown with
    the ``with`` context managing::

        with when(os.path).exists('/foo').thenReturn(True):
            ...

    Most of the time verifying your interactions is not necessary, because
    your code under tests implicitly verifies the return value by evaluating
    it. See ``verify`` if you need to, see also ``expect`` to setup expected
    call counts up front.

    If your function is pure side effect and does not return something, you
    can omit the specific answer. The default then is ``None``::

        when(manager).do_work()

    ``when`` verifies the method name, the expected argument signature, and the
    actual, factual arguments your code under test uses against the original
    object and its function so its easier to spot changing interfaces.

    Sometimes it's tedious to spell out all arguments::

        from mockito import ANY, ARGS, KWARGS
        when(requests).get('http://example.com/', **KWARGS).thenReturn(...)
        when(os.path).exists(ANY)
        when(os.path).exists(ANY(str))

    This is especially handy for subsequent ``verify`` calls::

        when(manager).add_tasks(1, 2, 3)
        ...
        # no need to duplicate the specification; every other argument pattern
        # would have raised anyway.
        verify(manager).add_tasks(*ARGS)
        verify(manager).add_tasks(...)       # Py3
        verify(manager).add_tasks(Ellipsis)  # Py2

    **MUST ``unstub`` after stubbing.** Or use ``with`` statement.

    Set ``strict=False`` to bypass the function signature checks.

    See related ``when2`` which has a more pythonic interface.

    """
    theMock = _get_mock(obj, strict=strict)

    class When(object):
        def __getattr__(self, method_name):
            return invocation.StubbedInvocation(theMock, method_name)

    return When()

def when2(fn, *args, **kwargs):
    """Stub a function call with the given arguments

    Exposes a more pythonic interface than ``when``. See ``when`` for more
    documentation.

    Returns ``AnswerSelector`` interface which exposes ``thenReturn``,
    ``thenRaise``, and ``thenAnswer`` as usual. Always ``strict``.

    Usage::

        # Given ``dog`` is an instance of a ``Dog``
        when2(dog.bark, 'Miau').thenReturn('Wuff')

    **MUST ``unstub`` after stubbing.** Or use ``with`` statement.

    """
    obj, name, fn = get_function_host(fn)
    theMock = Mock(obj, strict=True, spec=obj)
    mock_registry.register(obj, theMock)
    return invocation.StubbedInvocation(theMock, name)(*args, **kwargs)

def patch(fn, new_fn):
    when2(fn, Ellipsis).thenAnswer(new_fn)

def expect(obj, strict=True,
           times=None, atleast=None, atmost=None, between=None):
    """Stub a function call, and set up an expected call count.

    Usage::

        # Given ``dog`` is an instance of a ``Dog``
        expect(dog, times=1).bark('Wuff').thenReturn('Miau')
        dog.bark('Wuff')
        dog.bark('Wuff')  # will throw at call time: too many invocations

        # maybe if you need to ensure that ``dog.bark()`` was called at all
        verifyNoUnwantedInteractions()

    **MUST ``unstub`` after stubbing.** Or use ``with`` statement.

    See ``when``, ``when2``

    """

    theMock = _get_mock(obj, strict=strict)

    verification_fn = _get_wanted_verification(
        times=times, atleast=atleast, atmost=atmost, between=between)

    class Expect(object):
        def __getattr__(self, method_name):
            return invocation.StubbedInvocation(
                theMock, method_name, verification_fn)

    return Expect()

def unstub(*objs):
    """Unstubs all stubbed methods and functions

    If you don't pass in any argument, *all* registered mocks and
    patched modules, classes etc. will be unstubbed.

    Note that additionally, the underlying registry will be cleaned.
    After an ``unstub`` you can't ``verify`` anymore because all
    interactions will be forgotten.
    """

    if objs:
        for obj in objs:
            mock_registry.unstub(obj)
    else:
        mock_registry.unstub_all()

def verifyNoMoreInteractions(*objs):
    verifyNoUnwantedInteractions(*objs)

    for obj in objs:
        theMock = _get_mock(obj)

        for i in theMock.invocations:
            if not i.verified:
                raise VerificationError("\nUnwanted interaction: " + str(i))

def verifyZeroInteractions(*mocks):
    verifyNoMoreInteractions(*mocks)

def verifyNoUnwantedInteractions(*objs):
    """Verifies that expectations set via ``expect`` are met

    E.g.::

        expect(os.path, times=1).exists(...).thenReturn(True)
        os.path('/foo')
        verifyNoUnwantedInteractions(os.path)  # ok, called once

    If you leave out the argument *all* registered objects will
    be checked. **DANGERZONE**: If you did not ``unstub`` correctly,
    it is possible that old registered mocks, from other tests
    leak.
    """

    if objs:
        theMocks = map(_get_mock, objs)
    else:
        theMocks = mock_registry.get_registered_mocks()

    for mock in theMocks:
        for i in mock.stubbed_invocations:
            i.verify()
<commit_msg>Basic doc for `verify`<commit_after># Copyright (c) 2008-2016 Szczepan Faber, Serhiy Oplakanets, Herr Kaste
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from . import invocation
from . import verification

from .utils import get_function_host
from .mocking import Mock
from .mock_registry import mock_registry
from .verification import VerificationError

class ArgumentError(Exception):
    pass

def _multiple_arguments_in_use(*args):
    return len(filter(lambda x: x, args)) > 1

def _invalid_argument(value):
    return (value is not None and value < 1) or value == 0

def _invalid_between(between):
    if between is not None:
        start, end = between
        if start > end or start < 0:
            return True
    return False

def _get_wanted_verification(
        times=None, atleast=None, atmost=None, between=None):
    if times is not None and times < 0:
        raise ArgumentError("'times' argument has invalid value.\n"
                            "It should be at least 0. You wanted to set it to:"
                            " %i" % times)
    if _multiple_arguments_in_use(atleast, atmost, between):
        raise ArgumentError(
            "You can set only one of the arguments: 'atleast', "
            "'atmost' or 'between'.""")
    if _invalid_argument(atleast):
        raise ArgumentError("'atleast' argument has invalid value.\n"
                            "It should be at least 1.  You wanted to set it "
                            "to: %i" % atleast)
    if _invalid_argument(atmost):
        raise ArgumentError("'atmost' argument has invalid value.\n"
                            "It should be at least 1.  You wanted to set it "
                            "to: %i""" % atmost)
    if _invalid_between(between):
        raise ArgumentError("""'between' argument has invalid value.
            It should consist of positive values with second number not greater
            than first e.g. [1, 4] or [0, 3] or [2, 2]
            You wanted to set it to: %s""" % between)

    if atleast:
        return verification.AtLeast(atleast)
    elif atmost:
        return verification.AtMost(atmost)
    elif between:
        return verification.Between(*between)
    elif times is not None:
        return verification.Times(times)

def _get_mock(obj, strict=True):
    theMock = mock_registry.mock_for(obj)
    if theMock is None:
        theMock = Mock(obj, strict=strict, spec=obj)
        mock_registry.register(obj, theMock)
    return theMock

def verify(obj, times=1, atleast=None, atmost=None, between=None,
           inorder=False):
    """Central interface to verify interactions.

    ``verify`` uses a fluent interface::

        verify(<obj>, times=2).<method_name>(<args>)

    `args` can be as concrete as neccessary. Often a catch-all is enough,
    especially if you're working with strict mocks, bc they throw at call
    time on unwanted, unconfigured arguments::

        from mockito import ANY, ARGS, KWARGS
        when(manager).add_tasks(1, 2, 3)
        ...
        # no need to duplicate the specification; every other argument pattern
        # would have raised anyway.
        verify(manager).add_tasks(*ARGS)
        verify(manager).add_tasks(...)       # Py3
        verify(manager).add_tasks(Ellipsis)  # Py2

    """

    verification_fn = _get_wanted_verification(
        times=times, atleast=atleast, atmost=atmost, between=between)
    if inorder:
        verification_fn = verification.InOrder(verification_fn)

    # FIXME?: Catch error if obj is neither a Mock nor a known stubbed obj
    theMock = _get_mock(obj)

    class Verify(object):
        def __getattr__(self, method_name):
            return invocation.VerifiableInvocation(
                theMock, method_name, verification_fn)

    return Verify()

def when(obj, strict=True):
    """Central interface to stub functions on a given ``obj``

    ``obj`` should be a module, a class or an instance of a class; it can be
    a Dummy you created with ``mock``. ``when`` exposes a fluent interface
    where you configure a stub in three steps::

        when(<obj>).<method_name>(<args>).thenReturn(<value>)

    Compared to simple *patching*, stubbing in mockito requires you to specify
    conrete ``args`` for which the stub will answer with a concrete
    ``<value>``. All invocations that do not match this specific call signature
    will be rejected. They usually throw at call time.

    Stubbing in mockito's sense thus means not only to get rid of unwanted
    side effects, but effectively to turn function calls into constants.

    E.g.::

        # Given ``dog`` is an instance of a ``Dog``
        when(dog).bark('Grrr').thenReturn('Wuff')
        when(dog).bark('Miau').thenRaise(TypeError())

        # With this configuration set up:
        assert dog.bark('Grrr') == 'Wuff'
        dog.bark('Miau')  # will throw TypeError
        dog.bark('Wuff')  # will throw unwanted interaction

    Stubbing can effectively be used as monkeypatching; usage shown with
    the ``with`` context managing::

        with when(os.path).exists('/foo').thenReturn(True):
            ...

    Most of the time verifying your interactions is not necessary, because
    your code under tests implicitly verifies the return value by evaluating
    it. See ``verify`` if you need to, see also ``expect`` to setup expected
    call counts up front.

    If your function is pure side effect and does not return something, you
    can omit the specific answer. The default then is ``None``::

        when(manager).do_work()

    ``when`` verifies the method name, the expected argument signature, and the
    actual, factual arguments your code under test uses against the original
    object and its function so its easier to spot changing interfaces.

    Sometimes it's tedious to spell out all arguments::

        from mockito import ANY, ARGS, KWARGS
        when(requests).get('http://example.com/', **KWARGS).thenReturn(...)
        when(os.path).exists(ANY)
        when(os.path).exists(ANY(str))

    **MUST ``unstub`` after stubbing.** Or use ``with`` statement.

    Set ``strict=False`` to bypass the function signature checks.

    See related ``when2`` which has a more pythonic interface.

    """
    theMock = _get_mock(obj, strict=strict)

    class When(object):
        def __getattr__(self, method_name):
            return invocation.StubbedInvocation(theMock, method_name)

    return When()

def when2(fn, *args, **kwargs):
    """Stub a function call with the given arguments

    Exposes a more pythonic interface than ``when``. See ``when`` for more
    documentation.

    Returns ``AnswerSelector`` interface which exposes ``thenReturn``,
    ``thenRaise``, and ``thenAnswer`` as usual. Always ``strict``.

    Usage::

        # Given ``dog`` is an instance of a ``Dog``
        when2(dog.bark, 'Miau').thenReturn('Wuff')

    **MUST ``unstub`` after stubbing.** Or use ``with`` statement.

    """
    obj, name, fn = get_function_host(fn)
    theMock = Mock(obj, strict=True, spec=obj)
    mock_registry.register(obj, theMock)
    return invocation.StubbedInvocation(theMock, name)(*args, **kwargs)

def patch(fn, new_fn):
    when2(fn, Ellipsis).thenAnswer(new_fn)

def expect(obj, strict=True,
           times=None, atleast=None, atmost=None, between=None):
    """Stub a function call, and set up an expected call count.

    Usage::

        # Given ``dog`` is an instance of a ``Dog``
        expect(dog, times=1).bark('Wuff').thenReturn('Miau')
        dog.bark('Wuff')
        dog.bark('Wuff')  # will throw at call time: too many invocations

        # maybe if you need to ensure that ``dog.bark()`` was called at all
        verifyNoUnwantedInteractions()

    **MUST ``unstub`` after stubbing.** Or use ``with`` statement.

    See ``when``, ``when2``

    """

    theMock = _get_mock(obj, strict=strict)

    verification_fn = _get_wanted_verification(
        times=times, atleast=atleast, atmost=atmost, between=between)

    class Expect(object):
        def __getattr__(self, method_name):
            return invocation.StubbedInvocation(
                theMock, method_name, verification_fn)

    return Expect()

def unstub(*objs):
    """Unstubs all stubbed methods and functions

    If you don't pass in any argument, *all* registered mocks and
    patched modules, classes etc. will be unstubbed.

    Note that additionally, the underlying registry will be cleaned.
    After an ``unstub`` you can't ``verify`` anymore because all
    interactions will be forgotten.
    """

    if objs:
        for obj in objs:
            mock_registry.unstub(obj)
    else:
        mock_registry.unstub_all()

def verifyNoMoreInteractions(*objs):
    verifyNoUnwantedInteractions(*objs)

    for obj in objs:
        theMock = _get_mock(obj)

        for i in theMock.invocations:
            if not i.verified:
                raise VerificationError("\nUnwanted interaction: " + str(i))

def verifyZeroInteractions(*mocks):
    verifyNoMoreInteractions(*mocks)

def verifyNoUnwantedInteractions(*objs):
    """Verifies that expectations set via ``expect`` are met

    E.g.::

        expect(os.path, times=1).exists(...).thenReturn(True)
        os.path('/foo')
        verifyNoUnwantedInteractions(os.path)  # ok, called once

    If you leave out the argument *all* registered objects will
    be checked. **DANGERZONE**: If you did not ``unstub`` correctly,
    it is possible that old registered mocks, from other tests
    leak.
    """

    if objs:
        theMocks = map(_get_mock, objs)
    else:
        theMocks = mock_registry.get_registered_mocks()

    for mock in theMocks:
        for i in mock.stubbed_invocations:
            i.verify()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# vim: ai ts=4 sts=4 et sw=4 encoding=utf-8

import time
import pygsm
import Queue

from rapidsms.message import Message
from rapidsms.connection import Connection
from rapidsms.backends import Backend
import backend
from rapidsms import log
from rapidsms import utils
from datetime import datetime

POLL_INTERVAL=2 # num secs to wait between checking for inbound texts
LOG_LEVEL_MAP = {
    'traffic':'info',
    'read':'info',
    'write':'info',
    'debug':'debug',
    'warn':'warning',
    'error':'error'
}

class Backend(Backend):
    _title = "pyGSM"

    def _log(self, modem, msg, level):
        """
        convert GsmModem levels to levels understood by
        the rapidsms logger

        """
        level = 'info'
        try:
            level = LOG_LEVEL_MAP[level]
        except:
            # inbound level was bofus
            pass

        if self.modem_logger is not None:
            self.modem_logger.write(self,level,msg)
        else:
            self.router.log(level, msg)

    def configure(self, *args, **kwargs):
        self.modem = None
        self.modem_args = args

        # set max outbound text size
        if 'max_csm' in kwargs:
            self.max_csm = int(kwargs['max_csm'])
        else:
            self.max_csm = 1

        if self.max_csm>255:
            self.max_csm = 255
        if self.max_csm<1:
                self.max_csm = 1

        # make a modem log
        self.modem_logger = None
        if 'modem_log' in  kwargs:
            mlog = kwargs.pop('modem_log')
            level='info'
            if 'modem_log_level' in kwargs:
                level=kwargs.pop('modem_log_level')
            self.modem_logger = log.Logger(level=level, file=mlog, channel='pygsm')

        kwargs['logger'] = self._log
        self.modem_kwargs = kwargs

    def __send_sms(self, message):
        try:
            self.modem.send_sms(
                str(message.connection.identity),
                message.text,
                max_messages=self.max_csm)
        except ValueError, err:
            # TODO: Pass this error info on to caller!
            self.error('Error sending message: %s' % err)

    def run(self):
        while self._running:
            # check for new messages
            msg = self.modem.next_message()

            if msg is not None:
                # we got an sms! create RapidSMS Connection and
                # Message objects, and hand it off to the router
                c = Connection(self, msg.sender)

                # Try to use message sent date as timestamp
                # SOMETIMES this doesn't come over, in which
                # case use the current time
                try:
                    date=utils.to_naive_utc_dt(msg.sent)
                except:
                    date=datetime.utcnow()

                m = Message(
                            connection=c,
                            text=msg.text,
                            date=date
                            )
                self.router.send(m)

            # process all outbound messages
            while True:
                try:
                    self.__send_sms(self._queue.get_nowait())
                except Queue.Empty:
                    # break out of while
                    break

            # poll for new messages
            # every POLL_INTERVAL seconds
            time.sleep(POLL_INTERVAL)

    def start(self):
        self.modem = pygsm.GsmModem(
            *self.modem_args,
            **self.modem_kwargs)

        # If we got the connection, call superclass to
        # start the run loop--it just sets self._running to True
        # and calls run.
        if self.modem is not None:
            backend.Backend.start(self)

    def stop(self):
        # call superclass to stop--sets self._running
        # to False so that the 'run' loop will exit cleanly.
        backend.Backend.stop(self)

        # disconnect from modem
        if self.modem is not None:
            self.modem.disconnect()

<commit_msg>set default logger_level and trap any exception from level lookup<commit_after>#!/usr/bin/env python
# vim: ai ts=4 sts=4 et sw=4 encoding=utf-8

import time
import pygsm
import Queue

from rapidsms.message import Message
from rapidsms.connection import Connection
from rapidsms.backends import Backend
import backend
from rapidsms import log
from rapidsms import utils
from datetime import datetime

POLL_INTERVAL=2 # num secs to wait between checking for inbound texts
LOG_LEVEL_MAP = {
    'traffic':'info',
    'read':'info',
    'write':'info',
    'debug':'debug',
    'warn':'warning',
    'error':'error'
}

class Backend(Backend):
    _title = "pyGSM"

    def _log(self, modem, msg, level):
        """
        convert GsmModem levels to levels understood by
        the rapidsms logger

        """
        logger_level = 'info'
        try:
            logger_level = LOG_LEVEL_MAP[level]
        except:
            # inbound level was bofus
            pass

        if self.modem_logger is not None:
            self.modem_logger.write(self,logger_level,msg)
        else:
            self.router.log(logger_level, msg)

    def configure(self, *args, **kwargs):
        self.modem = None
        self.modem_args = args

        # set max outbound text size
        if 'max_csm' in kwargs:
            self.max_csm = int(kwargs['max_csm'])
        else:
            self.max_csm = 1

        if self.max_csm>255:
            self.max_csm = 255
        if self.max_csm<1:
                self.max_csm = 1

        # make a modem log
        self.modem_logger = None
        if 'modem_log' in  kwargs:
            mlog = kwargs.pop('modem_log')
            level='info'
            if 'modem_log_level' in kwargs:
                level=kwargs.pop('modem_log_level')
            self.modem_logger = log.Logger(level=level, file=mlog, channel='pygsm')

        kwargs['logger'] = self._log
        self.modem_kwargs = kwargs

    def __send_sms(self, message):
        try:
            self.modem.send_sms(
                str(message.connection.identity),
                message.text,
                max_messages=self.max_csm)
        except ValueError, err:
            # TODO: Pass this error info on to caller!
            self.error('Error sending message: %s' % err)

    def run(self):
        while self._running:
            # check for new messages
            msg = self.modem.next_message()

            if msg is not None:
                # we got an sms! create RapidSMS Connection and
                # Message objects, and hand it off to the router
                c = Connection(self, msg.sender)

                # Try to use message sent date as timestamp
                # SOMETIMES this doesn't come over, in which
                # case use the current time
                try:
                    date=utils.to_naive_utc_dt(msg.sent)
                except:
                    date=datetime.utcnow()

                m = Message(
                            connection=c,
                            text=msg.text,
                            date=date
                            )
                self.router.send(m)

            # process all outbound messages
            while True:
                try:
                    self.__send_sms(self._queue.get_nowait())
                except Queue.Empty:
                    # break out of while
                    break

            # poll for new messages
            # every POLL_INTERVAL seconds
            time.sleep(POLL_INTERVAL)

    def start(self):
        self.modem = pygsm.GsmModem(
            *self.modem_args,
            **self.modem_kwargs)

        # If we got the connection, call superclass to
        # start the run loop--it just sets self._running to True
        # and calls run.
        if self.modem is not None:
            backend.Backend.start(self)

    def stop(self):
        # call superclass to stop--sets self._running
        # to False so that the 'run' loop will exit cleanly.
        backend.Backend.stop(self)

        # disconnect from modem
        if self.modem is not None:
            self.modem.disconnect()

<|endoftext|>


<commit_before>import os
import click
import shutil
from click.testing import CliRunner
from flaky import flaky

from populus.utils.networking import (
    get_open_port,
)
from populus.utils.transactions import (
    wait_for_peers,
    wait_for_block_number,
    wait_for_syncing,
)
from populus.utils.cli import (
    show_chain_sync_progress,
)
from populus.project import Project

@flaky
def test_show_chain_sync_progress():
    project = Project()

    main_chain = project.get_chain('temp',
                                   no_discover=False,
                                   max_peers=None,
                                   port=str(get_open_port()),
                                   rpc_enabled=False,
                                   ws_enabled=False)
    with main_chain:
        sync_chain = project.get_chain('temp',
                                       no_discover=False,
                                       max_peers=None,
                                       mine=False,
                                       miner_threads=None,
                                       port=str(get_open_port()),
                                       rpc_enabled=False,
                                       ws_enabled=False)

        main_chain_data_dir = main_chain.geth.data_dir
        sync_chain_data_dir = sync_chain.geth.data_dir

        main_genesis_file = os.path.join(main_chain_data_dir, 'genesis.json')
        sync_genesis_file = os.path.join(sync_chain_data_dir, 'genesis.json')

        main_chaindata_dir = os.path.join(main_chain_data_dir, 'chaindata')
        sync_chaindata_dir = os.path.join(sync_chain_data_dir, 'chaindata')

        os.remove(sync_genesis_file)
        shutil.rmtree(sync_chaindata_dir)

        shutil.copyfile(main_genesis_file, sync_genesis_file)
        shutil.copytree(main_chaindata_dir, sync_chaindata_dir)

        block_numbers = []

        runner = CliRunner()

        @click.command()
        def wrapper():
            block_numbers.append(sync_chain.web3.eth.blockNumber)
            show_chain_sync_progress(sync_chain)
            block_numbers.append(sync_chain.web3.eth.blockNumber)

        with sync_chain:
            main_node_info = main_chain.web3.admin.nodeInfo
            main_enode = "enode://{node_id}@127.0.0.1:{node_port}".format(
                node_id=main_node_info['id'],
                node_port=main_node_info['ports']['listener'],
            )
            sync_node_info = sync_chain.web3.admin.nodeInfo
            sync_enode = "enode://{node_id}@127.0.0.1:{node_port}".format(
                node_id=sync_node_info['id'],
                node_port=sync_node_info['ports']['listener'],
            )

            wait_for_block_number(main_chain.web3, 40, 120)

            main_chain_start_block = main_chain.web3.eth.blockNumber
            sync_chain_start_block = sync_chain.web3.eth.blockNumber

            assert main_chain_start_block - sync_chain_start_block >= 10

            assert sync_chain.web3.net.peerCount == 0

            sync_chain.web3.admin.addPeer(main_enode)
            main_chain.web3.admin.addPeer(sync_enode)

            wait_for_peers(sync_chain.web3, timeout=30)
            wait_for_syncing(sync_chain.web3, timeout=30)

            result = runner.invoke(wrapper, [])

            assert result.exit_code == 0
            assert len(block_numbers) == 2

            start_block = block_numbers[0]
            end_block = block_numbers[1]

            assert start_block <= 1
            assert end_block >= 20
<commit_msg>bump the timeout<commit_after>import os
import click
import shutil
from click.testing import CliRunner
from flaky import flaky

from populus.utils.networking import (
    get_open_port,
)
from populus.utils.transactions import (
    wait_for_peers,
    wait_for_block_number,
    wait_for_syncing,
)
from populus.utils.cli import (
    show_chain_sync_progress,
)
from populus.project import Project

@flaky
def test_show_chain_sync_progress():
    project = Project()

    main_chain = project.get_chain('temp',
                                   no_discover=False,
                                   max_peers=None,
                                   port=str(get_open_port()),
                                   rpc_enabled=False,
                                   ws_enabled=False)
    with main_chain:
        sync_chain = project.get_chain('temp',
                                       no_discover=False,
                                       max_peers=None,
                                       mine=False,
                                       miner_threads=None,
                                       port=str(get_open_port()),
                                       rpc_enabled=False,
                                       ws_enabled=False)

        main_chain_data_dir = main_chain.geth.data_dir
        sync_chain_data_dir = sync_chain.geth.data_dir

        main_genesis_file = os.path.join(main_chain_data_dir, 'genesis.json')
        sync_genesis_file = os.path.join(sync_chain_data_dir, 'genesis.json')

        main_chaindata_dir = os.path.join(main_chain_data_dir, 'chaindata')
        sync_chaindata_dir = os.path.join(sync_chain_data_dir, 'chaindata')

        os.remove(sync_genesis_file)
        shutil.rmtree(sync_chaindata_dir)

        shutil.copyfile(main_genesis_file, sync_genesis_file)
        shutil.copytree(main_chaindata_dir, sync_chaindata_dir)

        block_numbers = []

        runner = CliRunner()

        @click.command()
        def wrapper():
            block_numbers.append(sync_chain.web3.eth.blockNumber)
            show_chain_sync_progress(sync_chain)
            block_numbers.append(sync_chain.web3.eth.blockNumber)

        with sync_chain:
            main_node_info = main_chain.web3.admin.nodeInfo
            main_enode = "enode://{node_id}@127.0.0.1:{node_port}".format(
                node_id=main_node_info['id'],
                node_port=main_node_info['ports']['listener'],
            )
            sync_node_info = sync_chain.web3.admin.nodeInfo
            sync_enode = "enode://{node_id}@127.0.0.1:{node_port}".format(
                node_id=sync_node_info['id'],
                node_port=sync_node_info['ports']['listener'],
            )

            wait_for_block_number(main_chain.web3, 40, 180)

            main_chain_start_block = main_chain.web3.eth.blockNumber
            sync_chain_start_block = sync_chain.web3.eth.blockNumber

            assert main_chain_start_block - sync_chain_start_block >= 10

            assert sync_chain.web3.net.peerCount == 0

            sync_chain.web3.admin.addPeer(main_enode)
            main_chain.web3.admin.addPeer(sync_enode)

            wait_for_peers(sync_chain.web3, timeout=30)
            wait_for_syncing(sync_chain.web3, timeout=30)

            result = runner.invoke(wrapper, [])

            assert result.exit_code == 0
            assert len(block_numbers) == 2

            start_block = block_numbers[0]
            end_block = block_numbers[1]

            assert start_block <= 1
            assert end_block >= 20
<|endoftext|>


<commit_before>#system information gather script.

import subprocess

def uname_func():
	uname = 'uname'
	uname_arg='-a'
	print "collecting system information with %s command:\n" %uname
	subprocess.call([uname,uname_arg])

def disk_func():
	diskspace = "df"
	diskspace_arg = "-h"
	print "collecting diskspace information %s command: \n" % diskspace
	subprocess.call([diskspace,diskspace_arg])

def main():
	uname_func()
	disk_func()

main()
<commit_msg>added some more functions.<commit_after>'''
python script to collect system information using subprocess library.
'''

#system information gather script.

import subprocess

def shell_func():
	print "Setting the shell argument to a true value"
	subprocess.call('echo $HOME',shell=True)

def uname_func():
	uname = 'uname'
	uname_arg='-a'
	print "gathering system information with %s command:\n" %uname
	subprocess.call([uname,uname_arg])

def disk_func():
	diskspace = "df"
	diskspace_arg = "-h"
	print "gathering diskspace information %s command: \n" % diskspace
	subprocess.call([diskspace,diskspace_arg])

def in_out_func():
	output = subprocess.check_output(['ls','-1'])
	print 'have %d bytes in output' % len(output)
	print output

def main():
	shell_func()
	uname_func()
	disk_func()
	in_out_func()

main()
<|endoftext|>


<commit_before># migrate_database.py: Migrate dumps of the Cognitive Atlas database into Neo4j
# see russ_ubergraph.pdf and quasi models below

from py2neo import Graph, Path, Node, Relationship, authenticate
from cognitiveatlas.api import get_task, get_concept
import pandas
import os
import re

# Function to clean up column names
# Note that Cognitive Atlas exports tables with different header delimiter than rest, I fixed this for
# the data below so all delims are ";"
def cleancolumns(df):
    df.columns = [x.strip() for x in df.columns]
    return df

# Function to make a node
def make_node(nodetype,uid,name,properties=None,property_key="id"):
    node = None
    if graph.find_one(nodetype,property_key='id', property_value=uid) == None:
        print("Creating %s:%s, %s" %(nodetype,name,uid))
        timestamp = graph.cypher.execute("RETURN timestamp()").one
        node = Node(nodetype, name=name,id=uid,creation_time=timestamp,last_updated=timestamp)
        graph.create(node)
        if properties != None:
            for property_name in properties.keys():
                node.properties[property_name] = properties[property_name]
            node.push()
    return node

# Function to make a relation
def make_relation(startnode,rel_type,endnode,properties=None):
    relation = None
    if graph.match_one(start_node=startnode, rel_type=rel_type, end_node=endnode) == None:
        relation = Relationship(startnode, rel_type, endnode)
        print("Creating relation %s [%s] %s" %(startnode.properties["name"],rel_type,endnode.properties["name"]))
        graph.create(relation)
        if properties != None:
            for property_name in properties.keys():
                relation.properties[property_name] = properties[property_name]
            relation.push()
    return relation

# Function to look up node - return None if doesn't exist
def find_node(nodetype,property_value,property_key='id'):
    try:
        node = graph.find_one(nodetype,property_key='id',property_value=property_value)
    except:
        node = None
    return node

# First we want to read in all of our data types
concepts = cleancolumns(pandas.read_csv("data/Dump_concept_2015-12-25_567dd9b28fabb.csv",sep=";"))
tasks = cleancolumns(pandas.read_csv("data/Dump_task_2015-12-25_567dd9bb7ff74.csv",sep=";"))
contrasts = cleancolumns(pandas.read_csv("data/Dump_contrast_2015-12-25_567dd9c4b46c4.csv",sep=";"))
battery = cleancolumns(pandas.read_csv("data/Dump_battery_2015-12-25_567dd9d2866fd.csv",sep=";"))
conditions = cleancolumns(pandas.read_csv("data/Dump_condition_2015-12-25_567dd9bf079bb.csv",sep=";"))
disorders = cleancolumns(pandas.read_csv("data/Dump_disorder_2015-12-26_567f21649c2cb.csv",sep=";"))
assertions = cleancolumns(pandas.read_csv("data/Dump_assertion_2015-12-25_567dd9cd411c5.csv",sep=";"))

# connect to graph database
#graph = Graph("http://graphdb:7474/db/data/")

# Just for local development
authenticate("localhost:7474", "neo4j", "noodles")
graph = Graph()

# clear database
graph.delete_all()

tx = graph.cypher.begin()
conceptnodes={}
tasknodes={}
contrastnodes={}

#class Task(models.NodeModel):
#    name = models.StringProperty()
#    uid = models.StringProperty(indexed=True)
#    derived_from = models.Relationship('self',rel_type='DERIVEDFROM')
#    definition = models.StringProperty()
#    has_condition = models.Relationship(Condition,rel_type='HASCONDITION')
#    has_implementation = models.Relationship(Experiment,rel_type='HASIMPLEMENTATION') # ExperimentFactory?
#    mentioned_in = models.Relationship('PMID',rel_type='MENTIONEDIN')

for row in tasks.iterrows():
    name = row[1].term
    uid = row[1].url.split("/")[-1]
    try:
        task = get_task(id=uid,silent=True).json
        definition = task[0]["definition_text"]
    except:
        definition = ""
    if not str(name) =="nan":
        properties = {"definition":definition}
        node = make_node("task",uid,name,properties)

#class Condition(models.NodeModel):
#    name = models.StringProperty()
#    uid = models.StringProperty(indexed=True)
#    has_contrast = models.Relationship(Contrast,rel_type='HASCONTRAST')

for row in conditions.iterrows():
    name = row[1].condition_text
    user = row[1].id_user
    uid = row[1].id
    task = row[1].id_term
    description = row[1].condition_description
    timestamp = row[1].event_stamp
    if not str(name) =="nan":
        properties = {"description":description}
        node = make_node("condition",uid,name,properties)
        tasknode = find_node("task",property_value=task)
        # If the tasknode is node found, then we do not create the relation, and we delete the condition
        if tasknode == None:
            graph.delete(node)
        else:
            relation = make_relation(tasknode,"HASCONDITION",node)

#class Disorder(models.NodeModel):
#    has_difference = models.Relationship(Contrast,rel_type='HASDIFFERENCE')
#    mentioned_in = models.Relationship('PMID',rel_type='MENTIONEDIN')

disorders.columns = ["id","term","classification"] # Bug in returning column index name
for row in disorders.iterrows():
    uid = row[1].id
    classification = row[1].classification
    name = row[1].term
    properties = {"classification":classification}
    node = make_node("disorder",uid,name,properties)

#class Contrast(models.NodeModel):
#    name = models.StringProperty()
#    uid = models.StringProperty(indexed=True)
#    measured_by_phenomenon = models.Relationship(Phenomenon,rel_type='MEASUREDBYPHENOMENON')
#    measured_by_trait = models.Relationship(Trait,rel_type='MEASUREDBYTRAIT')

for row in contrasts.iterrows():
    uid = row[1].id
    user = row[1].id_user
    name = row[1].contrast_text
    timestamp = row[1].event_stamp
    node = make_node("contrast",uid,name)

#class Concept(models.NodeModel):
#    name = models.StringProperty()
#    uid = models.StringProperty(indexed=True)
#    definition = models.StringProperty()
#    related_to = models.Relationship('self',rel_type='RELATEDTO')
#    part_of = models.Relationship('self',rel_type='PARTOF')
#    is_a = models.Relationship('self',rel_type='ISA')
#    measured_by = models.Relationship(Contrast,rel_type='MEASUREDBY')
#    mentioned_in = models.Relationship('PMID',rel_type='MENTIONEDIN')

for row in concepts.iterrows():
    uid = row[1].url.split("/")[-1]
    name = row[1].term
    try:
        concept = get_concept(id=uid,silent=True).json
        definition = concept[0]["definition_text"]
    except:
        definition = ""
    properties={"definition":definition}
    node = make_node("concept",uid,name,properties)

# Assertions!
# We will store the old uid as a property, in case we need to map back to original data

for row in assertions.iterrows():
    uid = row[1].id
    user = row[1].id_user
    subject = row[1].id_subject
    relation = row[1].id_relation
    id_type = row[1].id_type
    predicate = row[1].id_predicate
    confidence_level = row[1].confidence_level # Not sure this was useful
    description = row[1].text_description
    timestamp = row[1].event_stamp
    id_subject_def = row[1].id_subject_def
    id_predicate_def = row[1].id_predicate_def
    # truth_value = row[1].truth_value # I don't know what this is
    # I am not including "flag for curator" because that wasn't very useful
    properties = {"user_id":user,"id":uid}

    # Don't add properties that aren't defined
    if str(confidence_level)!= "nan":
        properties["confidence"] = confidence_level
    if str(description)!= "nan":
        properties["description"] = description
    if str(timestamp)!= "nan":
        properties["timestamp"] = timestamp

    if id_type == "concept-task":
        tasknode = find_node("task",property_value=predicate)
        conceptnode = find_node("concept",property_value=subject)
        contrastnode = find_node("contrast",property_value=id_predicate_def)

        # Contrast is measured by contrast
        if contrastnode != None and conceptnode != None:
            relation = make_relation(contrastnode,"MEASUREDBY",conceptnode,properties)

        # We do not model task --> contrast, this comes by way of conditions (task --> conditions --> contrasts)

    elif id_type == "contrast-contrast":
        conceptnode1 = find_node("concept",property_value=subject)
        conceptnode2 = find_node("concept",property_value=predicate)

        # Figure out relationship type
        if re.search("is a part of",description):
            relationship_type = "PARTOF"
        elif re.search("is a kind of",description):
            relationship_type = "KINDOF"

        # Contrast is measured by contrast
        if conceptnode1 and conceptnode2:
            relation = make_relation(conceptnode1,relationship_type,conceptnode2)

    elif id_type == "task-task":
        tasknode1 = find_node("task",property_value=subject)
        tasknode2 = find_node("task",property_value=predicate)

        # Task is descended from task
        if tasknode1 and tasknode2:
            relation = make_relation(tasknode1,"DERIVEDFROM",tasknode2)

# The following do not exist
#class Trait(models.aNodeModel):
#    name = models.StringProperty()
#    uid = models.StringProperty(indexed=True)

#class Phenomenon(models.NodeModel):
#    name = models.StringProperty()
#    uid = models.StringProperty(indexed=True)

#class Experiment(models.NodeModel):
#    name = models.StringProperty()
#    uid = models.StringProperty(indexed=True)
<commit_msg>create relation between conditions and contrasts in database import script<commit_after># migrate_database.py: Migrate dumps of the Cognitive Atlas database into Neo4j
# see russ_ubergraph.pdf and quasi models below

from py2neo import Graph, Path, Node, Relationship, authenticate
from cognitiveatlas.api import get_task, get_concept
import pandas
import os
import re

# Function to clean up column names
# Note that Cognitive Atlas exports tables with different header delimiter than rest, I fixed this for
# the data below so all delims are ";"
def cleancolumns(df):
    df.columns = [x.strip() for x in df.columns]
    return df

# Function to make a node
def make_node(nodetype,uid,name,properties=None,property_key="id"):
    node = None
    if graph.find_one(nodetype,property_key='id', property_value=uid) == None:
        print("Creating %s:%s, %s" %(nodetype,name,uid))
        timestamp = graph.cypher.execute("RETURN timestamp()").one
        node = Node(nodetype, name=name,id=uid,creation_time=timestamp,last_updated=timestamp)
        graph.create(node)
        if properties != None:
            for property_name in properties.keys():
                node.properties[property_name] = properties[property_name]
            node.push()
    return node

# Function to make a relation
def make_relation(startnode,rel_type,endnode,properties=None):
    relation = None
    if graph.match_one(start_node=startnode, rel_type=rel_type, end_node=endnode) == None:
        relation = Relationship(startnode, rel_type, endnode)
        print("Creating relation %s [%s] %s" %(startnode.properties["name"],rel_type,endnode.properties["name"]))
        graph.create(relation)
        if properties != None:
            for property_name in properties.keys():
                relation.properties[property_name] = properties[property_name]
            relation.push()
    return relation

# Function to look up node - return None if doesn't exist
def find_node(nodetype,property_value,property_key='id'):
    try:
        node = graph.find_one(nodetype,property_key='id',property_value=property_value)
    except:
        node = None
    return node

# First we want to read in all of our data types
concepts = cleancolumns(pandas.read_csv("data/Dump_concept_2015-12-25_567dd9b28fabb.csv",sep=";"))
tasks = cleancolumns(pandas.read_csv("data/Dump_task_2015-12-25_567dd9bb7ff74.csv",sep=";"))
contrasts = cleancolumns(pandas.read_csv("data/Dump_contrast_2015-12-25_567dd9c4b46c4.csv",sep=";"))
battery = cleancolumns(pandas.read_csv("data/Dump_battery_2015-12-25_567dd9d2866fd.csv",sep=";"))
conditions = cleancolumns(pandas.read_csv("data/Dump_condition_2015-12-25_567dd9bf079bb.csv",sep=";"))
disorders = cleancolumns(pandas.read_csv("data/Dump_disorder_2015-12-26_567f21649c2cb.csv",sep=";"))
assertions = cleancolumns(pandas.read_csv("data/Dump_assertion_2015-12-25_567dd9cd411c5.csv",sep=";"))

# connect to graph database
graph = Graph("http://graphdb:7474/db/data/")

# Just for local development
#authenticate("localhost:7474", "neo4j", "noodles")
#graph = Graph()

# clear database
graph.delete_all()

tx = graph.cypher.begin()
conceptnodes={}
tasknodes={}
contrastnodes={}

#class Task(models.NodeModel):
#    name = models.StringProperty()
#    uid = models.StringProperty(indexed=True)
#    derived_from = models.Relationship('self',rel_type='DERIVEDFROM')
#    definition = models.StringProperty()
#    has_condition = models.Relationship(Condition,rel_type='HASCONDITION')
#    has_implementation = models.Relationship(Experiment,rel_type='HASIMPLEMENTATION') # ExperimentFactory?
#    mentioned_in = models.Relationship('PMID',rel_type='MENTIONEDIN')

for row in tasks.iterrows():
    name = row[1].term
    uid = row[1].url.split("/")[-1]
    try:
        task = get_task(id=uid,silent=True).json
        definition = task[0]["definition_text"]
    except:
        definition = ""
    if not str(name) =="nan":
        properties = {"definition":definition}
        node = make_node("task",uid,name,properties)

#class Condition(models.NodeModel):
#    name = models.StringProperty()
#    uid = models.StringProperty(indexed=True)
#    has_contrast = models.Relationship(Contrast,rel_type='HASCONTRAST')

for row in conditions.iterrows():
    name = row[1].condition_text
    user = row[1].id_user
    uid = row[1].id
    task = row[1].id_term
    description = row[1].condition_description
    timestamp = row[1].event_stamp
    if not str(name) =="nan":
        properties = {"description":description}
        node = make_node("condition",uid,name,properties)
        tasknode = find_node("task",property_value=task)
        # If the tasknode is node found, then we do not create the relation, and we delete the condition
        if tasknode == None:
            graph.delete(node)
        else:
            relation = make_relation(tasknode,"HASCONDITION",node)

#class Disorder(models.NodeModel):
#    has_difference = models.Relationship(Contrast,rel_type='HASDIFFERENCE')
#    mentioned_in = models.Relationship('PMID',rel_type='MENTIONEDIN')

disorders.columns = ["id","term","classification"] # Bug in returning column index name
for row in disorders.iterrows():
    uid = row[1].id
    classification = row[1].classification
    name = row[1].term
    properties = {"classification":classification}
    node = make_node("disorder",uid,name,properties)

#class Contrast(models.NodeModel):
#    name = models.StringProperty()
#    uid = models.StringProperty(indexed=True)
#    measured_by_phenomenon = models.Relationship(Phenomenon,rel_type='MEASUREDBYPHENOMENON')
#    measured_by_trait = models.Relationship(Trait,rel_type='MEASUREDBYTRAIT')

for row in contrasts.iterrows():
    uid = row[1].id
    user = row[1].id_user
    name = row[1].contrast_text
    id_term = row[1].id_term
    timestamp = row[1].event_stamp
    node = make_node("contrast",uid,name)

    # id_term in the database dumps appears to point at a task, where should
    # that relation be mapped?
    for condition in conditions.iterrows():
        if id_term == condition[1].id_term:
            condition_node = find_node("condition", property_value=condition[1].id)
            if condition_node is not None:
                make_relation(condition_node, "HASCONTRAST", node)

#class Concept(models.NodeModel):
#    name = models.StringProperty()
#    uid = models.StringProperty(indexed=True)
#    definition = models.StringProperty()
#    related_to = models.Relationship('self',rel_type='RELATEDTO')
#    part_of = models.Relationship('self',rel_type='PARTOF')
#    is_a = models.Relationship('self',rel_type='ISA')
#    measured_by = models.Relationship(Contrast,rel_type='MEASUREDBY')
#    mentioned_in = models.Relationship('PMID',rel_type='MENTIONEDIN')

for row in concepts.iterrows():
    uid = row[1].url.split("/")[-1]
    name = row[1].term
    try:
        concept = get_concept(id=uid,silent=True).json
        definition = concept[0]["definition_text"]
    except:
        definition = ""
    properties={"definition":definition}
    node = make_node("concept",uid,name,properties)

# Assertions!
# We will store the old uid as a property, in case we need to map back to original data

for row in assertions.iterrows():
    uid = row[1].id
    user = row[1].id_user
    subject = row[1].id_subject
    relation = row[1].id_relation
    id_type = row[1].id_type
    predicate = row[1].id_predicate
    confidence_level = row[1].confidence_level # Not sure this was useful
    description = row[1].text_description
    timestamp = row[1].event_stamp
    id_subject_def = row[1].id_subject_def
    id_predicate_def = row[1].id_predicate_def
    # truth_value = row[1].truth_value # I don't know what this is
    # I am not including "flag for curator" because that wasn't very useful
    properties = {"user_id":user,"id":uid}

    # Don't add properties that aren't defined
    if str(confidence_level)!= "nan":
        properties["confidence"] = confidence_level
    if str(description)!= "nan":
        properties["description"] = description
    if str(timestamp)!= "nan":
        properties["timestamp"] = timestamp

    if id_type == "concept-task":
        tasknode = find_node("task",property_value=predicate)
        conceptnode = find_node("concept",property_value=subject)
        contrastnode = find_node("contrast",property_value=id_predicate_def)

        # Contrast is measured by contrast
        if contrastnode != None and conceptnode != None:
            relation = make_relation(conceptnode,"MEASUREDBY",contrastnode,properties)

        if tasknode and conceptnode:
            make_relation(tasknode,"ASSERTS", conceptnode)
        # We do not model task --> contrast, this comes by way of conditions (task --> conditions --> contrasts)

    elif id_type == "contrast-contrast":
        conceptnode1 = find_node("concept",property_value=subject)
        conceptnode2 = find_node("concept",property_value=predicate)

        # Figure out relationship type
        if re.search("is a part of",description):
            relationship_type = "PARTOF"
        elif re.search("is a kind of",description):
            relationship_type = "KINDOF"

        # Contrast is measured by contrast
        if conceptnode1 and conceptnode2:
            relation = make_relation(conceptnode1,relationship_type,conceptnode2)

    elif id_type == "task-task":
        tasknode1 = find_node("task",property_value=subject)
        tasknode2 = find_node("task",property_value=predicate)

        # Task is descended from task
        if tasknode1 and tasknode2:
            relation = make_relation(tasknode1,"DERIVEDFROM",tasknode2)

# The following do not exist
#class Trait(models.aNodeModel):
#    name = models.StringProperty()
#    uid = models.StringProperty(indexed=True)

#class Phenomenon(models.NodeModel):
#    name = models.StringProperty()
#    uid = models.StringProperty(indexed=True)

#class Experiment(models.NodeModel):
#    name = models.StringProperty()
#    uid = models.StringProperty(indexed=True)
<|endoftext|>