<commit_before># -*- coding: utf-8 -*-
import json
import time

from mock import Mock, patch

from nailgun.settings import settings

import nailgun
import nailgun.rpc as rpc
from nailgun.task.manager import DeploymentTaskManager
from nailgun.task.fake import FAKE_THREADS
from nailgun.task.errors import WrongNodeStatus
from nailgun.test.base import BaseHandlers
from nailgun.test.base import reverse
from nailgun.api.models import Cluster, Attributes, Task, Notification, Node

class TestTaskManagers(BaseHandlers):

    def setUp(self):
        super(TestTaskManagers, self).setUp()

    def tearDown(self):
        # wait for fake task thread termination
        import threading
        for thread in threading.enumerate():
            if thread is not threading.currentThread():
                if hasattr(thread, "rude_join"):
                    timer = time.time()
                    timeout = 25
                    thread.rude_join(timeout)
                    if time.time() - timer > timeout:
                        raise Exception(
                            '{0} seconds is not enough'
                            ' - possible hanging'.format(
                                timeout
                            )
                        )

    @patch('nailgun.task.task.rpc.cast', nailgun.task.task.fake_cast)
    @patch('nailgun.task.task.settings.FAKE_TASKS', True)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_COUNT', 80)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_INTERVAL', 1)
    def test_deployment_task_managers(self):
        cluster = self.create_cluster_api()
        node1 = self.create_default_node(cluster_id=cluster['id'],
                                         status="discover",
                                         pending_addition=True)
        node2 = self.create_default_node(cluster_id=cluster['id'],
                                         status="ready",
                                         pending_addition=True)
        node3 = self.create_default_node(cluster_id=cluster['id'],
                                         pending_deletion=True)
        resp = self.app.put(
            reverse(
                'ClusterChangesHandler',
                kwargs={'cluster_id': cluster['id']}),
            headers=self.default_headers
        )
        self.assertEquals(200, resp.status)
        response = json.loads(resp.body)
        supertask_uuid = response['uuid']
        supertask = self.db.query(Task).filter_by(
            uuid=supertask_uuid
        ).first()
        self.assertEquals(supertask.name, 'deploy')
        self.assertIn(supertask.status, ('running', 'ready'))
        self.assertEquals(len(supertask.subtasks), 2)

        timer = time.time()
        timeout = 10
        while True:
            self.db.refresh(node1)
            self.db.refresh(node2)
            if node1.status in ('provisioning', 'provisioned') and \
                    node2.status == 'provisioned':
                break
            if time.time() - timer > timeout:
                raise Exception("Something wrong with the statuses")
            time.sleep(1)

        timer = time.time()
        timeout = 60
        while supertask.status == 'running':
            self.db.refresh(supertask)
            if time.time() - timer > timeout:
                raise Exception("Deployment seems to be hanged")
            time.sleep(1)
        self.db.refresh(node1)
        self.db.refresh(node2)
        self.assertEquals(node1.status, 'ready')
        self.assertEquals(node2.status, 'ready')
        self.assertEquals(node1.progress, 100)
        self.assertEquals(node2.progress, 100)
        self.assertEquals(supertask.status, 'ready')
        self.assertEquals(supertask.progress, 100)
        self.assertEquals(supertask.message, (
            "Successfully removed 1 node(s). No errors occured; "
            "Deployment of installation '{0}' is done").format(
                cluster['name']))

    @patch('nailgun.task.task.rpc.cast', nailgun.task.task.fake_cast)
    @patch('nailgun.task.task.settings.FAKE_TASKS', True)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_COUNT', 80)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_INTERVAL', 1)
    def test_deployment_fails_if_node_offline(self):
        cluster = self.create_cluster_api()
        node1 = self.create_default_node(cluster_id=cluster['id'],
                                         role="controller",
                                         status="offline",
                                         pending_addition=True)
        resp = self.app.put(
            reverse(
                'ClusterChangesHandler',
                kwargs={'cluster_id': cluster['id']}),
            headers=self.default_headers
        )
        response = json.loads(resp.body)
        supertask_uuid = response['uuid']
        supertask = self.db.query(Task).filter_by(
            uuid=supertask_uuid
        ).first()
        timer = time.time()
        timeout = 60
        while supertask.status == 'running':
            self.db.refresh(supertask)
            if time.time() - timer > timeout:
                raise Exception("Deployment seems to be hanged")
            time.sleep(1)
        self.assertEqual(supertask.status, 'error')
        self.assertEqual(
            supertask.message,
            'Cannot provision/deploy offline node'
        )

    @patch('nailgun.task.task.rpc.cast', nailgun.task.task.fake_cast)
    @patch('nailgun.task.task.settings.FAKE_TASKS', True)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_COUNT', 80)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_INTERVAL', 1)
    def test_redeployment_works(self):
        cluster = self.create_cluster_api(mode="ha")
        node1 = self.create_default_node(cluster_id=cluster['id'],
                                         role="controller",
                                         pending_addition=True)
        node2 = self.create_default_node(cluster_id=cluster['id'],
                                         role="compute",
                                         pending_addition=True)
        resp = self.app.put(
            reverse(
                'ClusterChangesHandler',
                kwargs={'cluster_id': cluster['id']}),
            headers=self.default_headers
        )

        response = json.loads(resp.body)
        supertask_uuid = response['uuid']
        supertask = self.db.query(Task).filter_by(
            uuid=supertask_uuid
        ).first()

        timer = time.time()
        timeout = 60
        while supertask.status == 'running':
            self.db.refresh(supertask)
            if time.time() - timer > timeout:
                raise Exception("First deployment seems to be hanged")
            time.sleep(1)
        self.db.refresh(node1)
        self.db.refresh(node2)
        self.assertEquals(node1.status, 'ready')
        self.assertEquals(node2.status, 'ready')
        self.assertEquals(node1.progress, 100)
        self.assertEquals(node2.progress, 100)
        self.assertEquals(supertask.status, 'ready')
        self.assertEquals(supertask.progress, 100)

        node3 = self.create_default_node(cluster_id=cluster['id'],
                                         role="controller",
                                         pending_addition=True)

        resp = self.app.put(
            reverse(
                'ClusterChangesHandler',
                kwargs={'cluster_id': cluster['id']}),
            headers=self.default_headers
        )
        response = json.loads(resp.body)
        supertask_uuid = response['uuid']
        supertask = self.db.query(Task).filter_by(
            uuid=supertask_uuid
        ).first()

        timer = time.time()
        timeout = 60
        while supertask.status == 'running':
            self.db.refresh(supertask)
            if time.time() - timer > timeout:
                raise Exception("Second deployment seems to be hanged")
            time.sleep(1)
        self.db.refresh(node1)
        self.db.refresh(node2)
        self.db.refresh(node3)
        self.assertEquals(node1.status, 'ready')
        self.assertEquals(node2.status, 'ready')
        self.assertEquals(node3.status, 'ready')
        self.assertEquals(node1.progress, 100)
        self.assertEquals(node2.progress, 100)
        self.assertEquals(node3.progress, 100)
        self.assertEquals(supertask.status, 'ready')
        self.assertEquals(supertask.progress, 100)

    @patch('nailgun.task.task.rpc.cast', nailgun.task.task.fake_cast)
    @patch('nailgun.task.task.settings.FAKE_TASKS', True)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_COUNT', 80)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_INTERVAL', 1)
    def test_network_verify_task_managers(self):
        cluster = self.create_cluster_api()
        node1 = self.create_default_node(cluster_id=cluster['id'])
        node2 = self.create_default_node(cluster_id=cluster['id'])
        resp = self.app.put(
            reverse(
                'ClusterNetworksHandler',
                kwargs={'cluster_id': cluster['id']}),
            headers=self.default_headers
        )
        self.assertEquals(200, resp.status)
        response = json.loads(resp.body)
        task_uuid = response['uuid']
        task = self.db.query(Task).filter_by(uuid=task_uuid).first()
        self.assertEquals(task.name, 'verify_networks')
        self.assertIn(task.status, ('running', 'ready'))

    @patch('nailgun.task.task.rpc.cast', nailgun.task.task.fake_cast)
    @patch('nailgun.task.task.settings.FAKE_TASKS', True)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_COUNT', 80)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_INTERVAL', 1)
    def test_deletion_empty_cluster_task_manager(self):
        cluster = self.create_cluster_api()
        resp = self.app.delete(
            reverse(
                'ClusterHandler',
                kwargs={'cluster_id': cluster['id']}),
            headers=self.default_headers
        )
        self.assertEquals(202, resp.status)

        notification = self.db.query(Notification)\
            .filter(Notification.topic == "done")\
            .filter(Notification.message == "Cluster %s and all cluster "
                    "nodes are deleted" % cluster["name"])
        self.assertIsNotNone(notification)

        timer = time.time()
        timeout = 25
        clstr = self.db.query(Cluster).get(cluster["id"])
        while clstr:
            time.sleep(1)
            clstr = self.db.query(Cluster).get(cluster["id"])
            if time.time() - timer > timeout:
                raise Exception("Cluster deletion seems to be hanged")

        tasks = self.db.query(Task).all()
        self.assertEqual(tasks, [])

    @patch('nailgun.task.task.rpc.cast', nailgun.task.task.fake_cast)
    @patch('nailgun.task.task.settings.FAKE_TASKS', True)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_COUNT', 80)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_INTERVAL', 1)
    def test_deletion_cluster_task_manager(self):
        cluster = self.create_cluster_api()
        node1 = self.create_default_node(cluster_id=cluster['id'],
                                         role="controller",
                                         status="ready",
                                         progress=100)
        node2 = self.create_default_node(cluster_id=cluster['id'],
                                         role="compute",
                                         status="ready",
                                         progress=100)
        node3 = self.create_default_node(cluster_id=cluster['id'],
                                         role="compute",
                                         pending_addition=True)
        nodes_ids = [node1.id, node2.id, node3.id]
        resp = self.app.delete(
            reverse(
                'ClusterHandler',
                kwargs={'cluster_id': cluster['id']}),
            headers=self.default_headers
        )
        self.assertEquals(202, resp.status)

        notification = self.db.query(Notification)\
            .filter(Notification.topic == "done")\
            .filter(Notification.message == "Cluster %s and all cluster "
                    "nodes are deleted" % cluster["name"])
        self.assertIsNotNone(notification)

        timer = time.time()
        timeout = 15
        clstr = self.db.query(Cluster).get(cluster["id"])
        while clstr:
            time.sleep(1)
            clstr = self.db.query(Cluster).get(cluster["id"])
            if time.time() - timer > timeout:
                raise Exception("Cluster deletion seems to be hanged")

        tasks = self.db.query(Task).all()
        self.assertEqual(tasks, [])

    @patch('nailgun.task.task.rpc.cast', nailgun.task.task.fake_cast)
    @patch('nailgun.task.task.settings.FAKE_TASKS', True)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_COUNT', 80)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_INTERVAL', 1)
    def test_deletion_during_deployment(self):
        cluster = self.create_cluster_api()
        node1 = self.create_default_node(cluster_id=cluster['id'],
                                         pending_addition=True)
        resp = self.app.put(
            reverse(
                'ClusterChangesHandler',
                kwargs={'cluster_id': cluster['id']}),
            headers=self.default_headers
        )
        deploy_uuid = json.loads(resp.body)['uuid']
        resp = self.app.delete(
            reverse(
                'ClusterHandler',
                kwargs={'cluster_id': cluster['id']}),
            headers=self.default_headers
        )
        timeout = 120
        timer = time.time()
        while True:
            task_deploy = self.db.query(Task).filter_by(
                uuid=deploy_uuid
            ).first()
            task_delete = self.db.query(Task).filter_by(
                cluster_id=cluster['id'],
                name="cluster_deletion"
            ).first()
            if not task_delete:
                break
            self.db.expire(task_deploy)
            self.db.expire(task_delete)
            if (time.time() - timer) > timeout:
                break
            time.sleep(0.24)

        cluster_db = self.db.query(Cluster).get(cluster['id'])
        self.assertIsNone(cluster_db)

    @patch('nailgun.task.task.rpc.cast', nailgun.task.task.fake_cast)
    @patch('nailgun.task.task.settings.FAKE_TASKS', True)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_COUNT', 80)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_INTERVAL', 1)
    def test_node_fqdn_is_assigned(self):
        cluster = self.create_cluster_api()
        node1 = self.create_default_node(cluster_id=cluster['id'],
                                         pending_addition=True)
        node2 = self.create_default_node(cluster_id=cluster['id'],
                                         pending_addition=True)
        resp = self.app.put(
            reverse(
                'ClusterChangesHandler',
                kwargs={'cluster_id': cluster['id']}),
            headers=self.default_headers
        )
        self.assertEquals(200, resp.status)
        nodes = self.db.query(Node).all()
        for node in (node1, node2):
            self.db.refresh(node)
            fqdn = "slave-%s.%s" % (node.id, settings.DNS_DOMAIN)
            self.assertEquals(fqdn, node.fqdn)

    @patch('nailgun.task.task.rpc.cast', nailgun.task.task.fake_cast)
    @patch('nailgun.task.task.settings.FAKE_TASKS', True)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_COUNT', 80)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_INTERVAL', 1)
    def test_no_node_no_cry(self):
        cluster = self.create_cluster_api()
        rcvr = rpc.receiver.NailgunReceiver
        manager = DeploymentTaskManager(cluster["id"])
        rcvr.deploy_resp(nodes=[
            {'uid': 666, 'id': 666, 'status': 'discover'}
        ], uuid='no_freaking_way')  # and wrong task also
        self.assertRaises(WrongNodeStatus, manager.execute)
<commit_msg>cluster deletion check via refresh<commit_after># -*- coding: utf-8 -*-
import json
import time

from mock import Mock, patch

from nailgun.settings import settings

import nailgun
import nailgun.rpc as rpc
from nailgun.task.manager import DeploymentTaskManager
from nailgun.task.fake import FAKE_THREADS
from nailgun.task.errors import WrongNodeStatus
from nailgun.test.base import BaseHandlers
from nailgun.test.base import reverse
from nailgun.api.models import Cluster, Attributes, Task, Notification, Node

class TestTaskManagers(BaseHandlers):

    def setUp(self):
        super(TestTaskManagers, self).setUp()

    def tearDown(self):
        # wait for fake task thread termination
        import threading
        for thread in threading.enumerate():
            if thread is not threading.currentThread():
                if hasattr(thread, "rude_join"):
                    timer = time.time()
                    timeout = 25
                    thread.rude_join(timeout)
                    if time.time() - timer > timeout:
                        raise Exception(
                            '{0} seconds is not enough'
                            ' - possible hanging'.format(
                                timeout
                            )
                        )

    @patch('nailgun.task.task.rpc.cast', nailgun.task.task.fake_cast)
    @patch('nailgun.task.task.settings.FAKE_TASKS', True)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_COUNT', 80)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_INTERVAL', 1)
    def test_deployment_task_managers(self):
        cluster = self.create_cluster_api()
        node1 = self.create_default_node(cluster_id=cluster['id'],
                                         status="discover",
                                         pending_addition=True)
        node2 = self.create_default_node(cluster_id=cluster['id'],
                                         status="ready",
                                         pending_addition=True)
        node3 = self.create_default_node(cluster_id=cluster['id'],
                                         pending_deletion=True)
        resp = self.app.put(
            reverse(
                'ClusterChangesHandler',
                kwargs={'cluster_id': cluster['id']}),
            headers=self.default_headers
        )
        self.assertEquals(200, resp.status)
        response = json.loads(resp.body)
        supertask_uuid = response['uuid']
        supertask = self.db.query(Task).filter_by(
            uuid=supertask_uuid
        ).first()
        self.assertEquals(supertask.name, 'deploy')
        self.assertIn(supertask.status, ('running', 'ready'))
        self.assertEquals(len(supertask.subtasks), 2)

        timer = time.time()
        timeout = 10
        while True:
            self.db.refresh(node1)
            self.db.refresh(node2)
            if node1.status in ('provisioning', 'provisioned') and \
                    node2.status == 'provisioned':
                break
            if time.time() - timer > timeout:
                raise Exception("Something wrong with the statuses")
            time.sleep(1)

        timer = time.time()
        timeout = 60
        while supertask.status == 'running':
            self.db.refresh(supertask)
            if time.time() - timer > timeout:
                raise Exception("Deployment seems to be hanged")
            time.sleep(1)
        self.db.refresh(node1)
        self.db.refresh(node2)
        self.assertEquals(node1.status, 'ready')
        self.assertEquals(node2.status, 'ready')
        self.assertEquals(node1.progress, 100)
        self.assertEquals(node2.progress, 100)
        self.assertEquals(supertask.status, 'ready')
        self.assertEquals(supertask.progress, 100)
        self.assertEquals(supertask.message, (
            "Successfully removed 1 node(s). No errors occured; "
            "Deployment of installation '{0}' is done").format(
                cluster['name']))

    @patch('nailgun.task.task.rpc.cast', nailgun.task.task.fake_cast)
    @patch('nailgun.task.task.settings.FAKE_TASKS', True)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_COUNT', 80)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_INTERVAL', 1)
    def test_deployment_fails_if_node_offline(self):
        cluster = self.create_cluster_api()
        node1 = self.create_default_node(cluster_id=cluster['id'],
                                         role="controller",
                                         status="offline",
                                         pending_addition=True)
        resp = self.app.put(
            reverse(
                'ClusterChangesHandler',
                kwargs={'cluster_id': cluster['id']}),
            headers=self.default_headers
        )
        response = json.loads(resp.body)
        supertask_uuid = response['uuid']
        supertask = self.db.query(Task).filter_by(
            uuid=supertask_uuid
        ).first()
        timer = time.time()
        timeout = 60
        while supertask.status == 'running':
            self.db.refresh(supertask)
            if time.time() - timer > timeout:
                raise Exception("Deployment seems to be hanged")
            time.sleep(1)
        self.assertEqual(supertask.status, 'error')
        self.assertEqual(
            supertask.message,
            'Cannot provision/deploy offline node'
        )

    @patch('nailgun.task.task.rpc.cast', nailgun.task.task.fake_cast)
    @patch('nailgun.task.task.settings.FAKE_TASKS', True)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_COUNT', 80)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_INTERVAL', 1)
    def test_redeployment_works(self):
        cluster = self.create_cluster_api(mode="ha")
        node1 = self.create_default_node(cluster_id=cluster['id'],
                                         role="controller",
                                         pending_addition=True)
        node2 = self.create_default_node(cluster_id=cluster['id'],
                                         role="compute",
                                         pending_addition=True)
        resp = self.app.put(
            reverse(
                'ClusterChangesHandler',
                kwargs={'cluster_id': cluster['id']}),
            headers=self.default_headers
        )

        response = json.loads(resp.body)
        supertask_uuid = response['uuid']
        supertask = self.db.query(Task).filter_by(
            uuid=supertask_uuid
        ).first()

        timer = time.time()
        timeout = 60
        while supertask.status == 'running':
            self.db.refresh(supertask)
            if time.time() - timer > timeout:
                raise Exception("First deployment seems to be hanged")
            time.sleep(1)
        self.db.refresh(node1)
        self.db.refresh(node2)
        self.assertEquals(node1.status, 'ready')
        self.assertEquals(node2.status, 'ready')
        self.assertEquals(node1.progress, 100)
        self.assertEquals(node2.progress, 100)
        self.assertEquals(supertask.status, 'ready')
        self.assertEquals(supertask.progress, 100)

        node3 = self.create_default_node(cluster_id=cluster['id'],
                                         role="controller",
                                         pending_addition=True)

        resp = self.app.put(
            reverse(
                'ClusterChangesHandler',
                kwargs={'cluster_id': cluster['id']}),
            headers=self.default_headers
        )
        response = json.loads(resp.body)
        supertask_uuid = response['uuid']
        supertask = self.db.query(Task).filter_by(
            uuid=supertask_uuid
        ).first()

        timer = time.time()
        timeout = 60
        while supertask.status == 'running':
            self.db.refresh(supertask)
            if time.time() - timer > timeout:
                raise Exception("Second deployment seems to be hanged")
            time.sleep(1)
        self.db.refresh(node1)
        self.db.refresh(node2)
        self.db.refresh(node3)
        self.assertEquals(node1.status, 'ready')
        self.assertEquals(node2.status, 'ready')
        self.assertEquals(node3.status, 'ready')
        self.assertEquals(node1.progress, 100)
        self.assertEquals(node2.progress, 100)
        self.assertEquals(node3.progress, 100)
        self.assertEquals(supertask.status, 'ready')
        self.assertEquals(supertask.progress, 100)

    @patch('nailgun.task.task.rpc.cast', nailgun.task.task.fake_cast)
    @patch('nailgun.task.task.settings.FAKE_TASKS', True)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_COUNT', 80)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_INTERVAL', 1)
    def test_network_verify_task_managers(self):
        cluster = self.create_cluster_api()
        node1 = self.create_default_node(cluster_id=cluster['id'])
        node2 = self.create_default_node(cluster_id=cluster['id'])
        resp = self.app.put(
            reverse(
                'ClusterNetworksHandler',
                kwargs={'cluster_id': cluster['id']}),
            headers=self.default_headers
        )
        self.assertEquals(200, resp.status)
        response = json.loads(resp.body)
        task_uuid = response['uuid']
        task = self.db.query(Task).filter_by(uuid=task_uuid).first()
        self.assertEquals(task.name, 'verify_networks')
        self.assertIn(task.status, ('running', 'ready'))

    @patch('nailgun.task.task.rpc.cast', nailgun.task.task.fake_cast)
    @patch('nailgun.task.task.settings.FAKE_TASKS', True)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_COUNT', 80)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_INTERVAL', 1)
    def test_deletion_empty_cluster_task_manager(self):
        cluster = self.create_cluster_api()
        resp = self.app.delete(
            reverse(
                'ClusterHandler',
                kwargs={'cluster_id': cluster['id']}),
            headers=self.default_headers
        )
        self.assertEquals(202, resp.status)

        notification = self.db.query(Notification)\
            .filter(Notification.topic == "done")\
            .filter(Notification.message == "Cluster %s and all cluster "
                    "nodes are deleted" % cluster["name"])
        self.assertIsNotNone(notification)

        timer = time.time()
        timeout = 15
        clstr = self.db.query(Cluster).get(cluster["id"])
        while clstr:
            time.sleep(1)
            try:
                self.db.refresh(clstr)
            except:
                break
            if time.time() - timer > timeout:
                raise Exception("Cluster deletion seems to be hanged")

        tasks = self.db.query(Task).all()
        self.assertEqual(tasks, [])

    @patch('nailgun.task.task.rpc.cast', nailgun.task.task.fake_cast)
    @patch('nailgun.task.task.settings.FAKE_TASKS', True)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_COUNT', 80)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_INTERVAL', 1)
    def test_deletion_cluster_task_manager(self):
        cluster = self.create_cluster_api()
        node1 = self.create_default_node(cluster_id=cluster['id'],
                                         role="controller",
                                         status="ready",
                                         progress=100)
        node2 = self.create_default_node(cluster_id=cluster['id'],
                                         role="compute",
                                         status="ready",
                                         progress=100)
        node3 = self.create_default_node(cluster_id=cluster['id'],
                                         role="compute",
                                         pending_addition=True)
        nodes_ids = [node1.id, node2.id, node3.id]
        resp = self.app.delete(
            reverse(
                'ClusterHandler',
                kwargs={'cluster_id': cluster['id']}),
            headers=self.default_headers
        )
        self.assertEquals(202, resp.status)

        notification = self.db.query(Notification)\
            .filter(Notification.topic == "done")\
            .filter(Notification.message == "Cluster %s and all cluster "
                    "nodes are deleted" % cluster["name"])
        self.assertIsNotNone(notification)

        timer = time.time()
        timeout = 15
        clstr = self.db.query(Cluster).get(cluster["id"])
        while clstr:
            time.sleep(1)
            clstr = self.db.query(Cluster).get(cluster["id"])
            if time.time() - timer > timeout:
                raise Exception("Cluster deletion seems to be hanged")

        tasks = self.db.query(Task).all()
        self.assertEqual(tasks, [])

    @patch('nailgun.task.task.rpc.cast', nailgun.task.task.fake_cast)
    @patch('nailgun.task.task.settings.FAKE_TASKS', True)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_COUNT', 80)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_INTERVAL', 1)
    def test_deletion_during_deployment(self):
        cluster = self.create_cluster_api()
        node1 = self.create_default_node(cluster_id=cluster['id'],
                                         pending_addition=True)
        resp = self.app.put(
            reverse(
                'ClusterChangesHandler',
                kwargs={'cluster_id': cluster['id']}),
            headers=self.default_headers
        )
        deploy_uuid = json.loads(resp.body)['uuid']
        resp = self.app.delete(
            reverse(
                'ClusterHandler',
                kwargs={'cluster_id': cluster['id']}),
            headers=self.default_headers
        )
        timeout = 120
        timer = time.time()
        while True:
            task_deploy = self.db.query(Task).filter_by(
                uuid=deploy_uuid
            ).first()
            task_delete = self.db.query(Task).filter_by(
                cluster_id=cluster['id'],
                name="cluster_deletion"
            ).first()
            if not task_delete:
                break
            self.db.expire(task_deploy)
            self.db.expire(task_delete)
            if (time.time() - timer) > timeout:
                break
            time.sleep(0.24)

        cluster_db = self.db.query(Cluster).get(cluster['id'])
        self.assertIsNone(cluster_db)

    @patch('nailgun.task.task.rpc.cast', nailgun.task.task.fake_cast)
    @patch('nailgun.task.task.settings.FAKE_TASKS', True)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_COUNT', 80)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_INTERVAL', 1)
    def test_node_fqdn_is_assigned(self):
        cluster = self.create_cluster_api()
        node1 = self.create_default_node(cluster_id=cluster['id'],
                                         pending_addition=True)
        node2 = self.create_default_node(cluster_id=cluster['id'],
                                         pending_addition=True)
        resp = self.app.put(
            reverse(
                'ClusterChangesHandler',
                kwargs={'cluster_id': cluster['id']}),
            headers=self.default_headers
        )
        self.assertEquals(200, resp.status)
        nodes = self.db.query(Node).all()
        for node in (node1, node2):
            self.db.refresh(node)
            fqdn = "slave-%s.%s" % (node.id, settings.DNS_DOMAIN)
            self.assertEquals(fqdn, node.fqdn)

    @patch('nailgun.task.task.rpc.cast', nailgun.task.task.fake_cast)
    @patch('nailgun.task.task.settings.FAKE_TASKS', True)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_COUNT', 80)
    @patch('nailgun.task.fake.settings.FAKE_TASKS_TICK_INTERVAL', 1)
    def test_no_node_no_cry(self):
        cluster = self.create_cluster_api()
        rcvr = rpc.receiver.NailgunReceiver
        manager = DeploymentTaskManager(cluster["id"])
        rcvr.deploy_resp(nodes=[
            {'uid': 666, 'id': 666, 'status': 'discover'}
        ], uuid='no_freaking_way')  # and wrong task also
        self.assertRaises(WrongNodeStatus, manager.execute)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Algorithms for chordal graphs.

A graph is chordal if every cycle of length at least 4 has a chord
(an edge joining two nodes not adjacent in the cycle).
http://en.wikipedia.org/wiki/Chordal_graph

"""
import networkx as nx
import random
import sys

__authors__ = "\n".join(['Jesus Cerquides <cerquide@iiia.csic.es>'])
#    Copyright (C) 2010 by
#    Jesus Cerquides <cerquide@iiia.csic.es>
#    All rights reserved.
#    BSD license.

__all__ = ['is_chordal',
           'find_induced_nodes',
           'chordal_graph_cliques',
           'chordal_graph_treewidth',
           'NetworkXTreewidthBoundExceeded']

class NetworkXTreewidthBoundExceeded(nx.NetworkXException):
    """Exception raised when a treewidth bound has been provided and it has
    been exceeded"""

def is_chordal(G):
    """Checks whether G is a chordal graph.

    A graph is chordal if every cycle of length at least 4 has a chord
    (an edge joining two nodes not adjacent in the cycle).

    Parameters
    ----------
    G : NetworkX graph

    Returns
    -------
    chordal : boolean.
	True if G is a chordal graph and False otherwise

    Raises
    ------
    NetworkXError
        The algorithm does not support DiGraph, MultiGraph and MultiDiGraph.
        If the input graph is an instance of one of these classes, a
        NetworkXError is raised.

    Examples
    --------
    >>> import networkx as nx
    >>> e=[(1,2),(1,3),(2,3),(2,4),(3,4),(3,5),(3,6),(4,5),(4,6),(5,6)]
    >>> G=nx.Graph(e)
    >>> nx.is_chordal(G)
    True

    Notes
    -----
    The routine tries to go through every node following maximum cardinality
    search. It returns False when it founds that the separator for any node
    is not a clique.  Based on the algorithms in [1]_.

    References
    ----------
    .. [1] R. E. Tarjan and M. Yannakakis, Simple linear-time algorithms
       to test chordality of graphs, test acyclicity of hypergraphs, and
       selectively reduce acyclic hypergraphs, SIAM J. Comput., 13 (1984),
       pp. 566–579.
    """
    if G.is_directed():
        raise nx.NetworkXError('Directed graphs not supported')
    if G.is_multigraph():
        raise nx.NetworkXError('Multiply connected graphs not supported.')
    if len(_find_chordality_breaker(G))==0:
        return True
    else:
        return False

def find_induced_nodes(G,s,t,treewidth_bound=sys.maxint):
    """Returns the set of induced nodes in the path from s to t.

    Parameters
    ----------
    G : NetworkX graph
	Chordal graph.

    s : node
	Source node to look for induced nodes

    t : node
	Destination node to look for induced nodes

    treewith_bound: float
        Maximum treewidth acceptable for the graph H. The search
        for induced nodes will end as soon as the treewidth_bound is exceeded.

    Returns
    -------
    I : Set of nodes
	The set of induced nodes in the path from s to t in G

    Raises
    ------
    NetworkXError
        The algorithm does not support DiGraph, MultiGraph and MultiDiGraph.
        If the input graph is an instance of one of these classes, a
        NetworkXError is raised.
        The algorithm can only be applied to chordal graphs. If
        the input graph is found to be non-chordal, a NetworkXError is raised.

    Examples
    --------
    >>> import networkx as nx
    >>> G=nx.Graph()
    >>> G = nx.generators.classic.path_graph(10)
    >>> I = nx.find_induced_nodes(G,1,9,2)
    >>> list(I)
    [1, 2, 3, 4, 5, 6, 7, 8, 9]

    Notes
    -----
    G must be a chordal graph and (s,t) an edge that is not in G.

    If a treewidth_bound is provided, the search for induced nodes will end
    as soon as the treewidth_bound is exceeded.

    The algorithm is inspired by Algorithm 4 in [1]_.
    A formal definition of induced node can also be found on that reference.

    References
    ----------
    .. [1] Learning Bounded Treewidth Bayesian Networks.
       Gal Elidan, Stephen Gould; JMLR, 9(Dec):2699--2731, 2008.
       http://jmlr.csail.mit.edu/papers/volume9/elidan08a/elidan08a.pdf
    """
    if not is_chordal(G):
        raise nx.NetworkXError("Input graph is not chordal.")

    H = nx.Graph(G)
    H.add_edge(s,t)
    I = set()
    triplet =  _find_chordality_breaker(H,s,treewidth_bound)
    while triplet:
        (u,v,w) = triplet
        I.update(triplet)
        for n in triplet:
            if n!=s:
                H.add_edge(s,n)
        triplet =  _find_chordality_breaker(H,s,treewidth_bound)
    if I:
        # Add t and the second node in the induced path from s to t.
        I.add(t)
        for u in G[s]:
            if len(I & set(G[u]))==2:
                I.add(u)
                break
    return I

def chordal_graph_cliques(G):
    """Returns the set of maximal cliques of a chordal graph.

    The algorithm breaks the graph in connected components and performs a
    maximum cardinality search in each component to get the cliques.

    Parameters
    ----------
    G: NetworkX graph

    Returns
    -------
    cliques: A set containing the maximal cliques in G.

    Raises
    ------
    NetworkXError
        The algorithm does not support DiGraph, MultiGraph and MultiDiGraph.
        If the input graph is an instance of one of these classes, a
        NetworkXError is raised.
        The algorithm can only be applied to chordal graphs. If the
        input graph is found to be non-chordal, a NetworkXError is raised.

    Examples
    --------
    >>> import networkx as nx
    >>> e= [(1,2),(1,3),(2,3),(2,4),(3,4),(3,5),(3,6),(4,5),(4,6),(5,6),(7,8)]
    >>> G = nx.Graph(e)
    >>> G.add_node(9)
    >>> setlist = nx.chordal_graph_cliques(G)
    """
    if not is_chordal(G):
        raise nx.NetworkXError("Input graph is not chordal.")

    cliques = set()
    for C in nx.connected.connected_component_subgraphs(G):
        cliques |= _connected_chordal_graph_cliques(C)

    return cliques

def chordal_graph_treewidth(G):
    """Returns the treewidth of the chordal graph G.

    Parameters
    ----------
    G : NetworkX graph
	The chordal graph which we want to measure its treewidth.

    Returns
    -------
    treewidth : int
	The size of the largest clique in the graph minus one.

    Raises
    ------
    NetworkXError
        The algorithm does not support DiGraph, MultiGraph and MultiDiGraph.
        If the input graph is an instance of one of these classes, a
        NetworkXError is raised.
        The algorithm can only be applied to chordal graphs. If
        the input graph is found to be non-chordal, a NetworkXError is raised.

    Examples
    --------
    >>> import networkx as nx
    >>> e = [(1,2),(1,3),(2,3),(2,4),(3,4),(3,5),(3,6),(4,5),(4,6),(5,6),(7,8)]
    >>> G = nx.Graph(e)
    >>> G.add_node(9)
    >>> nx.chordal_graph_treewidth(G)
    3

    References
    ----------
    .. [1] http://en.wikipedia.org/wiki/Tree_decomposition#Treewidth
    """
    if not is_chordal(G):
        raise nx.NetworkXError("Input graph is not chordal.")

    max_clique = -1
    for clique in nx.chordal_graph_cliques(G):
        max_clique = max(max_clique,len(clique))
    return max_clique - 1

def _is_complete_graph(G):
    """Returns True if G is a complete graph."""
    if G.number_of_selfloops()>0:
        raise nx.NetworkXError("Self loop found in _is_complete_graph()")
    n = G.number_of_nodes()
    if n < 2:
        return True
    e = G.number_of_edges()
    max_edges = ((n * (n-1))/2)
    return e == max_edges

def _find_missing_edge(G):
    """ Given a non-complete graph G, returns a missing edge."""
    nodes=set(G)
    for u in G:
        missing=nodes-set(list(G[u].keys())+[u])
        if missing:
            return (u,missing.pop())

def _max_cardinality_node(G,choices,wanna_connect):
    """Returns a the node in choices that has more connections in G
    to nodes in wanna_connect.
    """
#    max_number = None
    max_number = -1
    for x in choices:
        number=len([y for y in G[x] if y in wanna_connect])
        if number > max_number:
            max_number = number
            max_cardinality_node = x
    return max_cardinality_node

def _find_chordality_breaker(G,s=None,treewidth_bound=sys.maxint):
    """ Given a graph G, starts a max cardinality search
    (starting from s if s is given and from a random node otherwise)
    trying to find a non-chordal cycle.

    If it does find one, it returns (u,v,w) where u,v,w are the three
    nodes that together with s are involved in the cycle.
    """

    unnumbered = set(G)
    if s is None:
        s = random.choice(list(unnumbered))
    unnumbered.remove(s)
    numbered = set([s])
#    current_treewidth = None
    current_treewidth = -1
    while unnumbered:# and current_treewidth <= treewidth_bound:
        v = _max_cardinality_node(G,unnumbered,numbered)
        unnumbered.remove(v)
        numbered.add(v)
        clique_wanna_be = set(G[v]) & numbered
        sg = G.subgraph(clique_wanna_be)
        if _is_complete_graph(sg):
            # The graph seems to be chordal by now. We update the treewidth
            current_treewidth = max(current_treewidth,len(clique_wanna_be))
            if current_treewidth > treewidth_bound:
                raise nx.NetworkXTreewidthBoundExceeded(\
                    "treewidth_bound exceeded: %s"%current_treewidth)
        else:
            # sg is not a clique,
            # look for an edge that is not included in sg
            (u,w) = _find_missing_edge(sg)
            return (u,v,w)
    return ()

def _connected_chordal_graph_cliques(G):
    """Return the set of maximal cliques of a connected chordal graph."""
    if G.number_of_nodes() == 1:
        x = frozenset(G.nodes())
        return set([x])
    else:
        cliques = set()
        unnumbered = set(G.nodes())
        v = random.choice(list(unnumbered))
        unnumbered.remove(v)
        numbered = set([v])
        clique_wanna_be = set([v])
        while unnumbered:
            v = _max_cardinality_node(G,unnumbered,numbered)
            unnumbered.remove(v)
            numbered.add(v)
            new_clique_wanna_be = set(G.neighbors(v)) & numbered
            sg = G.subgraph(clique_wanna_be)
            if _is_complete_graph(sg):
                new_clique_wanna_be.add(v)
                if not new_clique_wanna_be >= clique_wanna_be:
                    cliques.add(frozenset(clique_wanna_be))
                clique_wanna_be = new_clique_wanna_be
            else:
                raise nx.NetworkXError("Input graph is not chordal.")
        cliques.add(frozenset(clique_wanna_be))
        return cliques

<commit_msg>Use maxsize instead of maxint. Addresses #417<commit_after># -*- coding: utf-8 -*-
"""
Algorithms for chordal graphs.

A graph is chordal if every cycle of length at least 4 has a chord
(an edge joining two nodes not adjacent in the cycle).
http://en.wikipedia.org/wiki/Chordal_graph

"""
import networkx as nx
import random
import sys

__authors__ = "\n".join(['Jesus Cerquides <cerquide@iiia.csic.es>'])
#    Copyright (C) 2010 by
#    Jesus Cerquides <cerquide@iiia.csic.es>
#    All rights reserved.
#    BSD license.

__all__ = ['is_chordal',
           'find_induced_nodes',
           'chordal_graph_cliques',
           'chordal_graph_treewidth',
           'NetworkXTreewidthBoundExceeded']

class NetworkXTreewidthBoundExceeded(nx.NetworkXException):
    """Exception raised when a treewidth bound has been provided and it has
    been exceeded"""

def is_chordal(G):
    """Checks whether G is a chordal graph.

    A graph is chordal if every cycle of length at least 4 has a chord
    (an edge joining two nodes not adjacent in the cycle).

    Parameters
    ----------
    G : NetworkX graph

    Returns
    -------
    chordal : boolean.
	True if G is a chordal graph and False otherwise

    Raises
    ------
    NetworkXError
        The algorithm does not support DiGraph, MultiGraph and MultiDiGraph.
        If the input graph is an instance of one of these classes, a
        NetworkXError is raised.

    Examples
    --------
    >>> import networkx as nx
    >>> e=[(1,2),(1,3),(2,3),(2,4),(3,4),(3,5),(3,6),(4,5),(4,6),(5,6)]
    >>> G=nx.Graph(e)
    >>> nx.is_chordal(G)
    True

    Notes
    -----
    The routine tries to go through every node following maximum cardinality
    search. It returns False when it founds that the separator for any node
    is not a clique.  Based on the algorithms in [1]_.

    References
    ----------
    .. [1] R. E. Tarjan and M. Yannakakis, Simple linear-time algorithms
       to test chordality of graphs, test acyclicity of hypergraphs, and
       selectively reduce acyclic hypergraphs, SIAM J. Comput., 13 (1984),
       pp. 566–579.
    """
    if G.is_directed():
        raise nx.NetworkXError('Directed graphs not supported')
    if G.is_multigraph():
        raise nx.NetworkXError('Multiply connected graphs not supported.')
    if len(_find_chordality_breaker(G))==0:
        return True
    else:
        return False

def find_induced_nodes(G,s,t,treewidth_bound=sys.maxsize):
    """Returns the set of induced nodes in the path from s to t.

    Parameters
    ----------
    G : NetworkX graph
	Chordal graph.

    s : node
	Source node to look for induced nodes

    t : node
	Destination node to look for induced nodes

    treewith_bound: float
        Maximum treewidth acceptable for the graph H. The search
        for induced nodes will end as soon as the treewidth_bound is exceeded.

    Returns
    -------
    I : Set of nodes
	The set of induced nodes in the path from s to t in G

    Raises
    ------
    NetworkXError
        The algorithm does not support DiGraph, MultiGraph and MultiDiGraph.
        If the input graph is an instance of one of these classes, a
        NetworkXError is raised.
        The algorithm can only be applied to chordal graphs. If
        the input graph is found to be non-chordal, a NetworkXError is raised.

    Examples
    --------
    >>> import networkx as nx
    >>> G=nx.Graph()
    >>> G = nx.generators.classic.path_graph(10)
    >>> I = nx.find_induced_nodes(G,1,9,2)
    >>> list(I)
    [1, 2, 3, 4, 5, 6, 7, 8, 9]

    Notes
    -----
    G must be a chordal graph and (s,t) an edge that is not in G.

    If a treewidth_bound is provided, the search for induced nodes will end
    as soon as the treewidth_bound is exceeded.

    The algorithm is inspired by Algorithm 4 in [1]_.
    A formal definition of induced node can also be found on that reference.

    References
    ----------
    .. [1] Learning Bounded Treewidth Bayesian Networks.
       Gal Elidan, Stephen Gould; JMLR, 9(Dec):2699--2731, 2008.
       http://jmlr.csail.mit.edu/papers/volume9/elidan08a/elidan08a.pdf
    """
    if not is_chordal(G):
        raise nx.NetworkXError("Input graph is not chordal.")

    H = nx.Graph(G)
    H.add_edge(s,t)
    I = set()
    triplet =  _find_chordality_breaker(H,s,treewidth_bound)
    while triplet:
        (u,v,w) = triplet
        I.update(triplet)
        for n in triplet:
            if n!=s:
                H.add_edge(s,n)
        triplet =  _find_chordality_breaker(H,s,treewidth_bound)
    if I:
        # Add t and the second node in the induced path from s to t.
        I.add(t)
        for u in G[s]:
            if len(I & set(G[u]))==2:
                I.add(u)
                break
    return I

def chordal_graph_cliques(G):
    """Returns the set of maximal cliques of a chordal graph.

    The algorithm breaks the graph in connected components and performs a
    maximum cardinality search in each component to get the cliques.

    Parameters
    ----------
    G: NetworkX graph

    Returns
    -------
    cliques: A set containing the maximal cliques in G.

    Raises
    ------
    NetworkXError
        The algorithm does not support DiGraph, MultiGraph and MultiDiGraph.
        If the input graph is an instance of one of these classes, a
        NetworkXError is raised.
        The algorithm can only be applied to chordal graphs. If the
        input graph is found to be non-chordal, a NetworkXError is raised.

    Examples
    --------
    >>> import networkx as nx
    >>> e= [(1,2),(1,3),(2,3),(2,4),(3,4),(3,5),(3,6),(4,5),(4,6),(5,6),(7,8)]
    >>> G = nx.Graph(e)
    >>> G.add_node(9)
    >>> setlist = nx.chordal_graph_cliques(G)
    """
    if not is_chordal(G):
        raise nx.NetworkXError("Input graph is not chordal.")

    cliques = set()
    for C in nx.connected.connected_component_subgraphs(G):
        cliques |= _connected_chordal_graph_cliques(C)

    return cliques

def chordal_graph_treewidth(G):
    """Returns the treewidth of the chordal graph G.

    Parameters
    ----------
    G : NetworkX graph
	The chordal graph which we want to measure its treewidth.

    Returns
    -------
    treewidth : int
	The size of the largest clique in the graph minus one.

    Raises
    ------
    NetworkXError
        The algorithm does not support DiGraph, MultiGraph and MultiDiGraph.
        If the input graph is an instance of one of these classes, a
        NetworkXError is raised.
        The algorithm can only be applied to chordal graphs. If
        the input graph is found to be non-chordal, a NetworkXError is raised.

    Examples
    --------
    >>> import networkx as nx
    >>> e = [(1,2),(1,3),(2,3),(2,4),(3,4),(3,5),(3,6),(4,5),(4,6),(5,6),(7,8)]
    >>> G = nx.Graph(e)
    >>> G.add_node(9)
    >>> nx.chordal_graph_treewidth(G)
    3

    References
    ----------
    .. [1] http://en.wikipedia.org/wiki/Tree_decomposition#Treewidth
    """
    if not is_chordal(G):
        raise nx.NetworkXError("Input graph is not chordal.")

    max_clique = -1
    for clique in nx.chordal_graph_cliques(G):
        max_clique = max(max_clique,len(clique))
    return max_clique - 1

def _is_complete_graph(G):
    """Returns True if G is a complete graph."""
    if G.number_of_selfloops()>0:
        raise nx.NetworkXError("Self loop found in _is_complete_graph()")
    n = G.number_of_nodes()
    if n < 2:
        return True
    e = G.number_of_edges()
    max_edges = ((n * (n-1))/2)
    return e == max_edges

def _find_missing_edge(G):
    """ Given a non-complete graph G, returns a missing edge."""
    nodes=set(G)
    for u in G:
        missing=nodes-set(list(G[u].keys())+[u])
        if missing:
            return (u,missing.pop())

def _max_cardinality_node(G,choices,wanna_connect):
    """Returns a the node in choices that has more connections in G
    to nodes in wanna_connect.
    """
#    max_number = None
    max_number = -1
    for x in choices:
        number=len([y for y in G[x] if y in wanna_connect])
        if number > max_number:
            max_number = number
            max_cardinality_node = x
    return max_cardinality_node

def _find_chordality_breaker(G,s=None,treewidth_bound=sys.maxsize):
    """ Given a graph G, starts a max cardinality search
    (starting from s if s is given and from a random node otherwise)
    trying to find a non-chordal cycle.

    If it does find one, it returns (u,v,w) where u,v,w are the three
    nodes that together with s are involved in the cycle.
    """

    unnumbered = set(G)
    if s is None:
        s = random.choice(list(unnumbered))
    unnumbered.remove(s)
    numbered = set([s])
#    current_treewidth = None
    current_treewidth = -1
    while unnumbered:# and current_treewidth <= treewidth_bound:
        v = _max_cardinality_node(G,unnumbered,numbered)
        unnumbered.remove(v)
        numbered.add(v)
        clique_wanna_be = set(G[v]) & numbered
        sg = G.subgraph(clique_wanna_be)
        if _is_complete_graph(sg):
            # The graph seems to be chordal by now. We update the treewidth
            current_treewidth = max(current_treewidth,len(clique_wanna_be))
            if current_treewidth > treewidth_bound:
                raise nx.NetworkXTreewidthBoundExceeded(\
                    "treewidth_bound exceeded: %s"%current_treewidth)
        else:
            # sg is not a clique,
            # look for an edge that is not included in sg
            (u,w) = _find_missing_edge(sg)
            return (u,v,w)
    return ()

def _connected_chordal_graph_cliques(G):
    """Return the set of maximal cliques of a connected chordal graph."""
    if G.number_of_nodes() == 1:
        x = frozenset(G.nodes())
        return set([x])
    else:
        cliques = set()
        unnumbered = set(G.nodes())
        v = random.choice(list(unnumbered))
        unnumbered.remove(v)
        numbered = set([v])
        clique_wanna_be = set([v])
        while unnumbered:
            v = _max_cardinality_node(G,unnumbered,numbered)
            unnumbered.remove(v)
            numbered.add(v)
            new_clique_wanna_be = set(G.neighbors(v)) & numbered
            sg = G.subgraph(clique_wanna_be)
            if _is_complete_graph(sg):
                new_clique_wanna_be.add(v)
                if not new_clique_wanna_be >= clique_wanna_be:
                    cliques.add(frozenset(clique_wanna_be))
                clique_wanna_be = new_clique_wanna_be
            else:
                raise nx.NetworkXError("Input graph is not chordal.")
        cliques.add(frozenset(clique_wanna_be))
        return cliques

<|endoftext|>


<commit_before># Setup and find Custom config dir
from __future__ import print_function

import sys
import os
from os.path import exists, expanduser, dirname, realpath, join as joinpath
import logging
import shutil

from sasmodels.custom import load_module_from_path

logger = logging.getLogger(__name__)

# TODO: REMOVE
def make_custom_config_path(user_dir):
    raise RuntimeError("Tried to run make_custom_config_path - this is now forbidden")
    # """
    # The location of the cusstom config file.
    #
    # Returns ~/.sasview/config/custom_config.py
    # """
    # dirname = os.path.join(user_dir, 'config')
    # # If the directory doesn't exist, create it
    # if not os.path.exists(dirname):
    #     os.makedirs(dirname)
    # path = os.path.join(dirname, "custom_config.py")
    # return path

def setup_custom_config(app_dir, user_dir):
    raise RuntimeError("Tried to run setup_custom_config - this is now forbidden")

    # path = make_custom_config_path(user_dir)
    # if not os.path.isfile(path):
    #     try:
    #         # if the custom config file does not exist, copy the default from
    #         # the app dir
    #         shutil.copyfile(os.path.join(app_dir, "custom_config.py"), path)
    #     except Exception:
    #         logger.error("Could not copy default custom config.")
    #
    # #Adding SAS_OPENCL if it doesn't exist in the config file
    # # - to support backcompability
    # if not "SAS_OPENCL" in open(path).read():
    #     try:
    #         open(path, "a+").write("SAS_OPENCL = \"None\"\n")
    #     except Exception:
    #         logger.error("Could not update custom config with SAS_OPENCL.")
    #
    # custom_config = load_custom_config(path)
    # return custom_config

def load_custom_config(path):
    raise RuntimeError("Tried to call load_custom_config - this is now forbidden")
    # if os.path.exists(path):
    #     try:
    #         module = load_module_from_path('sas.custom_config', path)
    #         #logger.info("GuiManager loaded %s", path)
    #         return module
    #     except Exception as exc:
    #         logger.error("Error loading %s: %s", path, exc)
    #
    # from sas.sasview import custom_config
    # logger.info("GuiManager custom_config defaults to sas.sasview.custom_config")
    # return custom_config<commit_msg>Deleted _config<commit_after><|endoftext|>


<commit_before>#!/usr/bin/python3

import time
import socketserver
import http.server
import http.client
import os.path
import pickle
import tweepy
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
from models import Tweet

tweets = []

class StreamHandler(StreamListener):

    def on_status(self, status):
        tweets.append(Tweet(status))
        return True

    def on_error(self, status):
        print(status)

class NetworkRequestHandler(http.server.BaseHTTPRequestHandler):

    """ Get every tweet with id > of path"""
    def do_GET(self):
        try:
            if "/status/from_id/" in self.path:
                status_id = int(self.path.replace("/status/from_id/", ""))
                self.send_tweets(status_id)
            elif "/status/length" in self.path:
                self.size_tweets()

            else:
                raise Exception("Unknown Request")

        except Exception as ex:
            self.send_response(500)
            self.send_header("Content-type", "stacktrace")
            self.end_headers()

            self.wfile.write(str(ex).encode("utf-8"))

    def size_tweets(self):
        self.send_response(200)
        self.end_headers()
        self.wfile.write(str(len(tweets)).encode("utf-8"))

    def send_tweets(self, status_id):
        if tweets[-1].tid <= status_id:
            self.send_response(204)
            self.end_headers()

        else:
            to_send = []

            i = len(tweets) - 1
            while True:
                cur = tweets[i]
                if cur.tid <= status_id or i <= 0:
                    break
                to_send.append(cur)
                i -= 1

            self.send_response(200)
            self.send_header("Content-type", "application/octet-stream")
            self.end_headers()
            out = pickle.dump(to_send, self.wfile)
            self.wfile.write(out)

def load_auth():
    f = open(os.path.expanduser('~/.config/pyt'))

    consKey = f.readline().replace('\n', '')
    consSec = f.readline().replace('\n', '')

    accessTok = f.readline().replace('\n', '')
    accessSecret = f.readline().replace('\n', '')

    f.close()

    auth = OAuthHandler(consKey, consSec)
    auth.set_access_token(accessTok, accessSecret)
    return auth

def main():
    auth = load_auth()
    api = tweepy.API(auth)
    st = api.home_timeline()
    for i in st:
        tweets.append(Tweet(i))

    start_stream(auth)
    start_server()

def start_stream(auth):
    l = StreamHandler()
    stream = Stream(auth, l)
    stream.userstream(async=True)

def start_server():
    import threading
    address = ('localhost', 8080)
    server = socketserver.TCPServer(address, NetworkRequestHandler)
    ip, port = server.server_address

    t = threading.Thread(target=server.serve_forever)
    t.setDaemon(True)
    t.start()

    # Connect to the server
    conn = http.client.HTTPConnection("localhost", 8080)
    """conn.request("GET", "/status/from_id/5880")"""
    conn.request("GET", "/status/length")
    resp = conn.getresponse()
    print(resp.status)
    print(resp.read().decode("utf-8"))
    time.sleep(500)

if __name__ == '__main__':
    main()
<commit_msg>Added models.py<commit_after>#!/usr/bin/python3

import time
import socketserver
import http.server
import http.client
import os.path
import pickle
import tweepy
from tweepy.streaming import StreamListener
from tweepy import OAuthHandler
from tweepy import Stream
from models import Tweet

tweets = []

class StreamHandler(StreamListener):

    def on_status(self, status):
        tweets.append(Tweet(status))
        return True

    def on_error(self, status):
        print(status)

class NetworkRequestHandler(http.server.BaseHTTPRequestHandler):

    """ Get every tweet with id > of path"""
    def do_GET(self):
        try:
            if "/status/from_id/" in self.path:
                status_id = int(self.path.replace("/status/from_id/", ""))
                self.send_tweets(status_id)
            elif "/status/length" in self.path:
                self.size_tweets()

            else:
                raise Exception("Unknown Request")

        except Exception as ex:
            self.send_response(500)
            self.send_header("Content-type", "stacktrace")
            self.end_headers()

            self.wfile.write(str(ex).encode("utf-8"))

    def size_tweets(self):
        self.send_response(200)
        self.end_headers()
        self.wfile.write(str(len(tweets)).encode("utf-8"))

    def send_tweets(self, status_id):
        if tweets[-1].tid <= status_id:
            self.send_response(204)
            self.end_headers()

        else:
            to_send = []

            i = len(tweets) - 1
            while True:
                cur = tweets[i]
                if cur.tid <= status_id or i <= 0:
                    break
                to_send.append(cur)
                i -= 1

            self.send_response(200)
            self.send_header("Content-type", "application/octet-stream")
            self.end_headers()
            pickle.dump(to_send, self.wfile)

def load_auth():
    f = open(os.path.expanduser('~/.config/pyt'))

    consKey = f.readline().replace('\n', '')
    consSec = f.readline().replace('\n', '')

    accessTok = f.readline().replace('\n', '')
    accessSecret = f.readline().replace('\n', '')

    f.close()

    auth = OAuthHandler(consKey, consSec)
    auth.set_access_token(accessTok, accessSecret)
    return auth

def main():
    auth = load_auth()
    api = tweepy.API(auth)
    st = api.home_timeline()
    for i in st:
        tweets.append(Tweet(i))

    start_stream(auth)
    start_server()

def start_stream(auth):
    l = StreamHandler()
    stream = Stream(auth, l)
    stream.userstream(async=True)

def start_server():
    import threading
    address = ('localhost', 8080)
    server = socketserver.TCPServer(address, NetworkRequestHandler)
    ip, port = server.server_address

    t = threading.Thread(target=server.serve_forever)
    t.setDaemon(True)
    t.start()

    # Connect to the server
    conn = http.client.HTTPConnection("localhost", 8080)
    """conn.request("GET", "/status/from_id/5880")"""
    conn.request("GET", "/status/length")
    resp = conn.getresponse()
    print(resp.status)
    print(resp.read().decode("utf-8"))
    time.sleep(500)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
:mod:`herpderper` --- Derping the herp since 1895
=================================================
"""

__docformat__ = "restructuredtext en"

import random
import re

from Skype4Py.enums import cmsReceived, cmeEmoted

from plugin import Plugin

def all_same(myiter):
    """
    Check whether all elements in iterable are the same.

    >>> all_same([1, 1, 1, 1])
    True
    >>> all_same([0, 1, 1])
    False
    >>> all_same("aaa")
    True
    >>> all_same("aab")
    False
    """

    return all([myiter[0] == item for item in myiter])

def parse_linux_quote(message):
    """
    >>> message = ur'''
    ... > [Wednesday, June 26, 2013 1:37:59 PM username derp] a quoted message
    ...
    ... some other message.
    ... '''
    >>> found = parse_linux_quote(message)
    >>> assert found is not None
    >>> assert found.group("quote") == "a quoted message"
    """

    pattern = re.compile(
        r"""
        # Conversation message pattern which matches Linux Skype client message
        # quotation.
        >\s\[
        (?P<datetime>
            (?P<day>
                \w+
            ),\s
            (?P<month>
                \w+
            )\s
            (?P<date>
                \d{1,2}
            ),\s
            (?P<year>
                \d{4}
            )\s
            (?P<time>
                \d{1,2}:\d{2}:\d{2}
            )\s
            (?P<ampm>
                \w{2}
            )
        )\s
        (?P<sender>
            .*
        )
        ]\s
        (?P<quote>
            .*
        )\n?
        (?P<message>
            .*
        )
        """,
        re.UNICODE | re.VERBOSE)

    return re.search(pattern, message)

def parse_macosx_quote(message):
    """
    >>> message = ur'''
    ... 26.06.13 в 13:37 skypename.derp написал (-а):
    ... > a quoted message
    ... some other message.
    ... '''
    >>> found = parse_macosx_quote(message)
    >>> assert found is not None
    >>> assert found.group("quote") == "a quoted message"
    """

    pattern = re.compile(
        r"""
        # Conversation message pattern which matches Mac OS X Skype client
        # message quotation.
        (?P<date>
            \d{2}.\d{2}.\d{2}
        )\s\w+\s
        (?P<time>
            \d{2}:\d{2}
        )\s
        # There are no space characters in Skype user names.
        (?P<sender>
            [a-zA-Z0-9_\-\.]+
        )\s.*:\n
        >\s*
        (?P<quote>
            .*
        )\n*
        (?P<message>
            .*
        )
        """,
        re.UNICODE | re.VERBOSE)

    return re.search(pattern, message)

def parse_windows_quote(message):
    """
    >>> message = ur'''[1:37:59] username derp: a quoted message
    ... <<<'''
    >>> found = parse_windows_quote(message)
    >>> assert found is not None
    >>> assert found.group("quote") == "a quoted message"
    """
    # FIXME: Does not work with actual client quotes for some reason.

    pattern = re.compile(
        r"""
        # Conversation message pattern which matches Windows Skype client
        # message quotation.
        \[
        (?P<time>
            \d{1,2}:\d{2}:\d{2}
        )\s*
        (?P<ampm>
            \w{2}
        )?]\s
        (?P<sender>
            .*
        ):\s
        (?P<quote>
            .*
        )\n*
        <<<\s*
        (?P<message>
            .*
        )
        """,
        re.UNICODE | re.VERBOSE)

    return re.search(pattern, message)

def message_is_quoted(message):
    """
    >>> # Windows client.
    >>> message = ur'''
    ... [1:37:59 PM] юзернейм derp: a quoted message
    ...
    ... <<< тест
    ... '''
    >>> assert message_is_quoted(message) is True

    >>> # Linux client.
    >>> message = ur'''
    ... > [Wednesday, June 26, 2013 1:37:59 PM юзернейм derp] a quoted message
    ...
    ... some other message.
    ... '''
    >>> assert message_is_quoted(message) is True

    >>> # Mac OS X client.
    >>> message = ur'''
    ... 26.06.13 в 13:37 skypename.derp написал (-а):
    ... > a quoted message
    ... some other message.
    ... '''
    >>> assert message_is_quoted(message) is True
    """

    if parse_macosx_quote(message) is not None:
        return True
    if parse_linux_quote(message) is not None:
        return True
    if parse_windows_quote(message) is not None:
        return True
    return False

class HerpDerper(Plugin):
    """
    A very simple and "fun" plugin which reacts on certain keywords by replying
    messages with random amount of "herp"'s and "derp"'s while maintaining
    meaningless statistics.
    """

    _triggers = [
        u"губи",
        u"ue,b",
        u"gooby",
        u"пщщин",
    ]

    def on_message_status(self, message, status):
        if status != cmsReceived or message.Type == cmeEmoted:
            return

        if not any(t.lower() in message.Body.lower() for t in self._triggers):
            return

        if message_is_quoted(message.Body):
            return

        msg = []

        max_words = 3
        words_count = random.randint(1, max_words)
        words = []
        for _ in xrange(words_count):
            words.append(random.choice(["herp", "derp", "hurr", "durr"]))

        msg.append(" ".join(words))

        if len(words) is max_words and all_same(words):
            self._logger.info("Jackpot winner {0}".format(message.FromHandle))

            try:
                wins = int(self._cache.get(message.FromHandle))
            except TypeError:
                wins = 0
            wins += 1
            self._cache.set(message.FromHandle, wins)

            msg.append("{0} wins the jackpot! {1} in total".format(
                message.FromDisplayName, wins
            ))

        message.Chat.SendMessage(u"\n".join(msg))

if __name__ == "__main__":
    import doctest
    doctest.testmod()
<commit_msg>Removed trigger phrase<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
:mod:`herpderper` --- Derping the herp since 1895
=================================================
"""

__docformat__ = "restructuredtext en"

import random
import re

from Skype4Py.enums import cmsReceived, cmeEmoted

from plugin import Plugin

def all_same(myiter):
    """
    Check whether all elements in iterable are the same.

    >>> all_same([1, 1, 1, 1])
    True
    >>> all_same([0, 1, 1])
    False
    >>> all_same("aaa")
    True
    >>> all_same("aab")
    False
    """

    return all([myiter[0] == item for item in myiter])

def parse_linux_quote(message):
    """
    >>> message = ur'''
    ... > [Wednesday, June 26, 2013 1:37:59 PM username derp] a quoted message
    ...
    ... some other message.
    ... '''
    >>> found = parse_linux_quote(message)
    >>> assert found is not None
    >>> assert found.group("quote") == "a quoted message"
    """

    pattern = re.compile(
        r"""
        # Conversation message pattern which matches Linux Skype client message
        # quotation.
        >\s\[
        (?P<datetime>
            (?P<day>
                \w+
            ),\s
            (?P<month>
                \w+
            )\s
            (?P<date>
                \d{1,2}
            ),\s
            (?P<year>
                \d{4}
            )\s
            (?P<time>
                \d{1,2}:\d{2}:\d{2}
            )\s
            (?P<ampm>
                \w{2}
            )
        )\s
        (?P<sender>
            .*
        )
        ]\s
        (?P<quote>
            .*
        )\n?
        (?P<message>
            .*
        )
        """,
        re.UNICODE | re.VERBOSE)

    return re.search(pattern, message)

def parse_macosx_quote(message):
    """
    >>> message = ur'''
    ... 26.06.13 в 13:37 skypename.derp написал (-а):
    ... > a quoted message
    ... some other message.
    ... '''
    >>> found = parse_macosx_quote(message)
    >>> assert found is not None
    >>> assert found.group("quote") == "a quoted message"
    """

    pattern = re.compile(
        r"""
        # Conversation message pattern which matches Mac OS X Skype client
        # message quotation.
        (?P<date>
            \d{2}.\d{2}.\d{2}
        )\s\w+\s
        (?P<time>
            \d{2}:\d{2}
        )\s
        # There are no space characters in Skype user names.
        (?P<sender>
            [a-zA-Z0-9_\-\.]+
        )\s.*:\n
        >\s*
        (?P<quote>
            .*
        )\n*
        (?P<message>
            .*
        )
        """,
        re.UNICODE | re.VERBOSE)

    return re.search(pattern, message)

def parse_windows_quote(message):
    """
    >>> message = ur'''[1:37:59] username derp: a quoted message
    ... <<<'''
    >>> found = parse_windows_quote(message)
    >>> assert found is not None
    >>> assert found.group("quote") == "a quoted message"
    """
    # FIXME: Does not work with actual client quotes for some reason.

    pattern = re.compile(
        r"""
        # Conversation message pattern which matches Windows Skype client
        # message quotation.
        \[
        (?P<time>
            \d{1,2}:\d{2}:\d{2}
        )\s*
        (?P<ampm>
            \w{2}
        )?]\s
        (?P<sender>
            .*
        ):\s
        (?P<quote>
            .*
        )\n*
        <<<\s*
        (?P<message>
            .*
        )
        """,
        re.UNICODE | re.VERBOSE)

    return re.search(pattern, message)

def message_is_quoted(message):
    """
    >>> # Windows client.
    >>> message = ur'''
    ... [1:37:59 PM] юзернейм derp: a quoted message
    ...
    ... <<< тест
    ... '''
    >>> assert message_is_quoted(message) is True

    >>> # Linux client.
    >>> message = ur'''
    ... > [Wednesday, June 26, 2013 1:37:59 PM юзернейм derp] a quoted message
    ...
    ... some other message.
    ... '''
    >>> assert message_is_quoted(message) is True

    >>> # Mac OS X client.
    >>> message = ur'''
    ... 26.06.13 в 13:37 skypename.derp написал (-а):
    ... > a quoted message
    ... some other message.
    ... '''
    >>> assert message_is_quoted(message) is True
    """

    if parse_macosx_quote(message) is not None:
        return True
    if parse_linux_quote(message) is not None:
        return True
    if parse_windows_quote(message) is not None:
        return True
    return False

class HerpDerper(Plugin):
    """
    A very simple and "fun" plugin which reacts on certain keywords by replying
    messages with random amount of "herp"'s and "derp"'s while maintaining
    meaningless statistics.
    """

    _triggers = [
        u"губи",
        u"ue,b",
        # u"gooby",
        u"пщщин",
    ]

    def on_message_status(self, message, status):
        if status != cmsReceived or message.Type == cmeEmoted:
            return

        if not any(t.lower() in message.Body.lower() for t in self._triggers):
            return

        if message_is_quoted(message.Body):
            return

        msg = []

        max_words = 3
        words_count = random.randint(1, max_words)
        words = []
        for _ in xrange(words_count):
            words.append(random.choice(["herp", "derp", "hurr", "durr"]))

        msg.append(" ".join(words))

        if len(words) is max_words and all_same(words):
            self._logger.info("Jackpot winner {0}".format(message.FromHandle))

            try:
                wins = int(self._cache.get(message.FromHandle))
            except TypeError:
                wins = 0
            wins += 1
            self._cache.set(message.FromHandle, wins)

            msg.append("{0} wins the jackpot! {1} in total".format(
                message.FromDisplayName, wins
            ))

        message.Chat.SendMessage(u"\n".join(msg))

if __name__ == "__main__":
    import doctest
    doctest.testmod()
<|endoftext|>


<commit_before>import datetime
import hashlib
import chardet
from rdflib import Graph, Dataset, URIRef, Literal, Namespace, RDF, RDFS

# Global variable holding the dataset that accumulates all provenance graphs
_ds = Dataset()

def clear_dataset():
    _ds = Dataset()

def get_dataset():
    return _ds

def get_graph():
    PROV = Namespace('http://www.w3.org/ns/prov#')
    PROVOMATIC = Namespace('http://provomatic.org/resource/')
    SKOS = Namespace('http://www.w3.org/2004/02/skos/core#')
    DCT = Namespace('http://purl.org/dc/terms/')

    graph = Graph()

    # Bind namespaces to prefixes
    graph.bind('prov',PROV)
    graph.bind('provomatic',PROVOMATIC)
    graph.bind('skos',SKOS)
    graph.bind('dcterms',DCT)

    for s,p,o,_ in get_dataset().quads(None) :
        graph.add((s,p,o))

    return graph

def save_prov(trail_filename='provenance-trail.ttl'):
    graph = get_graph()

    try :
        graph.serialize(open(trail_filename,'w'),format='turtle')
        print "File saved to {}".format(trail_filename)
    except:
        print "Problem writing to {}".format(trail_filename)

    return

def add_prov(uri, prov):
    """
        prov should be a Turtle serialization of an RDF PROV-O graph
        uri is a unique id of the graph

        """
    PROV = Namespace('http://www.w3.org/ns/prov#')
    PROVOMATIC = Namespace('http://provomatic.org/resource/')
    SKOS = Namespace('http://www.w3.org/2004/02/skos/core#')
    DCT = Namespace('http://purl.org/dc/terms/')

    ds = get_dataset()

    graph = ds.graph(URIRef(uri))
    graph.parse(data=prov,format='turtle')

    print "Loaded provenance graph with id {}".format(uri)
    return

class ProvBuilder(object):
    PROV = Namespace('http://www.w3.org/ns/prov#')
    PROVOMATIC = Namespace('http://provomatic.org/resource/')
    SKOS = Namespace('http://www.w3.org/2004/02/skos/core#')
    DCT = Namespace('http://purl.org/dc/terms/')

    # The variable ticker keeps the latest version of the value of a variable, to make sure that no cycles occur in the provenance graph.
    variable_ticker = {}

    def __init__(self):
        # Bind namespaces to prefixes
        _ds.bind('prov',self.PROV)
        _ds.bind('provomatic',self.PROVOMATIC)
        _ds.bind('skos',self.SKOS)
        _ds.bind('dcterms',self.DCT)

    def tick(self, variable):
        # We increment the re-use of the variable by 1
        self.variable_ticker[variable] = self.variable_ticker.setdefault(variable,0) + 1
        return self.variable_ticker[variable]

    def get_tick(self, variable):
        if variable in self.variable_ticker:
            return self.variable_ticker[variable]
        else :
            return self.tick(variable)

    def add_activity(self, name, description, inputs, outputs, dependencies={}, output_names=[], expand_output_dict=False, source=None):
        """Adds an activity to the graph. Inputs should be a dictionary of inputs & values, outputs a list or tuple of just values

           If expand_output_dict is set, the keys of the dictionary are used to generate individual outputs, otherwise the output dictionary is a single output.

        """

        if not source:
            source = description

        source, digest = self.get_value(source)

        description = unicode(description)

        timestamp = self.now()
        # Determine the plan and activity URI based on a digest of the source code of the function.
        plan_uri = self.PROVOMATIC['id-'+digest]
        activity_uri = self.PROVOMATIC['id-'+digest + "/" + timestamp]

        # print "Adding activity with name '{}'".format(name)
        # Initialise a graph with the same identifier as the activity uri
        self.g = _ds.graph(identifier=activity_uri)

        # Bind namespaces to prefixes
        self.g.bind('prov',self.PROV)
        self.g.bind('provomatic',self.PROVOMATIC)
        self.g.bind('skos',self.SKOS)
        self.g.bind('dcterms',self.DCT)

        # TODO: The PLAN part should be a qualified association
        self.g.add((plan_uri, RDF.type, self.PROV['Plan']))
        self.g.add((plan_uri, RDF.type, self.PROV['Entity']))
        self.g.add((plan_uri, RDFS.label, Literal(name)))
        self.g.add((plan_uri, self.DCT.description, Literal(description)))
        self.g.add((plan_uri,self.SKOS.note,Literal(source)))

        # print "Relating Activity '{} ({})' to Plan '{}'".format(name, timestamp, name)
        self.g.add((activity_uri,RDF.type,self.PROV['Activity']))
        self.g.add((activity_uri,RDFS.label,Literal("{} ({})".format(name, timestamp))))

        self.g.add((activity_uri,self.PROV['used'],plan_uri))
        self.g.add((activity_uri,self.DCT.description,Literal(description)))

        # For each input, create a 'used' relation
        for iname, value in inputs.items():
            value, vdigest = self.get_value(value)

            input_uri = self.add_entity(iname, vdigest, value)

            # print "Relating Activity '{} ({})' to input Entity '{}'".format(name, timestamp, input_uri)
            self.g.add((activity_uri, self.PROV['used'], input_uri))

        # For each output, create a 'generated' relation
        # Always expand tuples, to capture the variables separately.
        if isinstance(outputs, tuple) and len(outputs) == len(output_names):
            count = 0
            print "names: ", output_names
            print "outputs: ", outputs
            for value in outputs:

                value, vdigest = self.get_value(value)

                # If we know the output names (captured e.g. by 'replace'), we can also use them to generate nice names
                # Otherwise we create a nameless output
                print count
                if output_names != [] :
                    print "Generating entity for {}".format(output_names[count])
                    self.tick(output_names[count])
                    output_uri = self.add_entity(output_names[count],vdigest,value)
                else :
                    output_uri = self.add_entity("{} output {}".format(name,count),vdigest,value)

                # print "Relating Activity '{} ({})' to output Entity '{}'".format(name, timestamp, output_uri)
                self.g.add((activity_uri, self.PROV['generated'], output_uri))

                count += 1
        # Only expand dictionaries when explicitly told to do so
        elif expand_output_dict and isinstance(outputs,dict):
            for oname, value in outputs.items() :
                value, vdigest = self.get_value(value)

                output_uri = self.add_entity(oname, vdigest, value)

                # print "Relating Activity '{}' to output Entity '{}'".format(name, output_uri)
                self.g.add((activity_uri, self.PROV['generated'], output_uri))
        # Otherwise we'll take the value at 'face value'
        else :
            value, vdigest = self.get_value(outputs)

            # If we know the output name (captured e.g. by 'replace'), we can also use them to generate nice names
            # Otherwise we create a nameless output
            if output_names != []:
                print "Generating entity for {}".format(output_names[0])
                self.tick(output_names[0])
                output_uri = self.add_entity(output_names[0],vdigest,value)
            else :
                output_uri = self.add_entity("{} output".format(name), vdigest, value)

            # print "Relating Activity '{} ({})' to output Entity '{}'".format(name, timestamp, output_uri)
            self.g.add((activity_uri, self.PROV['generated'], output_uri))

        # For each dependency, create a 'wasInformedBy' relation

        # TODO: SKIPPING THIS FOR NOW
        # for dname, value in dependencies.items():
        #     value, vdigest = self.get_value(value)
        #
        #     dependency_uri = self.PROVOMATIC[vdigest]
        #
        #     self.g.add((dependency_uri,RDF.type,self.PROV['Activity']))
        #     # self.g.add((dependency_uri,RDFS.label,Literal(dname)))
        #     self.g.add((dependency_uri,self.SKOS.note,Literal(value)))
        #
        #     self.g.add((activity_uri, self.PROV['wasInformedBy'], dependency_uri))

        return activity_uri

    def add_entity(self, name, digest, description):

        tick = self.get_tick(name)

        entity_uri = self.PROVOMATIC['{}/{}/{}'.format(name.replace(' ','_').replace('%','_'),tick,digest)]

        # print "Adding Entity with label '{}' ({})".format(name,entity_uri)

        self.g.add((entity_uri,RDF.type,self.PROV['Entity']))
        self.g.add((entity_uri,RDFS.label,Literal(name)))
        self.g.add((entity_uri,self.SKOS['note'],Literal(description)))

        return entity_uri

    def get_value(self, io):
        """We'll just use the __unicode__ representation as source for the hash digest"""

        try :
            value = unicode(io)
        except :
            print "Type: ", type(io)
            print "IO cannot be decoded"
            encoding = chardet.detect(io)['encoding']
            if encoding == None :
                print "No encoding"
                value = io.encode('string-escape').decode('utf-8')
            else :
                print "Encoding: {}".format(encoding)
                value = io.decode(encoding)

        vdigest = hashlib.md5(value.encode('utf-8')).hexdigest()

        if len(value) > 200:
            value = value[:99] + u"..." + value[-100:]

        return value, vdigest

    def get_graph(self):
        return self.g

    def now(self):
        return datetime.datetime.now().isoformat()<commit_msg>Tick-aware entity adding (will introduce a prov:wasDerivedFrom relation if the variable existed previously)<commit_after>import datetime
import hashlib
import chardet
from rdflib import Graph, Dataset, URIRef, Literal, Namespace, RDF, RDFS

# Global variable holding the dataset that accumulates all provenance graphs
_ds = Dataset()

def clear_dataset():
    _ds = Dataset()

def get_dataset():
    return _ds

def get_graph():
    PROV = Namespace('http://www.w3.org/ns/prov#')
    PROVOMATIC = Namespace('http://provomatic.org/resource/')
    SKOS = Namespace('http://www.w3.org/2004/02/skos/core#')
    DCT = Namespace('http://purl.org/dc/terms/')

    graph = Graph()

    # Bind namespaces to prefixes
    graph.bind('prov',PROV)
    graph.bind('provomatic',PROVOMATIC)
    graph.bind('skos',SKOS)
    graph.bind('dcterms',DCT)

    for s,p,o,_ in get_dataset().quads(None) :
        graph.add((s,p,o))

    return graph

def save_prov(trail_filename='provenance-trail.ttl'):
    graph = get_graph()

    try :
        graph.serialize(open(trail_filename,'w'),format='turtle')
        print "File saved to {}".format(trail_filename)
    except:
        print "Problem writing to {}".format(trail_filename)

    return

def add_prov(uri, prov):
    """
        prov should be a Turtle serialization of an RDF PROV-O graph
        uri is a unique id of the graph

        """
    PROV = Namespace('http://www.w3.org/ns/prov#')
    PROVOMATIC = Namespace('http://provomatic.org/resource/')
    SKOS = Namespace('http://www.w3.org/2004/02/skos/core#')
    DCT = Namespace('http://purl.org/dc/terms/')

    ds = get_dataset()

    graph = ds.graph(URIRef(uri))
    graph.parse(data=prov,format='turtle')

    print "Loaded provenance graph with id {}".format(uri)
    return

class ProvBuilder(object):
    PROV = Namespace('http://www.w3.org/ns/prov#')
    PROVOMATIC = Namespace('http://provomatic.org/resource/')
    SKOS = Namespace('http://www.w3.org/2004/02/skos/core#')
    DCT = Namespace('http://purl.org/dc/terms/')

    # The variable ticker keeps the latest version of the value of a variable, to make sure that no cycles occur in the provenance graph.
    variable_ticker = {}

    def __init__(self):
        # Bind namespaces to prefixes
        _ds.bind('prov',self.PROV)
        _ds.bind('provomatic',self.PROVOMATIC)
        _ds.bind('skos',self.SKOS)
        _ds.bind('dcterms',self.DCT)

    def tick(self, variable):
        # We increment the re-use of the variable by 1
        self.variable_ticker[variable] = self.variable_ticker.setdefault(variable,0) + 1
        return self.variable_ticker[variable]

    def tick_and_add_entity(self, variable, digest, description):
        # We increment the re-use of the variable by 1, and create a dependency
        if variable in self.variable_ticker:
            old_entity_uri = self.add_entity(variable, digest, description)

            self.tick(variable)

            entity_uri = self.add_entity(variable, digest, description)

            self.g.add((entity_uri,self.PROV['wasDerivedFrom'],old_entity_uri))
        else :
            self.tick(variable)
            entity_uri = self.add_entity(variable, digest, description)

        return entity_uri

    def get_tick(self, variable):
        if variable in self.variable_ticker:
            return self.variable_ticker[variable]
        else :
            return self.tick(variable)

    def add_activity(self, name, description, inputs, outputs, dependencies={}, output_names=[], expand_output_dict=False, source=None):
        """Adds an activity to the graph. Inputs should be a dictionary of inputs & values, outputs a list or tuple of just values

           If expand_output_dict is set, the keys of the dictionary are used to generate individual outputs, otherwise the output dictionary is a single output.

        """

        if not source:
            source = description

        source, digest = self.get_value(source)

        description = unicode(description)

        timestamp = self.now()
        # Determine the plan and activity URI based on a digest of the source code of the function.
        plan_uri = self.PROVOMATIC['id-'+digest]
        activity_uri = self.PROVOMATIC['id-'+digest + "/" + timestamp]

        # print "Adding activity with name '{}'".format(name)
        # Initialise a graph with the same identifier as the activity uri
        self.g = _ds.graph(identifier=activity_uri)

        # Bind namespaces to prefixes
        self.g.bind('prov',self.PROV)
        self.g.bind('provomatic',self.PROVOMATIC)
        self.g.bind('skos',self.SKOS)
        self.g.bind('dcterms',self.DCT)

        # TODO: The PLAN part should be a qualified association
        self.g.add((plan_uri, RDF.type, self.PROV['Plan']))
        self.g.add((plan_uri, RDF.type, self.PROV['Entity']))
        self.g.add((plan_uri, RDFS.label, Literal(name)))
        self.g.add((plan_uri, self.DCT.description, Literal(description)))
        self.g.add((plan_uri,self.SKOS.note,Literal(source)))

        # print "Relating Activity '{} ({})' to Plan '{}'".format(name, timestamp, name)
        self.g.add((activity_uri,RDF.type,self.PROV['Activity']))
        self.g.add((activity_uri,RDFS.label,Literal("{} ({})".format(name, timestamp))))

        self.g.add((activity_uri,self.PROV['used'],plan_uri))
        self.g.add((activity_uri,self.DCT.description,Literal(description)))

        # For each input, create a 'used' relation
        for iname, value in inputs.items():
            value, vdigest = self.get_value(value)

            input_uri = self.add_entity(iname, vdigest, value)

            # print "Relating Activity '{} ({})' to input Entity '{}'".format(name, timestamp, input_uri)
            self.g.add((activity_uri, self.PROV['used'], input_uri))

        # For each output, create a 'generated' relation
        # Always expand tuples, to capture the variables separately.
        if isinstance(outputs, tuple) and len(outputs) == len(output_names):
            count = 0
            print "names: ", output_names
            print "outputs: ", outputs
            for value in outputs:

                value, vdigest = self.get_value(value)

                # If we know the output names (captured e.g. by 'replace'), we can also use them to generate nice names
                # Otherwise we create a nameless output
                print count
                if output_names != [] :
                    print "Generating entity for {}".format(output_names[count])
                    self.tick(output_names[count])
                    output_uri = self.add_entity(output_names[count],vdigest,value)
                else :
                    output_uri = self.add_entity("{} output {}".format(name,count),vdigest,value)

                # print "Relating Activity '{} ({})' to output Entity '{}'".format(name, timestamp, output_uri)
                self.g.add((activity_uri, self.PROV['generated'], output_uri))

                count += 1
        # Only expand dictionaries when explicitly told to do so
        elif expand_output_dict and isinstance(outputs,dict):
            for oname, value in outputs.items() :
                value, vdigest = self.get_value(value)

                output_uri = self.add_entity(oname, vdigest, value)

                # print "Relating Activity '{}' to output Entity '{}'".format(name, output_uri)
                self.g.add((activity_uri, self.PROV['generated'], output_uri))
        # Otherwise we'll take the value at 'face value'
        else :
            value, vdigest = self.get_value(outputs)

            # If we know the output name (captured e.g. by 'replace'), we can also use them to generate nice names
            # Otherwise we create a nameless output
            if output_names != []:
                print "Generating entity for {}".format(output_names[0])
                self.tick(output_names[0])
                output_uri = self.add_entity(output_names[0],vdigest,value)
            else :
                output_uri = self.add_entity("{} output".format(name), vdigest, value)

            # print "Relating Activity '{} ({})' to output Entity '{}'".format(name, timestamp, output_uri)
            self.g.add((activity_uri, self.PROV['generated'], output_uri))

        # For each dependency, create a 'wasInformedBy' relation

        # TODO: SKIPPING THIS FOR NOW
        # for dname, value in dependencies.items():
        #     value, vdigest = self.get_value(value)
        #
        #     dependency_uri = self.PROVOMATIC[vdigest]
        #
        #     self.g.add((dependency_uri,RDF.type,self.PROV['Activity']))
        #     # self.g.add((dependency_uri,RDFS.label,Literal(dname)))
        #     self.g.add((dependency_uri,self.SKOS.note,Literal(value)))
        #
        #     self.g.add((activity_uri, self.PROV['wasInformedBy'], dependency_uri))

        return activity_uri

    def add_entity(self, name, digest, description):

        tick = self.get_tick(name)

        entity_uri = self.PROVOMATIC['{}/{}/{}'.format(name.replace(' ','_').replace('%','_'),tick,digest)]

        # print "Adding Entity with label '{}' ({})".format(name,entity_uri)

        self.g.add((entity_uri,RDF.type,self.PROV['Entity']))
        self.g.add((entity_uri,RDFS.label,Literal(name)))
        self.g.add((entity_uri,self.SKOS['note'],Literal(description)))

        return entity_uri

    def get_value(self, io):
        """We'll just use the __unicode__ representation as source for the hash digest"""

        try :
            value = unicode(io)
        except :
            print "Type: ", type(io)
            print "IO cannot be decoded"
            encoding = chardet.detect(io)['encoding']
            if encoding == None :
                print "No encoding"
                value = io.encode('string-escape').decode('utf-8')
            else :
                print "Encoding: {}".format(encoding)
                value = io.decode(encoding)

        vdigest = hashlib.md5(value.encode('utf-8')).hexdigest()

        if len(value) > 200:
            value = value[:99] + u"..." + value[-100:]

        return value, vdigest

    def get_graph(self):
        return self.g

    def now(self):
        return datetime.datetime.now().isoformat()<|endoftext|>