<commit_before>import time
from nose.tools import timed
from cassandra import ReadTimeout, ReadFailure
from cassandra import ConsistencyLevel as CL
from cassandra.query import SimpleStatement
from dtest import Tester, debug
from tools import no_vnodes, require, since
from threading import Event
from assertions import assert_invalid

class NotificationWaiter(object):
    """
    A helper class for waiting for pushed notifications from
    Cassandra over the native protocol.
    """

    def __init__(self, tester, node, notification_types, keyspace=None):
        """
        `address` should be a ccmlib.node.Node instance
        `notification_types` should be a list of
        "TOPOLOGY_CHANGE", "STATUS_CHANGE", and "SCHEMA_CHANGE".
        """
        self.node = node
        self.address = node.network_interfaces['binary'][0]
        self.notification_types = notification_types
        self.keyspace = keyspace

        # get a single, new connection
        session = tester.patient_cql_connection(node)
        connection = session.cluster.connection_factory(self.address, is_control_connection=True)

        # coordinate with an Event
        self.event = Event()

        # the pushed notification
        self.notifications = []

        # register a callback for the notification type
        for notification_type in notification_types:
            connection.register_watcher(notification_type, self.handle_notification, register_timeout=5.0)

    def handle_notification(self, notification):
        """
        Called when a notification is pushed from Cassandra.
        """
        debug("Source {} sent {}".format(self.address, notification))

        if self.keyspace and notification['keyspace'] and self.keyspace != notification['keyspace']:
            return  # we are not interested in this schema change

        self.notifications.append(notification)
        self.event.set()

    def wait_for_notifications(self, timeout, num_notifications=1):
        """
        Waits up to `timeout` seconds for notifications from Cassandra. If
        passed `num_notifications`, stop waiting when that many notifications
        are observed.
        """

        deadline = time.time() + timeout
        while time.time() < deadline:
            self.event.wait(deadline - time.time())
            self.event.clear()
            if len(self.notifications) >= num_notifications:
                break

        return self.notifications

    def clear_notifications(self):
        self.notifications = []
        self.event.clear()

class TestPushedNotifications(Tester):
    """
    Tests for pushed native protocol notification from Cassandra.
    """

    @no_vnodes()
    def move_single_node_test(self):
        """
        @jira_ticket CASSANDRA-8516
        Moving a token should result in NODE_MOVED notifications.
        """
        self.cluster.populate(3).start(wait_for_binary_proto=True, wait_other_notice=True)

        # Despite waiting for each node to see the other nodes as UP, there is apparently
        # still a race condition that can result in NEW_NODE events being sent.  We don't
        # want to accidentally collect those, so for now we will just sleep a few seconds.
        time.sleep(3)

        waiters = [NotificationWaiter(self, node, ["TOPOLOGY_CHANGE"])
                   for node in self.cluster.nodes.values()]

        node1 = self.cluster.nodes.values()[0]
        node1.move("123")

        for waiter in waiters:
            debug("Waiting for notification from {}".format(waiter.address,))
            notifications = waiter.wait_for_notifications(60.0)
            self.assertEquals(1, len(notifications))
            notification = notifications[0]
            change_type = notification["change_type"]
            address, port = notification["address"]
            self.assertEquals("MOVED_NODE", change_type)
            self.assertEquals(self.get_ip_from_node(node1), address)

    @no_vnodes()
    @since("2.1")
    @require("10052")
    def move_single_node_localhost_test(self):
        """
        @jira_ticket  CASSANDRA-10052
        Test that we don't get NODE_MOVED notifications from nodes other than the local one,
        when rpc_address is set to localhost.

        To set-up this test we override the rpc_address to "localhost" for all nodes, and
        therefore we must change the rpc port or else processes won't start.
        """
        cluster = self.cluster
        cluster.populate(3)
        node1, node2, node3 = cluster.nodelist()

        # change node3 'rpc_address' from '127.0.0.x' to 'localhost', increase port numbers
        i = 0
        for node in cluster.nodelist():
            node.network_interfaces['thrift'] = ('localhost', node.network_interfaces['thrift'][1] + i)
            node.network_interfaces['binary'] = ('localhost', node.network_interfaces['thrift'][1] + 1)
            node.import_config_files()  # this regenerates the yaml file and sets 'rpc_address' to the 'thrift' address
            debug(node.show())
            i = i + 2

        cluster.start(wait_for_binary_proto=True, wait_other_notice=True)

        # Despite waiting for each node to see the other nodes as UP, there is apparently
        # still a race condition that can result in NEW_NODE events being sent.  We don't
        # want to accidentally collect those, so for now we will just sleep a few seconds.
        time.sleep(3)

        waiters = [NotificationWaiter(self, node, ["TOPOLOGY_CHANGE"])
                   for node in self.cluster.nodes.values()]

        node1 = self.cluster.nodes.values()[0]
        node1.move("123")

        for waiter in waiters:
            debug("Waiting for notification from {}".format(waiter.address,))
            notifications = waiter.wait_for_notifications(30.0)
            self.assertEquals(1 if waiter.node is node1 else 0, len(notifications))

    @require("10052")
    def restart_node_test(self):
        """
        @jira_ticket CASSANDRA-7816
        Restarting a node should generate exactly one DOWN and one UP notification
        """

        self.cluster.populate(2).start()
        node1, node2 = self.cluster.nodelist()

        waiter = NotificationWaiter(self, node1, ["STATUS_CHANGE", "TOPOLOGY_CHANGE"])

        for i in range(5):
            debug("Restarting second node...")
            node2.stop(wait_other_notice=True)
            node2.start(wait_other_notice=True)
            debug("Waiting for notifications from {}".format(waiter.address,))
            notifications = waiter.wait_for_notifications(timeout=60.0, num_notifications=3)
            self.assertEquals(3, len(notifications))
            for notification in notifications:
                self.assertEquals(self.get_ip_from_node(node2), notification["address"][0])
            self.assertEquals("DOWN", notifications[0]["change_type"])
            self.assertEquals("UP", notifications[1]["change_type"])
            self.assertEquals("NEW_NODE", notifications[2]["change_type"])
            waiter.clear_notifications()

    @since("2.1")
    @require("10052")
    def restart_node_localhost_test(self):
        """
        Test that we don't get client notifications when rpc_address is set to localhost.
        @jira_ticket  CASSANDRA-10052

        To set-up this test we override the rpc_address to "localhost" for all nodes, and
        therefore we must change the rpc port or else processes won't start.
        """
        cluster = self.cluster
        cluster.populate(2)
        node1, node2 = cluster.nodelist()

        i = 0  # change 'rpc_address' from '127.0.0.x' to 'localhost' and diversify port numbers
        for node in cluster.nodelist():
            node.network_interfaces['thrift'] = ('localhost', node.network_interfaces['thrift'][1] + i)
            node.network_interfaces['binary'] = ('localhost', node.network_interfaces['thrift'][1] + 1)
            node.import_config_files()  # this regenerates the yaml file and sets 'rpc_address' to the 'thrift' address
            debug(node.show())
            i = i + 2

        cluster.start(wait_for_binary_proto=True)

        # register for notification with node1
        waiter = NotificationWaiter(self, node1, ["STATUS_CHANGE", "TOPOLOGY_CHANGE"])

        # restart node 2
        debug("Restarting second node...")
        node2.stop(wait_other_notice=True)
        node2.start(wait_other_notice=True)

        # check that node1 did not send UP or DOWN notification for node2
        debug("Waiting for notifications from {}".format(waiter.address,))
        notifications = waiter.wait_for_notifications(timeout=30.0, num_notifications=3)
        self.assertEquals(0, len(notifications))

    @since("3.0")
    @require("9961")
    def schema_changes_test(self):
        """
        @jira_ticket CASSANDRA-10328
        Creating, updating and dropping a keyspace, a table and a materialized view
        will generate the correct schema change notifications.
        """

        self.cluster.populate(2).start(wait_for_binary_proto=True)
        node1, node2 = self.cluster.nodelist()

        session = self.patient_cql_connection(node1)
        waiter = NotificationWaiter(self, node2, ["SCHEMA_CHANGE"], keyspace='ks')

        self.create_ks(session, 'ks', 3)
        session.execute("create TABLE t (k int PRIMARY KEY , v int)")
        session.execute("alter TABLE t add v1 int;")

        session.execute("create MATERIALIZED VIEW mv as select * from t WHERE v IS NOT NULL AND t IS NOT NULL PRIMARY KEY (v, k)")
        session.execute(" alter materialized view mv with min_index_interval = 100")

        session.execute("drop MATERIALIZED VIEW mv")
        session.execute("drop TABLE t")
        session.execute("drop KEYSPACE ks")

        debug("Waiting for notifications from {}".format(waiter.address,))
        notifications = waiter.wait_for_notifications(timeout=60.0, num_notifications=14)
        self.assertEquals(14, len(notifications))
        self.assertDictContainsSubset({'change_type': u'CREATED', 'target_type': u'KEYSPACE'}, notifications[0])
        self.assertDictContainsSubset({'change_type': u'UPDATED', 'target_type': u'KEYSPACE'}, notifications[1])
        self.assertDictContainsSubset({'change_type': u'CREATED', 'target_type': u'TABLE', u'table': u't'}, notifications[2])
        self.assertDictContainsSubset({'change_type': u'UPDATED', 'target_type': u'KEYSPACE'}, notifications[3])
        self.assertDictContainsSubset({'change_type': u'UPDATED', 'target_type': u'TABLE', u'table': u't'}, notifications[4])
        self.assertDictContainsSubset({'change_type': u'UPDATED', 'target_type': u'KEYSPACE'}, notifications[5])
        self.assertDictContainsSubset({'change_type': u'CREATED', 'target_type': u'TABLE', u'table': u'mv'}, notifications[6])
        self.assertDictContainsSubset({'change_type': u'UPDATED', 'target_type': u'KEYSPACE'}, notifications[7])
        self.assertDictContainsSubset({'change_type': u'UPDATED', 'target_type': u'TABLE', u'table': u'mv'}, notifications[8])
        self.assertDictContainsSubset({'change_type': u'UPDATED', 'target_type': u'KEYSPACE'}, notifications[9])
        self.assertDictContainsSubset({'change_type': u'DROPPED', 'target_type': u'TABLE', u'table': u'mv'}, notifications[10])
        self.assertDictContainsSubset({'change_type': u'UPDATED', 'target_type': u'KEYSPACE'}, notifications[11])
        self.assertDictContainsSubset({'change_type': u'DROPPED', 'target_type': u'TABLE', u'table': u't'}, notifications[12])
        self.assertDictContainsSubset({'change_type': u'DROPPED', 'target_type': u'KEYSPACE'}, notifications[13])

class TestVariousNotifications(Tester):
    """
    Tests for various notifications/messages from Cassandra.
    """

    @since('2.2')
    def tombstone_failure_threshold_message_test(self):
        """
        Ensure nodes return an error message in case of TombstoneOverwhelmingExceptions rather
        than dropping the request. A drop makes the coordinator waits for the specified
        read_request_timeout_in_ms.
        @jira_ticket CASSANDRA-7886
        """

        self.allow_log_errors = True
        self.cluster.set_configuration_options(
            values={
                'tombstone_failure_threshold': 500,
                'read_request_timeout_in_ms': 30000,  # 30 seconds
                'range_request_timeout_in_ms': 40000
            }
        )
        self.cluster.populate(3).start()
        node1, node2, node3 = self.cluster.nodelist()
        session = self.patient_cql_connection(node1)

        self.create_ks(session, 'test', 3)
        session.execute(
            "CREATE TABLE test ( "
            "id int, mytext text, col1 int, col2 int, col3 int, "
            "PRIMARY KEY (id, mytext) )"
        )

        # Add data with tombstones
        values = map(lambda i: str(i), range(1000))
        for value in values:
            session.execute(SimpleStatement(
                "insert into test (id, mytext, col1) values (1, '{}', null) ".format(
                    value
                ),
                consistency_level=CL.ALL
            ))

        failure_msg = ("Scanned over.* tombstones.* query aborted")

        @timed(25)
        def read_failure_query():
            assert_invalid(
                session, SimpleStatement("select * from test where id in (1,2,3,4,5)", consistency_level=CL.ALL),
                expected=ReadTimeout if self.cluster.version() < '3' else ReadFailure,
            )

        read_failure_query()

        failure = (node1.grep_log(failure_msg) or
                   node2.grep_log(failure_msg) or
                   node3.grep_log(failure_msg))

        self.assertTrue(failure, ("Cannot find tombstone failure threshold error in log "
                                  "after failed query"))
        mark1 = node1.mark_log()
        mark2 = node2.mark_log()
        mark3 = node3.mark_log()

        @timed(35)
        def range_request_failure_query():
            assert_invalid(
                session, SimpleStatement("select * from test", consistency_level=CL.ALL),
                expected=ReadTimeout if self.cluster.version() < '3' else ReadFailure,
            )

        range_request_failure_query()

        failure = (node1.watch_log_for(failure_msg, from_mark=mark1, timeout=5) or
                   node2.watch_log_for(failure_msg, from_mark=mark2, timeout=5) or
                   node3.watch_log_for(failure_msg, from_mark=mark3, timeout=5))

        self.assertTrue(failure, ("Cannot find tombstone failure threshold error in log "
                                  "after range_request_timeout_query"))
<commit_msg>unrequire CASSANDRA-10052 tests<commit_after>import time
from nose.tools import timed
from cassandra import ReadTimeout, ReadFailure
from cassandra import ConsistencyLevel as CL
from cassandra.query import SimpleStatement
from dtest import Tester, debug
from tools import no_vnodes, require, since
from threading import Event
from assertions import assert_invalid

class NotificationWaiter(object):
    """
    A helper class for waiting for pushed notifications from
    Cassandra over the native protocol.
    """

    def __init__(self, tester, node, notification_types, keyspace=None):
        """
        `address` should be a ccmlib.node.Node instance
        `notification_types` should be a list of
        "TOPOLOGY_CHANGE", "STATUS_CHANGE", and "SCHEMA_CHANGE".
        """
        self.node = node
        self.address = node.network_interfaces['binary'][0]
        self.notification_types = notification_types
        self.keyspace = keyspace

        # get a single, new connection
        session = tester.patient_cql_connection(node)
        connection = session.cluster.connection_factory(self.address, is_control_connection=True)

        # coordinate with an Event
        self.event = Event()

        # the pushed notification
        self.notifications = []

        # register a callback for the notification type
        for notification_type in notification_types:
            connection.register_watcher(notification_type, self.handle_notification, register_timeout=5.0)

    def handle_notification(self, notification):
        """
        Called when a notification is pushed from Cassandra.
        """
        debug("Source {} sent {}".format(self.address, notification))

        if self.keyspace and notification['keyspace'] and self.keyspace != notification['keyspace']:
            return  # we are not interested in this schema change

        self.notifications.append(notification)
        self.event.set()

    def wait_for_notifications(self, timeout, num_notifications=1):
        """
        Waits up to `timeout` seconds for notifications from Cassandra. If
        passed `num_notifications`, stop waiting when that many notifications
        are observed.
        """

        deadline = time.time() + timeout
        while time.time() < deadline:
            self.event.wait(deadline - time.time())
            self.event.clear()
            if len(self.notifications) >= num_notifications:
                break

        return self.notifications

    def clear_notifications(self):
        self.notifications = []
        self.event.clear()

class TestPushedNotifications(Tester):
    """
    Tests for pushed native protocol notification from Cassandra.
    """

    @no_vnodes()
    def move_single_node_test(self):
        """
        @jira_ticket CASSANDRA-8516
        Moving a token should result in NODE_MOVED notifications.
        """
        self.cluster.populate(3).start(wait_for_binary_proto=True, wait_other_notice=True)

        # Despite waiting for each node to see the other nodes as UP, there is apparently
        # still a race condition that can result in NEW_NODE events being sent.  We don't
        # want to accidentally collect those, so for now we will just sleep a few seconds.
        time.sleep(3)

        waiters = [NotificationWaiter(self, node, ["TOPOLOGY_CHANGE"])
                   for node in self.cluster.nodes.values()]

        node1 = self.cluster.nodes.values()[0]
        node1.move("123")

        for waiter in waiters:
            debug("Waiting for notification from {}".format(waiter.address,))
            notifications = waiter.wait_for_notifications(60.0)
            self.assertEquals(1, len(notifications))
            notification = notifications[0]
            change_type = notification["change_type"]
            address, port = notification["address"]
            self.assertEquals("MOVED_NODE", change_type)
            self.assertEquals(self.get_ip_from_node(node1), address)

    @no_vnodes()
    @since("2.1")
    @require("10052")
    def move_single_node_localhost_test(self):
        """
        @jira_ticket  CASSANDRA-10052
        Test that we don't get NODE_MOVED notifications from nodes other than the local one,
        when rpc_address is set to localhost.

        To set-up this test we override the rpc_address to "localhost" for all nodes, and
        therefore we must change the rpc port or else processes won't start.
        """
        cluster = self.cluster
        cluster.populate(3)
        node1, node2, node3 = cluster.nodelist()

        # change node3 'rpc_address' from '127.0.0.x' to 'localhost', increase port numbers
        i = 0
        for node in cluster.nodelist():
            node.network_interfaces['thrift'] = ('localhost', node.network_interfaces['thrift'][1] + i)
            node.network_interfaces['binary'] = ('localhost', node.network_interfaces['thrift'][1] + 1)
            node.import_config_files()  # this regenerates the yaml file and sets 'rpc_address' to the 'thrift' address
            debug(node.show())
            i = i + 2

        cluster.start(wait_for_binary_proto=True, wait_other_notice=True)

        # Despite waiting for each node to see the other nodes as UP, there is apparently
        # still a race condition that can result in NEW_NODE events being sent.  We don't
        # want to accidentally collect those, so for now we will just sleep a few seconds.
        time.sleep(3)

        waiters = [NotificationWaiter(self, node, ["TOPOLOGY_CHANGE"])
                   for node in self.cluster.nodes.values()]

        node1 = self.cluster.nodes.values()[0]
        node1.move("123")

        for waiter in waiters:
            debug("Waiting for notification from {}".format(waiter.address,))
            notifications = waiter.wait_for_notifications(30.0)
            self.assertEquals(1 if waiter.node is node1 else 0, len(notifications))

    def restart_node_test(self):
        """
        @jira_ticket CASSANDRA-7816
        Restarting a node should generate exactly one DOWN and one UP notification
        """

        self.cluster.populate(2).start()
        node1, node2 = self.cluster.nodelist()

        waiter = NotificationWaiter(self, node1, ["STATUS_CHANGE", "TOPOLOGY_CHANGE"])

        for i in range(5):
            debug("Restarting second node...")
            node2.stop(wait_other_notice=True)
            node2.start(wait_other_notice=True)
            debug("Waiting for notifications from {}".format(waiter.address,))
            notifications = waiter.wait_for_notifications(timeout=60.0, num_notifications=3)
            self.assertEquals(3, len(notifications))
            for notification in notifications:
                self.assertEquals(self.get_ip_from_node(node2), notification["address"][0])
            self.assertEquals("DOWN", notifications[0]["change_type"])
            self.assertEquals("UP", notifications[1]["change_type"])
            self.assertEquals("NEW_NODE", notifications[2]["change_type"])
            waiter.clear_notifications()

    @since("2.1")
    def restart_node_localhost_test(self):
        """
        Test that we don't get client notifications when rpc_address is set to localhost.
        @jira_ticket  CASSANDRA-10052

        To set-up this test we override the rpc_address to "localhost" for all nodes, and
        therefore we must change the rpc port or else processes won't start.
        """
        cluster = self.cluster
        cluster.populate(2)
        node1, node2 = cluster.nodelist()

        i = 0  # change 'rpc_address' from '127.0.0.x' to 'localhost' and diversify port numbers
        for node in cluster.nodelist():
            node.network_interfaces['thrift'] = ('localhost', node.network_interfaces['thrift'][1] + i)
            node.network_interfaces['binary'] = ('localhost', node.network_interfaces['thrift'][1] + 1)
            node.import_config_files()  # this regenerates the yaml file and sets 'rpc_address' to the 'thrift' address
            debug(node.show())
            i = i + 2

        cluster.start(wait_for_binary_proto=True)

        # register for notification with node1
        waiter = NotificationWaiter(self, node1, ["STATUS_CHANGE", "TOPOLOGY_CHANGE"])

        # restart node 2
        debug("Restarting second node...")
        node2.stop(wait_other_notice=True)
        node2.start(wait_other_notice=True)

        # check that node1 did not send UP or DOWN notification for node2
        debug("Waiting for notifications from {}".format(waiter.address,))
        notifications = waiter.wait_for_notifications(timeout=30.0, num_notifications=3)
        self.assertEquals(0, len(notifications))

    @since("3.0")
    @require("9961")
    def schema_changes_test(self):
        """
        @jira_ticket CASSANDRA-10328
        Creating, updating and dropping a keyspace, a table and a materialized view
        will generate the correct schema change notifications.
        """

        self.cluster.populate(2).start(wait_for_binary_proto=True)
        node1, node2 = self.cluster.nodelist()

        session = self.patient_cql_connection(node1)
        waiter = NotificationWaiter(self, node2, ["SCHEMA_CHANGE"], keyspace='ks')

        self.create_ks(session, 'ks', 3)
        session.execute("create TABLE t (k int PRIMARY KEY , v int)")
        session.execute("alter TABLE t add v1 int;")

        session.execute("create MATERIALIZED VIEW mv as select * from t WHERE v IS NOT NULL AND t IS NOT NULL PRIMARY KEY (v, k)")
        session.execute(" alter materialized view mv with min_index_interval = 100")

        session.execute("drop MATERIALIZED VIEW mv")
        session.execute("drop TABLE t")
        session.execute("drop KEYSPACE ks")

        debug("Waiting for notifications from {}".format(waiter.address,))
        notifications = waiter.wait_for_notifications(timeout=60.0, num_notifications=14)
        self.assertEquals(14, len(notifications))
        self.assertDictContainsSubset({'change_type': u'CREATED', 'target_type': u'KEYSPACE'}, notifications[0])
        self.assertDictContainsSubset({'change_type': u'UPDATED', 'target_type': u'KEYSPACE'}, notifications[1])
        self.assertDictContainsSubset({'change_type': u'CREATED', 'target_type': u'TABLE', u'table': u't'}, notifications[2])
        self.assertDictContainsSubset({'change_type': u'UPDATED', 'target_type': u'KEYSPACE'}, notifications[3])
        self.assertDictContainsSubset({'change_type': u'UPDATED', 'target_type': u'TABLE', u'table': u't'}, notifications[4])
        self.assertDictContainsSubset({'change_type': u'UPDATED', 'target_type': u'KEYSPACE'}, notifications[5])
        self.assertDictContainsSubset({'change_type': u'CREATED', 'target_type': u'TABLE', u'table': u'mv'}, notifications[6])
        self.assertDictContainsSubset({'change_type': u'UPDATED', 'target_type': u'KEYSPACE'}, notifications[7])
        self.assertDictContainsSubset({'change_type': u'UPDATED', 'target_type': u'TABLE', u'table': u'mv'}, notifications[8])
        self.assertDictContainsSubset({'change_type': u'UPDATED', 'target_type': u'KEYSPACE'}, notifications[9])
        self.assertDictContainsSubset({'change_type': u'DROPPED', 'target_type': u'TABLE', u'table': u'mv'}, notifications[10])
        self.assertDictContainsSubset({'change_type': u'UPDATED', 'target_type': u'KEYSPACE'}, notifications[11])
        self.assertDictContainsSubset({'change_type': u'DROPPED', 'target_type': u'TABLE', u'table': u't'}, notifications[12])
        self.assertDictContainsSubset({'change_type': u'DROPPED', 'target_type': u'KEYSPACE'}, notifications[13])

class TestVariousNotifications(Tester):
    """
    Tests for various notifications/messages from Cassandra.
    """

    @since('2.2')
    def tombstone_failure_threshold_message_test(self):
        """
        Ensure nodes return an error message in case of TombstoneOverwhelmingExceptions rather
        than dropping the request. A drop makes the coordinator waits for the specified
        read_request_timeout_in_ms.
        @jira_ticket CASSANDRA-7886
        """

        self.allow_log_errors = True
        self.cluster.set_configuration_options(
            values={
                'tombstone_failure_threshold': 500,
                'read_request_timeout_in_ms': 30000,  # 30 seconds
                'range_request_timeout_in_ms': 40000
            }
        )
        self.cluster.populate(3).start()
        node1, node2, node3 = self.cluster.nodelist()
        session = self.patient_cql_connection(node1)

        self.create_ks(session, 'test', 3)
        session.execute(
            "CREATE TABLE test ( "
            "id int, mytext text, col1 int, col2 int, col3 int, "
            "PRIMARY KEY (id, mytext) )"
        )

        # Add data with tombstones
        values = map(lambda i: str(i), range(1000))
        for value in values:
            session.execute(SimpleStatement(
                "insert into test (id, mytext, col1) values (1, '{}', null) ".format(
                    value
                ),
                consistency_level=CL.ALL
            ))

        failure_msg = ("Scanned over.* tombstones.* query aborted")

        @timed(25)
        def read_failure_query():
            assert_invalid(
                session, SimpleStatement("select * from test where id in (1,2,3,4,5)", consistency_level=CL.ALL),
                expected=ReadTimeout if self.cluster.version() < '3' else ReadFailure,
            )

        read_failure_query()

        failure = (node1.grep_log(failure_msg) or
                   node2.grep_log(failure_msg) or
                   node3.grep_log(failure_msg))

        self.assertTrue(failure, ("Cannot find tombstone failure threshold error in log "
                                  "after failed query"))
        mark1 = node1.mark_log()
        mark2 = node2.mark_log()
        mark3 = node3.mark_log()

        @timed(35)
        def range_request_failure_query():
            assert_invalid(
                session, SimpleStatement("select * from test", consistency_level=CL.ALL),
                expected=ReadTimeout if self.cluster.version() < '3' else ReadFailure,
            )

        range_request_failure_query()

        failure = (node1.watch_log_for(failure_msg, from_mark=mark1, timeout=5) or
                   node2.watch_log_for(failure_msg, from_mark=mark2, timeout=5) or
                   node3.watch_log_for(failure_msg, from_mark=mark3, timeout=5))

        self.assertTrue(failure, ("Cannot find tombstone failure threshold error in log "
                                  "after range_request_timeout_query"))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import librosa
import pandas as pd
from amen.feature import Feature

class Audio(object):
    """
    Audio object: should wrap the output from libRosa.
    """

    def __init__(self, file_path, convert_to_mono=False, sample_rate=22050):
        """
        Opens a file path, loads it with librosa.
        """
        self.file_path = file_path
        y, sr = librosa.load(file_path, mono=convert_to_mono, sr=sample_rate)
        self.sample_rate = float(sr)
        self.raw_samples = y
        self.num_channels = y.ndim
        self.duration = librosa.get_duration(y=y, sr=sr)
        self.features = self.create_features()

    def create_features(self):
        features = {}
        features['centroid'] = self.get_centroid()
        return features

    def get_centroid(self):
        """
        Gets spectral centroid data from librosa and loads it into a feature
        """
        hop_length = 512
        mono_samples = librosa.to_mono(self.raw_samples)
        centroid = librosa.feature.spectral_centroid(mono_samples, hop_length=hop_length)
        centroid = centroid[0]

        indexes = range(len(centroid))
        indexes = [((hop_length * i) / self.sample_rate) for i in indexes]
        indexes = pd.to_timedelta(indexes, unit='s')

        data = pd.DataFrame(data=centroid, index=indexes)
        return data

<commit_msg>Using extant code, adding column names<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import librosa
import pandas as pd
from amen.feature import Feature

class Audio(object):
    """
    Audio object: should wrap the output from libRosa.
    """

    def __init__(self, file_path, convert_to_mono=False, sample_rate=22050):
        """
        Opens a file path, loads it with librosa.
        """
        self.file_path = file_path
        y, sr = librosa.load(file_path, mono=convert_to_mono, sr=sample_rate)
        self.sample_rate = float(sr)
        self.raw_samples = y
        self.num_channels = y.ndim
        self.duration = librosa.get_duration(y=y, sr=sr)
        self.features = self.create_features()

    def create_features(self):
        features = {}
        features['centroid'] = self.get_centroid()
        return features

    def get_centroid(self):
        """
        Gets spectral centroid data from librosa and loads it into a feature
        """
        mono_samples = librosa.to_mono(self.raw_samples)
        centroid = librosa.feature.spectral_centroid(mono_samples)
        centroid = centroid[0]

        frame_numbers = range(len(centroid))
        indexes = librosa.frames_to_time(frame_numbers)
        indexes = pd.to_timedelta(indexes, unit='s')

        data = pd.DataFrame(data=centroid, index=indexes, columns=['spectral_centroid'])
        return data

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Created on Tue Mar 26 13:49:04 2013

@author: mmikofski
"""

from functools import partial
from multiprocessing import current_process, Pool, cpu_count
from pvmismatch.pvconstants import npinterpx
from pvmismatch.pvexceptions import PVparallel_calcError
import numpy as np

def parallel_calcSystem(pvsys, Vsys):
    """
    Embarrassingly parallel calculations.
    """

    # protect main thread
    if current_process().name != 'MainProcess':
        raise PVparallel_calcError(__name__)
    # pool overhead is high, create once and reuse processes
    # if procs is None, set it cpu_count - 2, so 2 cores are free
    procs = pvsys.pvconst.procs
    if not procs:
        procs = cpu_count() - 2
    pool = Pool(processes=procs,
                maxtasksperchild=pvsys.pvconst.maxtasksperchild)
    # if chunksize is None, then divide tasks evenly between all procs
    tot_mods = pvsys.numberStrs * pvsys.numberMods
    chunksize = pvsys.pvconst.chunksize
    if not chunksize:
        chunksize = tot_mods / procs
    # limit data pickled and sent to process to reduce overhead
    # flatten pvmods to 1-D
    pvmods = np.reshape(pvsys.pvmods, (tot_mods, ))
    # extract Imod[0], Imod[-1] and Ee, reorganize back into strings of modules
    Istring_args = np.array([(pvmod.Imod[0], pvmod.Imod[-1],
                              np.mean(pvmod.Ee, dtype=float)) for pvmod in
                             pvmods], dtype=float).reshape(pvsys.numberStrs,
                                                           pvsys.numberMods, 3)
    # index-0: strings, index-1: modules, index-2: (Imax, Imin, Eavg)
    Istring_args = np.concatenate((np.min(Istring_args[:, :, 0], 1,
                                          keepdims=True),
                                   np.max(Istring_args[:, :, 1], 1,
                                          keepdims=True),
                                   (np.mean(Istring_args[:, :, 2], 1,
                                            dtype=float, keepdims=True) *
                                    pvsys.pvconst.Isc0)), 1)
    # only use pool if more than one string
    if pvsys.numberStrs == 1:
        # transpose from (<npts>, 1) to (1, <npts>) to match pool.map
        Istring = calcIstring(Istring_args.squeeze(), pvsys.pvconst.Imod_pts,
                              pvsys.pvconst.Imod_negpts).T
    else:
        partial_calcIstring = partial(calcIstring,
                                      Imod_pts=pvsys.pvconst.Imod_pts,
                                      Imod_negpts=pvsys.pvconst.Imod_negpts)
        Istring = np.squeeze(pool.map(partial_calcIstring, Istring_args,
                                      chunksize))
        # squeeze take array-like, removes singleton dimensions
        # converts (<numberStrs>, <npts>, 1) to (<numberStrs>, <npts>)
    # only use pool if more than one module
    if tot_mods == 1:
        Vstring = interpMods(np.squeeze([pvsys.pvmods[0][0].Imod.T,
                                         pvsys.pvmods[0][0].Vmod.T, Istring]))
    else:
        # expand Istring for every module in each string
        # repeat is smart - it repeats the each row <repeats> times,
        # so each string is kept together [[str1-mod1],...,[str2-mod1],...]
        Imodstr = Istring.repeat(pvsys.numberMods,
                                 axis=0).reshape(tot_mods, 1,
                                                 2 * pvsys.pvconst.npts)
        IVmods = np.array([(pvmod.Imod, pvmod.Vmod) for pvmod in pvmods],
                          dtype=float).reshape(tot_mods, 2,
                                               2 * pvsys.pvconst.npts)
        Vstring_args = np.append(IVmods, Imodstr,1)
        Vstring = np.sum(np.reshape(pool.map(interpMods, Vstring_args,
                                             chunksize),
                                    (pvsys.numberStrs, pvsys.numberMods,
                                     2 * pvsys.pvconst.npts)), axis=1)
        # reshape Vstring to reorganize mods in each string
        # IE (<numberStrs>, <numberMods>, <npts>)
        # add up voltages from each mod to get Vstring
        # NOTE: str: 0th dim, mod: 1st dim, npts: 2nd dim
    Pstring = Istring * Vstring
    for (pvstr, P, I, V) in zip(pvsys.pvstrs, Pstring, Istring, Vstring):
        pvstr.Pstring, pvstr.Istring, pvstr.Vstring = P, I, V
    # only use pool if more than one string
    if pvsys.numberStrs == 1:
        Isys = interpString((Vstring, Istring), Vsys)
    else:
        partial_interpString = partial(interpString, Vsys=Vsys)
        Isys = pool.map(partial_interpString, zip(Vstring, Istring),
                        chunksize)
        Isys = np.sum(Isys, axis=0)
    pool.close()
    pool.join()
    # np.sum takes array-like, Isys is already in (<numberStrs>, <npts>, 1)
    # summation along 0th dim reduces shape to (<npts>, 1)
    return Isys

def calcIstring(Istring_args, Imod_pts, Imod_negpts):
    """
    Calculate Istring appropriate to interpolate Vmod.
    :param args: min Imod, max Imod and mean Ee for all modules in each string.
    :returns: Istring
    :rtype: {float, ndarray (PVconstants.npts 1)}
    """
    Isc = Istring_args[2]
    Imax = (Istring_args[1] - Isc) * Imod_pts + Isc
    Ineg = (Istring_args[0] - Isc) * Imod_negpts + Isc
    return np.concatenate((Ineg, Imax), axis=0)

def interpMods(Vstring_arg):
    """
    Interpolate Vmod from Istring.
    :param pvmod: A PVmodule class instance.
    :type pvstr: :class:`PVmodule`
    :returns: Vmod at Istring
    :rtype: {float, ndarray (1 PVconstants.npts)}
    """
    xp = Vstring_arg[0].flatten()
    fp = Vstring_arg[1].flatten()
    return npinterpx(Vstring_arg[2], xp, fp)

def interpString((Vstring, Istring), Vsys):
    """
    Interpolate Istring from Vsys.
    :param pvstr: A PVstring class instance.
    :type pvstr: :class:`PVstring`
    :returns: Istring at Vsys
    :rtype: {float, ndarray (PVconstants.npts, 1)}
    """
    xp = np.flipud(Vstring.flatten())
    fp = np.flipud(Istring.flatten())
    return npinterpx(Vsys, xp, fp)

def parallel_calcMod(pvmod):
    # protect main thread
    if current_process().name != 'MainProcess':
        raise PVparallel_calcError(__name__)

    procs = pvmod.pvconst.procs
    if not procs:
        procs = cpu_count() - 2
    pool = Pool(processes=procs,
                maxtasksperchild=pvmod.pvconst.maxtasksperchild)
    # if chunksize is None, then divide tasks evenly between all procs
    chunksize = pvmod.pvconst.chunksize
    if not chunksize:
        chunksize = pvmod.numberCells / procs
    # create partial funcion with VRBD constant
    partial_calcIatVrbd = partial(calcIatVrbd, VRBD=pvmod.pvconst.VRBD)
    # calculate I at VRBD for every cell
    IatVrbd = pool.map(partial_calcIatVrbd, zip(pvmod.Vcell.T, pvmod.Icell.T),
                       chunksize)
    Isc = np.mean(pvmod.Ee) * pvmod.pvconst.Isc0
    Imax = (np.max(IatVrbd) - Isc) * pvmod.pvconst.Imod_pts + Isc # max current
    Imin = np.min(pvmod.Icell)
    Imin = Imin if Imin < 0 else 0
    Ineg = (Imin - Isc) * pvmod.pvconst.Imod_negpts + Isc  # min current
    Imod = np.concatenate((Ineg, Imax), axis=0)  # interpolation range
    Vsubstr = np.zeros((2 * pvmod.pvconst.npts, 3))
    start = np.cumsum(pvmod.subStrCells) - pvmod.subStrCells
    stop = np.cumsum(pvmod.subStrCells)
    # partial with Imod constant
    partial_interpVsubstrcells = partial(interpVsubstrcells,Imod=Imod)
    # interpolate cell voltages
    Vsubstrcells = np.squeeze(pool.map(partial_interpVsubstrcells,
                              zip(pvmod.Icell.T, pvmod.Vcell.T), chunksize))
    # loop over substrings
    for substr in range(pvmod.numSubStr):
        cells = range(start[substr], stop[substr])
        # calculate substring voltages
        Vsubstr[:, substr] = np.sum(Vsubstrcells[cells][:], axis=0).flatten()
    bypassed = Vsubstr < pvmod.pvconst.Vbypass
    Vsubstr[bypassed] = pvmod.pvconst.Vbypass
    Vmod = np.sum(Vsubstr, 1).reshape(2 * pvmod.pvconst.npts, 1)
    Pmod = Imod * Vmod
    pool.close()
    pool.join()
    return (Imod, Vmod, Pmod, Vsubstr)

def calcIatVrbd((Vcell, Icell), VRBD):
    return np.interp(VRBD, Vcell, Icell)

def interpVsubstrcells((Icell, Vcell), Imod):
    xp = np.flipud(Icell)
    fp = np.flipud(Vcell)
    return npinterpx(Imod, xp, fp)

def parallel_calcString():
    pass
#     pool.map()
<commit_msg>limit min procs/chunksize to 1<commit_after># -*- coding: utf-8 -*-

"""
Created on Tue Mar 26 13:49:04 2013

@author: mmikofski
"""

from functools import partial
from multiprocessing import current_process, Pool, cpu_count
from pvmismatch.pvconstants import npinterpx
from pvmismatch.pvexceptions import PVparallel_calcError
import numpy as np

def parallel_calcSystem(pvsys, Vsys):
    """
    Embarrassingly parallel calculations.
    """

    # protect main thread
    if current_process().name != 'MainProcess':
        raise PVparallel_calcError(__name__)
    # pool overhead is high, create once and reuse processes
    # if procs is None, set it cpu_count - 2, so 2 cores are free
    procs = pvsys.pvconst.procs
    if not procs:
        procs = max(cpu_count() - 2, 1)
    pool = Pool(processes=procs,
                maxtasksperchild=pvsys.pvconst.maxtasksperchild)
    # if chunksize is None, then divide tasks evenly between all procs
    tot_mods = pvsys.numberStrs * pvsys.numberMods
    chunksize = pvsys.pvconst.chunksize
    if not chunksize:
        chunksize = max(tot_mods / procs, 1)
    # limit data pickled and sent to process to reduce overhead
    # flatten pvmods to 1-D
    pvmods = np.reshape(pvsys.pvmods, (tot_mods, ))
    # extract Imod[0], Imod[-1] and Ee, reorganize back into strings of modules
    Istring_args = np.array([(pvmod.Imod[0], pvmod.Imod[-1],
                              np.mean(pvmod.Ee, dtype=float)) for pvmod in
                             pvmods], dtype=float).reshape(pvsys.numberStrs,
                                                           pvsys.numberMods, 3)
    # index-0: strings, index-1: modules, index-2: (Imax, Imin, Eavg)
    Istring_args = np.concatenate((np.min(Istring_args[:, :, 0], 1,
                                          keepdims=True),
                                   np.max(Istring_args[:, :, 1], 1,
                                          keepdims=True),
                                   (np.mean(Istring_args[:, :, 2], 1,
                                            dtype=float, keepdims=True) *
                                    pvsys.pvconst.Isc0)), 1)
    # only use pool if more than one string
    if pvsys.numberStrs == 1:
        # transpose from (<npts>, 1) to (1, <npts>) to match pool.map
        Istring = calcIstring(Istring_args.squeeze(), pvsys.pvconst.Imod_pts,
                              pvsys.pvconst.Imod_negpts).T
    else:
        partial_calcIstring = partial(calcIstring,
                                      Imod_pts=pvsys.pvconst.Imod_pts,
                                      Imod_negpts=pvsys.pvconst.Imod_negpts)
        Istring = np.squeeze(pool.map(partial_calcIstring, Istring_args,
                                      chunksize))
        # squeeze take array-like, removes singleton dimensions
        # converts (<numberStrs>, <npts>, 1) to (<numberStrs>, <npts>)
    # only use pool if more than one module
    if tot_mods == 1:
        Vstring = interpMods(np.squeeze([pvsys.pvmods[0][0].Imod.T,
                                         pvsys.pvmods[0][0].Vmod.T, Istring]))
    else:
        # expand Istring for every module in each string
        # repeat is smart - it repeats the each row <repeats> times,
        # so each string is kept together [[str1-mod1],...,[str2-mod1],...]
        Imodstr = Istring.repeat(pvsys.numberMods,
                                 axis=0).reshape(tot_mods, 1,
                                                 2 * pvsys.pvconst.npts)
        IVmods = np.array([(pvmod.Imod, pvmod.Vmod) for pvmod in pvmods],
                          dtype=float).reshape(tot_mods, 2,
                                               2 * pvsys.pvconst.npts)
        Vstring_args = np.append(IVmods, Imodstr,1)
        Vstring = np.sum(np.reshape(pool.map(interpMods, Vstring_args,
                                             chunksize),
                                    (pvsys.numberStrs, pvsys.numberMods,
                                     2 * pvsys.pvconst.npts)), axis=1)
        # reshape Vstring to reorganize mods in each string
        # IE (<numberStrs>, <numberMods>, <npts>)
        # add up voltages from each mod to get Vstring
        # NOTE: str: 0th dim, mod: 1st dim, npts: 2nd dim
    Pstring = Istring * Vstring
    for (pvstr, P, I, V) in zip(pvsys.pvstrs, Pstring, Istring, Vstring):
        pvstr.Pstring, pvstr.Istring, pvstr.Vstring = P, I, V
    # only use pool if more than one string
    if pvsys.numberStrs == 1:
        Isys = interpString((Vstring, Istring), Vsys)
    else:
        partial_interpString = partial(interpString, Vsys=Vsys)
        Isys = pool.map(partial_interpString, zip(Vstring, Istring),
                        chunksize)
        Isys = np.sum(Isys, axis=0)
    pool.close()
    pool.join()
    # np.sum takes array-like, Isys is already in (<numberStrs>, <npts>, 1)
    # summation along 0th dim reduces shape to (<npts>, 1)
    return Isys

def calcIstring(Istring_args, Imod_pts, Imod_negpts):
    """
    Calculate Istring appropriate to interpolate Vmod.
    :param args: min Imod, max Imod and mean Ee for all modules in each string.
    :returns: Istring
    :rtype: {float, ndarray (PVconstants.npts 1)}
    """
    Isc = Istring_args[2]
    Imax = (Istring_args[1] - Isc) * Imod_pts + Isc
    Ineg = (Istring_args[0] - Isc) * Imod_negpts + Isc
    return np.concatenate((Ineg, Imax), axis=0)

def interpMods(Vstring_arg):
    """
    Interpolate Vmod from Istring.
    :param pvmod: A PVmodule class instance.
    :type pvstr: :class:`PVmodule`
    :returns: Vmod at Istring
    :rtype: {float, ndarray (1 PVconstants.npts)}
    """
    xp = Vstring_arg[0].flatten()
    fp = Vstring_arg[1].flatten()
    return npinterpx(Vstring_arg[2], xp, fp)

def interpString((Vstring, Istring), Vsys):
    """
    Interpolate Istring from Vsys.
    :param pvstr: A PVstring class instance.
    :type pvstr: :class:`PVstring`
    :returns: Istring at Vsys
    :rtype: {float, ndarray (PVconstants.npts, 1)}
    """
    xp = np.flipud(Vstring.flatten())
    fp = np.flipud(Istring.flatten())
    return npinterpx(Vsys, xp, fp)

def parallel_calcMod(pvmod):
    # protect main thread
    if current_process().name != 'MainProcess':
        raise PVparallel_calcError(__name__)

    procs = pvmod.pvconst.procs
    if not procs:
        procs = max(cpu_count() - 2, 1)
    pool = Pool(processes=procs,
                maxtasksperchild=pvmod.pvconst.maxtasksperchild)
    # if chunksize is None, then divide tasks evenly between all procs
    chunksize = pvmod.pvconst.chunksize
    if not chunksize:
        chunksize = max(pvmod.numberCells / procs, 1)
    # create partial funcion with VRBD constant
    partial_calcIatVrbd = partial(calcIatVrbd, VRBD=pvmod.pvconst.VRBD)
    # calculate I at VRBD for every cell
    IatVrbd = pool.map(partial_calcIatVrbd, zip(pvmod.Vcell.T, pvmod.Icell.T),
                       chunksize)
    Isc = np.mean(pvmod.Ee) * pvmod.pvconst.Isc0
    Imax = (np.max(IatVrbd) - Isc) * pvmod.pvconst.Imod_pts + Isc # max current
    Imin = np.min(pvmod.Icell)
    Imin = Imin if Imin < 0 else 0
    Ineg = (Imin - Isc) * pvmod.pvconst.Imod_negpts + Isc  # min current
    Imod = np.concatenate((Ineg, Imax), axis=0)  # interpolation range
    Vsubstr = np.zeros((2 * pvmod.pvconst.npts, 3))
    start = np.cumsum(pvmod.subStrCells) - pvmod.subStrCells
    stop = np.cumsum(pvmod.subStrCells)
    # partial with Imod constant
    partial_interpVsubstrcells = partial(interpVsubstrcells,Imod=Imod)
    # interpolate cell voltages
    Vsubstrcells = np.squeeze(pool.map(partial_interpVsubstrcells,
                              zip(pvmod.Icell.T, pvmod.Vcell.T), chunksize))
    # loop over substrings
    for substr in range(pvmod.numSubStr):
        cells = range(start[substr], stop[substr])
        # calculate substring voltages
        Vsubstr[:, substr] = np.sum(Vsubstrcells[cells][:], axis=0).flatten()
    bypassed = Vsubstr < pvmod.pvconst.Vbypass
    Vsubstr[bypassed] = pvmod.pvconst.Vbypass
    Vmod = np.sum(Vsubstr, 1).reshape(2 * pvmod.pvconst.npts, 1)
    Pmod = Imod * Vmod
    pool.close()
    pool.join()
    return (Imod, Vmod, Pmod, Vsubstr)

def calcIatVrbd((Vcell, Icell), VRBD):
    return np.interp(VRBD, Vcell, Icell)

def interpVsubstrcells((Icell, Vcell), Imod):
    xp = np.flipud(Icell)
    fp = np.flipud(Vcell)
    return npinterpx(Imod, xp, fp)

def parallel_calcString():
    pass
#     pool.map()
<|endoftext|>


<commit_before>
import pyxnat

def __pyxnat_assessor_type():
    return pyxnat.core.resources.Assessor

def full_label(project_id, subject_id, session_id,
               assessor_label):
    return '-x-'.join([project_id, subject_id, session_id, assessor_label])

# TODO: BenM/assessor_of_assessor/behaviour contingent on assessor version;
# the name shouldn't be used to test this
def full_label_from_assessor(assessor):
    assessor_label = assessor.label()
    if '-x-' in assessor_label:
        return assessor_label

    components = list()
    components.append(assessor_label)
    entity = assessor
    while True:
        entity = entity.parent()
        if entity is None:
            break
        components.append(entity.label())
    return '-x-'.join(reversed(components))

def parse_full_assessor_name(assessor_name):
    elements = assessor_name.split('-x-')

    if len(elements) == 5:
        # relabel is in use so no need to parse
        return assessor_name

    if len(elements) != 4:
        raise ValueError(("'assessor_name' parameter '{}' is not a valid full "
                          "assessor name".format(assessor_name)))
    return dict(zip(
        ['project_id', 'subject_label', 'session_label', 'label'],
        elements))
<commit_msg>Handle old and new assr labels<commit_after>
import pyxnat

def __pyxnat_assessor_type():
    return pyxnat.core.resources.Assessor

def full_label(project_id, subject_id, session_id,
               assessor_label):
    return '-x-'.join([project_id, subject_id, session_id, assessor_label])

# TODO: BenM/assessor_of_assessor/behaviour contingent on assessor version;
# the name shouldn't be used to test this
def full_label_from_assessor(assessor):
    assessor_label = assessor.label()
    if '-x-' in assessor_label:
        return assessor_label

    components = list()
    components.append(assessor_label)
    entity = assessor
    while True:
        entity = entity.parent()
        if entity is None:
            break
        components.append(entity.label())
    return '-x-'.join(reversed(components))

def parse_full_assessor_name(assessor_name):
    elements = assessor_name.split('-x-')
    assrdict = dict()

    if len(elements) == 5:
        # relabel is in use or old style with scan id in label
        assrdict = zip(
            ['project_id', 'subject_label', 'session_label', 'label'],
            [elements[0], elements[1], elements[2], assessor_name])

    if len(elements) != 4:
        raise ValueError(("'assessor_name' parameter '{}' is not a valid full "
                          "assessor name".format(assessor_name)))

    if len(elements[3]) == 36:
        # new style label with uuid
        assrdict = zip(
            ['project_id', 'subject_label', 'session_label', 'label'],
            elements)
    else:
        # old style label
        assrdict = zip(
            ['project_id', 'subject_label', 'session_label', 'label'],
            [elements[0], elements[1], elements[2], assessor_name])

    return assrdict
<|endoftext|>


<commit_before>import logging
l = logging.getLogger('angr.block')

import pyvex
from archinfo import ArchARM

class Block(object):
    BLOCK_MAX_SIZE = 4096

    __slots__ = ['_project', '_bytes', '_vex', 'thumb', '_capstone', 'addr', 'size', 'instructions', '_instruction_addrs']

    def __init__(self, project, addr, size=None, byte_string=None, vex=None, thumb=False, backup_state=None,
                 opt_level=None, num_inst=None):
        if isinstance(project.arch, ArchARM):
            if addr & 1 == 1:
                thumb = True
            if thumb:
                addr |= 1
        else:
            thumb = False

        self._project = project
        self.thumb = thumb
        self.addr = addr

        if size is None:
            if byte_string is not None:
                size = len(byte_string)
            elif vex is not None:
                size = vex.size
            else:
                vex = project.factory.default_engine.lift(
                        clemory=project.loader.memory,
                        state=backup_state,
                        insn_bytes=byte_string,
                        addr=addr,
                        thumb=thumb,
                        opt_level=opt_level,
                        num_inst=num_inst)
                size = vex.size

        self._vex = vex
        self._capstone = None
        self.size = size

        self.instructions = None
        self._instruction_addrs = []

        self._parse_vex_info()

        if byte_string is None:
            self._bytes = None
        elif type(byte_string) is str:
            if self.size is not None:
                self._bytes = byte_string[:self.size]
            else:
                self._bytes = byte_string
        else:
            # Convert bytestring to a str
            # size will ALWAYS be known at this point
            self._bytes = str(pyvex.ffi.buffer(byte_string, self.size))

    def _parse_vex_info(self):
        vex = self._vex
        if vex is not None:
            self.instructions = vex.instructions
            self._instruction_addrs = []

            for stmt in vex.statements:
                if stmt.tag != 'Ist_IMark':
                    continue
                if self.addr is None:
                    self.addr = stmt.addr + stmt.delta
                self._instruction_addrs.append(stmt.addr + stmt.delta)

    def __repr__(self):
        return '<Block for %#x, %d bytes>' % (self.addr, self.size)

    def __getstate__(self):
        return dict((k, getattr(self, k)) for k in self.__slots__ if k not in ('_capstone', ))

    def __setstate__(self, data):
        for k, v in data.iteritems():
            setattr(self, k, v)

    def __hash__(self):
        return hash((type(self), self.addr, self.bytes))

    def __eq__(self, other):
        return type(self) is type(other) and \
               self.addr == other.addr and \
               self.bytes == other.bytes

    def __ne__(self, other):
        return not self == other

    def pp(self):
        return self.capstone.pp()

    @property
    def vex(self):
        if not self._vex:
            self._vex = self._project.factory.default_engine.lift(
                    clemory=self._project.loader.memory,
                    insn_bytes=self._bytes,
                    addr=self.addr,
                    thumb=self.thumb)
            self._parse_vex_info()

        return self._vex

    @property
    def capstone(self):
        if self._capstone: return self._capstone

        cs = self._project.arch.capstone if not self.thumb else self._project.arch.capstone_thumb

        insns = []

        for cs_insn in cs.disasm(self.bytes, self.addr):
            insns.append(CapstoneInsn(cs_insn))
        block = CapstoneBlock(self.addr, insns, self.thumb, self._project.arch)

        self._capstone = block
        return block

    @property
    def codenode(self):
        return BlockNode(self.addr, self.size, bytestr=self.bytes)

    @property
    def bytes(self):
        if self._bytes is None:
            addr = self.addr
            if self.thumb:
                addr = (addr >> 1) << 1
            self._bytes = ''.join(self._project.loader.memory.read_bytes(addr, self.size))
        return self._bytes

    @property
    def instruction_addrs(self):
        if not self._instruction_addrs and self._vex is None:
            # initialize instruction addrs
            _ = self.vex

        return self._instruction_addrs

class CapstoneBlock(object):
    """
    Deep copy of the capstone blocks, which have serious issues with having extended lifespans
    outside of capstone itself
    """
    __slots__ = [ 'addr', 'insns', 'thumb', 'arch' ]

    def __init__(self, addr, insns, thumb, arch):
        self.addr = addr
        self.insns = insns
        self.thumb = thumb
        self.arch = arch

    def pp(self):
        print str(self)

    def __str__(self):
        return '\n'.join(map(str, self.insns))

    def __repr__(self):
        return '<CapstoneBlock for %#x>' % self.addr

class CapstoneInsn(object):
    def __init__(self, capstone_insn):
        self.insn = capstone_insn

    def __getattr__(self, item):
        if item in ('__str__', '__repr__'):
            return self.__getattribute__(item)
        if hasattr(self.insn, item):
            return getattr(self.insn, item)
        raise AttributeError()

    def __str__(self):
        return "%#x:\t%s\t%s" % (self.address, self.mnemonic, self.op_str)

    def __repr__(self):
        return '<CapstoneInsn "%s" for %#x>' % (self.mnemonic, self.address)

from .knowledge.codenode import BlockNode
<commit_msg>Block: pass self.size to SimEngineVEX.lift().<commit_after>import logging
l = logging.getLogger('angr.block')

import pyvex
from archinfo import ArchARM

class Block(object):
    BLOCK_MAX_SIZE = 4096

    __slots__ = ['_project', '_bytes', '_vex', 'thumb', '_capstone', 'addr', 'size', 'instructions', '_instruction_addrs']

    def __init__(self, project, addr, size=None, byte_string=None, vex=None, thumb=False, backup_state=None,
                 opt_level=None, num_inst=None):
        if isinstance(project.arch, ArchARM):
            if addr & 1 == 1:
                thumb = True
            if thumb:
                addr |= 1
        else:
            thumb = False

        self._project = project
        self.thumb = thumb
        self.addr = addr

        if size is None:
            if byte_string is not None:
                size = len(byte_string)
            elif vex is not None:
                size = vex.size
            else:
                vex = project.factory.default_engine.lift(
                        clemory=project.loader.memory,
                        state=backup_state,
                        insn_bytes=byte_string,
                        addr=addr,
                        thumb=thumb,
                        opt_level=opt_level,
                        num_inst=num_inst)
                size = vex.size

        self._vex = vex
        self._capstone = None
        self.size = size

        self.instructions = None
        self._instruction_addrs = []

        self._parse_vex_info()

        if byte_string is None:
            self._bytes = None
        elif type(byte_string) is str:
            if self.size is not None:
                self._bytes = byte_string[:self.size]
            else:
                self._bytes = byte_string
        else:
            # Convert bytestring to a str
            # size will ALWAYS be known at this point
            self._bytes = str(pyvex.ffi.buffer(byte_string, self.size))

    def _parse_vex_info(self):
        vex = self._vex
        if vex is not None:
            self.instructions = vex.instructions
            self._instruction_addrs = []

            for stmt in vex.statements:
                if stmt.tag != 'Ist_IMark':
                    continue
                if self.addr is None:
                    self.addr = stmt.addr + stmt.delta
                self._instruction_addrs.append(stmt.addr + stmt.delta)

    def __repr__(self):
        return '<Block for %#x, %d bytes>' % (self.addr, self.size)

    def __getstate__(self):
        return dict((k, getattr(self, k)) for k in self.__slots__ if k not in ('_capstone', ))

    def __setstate__(self, data):
        for k, v in data.iteritems():
            setattr(self, k, v)

    def __hash__(self):
        return hash((type(self), self.addr, self.bytes))

    def __eq__(self, other):
        return type(self) is type(other) and \
               self.addr == other.addr and \
               self.bytes == other.bytes

    def __ne__(self, other):
        return not self == other

    def pp(self):
        return self.capstone.pp()

    @property
    def vex(self):
        if not self._vex:
            self._vex = self._project.factory.default_engine.lift(
                    clemory=self._project.loader.memory,
                    insn_bytes=self._bytes,
                    addr=self.addr,
                    thumb=self.thumb,
                    max_size=self.size,
            )
            self._parse_vex_info()

        return self._vex

    @property
    def capstone(self):
        if self._capstone: return self._capstone

        cs = self._project.arch.capstone if not self.thumb else self._project.arch.capstone_thumb

        insns = []

        for cs_insn in cs.disasm(self.bytes, self.addr):
            insns.append(CapstoneInsn(cs_insn))
        block = CapstoneBlock(self.addr, insns, self.thumb, self._project.arch)

        self._capstone = block
        return block

    @property
    def codenode(self):
        return BlockNode(self.addr, self.size, bytestr=self.bytes)

    @property
    def bytes(self):
        if self._bytes is None:
            addr = self.addr
            if self.thumb:
                addr = (addr >> 1) << 1
            self._bytes = ''.join(self._project.loader.memory.read_bytes(addr, self.size))
        return self._bytes

    @property
    def instruction_addrs(self):
        if not self._instruction_addrs and self._vex is None:
            # initialize instruction addrs
            _ = self.vex

        return self._instruction_addrs

class CapstoneBlock(object):
    """
    Deep copy of the capstone blocks, which have serious issues with having extended lifespans
    outside of capstone itself
    """
    __slots__ = [ 'addr', 'insns', 'thumb', 'arch' ]

    def __init__(self, addr, insns, thumb, arch):
        self.addr = addr
        self.insns = insns
        self.thumb = thumb
        self.arch = arch

    def pp(self):
        print str(self)

    def __str__(self):
        return '\n'.join(map(str, self.insns))

    def __repr__(self):
        return '<CapstoneBlock for %#x>' % self.addr

class CapstoneInsn(object):
    def __init__(self, capstone_insn):
        self.insn = capstone_insn

    def __getattr__(self, item):
        if item in ('__str__', '__repr__'):
            return self.__getattribute__(item)
        if hasattr(self.insn, item):
            return getattr(self.insn, item)
        raise AttributeError()

    def __str__(self):
        return "%#x:\t%s\t%s" % (self.address, self.mnemonic, self.op_str)

    def __repr__(self):
        return '<CapstoneInsn "%s" for %#x>' % (self.mnemonic, self.address)

from .knowledge.codenode import BlockNode
<|endoftext|>


<commit_before>#
# GRAKN.AI - THE KNOWLEDGE GRAPH
# Copyright (C) 2019 Grakn Labs Ltd
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
#

load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")

def graknlabs_build_tools():
    git_repository(
        name = "graknlabs_build_tools",
        remote = "https://github.com/graknlabs/build-tools",
        commit = "cdb69d6fd4f8384c910802df7d949edabd246e24", # sync-marker: do not remove this comment, this is used for sync-dependencies by @graknlabs_build_tools
    )

def graknlabs_common():
    git_repository(
        name = "graknlabs_common",
        remote = "https://github.com/graknlabs/common",
        commit = "6c5c085810f02a1b6958386d78ffe76d8510f7f2", # sync-marker: do not remove this comment, this is used for sync-dependencies by @graknlabs_common
    )

def graknlabs_graql():
     git_repository(
         name = "graknlabs_graql",
         remote = "https://github.com/graknlabs/graql",
         commit = "102d14c4c3cb7a87961f325778db84a8f536197f", # sync-marker: do not remove this comment, this is used for sync-dependencies by @graknlabs_graql
     )

def graknlabs_protocol():
    git_repository(
        name = "graknlabs_protocol",
        remote = "https://github.com/graknlabs/protocol",
        commit = "0f59f62d8ce7002bea385a41959b4dbd803b0caf", # sync-marker: do not remove this comment, this is used for sync-dependencies by @graknlabs_protocol
    )

def graknlabs_client_java():
     git_repository(
         name = "graknlabs_client_java",
         remote = "https://github.com/graknlabs/client-java",
         commit = "b9c379a90cb670ac720879ffbbbeb38ca049b67d",
     )

def graknlabs_console():
    git_repository(
        name = "graknlabs_console",
        remote = "https://github.com/graknlabs/console",
        commit = "bf3a8ce00025e230ab7211af5421509a814708b8"
    )

def graknlabs_benchmark():
    git_repository(
        name = "graknlabs_benchmark",
        remote = "https://github.com/graknlabs/benchmark.git",
        commit = "186eeabf8122c209cc7d0ab290c9fe82b2185cc8" # keep in sync with protocol changes
    )
<commit_msg>//ci:sync-dependencies: graknlabs/build-tools@499a203<commit_after>#
# GRAKN.AI - THE KNOWLEDGE GRAPH
# Copyright (C) 2019 Grakn Labs Ltd
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
#

load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")

def graknlabs_build_tools():
    git_repository(
        name = "graknlabs_build_tools",
        remote = "https://github.com/graknlabs/build-tools",
        commit = "499a203981325e3ee50e3fc2f5298bad8a6bf683", # sync-marker: do not remove this comment, this is used for sync-dependencies by @graknlabs_build_tools
    )

def graknlabs_common():
    git_repository(
        name = "graknlabs_common",
        remote = "https://github.com/graknlabs/common",
        commit = "72132e57cfea58391e64def6ef76155bfb0316d7", # sync-marker: do not remove this comment, this is used for sync-dependencies by @graknlabs_common
    )

def graknlabs_graql():
     git_repository(
         name = "graknlabs_graql",
         remote = "https://github.com/graknlabs/graql",
         commit = "a81eacf87eebe71a68ea32e74e11b6e6a101e2bf", # sync-marker: do not remove this comment, this is used for sync-dependencies by @graknlabs_graql
     )

def graknlabs_protocol():
    git_repository(
        name = "graknlabs_protocol",
        remote = "https://github.com/graknlabs/protocol",
        commit = "e081ddbdd2c94ed40dafc740d1179b4c195e869c", # sync-marker: do not remove this comment, this is used for sync-dependencies by @graknlabs_protocol
    )

def graknlabs_client_java():
     git_repository(
         name = "graknlabs_client_java",
         remote = "https://github.com/graknlabs/client-java",
         commit = "b9c379a90cb670ac720879ffbbbeb38ca049b67d",
     )

def graknlabs_console():
    git_repository(
        name = "graknlabs_console",
        remote = "https://github.com/graknlabs/console",
        commit = "bf3a8ce00025e230ab7211af5421509a814708b8"
    )

def graknlabs_benchmark():
    git_repository(
        name = "graknlabs_benchmark",
        remote = "https://github.com/graknlabs/benchmark.git",
        commit = "186eeabf8122c209cc7d0ab290c9fe82b2185cc8" # keep in sync with protocol changes
    )
<|endoftext|>