<commit_before># -*- coding: utf-8 -*-
import logging
from util import full_stack
from util import get_credentials_for
from workflow.steps.utils.base import BaseStep
from workflow.exceptions.error_codes import DBAAS_0020
from dbaas_credentials.models import CredentialType
from dbaas_cloudstack.models import DatabaseInfraAttr
from dbaas_aclapi.models import DatabaseInfraInstanceBind
from dbaas_aclapi.acl_base_client import AclClient
from dbaas_aclapi import helpers
from dbaas_aclapi.models import ERROR

LOG = logging.getLogger(__name__)

class BindNewInstances(BaseStep):

    def __unicode__(self):
        return "Binding new instances ..."

    def do(self, workflow_dict):

        try:
            database = workflow_dict['database']
            databaseinfra = workflow_dict['databaseinfra']

            acl_credential = get_credentials_for(environment=database.environment,
                                                 credential_type=CredentialType.ACLAPI)
            acl_client = AclClient(acl_credential.endpoint,
                                   acl_credential.user,
                                   acl_credential.password,
                                   database.environment)

            instances = databaseinfra.instances.filter(
                future_instance__isnull=True)

            databaseinfraattr_instances = DatabaseInfraAttr.objects.filter(databaseinfra=databaseinfra,
                                                                           equivalent_dbinfraattr__isnull=True)
            instance_address_list = []
            for instance in instances:
                instance_address_list.append(instance.address)
            for instance in databaseinfraattr_instances:
                instance_address_list.append(instance.ip)

            for database_bind in database.acl_binds.all():
                if helpers.bind_address(
                        database_bind, acl_client, instances=instances,
                        infra_attr_instances=databaseinfraattr_instances):
                    continue
                else:
                    LOG.error("The AclApi is not working properly.")
                    database_bind.bind_status = ERROR
                    database_bind.save()
                    DatabaseInfraInstanceBind.objects.filter(databaseinfra=databaseinfra,
                                                             bind_address=database_bind.bind_address,
                                                             instance__in=instance_address_list
                                                             ).update(bind_status=ERROR)
            return True

        except Exception:
            traceback = full_stack()

            workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
            workflow_dict['exceptions']['traceback'].append(traceback)

            return False

    def undo(self, workflow_dict):

        LOG.info("Running undo...")
        try:
            database = workflow_dict['database']
            databaseinfra = workflow_dict['databaseinfra']

            acl_credential = get_credentials_for(environment=database.environment,
                                                 credential_type=CredentialType.ACLAPI)
            acl_client = AclClient(acl_credential.endpoint,
                                   acl_credential.user,
                                   acl_credential.password,
                                   database.environment)

            instances = databaseinfra.instances.filter(
                future_instance__isnull=True)
            databaseinfraattr_instances = DatabaseInfraAttr.objects.filter(databaseinfra=databaseinfra,
                                                                           equivalent_dbinfraattr__isnull=True)

            instance_address_list = []
            for instance in instances:
                instance_address_list.append(instance.address)
            for instance in databaseinfraattr_instances:
                instance_address_list.append(instance.ip)

            for database_bind in database.acl_binds.all():
                infra_instances_binds = DatabaseInfraInstanceBind.objects.filter(
                    databaseinfra=databaseinfra,
                    instance__in=instance_address_list,
                    bind_address=database_bind.bind_address)
                if helpers.unbind_address(database_bind, acl_client, infra_instances_binds):
                    continue

            return True

        except Exception:
            traceback = full_stack()

            workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
            workflow_dict['exceptions']['traceback'].append(traceback)

            return False

class UnbindOldInstances(BaseStep):

    def __unicode__(self):
        return "Unbinding old instances ..."

    def do(self, workflow_dict):

        try:
            database = workflow_dict['database']
            databaseinfra = workflow_dict['databaseinfra']

            acl_credential = get_credentials_for(environment=database.environment,
                                                 credential_type=CredentialType.ACLAPI)
            acl_client = AclClient(acl_credential.endpoint,
                                   acl_credential.user,
                                   acl_credential.password,
                                   database.environment)

            instances = databaseinfra.instances.filter(
                future_instance__isnull=False)
            databaseinfraattr_instances = DatabaseInfraAttr.objects.filter(databaseinfra=databaseinfra,
                                                                           equivalent_dbinfraattr__isnull=False)

            instance_address_list = []
            for instance in instances:
                instance_address_list.append(instance.address)
            for instance in databaseinfraattr_instances:
                instance_address_list.append(instance.ip)

            for database_bind in database.acl_binds.all():
                infra_instances_binds = DatabaseInfraInstanceBind.objects.filter(
                    databaseinfra=databaseinfra,
                    instance__in=instance_address_list,
                    bind_address=database_bind.bind_address)
                if helpers.unbind_address(database_bind, acl_client, infra_instances_binds):
                    continue

            return True

        except Exception:
            traceback = full_stack()

            workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
            workflow_dict['exceptions']['traceback'].append(traceback)

            return False

    def undo(self, workflow_dict):

        LOG.info("Running undo...")
        try:
            database = workflow_dict['database']
            databaseinfra = workflow_dict['databaseinfra']

            acl_credential = get_credentials_for(environment=database.environment,
                                                 credential_type=CredentialType.ACLAPI)
            acl_client = AclClient(acl_credential.endpoint,
                                   acl_credential.user,
                                   acl_credential.password,
                                   database.environment)

            instances = databaseinfra.instances.filter(
                future_instance__isnull=False)

            databaseinfraattr_instances = DatabaseInfraAttr.objects.filter(databaseinfra=databaseinfra,
                                                                           equivalent_dbinfraattr__isnull=False)
            instance_address_list = []
            for instance in instances:
                instance_address_list.append(instance.address)
            for instance in databaseinfraattr_instances:
                instance_address_list.append(instance.ip)

            for database_bind in database.acl_binds.all():
                if helpers.bind_address(
                        database_bind, acl_client, instances=instances,
                        infra_attr_instances=databaseinfraattr_instances):
                    continue
                else:
                    LOG.error("The AclApi is not working properly.")
                    database_bind.bind_status = ERROR
                    database_bind.save()
                    DatabaseInfraInstanceBind.objects.filter(databaseinfra=databaseinfra,
                                                             bind_address=database_bind.bind_address,
                                                             instance__in=instance_address_list
                                                             ).update(bind_status=ERROR)
            return True

        except Exception:
            traceback = full_stack()

            workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
            workflow_dict['exceptions']['traceback'].append(traceback)

            return False
<commit_msg>Fix import error<commit_after># -*- coding: utf-8 -*-
import logging
from util import full_stack
from util import get_credentials_for
from workflow.steps.util.base import BaseStep
from workflow.exceptions.error_codes import DBAAS_0020
from dbaas_credentials.models import CredentialType
from dbaas_cloudstack.models import DatabaseInfraAttr
from dbaas_aclapi.models import DatabaseInfraInstanceBind
from dbaas_aclapi.acl_base_client import AclClient
from dbaas_aclapi import helpers
from dbaas_aclapi.models import ERROR

LOG = logging.getLogger(__name__)

class BindNewInstances(BaseStep):

    def __unicode__(self):
        return "Binding new instances ..."

    def do(self, workflow_dict):

        try:
            database = workflow_dict['database']
            databaseinfra = workflow_dict['databaseinfra']

            acl_credential = get_credentials_for(environment=database.environment,
                                                 credential_type=CredentialType.ACLAPI)
            acl_client = AclClient(acl_credential.endpoint,
                                   acl_credential.user,
                                   acl_credential.password,
                                   database.environment)

            instances = databaseinfra.instances.filter(
                future_instance__isnull=True)

            databaseinfraattr_instances = DatabaseInfraAttr.objects.filter(databaseinfra=databaseinfra,
                                                                           equivalent_dbinfraattr__isnull=True)
            instance_address_list = []
            for instance in instances:
                instance_address_list.append(instance.address)
            for instance in databaseinfraattr_instances:
                instance_address_list.append(instance.ip)

            for database_bind in database.acl_binds.all():
                if helpers.bind_address(
                        database_bind, acl_client, instances=instances,
                        infra_attr_instances=databaseinfraattr_instances):
                    continue
                else:
                    LOG.error("The AclApi is not working properly.")
                    database_bind.bind_status = ERROR
                    database_bind.save()
                    DatabaseInfraInstanceBind.objects.filter(databaseinfra=databaseinfra,
                                                             bind_address=database_bind.bind_address,
                                                             instance__in=instance_address_list
                                                             ).update(bind_status=ERROR)
            return True

        except Exception:
            traceback = full_stack()

            workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
            workflow_dict['exceptions']['traceback'].append(traceback)

            return False

    def undo(self, workflow_dict):

        LOG.info("Running undo...")
        try:
            database = workflow_dict['database']
            databaseinfra = workflow_dict['databaseinfra']

            acl_credential = get_credentials_for(environment=database.environment,
                                                 credential_type=CredentialType.ACLAPI)
            acl_client = AclClient(acl_credential.endpoint,
                                   acl_credential.user,
                                   acl_credential.password,
                                   database.environment)

            instances = databaseinfra.instances.filter(
                future_instance__isnull=True)
            databaseinfraattr_instances = DatabaseInfraAttr.objects.filter(databaseinfra=databaseinfra,
                                                                           equivalent_dbinfraattr__isnull=True)

            instance_address_list = []
            for instance in instances:
                instance_address_list.append(instance.address)
            for instance in databaseinfraattr_instances:
                instance_address_list.append(instance.ip)

            for database_bind in database.acl_binds.all():
                infra_instances_binds = DatabaseInfraInstanceBind.objects.filter(
                    databaseinfra=databaseinfra,
                    instance__in=instance_address_list,
                    bind_address=database_bind.bind_address)
                if helpers.unbind_address(database_bind, acl_client, infra_instances_binds):
                    continue

            return True

        except Exception:
            traceback = full_stack()

            workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
            workflow_dict['exceptions']['traceback'].append(traceback)

            return False

class UnbindOldInstances(BaseStep):

    def __unicode__(self):
        return "Unbinding old instances ..."

    def do(self, workflow_dict):

        try:
            database = workflow_dict['database']
            databaseinfra = workflow_dict['databaseinfra']

            acl_credential = get_credentials_for(environment=database.environment,
                                                 credential_type=CredentialType.ACLAPI)
            acl_client = AclClient(acl_credential.endpoint,
                                   acl_credential.user,
                                   acl_credential.password,
                                   database.environment)

            instances = databaseinfra.instances.filter(
                future_instance__isnull=False)
            databaseinfraattr_instances = DatabaseInfraAttr.objects.filter(databaseinfra=databaseinfra,
                                                                           equivalent_dbinfraattr__isnull=False)

            instance_address_list = []
            for instance in instances:
                instance_address_list.append(instance.address)
            for instance in databaseinfraattr_instances:
                instance_address_list.append(instance.ip)

            for database_bind in database.acl_binds.all():
                infra_instances_binds = DatabaseInfraInstanceBind.objects.filter(
                    databaseinfra=databaseinfra,
                    instance__in=instance_address_list,
                    bind_address=database_bind.bind_address)
                if helpers.unbind_address(database_bind, acl_client, infra_instances_binds):
                    continue

            return True

        except Exception:
            traceback = full_stack()

            workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
            workflow_dict['exceptions']['traceback'].append(traceback)

            return False

    def undo(self, workflow_dict):

        LOG.info("Running undo...")
        try:
            database = workflow_dict['database']
            databaseinfra = workflow_dict['databaseinfra']

            acl_credential = get_credentials_for(environment=database.environment,
                                                 credential_type=CredentialType.ACLAPI)
            acl_client = AclClient(acl_credential.endpoint,
                                   acl_credential.user,
                                   acl_credential.password,
                                   database.environment)

            instances = databaseinfra.instances.filter(
                future_instance__isnull=False)

            databaseinfraattr_instances = DatabaseInfraAttr.objects.filter(databaseinfra=databaseinfra,
                                                                           equivalent_dbinfraattr__isnull=False)
            instance_address_list = []
            for instance in instances:
                instance_address_list.append(instance.address)
            for instance in databaseinfraattr_instances:
                instance_address_list.append(instance.ip)

            for database_bind in database.acl_binds.all():
                if helpers.bind_address(
                        database_bind, acl_client, instances=instances,
                        infra_attr_instances=databaseinfraattr_instances):
                    continue
                else:
                    LOG.error("The AclApi is not working properly.")
                    database_bind.bind_status = ERROR
                    database_bind.save()
                    DatabaseInfraInstanceBind.objects.filter(databaseinfra=databaseinfra,
                                                             bind_address=database_bind.bind_address,
                                                             instance__in=instance_address_list
                                                             ).update(bind_status=ERROR)
            return True

        except Exception:
            traceback = full_stack()

            workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
            workflow_dict['exceptions']['traceback'].append(traceback)

            return False
<|endoftext|>


<commit_before>from datetime import datetime
from django.http import Http404
from django.test import TestCase, RequestFactory

from devilry.devilry_detektor.models import DetektorAssignment
from devilry.devilry_detektor.tasks import AssignmentParser
from devilry.devilry_detektor.views.admin_assignmentassembly import AssignmentAssemblyView
from devilry.devilry_detektor.comparer import CompareManyCollection
from devilry.project.develop.testhelpers.corebuilder import UserBuilder
from devilry.project.develop.testhelpers.corebuilder import PeriodBuilder
from devilry.project.develop.testhelpers.soupselect import cssFind

class TestAssignmentAssemblyView(TestCase):
    def setUp(self):
        self.testuser = UserBuilder('testuser').user
        self.assignmentbuilder = PeriodBuilder\
            .quickadd_ducku_duck1010_active()\
            .add_admins(self.testuser)\
            .add_assignment('testassignment')
        self.factory = RequestFactory()

    def test_404(self):
        request = self.factory.get('/test')
        request.user = self.testuser
        with self.assertRaises(Http404):
            AssignmentAssemblyView.as_view()(request, assignmentid=200001)

    def _create_mock_getrequest(self, data={}):
        request = self.factory.get('/test', data=data)
        request.user = self.testuser
        request.session = {}
        return request

    def test_render_first_visit(self):
        request = self._create_mock_getrequest()
        response = AssignmentAssemblyView.as_view()(
            request, assignmentid=self.assignmentbuilder.assignment.id)
        response.render()
        self.assertNotIn('Similarity check processing was started by testuser', response.content)
        self.assertIn('Run/re-run similarity check', response.content)

    def test_processing_subsequent(self):
        DetektorAssignment.objects.create(
            assignment_id=self.assignmentbuilder.assignment.id,
            processing_started_by=self.testuser)
        request = self._create_mock_getrequest()
        response = AssignmentAssemblyView.as_view()(
            request, assignmentid=self.assignmentbuilder.assignment.id)
        response.render()
        self.assertNotIn('Similarity check processing was started by testuser', response.content)
        self.assertIn('Run/re-run similarity check', response.content)

    def test_already_running(self):
        processing_started_datetime = datetime.now()
        DetektorAssignment.objects.create(
            assignment_id=self.assignmentbuilder.assignment.id,
            processing_started_by=self.testuser,
            processing_started_datetime=processing_started_datetime,
            status='running')
        request = self._create_mock_getrequest()
        response = AssignmentAssemblyView.as_view()(
            request, assignmentid=self.assignmentbuilder.assignment.id)
        response.render()
        self.assertIn('Similarity check processing was started by testuser', response.content)
        self.assertNotIn('Run/re-run similarity check', response.content)

    def test_results_ordering(self):
        self.assignmentbuilder\
            .add_group(name='Group A')\
            .add_deadline_in_x_weeks(weeks=1)\
            .add_delivery()\
            .add_filemeta(filename='Test1.java', data='if(i==10) {}')
        self.assignmentbuilder\
            .add_group(name='Group B')\
            .add_deadline_in_x_weeks(weeks=1)\
            .add_delivery()\
            .add_filemeta(filename='Test2.java', data='class Test {if(i==10) {}}')
        self.assignmentbuilder\
            .add_group(name='Group C')\
            .add_deadline_in_x_weeks(weeks=1)\
            .add_delivery()\
            .add_filemeta(filename='Test3.java', data='class Test {}')
        detektorassignment = DetektorAssignment.objects.create(
            assignment_id=self.assignmentbuilder.assignment.id,
            processing_started_by=self.testuser)
        AssignmentParser(assignment_id=self.assignmentbuilder.assignment.id).run_detektor()
        CompareManyCollection(detektorassignment).save()

        request = self._create_mock_getrequest(data={
            'language': 'java'
        })
        response = AssignmentAssemblyView.as_view()(
            request, assignmentid=self.assignmentbuilder.assignment.id)
        response.render()

        displaynames = []
        for displayname1, displayname2 in zip(
                [span.text.strip() for span in
                 cssFind(response.content, '#detektorassembly-results .detektorassembly-delivery1-displayname')],
                [span.text.strip() for span in
                 cssFind(response.content, '#detektorassembly-results .detektorassembly-delivery2-displayname')]):
            displaynames.append({displayname1, displayname2})
        self.assertEqual(
            displaynames,
            [
                {'Group A', 'Group B'},
                {'Group A', 'Group C'},
            ])

    def _create_mock_postrequest(self):
        request = self.factory.post('/test')
        request.user = self.testuser
        request.session = {}
        return request

    def test_post(self):
        self.assertEqual(DetektorAssignment.objects.count(), 0)
        request = self._create_mock_postrequest()
        response = AssignmentAssemblyView.as_view()(
            request, assignmentid=self.assignmentbuilder.assignment.id)
        self.assertEqual(DetektorAssignment.objects.count(), 1)
        self.assertEquals(response.status_code, 302)
        detektorassignment = DetektorAssignment.objects.all()[0]
        self.assertEqual(detektorassignment.processing_started_by, self.testuser)
<commit_msg>devilry_detector: Skip the view tests.<commit_after>from datetime import datetime
import unittest
from django.http import Http404
from django.test import TestCase, RequestFactory

from devilry.devilry_detektor.models import DetektorAssignment
from devilry.devilry_detektor.tasks import AssignmentParser
from devilry.devilry_detektor.views.admin_assignmentassembly import AssignmentAssemblyView
from devilry.devilry_detektor.comparer import CompareManyCollection
from devilry.project.develop.testhelpers.corebuilder import UserBuilder
from devilry.project.develop.testhelpers.corebuilder import PeriodBuilder
from devilry.project.develop.testhelpers.soupselect import cssFind

@unittest.skip('devilry_detector must be updated to use cradmin')
class TestAssignmentAssemblyView(TestCase):
    def setUp(self):
        self.testuser = UserBuilder('testuser').user
        self.assignmentbuilder = PeriodBuilder\
            .quickadd_ducku_duck1010_active()\
            .add_admins(self.testuser)\
            .add_assignment('testassignment')
        self.factory = RequestFactory()

    def test_404(self):
        request = self.factory.get('/test')
        request.user = self.testuser
        with self.assertRaises(Http404):
            AssignmentAssemblyView.as_view()(request, assignmentid=200001)

    def _create_mock_getrequest(self, data={}):
        request = self.factory.get('/test', data=data)
        request.user = self.testuser
        request.session = {}
        return request

    def test_render_first_visit(self):
        request = self._create_mock_getrequest()
        response = AssignmentAssemblyView.as_view()(
            request, assignmentid=self.assignmentbuilder.assignment.id)
        response.render()
        self.assertNotIn('Similarity check processing was started by testuser', response.content)
        self.assertIn('Run/re-run similarity check', response.content)

    def test_processing_subsequent(self):
        DetektorAssignment.objects.create(
            assignment_id=self.assignmentbuilder.assignment.id,
            processing_started_by=self.testuser)
        request = self._create_mock_getrequest()
        response = AssignmentAssemblyView.as_view()(
            request, assignmentid=self.assignmentbuilder.assignment.id)
        response.render()
        self.assertNotIn('Similarity check processing was started by testuser', response.content)
        self.assertIn('Run/re-run similarity check', response.content)

    def test_already_running(self):
        processing_started_datetime = datetime.now()
        DetektorAssignment.objects.create(
            assignment_id=self.assignmentbuilder.assignment.id,
            processing_started_by=self.testuser,
            processing_started_datetime=processing_started_datetime,
            status='running')
        request = self._create_mock_getrequest()
        response = AssignmentAssemblyView.as_view()(
            request, assignmentid=self.assignmentbuilder.assignment.id)
        response.render()
        self.assertIn('Similarity check processing was started by testuser', response.content)
        self.assertNotIn('Run/re-run similarity check', response.content)

    def test_results_ordering(self):
        self.assignmentbuilder\
            .add_group(name='Group A')\
            .add_deadline_in_x_weeks(weeks=1)\
            .add_delivery()\
            .add_filemeta(filename='Test1.java', data='if(i==10) {}')
        self.assignmentbuilder\
            .add_group(name='Group B')\
            .add_deadline_in_x_weeks(weeks=1)\
            .add_delivery()\
            .add_filemeta(filename='Test2.java', data='class Test {if(i==10) {}}')
        self.assignmentbuilder\
            .add_group(name='Group C')\
            .add_deadline_in_x_weeks(weeks=1)\
            .add_delivery()\
            .add_filemeta(filename='Test3.java', data='class Test {}')
        detektorassignment = DetektorAssignment.objects.create(
            assignment_id=self.assignmentbuilder.assignment.id,
            processing_started_by=self.testuser)
        AssignmentParser(assignment_id=self.assignmentbuilder.assignment.id).run_detektor()
        CompareManyCollection(detektorassignment).save()

        request = self._create_mock_getrequest(data={
            'language': 'java'
        })
        response = AssignmentAssemblyView.as_view()(
            request, assignmentid=self.assignmentbuilder.assignment.id)
        response.render()

        displaynames = []
        for displayname1, displayname2 in zip(
                [span.text.strip() for span in
                 cssFind(response.content, '#detektorassembly-results .detektorassembly-delivery1-displayname')],
                [span.text.strip() for span in
                 cssFind(response.content, '#detektorassembly-results .detektorassembly-delivery2-displayname')]):
            displaynames.append({displayname1, displayname2})
        self.assertEqual(
            displaynames,
            [
                {'Group A', 'Group B'},
                {'Group A', 'Group C'},
            ])

    def _create_mock_postrequest(self):
        request = self.factory.post('/test')
        request.user = self.testuser
        request.session = {}
        return request

    def test_post(self):
        self.assertEqual(DetektorAssignment.objects.count(), 0)
        request = self._create_mock_postrequest()
        response = AssignmentAssemblyView.as_view()(
            request, assignmentid=self.assignmentbuilder.assignment.id)
        self.assertEqual(DetektorAssignment.objects.count(), 1)
        self.assertEquals(response.status_code, 302)
        detektorassignment = DetektorAssignment.objects.all()[0]
        self.assertEqual(detektorassignment.processing_started_by, self.testuser)
<|endoftext|>


<commit_before>import json
import urllib

from django.views import generic
from django.shortcuts import render
from django.conf import settings
from django.core.urlresolvers import reverse
from django.http import HttpResponse

class IndexView(generic.TemplateView):
    template_name = 'metadata_categorization/index.html'

class QueueView(generic.TemplateView):
    template_name = 'metadata_categorization/queue.html'

    def getSummaryRecords(self, individualRecords):
        """
        Aggregates individual Biosample (SRA or GEO) records into
        "summary records" shown as top-level table rows in the UI.  Individual
        records are grouped into the same summary record only if they have
        the same 'sourceCellLine' value.

        Summary records have most of the same fields (columns) as
        the individual records.  Each summary record also has a list of the
        individual records that it contains/aggregates.

        It was unclear whether/how such aggregation was possible in Solr itself,
        so we're doing it in the web tier.
        """

        sourceFields = {
            "sourceCellLine": "",
            "sourceCellType": "",
            "sourceCellTreatment": "",
            "sourceAnatomy": "",
            "sourceTreatment": "",
            "sourceSpecies": "Homo Sapiens",
            "sourceDisease": ""
        }

        annotFields = {
            "annotCellLine": "",
            "annotCellType": "",
            "annotCellTreatment": "",
            "annotAnatomy": "",
            "annotSpecies": "",
            "annotSpecies": "",
            "annotDisease": ""
        }

        summaryRecords = []

        summaryRecord = {"individualRecords": []}
        summaryRecord.update(sourceFields)
        summaryRecord.update(annotFields)

        individualRecords = sorted(
                        individualRecords,
                        key=lambda k: k['sourceCellLine']
                    )

        tmpIRs = []
        for i, individualRecord in enumerate(individualRecords):
            tmpIR = {}
            for field in individualRecord:
                thisField = individualRecord[field]
                tmpIR[field] = thisField
                if thisField == '0' or thisField == "   ":
                    individualRecords[i][field] = ""
                    individualRecord[field] = ""
                    tmpIR[field] = ""

            individualRecord = tmpIR
            tmpIRs.append(individualRecord)

        individualRecords = tmpIRs

        for i, individualRecord in enumerate(individualRecords):
            if "sourceCellLine" not in individualRecord:
                print('"sourceCellLine" not in individualRecord')
                individualRecords[i]["sourceCellLine"] = ""

            cellLine = individualRecords[i]["sourceCellLine"]

            prevCellLine = individualRecords[i-1]["sourceCellLine"]

            if cellLine == prevCellLine or i == 0:
                summaryRecord["individualRecords"].append(individualRecord)
            else:
                recordsCount = summaryRecord["individualRecords"]
                summaryRecord['recordsCount'] = len(recordsCount)

                if len(summaryRecord["individualRecords"]) > 0:
                    summaryRecord['sourceCellLine'] = prevCellLine

                summaryRecords.append(summaryRecord)
                summaryRecord = {
                    "individualRecords": [individualRecord],
                    "index": i
                }
                summaryRecord.update(sourceFields)
                summaryRecord.update(annotFields)

        return summaryRecords

    # curationQueue
    # ID
    # list summaryRecord
    # summaryRecord
    # list indivdualRecord
    # list
    # individualRecord term|cell line|cell type|anatomy|species|disease|note
    def get_context_data(self, queueId, **kwargs):

        context = super(QueueView, self).get_context_data()

        # TODO:
        # Refactor into modular methods and models

        # Call Solr
        #solr_host = "http://localhost:8983/solr/AnnotationsDev"
        solr_host = "http://localhost:8983/solr/annotation"

        # %3A is :
        # sampleName%3A* is sampleName:*"
        url = (solr_host + "/select?" +
              #"q=queueId%3A42&" +
              'q=queueId%3A' + queueId + '+AND+NOT+sourceCellLine%3A%220%22'
              "start=0&" +
              "rows=10000&" +
              "wt=json&" +
              "indent=true&")

        request = urllib.request.Request(url)
        response = urllib.request.urlopen(request)
        str_response = response.readall().decode('utf-8')

        solr_data = json.loads(str_response)["response"]["docs"]

        #print(solr_data)

        context['solr_data'] = self.getSummaryRecords(solr_data)

        summaryRecords = self.getSummaryRecords(solr_data)

        '''
        # TODO
        # There's a bug in the summary sourceCellLine
        # Remove this hack once fixed
        tmp = []
        for summaryRecord in summaryRecords:
            if summaryRecord['sourceCellLine'] != '':
                tmp.append(summaryRecord)
        summaryRecords = tmp
        '''

        # Demo'd URL:
        # http://localhost:8000/queue/6

        context["id"] = queueId
        context["summaryRecords"] = summaryRecords

        return context

class RecordView(generic.TemplateView):

    def transform_record(self, id, data):
        # Transforms data to record format used by this app's Solr core
        record = {
            'annotCellLine': data.get('annotCellLine'),
            'annotCellType': data.get('annotCellType'),
            'annotAnatomy': data.get('annotAnatomy'),
            'annotSpecies': data.get('annotSpecies'),
            'annotDisease': data.get('annotDisease')
        }

        nr = {'id': id} # new record
        for key in record:
            value = record[key]
            if value == '':
                # We store empty strings as "0" in Solr.  I forget exactly why.
                # TODO: Investigate why.  If needed, document, else refactor.
                nr[key] = {'set': '0'}
            else:
                nr[key] = {'set': value}

        return [nr]

    def post(self, request, *args, **kwargs):

        data = request.POST

        id = kwargs['recordId']

        record = self.transform_record(id, data)

        # POST request body, representing an edited individual record
        body = str(record).encode('utf-8')

        solr_host = "http://localhost:8983/solr/annotation"
        url = solr_host + "/update?commit=true"

        # Example POST to update part of a Solr document in "annotation" core:
        # curl 'http://localhost:8983/solr/annotation/update?commit=true' -d "[{'annotAnatomy': {'set': '0'}, 'annotDisease': {'set': '0'}, 'annotSpecies': {'set': '0'}, 'id': '1090570', 'annotCellType': {'set': '0'}, 'annotCellLine': {'set': 'test'}}]"
        # Example read:
        # curl 'http://localhost:8983/solr/annotation/select?wt=json&q=id:1090570'
        # Another POST / partial update example:
        # curl 'http://localhost:8983/solr/annotation/update?commit=true' -d '[{"id": 3854415, "sourceCellLine": {"set": "testfoo"}}]'
        request = urllib.request.Request(url, body)
        request.add_header('Content-Type', 'application/json')
        response = urllib.request.urlopen(request)
        str_response = response.readall().decode('utf-8')

        return HttpResponse()
<commit_msg>Set summary record value if all individual records match for that field<commit_after>import json
import urllib

from django.views import generic
from django.shortcuts import render
from django.conf import settings
from django.core.urlresolvers import reverse
from django.http import HttpResponse

class IndexView(generic.TemplateView):
    template_name = 'metadata_categorization/index.html'

class QueueView(generic.TemplateView):
    template_name = 'metadata_categorization/queue.html'

    def getSummaryRecords(self, individualRecords):
        """
        Aggregates individual Biosample (SRA or GEO) records into
        "summary records" shown as top-level table rows in the UI.  Individual
        records are grouped into the same summary record only if they have
        the same 'sourceCellLine' value.

        Summary records have most of the same fields (columns) as
        the individual records.  Each summary record also has a list of the
        individual records that it contains/aggregates.

        It was unclear whether/how such aggregation was possible in Solr itself,
        so we're doing it in the web tier.
        """

        sourceFields = {
            "sourceCellLine": "",
            "sourceCellType": "",
            "sourceCellTreatment": "",
            "sourceAnatomy": "",
            "sourceTreatment": "",
            "sourceSpecies": "",
            "sourceDisease": ""
        }

        annotFields = {
            "annotCellLine": "",
            "annotCellType": "",
            "annotCellTreatment": "",
            "annotAnatomy": "",
            "annotSpecies": "",
            "annotSpecies": "",
            "annotDisease": ""
        }

        summaryRecords = []

        summaryRecord = {"individualRecords": []}
        summaryRecord.update(sourceFields)
        summaryRecord.update(annotFields)

        individualRecords = sorted(
                        individualRecords,
                        key=lambda k: k['sourceCellLine']
                    )

        tmpIRs = []
        for i, individualRecord in enumerate(individualRecords):
            tmpIR = {}
            for field in individualRecord:
                thisField = individualRecord[field]
                tmpIR[field] = thisField
                if thisField == '0' or thisField == "   ":
                    individualRecords[i][field] = ""
                    individualRecord[field] = ""
                    tmpIR[field] = ""

            individualRecord = tmpIR
            tmpIRs.append(individualRecord)

        individualRecords = tmpIRs

        for i, individualRecord in enumerate(individualRecords):
            if "sourceCellLine" not in individualRecord:
                print('"sourceCellLine" not in individualRecord')
                individualRecords[i]["sourceCellLine"] = ""

            cellLine = individualRecords[i]["sourceCellLine"]

            prevCellLine = individualRecords[i-1]["sourceCellLine"]

            if cellLine == prevCellLine or i == 0:
                summaryRecord["individualRecords"].append(individualRecord)
            else:
                recordsCount = summaryRecord["individualRecords"]
                summaryRecord['recordsCount'] = len(recordsCount)

                if len(summaryRecord["individualRecords"]) > 0:
                    summaryRecord['sourceCellLine'] = prevCellLine

                summaryRecords.append(summaryRecord)
                summaryRecord = {
                    "individualRecords": [individualRecord],
                    "index": i
                }
                summaryRecord.update(sourceFields)
                summaryRecord.update(annotFields)

        newSRs = []
        for i, summaryRecord in enumerate(summaryRecords[:10]):
            prevFields = {}
            newSR = summaryRecord
            individualRecords = summaryRecord['individualRecords']

            if len(individualRecords) == 1:
                newSR.update(individualRecords[0])
                newSRs.append(newSR)
                continue

            for j, individualRecord in enumerate(individualRecords):
                for field in individualRecord:
                    if j == 0:
                        prevFields[field] = individualRecord[field]
                    else:
                        if (
                            prevFields[field] != "" and
                            prevFields[field] == individualRecord[field]
                        ):
                            newSR[field] = prevFields[field]
                        else:
                            newSR[field] = ""
            newSRs.append(newSR)
        summaryRecords = newSRs

        return summaryRecords

    # curationQueue
    # ID
    # list summaryRecord
    # summaryRecord
    # list indivdualRecord
    # list
    # individualRecord term|cell line|cell type|anatomy|species|disease|note
    def get_context_data(self, queueId, **kwargs):

        context = super(QueueView, self).get_context_data()

        # TODO:
        # Refactor into modular methods and models

        # Call Solr
        #solr_host = "http://localhost:8983/solr/AnnotationsDev"
        solr_host = "http://localhost:8983/solr/annotation"

        # %3A is :
        # sampleName%3A* is sampleName:*"
        url = (solr_host + "/select?" +
              #"q=queueId%3A42&" +
              'q=queueId%3A' + queueId + '+AND+NOT+sourceCellLine%3A%220%22'
              "start=0&" +
              "rows=10000&" +
              "wt=json&" +
              "indent=true&")

        request = urllib.request.Request(url)
        response = urllib.request.urlopen(request)
        str_response = response.readall().decode('utf-8')

        solr_data = json.loads(str_response)["response"]["docs"]

        #print(solr_data)

        context['solr_data'] = self.getSummaryRecords(solr_data)

        summaryRecords = self.getSummaryRecords(solr_data)

        '''
        # TODO
        # There's a bug in the summary sourceCellLine
        # Remove this hack once fixed
        tmp = []
        for summaryRecord in summaryRecords:
            if summaryRecord['sourceCellLine'] != '':
                tmp.append(summaryRecord)
        summaryRecords = tmp
        '''

        # Demo'd URL:
        # http://localhost:8000/queue/6

        context["id"] = queueId
        context["summaryRecords"] = summaryRecords

        return context

class RecordView(generic.TemplateView):

    def transform_record(self, id, data):
        # Transforms data to record format used by this app's Solr core
        record = {
            'annotCellLine': data.get('annotCellLine'),
            'annotCellType': data.get('annotCellType'),
            'annotAnatomy': data.get('annotAnatomy'),
            'annotSpecies': data.get('annotSpecies'),
            'annotDisease': data.get('annotDisease')
        }

        nr = {'id': id} # new record
        for key in record:
            value = record[key]
            if value == '':
                # We store empty strings as "0" in Solr.  I forget exactly why.
                # TODO: Investigate why.  If needed, document, else refactor.
                nr[key] = {'set': '0'}
            else:
                nr[key] = {'set': value}

        return [nr]

    def post(self, request, *args, **kwargs):

        data = request.POST

        id = kwargs['recordId']

        record = self.transform_record(id, data)

        # POST request body, representing an edited individual record
        body = str(record).encode('utf-8')

        solr_host = "http://localhost:8983/solr/annotation"
        url = solr_host + "/update?commit=true"

        # Example POST to update part of a Solr document in "annotation" core:
        # curl 'http://localhost:8983/solr/annotation/update?commit=true' -d "[{'annotAnatomy': {'set': '0'}, 'annotDisease': {'set': '0'}, 'annotSpecies': {'set': '0'}, 'id': '1090570', 'annotCellType': {'set': '0'}, 'annotCellLine': {'set': 'test'}}]"
        # Example read:
        # curl 'http://localhost:8983/solr/annotation/select?wt=json&q=id:1090570'
        # Another POST / partial update example:
        # curl 'http://localhost:8983/solr/annotation/update?commit=true' -d '[{"id": 3854415, "sourceCellLine": {"set": "testfoo"}}]'
        request = urllib.request.Request(url, body)
        request.add_header('Content-Type', 'application/json')
        response = urllib.request.urlopen(request)
        str_response = response.readall().decode('utf-8')

        return HttpResponse()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# Copyright (c) 2016 MagicStack Inc.
# All rights reserved.
#
# See LICENSE for details.
##

import argparse
import asyncio
from concurrent import futures
import json
import re
import sys
import time

import numpy as np
import uvloop

import aiopg
import asyncpg
import postgresql
import psycopg2
import psycopg2.extras

def psycopg_connect(args):
    conn = psycopg2.connect(user=args.pguser, host=args.pghost,
                            port=args.pgport)
    return conn

def psycopg_execute(conn, query, args):
    cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
    cur.execute(query, args)
    return len(cur.fetchall())

def pypostgresql_connect(args):
    conn = postgresql.open(user=args.pguser, host=args.pghost,
                           port=args.pgport)
    return conn

def pypostgresql_execute(conn, query, args):
    stmt = conn.prepare(query)
    return len(list(stmt.rows(*args)))

async def aiopg_connect(args):
    conn = await aiopg.connect(user=args.pguser, host=args.pghost,
                               port=args.pgport)
    return conn

async def aiopg_execute(conn, query, args):
    cur = await conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
    await cur.execute(query, args)
    return len(await cur.fetchall())

async def asyncpg_connect(args):
    conn = await asyncpg.connect(user=args.pguser, host=args.pghost,
                                 port=args.pgport)
    return conn

async def asyncpg_execute(conn, query, args):
    return len(await conn.fetch(query, *args))

async def worker(executor, eargs, start, duration, timeout):
    queries = 0
    rows = 0
    latency_stats = np.zeros((timeout * 100,))
    min_latency = float('inf')
    max_latency = 0.0

    while time.monotonic() - start < duration:
        req_start = time.monotonic()
        rows += await executor(*eargs)
        req_time = round((time.monotonic() - req_start) * 1000 * 100)

        if req_time > max_latency:
            max_latency = req_time
        if req_time < min_latency:
            min_latency = req_time
        latency_stats[req_time] += 1
        queries += 1

    return queries, rows, latency_stats, min_latency, max_latency

def sync_worker(executor, eargs, start, duration, timeout):
    queries = 0
    rows = 0
    latency_stats = np.zeros((timeout * 100,))
    min_latency = float('inf')
    max_latency = 0.0

    while time.monotonic() - start < duration:
        req_start = time.monotonic()
        rows += executor(*eargs)
        req_time = round((time.monotonic() - req_start) * 1000 * 100)

        if req_time > max_latency:
            max_latency = req_time
        if req_time < min_latency:
            min_latency = req_time
        latency_stats[req_time] += 1
        queries += 1

    return queries, rows, latency_stats, min_latency, max_latency

async def runner(args, connector, executor, is_async, arg_format, query,
                 query_args, setup, teardown):

    timeout = args.timeout * 1000
    concurrency = args.concurrency

    if arg_format == 'python':
        query = re.sub(r'\$\d+', '%s', query)

    conns = []

    for i in range(concurrency):
        if is_async:
            conn = await connector(args)
        else:
            conn = connector(args)
        conns.append(conn)

    async def _do_run(run_duration):
        start = time.monotonic()

        tasks = []

        if is_async:
            # Asyncio driver
            for i in range(concurrency):
                task = worker(executor, [conns[i], query, query_args],
                              start, args.duration, timeout)
                tasks.append(task)

            results = await asyncio.gather(*tasks)
        else:
            # Sync driver
            with futures.ThreadPoolExecutor(max_workers=concurrency) as e:
                for i in range(concurrency):
                    task = e.submit(sync_worker, executor,
                                    [conns[i], query, query_args],
                                    start, run_duration, timeout)
                    tasks.append(task)

                results = [fut.result() for fut in futures.wait(tasks).done]

        end = time.monotonic()

        return results, end - start

    if setup:
        admin_conn = await asyncpg.connect(user=args.pguser, host=args.pghost,
                                           port=args.pgport)
        await admin_conn.execute(setup)

    try:
        if args.warmup_time:
            await _do_run(args.warmup_time)

        results, duration = await _do_run(args.duration)

    finally:
        if teardown:
            await admin_conn.execute(teardown)
            await admin_conn.close()

    for conn in conns:
        if is_async:
            await conn.close()
        else:
            conn.close()

    min_latency = float('inf')
    max_latency = 0.0
    queries = 0
    rows = 0
    latency_stats = None

    for result in results:
        t_queries, t_rows, t_latency_stats, t_min_latency, t_max_latency = \
            result
        queries += t_queries
        rows += t_rows
        if latency_stats is None:
            latency_stats = t_latency_stats
        else:
            latency_stats = np.add(latency_stats, t_latency_stats)
        if t_max_latency > max_latency:
            max_latency = t_max_latency
        if t_min_latency < min_latency:
            min_latency = t_min_latency

    data = {
        'queries': queries,
        'rows': rows,
        'duration': duration,
        'min_latency': min_latency,
        'max_latency': max_latency,
        'latency_stats': latency_stats.tolist(),
        'output_format': args.output_format
    }

    print(json.dumps(data))

def die(msg):
    print('fatal: {}'.format(msg), file=sys.stderr)
    sys.exit(1)

if __name__ == '__main__':
    asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
    loop = asyncio.get_event_loop()

    parser = argparse.ArgumentParser(
        description='async pg driver benchmark [concurrent]')
    parser.add_argument(
        '-C', '--concurrency', type=int, default=10,
        help='number of concurrent connections')
    parser.add_argument(
        '-D', '--duration', type=int, default=30,
        help='duration of test in seconds')
    parser.add_argument(
        '--timeout', default=2, type=int,
        help='server timeout in seconds')
    parser.add_argument(
        '--warmup-time', type=int, default=5,
        help='duration of warmup period for each benchmark in seconds')
    parser.add_argument(
        '--output-format', default='text', type=str,
        help='output format', choices=['text', 'json'])
    parser.add_argument(
        '--pghost', type=str, default='127.0.0.1',
        help='PostgreSQL server host')
    parser.add_argument(
        '--pgport', type=int, default=5432,
        help='PostgreSQL server port')
    parser.add_argument(
        '--pguser', type=str, default='postgres',
        help='PostgreSQL server user')
    parser.add_argument(
        'driver', help='driver implementation to use',
        choices=['aiopg', 'asyncpg', 'psycopg', 'postgresql'])
    parser.add_argument(
        'queryfile', help='file to read benchmark query information from')

    args = parser.parse_args()

    if args.queryfile == '-':
        querydata_text = sys.stdin.read()
    else:
        with open(args.queryfile, 'rt') as f:
            querydata_text = f.read()

    querydata = json.loads(querydata_text)

    query = querydata.get('query')
    if not query:
        die('missing "query" in query JSON')

    query_args = querydata.get('args')
    if not query_args:
        query_args = []

    setup = querydata.get('setup')
    teardown = querydata.get('teardown')
    if setup and not teardown:
        die('"setup" is present, but "teardown" is missing in query JSON')

    if args.driver == 'aiopg':
        connector, executor = aiopg_connect, aiopg_execute
        is_async = True
        arg_format = 'python'
    elif args.driver == 'asyncpg':
        connector, executor = asyncpg_connect, asyncpg_execute
        is_async = True
        arg_format = 'native'
    elif args.driver == 'psycopg':
        connector, executor = psycopg_connect, psycopg_execute
        is_async = False
        arg_format = 'python'
    elif args.driver == 'postgresql':
        connector, executor = pypostgresql_connect, pypostgresql_execute
        is_async = False
        arg_format = 'native'
    else:
        raise ValueError('unexpected driver: {!r}'.format(args.driver))

    runner_coro = runner(args, connector, executor, is_async, arg_format,
                         query, query_args, setup, teardown)
    loop.run_until_complete(runner_coro)
<commit_msg>Add aiopg runner variant returning tuples<commit_after>#!/usr/bin/env python3
#
# Copyright (c) 2016 MagicStack Inc.
# All rights reserved.
#
# See LICENSE for details.
##

import argparse
import asyncio
from concurrent import futures
import json
import re
import sys
import time

import numpy as np
import uvloop

import aiopg
import asyncpg
import postgresql
import psycopg2
import psycopg2.extras

def psycopg_connect(args):
    conn = psycopg2.connect(user=args.pguser, host=args.pghost,
                            port=args.pgport)
    return conn

def psycopg_execute(conn, query, args):
    cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
    cur.execute(query, args)
    return len(cur.fetchall())

def pypostgresql_connect(args):
    conn = postgresql.open(user=args.pguser, host=args.pghost,
                           port=args.pgport)
    return conn

def pypostgresql_execute(conn, query, args):
    stmt = conn.prepare(query)
    return len(list(stmt.rows(*args)))

async def aiopg_connect(args):
    conn = await aiopg.connect(user=args.pguser, host=args.pghost,
                               port=args.pgport)
    return conn

async def aiopg_execute(conn, query, args):
    cur = await conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
    await cur.execute(query, args)
    return len(await cur.fetchall())

aiopg_tuples_connect = aiopg_connect

async def aiopg_tuples_execute(conn, query, args):
    cur = await conn.cursor()
    await cur.execute(query, args)
    return len(await cur.fetchall())

async def asyncpg_connect(args):
    conn = await asyncpg.connect(user=args.pguser, host=args.pghost,
                                 port=args.pgport)
    return conn

async def asyncpg_execute(conn, query, args):
    return len(await conn.fetch(query, *args))

async def worker(executor, eargs, start, duration, timeout):
    queries = 0
    rows = 0
    latency_stats = np.zeros((timeout * 100,))
    min_latency = float('inf')
    max_latency = 0.0

    while time.monotonic() - start < duration:
        req_start = time.monotonic()
        rows += await executor(*eargs)
        req_time = round((time.monotonic() - req_start) * 1000 * 100)

        if req_time > max_latency:
            max_latency = req_time
        if req_time < min_latency:
            min_latency = req_time
        latency_stats[req_time] += 1
        queries += 1

    return queries, rows, latency_stats, min_latency, max_latency

def sync_worker(executor, eargs, start, duration, timeout):
    queries = 0
    rows = 0
    latency_stats = np.zeros((timeout * 100,))
    min_latency = float('inf')
    max_latency = 0.0

    while time.monotonic() - start < duration:
        req_start = time.monotonic()
        rows += executor(*eargs)
        req_time = round((time.monotonic() - req_start) * 1000 * 100)

        if req_time > max_latency:
            max_latency = req_time
        if req_time < min_latency:
            min_latency = req_time
        latency_stats[req_time] += 1
        queries += 1

    return queries, rows, latency_stats, min_latency, max_latency

async def runner(args, connector, executor, is_async, arg_format, query,
                 query_args, setup, teardown):

    timeout = args.timeout * 1000
    concurrency = args.concurrency

    if arg_format == 'python':
        query = re.sub(r'\$\d+', '%s', query)

    conns = []

    for i in range(concurrency):
        if is_async:
            conn = await connector(args)
        else:
            conn = connector(args)
        conns.append(conn)

    async def _do_run(run_duration):
        start = time.monotonic()

        tasks = []

        if is_async:
            # Asyncio driver
            for i in range(concurrency):
                task = worker(executor, [conns[i], query, query_args],
                              start, args.duration, timeout)
                tasks.append(task)

            results = await asyncio.gather(*tasks)
        else:
            # Sync driver
            with futures.ThreadPoolExecutor(max_workers=concurrency) as e:
                for i in range(concurrency):
                    task = e.submit(sync_worker, executor,
                                    [conns[i], query, query_args],
                                    start, run_duration, timeout)
                    tasks.append(task)

                results = [fut.result() for fut in futures.wait(tasks).done]

        end = time.monotonic()

        return results, end - start

    if setup:
        admin_conn = await asyncpg.connect(user=args.pguser, host=args.pghost,
                                           port=args.pgport)
        await admin_conn.execute(setup)

    try:
        if args.warmup_time:
            await _do_run(args.warmup_time)

        results, duration = await _do_run(args.duration)

    finally:
        if teardown:
            await admin_conn.execute(teardown)
            await admin_conn.close()

    for conn in conns:
        if is_async:
            await conn.close()
        else:
            conn.close()

    min_latency = float('inf')
    max_latency = 0.0
    queries = 0
    rows = 0
    latency_stats = None

    for result in results:
        t_queries, t_rows, t_latency_stats, t_min_latency, t_max_latency = \
            result
        queries += t_queries
        rows += t_rows
        if latency_stats is None:
            latency_stats = t_latency_stats
        else:
            latency_stats = np.add(latency_stats, t_latency_stats)
        if t_max_latency > max_latency:
            max_latency = t_max_latency
        if t_min_latency < min_latency:
            min_latency = t_min_latency

    data = {
        'queries': queries,
        'rows': rows,
        'duration': duration,
        'min_latency': min_latency,
        'max_latency': max_latency,
        'latency_stats': latency_stats.tolist(),
        'output_format': args.output_format
    }

    print(json.dumps(data))

def die(msg):
    print('fatal: {}'.format(msg), file=sys.stderr)
    sys.exit(1)

if __name__ == '__main__':
    asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
    loop = asyncio.get_event_loop()

    parser = argparse.ArgumentParser(
        description='async pg driver benchmark [concurrent]')
    parser.add_argument(
        '-C', '--concurrency', type=int, default=10,
        help='number of concurrent connections')
    parser.add_argument(
        '-D', '--duration', type=int, default=30,
        help='duration of test in seconds')
    parser.add_argument(
        '--timeout', default=2, type=int,
        help='server timeout in seconds')
    parser.add_argument(
        '--warmup-time', type=int, default=5,
        help='duration of warmup period for each benchmark in seconds')
    parser.add_argument(
        '--output-format', default='text', type=str,
        help='output format', choices=['text', 'json'])
    parser.add_argument(
        '--pghost', type=str, default='127.0.0.1',
        help='PostgreSQL server host')
    parser.add_argument(
        '--pgport', type=int, default=5432,
        help='PostgreSQL server port')
    parser.add_argument(
        '--pguser', type=str, default='postgres',
        help='PostgreSQL server user')
    parser.add_argument(
        'driver', help='driver implementation to use',
        choices=['aiopg', 'aiopg-tuples', 'asyncpg', 'psycopg', 'postgresql'])
    parser.add_argument(
        'queryfile', help='file to read benchmark query information from')

    args = parser.parse_args()

    if args.queryfile == '-':
        querydata_text = sys.stdin.read()
    else:
        with open(args.queryfile, 'rt') as f:
            querydata_text = f.read()

    querydata = json.loads(querydata_text)

    query = querydata.get('query')
    if not query:
        die('missing "query" in query JSON')

    query_args = querydata.get('args')
    if not query_args:
        query_args = []

    setup = querydata.get('setup')
    teardown = querydata.get('teardown')
    if setup and not teardown:
        die('"setup" is present, but "teardown" is missing in query JSON')

    if args.driver == 'aiopg':
        connector, executor = aiopg_connect, aiopg_execute
        is_async = True
        arg_format = 'python'
    if args.driver == 'aiopg-tuples':
        connector, executor = aiopg_tuples_connect, aiopg_tuples_execute
        is_async = True
        arg_format = 'python'
    elif args.driver == 'asyncpg':
        connector, executor = asyncpg_connect, asyncpg_execute
        is_async = True
        arg_format = 'native'
    elif args.driver == 'psycopg':
        connector, executor = psycopg_connect, psycopg_execute
        is_async = False
        arg_format = 'python'
    elif args.driver == 'postgresql':
        connector, executor = pypostgresql_connect, pypostgresql_execute
        is_async = False
        arg_format = 'native'
    else:
        raise ValueError('unexpected driver: {!r}'.format(args.driver))

    runner_coro = runner(args, connector, executor, is_async, arg_format,
                         query, query_args, setup, teardown)
    loop.run_until_complete(runner_coro)
<|endoftext|>


<commit_before>import bibtexparser
import argparse
import pdb
import json
from itertools import groupby
import operator
from bibtexparser.bwriter import BibTexWriter
from bibtexparser.bibdatabase import BibDatabase

# with open("../scrapper/list_abbrev.json") as file_data:
with open("../scrapper/list_abbrev.json") as file_data:
    list_abbrev = json.load(file_data)

def get_status(bib):

    def is_journal(item):
        return item["name"].lower() == bib["journal"].lower()

    def is_abbrev(item):
        return item["abbrev"].lower() == bib["journal"].lower()
    bib["_text"] = ""
    bib["_type"] = "out_db"
    a_journal = filter(is_journal, list_abbrev)
    a_abbrev = filter(is_abbrev, list_abbrev)
    # pdb.set_trace()
    if len(a_journal) > 0:
        # pdb.set_trace()
        bib["_text"] = a_journal[0]["abbrev"]
        bib["_type"] = "abreviated"
    elif len(a_abbrev) > 0:
        # pdb.set_trace()
        bib["_text"] = a_abbrev[0]["name"]
        bib["_type"] = "expanded"
    return bib

def manual_update(bibs):
    actions = {
        "y": lambda: update_bibs(bibs),
        "n": lambda: bibs,
        "c": lambda: update_bibs(bibs, custom=True)
    }
    question = "Replace '{}' for {}? y(yes)/n(no)/c(custom): "
    question = question.format(bibs[0]["journal"], bibs[0]["_text"])
    action = raw_input(question)
    try:
        return actions.get(action)()
    except TypeError:
        return manual_update(bibs)

def manual_update_out(bibs):
    actions = {
        "y": lambda: update_bibs(bibs, custom=True),
        "n": lambda: bibs,
    }
    question = "Replace '{}' ? y(yes)/n(no): "
    question = question.format(bibs[0]["journal"])
    action = raw_input(question)
    # try:
    try:
        return actions.get(action)()
    except TypeError:
        return manual_update_out(bibs)

def update_bibs_out(bibs):
    is_abreviation  = raw_input(
        "'{}' is a abreviation?y(yes)n(no): ".format(bibs[0]["journal"])
    )
    if is_abreviation == "y":
        full_name = raw_input("Insert journal name: ")
        abreviation = bibs[0]["journal"]
    elif is_abreviation == "n":
        abreviation = raw_input("Insert abreviation: ")
        full_name = bibs[0]["journal"]
    else:
        return update_bibs_out(bibs)
    list_abbrev.apend({"name": full_name, "abbrev": abreviation})
    for i, bib in enumerate(bibs):
        bibs[i]["journal"] = abreviation
    return bibs

def update_bibs(bibs, custom=False):
    if custom:
        journal = raw_input("Insert abreviation: ")
    else:
        journal = bibs[0]["_text"]

    for i, bib in enumerate(bibs):
        bibs[i]["journal"] = journal
    return bibs

def update_bibs_in(grouped_bibs):
    actions = {
        "a": lambda items: map(lambda bibs: update_bibs(bibs), items),
        "m": lambda items: map(lambda bibs: manual_update(bibs), items),
        "n": lambda items: items
    }
    print "{:d} itens can be {}".format(
        len(grouped_bibs), grouped_bibs[0]["_type"])
    action = raw_input("What you want? a(update all)/m(manual)/n(do nothing)")
    grouped_bibs.sort(key=operator.itemgetter('journal'))
    grouped_by_journal = []
    for key, items in groupby(grouped_bibs, lambda i: i["journal"]):
        grouped_by_journal.append(list(items))

    try:
        updated_bibs = actions.get(action)(grouped_by_journal)
    except TypeError:
        return update_bibs_in(grouped_bibs)

    updated_bibs = reduce(lambda a, b: a+b, updated_bibs)
    return updated_bibs

def update_bibs_out(grouped_bibs):
    grouped_bibs.sort(key=operator.itemgetter('journal'))
    grouped_by_journal = []
    for key, items in groupby(grouped_bibs, lambda i: i["journal"]):
        grouped_by_journal.append(list(items))

    updated_bibs = map(manual_update_out, grouped_by_journal)

    updated_bibs = reduce(lambda a, b: a+b, updated_bibs)
    return updated_bibs

def main():
    parser = argparse.ArgumentParser(
        prog="jta",
        description="Abreviate the journals name inside a bibtex file.")
    parser.add_argument(
        "--input", "-i",
        required=True,
        type=argparse.FileType("r"),
        help="bibtex input file"
    )
    parser.add_argument(
        "--output", "-o",
        required=True,
        # type=argparse.FileType("w"),
        help="bibtex output file")

    args = parser.parse_args()
    bibtex = bibtexparser.loads(args.input.read())
    bibs_journal, bibs_not_journal = [], []
    for bib in bibtex.entries:
        (bibs_journal
         if "journal" in bib else bibs_not_journal).append(bib)
    bibs_published, bibs_arxiv = [], []
    for bib in bibs_journal:
        (bibs_arxiv
         if "arxiv" in bib["journal"].lower()
         else bibs_published).append(bib)
    # pdb.set_trace()
    bibs_status = map(get_status, bibs_published)
    bibs_status.sort(key=operator.itemgetter('_type'))
    grouped_bibs = []
    for key, items in groupby(bibs_status, lambda i: i["_type"]):
        grouped_bibs.append(list(items))

    bibs_in_db, bibs_expanded, bibs_out_db = [], [], []
    for bib in grouped_bibs:
        if bib[0]["_type"] == "out_db":
            bibs_out_db.append(bib)
        elif bib[0]["_type"] == "expanded":
            bibs_expanded.append(bib)
        else:
            bibs_in_db.append(bib)

    # bib_status = map(
    # lambda group: list(group[1]),
    # groupby(
    # map(get_status, bibs_journal),
    # operator.itemgetter("_type"))
    # )

    # pdb.set_trace()
    updated_bibs = reduce(
        lambda a, b: a + b,  map(update_bibs_in, bibs_in_db)
    )
    updated_bibs += reduce(
        lambda a, b: a + b, map(update_bibs_out, bibs_out_db)
    )
    # updated_bibs += reduce(
        # lambda a, b: a + b, bibs_expanded
    # )
    updated_bibs += bibs_expanded[0]

    # bibs = bibs_in_db
    # bibs = bibs_out_db + bibs_expanded
    # pdb.set_trace()
    [
        [
            bib.pop(t, None) for bib in updated_bibs
        ]
        for t in ["_text", "_type"]
    ]
    # for item in updated_bibs:
            # del item["_text"]
            # del item["_type"]
    updated_bibs += bibs_arxiv + bibs_not_journal
    # bibtex.entries = bibs

    # pdb.set_trace()
    # new_bibtex = bibtexparser.bibdatabase.BibDatabase()
    # new_bibtex.entries = bibs
    # bibtex.entries = bibs
    # pdb.set_trace()
    # with open(args.output, "w") as bib_file:
        # bibtexparser.dumps(bibtex, bib_file)
        # bibtexparser.dump(bibtex, bib_file)
    writer = BibTexWriter()
    new_bibtex = BibDatabase()
    new_bibtex.entries = updated_bibs
    # pdb.set_trace()
    with open(args.output, 'w') as bibfile:
        bibfile.write(writer.write(new_bibtex).encode("utf8"))
    # bibtexparser.dump(bibtex, args.output.read())
    # pdb.set_trace()
    # return bibs

if __name__ == "__main__":
    main()
<commit_msg>Removes trash. Implements multiprocessing fix #1<commit_after>import bibtexparser
import argparse
import pdb
import json
from itertools import groupby
import operator
from bibtexparser.bwriter import BibTexWriter
from bibtexparser.bibdatabase import BibDatabase
from multiprocessing import Pool

# with open("../scrapper/list_abbrev.json") as file_data:
with open("../scrapper/list_abbrev.json") as file_data:
    list_abbrev = json.load(file_data)

def get_status(bib):
    def is_journal(item):
        return item["name"].lower() == bib["journal"].lower()

    def is_abbrev(item):
        return item["abbrev"].lower() == bib["journal"].lower()
    bib["_text"] = ""
    bib["_type"] = "out_db"
    a_journal = filter(is_journal, list_abbrev)
    a_abbrev = filter(is_abbrev, list_abbrev)
    # pdb.set_trace()
    if len(a_journal) > 0:
        # pdb.set_trace()
        bib["_text"] = a_journal[0]["abbrev"]
        bib["_type"] = "abreviated"
    elif len(a_abbrev) > 0:
        # pdb.set_trace()
        bib["_text"] = a_abbrev[0]["name"]
        bib["_type"] = "expanded"
    return bib

def manual_update(bibs):
    actions = {
        "y": lambda: update_bibs(bibs),
        "n": lambda: bibs,
        "c": lambda: update_bibs(bibs, custom=True)
    }
    question = "Replace '{}' for {}? y(yes)/n(no)/c(custom): "
    question = question.format(bibs[0]["journal"], bibs[0]["_text"])
    action = raw_input(question)
    try:
        return actions.get(action)()
    except TypeError:
        return manual_update(bibs)

def manual_update_out(bibs):
    actions = {
        "y": lambda: update_bibs(bibs, custom=True),
        "n": lambda: bibs,
    }
    question = "Replace '{}' ? y(yes)/n(no): "
    question = question.format(bibs[0]["journal"])
    action = raw_input(question)
    # try:
    try:
        return actions.get(action)()
    except TypeError:
        return manual_update_out(bibs)

def update_bibs_out(bibs):
    is_abreviation = raw_input(
        "'{}' is a abreviation?y(yes)n(no): ".format(bibs[0]["journal"])
    )
    if is_abreviation == "y":
        full_name = raw_input("Insert journal name: ")
        abreviation = bibs[0]["journal"]
    elif is_abreviation == "n":
        abreviation = raw_input("Insert abreviation: ")
        full_name = bibs[0]["journal"]
    else:
        return update_bibs_out(bibs)
    list_abbrev.append({"name": full_name, "abbrev": abreviation})
    for i, bib in enumerate(bibs):
        bibs[i]["journal"] = abreviation
    return bibs

def update_bibs(bibs, custom=False):
    if custom:
        journal = raw_input("Insert abreviation: ")
    else:
        journal = bibs[0]["_text"]

    for i, bib in enumerate(bibs):
        bibs[i]["journal"] = journal
    return bibs

def update_bibs_in(grouped_bibs):
    actions = {
        "a": lambda items: map(lambda bibs: update_bibs(bibs), items),
        "m": lambda items: map(lambda bibs: manual_update(bibs), items),
        "n": lambda items: items
    }
    print "{:d} itens can be {}".format(
        len(grouped_bibs), grouped_bibs[0]["_type"])
    action = raw_input("What you want? a(update all)/m(manual)/n(do nothing)")
    grouped_bibs.sort(key=operator.itemgetter('journal'))
    grouped_by_journal = []
    for key, items in groupby(grouped_bibs, lambda i: i["journal"]):
        grouped_by_journal.append(list(items))

    try:
        updated_bibs = actions.get(action)(grouped_by_journal)
    except TypeError:
        return update_bibs_in(grouped_bibs)

    updated_bibs = reduce(lambda a, b: a+b, updated_bibs)
    return updated_bibs

def main():
    parser = argparse.ArgumentParser(
        prog="jta",
        description="Abreviate the journals name inside a bibtex file.")
    parser.add_argument(
        "--input", "-i",
        required=True,
        type=argparse.FileType("r"),
        help="bibtex input file"
    )
    parser.add_argument(
        "--output", "-o",
        required=True,
        # type=argparse.FileType("w"),
        help="bibtex output file")

    args = parser.parse_args()
    bibtex = bibtexparser.loads(args.input.read())
    bibs_journal, bibs_not_journal = [], []
    for bib in bibtex.entries:
        (bibs_journal
         if "journal" in bib else bibs_not_journal).append(bib)
    bibs_published, bibs_arxiv = [], []
    for bib in bibs_journal:
        (bibs_arxiv
         if "arxiv" in bib["journal"].lower()
         else bibs_published).append(bib)
    # pdb.set_trace()
    pool = Pool()
    bibs_status = pool.map(get_status, bibs_published)
    pool.close()
    pool.join()
    bibs_status.sort(key=operator.itemgetter('_type'))
    grouped_bibs = []
    for key, items in groupby(bibs_status, lambda i: i["_type"]):
        grouped_bibs.append(list(items))

    bibs_in_db, bibs_expanded, bibs_out_db = [], [], []
    for bib in grouped_bibs:
        if bib[0]["_type"] == "out_db":
            bibs_out_db.append(bib)
        elif bib[0]["_type"] == "expanded":
            bibs_expanded.append(bib)
        else:
            bibs_in_db.append(bib)

    # bib_status = map(
    # lambda group: list(group[1]),
    # groupby(
    # map(get_status, bibs_journal),
    # operator.itemgetter("_type"))
    # )

    # pdb.set_trace()
    updated_bibs = reduce(
        lambda a, b: a + b,  map(update_bibs_in, bibs_in_db)
    )
    updated_bibs += reduce(
        lambda a, b: a + b, map(update_bibs_out, bibs_out_db)
    )
    updated_bibs += bibs_expanded[0]

    # bibs = bibs_in_db
    # bibs = bibs_out_db + bibs_expanded
    # pdb.set_trace()
    [
        [
            bib.pop(t, None) for bib in updated_bibs
        ]
        for t in ["_text", "_type"]
    ]
    updated_bibs += bibs_arxiv + bibs_not_journal
    # bibtex.entries = bibs

    writer = BibTexWriter()
    new_bibtex = BibDatabase()
    new_bibtex.entries = updated_bibs
    # pdb.set_trace()
    with open(args.output, 'w') as bibfile:
        bibfile.write(writer.write(new_bibtex).encode("utf8"))
    # bibtexparser.dump(bibtex, args.output.read())
    # pdb.set_trace()
    # return bibs

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""TensorAudio class."""

import numpy as np

from tensorflow_lite_support.python.task.audio.core.pybinds import _pywrap_audio_buffer
from tensorflow_lite_support.python.task.audio.core import audio_record

_CppAudioFormat = _pywrap_audio_buffer.AudioFormat
_LoadAudioBufferFromFile = _pywrap_audio_buffer.LoadAudioBufferFromFile

class TensorAudio(object):
  """A wrapper class to store the input audio."""

  def __init__(self,
               audio_format: _CppAudioFormat,
               buffer_size: int) -> None:
    """Initializes the `TensorAudio` object.

    Args:
      audio_format: format of the audio.
      buffer_size: buffer size of the audio.
      audio_data, optional: contains raw audio data.
    """
    self._format = audio_format
    self._buffer_size = buffer_size
    self._buffer = np.zeros(
        [self._buffer_size, self._format.channels], dtype=np.float32)

  def clear(self):
    """Clear the internal buffer and fill it with zeros."""
    self._buffer.fill(0)

  @classmethod
  def create_from_wav_file(cls,
                           file_name: str,
                           buffer_size: int) -> "TensorAudio":
    """Creates `TensorAudio` object from the WAV file.

    Args:
      file_name: WAV file name.
      buffer_size: Required input buffer size.

    Returns:
      `TensorAudio` object.

    Raises:
      status.StatusNotOk if the audio file can't be decoded.
    """
    # TODO(b/220931229): Raise RuntimeError instead of status.StatusNotOk.
    # Need to import the module to catch this error:
    # `from pybind11_abseil import status`
    # see https://github.com/pybind/pybind11_abseil#abslstatusor.
    audio = _LoadAudioBufferFromFile(
      file_name, buffer_size, np.zeros([buffer_size]))
    tensor = TensorAudio(audio.audio_format, audio.buffer_size)
    tensor.load_from_array(np.array(audio.float_buffer, copy=False))
    return tensor

  def load_from_audio_record(self, record: audio_record.AudioRecord) -> None:
    """Loads audio data from an AudioRecord instance.
    Args:
      record: An AudioRecord instance.
    Raises:
      ValueError: Raised if the audio record's config is invalid.
    """
    if record.buffer_size < self._buffer_size:
      raise ValueError(
        "The audio record's buffer size cannot be smaller than the tensor "
        "audio's sample count.")

    if record.channels != self._format.channels:
      raise ValueError(
        f"The audio record's channel count doesn't match. "
        f"Expects {self._format.channels} channel(s).")

    if record.sampling_rate != self._format.sample_rate:
      raise ValueError(
        f"The audio record's sampling rate doesn't match. "
        f"Expects {self._format.sample_rate}Hz.")

    # Load audio data from the AudioRecord instance.
    data = record.read(self._buffer_size)
    self.load_from_array(data.astype(np.float32))

  def load_from_array(self, src: np.ndarray) -> None:
    """Loads audio data from a NumPy array.

    Args:
      src: A NumPy array contains the input audio.

    Raises:
      ValueError if the input audio is too large or if it contains an invalid
      number of channels.
    """
    if len(src) != len(self._buffer):
      raise ValueError(
        f"Input audio contains an invalid number of samples. "
        f"Expect {len(self._buffer)}.")
    elif src.shape[1] != self._format.channels:
      raise ValueError(
        f"Input audio contains an invalid number of channels. "
        f"Expect {self._format.channels}.")

    self._buffer = src

  @property
  def format(self) -> _CppAudioFormat:
    """Gets the audio format of the audio."""
    return self._format

  @property
  def buffer_size(self) -> int:
    """Gets the sample count of the audio."""
    return self._buffer_size

  @property
  def buffer(self) -> np.ndarray:
    """Gets the internal buffer."""
    return self._buffer
<commit_msg>Updated create_from_wav_file method<commit_after># Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""TensorAudio class."""

import numpy as np

from tensorflow_lite_support.python.task.audio.core.pybinds import _pywrap_audio_buffer
from tensorflow_lite_support.python.task.audio.core import audio_record

_CppAudioFormat = _pywrap_audio_buffer.AudioFormat
_LoadAudioBufferFromFile = _pywrap_audio_buffer.LoadAudioBufferFromFile

class TensorAudio(object):
  """A wrapper class to store the input audio."""

  def __init__(self,
               audio_format: _CppAudioFormat,
               buffer_size: int) -> None:
    """Initializes the `TensorAudio` object.

    Args:
      audio_format: format of the audio.
      buffer_size: buffer size of the audio.
    """
    self._format = audio_format
    self._buffer_size = buffer_size
    self._buffer = np.zeros(
        [self._buffer_size, self._format.channels], dtype=np.float32)

  def clear(self):
    """Clear the internal buffer and fill it with zeros."""
    self._buffer.fill(0)

  @classmethod
  def create_from_wav_file(cls,
                           file_name: str,
                           buffer_size: int) -> "TensorAudio":
    """Creates `TensorAudio` object from the WAV file.

    Args:
      file_name: WAV file name.
      buffer_size: Required input buffer size.

    Returns:
      `TensorAudio` object.

    Raises:
      status.StatusNotOk if the audio file can't be decoded.
    """
    # TODO(b/220931229): Raise RuntimeError instead of status.StatusNotOk.
    # Need to import the module to catch this error:
    # `from pybind11_abseil import status`
    # see https://github.com/pybind/pybind11_abseil#abslstatusor.
    audio = _LoadAudioBufferFromFile(
      file_name, buffer_size, np.zeros([buffer_size]))
    tensor = TensorAudio(audio.audio_format, audio.buffer_size)
    tensor.load_from_array(np.array(audio.float_buffer, copy=False))
    return tensor

  def load_from_audio_record(self, record: audio_record.AudioRecord) -> None:
    """Loads audio data from an AudioRecord instance.
    Args:
      record: An AudioRecord instance.
    Raises:
      ValueError: Raised if the audio record's config is invalid.
    """
    if record.buffer_size < self._buffer_size:
      raise ValueError(
        "The audio record's buffer size cannot be smaller than the tensor "
        "audio's sample count.")

    if record.channels != self._format.channels:
      raise ValueError(
        f"The audio record's channel count doesn't match. "
        f"Expects {self._format.channels} channel(s).")

    if record.sampling_rate != self._format.sample_rate:
      raise ValueError(
        f"The audio record's sampling rate doesn't match. "
        f"Expects {self._format.sample_rate}Hz.")

    # Load audio data from the AudioRecord instance.
    data = record.read(self._buffer_size)
    self.load_from_array(data.astype(np.float32))

  def load_from_array(self, src: np.ndarray) -> None:
    """Loads audio data from a NumPy array.

    Args:
      src: A NumPy array contains the input audio.

    Raises:
      ValueError if the input audio is too large or if it contains an invalid
      number of channels.
    """
    if len(src) != len(self._buffer):
      raise ValueError(
        f"Input audio contains an invalid number of samples. "
        f"Expect {len(self._buffer)}.")
    elif src.shape[1] != self._format.channels:
      raise ValueError(
        f"Input audio contains an invalid number of channels. "
        f"Expect {self._format.channels}.")

    self._buffer = src

  @property
  def format(self) -> _CppAudioFormat:
    """Gets the audio format of the audio."""
    return self._format

  @property
  def buffer_size(self) -> int:
    """Gets the sample count of the audio."""
    return self._buffer_size

  @property
  def buffer(self) -> np.ndarray:
    """Gets the internal buffer."""
    return self._buffer
<|endoftext|>