<commit_before>from datetime import datetime, timedelta
import uuid
import os
import hashlib

from django.conf import settings
from django.template import Template, Context

from django.test import TestCase
import lxml
from django.core.files.uploadedfile import UploadedFile
from mock import patch

from casexml.apps.case.tests.util import TEST_DOMAIN_NAME
from casexml.apps.case.xml import V2
from corehq.apps.receiverwrapper.util import submit_form_locally
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors, FormAccessors
from couchforms.models import XFormInstance
from dimagi.utils.parsing import json_format_datetime
from corehq.form_processor.interfaces.processor import FormProcessorInterface
from corehq.form_processor.tests.utils import FormProcessorTestUtils, run_with_all_backends
from corehq.util.test_utils import TestFileMixin

TEST_CASE_ID = "EOL9FIAKIQWOFXFOH0QAMWU64"
CREATE_XFORM_ID = "6RGAZTETE3Z2QC0PE2DKM88MO"
media_path = os.path.join(os.path.dirname(__file__), "data", "attachments")

MEDIA_FILES = {
    "fruity_file": os.path.join(media_path, "fruity.jpg"),  # first
    "dimagi_logo_file": os.path.join(media_path, "dimagi_logo.jpg"),
    "commcare_logo_file": os.path.join(media_path, "commcare-logo.png"),
    "globe_file": os.path.join(media_path, "globe.pdf"),
    "house_file": os.path.join(media_path, "house.jpg"),
}

class BaseCaseMultimediaTest(TestCase, TestFileMixin):

    file_path = ('data', 'multimedia')
    root = os.path.dirname(__file__)

    def setUp(self):
        super(BaseCaseMultimediaTest, self).setUp()
        self.formdb = FormAccessors()
        FormProcessorTestUtils.delete_all_cases()
        FormProcessorTestUtils.delete_all_xforms()

    def _formatXForm(self, doc_id, raw_xml, attachment_block, date=None):
        if date is None:
            date = datetime.utcnow()
        final_xml = Template(raw_xml).render(Context({
            "attachments": attachment_block,
            "time_start": json_format_datetime(date - timedelta(minutes=4)),
            "time_end": json_format_datetime(date),
            "date_modified": json_format_datetime(date),
            "doc_id": doc_id
        }))
        return final_xml

    def _prepAttachments(self, new_attachments, removes=[]):
        """
        Returns:
            attachment_block - An XML representation of the attachment
            dict_attachments - A key-value dict where the key is the name and the value is a Stream of the
            attachment
        """
        attachment_block = ''.join([self._singleAttachBlock(x) for x in new_attachments] + [self._singleAttachRemoveBlock(x) for x in removes])
        dict_attachments = dict((MEDIA_FILES[attach_name], self._attachmentFileStream(attach_name)) for attach_name in new_attachments)
        return attachment_block, dict_attachments

    def _singleAttachBlock(self, key):
        return '<n0:%s src="%s" from="local"/>' % (key, MEDIA_FILES[key])

    def _singleAttachRemoveBlock(self, key):
        return '<n0:%s />' % key

    def _attachmentFileStream(self, key):
        attachment_path = MEDIA_FILES[key]
        attachment = open(attachment_path, 'rb')
        uf = UploadedFile(attachment, key)
        return uf

    def _calc_file_hash(self, key):
        with open(MEDIA_FILES[key], 'rb') as attach:
            return hashlib.md5(attach.read()).hexdigest()

    def _do_submit(self, xml_data, dict_attachments, sync_token=None, date=None):
        """
        RequestFactory submitter - simulates direct submission to server directly (no need to call process case after fact)
        """
        response, form, cases = submit_form_locally(
            xml_data,
            TEST_DOMAIN_NAME,
            attachments=dict_attachments,
            last_sync_token=sync_token,
            received_on=date
        )
        attachments = form.attachments
        self.assertEqual(set(dict_attachments.keys()),
                         set(attachments.keys()))
        [case] = cases
        self.assertEqual(case.case_id, TEST_CASE_ID)

        return response, form, cases

    def _submit_and_verify(self, doc_id, xml_data, dict_attachments,
                           sync_token=None, date=None):
        response, form, [case] = self._do_submit(xml_data, dict_attachments, sync_token, date=date)

        attachments = form.attachments
        self.assertEqual(len(dict_attachments), len(attachments))
        for k, vstream in dict_attachments.items():
            fileback = form.get_attachment(k)
            # rewind the pointer before comparing
            orig_attachment = vstream
            orig_attachment.seek(0)
            self.assertEqual(hashlib.md5(fileback).hexdigest(), hashlib.md5(orig_attachment.read()).hexdigest())

        case = CaseAccessors(TEST_DOMAIN_NAME).get_case(case.case_id)  # re-fetch case
        return form, case

    def _doCreateCaseWithMultimedia(self, attachments=['fruity_file']):
        xml_data = self.get_xml('multimedia_create')
        attachment_block, dict_attachments = self._prepAttachments(attachments)
        final_xml = self._formatXForm(CREATE_XFORM_ID, xml_data, attachment_block)
        return self._submit_and_verify(CREATE_XFORM_ID, final_xml, dict_attachments)

    def _doSubmitUpdateWithMultimedia(self, new_attachments=None, removes=None,
                                      sync_token=None, date=None):
        new_attachments = new_attachments if new_attachments is not None \
            else ['commcare_logo_file', 'dimagi_logo_file']
        removes = removes if removes is not None else ['fruity_file']
        attachment_block, dict_attachments = self._prepAttachments(new_attachments, removes=removes)
        raw_xform = self.get_xml('multimedia_update')
        doc_id = uuid.uuid4().hex
        final_xform = self._formatXForm(doc_id, raw_xform, attachment_block, date)
        return self._submit_and_verify(doc_id, final_xform, dict_attachments,
                                sync_token, date=date)

class CaseMultimediaTest(BaseCaseMultimediaTest):
    """
    Tests new attachments for cases and case properties
    Spec: https://github.com/dimagi/commcare/wiki/CaseAttachmentAPI
    """
    @run_with_all_backends
    def testAttachInCreate(self):
        single_attach = 'fruity_file'
        xform, case = self._doCreateCaseWithMultimedia(attachments=[single_attach])

        self.assertEqual(1, len(case.case_attachments))
        self.assertTrue(single_attach in case.case_attachments)
        self.assertEqual(
            self._calc_file_hash(single_attach),
            hashlib.md5(case.get_attachment(single_attach)).hexdigest()
        )
        if not settings.TESTS_SHOULD_USE_SQL_BACKEND:
            self.assertEqual(1, len(filter(lambda x: x['action_type'] == 'attachment', case.actions)))

    @run_with_all_backends
    def testArchiveAfterAttach(self):
        single_attach = 'fruity_file'
        xform, case = self._doCreateCaseWithMultimedia(attachments=[single_attach])

        for xform_id in case.xform_ids:
            form = self.formdb.get_form(xform_id)

            form.archive()
            form = self.formdb.get_form(xform_id)
            self.assertTrue(form.is_archived)

            form.unarchive()
            form = self.formdb.get_form(xform_id)
            self.assertFalse(form.is_archived)

    @run_with_all_backends
    def testAttachRemoveSingle(self):
        self._doCreateCaseWithMultimedia()
        new_attachments = []
        removes = ['fruity_file']
        _, case = self._doSubmitUpdateWithMultimedia(new_attachments=new_attachments, removes=removes)

        self.assertEqual(0, len(case.case_attachments))

        if not settings.TESTS_SHOULD_USE_SQL_BACKEND:
            attach_actions = filter(lambda x: x['action_type'] == 'attachment', case.actions)
            self.assertEqual(2, len(attach_actions))
            last_action = attach_actions[-1]
            self.assertEqual(sorted(removes), sorted(last_action['attachments'].keys()))

    @run_with_all_backends
    def testAttachRemoveMultiple(self):
        self._doCreateCaseWithMultimedia()

        new_attachments = ['commcare_logo_file', 'dimagi_logo_file']
        removes = ['fruity_file']
        _, case = self._doSubmitUpdateWithMultimedia(new_attachments=new_attachments, removes=removes)

        self.assertEqual(sorted(new_attachments), sorted(case.case_attachments.keys()))

        if not settings.TESTS_SHOULD_USE_SQL_BACKEND:
            attach_actions = filter(lambda x: x['action_type'] == 'attachment', case.actions)
            self.assertEqual(2, len(attach_actions))

    @run_with_all_backends
    def testOTARestoreSingle(self):
        _, case = self._doCreateCaseWithMultimedia()
        restore_attachments = ['fruity_file']
        self._validateOTARestore(case.case_id, restore_attachments)

    @run_with_all_backends
    def testOTARestoreMultiple(self):
        _, case = self._doCreateCaseWithMultimedia()
        restore_attachments = ['commcare_logo_file', 'dimagi_logo_file']
        removes = ['fruity_file']
        _, case = self._doSubmitUpdateWithMultimedia(new_attachments=restore_attachments, removes=removes)

        self._validateOTARestore(case.case_id, restore_attachments)

    def _validateOTARestore(self, case_id, restore_attachments):
        case_xml = CaseAccessors().get_case(case_id).to_xml(V2)
        root_node = lxml.etree.fromstring(case_xml)
        attaches = root_node.find('{http://commcarehq.org/case/transaction/v2}attachment')
        self.assertEqual(len(restore_attachments), len(attaches))

        for attach in attaches:
            url = attach.values()[1]
            case_id = url.split('/')[-2]
            attach_key_from_url = url.split('/')[-1]
            tag = attach.tag
            clean_tag = tag.replace('{http://commcarehq.org/case/transaction/v2}', '')
            self.assertEqual(clean_tag, attach_key_from_url)
            self.assertEqual(case_id, TEST_CASE_ID)
            self.assertIn(attach_key_from_url, restore_attachments)
            restore_attachments.remove(clean_tag)

        self.assertEqual(0, len(restore_attachments))

    @run_with_all_backends
    def testAttachInUpdate(self):
        new_attachments = ['commcare_logo_file', 'dimagi_logo_file']

        self._doCreateCaseWithMultimedia()
        _, case = self._doSubmitUpdateWithMultimedia(new_attachments=new_attachments, removes=[])

        # 1 plus the 2 we had
        self.assertEqual(len(new_attachments) + 1, len(case.case_attachments))
        if not settings.TESTS_SHOULD_USE_SQL_BACKEND:
            attach_actions = filter(lambda x: x['action_type'] == 'attachment', case.actions)
            self.assertEqual(2, len(attach_actions))
            last_action = attach_actions[-1]
            self.assertEqual(sorted(new_attachments), sorted(last_action['attachments'].keys()))

        for attach_name in new_attachments:
            self.assertTrue(attach_name in case.case_attachments)
            self.assertEqual(
                self._calc_file_hash(attach_name),
                hashlib.md5(case.get_attachment(attach_name)).hexdigest()
            )

    def testUpdateWithNoNewAttachment(self):
        _, case = self._doCreateCaseWithMultimedia()
        bulk_save = XFormInstance.get_db().bulk_save
        bulk_save_attachments = []

        # pull out and record attachments to docs being bulk saved
        def new_bulk_save(docs, *args, **kwargs):
            for doc in docs:
                if doc['_id'] == TEST_CASE_ID:
                    bulk_save_attachments.append(doc['_attachments'])
            bulk_save(docs, *args, **kwargs)

        self._doSubmitUpdateWithMultimedia(
            new_attachments=[], removes=[])

        with patch('couchforms.models.XFormInstance._db.bulk_save', new_bulk_save):
            # submit from the 2 min in the past to trigger a rebuild
            self._doSubmitUpdateWithMultimedia(
                new_attachments=[], removes=[],
                date=datetime.utcnow() - timedelta(minutes=2))

        # make sure there's exactly one bulk save recorded
        self.assertEqual(len(bulk_save_attachments), 1)
        # make sure none of the attachments were re-saved in rebuild
        self.assertEqual(
            [key for key, value in bulk_save_attachments[0].items()
             if value.get('data')], [])

    @run_with_all_backends
    def test_sync_log_invalidation_bug(self):
        sync_log = FormProcessorInterface().sync_log_model(
            user_id='6dac4940-913e-11e0-9d4b-005056aa7fb5'
        )
        sync_log.save()
        self.addCleanup(FormProcessorTestUtils.delete_all_sync_logs)

        _, case = self._doCreateCaseWithMultimedia()

        # this used to fail before we fixed http://manage.dimagi.com/default.asp?158373
        self._doSubmitUpdateWithMultimedia(new_attachments=['commcare_logo_file'], removes=[],
                                           sync_token=sync_log._id)
<commit_msg>failing test for case attachments<commit_after>from datetime import datetime, timedelta
import uuid
import os
import hashlib
from unittest.case import skip

from django.conf import settings
from django.template import Template, Context

from django.test import TestCase
import lxml
from django.core.files.uploadedfile import UploadedFile
from mock import patch

from casexml.apps.case.tests.util import TEST_DOMAIN_NAME
from casexml.apps.case.xml import V2
from corehq.apps.receiverwrapper.util import submit_form_locally
from corehq.blobs import get_blob_db
from corehq.blobs.tests.util import TemporaryS3BlobDB
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors, FormAccessors
from couchforms.models import XFormInstance
from dimagi.utils.parsing import json_format_datetime
from corehq.form_processor.interfaces.processor import FormProcessorInterface
from corehq.form_processor.tests.utils import FormProcessorTestUtils, run_with_all_backends
from corehq.util.test_utils import TestFileMixin, trap_extra_setup

TEST_CASE_ID = "EOL9FIAKIQWOFXFOH0QAMWU64"
CREATE_XFORM_ID = "6RGAZTETE3Z2QC0PE2DKM88MO"
media_path = os.path.join(os.path.dirname(__file__), "data", "attachments")

MEDIA_FILES = {
    "fruity_file": os.path.join(media_path, "fruity.jpg"),  # first
    "dimagi_logo_file": os.path.join(media_path, "dimagi_logo.jpg"),
    "commcare_logo_file": os.path.join(media_path, "commcare-logo.png"),
    "globe_file": os.path.join(media_path, "globe.pdf"),
    "house_file": os.path.join(media_path, "house.jpg"),
}

class BaseCaseMultimediaTest(TestCase, TestFileMixin):

    file_path = ('data', 'multimedia')
    root = os.path.dirname(__file__)

    def setUp(self):
        super(BaseCaseMultimediaTest, self).setUp()
        self.formdb = FormAccessors()
        FormProcessorTestUtils.delete_all_cases()
        FormProcessorTestUtils.delete_all_xforms()

    def _formatXForm(self, doc_id, raw_xml, attachment_block, date=None):
        if date is None:
            date = datetime.utcnow()
        final_xml = Template(raw_xml).render(Context({
            "attachments": attachment_block,
            "time_start": json_format_datetime(date - timedelta(minutes=4)),
            "time_end": json_format_datetime(date),
            "date_modified": json_format_datetime(date),
            "doc_id": doc_id
        }))
        return final_xml

    def _prepAttachments(self, new_attachments, removes=[]):
        """
        Returns:
            attachment_block - An XML representation of the attachment
            dict_attachments - A key-value dict where the key is the name and the value is a Stream of the
            attachment
        """
        attachment_block = ''.join([self._singleAttachBlock(x) for x in new_attachments] + [self._singleAttachRemoveBlock(x) for x in removes])
        dict_attachments = dict((MEDIA_FILES[attach_name], self._attachmentFileStream(attach_name)) for attach_name in new_attachments)
        return attachment_block, dict_attachments

    def _singleAttachBlock(self, key):
        return '<n0:%s src="%s" from="local"/>' % (key, MEDIA_FILES[key])

    def _singleAttachRemoveBlock(self, key):
        return '<n0:%s />' % key

    def _attachmentFileStream(self, key):
        attachment_path = MEDIA_FILES[key]
        attachment = open(attachment_path, 'rb')
        uf = UploadedFile(attachment, key)
        return uf

    def _calc_file_hash(self, key):
        with open(MEDIA_FILES[key], 'rb') as attach:
            return hashlib.md5(attach.read()).hexdigest()

    def _do_submit(self, xml_data, dict_attachments, sync_token=None, date=None):
        """
        RequestFactory submitter - simulates direct submission to server directly (no need to call process case after fact)
        """
        response, form, cases = submit_form_locally(
            xml_data,
            TEST_DOMAIN_NAME,
            attachments=dict_attachments,
            last_sync_token=sync_token,
            received_on=date
        )
        attachments = form.attachments
        self.assertEqual(set(dict_attachments.keys()),
                         set(attachments.keys()))
        [case] = cases
        self.assertEqual(case.case_id, TEST_CASE_ID)

        return response, form, cases

    def _submit_and_verify(self, doc_id, xml_data, dict_attachments,
                           sync_token=None, date=None):
        response, form, [case] = self._do_submit(xml_data, dict_attachments, sync_token, date=date)

        attachments = form.attachments
        self.assertEqual(len(dict_attachments), len(attachments))
        for k, vstream in dict_attachments.items():
            fileback = form.get_attachment(k)
            # rewind the pointer before comparing
            orig_attachment = vstream
            orig_attachment.seek(0)
            self.assertEqual(hashlib.md5(fileback).hexdigest(), hashlib.md5(orig_attachment.read()).hexdigest())

        case = CaseAccessors(TEST_DOMAIN_NAME).get_case(case.case_id)  # re-fetch case
        return form, case

    def _doCreateCaseWithMultimedia(self, attachments=['fruity_file']):
        xml_data = self.get_xml('multimedia_create')
        attachment_block, dict_attachments = self._prepAttachments(attachments)
        final_xml = self._formatXForm(CREATE_XFORM_ID, xml_data, attachment_block)
        return self._submit_and_verify(CREATE_XFORM_ID, final_xml, dict_attachments)

    def _doSubmitUpdateWithMultimedia(self, new_attachments=None, removes=None,
                                      sync_token=None, date=None):
        new_attachments = new_attachments if new_attachments is not None \
            else ['commcare_logo_file', 'dimagi_logo_file']
        removes = removes if removes is not None else ['fruity_file']
        attachment_block, dict_attachments = self._prepAttachments(new_attachments, removes=removes)
        raw_xform = self.get_xml('multimedia_update')
        doc_id = uuid.uuid4().hex
        final_xform = self._formatXForm(doc_id, raw_xform, attachment_block, date)
        return self._submit_and_verify(doc_id, final_xform, dict_attachments,
                                sync_token, date=date)

class CaseMultimediaTest(BaseCaseMultimediaTest):
    """
    Tests new attachments for cases and case properties
    Spec: https://github.com/dimagi/commcare/wiki/CaseAttachmentAPI
    """
    @run_with_all_backends
    def testAttachInCreate(self):
        single_attach = 'fruity_file'
        xform, case = self._doCreateCaseWithMultimedia(attachments=[single_attach])

        self.assertEqual(1, len(case.case_attachments))
        self.assertTrue(single_attach in case.case_attachments)
        self.assertEqual(
            self._calc_file_hash(single_attach),
            hashlib.md5(case.get_attachment(single_attach)).hexdigest()
        )
        if not settings.TESTS_SHOULD_USE_SQL_BACKEND:
            self.assertEqual(1, len(filter(lambda x: x['action_type'] == 'attachment', case.actions)))

    @run_with_all_backends
    def testArchiveAfterAttach(self):
        single_attach = 'fruity_file'
        xform, case = self._doCreateCaseWithMultimedia(attachments=[single_attach])

        for xform_id in case.xform_ids:
            form = self.formdb.get_form(xform_id)

            form.archive()
            form = self.formdb.get_form(xform_id)
            self.assertTrue(form.is_archived)

            form.unarchive()
            form = self.formdb.get_form(xform_id)
            self.assertFalse(form.is_archived)

    @run_with_all_backends
    def testAttachRemoveSingle(self):
        self._doCreateCaseWithMultimedia()
        new_attachments = []
        removes = ['fruity_file']
        _, case = self._doSubmitUpdateWithMultimedia(new_attachments=new_attachments, removes=removes)

        self.assertEqual(0, len(case.case_attachments))

        if not settings.TESTS_SHOULD_USE_SQL_BACKEND:
            attach_actions = filter(lambda x: x['action_type'] == 'attachment', case.actions)
            self.assertEqual(2, len(attach_actions))
            last_action = attach_actions[-1]
            self.assertEqual(sorted(removes), sorted(last_action['attachments'].keys()))

    @run_with_all_backends
    def testAttachRemoveMultiple(self):
        self._doCreateCaseWithMultimedia()

        new_attachments = ['commcare_logo_file', 'dimagi_logo_file']
        removes = ['fruity_file']
        _, case = self._doSubmitUpdateWithMultimedia(new_attachments=new_attachments, removes=removes)

        self.assertEqual(sorted(new_attachments), sorted(case.case_attachments.keys()))

        if not settings.TESTS_SHOULD_USE_SQL_BACKEND:
            attach_actions = filter(lambda x: x['action_type'] == 'attachment', case.actions)
            self.assertEqual(2, len(attach_actions))

    @run_with_all_backends
    def testOTARestoreSingle(self):
        _, case = self._doCreateCaseWithMultimedia()
        restore_attachments = ['fruity_file']
        self._validateOTARestore(case.case_id, restore_attachments)

    @run_with_all_backends
    def testOTARestoreMultiple(self):
        _, case = self._doCreateCaseWithMultimedia()
        restore_attachments = ['commcare_logo_file', 'dimagi_logo_file']
        removes = ['fruity_file']
        _, case = self._doSubmitUpdateWithMultimedia(new_attachments=restore_attachments, removes=removes)

        self._validateOTARestore(case.case_id, restore_attachments)

    def _validateOTARestore(self, case_id, restore_attachments):
        case_xml = CaseAccessors().get_case(case_id).to_xml(V2)
        root_node = lxml.etree.fromstring(case_xml)
        attaches = root_node.find('{http://commcarehq.org/case/transaction/v2}attachment')
        self.assertEqual(len(restore_attachments), len(attaches))

        for attach in attaches:
            url = attach.values()[1]
            case_id = url.split('/')[-2]
            attach_key_from_url = url.split('/')[-1]
            tag = attach.tag
            clean_tag = tag.replace('{http://commcarehq.org/case/transaction/v2}', '')
            self.assertEqual(clean_tag, attach_key_from_url)
            self.assertEqual(case_id, TEST_CASE_ID)
            self.assertIn(attach_key_from_url, restore_attachments)
            restore_attachments.remove(clean_tag)

        self.assertEqual(0, len(restore_attachments))

    @run_with_all_backends
    def testAttachInUpdate(self):
        new_attachments = ['commcare_logo_file', 'dimagi_logo_file']

        self._doCreateCaseWithMultimedia()
        _, case = self._doSubmitUpdateWithMultimedia(new_attachments=new_attachments, removes=[])

        # 1 plus the 2 we had
        self.assertEqual(len(new_attachments) + 1, len(case.case_attachments))
        if not settings.TESTS_SHOULD_USE_SQL_BACKEND:
            attach_actions = filter(lambda x: x['action_type'] == 'attachment', case.actions)
            self.assertEqual(2, len(attach_actions))
            last_action = attach_actions[-1]
            self.assertEqual(sorted(new_attachments), sorted(last_action['attachments'].keys()))

        for attach_name in new_attachments:
            self.assertTrue(attach_name in case.case_attachments)
            self.assertEqual(
                self._calc_file_hash(attach_name),
                hashlib.md5(case.get_attachment(attach_name)).hexdigest()
            )

    def testUpdateWithNoNewAttachment(self):
        _, case = self._doCreateCaseWithMultimedia()
        bulk_save = XFormInstance.get_db().bulk_save
        bulk_save_attachments = []

        # pull out and record attachments to docs being bulk saved
        def new_bulk_save(docs, *args, **kwargs):
            for doc in docs:
                if doc['_id'] == TEST_CASE_ID:
                    bulk_save_attachments.append(doc['_attachments'])
            bulk_save(docs, *args, **kwargs)

        self._doSubmitUpdateWithMultimedia(
            new_attachments=[], removes=[])

        with patch('couchforms.models.XFormInstance._db.bulk_save', new_bulk_save):
            # submit from the 2 min in the past to trigger a rebuild
            self._doSubmitUpdateWithMultimedia(
                new_attachments=[], removes=[],
                date=datetime.utcnow() - timedelta(minutes=2))

        # make sure there's exactly one bulk save recorded
        self.assertEqual(len(bulk_save_attachments), 1)
        # make sure none of the attachments were re-saved in rebuild
        self.assertEqual(
            [key for key, value in bulk_save_attachments[0].items()
             if value.get('data')], [])

    @run_with_all_backends
    def test_sync_log_invalidation_bug(self):
        sync_log = FormProcessorInterface().sync_log_model(
            user_id='6dac4940-913e-11e0-9d4b-005056aa7fb5'
        )
        sync_log.save()
        self.addCleanup(FormProcessorTestUtils.delete_all_sync_logs)

        _, case = self._doCreateCaseWithMultimedia()

        # this used to fail before we fixed http://manage.dimagi.com/default.asp?158373
        self._doSubmitUpdateWithMultimedia(new_attachments=['commcare_logo_file'], removes=[],
                                           sync_token=sync_log._id)

@skip("Case attachments don't work with S3 API yet")
class CaseMultimediaS3DBTest(BaseCaseMultimediaTest):
    """
    Tests new attachments for cases and case properties
    Spec: https://github.com/dimagi/commcare/wiki/CaseAttachmentAPI
    """

    def setUp(self):
        super(CaseMultimediaS3DBTest, self).setUp()
        with trap_extra_setup(AttributeError, msg="S3_BLOB_DB_SETTINGS not configured"):
            config = settings.S3_BLOB_DB_SETTINGS
            self.s3db = TemporaryS3BlobDB(config)
            assert get_blob_db() is self.s3db, (get_blob_db(), self.s3db)

    def tearDown(self):
        self.s3db.close()
        super(CaseMultimediaS3DBTest, self).tearDown()

    @run_with_all_backends
    def test_case_attachment(self):
        single_attach = 'fruity_file'
        xform, case = self._doCreateCaseWithMultimedia(attachments=[single_attach])

        self.assertEqual(1, len(case.case_attachments))
        self.assertTrue(single_attach in case.case_attachments)
        self.assertEqual(
            self._calc_file_hash(single_attach),
            hashlib.md5(case.get_attachment(single_attach)).hexdigest()
        )
<|endoftext|>


<commit_before>import os
import re
import ConfigParser

from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from collections import defaultdict
from collections import Counter
from schema import *

def get_config(localfile="config.ini", default_file=True):
    """
    This grabs a configuration file and converts it into
    a dictionary.

    The default filename is called config.ini
    First we load the global file, then we load a local file
    """
    if default_file:
        openfile = "{0}/config.ini".format(os.path.dirname(os.path.realpath(__file__)))
    else:
        openfile = localfile
    config = defaultdict(dict)
    if os.path.isfile(openfile):
        cfg = ConfigParser.ConfigParser()
        cfg.read(openfile)
        for s in cfg.sections():
            for k, v in cfg.items(s):
                dec = re.compile('\d+(\.\d+)?')
                if v in ("True", "False") or v.isdigit() or dec.match(v):
                    v = eval(v)
                config[s][k] = v

    # this enables us to load a local file
    if default_file:
        newconfig = get_config(localfile, default_file=False)
        for section in newconfig:
            for item in newconfig[section]:
                config[section][item] = newconfig[section][item]

    return config

def fetch_session(db=None):
    """
    Read from config.ini file and load appropriate database
    """
    config = get_config()
    echo = config.get('global').get('echo')
    if not db:
        db = config.get('global').get('database')
    if db[:6] == "sqlite":
        sqlite_db_path = os.path.join(
            config.get(db).get('path'),
            config.get(db).get('database'))
        #Delete any existing database if we should refresh
        #Actually doesn't work properly - it calls itself on clean.py as well! Needs more work
        #if(config.get('sqlite').get('refresh')):
        #    try:
        #        os.remove(sqlite_db_path)
        #    except:
        #        pass
        engine = create_engine('sqlite:///{0}'.format(sqlite_db_path), echo=echo)
    else:
        engine = create_engine('mysql+mysqldb://{0}:{1}@{2}/{3}?charset=utf8'.format(
            config.get(db).get('user'),
            config.get(db).get('password'),
            config.get(db).get('host'),
            config.get(db).get('database')), echo=echo)
    Base.metadata.create_all(engine)
    Session = sessionmaker(bind=engine)
    session = Session()
    return session

session = fetch_session()

def match(objects=[], override={}):
    """
    Pass in several objects and make them equal
    Override is specified if there is a desire to override
    certain fields

    TODO: Add stuff to have a flag so it doesn't search
    TODO: Add stuff to default to certain variables
    TODO: Add better linkage of inventor/assignee > location
    """
    if type(objects).__name__ not in ('list', 'tuple'):
        objects = [objects]
    freq = defaultdict(Counter)
    param = {}
    all_objects = []
    all_objects.extend(objects)
    clean_objects = []

    # we extend our objects and determine the
    # previously associated items
    for obj in objects:
        clean = obj.__clean__
        # keep track of all the "clean" objects
        if clean:
            if clean not in clean_objects:
                clean_objects.append(clean)
            # add the "raw" objects as we want to
            # iterate these items
            for o in clean.__raw__:
                if o not in all_objects:
                    all_objects.append(o)

    # this helps us determine items to summarize
    # ques: how do we indicate most recent?
    #   like for people and their locations? hrm..
    for obj in all_objects:
        for k, v in obj.summarize.iteritems():
            if v not in (None, ""):
                freq[k][v] += 1
        if "id" not in param:
            param["id"] = obj.uuid
        param["id"] = min(param["id"], obj.uuid)

    # create parameters based on most frequent
    for k in freq:
        param[k] = freq[k].most_common(1)[0][0]
    param.update(override)

    # remove all clean objects
    for obj in clean_objects:
        session.delete(obj)
    session.commit()  # commit necessary

    relobj = objects[0].__related__(**param)
    # associate the data into the related object

    for obj in all_objects:
        relobj.__raw__.append(obj)
        if type(relobj.__many__).__name__ in ("dict"):
            # if it is a dictionary type, iterate and add
            for key in relobj.__many__.keys():
                if type(obj.__single__[key]).__name__ in ('list', 'tuple'):
                    relobj.__many__[key].extend(set(obj.__single__[key]) - set(relobj.__many__[key]))
                elif obj.__single__[key] not in relobj.__many__[key]:
                    relobj.__many__[key].append(obj.__single__[key])
        else:
            if obj.__single__ and obj.__single__ not in relobj.__many__:
                relobj.__many__.append(obj.__single__)
    session.merge(relobj)
    session.commit()

def add(obj, override=True, temp=False):
    """
    PatentGrant Object converting to tables via SQLAlchemy
    Necessary to convert dates to datetime because of SQLite (OK on MySQL)

    Case Sensitivity and Table Reflection
spr0us    MySQL has inconsistent support for case-sensitive identifier names,
    basing support on specific details of the underlying operating system.
    However, it has been observed that no matter what case sensitivity
    behavior is present, the names of tables in foreign key declarations
    are always received from the database as all-lower case, making it
    impossible to accurately reflect a schema where inter-related tables
    use mixed-case identifier names.

    Therefore it is strongly advised that table names be declared as all
    lower case both within SQLAlchemy as well as on the MySQL database
    itself, especially if database reflection features are to be used.
    """

    # if a patent exists, remove it so we can replace it
    pat_query = session.query(Patent).filter(Patent.number == obj.patent)
    if pat_query.count():
        if override:
            session.delete(pat_query.one())
        else:
            return
    if len(obj.pat["number"]) < 3:
        return

    #add
    # lots of abstracts seem to be missing. why?

    pat = Patent(**obj.pat)
    pat.application = Application(**obj.app)

    #+asg
    for asg, loc in obj.assignee_list:
        asg = RawAssignee(**asg)
        loc = RawLocation(**loc)
        session.merge(loc)
        asg.rawlocation = loc
        pat.rawassignees.append(asg)

    #+inv
    for inv, loc in obj.inventor_list:
        inv = RawInventor(**inv)
        loc = RawLocation(**loc)
        session.merge(loc)
        inv.rawlocation = loc
        pat.rawinventors.append(inv)

    #+law
    for law in obj.lawyer_list:
        law = RawLawyer(**law)
        pat.rawlawyers.append(law)

    #+usreldoc
    for usr in obj.us_relation_list:
        usr = USRelDoc(**usr)
        pat.usreldocs.append(usr)

    #+classes
    for uspc, mc, sc in obj.us_classifications:
        uspc = USPC(**uspc)
        mc = MainClass(**mc)
        sc = SubClass(**sc)
        session.merge(mc)
        session.merge(sc)
        uspc.mainclass = mc
        uspc.subclass = sc
        pat.classes.append(uspc)

    #+ipcr
    for ipc in obj.ipcr_classifications:
        ipc = IPCR(**ipc)
        pat.ipcrs.append(ipc)

    # citations are huge. this dumps them to
    # a temporary database which we can use for later
    if temp:
        cits, refs = obj.citation_list
        for cit in cits:
            cit["patent_id"] = obj.pat["number"]
            cit = TempCitation(**cit)
            session.add(cit)
        for ref in refs:
            ref["patent_id"] = obj.pat["number"]
            ref = TempOtherReference(**ref)
            session.add(ref)
    else:
        cits, refs = obj.citation_list
        for cit in cits:
            cit = Citation(**cit)
            pat.citations.append(cit)
        for ref in refs:
            ref = OtherReference(**ref)
            pat.otherreferences.append(ref)

    session.merge(pat)

def commit():
    try:
        session.commit()
    except Exception, e:
        session.rollback()
        print str(e)
<commit_msg>fix typo<commit_after>import os
import re
import ConfigParser

from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from collections import defaultdict
from collections import Counter
from schema import *

def get_config(localfile="config.ini", default_file=True):
    """
    This grabs a configuration file and converts it into
    a dictionary.

    The default filename is called config.ini
    First we load the global file, then we load a local file
    """
    if default_file:
        openfile = "{0}/config.ini".format(os.path.dirname(os.path.realpath(__file__)))
    else:
        openfile = localfile
    config = defaultdict(dict)
    if os.path.isfile(openfile):
        cfg = ConfigParser.ConfigParser()
        cfg.read(openfile)
        for s in cfg.sections():
            for k, v in cfg.items(s):
                dec = re.compile('\d+(\.\d+)?')
                if v in ("True", "False") or v.isdigit() or dec.match(v):
                    v = eval(v)
                config[s][k] = v

    # this enables us to load a local file
    if default_file:
        newconfig = get_config(localfile, default_file=False)
        for section in newconfig:
            for item in newconfig[section]:
                config[section][item] = newconfig[section][item]

    return config

def fetch_session(db=None):
    """
    Read from config.ini file and load appropriate database
    """
    config = get_config()
    echo = config.get('global').get('echo')
    if not db:
        db = config.get('global').get('database')
    if db[:6] == "sqlite":
        sqlite_db_path = os.path.join(
            config.get(db).get('path'),
            config.get(db).get('database'))
        #Delete any existing database if we should refresh
        #Actually doesn't work properly - it calls itself on clean.py as well! Needs more work
        #if(config.get('sqlite').get('refresh')):
        #    try:
        #        os.remove(sqlite_db_path)
        #    except:
        #        pass
        engine = create_engine('sqlite:///{0}'.format(sqlite_db_path), echo=echo)
    else:
        engine = create_engine('mysql+mysqldb://{0}:{1}@{2}/{3}?charset=utf8'.format(
            config.get(db).get('user'),
            config.get(db).get('password'),
            config.get(db).get('host'),
            config.get(db).get('database')), echo=echo)
    Base.metadata.create_all(engine)
    Session = sessionmaker(bind=engine)
    session = Session()
    return session

session = fetch_session()

def match(objects=[], override={}):
    """
    Pass in several objects and make them equal
    Override is specified if there is a desire to override
    certain fields

    TODO: Add stuff to have a flag so it doesn't search
    TODO: Add stuff to default to certain variables
    TODO: Add better linkage of inventor/assignee > location
    """
    if type(objects).__name__ not in ('list', 'tuple'):
        objects = [objects]
    freq = defaultdict(Counter)
    param = {}
    all_objects = []
    all_objects.extend(objects)
    clean_objects = []

    # we extend our objects and determine the
    # previously associated items
    for obj in objects:
        clean = obj.__clean__
        # keep track of all the "clean" objects
        if clean:
            if clean not in clean_objects:
                clean_objects.append(clean)
            # add the "raw" objects as we want to
            # iterate these items
            for o in clean.__raw__:
                if o not in all_objects:
                    all_objects.append(o)

    # this helps us determine items to summarize
    # ques: how do we indicate most recent?
    #   like for people and their locations? hrm..
    for obj in all_objects:
        for k, v in obj.summarize.iteritems():
            if v not in (None, ""):
                freq[k][v] += 1
        if "id" not in param:
            param["id"] = obj.uuid
        param["id"] = min(param["id"], obj.uuid)

    # create parameters based on most frequent
    for k in freq:
        param[k] = freq[k].most_common(1)[0][0]
    param.update(override)

    # remove all clean objects
    for obj in clean_objects:
        session.delete(obj)
    session.commit()  # commit necessary

    relobj = objects[0].__related__(**param)
    # associate the data into the related object

    for obj in all_objects:
        relobj.__raw__.append(obj)
        if type(relobj.__many__).__name__ in ("dict"):
            # if it is a dictionary type, iterate and add
            for key in relobj.__many__.keys():
                if type(obj.__single__[key]).__name__ in ('list', 'tuple'):
                    relobj.__many__[key].extend(set(obj.__single__[key]) - set(relobj.__many__[key]))
                elif obj.__single__[key] not in relobj.__many__[key]:
                    relobj.__many__[key].append(obj.__single__[key])
        else:
            if obj.__single__ and obj.__single__ not in relobj.__many__:
                relobj.__many__.append(obj.__single__)
    session.merge(relobj)
    session.commit()

def add(obj, override=True, temp=False):
    """
    PatentGrant Object converting to tables via SQLAlchemy
    Necessary to convert dates to datetime because of SQLite (OK on MySQL)

    Case Sensitivity and Table Reflection
    MySQL has inconsistent support for case-sensitive identifier names,
    basing support on specific details of the underlying operating system.
    However, it has been observed that no matter what case sensitivity
    behavior is present, the names of tables in foreign key declarations
    are always received from the database as all-lower case, making it
    impossible to accurately reflect a schema where inter-related tables
    use mixed-case identifier names.

    Therefore it is strongly advised that table names be declared as all
    lower case both within SQLAlchemy as well as on the MySQL database
    itself, especially if database reflection features are to be used.
    """

    # if a patent exists, remove it so we can replace it
    pat_query = session.query(Patent).filter(Patent.number == obj.patent)
    if pat_query.count():
        if override:
            session.delete(pat_query.one())
        else:
            return
    if len(obj.pat["number"]) < 3:
        return

    #add
    # lots of abstracts seem to be missing. why?

    pat = Patent(**obj.pat)
    pat.application = Application(**obj.app)

    #+asg
    for asg, loc in obj.assignee_list:
        asg = RawAssignee(**asg)
        loc = RawLocation(**loc)
        session.merge(loc)
        asg.rawlocation = loc
        pat.rawassignees.append(asg)

    #+inv
    for inv, loc in obj.inventor_list:
        inv = RawInventor(**inv)
        loc = RawLocation(**loc)
        session.merge(loc)
        inv.rawlocation = loc
        pat.rawinventors.append(inv)

    #+law
    for law in obj.lawyer_list:
        law = RawLawyer(**law)
        pat.rawlawyers.append(law)

    #+usreldoc
    for usr in obj.us_relation_list:
        usr = USRelDoc(**usr)
        pat.usreldocs.append(usr)

    #+classes
    for uspc, mc, sc in obj.us_classifications:
        uspc = USPC(**uspc)
        mc = MainClass(**mc)
        sc = SubClass(**sc)
        session.merge(mc)
        session.merge(sc)
        uspc.mainclass = mc
        uspc.subclass = sc
        pat.classes.append(uspc)

    #+ipcr
    for ipc in obj.ipcr_classifications:
        ipc = IPCR(**ipc)
        pat.ipcrs.append(ipc)

    # citations are huge. this dumps them to
    # a temporary database which we can use for later
    if temp:
        cits, refs = obj.citation_list
        for cit in cits:
            cit["patent_id"] = obj.pat["number"]
            cit = TempCitation(**cit)
            session.add(cit)
        for ref in refs:
            ref["patent_id"] = obj.pat["number"]
            ref = TempOtherReference(**ref)
            session.add(ref)
    else:
        cits, refs = obj.citation_list
        for cit in cits:
            cit = Citation(**cit)
            pat.citations.append(cit)
        for ref in refs:
            ref = OtherReference(**ref)
            pat.otherreferences.append(ref)

    session.merge(pat)

def commit():
    try:
        session.commit()
    except Exception, e:
        session.rollback()
        print str(e)
<|endoftext|>


<commit_before>import pytest
import subprocess
import sys
import urllib3.connection
import socket
import psutil
import time
import json
import urllib.parse
import string
import re
import base64
from functools import partial
from hypothesis import given, strategies as st, settings, Verbosity

@pytest.fixture(autouse=True, scope='module')
def server():
    server = subprocess.Popen([sys.executable, 'integration_tests/dump.py'])
    proc = psutil.Process(server.pid)

    collector = subprocess.Popen([
        sys.executable, 'integration_tests/collector.py', str(server.pid)])

    # wait until the server socket is open
    while 1:
        if proc.connections():
            break
        time.sleep(.001)

    yield server

    server.terminate()
    server.wait()
    assert server.returncode == 0

    collector.wait()
    assert collector.returncode == 0

@pytest.fixture(params=['example', 'test'])
def connect(request):
    if request.param == 'example':
        yield partial(urllib3.connection.HTTPConnection, 'localhost:8080')
    elif request.param == 'test':
        connection = urllib3.connection.HTTPConnection('localhost:8080')
        close = connection.close
        connection.close = lambda: None
        yield lambda: connection
        close()

method_alphabet = string.digits + string.ascii_letters + string.punctuation
st_method = method=st.text(method_alphabet, min_size=1)
@given(method=st_method)
@settings(verbosity=Verbosity.verbose)
def test_method(connect, method):
    connection = connect()
    connection.request(method, '/dump/1/2')
    response = connection.getresponse()
    json_body = json.loads(response.read().decode('utf-8'))

    assert response.status == 200
    assert json_body['method'] == method

    connection.close()

param_alphabet = st.characters(blacklist_characters='/?') \
    .filter(lambda x: not any(0xD800 <= ord(c) <= 0xDFFF for c in x))
st_param = st.text(param_alphabet, min_size=1)
@given(param1=st_param, param2=st_param)
@settings(verbosity=Verbosity.verbose)
def test_match_dict(connect, param1, param2):
    connection = connect()
    connection.request('GET', urllib.parse.quote('/dump/{}/{}'.format(param1, param2)))
    response = connection.getresponse()
    json_body = json.loads(response.read().decode('utf-8'))

    assert response.status == 200
    assert json_body['match_dict'] == {'p1': param1, 'p2': param2}

    connection.close()

st_query_string = st.one_of(st.text(), st.none())
@given(query_string=st_query_string)
@settings(verbosity=Verbosity.verbose)
def test_query_string(connect, query_string):
    connection = connect()
    url = '/dump/1/2'
    if query_string is not None:
        url += '?' + urllib.parse.quote(query_string)
    connection.request('GET', url)
    response = connection.getresponse()
    json_body = json.loads(response.read().decode('utf-8'))

    assert response.status == 200
    assert json_body['query_string'] == query_string

    connection.close()

name_alphabet = string.digits + string.ascii_letters + '!#$%&\'*+-.^_`|~'
names = st.text(name_alphabet, min_size=1).map(lambda x: 'X-' + x)
value_alphabet = ''.join(chr(x) for x in range(ord(' '), 256) if x != 127)
is_illegal_value = re.compile(r'\n(?![ \t])|\r(?![ \t\n])').search
values = st.text(value_alphabet, min_size=1) \
    .filter(lambda x: not is_illegal_value(x)).map(lambda x: x.strip())
st_headers = st.dictionaries(names, values, max_size=49).filter(
    lambda x: len(x) == len(set(n.lower() for n in x)))
@given(headers=st_headers)
@settings(verbosity=Verbosity.verbose)
def test_headers(connect, headers):
    connection = connect()
    connection.putrequest(
        'GET', '/dump/1/2', skip_host=True, skip_accept_encoding=True)
    for name, value in headers.items():
        connection.putheader(name, value)
    connection.endheaders()

    response = connection.getresponse()
    json_body = json.loads(response.read().decode('utf-8'))

    assert response.status == 200
    headers = {k.title(): v for k, v in headers.items()}
    assert json_body['headers'] == headers

    connection.close()

st_body = st.one_of(st.binary(), st.none())
@given(body=st_body)
@settings(verbosity=Verbosity.verbose)
@pytest.mark.parametrize(
    'size_k', [0, 1, 2, 4, 8], ids=['small', '1k', '2k', '4k', '8k'])
def test_body(connect, size_k, body):
    if size_k and body:
        body = body * ((size_k * 1024) // len(body) + 1)

    connection = connect()
    connection.putrequest('GET', '/dump/1/2')
    if body is not None:
        connection.putheader('Content-Length', len(body))
    connection.endheaders(body)
    response = connection.getresponse()

    assert response.status == 200
    json_body = json.loads(response.read().decode('utf-8'))

    if body is not None:
        assert base64.b64decode(json_body['body']) == body
    else:
        assert json_body['body'] == body

    connection.close()

@given(body=st.lists(st.binary(min_size=24)))
@settings(verbosity=Verbosity.verbose)
@pytest.mark.parametrize(
    'size_k', [0, 1, 2, 4, 8], ids=['small', '1k', '2k', '4k', '8k'])
def test_chunked(connect, size_k, body):
    length = sum(len(b) for b in body)
    if size_k and length:
        body = body * ((size_k * 1024) // length + 1)

    connection = connect()
    connection.request_chunked('POST', '/dump/1/2', body=body)
    response = connection.getresponse()
    assert response.status == 200
    json_body = json.loads(response.read().decode('utf-8'))

    assert base64.b64decode(json_body['body']) == b''.join(body)

    connection.close()

@given(
    method=st_method,
    param1=st_param, param2=st_param,
    query_string=st_query_string,
    headers=st_headers,
    body=st_body
)
@settings(verbosity=Verbosity.verbose)
@pytest.mark.parametrize(
    'size_k', [0, 1, 2, 4, 8], ids=['small', '1k', '2k', '4k', '8k'])
def test_all(connect, size_k, method, param1, param2, query_string, headers, body):
    connection = connect()
    if size_k and body:
        body = body * ((size_k * 1024) // len(body) + 1)
    url = urllib.parse.quote('/dump/{}/{}'.format(param1, param2))
    if query_string is not None:
        url += '?' + urllib.parse.quote(query_string)
    connection.putrequest(
        method, url, skip_host=True, skip_accept_encoding=True)
    for name, value in headers.items():
        connection.putheader(name, value)
    if body is not None:
        headers['Content-Length'] = str(len(body))
        connection.putheader('Content-Length', len(body))
    connection.endheaders(body)
    response = connection.getresponse()

    assert response.status == 200
    json_body = json.loads(response.read().decode('utf-8'))
    assert json_body['method'] == method
    assert json_body['match_dict'] == {'p1': param1, 'p2': param2}
    assert json_body['query_string'] == query_string
    headers = {k.title(): v for k, v in headers.items()}
    assert json_body['headers'] == headers
    if body is not None:
        assert base64.b64decode(json_body['body']) == body
    else:
        assert json_body['body'] is None

    connection.close()
<commit_msg>send marks at each test<commit_after>import pytest
import subprocess
import sys
import urllib3.connection
import socket
import psutil
import time
import json
import urllib.parse
import string
import re
import base64
from functools import partial
from hypothesis import given, strategies as st, settings, Verbosity

@pytest.fixture(autouse=True, scope='module')
def server():
    server = subprocess.Popen([sys.executable, 'integration_tests/dump.py'])
    proc = psutil.Process(server.pid)

    collector = subprocess.Popen([
        sys.executable, 'integration_tests/collector.py', str(server.pid)])
    c_proc = psutil.Process(collector.pid)

    # wait until the server socket is open
    while 1:
        if proc.connections():
            break
        time.sleep(.001)

    # wait until the collector socket is open
    while 1:
        if c_proc.connections():
            break
        time.sleep(.001)

    yield server

    server.terminate()
    server.mark('terminate')
    server.wait()
    assert server.returncode == 0

    collector.wait()
    assert collector.returncode == 0

@pytest.fixture(scope='module')
def mark(server):
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.connect(('localhost', 8081))
    sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)

    def send(data):
        if data.startswith('test-'):
            data = data[5:]
        data += '\n'
        sock.sendall(data.encode('utf-8'))

    server.mark = send

    yield send

    sock.close()

@pytest.fixture(autouse=True)
def marker(request, mark):
    mark(request.node.name)

@pytest.fixture(params=['example', 'test'])
def connect(request):
    if request.param == 'example':
        yield partial(urllib3.connection.HTTPConnection, 'localhost:8080')
    elif request.param == 'test':
        connection = urllib3.connection.HTTPConnection('localhost:8080')
        close = connection.close
        connection.close = lambda: None
        yield lambda: connection
        close()

method_alphabet = string.digits + string.ascii_letters + string.punctuation
st_method = method=st.text(method_alphabet, min_size=1)
@given(method=st_method)
@settings(verbosity=Verbosity.verbose)
def test_method(connect, method):
    connection = connect()
    connection.request(method, '/dump/1/2')
    response = connection.getresponse()
    json_body = json.loads(response.read().decode('utf-8'))

    assert response.status == 200
    assert json_body['method'] == method

    connection.close()

param_alphabet = st.characters(blacklist_characters='/?') \
    .filter(lambda x: not any(0xD800 <= ord(c) <= 0xDFFF for c in x))
st_param = st.text(param_alphabet, min_size=1)
@given(param1=st_param, param2=st_param)
@settings(verbosity=Verbosity.verbose)
def test_match_dict(connect, param1, param2):
    connection = connect()
    connection.request('GET', urllib.parse.quote('/dump/{}/{}'.format(param1, param2)))
    response = connection.getresponse()
    json_body = json.loads(response.read().decode('utf-8'))

    assert response.status == 200
    assert json_body['match_dict'] == {'p1': param1, 'p2': param2}

    connection.close()

st_query_string = st.one_of(st.text(), st.none())
@given(query_string=st_query_string)
@settings(verbosity=Verbosity.verbose)
def test_query_string(connect, query_string):
    connection = connect()
    url = '/dump/1/2'
    if query_string is not None:
        url += '?' + urllib.parse.quote(query_string)
    connection.request('GET', url)
    response = connection.getresponse()
    json_body = json.loads(response.read().decode('utf-8'))

    assert response.status == 200
    assert json_body['query_string'] == query_string

    connection.close()

name_alphabet = string.digits + string.ascii_letters + '!#$%&\'*+-.^_`|~'
names = st.text(name_alphabet, min_size=1).map(lambda x: 'X-' + x)
value_alphabet = ''.join(chr(x) for x in range(ord(' '), 256) if x != 127)
is_illegal_value = re.compile(r'\n(?![ \t])|\r(?![ \t\n])').search
values = st.text(value_alphabet, min_size=1) \
    .filter(lambda x: not is_illegal_value(x)).map(lambda x: x.strip())
st_headers = st.dictionaries(names, values, max_size=49).filter(
    lambda x: len(x) == len(set(n.lower() for n in x)))
@given(headers=st_headers)
@settings(verbosity=Verbosity.verbose)
def test_headers(connect, headers):
    connection = connect()
    connection.putrequest(
        'GET', '/dump/1/2', skip_host=True, skip_accept_encoding=True)
    for name, value in headers.items():
        connection.putheader(name, value)
    connection.endheaders()

    response = connection.getresponse()
    json_body = json.loads(response.read().decode('utf-8'))

    assert response.status == 200
    headers = {k.title(): v for k, v in headers.items()}
    assert json_body['headers'] == headers

    connection.close()

st_body = st.one_of(st.binary(), st.none())
@given(body=st_body)
@settings(verbosity=Verbosity.verbose)
@pytest.mark.parametrize(
    'size_k', [0, 1, 2, 4, 8], ids=['small', '1k', '2k', '4k', '8k'])
def test_body(connect, size_k, body):
    if size_k and body:
        body = body * ((size_k * 1024) // len(body) + 1)

    connection = connect()
    connection.putrequest('GET', '/dump/1/2')
    if body is not None:
        connection.putheader('Content-Length', len(body))
    connection.endheaders(body)
    response = connection.getresponse()

    assert response.status == 200
    json_body = json.loads(response.read().decode('utf-8'))

    if body is not None:
        assert base64.b64decode(json_body['body']) == body
    else:
        assert json_body['body'] == body

    connection.close()

@given(body=st.lists(st.binary(min_size=24)))
@settings(verbosity=Verbosity.verbose)
@pytest.mark.parametrize(
    'size_k', [0, 1, 2, 4, 8], ids=['small', '1k', '2k', '4k', '8k'])
def test_chunked(connect, size_k, body):
    length = sum(len(b) for b in body)
    if size_k and length:
        body = body * ((size_k * 1024) // length + 1)

    connection = connect()
    connection.request_chunked('POST', '/dump/1/2', body=body)
    response = connection.getresponse()
    assert response.status == 200
    json_body = json.loads(response.read().decode('utf-8'))

    assert base64.b64decode(json_body['body']) == b''.join(body)

    connection.close()

@given(
    method=st_method,
    param1=st_param, param2=st_param,
    query_string=st_query_string,
    headers=st_headers,
    body=st_body
)
@settings(verbosity=Verbosity.verbose)
@pytest.mark.parametrize(
    'size_k', [0, 1, 2, 4, 8], ids=['small', '1k', '2k', '4k', '8k'])
def test_all(connect, size_k, method, param1, param2, query_string, headers, body):
    connection = connect()
    if size_k and body:
        body = body * ((size_k * 1024) // len(body) + 1)
    url = urllib.parse.quote('/dump/{}/{}'.format(param1, param2))
    if query_string is not None:
        url += '?' + urllib.parse.quote(query_string)
    connection.putrequest(
        method, url, skip_host=True, skip_accept_encoding=True)
    for name, value in headers.items():
        connection.putheader(name, value)
    if body is not None:
        headers['Content-Length'] = str(len(body))
        connection.putheader('Content-Length', len(body))
    connection.endheaders(body)
    response = connection.getresponse()

    assert response.status == 200
    json_body = json.loads(response.read().decode('utf-8'))
    assert json_body['method'] == method
    assert json_body['match_dict'] == {'p1': param1, 'p2': param2}
    assert json_body['query_string'] == query_string
    headers = {k.title(): v for k, v in headers.items()}
    assert json_body['headers'] == headers
    if body is not None:
        assert base64.b64decode(json_body['body']) == body
    else:
        assert json_body['body'] is None

    connection.close()
<|endoftext|>


<commit_before>
import rospy
import urx
import numpy as np
from costar_robot import CostarArm
from sensor_msgs.msg import JointState
from std_msgs.msg import Header

mode = {'TEACH':'TeachArm', 'SERVO':'MoveArmJointServo', 'SHUTDOWN':'ShutdownArm', 'IDLE':'PauseArm'}

class CostarUR5Driver(CostarArm):

    def __init__(self,ip_address,simulation=False,
            world="/world",
            listener=None,
            traj_step_t=0.1,
            max_acc=1,
            max_vel=1,
            max_goal_diff = 0.02,
            goal_rotation_weight = 0.01,
            max_q_diff = 1e-6):

        if not simulation:
            self.ur = urx.Robot(ip_address)
        self.simulation = simulation

        base_link = "base_link"
        end_link = "ee_link"
        planning_group = "manipulator"
        super(CostarUR5Driver, self).__init__(base_link,end_link,planning_group,
            steps_per_meter=0,
            base_steps=1)

        self.js_publisher = rospy.Publisher('joint_states',JointState,queue_size=1000)

        self.q0 = np.array(self.ur.getj())
        self.old_q0 = self.q0
        self.set_goal(self.q0)

        self.joint_names = [joint.name for joint in self.robot.joints[:6]]
        self.planner.joint_names = self.joint_names

    '''
    Send a whole joint trajectory message to a robot...
    that is listening to individual joint states.
    '''
    def send_trajectory(self,traj,acceleration=0.5,velocity=0.5,cartesian=False):

        rate = rospy.Rate(30)
        t = rospy.Time(0)

        stamp = rospy.Time.now().to_sec()
        self.cur_stamp = stamp

        for pt in traj.points[:-1]:
          if not cartesian:
            self.send_q(pt.positions,acceleration,velocity)
          else:
            self.send_cart(pt.positions,acceleration,velocity) ##
          self.set_goal(pt.positions)

          print " -- %s"%(str(pt.positions))
          start_t = rospy.Time.now()

          if self.cur_stamp > stamp:
            return 'FAILURE - preempted'

          rospy.sleep(rospy.Duration(pt.time_from_start.to_sec() - t.to_sec()))
          t = pt.time_from_start

        print " -- GOAL: %s"%(str(traj.points[-1].positions))
        if not cartesian:
          self.send_q(traj.points[-1].positions,acceleration,velocity)
        else:
          self.send_cart(pt.positions,acceleration,velocity) ##
        self.set_goal(traj.points[-1].positions)
        start_t = rospy.Time.now()

        # wait until robot is at goal
        #while self.moving:
        while not self.at_goal:
            if (rospy.Time.now() - start_t).to_sec() > 3:
                return 'FAILURE - timeout'
            rate.sleep()

        if self.at_goal:
            return 'SUCCESS - moved to pose'
        else:
            return 'FAILURE - did not reach destination'

    '''
    set teach mode
    '''
    def set_teach_mode_call(self,req,cartesian=False):
        if req.enable == True:

            self.ur.set_freedrive(True)
            self.driver_status = 'TEACH'
            return 'SUCCESS - teach mode enabled'
        else:
            self.ur.set_freedrive(False)
            self.driver_status = 'IDLE'
            return 'SUCCESS - teach mode disabled'

    '''
    send a single joint space position
    '''
    def send_q(self,q,acceleration,velocity):
        if self.simulation:
            pt = JointTrajectoryPoint()
            pt.positions = q

            self.pt_publisher.publish(pt)
        else:
            self.ur.movej(q,wait=False,acc=acceleration,vel=velocity)

    '''
    '''
    def send_cart(self,q,acceleration,velocity):
        if self.simulation:
            pt = JointTrajectoryPoint()
            pt.positions = q

            self.pt_publisher.publish(pt)
        else:
          T = self.kdl_kin.forward(q)
          (angle,axis) = T.M.GetRotAngle()
          cmd = list(T.p) + [angle*axis[0],angle*axis[1],angle*axis[2]]
          self.ur.movel(cmd,acc=acceleration,vel=velocity)

    '''
    Send a whole sequence of points to a robot...
    that is listening to individual joint states.
    '''
    def send_sequence(self,traj,acceleration=0.5,velocity=0.5):
        q0 = self.q0
        for q in traj:
            self.send_q(q,acceleration,velocity)
            self.set_goal(q)

            #rospy.sleep(0.9*np.sqrt(np.sum((q-q0)**2)))

        if len(traj) > 0:
            self.send_q(traj[-1],acceleration,velocity)
            self.set_goal(traj[-1])
            rate = rospy.Rate(10)
            start_t = rospy.Time.now()

            # wait until robot is at goal
            while not self.at_goal:
                if (rospy.Time.now() - start_t).to_sec() > 10:
                    return 'FAILURE - timeout'
                rate.sleep()

            return 'SUCCESS - moved to pose'

    def handle_tick(self):

        # send out the joint states
        self.q0 = np.array(self.ur.getj())
        self.js_publisher.publish(JointState(
          header=Header(stamp=rospy.Time.now()),
          name=self.joint_names,
          position=self.q0,
          velocity=[0,0,0,0,0,0],
          effort=[0,0,0,0,0,0]))
        self.update_position()

        if self.driver_status in mode.keys():

            if self.driver_status == 'SHUTDOWN':
                self.ur.cleanup()
                self.ur.shutdown()
            elif self.driver_status == 'SERVO':
                pass
            elif self.driver_status == 'IDLE':
                pass
            elif self.driver_status == 'TEACH':
                pass

        else:
            #rospy.logwarn('IIWA mode for %s not specified!'%self.driver_status)
            pass

<commit_msg>wait=False to fix jitter on cartesian moves<commit_after>
import rospy
import urx
import numpy as np
from costar_robot import CostarArm
from sensor_msgs.msg import JointState
from std_msgs.msg import Header

mode = {'TEACH':'TeachArm', 'SERVO':'MoveArmJointServo', 'SHUTDOWN':'ShutdownArm', 'IDLE':'PauseArm'}

class CostarUR5Driver(CostarArm):

    def __init__(self,ip_address,simulation=False,
            world="/world",
            listener=None,
            traj_step_t=0.1,
            max_acc=1,
            max_vel=1,
            max_goal_diff = 0.02,
            goal_rotation_weight = 0.01,
            max_q_diff = 1e-6):

        if not simulation:
            self.ur = urx.Robot(ip_address)
        self.simulation = simulation

        base_link = "base_link"
        end_link = "ee_link"
        planning_group = "manipulator"
        super(CostarUR5Driver, self).__init__(base_link,end_link,planning_group,
            steps_per_meter=0,
            base_steps=1)

        self.js_publisher = rospy.Publisher('joint_states',JointState,queue_size=1000)

        self.q0 = np.array(self.ur.getj())
        self.old_q0 = self.q0
        self.set_goal(self.q0)

        self.joint_names = [joint.name for joint in self.robot.joints[:6]]
        self.planner.joint_names = self.joint_names

    '''
    Send a whole joint trajectory message to a robot...
    that is listening to individual joint states.
    '''
    def send_trajectory(self,traj,acceleration=0.5,velocity=0.5,cartesian=False):

        rate = rospy.Rate(30)
        t = rospy.Time(0)

        stamp = rospy.Time.now().to_sec()
        self.cur_stamp = stamp

        for pt in traj.points[:-1]:
          if not cartesian:
            self.send_q(pt.positions,acceleration,velocity)
          else:
            self.send_cart(pt.positions,acceleration,velocity) ##
          self.set_goal(pt.positions)

          print " -- %s"%(str(pt.positions))
          start_t = rospy.Time.now()

          if self.cur_stamp > stamp:
            return 'FAILURE - preempted'

          rospy.sleep(rospy.Duration(pt.time_from_start.to_sec() - t.to_sec()))
          t = pt.time_from_start

        print " -- GOAL: %s"%(str(traj.points[-1].positions))
        if not cartesian:
          self.send_q(traj.points[-1].positions,acceleration,velocity)
        else:
          self.send_cart(pt.positions,acceleration,velocity) ##
        self.set_goal(traj.points[-1].positions)
        start_t = rospy.Time.now()

        # wait until robot is at goal
        #while self.moving:
        while not self.at_goal:
            if (rospy.Time.now() - start_t).to_sec() > 3:
                return 'FAILURE - timeout'
            rate.sleep()

        if self.at_goal:
            return 'SUCCESS - moved to pose'
        else:
            return 'FAILURE - did not reach destination'

    '''
    set teach mode
    '''
    def set_teach_mode_call(self,req,cartesian=False):
        if req.enable == True:

            self.ur.set_freedrive(True)
            self.driver_status = 'TEACH'
            return 'SUCCESS - teach mode enabled'
        else:
            self.ur.set_freedrive(False)
            self.driver_status = 'IDLE'
            return 'SUCCESS - teach mode disabled'

    '''
    send a single joint space position
    '''
    def send_q(self,q,acceleration,velocity):
        if self.simulation:
            pt = JointTrajectoryPoint()
            pt.positions = q

            self.pt_publisher.publish(pt)
        else:
            self.ur.movej(q,wait=False,acc=acceleration,vel=velocity)

    '''
    URX supports Cartesian moves, so we will recover our forward kinematics
    and send a Cartesian pose.
    '''
    def send_cart(self,q,acceleration,velocity):
        if self.simulation:
            pt = JointTrajectoryPoint()
            pt.positions = q

            self.pt_publisher.publish(pt)
        else:
          T = self.kdl_kin.forward(q)
          (angle,axis) = T.M.GetRotAngle()
          cmd = list(T.p) + [angle*axis[0],angle*axis[1],angle*axis[2]]
          self.ur.movel(cmd,wait=False,acc=acceleration,vel=velocity)

    '''
    Send a whole sequence of points to a robot...
    that is listening to individual joint states.
    '''
    def send_sequence(self,traj,acceleration=0.5,velocity=0.5):
        q0 = self.q0
        for q in traj:
            self.send_q(q,acceleration,velocity)
            self.set_goal(q)

            #rospy.sleep(0.9*np.sqrt(np.sum((q-q0)**2)))

        if len(traj) > 0:
            self.send_q(traj[-1],acceleration,velocity)
            self.set_goal(traj[-1])
            rate = rospy.Rate(10)
            start_t = rospy.Time.now()

            # wait until robot is at goal
            while not self.at_goal:
                if (rospy.Time.now() - start_t).to_sec() > 10:
                    return 'FAILURE - timeout'
                rate.sleep()

            return 'SUCCESS - moved to pose'

    def handle_tick(self):

        # send out the joint states
        self.q0 = np.array(self.ur.getj())
        self.js_publisher.publish(JointState(
          header=Header(stamp=rospy.Time.now()),
          name=self.joint_names,
          position=self.q0,
          velocity=[0,0,0,0,0,0],
          effort=[0,0,0,0,0,0]))
        self.update_position()

        if self.driver_status in mode.keys():

            if self.driver_status == 'SHUTDOWN':
                self.ur.cleanup()
                self.ur.shutdown()
            elif self.driver_status == 'SERVO':
                pass
            elif self.driver_status == 'IDLE':
                pass
            elif self.driver_status == 'TEACH':
                pass

        else:
            #rospy.logwarn('IIWA mode for %s not specified!'%self.driver_status)
            pass

<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

# The MIT License (MIT)

# Copyright (c) 2017-2018 CNRS

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr

import numpy as np
from pyannote.audio.keras_utils import load_model
from pyannote.audio.labeling.base import SequenceLabeling
from pyannote.audio.signal import Binarize, Peak
from pyannote.audio.embedding.extraction import SequenceEmbedding
from pyannote.audio.embedding.clustering import Clustering
from pyannote.core import Annotation
from pyannote.audio.embedding.utils import l2_normalize, cdist
from pyannote.database import get_annotated

class SpeakerDiarization(object):
    """Speaker diarization pipeline

    1. Speech activity detection
    2. Speaker change detection
    3. Speech turn clustering
    TODO. 4. Resegmentation

    Parameters
    ----------
    sad : pyannote.audio.features.Precomputed
    scd : pyannote.audio.features.Precomputed
    emb : pyannote.audio.features.Precomputed
    sad__onset, sad__offset : float, optional
        See pyannote.audio.signal.Binarize. Default to 0.7 and 0.7.
    sad__dimension : int, optional
        See pyannote.audio.signal.Binarize.apply(). Defaults to 1.
    scd__alpha, scd__min_duration : float, optional
        See pyannote.audio.signal.Peak. Default to 0.5 and 1.
    scd__dimension : int, optional
        See pyannote.audio.signal.Peak.apply(). Defaults to 1.
    cls__min_cluster_size, cls__min_samples, cls__metric : optional
        See pyannote.audio.embedding.clustering.Clustering
    long_first : boolean, optional
        Start by clustering longer speech turns, then associate each smaller
        speech turns to the closest cluster. Defaults to cluster all speech
        turns at once.
    """

    def __init__(self, sad, scd, emb,
                 sad__onset=0.7, sad__offset=0.7, sad__dimension=1,
                 scd__alpha=0.5, scd__min_duration=1., scd__dimension=1,
                 cls__min_cluster_size=5, cls__min_samples=None,
                 cls__metric='cosine', long_first=False):

        super(SpeakerDiarization, self).__init__()

        # speech activity detection hyper-parameters
        self.sad = sad
        self.sad__onset = sad__onset
        self.sad__offset = sad__offset
        self.sad__dimension = sad__dimension

        # speaker change detection hyper-parameters
        self.scd = scd
        self.scd__alpha = scd__alpha
        self.scd__min_duration = scd__min_duration
        self.scd__dimension = scd__dimension

        # embedding hyper-parameters
        self.emb = emb

        # clustering hyper-parameters
        self.cls__min_cluster_size = cls__min_cluster_size
        self.cls__min_samples = cls__min_samples
        self.cls__metric = cls__metric
        self.long_first = long_first

        # initialize speech activity detection module
        self.sad_binarize_ = Binarize(onset=self.sad__onset,
                                      offset=self.sad__offset)

        # initialize speaker change detection module
        self.scd_peak_ = Peak(alpha=self.scd__alpha,
                              min_duration=self.scd__min_duration,
                              percentile=False)

        # initialize clustering module
        self.cls_ = Clustering(metric=self.cls__metric,
                               min_cluster_size=self.cls__min_cluster_size,
                               min_samples=self.cls__min_samples)

    def __call__(self, current_file, annotated=False):
        """Process current file

        Parameters
        ----------
        current_file : dict
        annotated : boolean, optional
            Only process annotated region.

        Return
        ------
        result : Annotation
            Speaker diarization result.
        """

        # speech activity detection
        soft_sad = self.sad(current_file)
        hard_sad = self.sad_binarize_.apply(
            soft_sad, dimension=self.sad__dimension)

        # speaker change detection
        soft_scd = self.scd(current_file)
        hard_scd = self.scd_peak_.apply(
            soft_scd, dimension=self.scd__dimension)

        # speech turns
        speech_turns = hard_scd.crop(hard_sad)

        if annotated:
            speech_turns = speech_turns.crop(
                get_annotated(current_file))

        hypothesis = Annotation(uri=current_file['uri'])
        if not speech_turns:
            return hypothesis

        # speech turns embedding
        emb = self.emb(current_file)

        # long speech turns = those with at least one embedding fully included
        # shrt speech turns = all the other speech turns
        long_speech_turns, long_fX = [], []
        shrt_speech_turns, shrt_fX = [], []

        # when long_first == False (i.e. mode == 'center'), there can only be
        # long speech turns (according to the above definition)
        mode = 'strict' if self.long_first else 'center'
        for speech_turn in speech_turns:
            fX = emb.crop(speech_turn, mode=mode)
            if len(fX):
                long_speech_turns.append(speech_turn)
                long_fX.append(np.sum(fX, axis=0))
            else:
                shrt_speech_turns.append(speech_turn)
                fX = emb.crop(speech_turn, mode='center')
                shrt_fX.append(np.sum(fX, axis=0))

        # stack long (and short) speech turns embedding into numpy arrays
        long_fX = np.vstack(long_fX)
        if shrt_speech_turns:
            shrt_fX = np.vstack(shrt_fX)

        # cluster long speech turns based on their normalized embeddings
        long_clusters = self.cls_.apply(l2_normalize(long_fX))

        # get a unique embedding for each cluster by summing the embedding of
        # all the included speech turns
        cluster_fX = np.vstack(
            [np.sum(long_fX[long_clusters == i], axis=0)
             for i in np.unique(long_clusters)])

        # associate each short speech turn to the closest cluster
        shrt_clusters = []
        if shrt_speech_turns:
            distances = cdist(l2_normalize(cluster_fX),
                              l2_normalize(shrt_fX),
                              metric=self.cls__metric)
            shrt_clusters = np.argmin(distances, axis=0)

        # build hypothesis from clustering results
        speech_turns = long_speech_turns + shrt_speech_turns
        clusters = list(long_clusters) + list(shrt_clusters)
        for speech_turn, cluster in zip(speech_turns, clusters):
            hypothesis[speech_turn] = cluster
        return hypothesis
<commit_msg>feat: add "normalize" option to SpeakerDiarization pipeline<commit_after>#!/usr/bin/env python
# encoding: utf-8

# The MIT License (MIT)

# Copyright (c) 2017-2018 CNRS

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr

import numpy as np
from pyannote.audio.keras_utils import load_model
from pyannote.audio.labeling.base import SequenceLabeling
from pyannote.audio.signal import Binarize, Peak
from pyannote.audio.embedding.extraction import SequenceEmbedding
from pyannote.audio.embedding.clustering import Clustering
from pyannote.core import Annotation
from pyannote.audio.embedding.utils import l2_normalize, cdist
from pyannote.database import get_annotated

class SpeakerDiarization(object):
    """Speaker diarization pipeline

    1. Speech activity detection
    2. Speaker change detection
    3. Speech turn clustering
    TODO. 4. Resegmentation

    Parameters
    ----------
    sad : pyannote.audio.features.Precomputed
    scd : pyannote.audio.features.Precomputed
    emb : pyannote.audio.features.Precomputed
    sad__onset, sad__offset : float, optional
        See pyannote.audio.signal.Binarize. Default to 0.7 and 0.7.
    sad__dimension : int, optional
        See pyannote.audio.signal.Binarize.apply(). Defaults to 1.
    scd__alpha, scd__min_duration : float, optional
        See pyannote.audio.signal.Peak. Default to 0.5 and 1.
    scd__dimension : int, optional
        See pyannote.audio.signal.Peak.apply(). Defaults to 1.
    cls__min_cluster_size, cls__min_samples, cls__metric : optional
        See pyannote.audio.embedding.clustering.Clustering
    normalize : boolean, optional
        Normalize embeddings before clustering. Defaults to not normalize.
    long_first : boolean, optional
        Start by clustering longer speech turns, then associate each smaller
        speech turns to the closest cluster. Defaults to cluster all speech
        turns at once.
    """

    def __init__(self, sad, scd, emb,
                 sad__onset=0.7, sad__offset=0.7, sad__dimension=1,
                 scd__alpha=0.5, scd__min_duration=1., scd__dimension=1,
                 cls__min_cluster_size=5, cls__min_samples=None,
                 cls__metric='cosine', long_first=False, normalize=False):

        super(SpeakerDiarization, self).__init__()

        # speech activity detection hyper-parameters
        self.sad = sad
        self.sad__onset = sad__onset
        self.sad__offset = sad__offset
        self.sad__dimension = sad__dimension

        # speaker change detection hyper-parameters
        self.scd = scd
        self.scd__alpha = scd__alpha
        self.scd__min_duration = scd__min_duration
        self.scd__dimension = scd__dimension

        # embedding hyper-parameters
        self.emb = emb

        # clustering hyper-parameters
        self.cls__min_cluster_size = cls__min_cluster_size
        self.cls__min_samples = cls__min_samples
        self.cls__metric = cls__metric
        self.long_first = long_first
        self.normalize = normalize

        # initialize speech activity detection module
        self.sad_binarize_ = Binarize(onset=self.sad__onset,
                                      offset=self.sad__offset)

        # initialize speaker change detection module
        self.scd_peak_ = Peak(alpha=self.scd__alpha,
                              min_duration=self.scd__min_duration,
                              percentile=False)

        # initialize clustering module
        self.cls_ = Clustering(metric=self.cls__metric,
                               min_cluster_size=self.cls__min_cluster_size,
                               min_samples=self.cls__min_samples)

    def __call__(self, current_file, annotated=False):
        """Process current file

        Parameters
        ----------
        current_file : dict
        annotated : boolean, optional
            Only process annotated region.

        Return
        ------
        result : Annotation
            Speaker diarization result.
        """

        # speech activity detection
        soft_sad = self.sad(current_file)
        hard_sad = self.sad_binarize_.apply(
            soft_sad, dimension=self.sad__dimension)

        # speaker change detection
        soft_scd = self.scd(current_file)
        hard_scd = self.scd_peak_.apply(
            soft_scd, dimension=self.scd__dimension)

        # speech turns
        speech_turns = hard_scd.crop(hard_sad)

        if annotated:
            speech_turns = speech_turns.crop(
                get_annotated(current_file))

        hypothesis = Annotation(uri=current_file['uri'])
        if not speech_turns:
            return hypothesis

        # speech turns embedding
        emb = self.emb(current_file)

        # normalize speech turns embeddings
        if self.normalize:
            emb.data = l2_normalize(emb.data)

        # long speech turns = those with at least one embedding fully included
        # shrt speech turns = all the other speech turns
        long_speech_turns, long_fX = [], []
        shrt_speech_turns, shrt_fX = [], []

        # when long_first == False (i.e. mode == 'center'), there can only be
        # long speech turns (according to the above definition)
        mode = 'strict' if self.long_first else 'center'
        for speech_turn in speech_turns:
            fX = emb.crop(speech_turn, mode=mode)
            if len(fX):
                long_speech_turns.append(speech_turn)
                long_fX.append(np.sum(fX, axis=0))
            else:
                shrt_speech_turns.append(speech_turn)
                fX = emb.crop(speech_turn, mode='center')
                shrt_fX.append(np.sum(fX, axis=0))

        # stack long (and short) speech turns embedding into numpy arrays
        long_fX = np.vstack(long_fX)
        if shrt_speech_turns:
            shrt_fX = np.vstack(shrt_fX)

        # cluster long speech turns based on their normalized embeddings
        # TODO: make sure it makes sense for embeddings trained with the
        # euclidean distance and not normalized
        long_clusters = self.cls_.apply(l2_normalize(long_fX))

        # get a unique embedding for each cluster by summing the embedding of
        # all the included speech turns
        cluster_fX = np.vstack(
            [np.sum(long_fX[long_clusters == i], axis=0)
             for i in np.unique(long_clusters)])

        # associate each short speech turn to the closest cluster
        shrt_clusters = []
        if shrt_speech_turns:
            distances = cdist(l2_normalize(cluster_fX),
                              l2_normalize(shrt_fX),
                              metric=self.cls__metric)
            shrt_clusters = np.argmin(distances, axis=0)

        # build hypothesis from clustering results
        speech_turns = long_speech_turns + shrt_speech_turns
        clusters = list(long_clusters) + list(shrt_clusters)
        for speech_turn, cluster in zip(speech_turns, clusters):
            hypothesis[speech_turn] = cluster

        return hypothesis
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import pyes

#import IPy
import re
import datetime
import time
import json
import time
import string
import dateutil

class WhoisQuery():

    def __init__(self,  es, es_index, es_type):
        self.es = pyes.ES(es)
        self.index = es_index
        self.es_type = es_type
    def checkindex(self):
        if len(self.es.status(indices=[self.index])["indices"]) == 0:
            try:
                self.es.open_index(self.index)
                time.sleep(20)
            except Exception, e:
                pass

    def validate(self, s):
        return filter(lambda x: x in string.ascii_letters +string.digits+'-'+':'+'.'+'*'+ '+' + '[' + ']' + '|' + '@' + '_', s)

    def list_to_str(self,l):
        if len(l) == 0:
            return ""
        return reduce(lambda x, y: "%s\r\n%s"%(x,y), map(self.printable_entry,  l))

    def printable_entry(self,  val):
       v = val['_source']

       return "%s\r\n%s"%(reduce(lambda x,y: "%s, %s"% (x,y), v.keys()), reduce (lambda x, y: "%s, %s" % (x,y), v.values()))
       #return "%s,"% (json.dumps(v))

    def whois_query(self, query):
        start_num = 0
        size_num = 100 # defaults
        query = self.validate(query)
        queries = []
        day = '%.4i%.2i%.2i%.2i'% (datetime.datetime.now().year, datetime.datetime.now().month, datetime.datetime.now().day, datetime.datetime.now().hour)
        if query.find('@') != -1:
            day = query.split('@', 1)
            query = day[1]
            day = day[0]
        qqq = query.split('+')
        for quer in qqq:
            qq = quer.split(':', 1)
            field = '_all'
            if len(qq) > 1:
                field = qq[0]
                query = qq[1]
		q = None
                if field == "count":
                    size_num = int(query)
                    continue
                elif field == "start":
                    start_num = int(query)
                    continue
		elif query.find('[') != -1:
			from_d = query[1:query.find('|')]
			to_d = query[query.find('|') + 1:query.find(']')]
			print from_d
                        print to_d
                        try:
                            dt_from_d = parser.parse(from_d)
                            dt_to_d = parser.parse(to_d)
                            from_d = dt_from_d.strftime("%Y-%m-%dT%H:%M:%S.000Z")
                            to_d = dt_to_d.strftime("%Y-%m-%dT%H:%M:%S.000Z")
                            print from_d
                            print to_d
                        except:
                            pass
			q = pyes.query.RangeQuery(qrange=pyes.query.ESRange(field, to_value=to_d,  from_value=from_d))
		else:
			q = pyes.query.TextQuery(field, query)
                queries.append(q)
        to_return = ""
        try:
            bq = pyes.query.BoolQuery(must=queries)
            rez = self.es.search(pyes.query.Search(query=bq, start=start_num, size=size_num), indices="%s-%s"% (self.index, day), doc_types=self.es_type)
            to_return = "Result, Total %s matches\r\n%s" % (rez.count(), self.list_to_str(rez._results['hits']['hits']))
        except Exception, e:
            to_return = "Result, Error: %s\r\n" % str(e)

        return to_return

if __name__ == "__main__":
    import os
    import sys
    import ConfigParser
    config = ConfigParser.RawConfigParser()
    config.read("../../etc/whois-server.conf")
    query_maker = WhoisQuery('localhost:9200', 'searchindex', 'searchindex-type')

    def usage():
        print "arin_query.py query"
        exit(1)

    if len(sys.argv) < 2:
        usage()

    query = sys.argv[1]
    print(query_maker.whois_query(query))
<commit_msg>handle dateutil date<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import pyes

#import IPy
import re
import datetime
import time
import json
import time
import string
from dateutil import parser

class WhoisQuery():

    def __init__(self,  es, es_index, es_type):
        self.es = pyes.ES(es)
        self.index = es_index
        self.es_type = es_type
    def checkindex(self):
        if len(self.es.status(indices=[self.index])["indices"]) == 0:
            try:
                self.es.open_index(self.index)
                time.sleep(20)
            except Exception, e:
                pass

    def validate(self, s):
        return filter(lambda x: x in string.ascii_letters +string.digits+'-'+':'+'.'+'*'+ '+' + '[' + ']' + '|' + '@' + '_', s)

    def list_to_str(self,l):
        if len(l) == 0:
            return ""
        return reduce(lambda x, y: "%s\r\n%s"%(x,y), map(self.printable_entry,  l))

    def printable_entry(self,  val):
       v = val['_source']

       return "%s\r\n%s"%(reduce(lambda x,y: "%s, %s"% (x,y), v.keys()), reduce (lambda x, y: "%s, %s" % (x,y), v.values()))
       #return "%s,"% (json.dumps(v))

    def whois_query(self, query):
        start_num = 0
        size_num = 100 # defaults
        query = self.validate(query)
        queries = []
        day = '%.4i%.2i%.2i%.2i'% (datetime.datetime.now().year, datetime.datetime.now().month, datetime.datetime.now().day, datetime.datetime.now().hour)
        if query.find('@') != -1:
            day = query.split('@', 1)
            query = day[1]
            day = day[0]
        qqq = query.split('+')
        for quer in qqq:
            qq = quer.split(':', 1)
            field = '_all'
            if len(qq) > 1:
                field = qq[0]
                query = qq[1]
		q = None
                if field == "count":
                    size_num = int(query)
                    continue
                elif field == "start":
                    start_num = int(query)
                    continue
		elif query.find('[') != -1:
			from_d = query[1:query.find('|')]
			to_d = query[query.find('|') + 1:query.find(']')]
			print from_d
                        print to_d
                        try:
                            dt_from_d = parser.parse(from_d)
                            dt_to_d = parser.parse(to_d)
                            from_d = dt_from_d.strftime("%Y-%m-%dT%H:%M:%S.000Z")
                            to_d = dt_to_d.strftime("%Y-%m-%dT%H:%M:%S.000Z")
                            print from_d
                            print to_d
                        except:
                            pass
			q = pyes.query.RangeQuery(qrange=pyes.query.ESRange(field, to_value=to_d,  from_value=from_d))
		else:
			q = pyes.query.TextQuery(field, query)
                queries.append(q)
        to_return = ""
        try:
            bq = pyes.query.BoolQuery(must=queries)
            rez = self.es.search(pyes.query.Search(query=bq, start=start_num, size=size_num), indices="%s-%s"% (self.index, day), doc_types=self.es_type)
            to_return = "Result, Total %s matches\r\n%s" % (rez.count(), self.list_to_str(rez._results['hits']['hits']))
        except Exception, e:
            to_return = "Result, Error: %s\r\n" % str(e)

        return to_return

if __name__ == "__main__":
    import os
    import sys
    import ConfigParser
    config = ConfigParser.RawConfigParser()
    config.read("../../etc/whois-server.conf")
    query_maker = WhoisQuery('localhost:9200', 'searchindex', 'searchindex-type')

    def usage():
        print "arin_query.py query"
        exit(1)

    if len(sys.argv) < 2:
        usage()

    query = sys.argv[1]
    print(query_maker.whois_query(query))
<|endoftext|>


<commit_before># ===============================================================================
# Copyright 2012 Jake Ross
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ===============================================================================

# ============= enthought library imports =======================
from traits.api import Any, Event, Property, Bool
from pyface.qt.QtCore import QThread
# from traitsui.api import View, Item, spring, ButtonEditor, HGroup
# ============= standard library imports ========================
from numpy import linspace
# ============= local library imports  ==========================
from pychron.loggable import Loggable
from threading import Thread

# from pychron.spectrometer.spectrometer import Spectrometer

class SpectrometerTask(Loggable):
    spectrometer = Any
    execute_button = Event
    execute_label = Property(depends_on='_alive')
    _alive = Bool
    execution_thread = None
    graph = Any

    def _get_execute_label(self):
        return 'Stop' if self.isAlive() else 'Start'

    def isAlive(self):
        return self._alive

    def stop(self):
        self._alive = False
        self.execution_thread = None

    def _execute_button_fired(self):
        if self.isAlive():
            self.stop()
            self._end()
        else:
            self.execute()

    def execute(self):
        self.debug('execute', self.__class__.__name__)
        self._alive = True
        t = QThread(name=self.__class__.__name__, target=self._execute)
        t.start()
        self.execution_thread = t
        self.debug('execution thread. {}'.format(t))
        return t

    def _execute(self):
        pass

    def _end(self):
        pass

    def _graph_factory(self):
        pass

    def _graph_default(self):
        return self._graph_factory()

    def _calc_step_values(self, start, end, width):
        sign = 1 if start < end else -1
        nsteps = abs(end - start + width * sign) / width
        self.debug('calculated step values: start={}, end={}, width={}, nsteps={}'.format(start, end, width, nsteps))
        return linspace(start, end, nsteps)

# ============= EOF =============================================
<commit_msg>fixed issue #1178<commit_after># ===============================================================================
# Copyright 2012 Jake Ross
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ===============================================================================

# from traitsui.api import View, Item, spring, ButtonEditor, HGroup
# ============= standard library imports ========================
from numpy import linspace
from pyface.qt.QtCore import QThread
# ============= enthought library imports =======================
from traits.api import Any, Event, Property, Bool

# ============= local library imports  ==========================
from pychron.loggable import Loggable

# from pychron.spectrometer.spectrometer import Spectrometer

class SpectrometerTask(Loggable):
    spectrometer = Any
    execute_button = Event
    execute_label = Property(depends_on='_alive')
    _alive = Bool
    execution_thread = None
    graph = Any

    def _get_execute_label(self):
        return 'Stop' if self.isAlive() else 'Start'

    def isAlive(self):
        return self._alive

    def stop(self):
        self._alive = False
        self.execution_thread = None

    def _execute_button_fired(self):
        if self.isAlive():
            self.stop()
            self._end()
        else:
            self.execute()

    def execute(self):
        self.debug('execute {}'.format(self.__class__.__name__))
        self._alive = True
        t = QThread(name=self.__class__.__name__, target=self._execute)
        t.start()
        self.execution_thread = t
        self.debug('execution thread. {}'.format(t))
        return t

    def _execute(self):
        pass

    def _end(self):
        pass

    def _graph_factory(self):
        pass

    def _graph_default(self):
        return self._graph_factory()

    def _calc_step_values(self, start, end, width):
        sign = 1 if start < end else -1
        nsteps = abs(end - start + width * sign) / width
        self.debug('calculated step values: start={}, end={}, width={}, nsteps={}'.format(start, end, width, nsteps))
        return linspace(start, end, nsteps)

# ============= EOF =============================================
<|endoftext|>


<commit_before>__version__ = "1.0.0rc1"
<commit_msg>Back version down.<commit_after>__version__ = "0.5.0"
<|endoftext|>