<commit_before>from datetime import datetime

import pytest
from freezegun import freeze_time
from flask import current_app

from app.models import (
    Job,
    Notification,
    NOTIFICATION_CREATED,
    NOTIFICATION_DELIVERED,
    NOTIFICATION_SENDING,
    NOTIFICATION_TECHNICAL_FAILURE
)
from app.celery.tasks import (
    process_updates_from_file,
    update_dvla_job_to_error,
    update_job_to_sent_to_dvla,
    update_letter_notifications_statuses,
    update_letter_notifications_to_error,
    update_letter_notifications_to_sent_to_dvla
)

from tests.app.db import create_notification
from tests.conftest import set_config

def test_update_job_to_sent_to_dvla(sample_letter_template, sample_letter_job):
    create_notification(template=sample_letter_template, job=sample_letter_job)
    create_notification(template=sample_letter_template, job=sample_letter_job)
    update_job_to_sent_to_dvla(job_id=sample_letter_job.id)

    updated_notifications = Notification.query.all()
    assert [(n.status == 'sending', n.sent_by == 'dvla') for n in updated_notifications]

    assert Job.query.filter_by(id=sample_letter_job.id).one().job_status == 'sent to dvla'

def test_update_dvla_job_to_error(sample_letter_template, sample_letter_job):
    create_notification(template=sample_letter_template, job=sample_letter_job)
    create_notification(template=sample_letter_template, job=sample_letter_job)
    update_dvla_job_to_error(job_id=sample_letter_job.id)

    updated_notifications = Notification.query.all()
    for n in updated_notifications:
        assert n.status == 'created'
        assert not n.sent_by

    assert Job.query.filter_by(id=sample_letter_job.id).one().job_status == 'error'

def test_update_letter_notifications_statuses_raises_for_invalid_format(notify_api, mocker):
    invalid_file = 'ref-foo|Sent|1|Unsorted\nref-bar|Sent|2'
    mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=invalid_file)

    with pytest.raises(TypeError):
        update_letter_notifications_statuses(filename='foo.txt')

def test_update_letter_notifications_statuses_calls_with_correct_bucket_location(notify_api, mocker):
    s3_mock = mocker.patch('app.celery.tasks.s3.get_s3_object')

    with set_config(notify_api, 'NOTIFY_EMAIL_DOMAIN', 'foo.bar'):
        update_letter_notifications_statuses(filename='foo.txt')
        s3_mock.assert_called_with('{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN']), 'foo.txt')

def test_update_letter_notifications_statuses_builds_updates_from_content(notify_api, mocker):
    valid_file = 'ref-foo|Sent|1|Unsorted\nref-bar|Sent|2|Sorted'
    mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
    update_mock = mocker.patch('app.celery.tasks.process_updates_from_file')

    update_letter_notifications_statuses(filename='foo.txt')

    update_mock.assert_called_with('ref-foo|Sent|1|Unsorted\nref-bar|Sent|2|Sorted')

def test_update_letter_notifications_statuses_builds_updates_list(notify_api, mocker):
    valid_file = 'ref-foo|Sent|1|Unsorted\nref-bar|Sent|2|Sorted'
    updates = process_updates_from_file(valid_file)

    assert len(updates) == 2

    assert updates[0].reference == 'ref-foo'
    assert updates[0].status == 'Sent'
    assert updates[0].page_count == '1'
    assert updates[0].cost_threshold == 'Unsorted'

    assert updates[1].reference == 'ref-bar'
    assert updates[1].status == 'Sent'
    assert updates[1].page_count == '2'
    assert updates[1].cost_threshold == 'Sorted'

def test_update_letter_notifications_statuses_persisted(notify_api, mocker, sample_letter_template):
    sent_letter = create_notification(sample_letter_template, reference='ref-foo', status=NOTIFICATION_SENDING,
                                      billable_units=0)
    failed_letter = create_notification(sample_letter_template, reference='ref-bar', status=NOTIFICATION_SENDING,
                                        billable_units=0)

    valid_file = '{}|Sent|1|Unsorted\n{}|Failed|2|Sorted'.format(
        sent_letter.reference, failed_letter.reference)
    mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)

    update_letter_notifications_statuses(filename='foo.txt')

    assert sent_letter.status == NOTIFICATION_DELIVERED
    assert sent_letter.billable_units == 1
    assert sent_letter.updated_at
    assert failed_letter.status == NOTIFICATION_TECHNICAL_FAILURE
    assert failed_letter.billable_units == 2
    assert failed_letter.updated

def test_update_letter_notifications_to_sent_to_dvla_updates_based_on_notification_references(
    client,
    sample_letter_template
):
    first = create_notification(sample_letter_template, reference='first ref')
    second = create_notification(sample_letter_template, reference='second ref')

    dt = datetime.utcnow()
    with freeze_time(dt):
        update_letter_notifications_to_sent_to_dvla([first.reference])

    assert first.status == NOTIFICATION_SENDING
    assert first.sent_by == 'dvla'
    assert first.sent_at == dt
    assert first.updated_at == dt
    assert second.status == NOTIFICATION_CREATED

def test_update_letter_notifications_to_error_updates_based_on_notification_references(
    client,
    sample_letter_template
):
    first = create_notification(sample_letter_template, reference='first ref')
    second = create_notification(sample_letter_template, reference='second ref')

    dt = datetime.utcnow()
    with freeze_time(dt):
        update_letter_notifications_to_error([first.reference])

    assert first.status == NOTIFICATION_TECHNICAL_FAILURE
    assert first.sent_by is None
    assert first.sent_at is None
    assert first.updated_at == dt
    assert second.status == NOTIFICATION_CREATED
<commit_msg>Fix typo<commit_after>from datetime import datetime

import pytest
from freezegun import freeze_time
from flask import current_app

from app.models import (
    Job,
    Notification,
    NOTIFICATION_CREATED,
    NOTIFICATION_DELIVERED,
    NOTIFICATION_SENDING,
    NOTIFICATION_TECHNICAL_FAILURE
)
from app.celery.tasks import (
    process_updates_from_file,
    update_dvla_job_to_error,
    update_job_to_sent_to_dvla,
    update_letter_notifications_statuses,
    update_letter_notifications_to_error,
    update_letter_notifications_to_sent_to_dvla
)

from tests.app.db import create_notification
from tests.conftest import set_config

def test_update_job_to_sent_to_dvla(sample_letter_template, sample_letter_job):
    create_notification(template=sample_letter_template, job=sample_letter_job)
    create_notification(template=sample_letter_template, job=sample_letter_job)
    update_job_to_sent_to_dvla(job_id=sample_letter_job.id)

    updated_notifications = Notification.query.all()
    assert [(n.status == 'sending', n.sent_by == 'dvla') for n in updated_notifications]

    assert Job.query.filter_by(id=sample_letter_job.id).one().job_status == 'sent to dvla'

def test_update_dvla_job_to_error(sample_letter_template, sample_letter_job):
    create_notification(template=sample_letter_template, job=sample_letter_job)
    create_notification(template=sample_letter_template, job=sample_letter_job)
    update_dvla_job_to_error(job_id=sample_letter_job.id)

    updated_notifications = Notification.query.all()
    for n in updated_notifications:
        assert n.status == 'created'
        assert not n.sent_by

    assert Job.query.filter_by(id=sample_letter_job.id).one().job_status == 'error'

def test_update_letter_notifications_statuses_raises_for_invalid_format(notify_api, mocker):
    invalid_file = 'ref-foo|Sent|1|Unsorted\nref-bar|Sent|2'
    mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=invalid_file)

    with pytest.raises(TypeError):
        update_letter_notifications_statuses(filename='foo.txt')

def test_update_letter_notifications_statuses_calls_with_correct_bucket_location(notify_api, mocker):
    s3_mock = mocker.patch('app.celery.tasks.s3.get_s3_object')

    with set_config(notify_api, 'NOTIFY_EMAIL_DOMAIN', 'foo.bar'):
        update_letter_notifications_statuses(filename='foo.txt')
        s3_mock.assert_called_with('{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN']), 'foo.txt')

def test_update_letter_notifications_statuses_builds_updates_from_content(notify_api, mocker):
    valid_file = 'ref-foo|Sent|1|Unsorted\nref-bar|Sent|2|Sorted'
    mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
    update_mock = mocker.patch('app.celery.tasks.process_updates_from_file')

    update_letter_notifications_statuses(filename='foo.txt')

    update_mock.assert_called_with('ref-foo|Sent|1|Unsorted\nref-bar|Sent|2|Sorted')

def test_update_letter_notifications_statuses_builds_updates_list(notify_api, mocker):
    valid_file = 'ref-foo|Sent|1|Unsorted\nref-bar|Sent|2|Sorted'
    updates = process_updates_from_file(valid_file)

    assert len(updates) == 2

    assert updates[0].reference == 'ref-foo'
    assert updates[0].status == 'Sent'
    assert updates[0].page_count == '1'
    assert updates[0].cost_threshold == 'Unsorted'

    assert updates[1].reference == 'ref-bar'
    assert updates[1].status == 'Sent'
    assert updates[1].page_count == '2'
    assert updates[1].cost_threshold == 'Sorted'

def test_update_letter_notifications_statuses_persisted(notify_api, mocker, sample_letter_template):
    sent_letter = create_notification(sample_letter_template, reference='ref-foo', status=NOTIFICATION_SENDING,
                                      billable_units=0)
    failed_letter = create_notification(sample_letter_template, reference='ref-bar', status=NOTIFICATION_SENDING,
                                        billable_units=0)

    valid_file = '{}|Sent|1|Unsorted\n{}|Failed|2|Sorted'.format(
        sent_letter.reference, failed_letter.reference)
    mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)

    update_letter_notifications_statuses(filename='foo.txt')

    assert sent_letter.status == NOTIFICATION_DELIVERED
    assert sent_letter.billable_units == 1
    assert sent_letter.updated_at
    assert failed_letter.status == NOTIFICATION_TECHNICAL_FAILURE
    assert failed_letter.billable_units == 2
    assert failed_letter.updated_at

def test_update_letter_notifications_to_sent_to_dvla_updates_based_on_notification_references(
    client,
    sample_letter_template
):
    first = create_notification(sample_letter_template, reference='first ref')
    second = create_notification(sample_letter_template, reference='second ref')

    dt = datetime.utcnow()
    with freeze_time(dt):
        update_letter_notifications_to_sent_to_dvla([first.reference])

    assert first.status == NOTIFICATION_SENDING
    assert first.sent_by == 'dvla'
    assert first.sent_at == dt
    assert first.updated_at == dt
    assert second.status == NOTIFICATION_CREATED

def test_update_letter_notifications_to_error_updates_based_on_notification_references(
    client,
    sample_letter_template
):
    first = create_notification(sample_letter_template, reference='first ref')
    second = create_notification(sample_letter_template, reference='second ref')

    dt = datetime.utcnow()
    with freeze_time(dt):
        update_letter_notifications_to_error([first.reference])

    assert first.status == NOTIFICATION_TECHNICAL_FAILURE
    assert first.sent_by is None
    assert first.sent_at is None
    assert first.updated_at == dt
    assert second.status == NOTIFICATION_CREATED
<|endoftext|>


<commit_before>#!/usr/bin/env python2.5
# ex: set expandtab softtabstop=4 shiftwidth=4: -*- cpy-indent-level: 4; indent-tabs-mode: nil -*-
#
# Copyright (C) 2008,2009  Contributor
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the EU DataGrid Software License.  You should
# have received a copy of the license with this program, and the
# license is published at
# http://eu-datagrid.web.cern.ch/eu-datagrid/license.html.
#
# THE FOLLOWING DISCLAIMER APPLIES TO ALL SOFTWARE CODE AND OTHER
# MATERIALS CONTRIBUTED IN CONNECTION WITH THIS PROGRAM.
#
# THIS SOFTWARE IS LICENSED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE AND ANY WARRANTY OF NON-INFRINGEMENT, ARE
# DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. THIS
# SOFTWARE MAY BE REDISTRIBUTED TO OTHERS ONLY BY EFFECTIVELY USING
# THIS OR ANOTHER EQUIVALENT DISCLAIMER AS WELL AS ANY OTHER LICENSE
# TERMS THAT MAY APPLY.
"""Module for testing commands that add virtual hardware."""

import os
import sys
import unittest

if __name__ == "__main__":
    BINDIR = os.path.dirname(os.path.realpath(sys.argv[0]))
    SRCDIR = os.path.join(BINDIR, "..", "..")
    sys.path.append(os.path.join(SRCDIR, "lib", "python2.5"))

from brokertest import TestBrokerCommand

class TestAddVirtualHardware(TestBrokerCommand):

    def test_000_addmachines(self):
        for i in range(1, 10):
            self.noouttest(["add", "machine", "--machine", "evm%s" % i,
                            "--cluster", "utecl1", "--cluster_type", "esx",
                            "--model", "utmedium"])

    def test_100_addinterfaces(self):
        for i in range(1, 8):
            self.noouttest(["add", "interface", "--machine", "evm%s" % i,
                            "--interface", "eth0", "--automac"])

    def test_110_addinterfaces(self):
        self.noouttest(["add", "interface", "--machine", "evm9",
                        "--interface", "eth0", "--mac", "00:50:56:3f:ff:ff"])

    def test_120_addinterfaces(self):
        # This should now fill in the 'hole' between 7 and 9
        self.noouttest(["add", "interface", "--machine", "evm8",
                        "--interface", "eth0", "--automac"])

    def test_500_verifyaddmachines(self):
        # Skipping evm9 since the mac is out of sequence
        for i in range(1, 9):
            command = "show machine --machine evm%s" % i
            out = self.commandtest(command.split(" "))
            self.matchoutput(out, "Virtual_machine: evm%s" % i, command)
            self.matchoutput(out, "Hosted by esx cluster: utecl1", command)
            self.matchoutput(out, "Building: ut", command)
            self.matchoutput(out, "Vendor: utvendor Model: utmedium", command)
            self.matchoutput(out, "Cpu: Cpu xeon_2500 x 1", command)
            self.matchoutput(out, "Memory: 8192 MB", command)
            self.matchoutput(out,
                             "Interface: eth0 00:50:56:01:00:%02x boot=True" %
                             (i - 1),
                             command)

    def test_500_verifycatmachines(self):
        # Skipping evm9 since the mac is out of sequence
        for i in range(1, 9):
            command = "cat --machine evm%s" % i
            out = self.commandtest(command.split(" "))
            self.matchoutput(out, """"location" = "ut.ny.na";""", command)
            self.matchoutput(out,
                             """include { """
                             """'hardware/machine/utvendor/utmedium' };""",
                             command)
            self.matchoutput(out,
                             """"ram" = list(create("hardware/ram/generic", """
                             """"size", 8192*MB));""",
                             command)
            self.matchoutput(out,
                             """"cpu" = list(create("""
                             """"hardware/cpu/intel/xeon_2500"));""",
                             command)
            self.matchoutput(out,
                             """"cards/nic/eth0/hwaddr" """
                             """= "00:50:56:01:00:%02x";""" % (i - 1),
                             command)
            self.matchoutput(out, """"cards/nic/eth0/boot" = true;""", command)

    # FIXME: Add a test for creating an aquilon host out of a virtual
    # machine.

    # FIXME: Test that cluster plenaries were updated correctly.

    # FIXME: Missing a test for add_interface non-esx automac.  (Might not
    # be possible to test with the current command set.)

    # FIXME: Missing a test for add_machine for a cluster without cluster_type.
    # (May not be possible with the aq client.)

    # Can't test this as there is no way to add a cluster without
    # an archetype of vmhost - yet.
#   def testfailaddnonvirtualcluster(self):
#       command = ["add", "machine", "--machine", "ut9s03p51",
#                  "--cluster", "utecl1", "--cluster_type", "esx",
#                  "--model", "utmedium"]
#       out = self.badrequesttest(command)
#       self.matchoutput(out,
#                        "Can only add virtual machines to "
#                        "clusters with archetype vmhost.",
#                        command)

    def testfailaddmissingcluster(self):
        command = ["add_machine", "--machine=ut9s03p51",
                   "--cluster=cluster-does-not-exist", "--cluster_type=esx",
                   "--model=utmedium"]
        out = self.badrequesttest(command)
        self.matchoutput(out, "esx cluster 'cluster-does-not-exist' not found",
                         command)

    # FIXME: Add a test for add_machine that tries to use a non vmhost cluster.
    # This may not be possible yet as only esx clusters can be created and aqdb
    # constrains them to be vmhost.

    # FIXME: Add a test for add_machine that tries to override the location
    # of the cluster.

    # FIXME: Add a test that tries to add a virtual_machine without attaching
    # it to a cluster.

    # FIXME: Add tests for update_machine.  (Or is that UpdateVirtualHardware?)

if __name__=='__main__':
    suite = unittest.TestLoader().loadTestsFromTestCase(TestAddVirtualHardware)
    unittest.TextTestRunner(verbosity=2).run(suite)

<commit_msg>Added an extra test for update_machine.<commit_after>#!/usr/bin/env python2.5
# ex: set expandtab softtabstop=4 shiftwidth=4: -*- cpy-indent-level: 4; indent-tabs-mode: nil -*-
#
# Copyright (C) 2008,2009  Contributor
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the EU DataGrid Software License.  You should
# have received a copy of the license with this program, and the
# license is published at
# http://eu-datagrid.web.cern.ch/eu-datagrid/license.html.
#
# THE FOLLOWING DISCLAIMER APPLIES TO ALL SOFTWARE CODE AND OTHER
# MATERIALS CONTRIBUTED IN CONNECTION WITH THIS PROGRAM.
#
# THIS SOFTWARE IS LICENSED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE AND ANY WARRANTY OF NON-INFRINGEMENT, ARE
# DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
# BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. THIS
# SOFTWARE MAY BE REDISTRIBUTED TO OTHERS ONLY BY EFFECTIVELY USING
# THIS OR ANOTHER EQUIVALENT DISCLAIMER AS WELL AS ANY OTHER LICENSE
# TERMS THAT MAY APPLY.
"""Module for testing commands that add virtual hardware."""

import os
import sys
import unittest

if __name__ == "__main__":
    BINDIR = os.path.dirname(os.path.realpath(sys.argv[0]))
    SRCDIR = os.path.join(BINDIR, "..", "..")
    sys.path.append(os.path.join(SRCDIR, "lib", "python2.5"))

from brokertest import TestBrokerCommand

class TestAddVirtualHardware(TestBrokerCommand):

    def test_000_addmachines(self):
        for i in range(1, 10):
            self.noouttest(["add", "machine", "--machine", "evm%s" % i,
                            "--cluster", "utecl1", "--cluster_type", "esx",
                            "--model", "utmedium"])

    def test_100_addinterfaces(self):
        for i in range(1, 8):
            self.noouttest(["add", "interface", "--machine", "evm%s" % i,
                            "--interface", "eth0", "--automac"])

    def test_110_addinterfaces(self):
        self.noouttest(["add", "interface", "--machine", "evm9",
                        "--interface", "eth0", "--mac", "00:50:56:3f:ff:ff"])

    def test_120_addinterfaces(self):
        # This should now fill in the 'hole' between 7 and 9
        self.noouttest(["add", "interface", "--machine", "evm8",
                        "--interface", "eth0", "--automac"])

    def test_200_updatemachine(self):
        # FIXME: Verify that plenary moved correctly.
        # Before: file should be in machine/americas/ut/ut10
        # After: file should be in machine/americas/ut/None
        self.noouttest(["update_machine", "--machine", "evm9",
                        "--cluster", "utecl2", "--cluster_type", "esx"])

    def test_500_verifyaddmachines(self):
        # Skipping evm9 since the mac is out of sequence and different cluster
        for i in range(1, 9):
            command = "show machine --machine evm%s" % i
            out = self.commandtest(command.split(" "))
            self.matchoutput(out, "Virtual_machine: evm%s" % i, command)
            self.matchoutput(out, "Hosted by esx cluster: utecl1", command)
            self.matchoutput(out, "Building: ut", command)
            self.matchoutput(out, "Vendor: utvendor Model: utmedium", command)
            self.matchoutput(out, "Cpu: Cpu xeon_2500 x 1", command)
            self.matchoutput(out, "Memory: 8192 MB", command)
            self.matchoutput(out,
                             "Interface: eth0 00:50:56:01:00:%02x boot=True" %
                             (i - 1),
                             command)

    def test_500_verifycatmachines(self):
        # Skipping evm9 since the mac is out of sequence
        for i in range(1, 9):
            command = "cat --machine evm%s" % i
            out = self.commandtest(command.split(" "))
            self.matchoutput(out, """"location" = "ut.ny.na";""", command)
            self.matchoutput(out,
                             """include { """
                             """'hardware/machine/utvendor/utmedium' };""",
                             command)
            self.matchoutput(out,
                             """"ram" = list(create("hardware/ram/generic", """
                             """"size", 8192*MB));""",
                             command)
            self.matchoutput(out,
                             """"cpu" = list(create("""
                             """"hardware/cpu/intel/xeon_2500"));""",
                             command)
            self.matchoutput(out,
                             """"cards/nic/eth0/hwaddr" """
                             """= "00:50:56:01:00:%02x";""" % (i - 1),
                             command)
            self.matchoutput(out, """"cards/nic/eth0/boot" = true;""", command)

    # FIXME: Add a test for creating an aquilon host out of a virtual
    # machine.

    # FIXME: Test that cluster plenaries were updated correctly.

    # FIXME: Missing a test for add_interface non-esx automac.  (Might not
    # be possible to test with the current command set.)

    # FIXME: Missing a test for add_machine for a cluster without cluster_type.
    # (May not be possible with the aq client.)

    # Can't test this as there is no way to add a cluster without
    # an archetype of vmhost - yet.
#   def testfailaddnonvirtualcluster(self):
#       command = ["add", "machine", "--machine", "ut9s03p51",
#                  "--cluster", "utecl1", "--cluster_type", "esx",
#                  "--model", "utmedium"]
#       out = self.badrequesttest(command)
#       self.matchoutput(out,
#                        "Can only add virtual machines to "
#                        "clusters with archetype vmhost.",
#                        command)

    def testfailaddmissingcluster(self):
        command = ["add_machine", "--machine=ut9s03p51",
                   "--cluster=cluster-does-not-exist", "--cluster_type=esx",
                   "--model=utmedium"]
        out = self.badrequesttest(command)
        self.matchoutput(out, "esx cluster 'cluster-does-not-exist' not found",
                         command)

    # FIXME: Add a test for add_machine that tries to use a non vmhost cluster.
    # This may not be possible yet as only esx clusters can be created and aqdb
    # constrains them to be vmhost.

    # FIXME: Add a test for add_machine that tries to override the location
    # of the cluster.

    # FIXME: Add a test that tries to add a virtual_machine without attaching
    # it to a cluster.

    # FIXME: Add tests for update_machine.  (Or is that UpdateVirtualHardware?)

if __name__=='__main__':
    suite = unittest.TestLoader().loadTestsFromTestCase(TestAddVirtualHardware)
    unittest.TextTestRunner(verbosity=2).run(suite)

<|endoftext|>


<commit_before># Copyright (c) 2012 Cloudera, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Tests to validate the Catalog Service continues to function even if the HMS fails.

import logging
import pytest
import os
from subprocess import check_call
from tests.common.test_vector import *
from tests.common.test_dimensions import *
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.impala_cluster import ImpalaCluster
from tests.common.skip import SkipIfIsilon
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException

class TestHiveMetaStoreFailure(ImpalaTestSuite):
  @classmethod
  def get_workload(cls):
    return 'functional-query'

  @classmethod
  def add_test_dimensions(cls):
    super(TestHiveMetaStoreFailure, cls).add_test_dimensions()
    cls.TestMatrix.add_dimension(create_single_exec_option_dimension())

    # There is no reason to run these tests using all dimensions.
    cls.TestMatrix.add_constraint(lambda v:\
        v.get_value('table_format').file_format == 'parquet' and\
        v.get_value('table_format').compression_codec == 'none')

  @classmethod
  def setup_class(cls):
    super(TestHiveMetaStoreFailure, cls).setup_class()

  @classmethod
  def teardown_class(cls):
    # Make sure the metastore is running even if the test aborts somewhere unexpected
    # before restarting the metastore itself.
    run_cmd = os.path.join(os.environ['IMPALA_HOME'], 'testdata/bin/run-hive-server.sh')
    check_call([run_cmd], close_fds=True)
    cls.client.execute("invalidate metadata")
    super(TestHiveMetaStoreFailure, cls).teardown_class()

  # TODO: remove skip once IMPALA-3108 is resolved
  @SkipIfIsilon.hive
  @pytest.mark.execute_serially
  def test_hms_service_dies(self, vector):
    """Regression test for IMPALA-823 to verify the catalog service works properly when
    HMS connections fail"""
    # Force all the tables to be reloaded and then kill the hive metastore.
    tbl_name = "functional.alltypes"
    self.client.execute("invalidate metadata")
    kill_cmd = os.path.join(os.environ['IMPALA_HOME'], 'testdata/bin/kill-hive-server.sh')
    check_call([kill_cmd], close_fds=True)

    try:
      self.client.execute("describe %s" % tbl_name)
      pytest.xfail("It is possible (although unlikely) that all tables completed " +
          "loading between the time 'invalidate metadata' was called and the HMS was " +
          "killed. TODO: Make the test case more deterministic to ensure this never " +
          "happens.")
    except ImpalaBeeswaxException as e:
      print str(e)
      assert "Failed to load metadata for table: %s. Running 'invalidate metadata %s' "\
          "may resolve this problem." % (tbl_name, tbl_name) in str(e)

    run_cmd = os.path.join(os.environ['IMPALA_HOME'], 'testdata/bin/run-hive-server.sh')
    check_call([run_cmd], close_fds=True)

    self.client.execute("invalidate metadata %s" % tbl_name)
    self.client.execute("describe %s" % tbl_name)
<commit_msg>IMPALA-3075: Fix Isilon Catalog test<commit_after># Copyright (c) 2012 Cloudera, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Tests to validate the Catalog Service continues to function even if the HMS fails.

import logging
import pytest
import os
from subprocess import check_call
from tests.common.test_vector import *
from tests.common.test_dimensions import *
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.impala_cluster import ImpalaCluster
from tests.common.skip import SkipIfIsilon
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.util.filesystem_utils import IS_ISILON, IS_LOCAL

class TestHiveMetaStoreFailure(ImpalaTestSuite):
  @classmethod
  def get_workload(cls):
    return 'functional-query'

  @classmethod
  def add_test_dimensions(cls):
    super(TestHiveMetaStoreFailure, cls).add_test_dimensions()
    cls.TestMatrix.add_dimension(create_single_exec_option_dimension())

    # There is no reason to run these tests using all dimensions.
    cls.TestMatrix.add_constraint(lambda v:\
        v.get_value('table_format').file_format == 'parquet' and\
        v.get_value('table_format').compression_codec == 'none')

  @classmethod
  def setup_class(cls):
    super(TestHiveMetaStoreFailure, cls).setup_class()

  @classmethod
  def run_hive_server(cls):
    script = os.path.join(os.environ['IMPALA_HOME'], 'testdata/bin/run-hive-server.sh')
    run_cmd = [script]
    if IS_LOCAL or IS_ISILON:
      run_cmd.append('-only_metastore')
    check_call(run_cmd, close_fds=True)

  @classmethod
  def teardown_class(cls):
    # Make sure the metastore is running even if the test aborts somewhere unexpected
    # before restarting the metastore itself.
    cls.run_hive_server()
    cls.client.execute("invalidate metadata")
    super(TestHiveMetaStoreFailure, cls).teardown_class()

  @pytest.mark.execute_serially
  def test_hms_service_dies(self, vector):
    """Regression test for IMPALA-823 to verify the catalog service works properly when
    HMS connections fail"""
    # Force all the tables to be reloaded and then kill the hive metastore.
    tbl_name = "functional.alltypes"
    self.client.execute("invalidate metadata")
    kill_cmd = os.path.join(os.environ['IMPALA_HOME'], 'testdata/bin/kill-hive-server.sh')
    check_call([kill_cmd], close_fds=True)

    try:
      self.client.execute("describe %s" % tbl_name)
      pytest.xfail("It is possible (although unlikely) that all tables completed " +
          "loading between the time 'invalidate metadata' was called and the HMS was " +
          "killed. TODO: Make the test case more deterministic to ensure this never " +
          "happens.")
    except ImpalaBeeswaxException as e:
      print str(e)
      assert "Failed to load metadata for table: %s. Running 'invalidate metadata %s' "\
          "may resolve this problem." % (tbl_name, tbl_name) in str(e)
    self.run_hive_server()

    self.client.execute("invalidate metadata %s" % tbl_name)
    self.client.execute("describe %s" % tbl_name)
<|endoftext|>


<commit_before>#!/usr/bin/python
# Common initialisation commands for each test.

import os, sys

if os.system('/sbin/mcetool --unblank-screen --set-tklock-mode=unlocked --set-inhibit-mode=stay-on'):
  print 'mcetool is missing!'

if os.system('/usr/bin/gconftool-2 --type bool --set /desktop/meego/notifications/previews_enabled false'):
  print 'cannot disable notifications'

if os.system('pidof mcompositor'):
  print 'mcompositor is not running'
  sys.exit(1)

os.system('aegis-su -o com.nokia.maemo -n /usr/bin/windowstack')
os.system('aegis-su -o com.nokia.maemo -n /usr/bin/windowctl')

if os.system('windowstack m | grep mdecorator'):
  print 'mdecorator is not mapped!'
  sys.exit(1)

os.system('uptime')
<commit_msg>make all tests pass even if there are notifications<commit_after>#!/usr/bin/python
# Common initialisation commands for each test.

import os, sys, re

if os.system('/sbin/mcetool --unblank-screen --set-tklock-mode=unlocked --set-inhibit-mode=stay-on'):
  print 'mcetool is missing!'

if os.system('/usr/bin/gconftool-2 --type bool --set /desktop/meego/notifications/previews_enabled false'):
  print 'cannot disable notifications'

# Find and unmap all existing notification windows, so they don't get in our way.
r = re.compile("^(0x[0-9a-fA-F]+) NOTIFICATION ")
for win in [ m.groups(1)[0] for m in [ r.match(l) for l in os.popen("windowstack m") ] if m is not None ]:
	os.system('windowctl U %s' % win)

if os.system('pidof mcompositor'):
  print 'mcompositor is not running'
  sys.exit(1)

os.system('aegis-su -o com.nokia.maemo -n /usr/bin/windowstack')
os.system('aegis-su -o com.nokia.maemo -n /usr/bin/windowctl')

if os.system('windowstack m | grep mdecorator'):
  print 'mdecorator is not mapped!'
  sys.exit(1)

os.system('uptime')
<|endoftext|>


<commit_before># Copyright 2013-2015 DataStax, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

try:
    from Queue import Queue, Empty
except ImportError:
    from queue import Queue, Empty  # noqa

from struct import pack
import unittest

from cassandra import ConsistencyLevel
from cassandra.cluster import Cluster
from cassandra.query import dict_factory
from cassandra.query import SimpleStatement
from tests.integration import use_singledc, PROTOCOL_VERSION
from tests.integration.long.utils import create_schema

def setup_module():
    use_singledc()

# Converts an integer to an string of letters
def create_column_name(i):
    letters = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j']

    column_name = ''
    while True:
        column_name += letters[i % 10]
        i = i // 10
        if not i:
            break

    if column_name == 'if':
        column_name = 'special_case'
    return column_name

class LargeDataTests(unittest.TestCase):

    def setUp(self):
        self.keyspace = 'large_data'

    def make_session_and_keyspace(self):
        cluster = Cluster(protocol_version=PROTOCOL_VERSION)
        session = cluster.connect()
        session.default_timeout = 20.0  # increase the default timeout
        session.row_factory = dict_factory

        create_schema(session, self.keyspace)
        return session

    def batch_futures(self, session, statement_generator):
        concurrency = 10
        futures = Queue(maxsize=concurrency)
        for i, statement in enumerate(statement_generator):
            if i > 0 and i % (concurrency - 1) == 0:
                # clear the existing queue
                while True:
                    try:
                        futures.get_nowait().result()
                    except Empty:
                        break

            future = session.execute_async(statement)
            futures.put_nowait(future)

        while True:
            try:
                futures.get_nowait().result()
            except Empty:
                break

    def test_wide_rows(self):
        table = 'wide_rows'
        session = self.make_session_and_keyspace()
        session.execute('CREATE TABLE %s (k INT, i INT, PRIMARY KEY(k, i))' % table)

        prepared = session.prepare('INSERT INTO %s (k, i) VALUES (0, ?)' % (table, ))

        # Write via async futures
        self.batch_futures(session, (prepared.bind((i, )) for i in range(100000)))

        # Read
        results = session.execute('SELECT i FROM %s WHERE k=0' % (table, ))

        # Verify
        for i, row in enumerate(results):
            self.assertEqual(row['i'], i)

        session.cluster.shutdown()

    def test_wide_batch_rows(self):
        table = 'wide_batch_rows'
        session = self.make_session_and_keyspace()
        session.execute('CREATE TABLE %s (k INT, i INT, PRIMARY KEY(k, i))' % table)

        # Write
        statement = 'BEGIN BATCH '
        for i in range(2000):
            statement += 'INSERT INTO %s (k, i) VALUES (%s, %s) ' % (table, 0, i)
        statement += 'APPLY BATCH'
        statement = SimpleStatement(statement, consistency_level=ConsistencyLevel.QUORUM)
        session.execute(statement)

        # Read
        results = session.execute('SELECT i FROM %s WHERE k=%s' % (table, 0))

        # Verify
        for i, row in enumerate(results):
            self.assertEqual(row['i'], i)

        session.cluster.shutdown()

    def test_wide_byte_rows(self):
        table = 'wide_byte_rows'
        session = self.make_session_and_keyspace()
        session.execute('CREATE TABLE %s (k INT, i INT, v BLOB, PRIMARY KEY(k, i))' % table)

        prepared = session.prepare('INSERT INTO %s (k, i, v) VALUES (0, ?, 0xCAFE)' % (table, ))

        # Write
        self.batch_futures(session, (prepared.bind((i, )) for i in range(100000)))

        # Read
        results = session.execute('SELECT i, v FROM %s WHERE k=0' % (table, ))

        # Verify
        bb = pack('>H', 0xCAFE)
        for row in results:
            self.assertEqual(row['v'], bb)

        session.cluster.shutdown()

    def test_large_text(self):
        table = 'large_text'
        session = self.make_session_and_keyspace()
        session.execute('CREATE TABLE %s (k int PRIMARY KEY, txt text)' % table)

        # Create ultra-long text
        text = 'a' * 1000000

        # Write
        session.execute(SimpleStatement("INSERT INTO %s (k, txt) VALUES (%s, '%s')"
                                        % (table, 0, text),
                                        consistency_level=ConsistencyLevel.QUORUM))

        # Read
        result = session.execute('SELECT * FROM %s WHERE k=%s' % (table, 0))

        # Verify
        for row in result:
            self.assertEqual(row['txt'], text)

        session.cluster.shutdown()

    def test_wide_table(self):
        table = 'wide_table'
        table_width = 330
        session = self.make_session_and_keyspace()
        table_declaration = 'CREATE TABLE %s (key INT PRIMARY KEY, '
        table_declaration += ' INT, '.join(create_column_name(i) for i in range(table_width))
        table_declaration += ' INT)'
        session.execute(table_declaration % table)

        # Write
        insert_statement = 'INSERT INTO %s (key, '
        insert_statement += ', '.join(create_column_name(i) for i in range(table_width))
        insert_statement += ') VALUES (%s, '
        insert_statement += ', '.join(str(i) for i in range(table_width))
        insert_statement += ')'
        insert_statement = insert_statement % (table, 0)

        session.execute(SimpleStatement(insert_statement, consistency_level=ConsistencyLevel.QUORUM))

        # Read
        result = session.execute('SELECT * FROM %s WHERE key=%s' % (table, 0))

        # Verify
        for row in result:
            for i in range(table_width):
                self.assertEqual(row[create_column_name(i)], i)

        session.cluster.shutdown()
<commit_msg>handle Timeout errors in LargeDataTests<commit_after># Copyright 2013-2015 DataStax, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

try:
    from Queue import Queue, Empty
except ImportError:
    from queue import Queue, Empty  # noqa

from struct import pack
import logging, sys, traceback, time

from cassandra import ConsistencyLevel, OperationTimedOut, WriteTimeout
from cassandra.cluster import Cluster
from cassandra.query import dict_factory
from cassandra.query import SimpleStatement
from tests.integration import use_singledc, PROTOCOL_VERSION
from tests.integration.long.utils import create_schema

try:
    import unittest2 as unittest
except ImportError:
    import unittest  # noqa

log = logging.getLogger(__name__)

def setup_module():
    use_singledc()

# Converts an integer to an string of letters
def create_column_name(i):
    letters = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j']

    column_name = ''
    while True:
        column_name += letters[i % 10]
        i = i // 10
        if not i:
            break

    if column_name == 'if':
        column_name = 'special_case'
    return column_name

class LargeDataTests(unittest.TestCase):

    def setUp(self):
        self.keyspace = 'large_data'

    def make_session_and_keyspace(self):
        cluster = Cluster(protocol_version=PROTOCOL_VERSION)
        session = cluster.connect()
        session.default_timeout = 20.0  # increase the default timeout
        session.row_factory = dict_factory

        create_schema(session, self.keyspace)
        return session

    def batch_futures(self, session, statement_generator):
        concurrency = 10
        futures = Queue(maxsize=concurrency)
        for i, statement in enumerate(statement_generator):
            if i > 0 and i % (concurrency - 1) == 0:
                # clear the existing queue
                while True:
                    try:
                        futures.get_nowait().result()
                    except (OperationTimedOut, WriteTimeout):
                        ex_type, ex, tb = sys.exc_info()
                        log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb)))
                        del tb
                        time.sleep(1)
                    except Empty:
                        break

            future = session.execute_async(statement)
            futures.put_nowait(future)

        while True:
            try:
                futures.get_nowait().result()
            except (OperationTimedOut, WriteTimeout):
                ex_type, ex, tb = sys.exc_info()
                log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb)))
                del tb
                time.sleep(1)
            except Empty:
                break

    def test_wide_rows(self):
        table = 'wide_rows'
        session = self.make_session_and_keyspace()
        session.execute('CREATE TABLE %s (k INT, i INT, PRIMARY KEY(k, i))' % table)

        prepared = session.prepare('INSERT INTO %s (k, i) VALUES (0, ?)' % (table, ))

        # Write via async futures
        self.batch_futures(session, (prepared.bind((i, )) for i in range(100000)))

        # Read
        results = session.execute('SELECT i FROM %s WHERE k=0' % (table, ))

        # Verify
        for i, row in enumerate(results):
            self.assertEqual(row['i'], i)

        session.cluster.shutdown()

    def test_wide_batch_rows(self):
        table = 'wide_batch_rows'
        session = self.make_session_and_keyspace()
        session.execute('CREATE TABLE %s (k INT, i INT, PRIMARY KEY(k, i))' % table)

        # Write
        statement = 'BEGIN BATCH '
        for i in range(2000):
            statement += 'INSERT INTO %s (k, i) VALUES (%s, %s) ' % (table, 0, i)
        statement += 'APPLY BATCH'
        statement = SimpleStatement(statement, consistency_level=ConsistencyLevel.QUORUM)
        session.execute(statement)

        # Read
        results = session.execute('SELECT i FROM %s WHERE k=%s' % (table, 0))

        # Verify
        for i, row in enumerate(results):
            self.assertEqual(row['i'], i)

        session.cluster.shutdown()

    def test_wide_byte_rows(self):
        table = 'wide_byte_rows'
        session = self.make_session_and_keyspace()
        session.execute('CREATE TABLE %s (k INT, i INT, v BLOB, PRIMARY KEY(k, i))' % table)

        prepared = session.prepare('INSERT INTO %s (k, i, v) VALUES (0, ?, 0xCAFE)' % (table, ))

        # Write
        self.batch_futures(session, (prepared.bind((i, )) for i in range(100000)))

        # Read
        results = session.execute('SELECT i, v FROM %s WHERE k=0' % (table, ))

        # Verify
        bb = pack('>H', 0xCAFE)
        for row in results:
            self.assertEqual(row['v'], bb)

        session.cluster.shutdown()

    def test_large_text(self):
        table = 'large_text'
        session = self.make_session_and_keyspace()
        session.execute('CREATE TABLE %s (k int PRIMARY KEY, txt text)' % table)

        # Create ultra-long text
        text = 'a' * 1000000

        # Write
        session.execute(SimpleStatement("INSERT INTO %s (k, txt) VALUES (%s, '%s')"
                                        % (table, 0, text),
                                        consistency_level=ConsistencyLevel.QUORUM))

        # Read
        result = session.execute('SELECT * FROM %s WHERE k=%s' % (table, 0))

        # Verify
        for row in result:
            self.assertEqual(row['txt'], text)

        session.cluster.shutdown()

    def test_wide_table(self):
        table = 'wide_table'
        table_width = 330
        session = self.make_session_and_keyspace()
        table_declaration = 'CREATE TABLE %s (key INT PRIMARY KEY, '
        table_declaration += ' INT, '.join(create_column_name(i) for i in range(table_width))
        table_declaration += ' INT)'
        session.execute(table_declaration % table)

        # Write
        insert_statement = 'INSERT INTO %s (key, '
        insert_statement += ', '.join(create_column_name(i) for i in range(table_width))
        insert_statement += ') VALUES (%s, '
        insert_statement += ', '.join(str(i) for i in range(table_width))
        insert_statement += ')'
        insert_statement = insert_statement % (table, 0)

        session.execute(SimpleStatement(insert_statement, consistency_level=ConsistencyLevel.QUORUM))

        # Read
        result = session.execute('SELECT * FROM %s WHERE key=%s' % (table, 0))

        # Verify
        for row in result:
            for i in range(table_width):
                self.assertEqual(row[create_column_name(i)], i)

        session.cluster.shutdown()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Import python libs
from __future__ import absolute_import, print_function, unicode_literals

import logging

# Import Salt Testing libs
from tests.support.case import ModuleCase, ShellCase
from tests.support.helpers import slowTest

log = logging.getLogger(__name__)

class ExecutorTest(ModuleCase, ShellCase):
    def setup(self):
        self.run_function("saltutil.sync_all")

    @slowTest
    def test_executor(self):
        """
        test that dunders are set
        """
        data = self.run_call("test.arg --module-executors=arg")
        self.assertIn("test.arg fired", "".join(data))

    def test_executor_with_multijob(self):
        """
        test that executor is fired when sending a multifunction job
        """
        data = self.run_salt("'*' test.arg,test.arg foo,bar --module-executors=arg")
        self.assertIn("test.arg fired", "".join(data))
<commit_msg>fixup! Fix executors not being called for multi-jobs<commit_after># -*- coding: utf-8 -*-

# Import python libs
from __future__ import absolute_import, print_function, unicode_literals

import logging

# Import Salt Testing libs
from tests.support.case import ModuleCase, ShellCase
from tests.support.helpers import slowTest

log = logging.getLogger(__name__)

class ExecutorTest(ModuleCase, ShellCase):
    def setup(self):
        self.run_function("saltutil.sync_all")

    @slowTest
    def test_executor(self):
        """
        test that dunders are set
        """
        data = self.run_call("test.arg --module-executors=arg")
        self.assertIn("test.arg fired", "".join(data))

    @slowTest
    def test_executor_with_multijob(self):
        """
        test that executor is fired when sending a multifunction job
        """
        data = self.run_salt("'*' test.arg,test.arg foo,bar --module-executors=arg")
        self.assertIn("test.arg fired", "".join(data))
<|endoftext|>


<commit_before>#
# Copyright (c) 2016 nexB Inc. and others. All rights reserved.
# http://nexb.com and https://github.com/nexB/scancode-toolkit/
# The ScanCode software is licensed under the Apache License version 2.0.
# Data generated with ScanCode require an acknowledgment.
# ScanCode is a trademark of nexB Inc.
#
# You may not use this software except in compliance with the License.
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
#
# When you publish or redistribute any data created with ScanCode or any ScanCode
# derivative work, you must accompany this data with the following acknowledgment:
#
#  Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
#  ScanCode should be considered or used as legal advice. Consult an Attorney
#  for any legal advice.
#  ScanCode is a free software code scanning tool from nexB Inc. and others.
#  Visit https://github.com/nexB/scancode-toolkit/ for support and download.

from __future__ import absolute_import
from __future__ import print_function

import os.path

from commoncode.testcase import FileBasedTesting

from packagedcode.models import AssertedLicense
from packagedcode import models
from packagedcode.models import Package
from packagedcode.models import Party

class TestModels(FileBasedTesting):
    test_data_dir = os.path.join(os.path.dirname(__file__), 'data')

    def test_model_creation_and_dump(self):
        aap = models.AndroidAppPackage()
        result = aap.as_dict()
        assert 'Android app' == result['type']
        assert 'archive' == result['packaging']
        assert 'Java' == result['primary_language']

    def test_validate_package(self):
        package = Package(dict(
            name='Sample',
            summary='Some package',
            payload_type='source',
            authors=[Party(
                dict(
                    name='Some Author',
                    email='some@email.com'
                    )
                )
            ],
            keywords=['some', 'keyword'],
            vcs_tool='git',
            asserted_licenses=[
                AssertedLicense(dict(
                    license='apache-2.0'
                    )
                )
            ],
            )
        )
        assert 'Sample' == package.name
        assert 'Some package' == package.summary
        assert 'source' == package.payload_type
        assert 'Some Author' == package.authors[0].name
        assert ['some', 'keyword'] == package.keywords
        assert 'apache-2.0' == package.asserted_licenses[0].license
<commit_msg>#241 Add test for complete serialized Package objects<commit_after>#
# Copyright (c) 2016 nexB Inc. and others. All rights reserved.
# http://nexb.com and https://github.com/nexB/scancode-toolkit/
# The ScanCode software is licensed under the Apache License version 2.0.
# Data generated with ScanCode require an acknowledgment.
# ScanCode is a trademark of nexB Inc.
#
# You may not use this software except in compliance with the License.
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
#
# When you publish or redistribute any data created with ScanCode or any ScanCode
# derivative work, you must accompany this data with the following acknowledgment:
#
#  Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
#  ScanCode should be considered or used as legal advice. Consult an Attorney
#  for any legal advice.
#  ScanCode is a free software code scanning tool from nexB Inc. and others.
#  Visit https://github.com/nexB/scancode-toolkit/ for support and download.

from __future__ import absolute_import
from __future__ import print_function

import os.path

from commoncode.testcase import FileBasedTesting

from packagedcode.models import AssertedLicense
from packagedcode import models
from packagedcode.models import Package
from packagedcode.models import Party

class TestModels(FileBasedTesting):
    test_data_dir = os.path.join(os.path.dirname(__file__), 'data')

    def test_model_creation_and_dump(self):
        aap = models.AndroidAppPackage()
        result = aap.as_dict()
        expected = {
            'as_archive': u'archive',
            'as_dir': u'directory',
            'as_file': u'file',
            'asserted_licenses': [],
            'authors': None,
            'bug_tracking_url': None,
            'code_view_url': None,
            'contributors': None,
            'copyright_top_level': None,
            'copyrights': None,
            'dependencies': {},
            'description': None,
            'distributors': None,
            'download_md5': None,
            'download_sha1': None,
            'download_sha256': None,
            'download_urls': [],
            'homepage_url': None,
            'id': None,
            'keywords': None,
            'keywords_doc_url': None,
            'legal_file_locations': None,
            'license_expression': None,
            'license_texts': None,
            'location': None,
            'maintainers': None,
            'metafile_locations': None,
            'metafile_urls': [],
            'name': None,
            'notes': None,
            'notice_texts': None,
            'owners': None,
            'packagers': None,
            'packaging': u'archive',
            'payload_bin': u'binary',
            'payload_doc': u'doc',
            'payload_src': u'source',
            'payload_type': None,
            'primary_language': u'Java',
            'related_packages': [],
            'summary': None,
            'support_contacts': None,
            'type': u'Android app',
            'vcs_repository': None,
            'vcs_revision': None,
            'vcs_tool': None,
            'vendors': None,
            'versioning': None
        }
        assert expected == result

    def test_validate_package(self):
        package = Package(dict(
            name='Sample',
            summary='Some package',
            payload_type='source',
            authors=[Party(
                dict(
                    name='Some Author',
                    email='some@email.com'
                    )
                )
            ],
            keywords=['some', 'keyword'],
            vcs_tool='git',
            asserted_licenses=[
                AssertedLicense(dict(
                    license='apache-2.0'
                    )
                )
            ],
            )
        )
        expected = {
            'as_archive': u'archive',
            'as_dir': u'directory',
            'as_file': u'file',
            'asserted_licenses': [{
                'license': u'apache-2.0',
                'notice': None,
                'text': None,
                'url': None}
            ],
            'authors': [{
                'email': u'some@email.com',
                'name': u'Some Author',
                'type': None,
                'url': None}
            ],
            'bug_tracking_url': None,
            'code_view_url': None,
            'contributors': None,
            'copyright_top_level': None,
            'copyrights': None,
            'dependencies': {},
            'description': None,
            'distributors': None,
            'download_md5': None,
            'download_sha1': None,
            'download_sha256': None,
            'download_urls': [],
            'homepage_url': None,
            'id': None,
            'keywords': [u'some', u'keyword'],
            'keywords_doc_url': None,
            'legal_file_locations': None,
            'license_expression': None,
            'license_texts': None,
            'location': None,
            'maintainers': None,
            'metafile_locations': None,
            'metafile_urls': [],
            'name': u'Sample',
            'notes': None,
            'notice_texts': None,
            'owners': None,
            'packagers': None,
            'packaging': None,
            'payload_bin': u'binary',
            'payload_doc': u'doc',
            'payload_src': u'source',
            'payload_type': u'source',
            'related_packages': [],
            'summary': u'Some package',
            'support_contacts': None,
            'type': None,
            'vcs_repository': None,
            'vcs_revision': None,
            'vcs_tool': u'git',
            'vendors': None,
            'versioning': None
    }
        assert expected == package.as_dict()
<|endoftext|>


<commit_before>try:
    from capture import capture
except RuntimeError:
    print("unable to import capture")

import time
import gzip
import json
import os
import subprocess

camera = capture()
subprocess.call('arecord -d 10 -c 1 -D plughw:1,0 -v -r 48000 -t wav -f S16_LE -M audio.wav', shell=True)
camera.start_capture()
time.sleep(10)
camera.stop_capture()
##print("capture complete. Zipping")
gzip.GzipFile('example.h264')
result = {"complete":True}
print json.dumps(result)
<commit_msg>fixing this<commit_after>try:
    from capture import capture
except RuntimeError:
    print("unable to import capture")

import time
import gzip
import json
import os
import subprocess
from multiprocessing import Process

camera = capture()
def func1:
    os.system('arecord -d 10 -c 1 -D plughw:1,0 -v -r 48000 -t wav -f S16_LE -M audio.wav', shell=True)

def func2:
    camera.start_capture()

p1 = Process(target = func1)
p1.start()
p2 = Process(target = func2)
p2.start()
time.sleep(10)
camera.stop_capture()
##print("capture complete. Zipping")
gzip.GzipFile('example.h264')
result = {"complete":True}
print json.dumps(result)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
###############################################################################
#
#    Copyright (C) 2001-2014 Micronaet SRL (<http://www.micronaet.it>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
import os
import pdb
import sys
import netsvc
import logging
from openerp.osv import osv, orm, fields
from datetime import datetime, timedelta
from openerp.tools import (DEFAULT_SERVER_DATE_FORMAT,
    DEFAULT_SERVER_DATETIME_FORMAT, DATETIME_FORMATS_MAP, float_compare)
import openerp.addons.decimal_precision as dp
from openerp.tools.translate import _
from openerp.report.report_sxw import rml_parse

_logger = logging.getLogger(__name__)

class Parser(rml_parse):
    """ Parser for report
    """
    # Private data:
    counters = {}

    def __init__(self, cr, uid, name, context):
        self.context = context
        super(Parser, self).__init__(cr, uid, name, context)
        self.localcontext.update({
            'get_order_list': self.get_order_list,
            'get_counter': self.get_counter,
            'set_counter': self.set_counter,
            'is_new_page': self.is_new_page,
            'get_province': self.get_province,
        })

    def get_province(self, partner):
        """ Provice form partner
        """
        cr = self.cr
        uid = self.uid
        context = {'lang': 'it_IT'}
        city_pool = self.pool.get('res.partner.city')

        city = partner.city
        if not city:
            return ''
        city_ids = city_pool.search(cr, uid, [
            ('name', '=', city),
        ], context=context)
        if city_ids:
            city = city_pool.browse(cr, uid, city_ids, context=context)[0]
            if city.province:
                return '(%s)' % city.province or ''
            else:
                _logger.error('Province not found: %s' % city)
        else:
            return ''

    def is_new_page(self, o, objects, counter, data):
        """ Jump page?
        """
        multi = data.get('multi', 1) - 1
        jump = not (o.id == objects[-1].id and counter >= multi)
        _logger.warning('%s [di %s] Ripetizione %s di %s Salto: %s' % (
            o.tour_id.name,
            len(objects),
            counter,
            multi,
            'SI' if jump else 'NO',
        ))
        return jump

    def get_order_list(self, trip):
        """ Pack same line
        """
        order_line = []
        if trip.report_line == 'packed':
            # Preload phase:
            preload = {}
            for order in trip.order_ids:
                key = (
                    order.partner_id,
                    order.destination_id,
                    order.order_mode,
                )
                if key in preload:
                    preload[key][1].append(order)
                else:
                    preload[key] = [order.sequence, [order]]

            # Prepare data phase:
            for key in sorted(preload):  # sequence is the sort key
                sequence, orders = preload[key]
                counter = len(orders)

                extra = {
                    'time': set(),
                    'deadline': set(),
                    'number': set(),
                    'quantity': 0.0,
                }
                for order in orders:
                    time = (order.time or '').strip()
                    if time:
                        extra['time'].add(time)
                    deadline = (order.date or '').strip()
                    if deadline:
                        extra['deadline'].add(deadline)
                    number = '/' if not order.name else \
                        order.name.split('-')[-1].strip()
                    extra['number'].add(number)
                    extra['quantity'] += order.prevision_load
                extra['time'] = ' '.join(extra['time'])
                extra['deadline'] = ' '.join(extra['deadline'])
                extra['number'] = ' '.join(extra['number'])

                # Add only first for collect data:
                order_line.append(
                    (counter, orders[0], extra)
                )
        else:  # detailed
            counter = 1
            extra = {}
            for order in trip.order_ids:
                order_line.append(
                    (counter, order, extra)
                )
        return order_line

    def get_counter(self, name):
        """ Get counter with name passed (else create an empty)
        """
        if name not in self.counters:
            self.counters[name] = False
        return self.counters[name]

    def set_counter(self, name, value):
        """ Set counter with name with value passed
        """
        self.counters[name] = value
        return ""  # empty so no write in module
<commit_msg>generic bugfix (fast message)<commit_after># -*- coding: utf-8 -*-
###############################################################################
#
#    Copyright (C) 2001-2014 Micronaet SRL (<http://www.micronaet.it>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
import os
import pdb
import sys
import netsvc
import logging
from openerp.osv import osv, orm, fields
from datetime import datetime, timedelta
from openerp.tools import (DEFAULT_SERVER_DATE_FORMAT,
    DEFAULT_SERVER_DATETIME_FORMAT, DATETIME_FORMATS_MAP, float_compare)
import openerp.addons.decimal_precision as dp
from openerp.tools.translate import _
from openerp.report.report_sxw import rml_parse

_logger = logging.getLogger(__name__)

class Parser(rml_parse):
    """ Parser for report
    """
    # Private data:
    counters = {}

    def __init__(self, cr, uid, name, context):
        self.context = context
        super(Parser, self).__init__(cr, uid, name, context)
        self.localcontext.update({
            'get_order_list': self.get_order_list,
            'get_counter': self.get_counter,
            'set_counter': self.set_counter,
            'is_new_page': self.is_new_page,
            'get_province': self.get_province,
        })

    def get_province(self, partner):
        """ Provice form partner
        """
        cr = self.cr
        uid = self.uid
        context = {'lang': 'it_IT'}
        city_pool = self.pool.get('res.partner.city')

        city = partner.city
        if not city:
            return ''
        city_ids = city_pool.search(cr, uid, [
            ('name', '=ilike', city),
        ], context=context)
        if city_ids:
            city = city_pool.browse(cr, uid, city_ids, context=context)[0]
            if city.province:
                return '(%s)' % city.province or ''
            else:
                _logger.error('Province not found: %s' % city)
        else:
            return ''

    def is_new_page(self, o, objects, counter, data):
        """ Jump page?
        """
        multi = data.get('multi', 1) - 1
        jump = not (o.id == objects[-1].id and counter >= multi)
        _logger.warning('%s [di %s] Ripetizione %s di %s Salto: %s' % (
            o.tour_id.name,
            len(objects),
            counter,
            multi,
            'SI' if jump else 'NO',
        ))
        return jump

    def get_order_list(self, trip):
        """ Pack same line
        """
        order_line = []
        if trip.report_line == 'packed':
            # Preload phase:
            preload = {}
            for order in trip.order_ids:
                key = (
                    order.partner_id,
                    order.destination_id,
                    order.order_mode,
                )
                if key in preload:
                    preload[key][1].append(order)
                else:
                    preload[key] = [order.sequence, [order]]

            # Prepare data phase:
            for key in sorted(preload):  # sequence is the sort key
                sequence, orders = preload[key]
                counter = len(orders)

                extra = {
                    'time': set(),
                    'deadline': set(),
                    'number': set(),
                    'quantity': 0.0,
                }
                for order in orders:
                    time = (order.time or '').strip()
                    if time:
                        extra['time'].add(time)
                    deadline = (order.date or '').strip()
                    if deadline:
                        extra['deadline'].add(deadline)
                    number = '/' if not order.name else \
                        order.name.split('-')[-1].strip()
                    extra['number'].add(number)
                    extra['quantity'] += order.prevision_load
                extra['time'] = ' '.join(extra['time'])
                extra['deadline'] = ' '.join(extra['deadline'])
                extra['number'] = ' '.join(extra['number'])

                # Add only first for collect data:
                order_line.append(
                    (counter, orders[0], extra)
                )
        else:  # detailed
            counter = 1
            extra = {}
            for order in trip.order_ids:
                order_line.append(
                    (counter, order, extra)
                )
        return order_line

    def get_counter(self, name):
        """ Get counter with name passed (else create an empty)
        """
        if name not in self.counters:
            self.counters[name] = False
        return self.counters[name]

    def set_counter(self, name, value):
        """ Set counter with name with value passed
        """
        self.counters[name] = value
        return ""  # empty so no write in module
<|endoftext|>


<commit_before>from . quote_fields import quote_definitions, quote_dtypes
from io import StringIO
import pandas as pd
import numpy as np
from datetime import datetime, timedelta
from requests import Session
from redis import StrictRedis

# TODO look into read_csv use_cols option for speedups
# TODO Fix doc comment formatting on methods

class ActiveTick:
    def __init__(self, host='127.0.0.1', port=5000):

        # Active tick HTTP proxy config
        self.host = host
        self.port = port
        self.r = Session()
        self.cache = StrictRedis(host='127.0.0.1')

        self._date_fmt = '%Y%m%d%H%M%S'

    def _format_symbols(self, symbols):
        """
        symbols - string (returns unchanged) or list of symbols to concat with + symbols (string, list)
        Formats list of symbols for ActiveTick URL
        :param symbols:
        :return:
        String
        """
        if not isinstance(symbols, str):
            symbols = '+'.join(symbols)
        return symbols

    def _date_parser(self, date_format):
        """
        Date parser function factory for pandas csv parsing of ActiveTick data
        :param date_format:
        Format used for parsing date (string)
        :return:
        (list of datetimes)
        """
        def date_parser(dates):
            return [ datetime.strptime(date, date_format) for date in dates ]
        return date_parser

    def quoteData(self, symbols, quoteFields):
        """
        symbols - Symbol (or iterable of multiple symbols) for contracts, ie SPY, AAPL--130308C00440000 (string, iter)
        quote_fields - List of all fields of interest (string, list)

        # Example:
        # look to quoteFields.py for the lookup table used and available fields

        atClient.quoteData('SPY', ['LastPrice' , 'BidSize', 'AskSize'])

        # returns pandas DataFrame with columns named

        :return:
        pandas.DataFrame() indexed on the symbol column with columns with requested quoteFields
        with extra status meta data regarding the request and symbols, to just get a DataFrame
        with the requested fields quoteData('SPY', fields)[fields]
        """
        print('quotefield:', quoteFields)
        names = ['symbol', 'symbol_status']
        def __name_fmt(names, field):
            names += ["{f}_field_id".format(f=field),
                      "{f}_status".format(f=field),
                      "{f}_datatype".format(f=field),
                      "{f}".format(f=field)]
            return names

        if not isinstance(quoteFields, str):

            # Create column names from quoteFields
            for field in quoteFields:
                names = __name_fmt(names, field)

            # TODO: Declare specific dtypes for each column in names
            # Translate from human readable quoteFields to IDs
            quoteFields = map(lambda field: quote_definitions[field], quoteFields)
            quoteFields = '+'.join(quoteFields)
        else:

            # Only one quoteField as string
            names = __name_fmt(names, quoteFields)
            quoteFields = quote_definitions[quoteFields]

        url = "http://{host}:{port}/quoteData?symbol={symbols}&field={quoteFields}".format(
            host=self.host,
            port=self.port,
            symbols=self._format_symbols(symbols),
            quoteFields=quoteFields
        )

        # GET request is made and the CSV is read into a Pandas DataFrame
        df = pd.read_csv(url, header=None, names=names, index_col='symbol')
        return df

    def quoteStream(self, symbols):
        """
        symbols - string or iter of symbols

        # Example
        # res is an instance of requests iter_lines()
        res = at.quoteStream('SPY')
        for quote in res:
            print(quote)

        :return:
        returns lazy iterator see requests iter_lines() that can be looped over to access streaming data
        """
        # TODO: Start, pause, stop quote stream

        def __tickParse(tick):
            tick = tick.decode('utf-8')
            if tick[0] is 'Q':
                names = ['type', 'symbol', 'cond', 'bid_ex', 'ask_ex', 'bid', 'ask', 'bidz', 'askz', 'datetime']
                dtype = {'type': object,
                         'symbol': object,
                         'cond': np.uint8,
                         'bid_ex': object,
                         'ask_ex': object,
                         'bid': np.float32,
                         'ask': np.float32,
                         'bidz': np.uint32,
                         'askz': np.uint32,
                         'datetime': object}
            else:
                names = ['type', 'symbol', 'flags', 'cond1', 'cond2', 'cond3', 'cond4', 'last_ex', 'last', 'lastz',
                         'datetime']
                dtype = {
                    'type': object,
                    'symbol': object,
                    'flags': object,
                    'cond1': np.int8,
                    'cond2': np.int8,
                    'cond3': np.int8,
                    'cond4': np.int8,
                    'last_ex': object,
                    'last': np.float32,
                    'lastz': np.uint32
                }
            date_format = '%Y%m%d%H%M%S%f'
            parse_date = self._date_parser(date_format)

            return pd.read_csv(StringIO(tick), names=names, index_col='type', dtype=dtype,
                               parse_dates=['datetime'], date_parser=parse_date)

        res = self.r.get('http://{host}:{port}/quoteStream?symbol={symbols}'.format(
            host=self.host,
            port=self.port,
            symbols=self._format_symbols(symbols)
        ), stream=True)
        lines = map(__tickParse, res.iter_lines())
        first_line = next(lines)
        return lines

    def barData(self, symbol, historyType='I', intradayMinutes=60,
                beginTime=datetime(datetime.now().year, datetime.now().month, 1), endTime=datetime.now()):
        """
        :param symbol:
         Takes only one symbol, string
        :param historyType:
         Takes 'I', 'D' or 'W' as a string (Intraday 0, Daily 1 or Weekly 0)
        :param intradayMinutes:
         If historyType is 'I' select a bar size: 0 to 60 minutes (int)
        :param beginTime:
         Beginning date for query (datetime)
        :param endTime:
         Ending date for query (datetime)
        :return:
         Pandas DataFrame OHLCV indexed on the datetime
        """
        history_lookup = {
            'I': 0,
            'D': 1,
            'W': 2
        }

        def __getIntradayMinutesAttr():
            # Returns URL segment for intraday minutes if needed
            if historyType is not 'I':
                attr_str = ''
            else:
                attr_str = 'intradayMinutes={intradayMinutes}&'.format(intradayMinutes=str(intradayMinutes))
            return attr_str

        beginTime_s = datetime.strftime(beginTime, format=self._date_fmt)
        endTime_s = datetime.strftime(endTime, format=self._date_fmt)

        cache_key = "AT:BARDATA:{symbol}:{historyType}:{intradayMinutes}:{beginTime}:{endTime}"
        cache_key = cache_key.format(
            symbol=symbol,
            historyType=history_lookup[historyType],
            intradayMinutes=intradayMinutes,
            beginTime=beginTime_s,
            endTime=endTime_s)

        # If the data is cached
        if self.cache.exists(cache_key):
            return pd.read_msgpack(self.cache.get(cache_key))

        url = 'http://{host}:{port}/barData?symbol={symbol}&historyType={historyType}' \
              '&{intradayMintuesAttr}beginTime={beginTime}&endTime={endTime}'
        url = url.format(
            host=self.host,
            port=self.port,
            symbol=symbol,
            historyType=history_lookup[historyType],
            intradayMintuesAttr=__getIntradayMinutesAttr(),
            beginTime=beginTime_s,
            endTime=endTime_s)

        dtypes = {'datetime': object,
                  'open': np.float32,
                  'high': np.float32,
                  'low': np.float32,
                  'close': np.float32,
                  'volume': np.uint32}

        df = pd.read_csv(url, header=None, names=['datetime', 'open', 'high', 'low', 'close', 'volume'],
                         index_col='datetime', parse_dates=['datetime'], dtype=dtypes)

        # Cache the data
        self.cache.set(cache_key, df.to_msgpack(compress='zlib'))
        return df

    def tickData(self, symbol, trades=False, quotes=True,
                 beginTime=datetime.now() - timedelta(minutes=15), endTime=datetime.now()):
        """
        Gets tick level data in between a time range, limited to returnedin 100,000 quotes/trades at a time
        :param symbol:
        String, ticker for symbol in ActiveTick format
        :param trades:
        Boolean, whether to return trade ticks
        :param quotes:
        Boolean whether to return quote ticks
        :param beginTime:
        datetime beginning of date range
        :param endTime:
        datetime end of date range
        :return:
        """

        tick_date_fmt = '%Y%m%d%H%M%S%f'
        date_parser = self._date_parser(tick_date_fmt)
        q_names = ['type',
                   'datetime',
                   'bid',
                   'ask',
                   'bidz',
                   'askz',
                   'bidx',
                   'askx',
                   'cond']
        t_names = ['type',
                   'datetime',
                   'last',
                   'lastz',
                   'lastx',
                   'cond1',
                   'cond2',
                   'cond3',
                   'cond4']

        def __get_trades(df):
            trades_df = df[df[0] == 'T'].copy()
            trades_df.columns = t_names
            trades_df.loc[:, 'last'] = trades_df.loc[:, 'last'].astype(np.float32)
            trades_df.loc[:, 'lastz'] = trades_df.loc[:, 'lastz'].astype(np.uint32)
            trades_df.loc[:, ['cond1', 'cond2', 'cond3', 'cond4']] = trades_df.loc[:, ['cond1',
                                                                                'cond2',
                                                                                'cond3',
                                                                                'cond4']].astype(np.uint8)
            return trades_df

        def __get_quotes(df):
            quotes_df = df[df[0] == 'Q'].copy()
            quotes_df.columns = q_names
            quotes_df.loc[:, ['bid', 'ask']] = quotes_df.loc[:, ['bid', 'ask']].astype(np.float32)
            quotes_df.loc[:, ['bidz', 'askz']] = quotes_df.loc[:, ['bidz', 'askz']].astype(np.uint32)
            quotes_df.loc[:, 'cond'] = quotes_df.loc[:, 'cond'].astype(np.uint8)
            return quotes_df

        def __at_request(url, names):
            if(names):
                date_col = 'datetime'
            else:
                date_col = 1
                del q_names[1]
                del t_names[1]
            df = pd.read_csv(url, header=None,
                             engine='c',
                             index_col=date_col,
                             parse_dates=[date_col],
                             names=names,
                             date_parser=date_parser)
            return df

        if not trades and not quotes:
            return pd.DataFrame()

        beginTime_s = datetime.strftime(beginTime, format=self._date_fmt)
        endTime_s = datetime.strftime(endTime, format=self._date_fmt)

        cache_key = 'AT:TICKDATA:{symbol}:{trades}:{quotes}:{beginTime}:{endTime}'
        cache_key = cache_key.format(
            symbol=symbol,
            trades=int(trades),
            quotes=int(quotes),
            beginTime=beginTime_s,
            endTime=endTime_s
        )

        # Return cached data
        if self.cache.exists(cache_key):
            return pd.read_msgpack(self.cache.get(cache_key))

        # Retrieve data not found in cache
        else:
            url = 'http://{host}:{port}/tickData?symbol={symbol}&trades={trades}' \
                  '&quotes={quotes}&beginTime={beginTime}&endTime={endTime}'

            url = url.format(
                host=self.host,
                port=self.port,
                symbol=symbol,
                trades=int(trades),
                quotes=int(quotes),
                beginTime=beginTime_s,
                endTime=endTime_s
            )
            print(url)

            # Quote column names
            if quotes and not trades:
                df = __at_request(url, q_names)

            # Trade columns names
            if trades and not quotes:
                df = __at_request(url, t_names)

            if trades and quotes:
                df = __at_request(url, None)
                df = __get_trades(df).append(__get_quotes(df)).sort_index(axis=0)

        self.cache.set(cache_key, df.to_msgpack(compress='zlib'))
        return df

    def optionChain(self, symbol):
        """
        Returns unnamed pandas dataframe of option symbols currently listed for underlying symbol
        :param symbol:
        String, ticker symbol for underlying
        :return:
        Raw unnamed dataframe from ActiveTick
        """
        url = 'http://{host}:{port}/optionChain?symbol={symbol}'.format(
            host=self.host,
            port=self.port,
            symbol=symbol)
        df = pd.read_csv(url)
        return df

if __name__ == '__main__':
    print('ActiveTick Python Module, attaches to ActiveTick HTTP Proxy, returns Pandas DataFrames.')
    at = ActiveTick()
    begin = datetime(year=2016, month=9, day=28, hour=9, minute=30)
    end = begin + timedelta(minutes=1)
    df = at.tickData('SPY', trades=False, quotes=False, beginTime=begin, endTime=end)
    print(df)
    # TODO Write/Run tests
<commit_msg>changed name to __init__<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from report import report_sxw
import xml.dom.minidom
import os, time
import osv
import re
import tools
import pooler
import re
import sys

class rml_parse(report_sxw.rml_parse):
    def __init__(self, cr, uid, name, context):
        super(rml_parse, self).__init__(cr, uid, name, context=context)
        self.localcontext.update({
            'comma_me': self.comma_me,
            'format_date': self._get_and_change_date_format_for_swiss,
            'strip_name' : self._strip_name,
            'explode_name' : self._explode_name,
        })

    def comma_me(self,amount):
        #print "#" + str(amount) + "#"
        if not amount:
            amount = 0.0
        if  type(amount) is float :
            amount = str('%.2f'%amount)
        else :
            amount = str(amount)
        if (amount == '0'):
             return ' '
        orig = amount
        new = re.sub("^(-?\d+)(\d{3})", "\g<1>'\g<2>", amount)
        if orig == new:
            return new
        else:
            return self.comma_me(new)
    def _ellipsis(self, string, maxlen=100, ellipsis = '...'):
        ellipsis = ellipsis or ''
        try:
            return string[:maxlen - len(ellipsis) ] + (ellipsis, '')[len(string) < maxlen]
        except Exception, e:
            return False
    def _strip_name(self, name, maxlen=50):
        return self._ellipsis(name, maxlen, '...')

    def _get_and_change_date_format_for_swiss (self,date_to_format):
        date_formatted=''
        if date_to_format:
            date_formatted = strptime (date_to_format,'%Y-%m-%d').strftime('%d.%m.%Y')
        return date_formatted

    def _explode_name(self,chaine,length):
        # We will test if the size is less then account
        full_string = ''
        if (len(str(chaine)) <= length):
            return chaine
        #
        else:
            chaine = unicode(chaine,'utf8').encode('iso-8859-1')
            rup = 0
            for carac in chaine:
                rup = rup + 1
                if rup == length:
                    full_string = full_string + '\n'
                    full_string = full_string + carac
                    rup = 0
                else:
                    full_string = full_string + carac

        return full_string

    def makeAscii(self,str):
        try:
            Stringer = str.encode("utf-8")
        except UnicodeDecodeError:
            try:
                Stringer = str.encode("utf-16")
            except UnicodeDecodeError:
                print "UTF_16 Error"
                Stringer = str
            else:
                return Stringer
        else:
            return Stringer
        return Stringer
    def explode_this(self,chaine,length):
        chaine = rstrip(chaine)
        ast = list(chaine)
        i = length
        while i <= len(ast):
            ast.insert(i,'\n')
            i = i + length
        chaine = str("".join(ast))
        return chaine
    def repair_string(self,chaine):
        ast = list(chaine)
        UnicodeAst = []
        _previouslyfound = False
        i = 0
        while i < len(ast):
            elem = ast[i]
            try:
                Stringer = elem.encode("utf-8")
            except UnicodeDecodeError:
                to_reencode = elem + ast[i+1]
                print str(to_reencode)
                Good_char = to_reencode.decode('utf-8')
                UnicodeAst.append(Good_char)
                i += i +2
            else:
                UnicodeAst.append(elem)
                i += i + 1

        return "".join(UnicodeAst)

    def ReencodeAscii(self,str):
        print sys.stdin.encoding
        try:
            Stringer = str.decode("ascii")
        except UnicodeEncodeError:
            print "REENCODING ERROR"
            return str.encode("ascii")
        except UnicodeDecodeError:
            print "DECODING ERROR"
            return str.encode("ascii")

        else:
            return Stringer

<commit_msg>[REF] account : Corrections in account report<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2010 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from report import report_sxw
import xml.dom.minidom
import time
import osv
import re
import pooler
import sys

class rml_parse(report_sxw.rml_parse):
    def __init__(self, cr, uid, name, context):
        super(rml_parse, self).__init__(cr, uid, name, context=context)
        self.localcontext.update({
            'comma_me': self.comma_me,
            'format_date': self._get_and_change_date_format_for_swiss,
            'strip_name' : self._strip_name,
            'explode_name' : self._explode_name,
        })

    def comma_me(self,amount):
        #print "#" + str(amount) + "#"
        if not amount:
            amount = 0.0
        if  type(amount) is float :
            amount = str('%.2f'%amount)
        else :
            amount = str(amount)
        if (amount == '0'):
             return ' '
        orig = amount
        new = re.sub("^(-?\d+)(\d{3})", "\g<1>'\g<2>", amount)
        if orig == new:
            return new
        else:
            return self.comma_me(new)
    def _ellipsis(self, string, maxlen=100, ellipsis = '...'):
        ellipsis = ellipsis or ''
        try:
            return string[:maxlen - len(ellipsis) ] + (ellipsis, '')[len(string) < maxlen]
        except Exception, e:
            return False
    def _strip_name(self, name, maxlen=50):
        return self._ellipsis(name, maxlen, '...')

    def _get_and_change_date_format_for_swiss (self,date_to_format):
        date_formatted=''
        if date_to_format:
            date_formatted = strptime (date_to_format,'%Y-%m-%d').strftime('%d.%m.%Y')
        return date_formatted

    def _explode_name(self,chaine,length):
        # We will test if the size is less then account
        full_string = ''
        if (len(str(chaine)) <= length):
            return chaine
        #
        else:
            chaine = unicode(chaine,'utf8').encode('iso-8859-1')
            rup = 0
            for carac in chaine:
                rup = rup + 1
                if rup == length:
                    full_string = full_string + '\n'
                    full_string = full_string + carac
                    rup = 0
                else:
                    full_string = full_string + carac

        return full_string

    def makeAscii(self,str):
        try:
            Stringer = str.encode("utf-8")
        except UnicodeDecodeError:
            try:
                Stringer = str.encode("utf-16")
            except UnicodeDecodeError:
                print "UTF_16 Error"
                Stringer = str
            else:
                return Stringer
        else:
            return Stringer
        return Stringer
    def explode_this(self,chaine,length):
        chaine = rstrip(chaine)
        ast = list(chaine)
        i = length
        while i <= len(ast):
            ast.insert(i,'\n')
            i = i + length
        chaine = str("".join(ast))
        return chaine
    def repair_string(self,chaine):
        ast = list(chaine)
        UnicodeAst = []
        _previouslyfound = False
        i = 0
        while i < len(ast):
            elem = ast[i]
            try:
                Stringer = elem.encode("utf-8")
            except UnicodeDecodeError:
                to_reencode = elem + ast[i+1]
                print str(to_reencode)
                Good_char = to_reencode.decode('utf-8')
                UnicodeAst.append(Good_char)
                i += i +2
            else:
                UnicodeAst.append(elem)
                i += i + 1

        return "".join(UnicodeAst)

    def ReencodeAscii(self,str):
        print sys.stdin.encoding
        try:
            Stringer = str.decode("ascii")
        except UnicodeEncodeError:
            print "REENCODING ERROR"
            return str.encode("ascii")
        except UnicodeDecodeError:
            print "DECODING ERROR"
            return str.encode("ascii")

        else:
            return Stringer

<|endoftext|>


<commit_before>import urllib.request
import sys
import json
from urllib.parse import quote

class WordSimilarity:

	# 1 - EasyESA client
	# a score of 1 and -1 results in a perfect match
	# treshold values to consider  0.07, 0.052 and 0.04
	def getEasyESAScore(word1,word2):
		url = "http://vmdeb20.deri.ie:8890/esaservice?task=esa&term1="+quote(word1)+'&term2='+quote(word2)
		request = urllib.request.Request(url)
		response = urllib.request.urlopen(request)
		score = str(response.read().decode('utf-8')).replace('\"','')
		return float(score)

	# 2 - ws4j client
	def getWs4jScore(word1,word2):
		url = "http://ws4jdemo.appspot.com/ws4j?measure=wup&args="+quote(word1)+"%3A%3A"+quote(word2)
		#print(url)
		request = urllib.request.Request(url)
		request.add_header('Accept', 'application/json')
		response = urllib.request.urlopen(request)
		responseStr = response.read().decode('utf-8')
		# fetch json from the response
		jsonStr = json.loads(responseStr)
		score = float(jsonStr['result'][0]['score'])
		return score

	# As of now using only EasyESA.
	# call the method 2 ws4j client if needed
	# a score of 1 and -1 results in a perfect match
	# treshold values to consider  0.07, 0.052 and 0.04
	def isPredicateSimilar(word1,word2):
		score = WordSimilarity.getEasyESAScore(word1,word2)
		if(score>0.04):
			return score
		else:
			return -1

	'''
	def isPredicateSimilar(word1,word2,option):
		score = -1
		if(option!=1 or option!=2):
			#score = WordSimilarity.getEasyESAScore(word1,word2)
			score = 0
			if((score!=0 or score!=-1) and score>0.1):
				return score
			else:
				score = WordSimilarity.getWs4jScore(word1,word2)
				if(score>0.8):
					return score
				else:
					return -1
	'''
<commit_msg>swoogle similarity added<commit_after>import urllib.request
import sys
import json
import math
from urllib.parse import quote

class WordSimilarity:

	# 1 - EasyESA client
	# a score of 1 and -1 results in a perfect match
	# treshold values to consider  0.07, 0.052 and 0.04
	def getEasyESAScore(word1,word2):
		url = "http://vmdeb20.deri.ie:8890/esaservice?task=esa&term1="+quote(word1)+'&term2='+quote(word2)
		request = urllib.request.Request(url)
		response = urllib.request.urlopen(request)
		score = str(response.read().decode('utf-8')).replace('\"','')
		return float(score)

	# 2 - ws4j client
	def getWs4jScore(word1,word2):
		url = "http://ws4jdemo.appspot.com/ws4j?measure=wup&args="+quote(word1)+"%3A%3A"+quote(word2)
		#print(url)
		request = urllib.request.Request(url)
		request.add_header('Accept', 'application/json')
		response = urllib.request.urlopen(request)
		responseStr = response.read().decode('utf-8')
		# fetch json from the response
		jsonStr = json.loads(responseStr)
		score = float(jsonStr['result'][0]['score'])
		return score

	# 3 - UMBC Semantic Similarity service
	#
	#  Documentation availabel at http://swoogle.umbc.edu/SimService/api.html
	def getEasyUMBCScore(word1,word2):
		url = "http://swoogle.umbc.edu/StsService/GetStsSim?operation=api&phrase1="+quote('birth name')+'&phrase2='+quote('last name')
		request = urllib.request.Request(url)
		response = urllib.request.urlopen(request)
		score = str(response.read().decode('utf-8')).replace('\"','')
		score = float(score)
		return score

	# As of now using only EasyESA.
	# call the method 2 ws4j client if needed
	# a score of 1 and -1 results in a perfect match
	# treshold values to consider  0.07, 0.052 and 0.04
	def isPredicateSimilar(word1,word2):
		#score = math.fabs(WordSimilarity.getEasyESAScore(word1,word2))
		score = (WordSimilarity.getEasyESAScore(word1,word2))
		if(score>0.07):
			return score
		else:
			return -1

	<|endoftext|>