<commit_before>from datetime import datetime
import os
import uuid

from django.conf import settings
from django.test import TestCase
from mock import patch
from couchdbkit import RequestFailed
from casexml.apps.case.mock import CaseBlock
from corehq.apps.hqcase.utils import submit_case_blocks
from corehq.apps.receiverwrapper.util import submit_form_locally
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors, FormAccessors
from couchforms.models import UnfinishedSubmissionStub

from corehq.form_processor.interfaces.processor import FormProcessorInterface
from corehq.form_processor.tests.utils import FormProcessorTestUtils, run_with_all_backends, post_xform
from corehq.util.test_utils import TestFileMixin

class EditFormTest(TestCase, TestFileMixin):
    ID = '7H46J37FGH3'
    domain = 'test-form-edits'

    file_path = ('data', 'deprecation')
    root = os.path.dirname(__file__)

    def setUp(self):
        super(EditFormTest, self).setUp()
        self.interface = FormProcessorInterface(self.domain)
        self.casedb = CaseAccessors(self.domain)
        self.formdb = FormAccessors(self.domain)

    def tearDown(self):
        FormProcessorTestUtils.delete_all_xforms(self.domain)
        FormProcessorTestUtils.delete_all_cases(self.domain)
        UnfinishedSubmissionStub.objects.all().delete()
        super(EditFormTest, self).tearDown()

    @run_with_all_backends
    def test_basic_edit(self):
        original_xml = self.get_xml('original')
        edit_xml = self.get_xml('edit')

        xform = post_xform(original_xml, domain=self.domain)

        self.assertEqual(self.ID, xform.form_id)
        self.assertTrue(xform.is_normal)
        self.assertEqual("", xform.form_data['vitals']['height'])
        self.assertEqual("other", xform.form_data['assessment']['categories'])

        xform = post_xform(edit_xml, domain=self.domain)
        self.assertEqual(self.ID, xform.form_id)
        self.assertTrue(xform.is_normal)
        self.assertEqual("100", xform.form_data['vitals']['height'])
        self.assertEqual("Edited Baby!", xform.form_data['assessment']['categories'])

        deprecated_xform = self.formdb.get_form(xform.deprecated_form_id)

        self.assertEqual(self.ID, deprecated_xform.orig_id)
        self.assertNotEqual(self.ID, deprecated_xform.form_id)
        self.assertTrue(deprecated_xform.is_deprecated)
        self.assertEqual("", deprecated_xform.form_data['vitals']['height'])
        self.assertEqual("other", deprecated_xform.form_data['assessment']['categories'])

        self.assertEqual(xform.received_on, deprecated_xform.received_on)
        self.assertEqual(xform.deprecated_form_id, deprecated_xform.form_id)
        self.assertTrue(xform.edited_on > deprecated_xform.received_on)

        self.assertEqual(
            deprecated_xform.get_xml(),
            original_xml
        )
        self.assertEqual(xform.get_xml(), edit_xml)

    @run_with_all_backends
    def test_broken_save(self):
        """
        Test that if the second form submission terminates unexpectedly
        and the main form isn't saved, then there are no side effects
        such as the original having been marked as deprecated.
        """

        original_xml = self.get_xml('original')
        edit_xml = self.get_xml('edit')

        _, xform, _ = submit_form_locally(original_xml, self.domain)
        self.assertEqual(self.ID, xform.form_id)
        self.assertTrue(xform.is_normal)
        self.assertEqual(self.domain, xform.domain)

        self.assertEqual(
            UnfinishedSubmissionStub.objects.filter(xform_id=self.ID).count(),
            0
        )

        with patch.object(self.interface.processor, 'save_processed_models', side_effect=RequestFailed):
            with self.assertRaises(RequestFailed):
                submit_form_locally(edit_xml, self.domain)

        # it didn't go through, so make sure there are no edits still
        self.assertIsNone(getattr(xform, 'deprecated_form_id', None))

        xform = self.formdb.get_form(self.ID)
        self.assertIsNotNone(xform)
        self.assertEqual(
            UnfinishedSubmissionStub.objects.filter(xform_id=self.ID,
                                                    saved=False).count(),
            1
        )
        self.assertEqual(
            UnfinishedSubmissionStub.objects.filter(xform_id=self.ID).count(),
            1
        )

    @run_with_all_backends
    def test_case_management(self):
        form_id = uuid.uuid4().hex
        case_id = uuid.uuid4().hex
        owner_id = uuid.uuid4().hex
        case_block = CaseBlock(
            create=True,
            case_id=case_id,
            case_type='person',
            owner_id=owner_id,
            update={
                'property': 'original value'
            }
        ).as_string()
        submit_case_blocks(case_block, domain=self.domain, form_id=form_id)

        # validate some assumptions
        case = self.casedb.get_case(case_id)
        self.assertEqual(case.type, 'person')
        self.assertEqual(case.dynamic_case_properties()['property'], 'original value')
        self.assertEqual([form_id], case.xform_ids)

        if not settings.TESTS_SHOULD_USE_SQL_BACKEND:
            self.assertEqual(2, len(case.actions))
            for a in case.actions:
                self.assertEqual(form_id, a.xform_id)

        # submit a new form with a different case update
        case_block = CaseBlock(
            create=True,
            case_id=case_id,
            case_type='newtype',
            owner_id=owner_id,
            update={
                'property': 'edited value'
            }
        ).as_string()
        submit_case_blocks(case_block, domain=self.domain, form_id=form_id)

        case = self.casedb.get_case(case_id)
        self.assertEqual(case.type, 'newtype')
        self.assertEqual(case.dynamic_case_properties()['property'], 'edited value')
        self.assertEqual([form_id], case.xform_ids)

        if not settings.TESTS_SHOULD_USE_SQL_BACKEND:
            self.assertEqual(2, len(case.actions))
            for a in case.actions:
                self.assertEqual(form_id, a.xform_id)

    @run_with_all_backends
    def test_second_edit_fails(self):
        form_id = uuid.uuid4().hex
        case_id = uuid.uuid4().hex
        case_block = CaseBlock(
            create=True,
            case_id=case_id,
            case_type='person',
        ).as_string()
        submit_case_blocks(case_block, domain=self.domain, form_id=form_id)

        # submit an edit form with a bad case update (for example a bad ID)
        case_block = CaseBlock(
            create=True,
            case_id='',
            case_type='person',
        ).as_string()
        submit_case_blocks(case_block, domain=self.domain, form_id=form_id)

        xform = self.formdb.get_form(form_id)
        self.assertTrue(xform.is_error)

        deprecated_xform = self.formdb.get_form(xform.deprecated_form_id)
        self.assertTrue(deprecated_xform.is_deprecated)

    @run_with_all_backends
    def test_case_management_ordering(self):
        case_id = uuid.uuid4().hex
        owner_id = uuid.uuid4().hex

        # create a case
        case_block = CaseBlock(
            create=True,
            case_id=case_id,
            case_type='person',
            owner_id=owner_id,
        ).as_string()
        create_form_id = submit_case_blocks(case_block, domain=self.domain)[0].form_id

        # validate that worked
        case = self.casedb.get_case(case_id)
        self.assertEqual([create_form_id], case.xform_ids)

        if not settings.TESTS_SHOULD_USE_SQL_BACKEND:
            self.assertTrue(create_form_id in [a.xform_id for a in case.actions])
            for a in case.actions:
                self.assertEqual(create_form_id, a.xform_id)

        edit_date = datetime.utcnow()
        # set some property value
        case_block = CaseBlock(
            create=False,
            case_id=case_id,
            date_modified=edit_date,
            update={
                'property': 'first value',
            }
        ).as_string()
        edit_form_id = submit_case_blocks(case_block, domain=self.domain)[0].form_id

        # validate that worked
        case = self.casedb.get_case(case_id)
        self.assertEqual(case.dynamic_case_properties()['property'], 'first value')
        self.assertEqual([create_form_id, edit_form_id], case.xform_ids)

        if not settings.TESTS_SHOULD_USE_SQL_BACKEND:
            self.assertTrue(all(
                form_id in [a.xform_id for a in case.actions]
                for form_id in [create_form_id, edit_form_id]
            ))

        # submit a second (new) form updating the value
        case_block = CaseBlock(
            create=False,
            case_id=case_id,
            update={
                'property': 'final value',
            }
        ).as_string()
        second_edit_form_id = submit_case_blocks(case_block, domain=self.domain)[0].form_id

        # validate that worked
        case = self.casedb.get_case(case_id)
        self.assertEqual(case.dynamic_case_properties()['property'], 'final value')
        self.assertEqual([create_form_id, edit_form_id, second_edit_form_id], case.xform_ids)

        if not settings.TESTS_SHOULD_USE_SQL_BACKEND:
            self.assertTrue(all(
                form_id in [a.xform_id for a in case.actions]
                for form_id in [create_form_id, edit_form_id, second_edit_form_id]
            ))

        # deprecate the middle edit
        case_block = CaseBlock(
            create=False,
            case_id=case_id,
            date_modified=edit_date,  # need to use the previous edit date for action sort comparisons
            update={
                'property': 'edited value',
                'added_property': 'added value',
            }
        ).as_string()
        submit_case_blocks(case_block, domain=self.domain, form_id=edit_form_id)

        # ensure that the middle edit stays in the right place and is applied
        # before the final one
        case = self.casedb.get_case(case_id)
        self.assertEqual(case.dynamic_case_properties()['property'], 'final value')
        self.assertEqual(case.dynamic_case_properties()['added_property'], 'added value')
        self.assertEqual([create_form_id, edit_form_id, second_edit_form_id], case.xform_ids)

        if not settings.TESTS_SHOULD_USE_SQL_BACKEND:
            self.assertEqual(
                [create_form_id, create_form_id, edit_form_id, second_edit_form_id],
                [a.xform_id for a in case.actions]
            )
<commit_msg>failing test for editing a form that has errors<commit_after>from datetime import datetime
import os
import uuid

from django.conf import settings
from django.test import TestCase
from mock import patch
from couchdbkit import RequestFailed
from casexml.apps.case.mock import CaseBlock
from corehq.apps.hqcase.utils import submit_case_blocks
from corehq.apps.receiverwrapper.util import submit_form_locally
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors, FormAccessors
from couchforms.models import UnfinishedSubmissionStub

from corehq.form_processor.interfaces.processor import FormProcessorInterface
from corehq.form_processor.tests.utils import FormProcessorTestUtils, run_with_all_backends, post_xform
from corehq.util.test_utils import TestFileMixin

class EditFormTest(TestCase, TestFileMixin):
    ID = '7H46J37FGH3'
    domain = 'test-form-edits'

    file_path = ('data', 'deprecation')
    root = os.path.dirname(__file__)

    def setUp(self):
        super(EditFormTest, self).setUp()
        self.interface = FormProcessorInterface(self.domain)
        self.casedb = CaseAccessors(self.domain)
        self.formdb = FormAccessors(self.domain)

    def tearDown(self):
        FormProcessorTestUtils.delete_all_xforms(self.domain)
        FormProcessorTestUtils.delete_all_cases(self.domain)
        UnfinishedSubmissionStub.objects.all().delete()
        super(EditFormTest, self).tearDown()

    @run_with_all_backends
    def test_basic_edit(self):
        original_xml = self.get_xml('original')
        edit_xml = self.get_xml('edit')

        xform = post_xform(original_xml, domain=self.domain)

        self.assertEqual(self.ID, xform.form_id)
        self.assertTrue(xform.is_normal)
        self.assertEqual("", xform.form_data['vitals']['height'])
        self.assertEqual("other", xform.form_data['assessment']['categories'])

        xform = post_xform(edit_xml, domain=self.domain)
        self.assertEqual(self.ID, xform.form_id)
        self.assertTrue(xform.is_normal)
        self.assertEqual("100", xform.form_data['vitals']['height'])
        self.assertEqual("Edited Baby!", xform.form_data['assessment']['categories'])

        deprecated_xform = self.formdb.get_form(xform.deprecated_form_id)

        self.assertEqual(self.ID, deprecated_xform.orig_id)
        self.assertNotEqual(self.ID, deprecated_xform.form_id)
        self.assertTrue(deprecated_xform.is_deprecated)
        self.assertEqual("", deprecated_xform.form_data['vitals']['height'])
        self.assertEqual("other", deprecated_xform.form_data['assessment']['categories'])

        self.assertEqual(xform.received_on, deprecated_xform.received_on)
        self.assertEqual(xform.deprecated_form_id, deprecated_xform.form_id)
        self.assertTrue(xform.edited_on > deprecated_xform.received_on)

        self.assertEqual(
            deprecated_xform.get_xml(),
            original_xml
        )
        self.assertEqual(xform.get_xml(), edit_xml)

    @run_with_all_backends
    def test_edit_an_error(self):
        form_id = uuid.uuid4().hex
        case_block = CaseBlock(
            create=True,
            case_id='',  # this should cause the submission to error
            case_type='person',
            owner_id='some-owner',
        )

        form, _ = submit_case_blocks(case_block.as_string(), domain=self.domain, form_id=form_id)
        self.assertTrue(form.is_error)
        self.assertTrue('IllegalCaseId' in form.problem)

        case_block.case_id = uuid.uuid4().hex
        form, _ = submit_case_blocks(case_block.as_string(), domain=self.domain, form_id=form_id)
        self.assertFalse(form.is_error)
        self.assertEqual(None, getattr(form, 'problem', None))

    @run_with_all_backends
    def test_broken_save(self):
        """
        Test that if the second form submission terminates unexpectedly
        and the main form isn't saved, then there are no side effects
        such as the original having been marked as deprecated.
        """

        original_xml = self.get_xml('original')
        edit_xml = self.get_xml('edit')

        _, xform, _ = submit_form_locally(original_xml, self.domain)
        self.assertEqual(self.ID, xform.form_id)
        self.assertTrue(xform.is_normal)
        self.assertEqual(self.domain, xform.domain)

        self.assertEqual(
            UnfinishedSubmissionStub.objects.filter(xform_id=self.ID).count(),
            0
        )

        with patch.object(self.interface.processor, 'save_processed_models', side_effect=RequestFailed):
            with self.assertRaises(RequestFailed):
                submit_form_locally(edit_xml, self.domain)

        # it didn't go through, so make sure there are no edits still
        self.assertIsNone(getattr(xform, 'deprecated_form_id', None))

        xform = self.formdb.get_form(self.ID)
        self.assertIsNotNone(xform)
        self.assertEqual(
            UnfinishedSubmissionStub.objects.filter(xform_id=self.ID,
                                                    saved=False).count(),
            1
        )
        self.assertEqual(
            UnfinishedSubmissionStub.objects.filter(xform_id=self.ID).count(),
            1
        )

    @run_with_all_backends
    def test_case_management(self):
        form_id = uuid.uuid4().hex
        case_id = uuid.uuid4().hex
        owner_id = uuid.uuid4().hex
        case_block = CaseBlock(
            create=True,
            case_id=case_id,
            case_type='person',
            owner_id=owner_id,
            update={
                'property': 'original value'
            }
        ).as_string()
        submit_case_blocks(case_block, domain=self.domain, form_id=form_id)

        # validate some assumptions
        case = self.casedb.get_case(case_id)
        self.assertEqual(case.type, 'person')
        self.assertEqual(case.dynamic_case_properties()['property'], 'original value')
        self.assertEqual([form_id], case.xform_ids)

        if not settings.TESTS_SHOULD_USE_SQL_BACKEND:
            self.assertEqual(2, len(case.actions))
            for a in case.actions:
                self.assertEqual(form_id, a.xform_id)

        # submit a new form with a different case update
        case_block = CaseBlock(
            create=True,
            case_id=case_id,
            case_type='newtype',
            owner_id=owner_id,
            update={
                'property': 'edited value'
            }
        ).as_string()
        submit_case_blocks(case_block, domain=self.domain, form_id=form_id)

        case = self.casedb.get_case(case_id)
        self.assertEqual(case.type, 'newtype')
        self.assertEqual(case.dynamic_case_properties()['property'], 'edited value')
        self.assertEqual([form_id], case.xform_ids)

        if not settings.TESTS_SHOULD_USE_SQL_BACKEND:
            self.assertEqual(2, len(case.actions))
            for a in case.actions:
                self.assertEqual(form_id, a.xform_id)

    @run_with_all_backends
    def test_second_edit_fails(self):
        form_id = uuid.uuid4().hex
        case_id = uuid.uuid4().hex
        case_block = CaseBlock(
            create=True,
            case_id=case_id,
            case_type='person',
        ).as_string()
        submit_case_blocks(case_block, domain=self.domain, form_id=form_id)

        # submit an edit form with a bad case update (for example a bad ID)
        case_block = CaseBlock(
            create=True,
            case_id='',
            case_type='person',
        ).as_string()
        submit_case_blocks(case_block, domain=self.domain, form_id=form_id)

        xform = self.formdb.get_form(form_id)
        self.assertTrue(xform.is_error)

        deprecated_xform = self.formdb.get_form(xform.deprecated_form_id)
        self.assertTrue(deprecated_xform.is_deprecated)

    @run_with_all_backends
    def test_case_management_ordering(self):
        case_id = uuid.uuid4().hex
        owner_id = uuid.uuid4().hex

        # create a case
        case_block = CaseBlock(
            create=True,
            case_id=case_id,
            case_type='person',
            owner_id=owner_id,
        ).as_string()
        create_form_id = submit_case_blocks(case_block, domain=self.domain)[0].form_id

        # validate that worked
        case = self.casedb.get_case(case_id)
        self.assertEqual([create_form_id], case.xform_ids)

        if not settings.TESTS_SHOULD_USE_SQL_BACKEND:
            self.assertTrue(create_form_id in [a.xform_id for a in case.actions])
            for a in case.actions:
                self.assertEqual(create_form_id, a.xform_id)

        edit_date = datetime.utcnow()
        # set some property value
        case_block = CaseBlock(
            create=False,
            case_id=case_id,
            date_modified=edit_date,
            update={
                'property': 'first value',
            }
        ).as_string()
        edit_form_id = submit_case_blocks(case_block, domain=self.domain)[0].form_id

        # validate that worked
        case = self.casedb.get_case(case_id)
        self.assertEqual(case.dynamic_case_properties()['property'], 'first value')
        self.assertEqual([create_form_id, edit_form_id], case.xform_ids)

        if not settings.TESTS_SHOULD_USE_SQL_BACKEND:
            self.assertTrue(all(
                form_id in [a.xform_id for a in case.actions]
                for form_id in [create_form_id, edit_form_id]
            ))

        # submit a second (new) form updating the value
        case_block = CaseBlock(
            create=False,
            case_id=case_id,
            update={
                'property': 'final value',
            }
        ).as_string()
        second_edit_form_id = submit_case_blocks(case_block, domain=self.domain)[0].form_id

        # validate that worked
        case = self.casedb.get_case(case_id)
        self.assertEqual(case.dynamic_case_properties()['property'], 'final value')
        self.assertEqual([create_form_id, edit_form_id, second_edit_form_id], case.xform_ids)

        if not settings.TESTS_SHOULD_USE_SQL_BACKEND:
            self.assertTrue(all(
                form_id in [a.xform_id for a in case.actions]
                for form_id in [create_form_id, edit_form_id, second_edit_form_id]
            ))

        # deprecate the middle edit
        case_block = CaseBlock(
            create=False,
            case_id=case_id,
            date_modified=edit_date,  # need to use the previous edit date for action sort comparisons
            update={
                'property': 'edited value',
                'added_property': 'added value',
            }
        ).as_string()
        submit_case_blocks(case_block, domain=self.domain, form_id=edit_form_id)

        # ensure that the middle edit stays in the right place and is applied
        # before the final one
        case = self.casedb.get_case(case_id)
        self.assertEqual(case.dynamic_case_properties()['property'], 'final value')
        self.assertEqual(case.dynamic_case_properties()['added_property'], 'added value')
        self.assertEqual([create_form_id, edit_form_id, second_edit_form_id], case.xform_ids)

        if not settings.TESTS_SHOULD_USE_SQL_BACKEND:
            self.assertEqual(
                [create_form_id, create_form_id, edit_form_id, second_edit_form_id],
                [a.xform_id for a in case.actions]
            )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import json
import logging
import os
import os.path
import shutil
import sys
import tempfile

from crawlcrunch.compat import GzipFile
from crawlcrunch.compat import StringIO
from crawlcrunch.tests import EXAMPLES_PATH
from crawlcrunch.tests import prepare_url_open
from crawlcrunch.tests import unittest

import mock

class ArgumentParserTests(unittest.TestCase):

    def setUp(self):
        self._old_err = sys.stderr
        sys.stderr = StringIO()

    def tearDown(self):
        sys.stderr = self._old_err

    def _make_one(self, *args):
        from crawlcrunch.scripts.cc_update import CCUpdateCommand
        effargs = ['crawlcrunch', ]
        effargs.extend(args)
        return CCUpdateCommand(effargs)

    def test_defaults(self):
        cmd = self._make_one('.')
        self.assertEqual(cmd.args.location, '.')
        self.assertEqual(cmd.args.classes, ['companies'])
        self.assertFalse(cmd.args.sql)

    def test_missing_argument(self):
        with self.assertRaises(SystemExit):
            self._make_one()

    def test_too_much_arguments(self):
        with self.assertRaises(SystemExit) as cm:
            cmd = self._make_one('one', 'two', )
        e = cm.exception
        self.assertEqual(e.code, 2)

    def test_non_existent_path(self):
        dst = os.path.join('non', 'existent', 'path', )
        with self.assertRaises(SystemExit) as cm:
            self._make_one(dst)
        e = cm.exception
        self.assertEqual(e.code, 2)
        err = sys.stderr.getvalue()
        self.assertTrue(
            err.endswith(
                "the directory 'non/existent/path' does not exist\n"\
                    .format(dst)))

    def test_verboseness_quietness_flags(self):
        cmd = self._make_one('-v', EXAMPLES_PATH['company_files_empty'])
        self.assertEqual(cmd.args.verbosity, logging.getLevelName('DEBUG'))
        cmd = self._make_one(EXAMPLES_PATH['company_files_empty'])
        self.assertEqual(cmd.args.verbosity, logging.getLevelName('INFO'))
        cmd = self._make_one('-q', EXAMPLES_PATH['company_files_empty'])
        self.assertEqual(cmd.args.verbosity, logging.getLevelName('WARNING'))
        cmd = self._make_one('-qq', EXAMPLES_PATH['company_files_empty'])
        self.assertEqual(cmd.args.verbosity, logging.getLevelName('ERROR'))
        cmd = self._make_one('-qqq', EXAMPLES_PATH['company_files_empty'])
        self.assertEqual(cmd.args.verbosity, logging.getLevelName('CRITICAL'))
        cmd = self._make_one('-qqvq', EXAMPLES_PATH['company_files_empty'])
        self.assertEqual(cmd.args.verbosity, logging.getLevelName('ERROR'))

    def test_passing_in_db_uri(self):
        cmd = self._make_one('--sql', 'sqlite:///:memory:')
        self.assertEqual(cmd.args.location, 'sqlite:///:memory:')

    def test_classes_flag(self):
        cmd = self._make_one('.', 'foo', 'bar')
        self.assertEqual(cmd.args.classes, ['foo', 'bar'])

class MainTests(unittest.TestCase):

    def setUp(self):
        self._old_err = sys.stderr
        sys.stderr = StringIO()

    def tearDown(self):
        sys.stderr = self._old_err

    def test_missing_argument(self):
        from crawlcrunch.scripts.cc_update import main
        with self.assertRaises(SystemExit) as cm:
            main(['cc_update'])
        self.assertEqual(cm.exception.code, 2)
        err = sys.stderr.getvalue()
        self.assertTrue(err.endswith('too few arguments\n'))

class IntegrationTests(unittest.TestCase):

    def setUp(self):
        self.tmpd = tempfile.mkdtemp()

    def tearDown(self):
        shutil.rmtree(self.tmpd)

    def _make_one(self, path):
        from crawlcrunch.scripts.cc_update import CCUpdateCommand
        return CCUpdateCommand(['cc_update', path])

    def test_with_wrong_class(self):
        from crawlcrunch.scripts.cc_update import main
        with self.assertRaises(ValueError) as cm:
            main(['cc_update', '-qqq', self.tmpd, 'no_such_class'])
        e = cm.exception
        self.assertEqual(len(e.args), 1)
        self.assertTrue(e.args[0].endswith("'no_such_class'"))

    @mock.patch('crawlcrunch.compat.urlopen')
    def test_on_empty_companies_list(self, url_open):
        url_return = (
            {'http://api.crunchbase.com/v/1/companies.js': []})
        prepare_url_open(url_open, url_return)
        from crawlcrunch.scripts.cc_update import main
        result = main(['cc_update', '-qqq', self.tmpd])
        self.assertEqual(result, 0)
        url_open.assert_called_once_with(
            'http://api.crunchbase.com/v/1/companies.js')
        listing = os.listdir(self.tmpd)
        self.assertEqual(listing, [])

    @mock.patch('crawlcrunch.compat.urlopen')
    def test_on_companies_list_with_elements(self, url_open):
        companies_url = 'http://api.crunchbase.com/v/1/companies.js'
        foo_url = 'http://api.crunchbase.com/v/1/company/foo.js'
        bar_url = 'http://api.crunchbase.com/v/1/company/bar.js'
        prepare_url_open(url_open,
                         {companies_url: [{'permalink': 'foo', },
                                          {'permalink': 'bar', }],
                          foo_url: ['some_foo'],
                          bar_url: ['some_bar']})
        cmd = self._make_one(self.tmpd)
        result = cmd.run()
        self.assertEqual(result, 0)
        url_open.assert_called_with(bar_url)
        listing = os.listdir(self.tmpd)
        listing.sort()
        self.assertEqual(listing, ['bar.json.gz',
                                   'foo.json.gz'])

        with GzipFile(os.path.join(self.tmpd,
                                   'bar.json.gz')) as fp:
            self.assertEqual(json.load(fp), ['some_bar', ])
        with GzipFile(os.path.join(self.tmpd,
                                   'foo.json.gz')) as fp:
            self.assertEqual(json.load(fp), ['some_foo', ])

if __name__ == '__main__':  # pragma: no cover
    unittest.main()
<commit_msg>test was wrong<commit_after># -*- coding: utf-8 -*-

import json
import logging
import os
import os.path
import shutil
import sys
import tempfile

from crawlcrunch.compat import GzipFile
from crawlcrunch.compat import StringIO
from crawlcrunch.tests import EXAMPLES_PATH
from crawlcrunch.tests import prepare_url_open
from crawlcrunch.tests import unittest

import mock

class ArgumentParserTests(unittest.TestCase):

    def setUp(self):
        self._old_err = sys.stderr
        sys.stderr = StringIO()

    def tearDown(self):
        sys.stderr = self._old_err

    def _make_one(self, *args):
        from crawlcrunch.scripts.cc_update import CCUpdateCommand
        effargs = ['crawlcrunch', ]
        effargs.extend(args)
        return CCUpdateCommand(effargs)

    def test_defaults(self):
        cmd = self._make_one('.')
        self.assertEqual(cmd.args.location, '.')
        self.assertEqual(cmd.args.classes, ['companies'])
        self.assertFalse(cmd.args.sql)

    def test_missing_argument(self):
        with self.assertRaises(SystemExit):
            self._make_one()

    def test_non_existent_path(self):
        dst = os.path.join('non', 'existent', 'path', )
        with self.assertRaises(SystemExit) as cm:
            self._make_one(dst)
        e = cm.exception
        self.assertEqual(e.code, 2)
        err = sys.stderr.getvalue()
        self.assertTrue(
            err.endswith(
                "the directory 'non/existent/path' does not exist\n"\
                    .format(dst)))

    def test_verboseness_quietness_flags(self):
        cmd = self._make_one('-v', EXAMPLES_PATH['company_files_empty'])
        self.assertEqual(cmd.args.verbosity, logging.getLevelName('DEBUG'))
        cmd = self._make_one(EXAMPLES_PATH['company_files_empty'])
        self.assertEqual(cmd.args.verbosity, logging.getLevelName('INFO'))
        cmd = self._make_one('-q', EXAMPLES_PATH['company_files_empty'])
        self.assertEqual(cmd.args.verbosity, logging.getLevelName('WARNING'))
        cmd = self._make_one('-qq', EXAMPLES_PATH['company_files_empty'])
        self.assertEqual(cmd.args.verbosity, logging.getLevelName('ERROR'))
        cmd = self._make_one('-qqq', EXAMPLES_PATH['company_files_empty'])
        self.assertEqual(cmd.args.verbosity, logging.getLevelName('CRITICAL'))
        cmd = self._make_one('-qqvq', EXAMPLES_PATH['company_files_empty'])
        self.assertEqual(cmd.args.verbosity, logging.getLevelName('ERROR'))

    def test_passing_in_db_uri(self):
        cmd = self._make_one('--sql', 'sqlite:///:memory:')
        self.assertEqual(cmd.args.location, 'sqlite:///:memory:')

    def test_classes_flag(self):
        cmd = self._make_one('.', 'foo', 'bar')
        self.assertEqual(cmd.args.classes, ['foo', 'bar'])

class MainTests(unittest.TestCase):

    def setUp(self):
        self._old_err = sys.stderr
        sys.stderr = StringIO()

    def tearDown(self):
        sys.stderr = self._old_err

    def test_missing_argument(self):
        from crawlcrunch.scripts.cc_update import main
        with self.assertRaises(SystemExit) as cm:
            main(['cc_update'])
        self.assertEqual(cm.exception.code, 2)
        err = sys.stderr.getvalue()
        self.assertTrue(err.endswith('too few arguments\n'))

class IntegrationTests(unittest.TestCase):

    def setUp(self):
        self.tmpd = tempfile.mkdtemp()

    def tearDown(self):
        shutil.rmtree(self.tmpd)

    def _make_one(self, path):
        from crawlcrunch.scripts.cc_update import CCUpdateCommand
        return CCUpdateCommand(['cc_update', path])

    def test_with_wrong_class(self):
        from crawlcrunch.scripts.cc_update import main
        with self.assertRaises(ValueError) as cm:
            main(['cc_update', '-qqq', self.tmpd, 'no_such_class'])
        e = cm.exception
        self.assertEqual(len(e.args), 1)
        self.assertTrue(e.args[0].endswith("'no_such_class'"))

    @mock.patch('crawlcrunch.compat.urlopen')
    def test_on_empty_companies_list(self, url_open):
        url_return = (
            {'http://api.crunchbase.com/v/1/companies.js': []})
        prepare_url_open(url_open, url_return)
        from crawlcrunch.scripts.cc_update import main
        result = main(['cc_update', '-qqq', self.tmpd])
        self.assertEqual(result, 0)
        url_open.assert_called_once_with(
            'http://api.crunchbase.com/v/1/companies.js')
        listing = os.listdir(self.tmpd)
        self.assertEqual(listing, [])

    @mock.patch('crawlcrunch.compat.urlopen')
    def test_on_companies_list_with_elements(self, url_open):
        companies_url = 'http://api.crunchbase.com/v/1/companies.js'
        foo_url = 'http://api.crunchbase.com/v/1/company/foo.js'
        bar_url = 'http://api.crunchbase.com/v/1/company/bar.js'
        prepare_url_open(url_open,
                         {companies_url: [{'permalink': 'foo', },
                                          {'permalink': 'bar', }],
                          foo_url: ['some_foo'],
                          bar_url: ['some_bar']})
        cmd = self._make_one(self.tmpd)
        result = cmd.run()
        self.assertEqual(result, 0)
        url_open.assert_called_with(bar_url)
        listing = os.listdir(self.tmpd)
        listing.sort()
        self.assertEqual(listing, ['bar.json.gz',
                                   'foo.json.gz'])

        with GzipFile(os.path.join(self.tmpd,
                                   'bar.json.gz')) as fp:
            self.assertEqual(json.load(fp), ['some_bar', ])
        with GzipFile(os.path.join(self.tmpd,
                                   'foo.json.gz')) as fp:
            self.assertEqual(json.load(fp), ['some_foo', ])

if __name__ == '__main__':  # pragma: no cover
    unittest.main()
<|endoftext|>


<commit_before># coding=utf-8
"""
All commands that call the Google API service.
"""
import re
import sys
from datetime import datetime, timedelta

import dateutil.parser

from ..connection import connect
from ..config import config

from .parser import subparsers

def quickadd(summary):
    service = connect()

    # Double up single-time events to be 0-length
    match = re.match(r'^\d\d:\d\d ', summary)
    if match:
        summary = match.group(0)[:-1] + '-' + summary

    # Make request
    print "Quick add >>", summary

    result = service.events().quickAdd(
        calendarId=config['calendar_id'],
        summary=summary
    ).execute()

    if result['status'] == 'confirmed':
        print "Added! Link: ", result['htmlLink']
        return True
    else:
        sys.stdout.write("Failed :( - status %s\n" % result['status'])
        return False

quickadd.parser = subparsers.add_parser('quickadd')
quickadd.parser.add_argument('summary')
quickadd.parser.set_defaults(func=quickadd)

def now(summary, offset=0, duration=None):
    if duration is None:
        duration = 0

    service = connect()

    start = datetime.now() + timedelta(minutes=offset)
    end = start + timedelta(minutes=duration)

    print "Adding %i-minute event >> %s" % (duration, summary)

    result = service.events().insert(
        calendarId=config['calendar_id'],
        body={
            'summary': summary,
            'start': {
                'dateTime': start.isoformat(),
                'timeZone': config['timezone']
            },
            'end': {
                'dateTime': end.isoformat(),
                'timeZone': config['timezone']
            }
        }
    ).execute()

    if result['status'] == 'confirmed':
        print "Added! Link: ", result['htmlLink']
        return True
    else:
        sys.stdout.write("Failed :( - status %s\n" % result['status'])
        return False

now.parser = subparsers.add_parser('now')
now.parser.add_argument('offset', type=int, default=0, nargs='?')
now.parser.add_argument('-d', '--duration', type=int)
now.parser.add_argument('summary')
now.parser.set_defaults(func=now)

def for_command(duration, summary):
    service = connect()

    times = [
        datetime.now(),
        datetime.now() + timedelta(minutes=duration)
    ]
    times.sort()
    start, end = times

    print "Adding %i-minute event >> %s" % (abs(duration), summary)

    result = service.events().insert(
        calendarId=config['calendar_id'],
        body={
            'summary': summary,
            'start': {
                'dateTime': start.isoformat(),
                'timeZone': config['timezone']
            },
            'end': {
                'dateTime': end.isoformat(),
                'timeZone': config['timezone']
            }
        }
    ).execute()

    if result['status'] == 'confirmed':
        print "Added! Link: ", result['htmlLink']
        return True
    else:
        sys.stdout.write("Failed :( - status %s\n" % result['status'])
        return False

for_command.parser = subparsers.add_parser('for')
for_command.parser.add_argument('duration', type=int)
for_command.parser.add_argument('summary')
for_command.parser.set_defaults(func=for_command)

def add(summary, when=None, duration=0):
    if when is None:
        when = datetime.now()
    else:
        when = dateutil.parser.parse(when)

    service = connect()

    times = [
        when,
        when + timedelta(minutes=duration)
    ]
    times.sort()
    start, end = times

    print "Adding %i-minute event at %s >> %s" % (abs(duration), when, summary)

    result = service.events().insert(
        calendarId=config['calendar_id'],
        body={
            'summary': summary,
            'start': {
                'dateTime': start.isoformat(),
                'timeZone': config['timezone']
            },
            'end': {
                'dateTime': end.isoformat(),
                'timeZone': config['timezone']
            }
        }
    ).execute()

    if result['status'] == 'confirmed':
        print "Added! Link: ", result['htmlLink']
        return True
    else:
        sys.stdout.write("Failed :( - status %s\n" % result['status'])
        return False

add.parser = subparsers.add_parser('add')
add.parser.add_argument('-w', '--when', default=None)
add.parser.add_argument('summary')
add.parser.set_defaults(func=add)

<commit_msg>Fix now command for the offset to be a commandline option<commit_after># coding=utf-8
"""
All commands that call the Google API service.
"""
import re
import sys
from datetime import datetime, timedelta

import dateutil.parser

from ..connection import connect
from ..config import config

from .parser import subparsers

def quickadd(summary):
    service = connect()

    # Double up single-time events to be 0-length
    match = re.match(r'^\d\d:\d\d ', summary)
    if match:
        summary = match.group(0)[:-1] + '-' + summary

    # Make request
    print "Quick add >>", summary

    result = service.events().quickAdd(
        calendarId=config['calendar_id'],
        summary=summary
    ).execute()

    if result['status'] == 'confirmed':
        print "Added! Link: ", result['htmlLink']
        return True
    else:
        sys.stdout.write("Failed :( - status %s\n" % result['status'])
        return False

quickadd.parser = subparsers.add_parser('quickadd')
quickadd.parser.add_argument('summary')
quickadd.parser.set_defaults(func=quickadd)

def now(summary, offset=None, duration=None):
    if offset is None:
        offset = 0

    if duration is None:
        duration = 0

    summary = ' '.join(summary)

    service = connect()

    start = datetime.now() + timedelta(minutes=offset)
    end = start + timedelta(minutes=duration)

    print "Adding %i-minute event >> %s" % (duration, summary)

    result = service.events().insert(
        calendarId=config['calendar_id'],
        body={
            'summary': summary,
            'start': {
                'dateTime': start.isoformat(),
                'timeZone': config['timezone']
            },
            'end': {
                'dateTime': end.isoformat(),
                'timeZone': config['timezone']
            }
        }
    ).execute()

    if result['status'] == 'confirmed':
        print "Added! Link: ", result['htmlLink']
        return True
    else:
        sys.stdout.write("Failed :( - status %s\n" % result['status'])
        return False

now.parser = subparsers.add_parser('now')
now.parser.add_argument('-o', '--offset', type=int)
now.parser.add_argument('-d', '--duration', type=int)
now.parser.add_argument('summary', nargs="+", type=unicode)
now.parser.set_defaults(func=now)

def for_command(duration, summary):
    service = connect()

    times = [
        datetime.now(),
        datetime.now() + timedelta(minutes=duration)
    ]
    times.sort()
    start, end = times

    print "Adding %i-minute event >> %s" % (abs(duration), summary)

    result = service.events().insert(
        calendarId=config['calendar_id'],
        body={
            'summary': summary,
            'start': {
                'dateTime': start.isoformat(),
                'timeZone': config['timezone']
            },
            'end': {
                'dateTime': end.isoformat(),
                'timeZone': config['timezone']
            }
        }
    ).execute()

    if result['status'] == 'confirmed':
        print "Added! Link: ", result['htmlLink']
        return True
    else:
        sys.stdout.write("Failed :( - status %s\n" % result['status'])
        return False

for_command.parser = subparsers.add_parser('for')
for_command.parser.add_argument('duration', type=int)
for_command.parser.add_argument('summary')
for_command.parser.set_defaults(func=for_command)

def add(summary, when=None, duration=0):
    if when is None:
        when = datetime.now()
    else:
        when = dateutil.parser.parse(when)

    service = connect()

    times = [
        when,
        when + timedelta(minutes=duration)
    ]
    times.sort()
    start, end = times

    print "Adding %i-minute event at %s >> %s" % (abs(duration), when, summary)

    result = service.events().insert(
        calendarId=config['calendar_id'],
        body={
            'summary': summary,
            'start': {
                'dateTime': start.isoformat(),
                'timeZone': config['timezone']
            },
            'end': {
                'dateTime': end.isoformat(),
                'timeZone': config['timezone']
            }
        }
    ).execute()

    if result['status'] == 'confirmed':
        print "Added! Link: ", result['htmlLink']
        return True
    else:
        sys.stdout.write("Failed :( - status %s\n" % result['status'])
        return False

add.parser = subparsers.add_parser('add')
add.parser.add_argument('-w', '--when', default=None)
add.parser.add_argument('summary')
add.parser.set_defaults(func=add)

<|endoftext|>


<commit_before>from __future__ import division
from __future__ import print_function
from builtins import bytes
from builtins import str
from builtins import range
from builtins import object
import binascii
import struct
import usb1
import os
from collections import deque
import time

class Icecube(object):
    def parseWavelengths(self, x):
        with open(x) as f:
            text = f.read()
        text = text.strip().split(",")
        wavelengths = [float(y) for y in text]
        return wavelengths

    def getAutonulling(self):
        # Somehow Flame moves the autonulling to post-production, so we have to do this
        _an = struct.unpack("<h", b"".join(self.getEEPROM(17)[4:6]))[0]
        return 65535/_an

    def getEEPROM(self, idx = None):
        # READ EEPROM
        """
            31 EEPROM values
            0  - Serial Number
            1  - 0th order Wavelength Calibration Coefficient
            2  - 1st order Wavelength Calibration Coefficient
            3  - 2nd order Wavelength Calibration Coefficient
            4  - 3rd order Wavelength Calibration Coefficient
            5  - Stray Light Constant
            6  - 0th order non-linearity correction coefficient
            7  - 1st order non-linearity correction coefficient
            8  - 2nd order non-linearity correction coefficient
            9  - 3rd order non-linearity correction coefficient
            10 - 4th order non-linearity correction coefficient
            11 - 5th order non-linearity correction coefficient
            12 - 6th order non-linearity correction coefficient
            13 - 7th order non-linearity correction coefficient
            14 - Polynomial order of non-linearity calibration
            15 - Optical Bench Configuration: gg ff sss
                gg  = Grating #
                fff = filter wavelengths
                sss = slit size
                https://oceanoptics.com/product-details/qe-pro-custom-configured-gratings-and-wavelength-range/
            16 - USB4000 configurations: AWL V
                A = Array coating Mfg
                W = Array wavelength (VIS, UV, OFLV)
                L = L2 Lens installed
                V = CPLD Version
            17 - Autonulling information
            18 - Power-up baud Rate Value
            19-30 - User-configured
        """
        _eeprom = list()
        _end = {
            "USB4000" : 31,
            "FLAME-S" : 20
        }
        self.eepromEnd = _end[self.type]
        if idx is None:
            for i in range(0, self.eepromEnd):
                _u = self.getSingleEEPROM(i)
                _eeprom.append(_u)
            return _eeprom
        else:
            return self.getSingleEEPROM(idx)

    def getSingleEEPROM(self, i):
        # print "Reading Byte " + str(i)
        self.icecube.bulkWrite(self.endpoints["EP1OUT"], b'\x05' + bytearray((i,)))
        data = self.icecube.bulkRead(self.endpoints["EP1OUT"], 17)
        # bytes = [data[i:i+2] for i in xrange(0, len(data), 2)]
        retCount = len(data)
        # unpacked = struct.unpack('<' + str(retCount) +'c', data)
        # if len(unpacked) >= 2: unpacked = unpacked[2:]
        # unpacked = deque(unpacked)                                       # Change the format to deque to pop from front
        unpacked = deque(struct.unpack('<' + str(retCount) +'c', data))
        if len(unpacked) > 2: unpacked.popleft(); unpacked.popleft();      # We discard the \x05 and Byte Number
        _u = list()

        while unpacked and unpacked[0] != b'\x00':                         # Discard everything after \x00
            _u.append(unpacked.popleft())

        try:
            _u = b"".join(_u).decode('ascii')
            _u = float(_u)
        except (UnicodeDecodeError, ValueError) as e:
            pass

        return _u

    def getTemp(self):
        self.icecube.bulkWrite(1, b'\x6C')
        data = self.icecube.bulkRead(1, 3)
        startByte = struct.unpack('<c', data[0:1])[0]
        if startByte == b'\x08':
            # Read successful
            ADC = struct.unpack('<h', data[1:])[0]
            return 0.003906 * ADC
        else:
            raise Exception('ADC Read unsuccessful')

    """ integrationTime is in milliseconds for convenience """
    def setIntegrationTime(self, x):
        t = struct.pack('<I', int(x * 1000))
        self.icecube.bulkWrite(1, b'\x02'+t)

    def getSpectra(self):
        data = []
        _data = []
        self.icecube.bulkWrite(self.endpoints["EP1OUT"], b'\x09')
        if self.type == "USB4000":
            for _ in range(4):
            	data.append(self.icecube.bulkRead(self.endpoints["EP6IN"], 512))
            for _ in range(11):
            	data.append(self.icecube.bulkRead(self.endpoints["EP2IN"], 512))
        else:
            for _ in range(8):
                data.append(self.icecube.bulkRead(self.endpoints["EP2IN"], 512))
        # Synchronization Byte
        self.icecube.bulkRead(self.endpoints["EP2IN"], 1)

        for j in range(len(data)):
            for i in range(256):
                x = data[j][2*i:(i+1)*2]
                _data.append(struct.unpack('<h', x)[0])
        return [(self.wavelengths[i], self.autonulling * _data[i]) for i in range(len(self.wavelengths))]

    def __enter__(self):
        dir_path = os.path.dirname(os.path.realpath(__file__))
        context = usb1.USBContext()

        # EP1OUT, EP2IN, EP6IN, EP1IN\
        self.endpoints = {
            "EP1OUT": 1,
            "EP2IN" : int("82", 16),
            "EP6IN" : int("86", 16),
            "EP1IN" : int("81", 16)
        }

        # Checking type
        self.icecube = context.openByVendorIDAndProductID(0x2457, 0x1022, skip_on_error=True)

        if self.icecube is not None:
            self.type = "USB4000"
            self.autonulling = 1
            wavelengths = self.parseWavelengths(dir_path + "/.USB4000.wv")
        else:
            self.icecube = context.openByVendorIDAndProductID(0x2457, 0x101e, skip_on_error=True)
            if self.icecube is not None:
                self.type = "FLAME-S"
                wavelengths = self.parseWavelengths(dir_path + "/.FLAME-S.wv")
            else:
                print("Unable to find appropriate spectrometer")
                exit(1)

        self.icecube.claimInterface(0)
        self.icecube.bulkWrite(self.endpoints["EP1OUT"], b'\x01') # init message

        if self.type == "FLAME-S":
            # print("Apparently we need 3 seconds to start up")
            # time.sleep(3) # wait for it to startup
            self.autonulling = self.getAutonulling()

        self.wavelengths = wavelengths

        return self

    def __exit__(self, type, value, traceback):
        self.icecube.releaseInterface(0)
        self.icecube.close()

if __name__ == '__main__':
    a = Icecube()
    #print a.getSpectra()
    #a.close()
<commit_msg>add comment about autonulling<commit_after>from __future__ import division
from __future__ import print_function
from builtins import bytes
from builtins import str
from builtins import range
from builtins import object
import binascii
import struct
import usb1
import os
from collections import deque
import time

class Icecube(object):
    def parseWavelengths(self, x):
        with open(x) as f:
            text = f.read()
        text = text.strip().split(",")
        wavelengths = [float(y) for y in text]
        return wavelengths

    def getAutonulling(self):
        # Somehow Flame moves the autonulling to post-production, so we have to do this
        _an = struct.unpack("<h", b"".join(self.getEEPROM(17)[4:6]))[0]
        return 65535/_an

    def getEEPROM(self, idx = None):
        # READ EEPROM
        """
            31 EEPROM values
            0  - Serial Number
            1  - 0th order Wavelength Calibration Coefficient
            2  - 1st order Wavelength Calibration Coefficient
            3  - 2nd order Wavelength Calibration Coefficient
            4  - 3rd order Wavelength Calibration Coefficient
            5  - Stray Light Constant
            6  - 0th order non-linearity correction coefficient
            7  - 1st order non-linearity correction coefficient
            8  - 2nd order non-linearity correction coefficient
            9  - 3rd order non-linearity correction coefficient
            10 - 4th order non-linearity correction coefficient
            11 - 5th order non-linearity correction coefficient
            12 - 6th order non-linearity correction coefficient
            13 - 7th order non-linearity correction coefficient
            14 - Polynomial order of non-linearity calibration
            15 - Optical Bench Configuration: gg ff sss
                gg  = Grating #
                fff = filter wavelengths
                sss = slit size
                https://oceanoptics.com/product-details/qe-pro-custom-configured-gratings-and-wavelength-range/
            16 - USB4000 configurations: AWL V
                A = Array coating Mfg
                W = Array wavelength (VIS, UV, OFLV)
                L = L2 Lens installed
                V = CPLD Version
            17 - Autonulling information
            18 - Power-up baud Rate Value
            19-30 - User-configured
        """
        _eeprom = list()
        _end = {
            "USB4000" : 31,
            "FLAME-S" : 20
        }
        self.eepromEnd = _end[self.type]
        if idx is None:
            for i in range(0, self.eepromEnd):
                _u = self.getSingleEEPROM(i)
                _eeprom.append(_u)
            return _eeprom
        else:
            return self.getSingleEEPROM(idx)

    def getSingleEEPROM(self, i):
        # print "Reading Byte " + str(i)
        self.icecube.bulkWrite(self.endpoints["EP1OUT"], b'\x05' + bytearray((i,)))
        data = self.icecube.bulkRead(self.endpoints["EP1OUT"], 17)
        # bytes = [data[i:i+2] for i in xrange(0, len(data), 2)]
        retCount = len(data)
        # unpacked = struct.unpack('<' + str(retCount) +'c', data)
        # if len(unpacked) >= 2: unpacked = unpacked[2:]
        # unpacked = deque(unpacked)                                       # Change the format to deque to pop from front
        unpacked = deque(struct.unpack('<' + str(retCount) +'c', data))
        if len(unpacked) > 2: unpacked.popleft(); unpacked.popleft();      # We discard the \x05 and Byte Number
        _u = list()

        while unpacked and unpacked[0] != b'\x00':                         # Discard everything after \x00
            _u.append(unpacked.popleft())

        try:
            _u = b"".join(_u).decode('ascii')
            _u = float(_u)
        except (UnicodeDecodeError, ValueError) as e:
            pass

        return _u

    def getTemp(self):
        self.icecube.bulkWrite(1, b'\x6C')
        data = self.icecube.bulkRead(1, 3)
        startByte = struct.unpack('<c', data[0:1])[0]
        if startByte == b'\x08':
            # Read successful
            ADC = struct.unpack('<h', data[1:])[0]
            return 0.003906 * ADC
        else:
            raise Exception('ADC Read unsuccessful')

    """ integrationTime is in milliseconds for convenience """
    def setIntegrationTime(self, x):
        t = struct.pack('<I', int(x * 1000))
        self.icecube.bulkWrite(1, b'\x02'+t)

    def getSpectra(self):
        """
        We auto multiply the autonulling factor here
        """

        data = []
        _data = []
        self.icecube.bulkWrite(self.endpoints["EP1OUT"], b'\x09')
        if self.type == "USB4000":
            for _ in range(4):
            	data.append(self.icecube.bulkRead(self.endpoints["EP6IN"], 512))
            for _ in range(11):
            	data.append(self.icecube.bulkRead(self.endpoints["EP2IN"], 512))
        else:
            for _ in range(8):
                data.append(self.icecube.bulkRead(self.endpoints["EP2IN"], 512))
        # Synchronization Byte
        self.icecube.bulkRead(self.endpoints["EP2IN"], 1)

        for j in range(len(data)):
            for i in range(256):
                x = data[j][2*i:(i+1)*2]
                _data.append(struct.unpack('<h', x)[0])
        return [(self.wavelengths[i], self.autonulling * _data[i]) for i in range(len(self.wavelengths))]

    def __enter__(self):
        dir_path = os.path.dirname(os.path.realpath(__file__))
        context = usb1.USBContext()

        # EP1OUT, EP2IN, EP6IN, EP1IN\
        self.endpoints = {
            "EP1OUT": 1,
            "EP2IN" : int("82", 16),
            "EP6IN" : int("86", 16),
            "EP1IN" : int("81", 16)
        }

        # Checking type
        self.icecube = context.openByVendorIDAndProductID(0x2457, 0x1022, skip_on_error=True)

        if self.icecube is not None:
            self.type = "USB4000"
            self.autonulling = 1
            wavelengths = self.parseWavelengths(dir_path + "/.USB4000.wv")
        else:
            self.icecube = context.openByVendorIDAndProductID(0x2457, 0x101e, skip_on_error=True)
            if self.icecube is not None:
                self.type = "FLAME-S"
                wavelengths = self.parseWavelengths(dir_path + "/.FLAME-S.wv")
            else:
                print("Unable to find appropriate spectrometer")
                exit(1)

        self.icecube.claimInterface(0)
        self.icecube.bulkWrite(self.endpoints["EP1OUT"], b'\x01') # init message

        if self.type == "FLAME-S":
            # print("Apparently we need 3 seconds to start up")
            # time.sleep(3) # wait for it to startup
            self.autonulling = self.getAutonulling()

        self.wavelengths = wavelengths

        return self

    def __exit__(self, type, value, traceback):
        self.icecube.releaseInterface(0)
        self.icecube.close()

if __name__ == '__main__':
    a = Icecube()
    #print a.getSpectra()
    #a.close()
<|endoftext|>


<commit_before>"""
* Copyright 2008 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http:#www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
"""

def addNamespace():
    if not doc().namespaces.has_key("v"):
        doc().namespaces.add("v", "urn:schemas-microsoft-com:vml")
        doc().createStyleSheet().cssText = "v\\:*{behavior:url(#default#VML);}"

<commit_msg>gwt canvas mshtml update<commit_after>"""
* Copyright 2008 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http:#www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
"""

def addNamespace():
    print doc().namespaces.item("v")
    if not doc().namespaces.item("v"):
        doc().namespaces.add("v", "urn:schemas-microsoft-com:vml")
        doc().createStyleSheet().cssText = "v\\:*{behavior:url(#default#VML);}"

<|endoftext|>


<commit_before>#!/usr/bin/env python3

from testUtils import Utils
from Cluster import Cluster, PFSetupPolicy
from TestHelper import TestHelper
from WalletMgr import WalletMgr
from Node import Node

import signal
import json
import time
import os
from os.path import join, exists
from datetime import datetime

###############################################################
# nodeos_multiple_version_protocol_feature_test
#
# Test for verifying that older versions of nodeos can work with newer versions of nodeos.
#
###############################################################

# Parse command line arguments
args = TestHelper.parse_args({"-v","--clean-run","--dump-error-details","--leave-running",
                              "--keep-logs", "--alternate-version-labels-file"})
Utils.Debug=args.v
killAll=args.clean_run
dumpErrorDetails=args.dump_error_details
dontKill=args.leave_running
killEosInstances=not dontKill
killWallet=not dontKill
keepLogs=args.keep_logs
alternateVersionLabelsFile=args.alternate_version_labels_file

walletMgr=WalletMgr(True)
cluster=Cluster(walletd=True)
cluster.setWalletMgr(walletMgr)

def restartNode(node: Node, chainArg=None, addSwapFlags=None, nodeosPath=None):
    if not node.killed:
        node.kill(signal.SIGTERM)
    isRelaunchSuccess = node.relaunch(chainArg, addSwapFlags=addSwapFlags,
                                      timeout=5, cachePopen=True, nodeosPath=nodeosPath)
    assert isRelaunchSuccess, "Fail to relaunch"

def shouldNodeContainPreactivateFeature(node):
    preactivateFeatureDigest = node.getSupportedProtocolFeatureDict()["PREACTIVATE_FEATURE"]["feature_digest"]
    assert preactivateFeatureDigest
    blockHeaderState = node.getLatestBlockHeaderState()
    activatedProtocolFeatures = blockHeaderState["activated_protocol_features"]["protocol_features"]
    return preactivateFeatureDigest in activatedProtocolFeatures

def waitUntilBeginningOfProdTurn(node, producerName, timeout=30, sleepTime=0.4):
    def isDesiredProdTurn():
        headBlockNum = node.getHeadBlockNum()
        res =  node.getBlock(headBlockNum)["producer"] == producerName and \
               node.getBlock(headBlockNum-1)["producer"] != producerName
        return res
    Utils.waitForTruth(isDesiredProdTurn, timeout, sleepTime)

def waitForOneRound():
    time.sleep(24) # We have 4 producers for this test

def setValidityOfActTimeSubjRestriction(node, codename, valid):
    invalidActTimeSubjRestriction = {
        "earliest_allowed_activation_time": "2030-01-01T00:00:00.000",
    }
    validActTimeSubjRestriction = {
        "earliest_allowed_activation_time": "1970-01-01T00:00:00.000",
    }
    actTimeSubjRestriction = validActTimeSubjRestriction if valid else invalidActTimeSubjRestriction
    node.modifyBuiltinPFSubjRestrictions(codename, actTimeSubjRestriction)
    restartNode(node)

def waitUntilBlockBecomeIrr(node, blockNum, timeout=60):
    def hasBlockBecomeIrr():
        return node.getIrreversibleBlockNum() >= blockNum
    return Utils.waitForTruth(hasBlockBecomeIrr, timeout)

# List to contain the test result message
testSuccessful = False
try:
    TestHelper.printSystemInfo("BEGIN")
    cluster.killall(allInstances=killAll)
    cluster.cleanup()

    # Create a cluster of 4 nodes, each node has 1 producer. The first 3 nodes use the latest vesion,
    # While the 4th node use the version that doesn't support protocol feature activation (i.e. 1.7.0)
    associatedNodeLabels = {
        "3": "170"
    }
    Utils.Print("Alternate Version Labels File is {}".format(alternateVersionLabelsFile))
    assert exists(alternateVersionLabelsFile), "Alternate version labels file does not exist"
    # version 1.7 did not provide a default value for "--last-block-time-offset-us" so this is needed to
    # avoid dropping late blocks
    assert cluster.launch(pnodes=4, totalNodes=4, prodCount=1, totalProducers=4,
                          extraNodeosArgs=" --plugin eosio::producer_api_plugin ",
                          useBiosBootFile=False,
                          specificExtraNodeosArgs={
                             0:"--http-max-response-time-ms 990000",
                             1:"--http-max-response-time-ms 990000",
                             2:"--http-max-response-time-ms 990000",
                             3:"--last-block-time-offset-us -200000"},
                          onlySetProds=True,
                          pfSetupPolicy=PFSetupPolicy.NONE,
                          alternateVersionLabelsFile=alternateVersionLabelsFile,
                          associatedNodeLabels=associatedNodeLabels), "Unable to launch cluster"

    newNodeIds = [0, 1, 2]
    oldNodeId = 3
    newNodes = list(map(lambda id: cluster.getNode(id), newNodeIds))
    oldNode = cluster.getNode(oldNodeId)
    allNodes = [*newNodes, oldNode]

    def pauseBlockProductions():
        for node in allNodes:
            if not node.killed: node.processCurlCmd("producer", "pause", "")

    def resumeBlockProductions():
        for node in allNodes:
            if not node.killed: node.processCurlCmd("producer", "resume", "")

    def areNodesInSync(nodes:[Node]):
        # Pause all block production to ensure the head is not moving
        pauseBlockProductions()
        time.sleep(2) # Wait for some time to ensure all blocks are propagated
        headBlockIds = []
        for node in nodes:
            headBlockId = node.getInfo()["head_block_id"]
            headBlockIds.append(headBlockId)
        resumeBlockProductions()
        return len(set(headBlockIds)) == 1

    # Before everything starts, all nodes (new version and old version) should be in sync
    assert areNodesInSync(allNodes), "Nodes are not in sync before preactivation"

    # First, we are going to test the case where:
    # - 1st node has valid earliest_allowed_activation_time
    # - While 2nd and 3rd node have invalid earliest_allowed_activation_time
    # Producer in the 1st node is going to activate PREACTIVATE_FEATURE during his turn
    # Immediately, in the next block PREACTIVATE_FEATURE should be active in 1st node, but not on 2nd and 3rd
    # Therefore, 1st node will be out of sync with 2nd, 3rd, and 4th node
    # After a round has passed though, 1st node will realize he's in minority fork and then join the other nodes
    # Hence, the PREACTIVATE_FEATURE that was previously activated will be dropped and all of the nodes should be in sync
    setValidityOfActTimeSubjRestriction(newNodes[1], "PREACTIVATE_FEATURE", False)
    setValidityOfActTimeSubjRestriction(newNodes[2], "PREACTIVATE_FEATURE", False)

    waitUntilBeginningOfProdTurn(newNodes[0], "defproducera")
    newNodes[0].activatePreactivateFeature()
    assert shouldNodeContainPreactivateFeature(newNodes[0]), "1st node should contain PREACTIVATE FEATURE"
    assert not (shouldNodeContainPreactivateFeature(newNodes[1]) or shouldNodeContainPreactivateFeature(newNodes[2])), \
           "2nd and 3rd node should not contain PREACTIVATE FEATURE"
    assert areNodesInSync([newNodes[1], newNodes[2], oldNode]), "2nd, 3rd and 4th node should be in sync"
    assert not areNodesInSync(allNodes), "1st node should be out of sync with the rest nodes"

    waitForOneRound()

    assert not shouldNodeContainPreactivateFeature(newNodes[0]), "PREACTIVATE_FEATURE should be dropped"
    assert areNodesInSync(allNodes), "All nodes should be in sync"

    # Then we set the earliest_allowed_activation_time of 2nd node and 3rd node with valid value
    # Once the 1st node activate PREACTIVATE_FEATURE, all of them should have PREACTIVATE_FEATURE activated in the next block
    # They will be in sync and their LIB will advance since they control > 2/3 of the producers
    # Also the LIB should be able to advance past the block that contains PREACTIVATE_FEATURE
    # However, the 4th node will be out of sync with them, and its LIB will stuck
    setValidityOfActTimeSubjRestriction(newNodes[1], "PREACTIVATE_FEATURE", True)
    setValidityOfActTimeSubjRestriction(newNodes[2], "PREACTIVATE_FEATURE", True)

    waitUntilBeginningOfProdTurn(newNodes[0], "defproducera")
    libBeforePreactivation = newNodes[0].getIrreversibleBlockNum()
    newNodes[0].activatePreactivateFeature()

    assert areNodesInSync(newNodes), "New nodes should be in sync"
    assert not areNodesInSync(allNodes), "Nodes should not be in sync after preactivation"
    for node in newNodes: assert shouldNodeContainPreactivateFeature(node), "New node should contain PREACTIVATE_FEATURE"

    activatedBlockNum = newNodes[0].getHeadBlockNum() # The PREACTIVATE_FEATURE should have been activated before or at this block num
    assert waitUntilBlockBecomeIrr(newNodes[0], activatedBlockNum), \
           "1st node LIB should be able to advance past the block that contains PREACTIVATE_FEATURE"
    assert newNodes[1].getIrreversibleBlockNum() >= activatedBlockNum and \
           newNodes[2].getIrreversibleBlockNum() >= activatedBlockNum, \
           "2nd and 3rd node LIB should also be able to advance past the block that contains PREACTIVATE_FEATURE"
    assert oldNode.getIrreversibleBlockNum() <= libBeforePreactivation, \
           "4th node LIB should stuck on LIB before PREACTIVATE_FEATURE is activated"

    # Restart old node with newest version
    # Before we are migrating to new version, use --export-reversible-blocks as the old version
    # and --import-reversible-blocks with the new version to ensure the compatibility of the reversible blocks
    # Finally, when we restart the 4th node with the version of nodeos that supports protocol feature,
    # all nodes should be in sync, and the 4th node will also contain PREACTIVATE_FEATURE
    portableRevBlkPath = os.path.join(Utils.getNodeDataDir(oldNodeId), "rev_blk_portable_format")
    oldNode.kill(signal.SIGTERM)
    # Note, for the following relaunch, these will fail to relaunch immediately (expected behavior of export/import), so the chainArg will not replace the old cmd
    oldNode.relaunch(oldNodeId, chainArg="--export-reversible-blocks {}".format(portableRevBlkPath), timeout=1)
    oldNode.relaunch(oldNodeId, chainArg="--import-reversible-blocks {}".format(portableRevBlkPath), timeout=1, nodeosPath="programs/nodeos/nodeos")
    os.remove(portableRevBlkPath)

    restartNode(oldNode, oldNodeId, chainArg="--replay", nodeosPath="programs/nodeos/nodeos")
    time.sleep(2) # Give some time to replay

    assert areNodesInSync(allNodes), "All nodes should be in sync"
    assert shouldNodeContainPreactivateFeature(oldNode), "4th node should contain PREACTIVATE_FEATURE"

    testSuccessful = True
finally:
    TestHelper.shutdown(cluster, walletMgr, testSuccessful, killEosInstances, killWallet, keepLogs, killAll, dumpErrorDetails)

exitCode = 0 if testSuccessful else 1
exit(exitCode)
<commit_msg>Missed some methods that were still passing nodeId.<commit_after>#!/usr/bin/env python3

from testUtils import Utils
from Cluster import Cluster, PFSetupPolicy
from TestHelper import TestHelper
from WalletMgr import WalletMgr
from Node import Node

import signal
import json
import time
import os
from os.path import join, exists
from datetime import datetime

###############################################################
# nodeos_multiple_version_protocol_feature_test
#
# Test for verifying that older versions of nodeos can work with newer versions of nodeos.
#
###############################################################

# Parse command line arguments
args = TestHelper.parse_args({"-v","--clean-run","--dump-error-details","--leave-running",
                              "--keep-logs", "--alternate-version-labels-file"})
Utils.Debug=args.v
killAll=args.clean_run
dumpErrorDetails=args.dump_error_details
dontKill=args.leave_running
killEosInstances=not dontKill
killWallet=not dontKill
keepLogs=args.keep_logs
alternateVersionLabelsFile=args.alternate_version_labels_file

walletMgr=WalletMgr(True)
cluster=Cluster(walletd=True)
cluster.setWalletMgr(walletMgr)

def restartNode(node: Node, chainArg=None, addSwapFlags=None, nodeosPath=None):
    if not node.killed:
        node.kill(signal.SIGTERM)
    isRelaunchSuccess = node.relaunch(chainArg, addSwapFlags=addSwapFlags,
                                      timeout=5, cachePopen=True, nodeosPath=nodeosPath)
    assert isRelaunchSuccess, "Fail to relaunch"

def shouldNodeContainPreactivateFeature(node):
    preactivateFeatureDigest = node.getSupportedProtocolFeatureDict()["PREACTIVATE_FEATURE"]["feature_digest"]
    assert preactivateFeatureDigest
    blockHeaderState = node.getLatestBlockHeaderState()
    activatedProtocolFeatures = blockHeaderState["activated_protocol_features"]["protocol_features"]
    return preactivateFeatureDigest in activatedProtocolFeatures

def waitUntilBeginningOfProdTurn(node, producerName, timeout=30, sleepTime=0.4):
    def isDesiredProdTurn():
        headBlockNum = node.getHeadBlockNum()
        res =  node.getBlock(headBlockNum)["producer"] == producerName and \
               node.getBlock(headBlockNum-1)["producer"] != producerName
        return res
    Utils.waitForTruth(isDesiredProdTurn, timeout, sleepTime)

def waitForOneRound():
    time.sleep(24) # We have 4 producers for this test

def setValidityOfActTimeSubjRestriction(node, codename, valid):
    invalidActTimeSubjRestriction = {
        "earliest_allowed_activation_time": "2030-01-01T00:00:00.000",
    }
    validActTimeSubjRestriction = {
        "earliest_allowed_activation_time": "1970-01-01T00:00:00.000",
    }
    actTimeSubjRestriction = validActTimeSubjRestriction if valid else invalidActTimeSubjRestriction
    node.modifyBuiltinPFSubjRestrictions(codename, actTimeSubjRestriction)
    restartNode(node)

def waitUntilBlockBecomeIrr(node, blockNum, timeout=60):
    def hasBlockBecomeIrr():
        return node.getIrreversibleBlockNum() >= blockNum
    return Utils.waitForTruth(hasBlockBecomeIrr, timeout)

# List to contain the test result message
testSuccessful = False
try:
    TestHelper.printSystemInfo("BEGIN")
    cluster.killall(allInstances=killAll)
    cluster.cleanup()

    # Create a cluster of 4 nodes, each node has 1 producer. The first 3 nodes use the latest vesion,
    # While the 4th node use the version that doesn't support protocol feature activation (i.e. 1.7.0)
    associatedNodeLabels = {
        "3": "170"
    }
    Utils.Print("Alternate Version Labels File is {}".format(alternateVersionLabelsFile))
    assert exists(alternateVersionLabelsFile), "Alternate version labels file does not exist"
    # version 1.7 did not provide a default value for "--last-block-time-offset-us" so this is needed to
    # avoid dropping late blocks
    assert cluster.launch(pnodes=4, totalNodes=4, prodCount=1, totalProducers=4,
                          extraNodeosArgs=" --plugin eosio::producer_api_plugin ",
                          useBiosBootFile=False,
                          specificExtraNodeosArgs={
                             0:"--http-max-response-time-ms 990000",
                             1:"--http-max-response-time-ms 990000",
                             2:"--http-max-response-time-ms 990000",
                             3:"--last-block-time-offset-us -200000"},
                          onlySetProds=True,
                          pfSetupPolicy=PFSetupPolicy.NONE,
                          alternateVersionLabelsFile=alternateVersionLabelsFile,
                          associatedNodeLabels=associatedNodeLabels), "Unable to launch cluster"

    newNodeIds = [0, 1, 2]
    oldNodeId = 3
    newNodes = list(map(lambda id: cluster.getNode(id), newNodeIds))
    oldNode = cluster.getNode(oldNodeId)
    allNodes = [*newNodes, oldNode]

    def pauseBlockProductions():
        for node in allNodes:
            if not node.killed: node.processCurlCmd("producer", "pause", "")

    def resumeBlockProductions():
        for node in allNodes:
            if not node.killed: node.processCurlCmd("producer", "resume", "")

    def areNodesInSync(nodes:[Node]):
        # Pause all block production to ensure the head is not moving
        pauseBlockProductions()
        time.sleep(2) # Wait for some time to ensure all blocks are propagated
        headBlockIds = []
        for node in nodes:
            headBlockId = node.getInfo()["head_block_id"]
            headBlockIds.append(headBlockId)
        resumeBlockProductions()
        return len(set(headBlockIds)) == 1

    # Before everything starts, all nodes (new version and old version) should be in sync
    assert areNodesInSync(allNodes), "Nodes are not in sync before preactivation"

    # First, we are going to test the case where:
    # - 1st node has valid earliest_allowed_activation_time
    # - While 2nd and 3rd node have invalid earliest_allowed_activation_time
    # Producer in the 1st node is going to activate PREACTIVATE_FEATURE during his turn
    # Immediately, in the next block PREACTIVATE_FEATURE should be active in 1st node, but not on 2nd and 3rd
    # Therefore, 1st node will be out of sync with 2nd, 3rd, and 4th node
    # After a round has passed though, 1st node will realize he's in minority fork and then join the other nodes
    # Hence, the PREACTIVATE_FEATURE that was previously activated will be dropped and all of the nodes should be in sync
    setValidityOfActTimeSubjRestriction(newNodes[1], "PREACTIVATE_FEATURE", False)
    setValidityOfActTimeSubjRestriction(newNodes[2], "PREACTIVATE_FEATURE", False)

    waitUntilBeginningOfProdTurn(newNodes[0], "defproducera")
    newNodes[0].activatePreactivateFeature()
    assert shouldNodeContainPreactivateFeature(newNodes[0]), "1st node should contain PREACTIVATE FEATURE"
    assert not (shouldNodeContainPreactivateFeature(newNodes[1]) or shouldNodeContainPreactivateFeature(newNodes[2])), \
           "2nd and 3rd node should not contain PREACTIVATE FEATURE"
    assert areNodesInSync([newNodes[1], newNodes[2], oldNode]), "2nd, 3rd and 4th node should be in sync"
    assert not areNodesInSync(allNodes), "1st node should be out of sync with the rest nodes"

    waitForOneRound()

    assert not shouldNodeContainPreactivateFeature(newNodes[0]), "PREACTIVATE_FEATURE should be dropped"
    assert areNodesInSync(allNodes), "All nodes should be in sync"

    # Then we set the earliest_allowed_activation_time of 2nd node and 3rd node with valid value
    # Once the 1st node activate PREACTIVATE_FEATURE, all of them should have PREACTIVATE_FEATURE activated in the next block
    # They will be in sync and their LIB will advance since they control > 2/3 of the producers
    # Also the LIB should be able to advance past the block that contains PREACTIVATE_FEATURE
    # However, the 4th node will be out of sync with them, and its LIB will stuck
    setValidityOfActTimeSubjRestriction(newNodes[1], "PREACTIVATE_FEATURE", True)
    setValidityOfActTimeSubjRestriction(newNodes[2], "PREACTIVATE_FEATURE", True)

    waitUntilBeginningOfProdTurn(newNodes[0], "defproducera")
    libBeforePreactivation = newNodes[0].getIrreversibleBlockNum()
    newNodes[0].activatePreactivateFeature()

    assert areNodesInSync(newNodes), "New nodes should be in sync"
    assert not areNodesInSync(allNodes), "Nodes should not be in sync after preactivation"
    for node in newNodes: assert shouldNodeContainPreactivateFeature(node), "New node should contain PREACTIVATE_FEATURE"

    activatedBlockNum = newNodes[0].getHeadBlockNum() # The PREACTIVATE_FEATURE should have been activated before or at this block num
    assert waitUntilBlockBecomeIrr(newNodes[0], activatedBlockNum), \
           "1st node LIB should be able to advance past the block that contains PREACTIVATE_FEATURE"
    assert newNodes[1].getIrreversibleBlockNum() >= activatedBlockNum and \
           newNodes[2].getIrreversibleBlockNum() >= activatedBlockNum, \
           "2nd and 3rd node LIB should also be able to advance past the block that contains PREACTIVATE_FEATURE"
    assert oldNode.getIrreversibleBlockNum() <= libBeforePreactivation, \
           "4th node LIB should stuck on LIB before PREACTIVATE_FEATURE is activated"

    # Restart old node with newest version
    # Before we are migrating to new version, use --export-reversible-blocks as the old version
    # and --import-reversible-blocks with the new version to ensure the compatibility of the reversible blocks
    # Finally, when we restart the 4th node with the version of nodeos that supports protocol feature,
    # all nodes should be in sync, and the 4th node will also contain PREACTIVATE_FEATURE
    portableRevBlkPath = os.path.join(Utils.getNodeDataDir(oldNodeId), "rev_blk_portable_format")
    oldNode.kill(signal.SIGTERM)
    # Note, for the following relaunch, these will fail to relaunch immediately (expected behavior of export/import), so the chainArg will not replace the old cmd
    oldNode.relaunch(chainArg="--export-reversible-blocks {}".format(portableRevBlkPath), timeout=1)
    oldNode.relaunch(chainArg="--import-reversible-blocks {}".format(portableRevBlkPath), timeout=1, nodeosPath="programs/nodeos/nodeos")
    os.remove(portableRevBlkPath)

    restartNode(oldNode, chainArg="--replay", nodeosPath="programs/nodeos/nodeos")
    time.sleep(2) # Give some time to replay

    assert areNodesInSync(allNodes), "All nodes should be in sync"
    assert shouldNodeContainPreactivateFeature(oldNode), "4th node should contain PREACTIVATE_FEATURE"

    testSuccessful = True
finally:
    TestHelper.shutdown(cluster, walletMgr, testSuccessful, killEosInstances, killWallet, keepLogs, killAll, dumpErrorDetails)

exitCode = 0 if testSuccessful else 1
exit(exitCode)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
r'''
Author: Bo Maryniuk <bo@suse.de>

Execution of Ansible modules from within states
===============================================

With `ansible.call` these states allow individual Ansible module calls to be
made via states. To call an Ansible module function use a :mod:`module.run <salt.states.ansible.call>`
state:

.. code-block:: yaml

    some_set_of_tasks:
      ansible:
        - system.ping
        - packaging.os.zypper
          - name: emacs
          - state: installed

'''
from __future__ import absolute_import
import sys
try:
    import ansible
except ImportError as err:
    ansible = None

__virtualname__ = 'ansible'

class AnsibleState(object):
    '''
    Ansible state caller.
    '''
    def get_args(self, argset):
        '''
        Get args and kwargs from the argset.

        :param argset:
        :return:
        '''
        args = []
        kwargs = {}
        for element in argset or []:
            if isinstance(element, dict):
                kwargs.update(element)
            else:
                args.append(element)
        return args, kwargs

    def __call__(self, **kwargs):
        '''
        Call Ansible module.

        :return:
        '''

        ret = {
            'name': kwargs.pop('name'),
            'changes': {},
            'comment': '',
            'result': True,
        }

        for mod_name, mod_params in kwargs.items():
            args, kwargs = self.get_args(mod_params)
            try:
                ans_mod_out = __salt__['ansible.{0}'.format(mod_name)](**{'__pub_arg': [args, kwargs]})
            except Exception as err:
                ans_mod_out = 'Module "{0}" failed. Error message: ({1}) {2}'.format(
                    mod_name, err.__class__.__name__, err)
                ret['result'] = False
            ret['changes'][mod_name] = ans_mod_out

        return ret

def __virtual__():
    '''
    Disable, if Ansible is not available around on the Minion.
    '''
    setattr(sys.modules[__name__], 'call', lambda **kwargs: AnsibleState()(**kwargs))
    return ansible is not None
<commit_msg>Add the license header<commit_after># -*- coding: utf-8 -*-
#
# Author: Bo Maryniuk <bo@suse.de>
#
# Copyright 2017 SUSE LLC
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

r'''
:codeauthor: :email:`Bo Maryniuk <bo@suse.de>`

Execution of Ansible modules from within states
===============================================

With `ansible.call` these states allow individual Ansible module calls to be
made via states. To call an Ansible module function use a :mod:`module.run <salt.states.ansible.call>`
state:

.. code-block:: yaml

    some_set_of_tasks:
      ansible:
        - system.ping
        - packaging.os.zypper
          - name: emacs
          - state: installed

'''
from __future__ import absolute_import
import sys
try:
    import ansible
except ImportError as err:
    ansible = None

__virtualname__ = 'ansible'

class AnsibleState(object):
    '''
    Ansible state caller.
    '''
    def get_args(self, argset):
        '''
        Get args and kwargs from the argset.

        :param argset:
        :return:
        '''
        args = []
        kwargs = {}
        for element in argset or []:
            if isinstance(element, dict):
                kwargs.update(element)
            else:
                args.append(element)
        return args, kwargs

    def __call__(self, **kwargs):
        '''
        Call Ansible module.

        :return:
        '''

        ret = {
            'name': kwargs.pop('name'),
            'changes': {},
            'comment': '',
            'result': True,
        }

        for mod_name, mod_params in kwargs.items():
            args, kwargs = self.get_args(mod_params)
            try:
                ans_mod_out = __salt__['ansible.{0}'.format(mod_name)](**{'__pub_arg': [args, kwargs]})
            except Exception as err:
                ans_mod_out = 'Module "{0}" failed. Error message: ({1}) {2}'.format(
                    mod_name, err.__class__.__name__, err)
                ret['result'] = False
            ret['changes'][mod_name] = ans_mod_out

        return ret

def __virtual__():
    '''
    Disable, if Ansible is not available around on the Minion.
    '''
    setattr(sys.modules[__name__], 'call', lambda **kwargs: AnsibleState()(**kwargs))
    return ansible is not None
<|endoftext|>