<commit_before>import unittest

import mock

from vmupdate.constants import *
from vmupdate.virtualizers import get_virtualizer, VirtualBox

from tests.constants import *
from tests.context import get_data_string, mock

class VirtualBoxTestCase(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.TEST_LIST_VMS = get_data_string('virtualbox/list_vms.txt')
        cls.TEST_SHOW_VM = get_data_string('virtualbox/show_vm.txt')

    def setUp(self):
        patch_popen = mock.patch('subprocess.Popen', autospec=True)

        self.addCleanup(patch_popen.stop)

        self.mock_popen = patch_popen.start()

        self.mock_cmd = mock.Mock()
        self.mock_cmd.wait.return_value = 0
        self.mock_cmd.communicate.return_value = 'stdoutdata', 'stderrdata'

        self.mock_popen.return_value = self.mock_cmd

        self.virt = VirtualBox(TEST_VIRTUALIZER_PATH)

    def test_get_virtualizer(self):
        virt = get_virtualizer('VirtualBox', None)

        self.assertIs(type(virt), VirtualBox)

    def test_list_vms(self):
        self.mock_cmd.communicate.return_value = VirtualBoxTestCase.TEST_LIST_VMS, 'stderrdata'

        vms = self.virt.list_vms()

        self.mock_popen.assert_called_once_with([TEST_VIRTUALIZER_PATH, 'list', 'vms'],
                                                stderr=mock.ANY, stdout=mock.ANY)

        self.assertListEqual(vms, [('Test Machine 1', 'b243d621-8003-43e5-9ff4-3e87b2c1b303'),
                                   ('Test Machine 2', 'd208316d-e977-4e56-8933-c942ff7848d4')])

    def test_start_vm(self):
        self.virt.start_vm(TEST_UID)

        self.mock_popen.assert_called_once_with([TEST_VIRTUALIZER_PATH, 'startvm', TEST_UID,
                                                 '--type', 'headless'], stderr=mock.ANY, stdout=mock.ANY)

        self.mock_cmd.wait.assert_called_once_with()

    def test_stop_vm(self):
        self.virt.stop_vm(TEST_UID)

        self.mock_popen.assert_called_once_with([TEST_VIRTUALIZER_PATH, 'controlvm',
                                                 TEST_UID, 'poweroff'],
                                                stderr=mock.ANY, stdout=mock.ANY)

        self.mock_cmd.wait.assert_called_once_with()

    def test_get_vm_status(self):
        status = self.virt.get_vm_status(TEST_UID)

        self.mock_popen.assert_called_once_with([TEST_VIRTUALIZER_PATH, 'showvminfo',
                                                 TEST_UID], stderr=mock.ANY, stdout=mock.ANY)

        self._test_get_vm_status('powered off', VM_STOPPED)
        self._test_get_vm_status('aborted', VM_STOPPED)
        self._test_get_vm_status('running', VM_RUNNING)
        self._test_get_vm_status('saved', VM_SUSPENDED)
        self._test_get_vm_status('paused', VM_PAUSED)
        self._test_get_vm_status('random test state', VM_UNKNOWN)

    def test_get_vm_os(self):
        status = self.virt.get_vm_status(TEST_UID)

        self.mock_popen.assert_called_once_with([TEST_VIRTUALIZER_PATH, 'showvminfo',
                                                 TEST_UID], stderr=mock.ANY, stdout=mock.ANY)

        self._test_get_vm_os('Windows', OS_WINDOWS)
        self._test_get_vm_os('Windows XP', OS_WINDOWS)
        self._test_get_vm_os('Windows Vista', OS_WINDOWS)
        self._test_get_vm_os('Other Windows', OS_WINDOWS)
        self._test_get_vm_os('Mac OS X', OS_MAC_OS_X)
        self._test_get_vm_os('Linux', OS_LINUX)
        self._test_get_vm_os('Arch Linux', OS_ARCH)
        self._test_get_vm_os('Ubuntu', OS_UBUNTU)
        self._test_get_vm_os('Red Hat', OS_REDHAT)
        self._test_get_vm_os('Debian', OS_DEBIAN)
        self._test_get_vm_os('Fedora', OS_FEDORA)
        self._test_get_vm_os('Gentoo', OS_GENTOO)
        self._test_get_vm_os('openSUSE', OS_OPENSUSE)
        self._test_get_vm_os('Mandriva', OS_MANDRIVA)
        self._test_get_vm_os('Turbolinux', OS_TURBOLINUX)
        self._test_get_vm_os('Xandros', OS_XANDROS)
        self._test_get_vm_os('Oracle', OS_ORACLE)
        self._test_get_vm_os('Random OS', OS_UNKNOWN)

    def test_get_ssh_info(self):
        self.mock_cmd.communicate.return_value = VirtualBoxTestCase.TEST_SHOW_VM, 'stderrdata'

        ssh_info = self.virt.get_ssh_info(TEST_UID, 0)

        self.mock_popen.assert_called_once_with([TEST_VIRTUALIZER_PATH, 'showvminfo',
                                                 TEST_UID], stderr=mock.ANY, stdout=mock.ANY)

        self.assertTupleEqual(ssh_info, (None, None))

        host, port = self.virt.get_ssh_info(TEST_UID, 22)

        self.assertEqual(host, '127.0.0.1')
        self.assertEqual(port, 49152)

    def test_enable_ssh(self):
        test_host_port = 22
        test_guest_port = 49152

        self.virt.enable_ssh(TEST_UID, test_host_port, test_guest_port)

        self.mock_popen.assert_called_once_with([TEST_VIRTUALIZER_PATH, 'modifyvm', TEST_UID,
                                                 '--natpf1', 'ssh,tcp,,{0},,{1}'.format(
                                                    test_host_port, test_guest_port)],
                                                stderr=mock.ANY, stdout=mock.ANY)

        self.mock_cmd.wait.assert_called_once_with()

    def _test_get_vm_status(self, status, expected):
        stdout = 'State:           %s (since 2016-03-25T23:11:46.252000000)' % status

        self.mock_cmd.communicate.return_value = stdout, 'stderrdata'

        vm_status = self.virt.get_vm_status(TEST_UID)

        self.assertEqual(vm_status, expected)

    def _test_get_vm_os(self, os, expected):
        stdout = 'Guest OS:        %s (64-bit)' % os

        self.mock_cmd.communicate.return_value = stdout, 'stderrdata'

        vm_os = self.virt.get_vm_os(TEST_UID)

        self.assertEqual(vm_os, expected)
<commit_msg>fixed tests<commit_after>import unittest

import mock

from vmupdate.constants import *
from vmupdate.virtualizers import get_virtualizer, VirtualBox

from tests.constants import *
from tests.context import get_data_string, mock

class VirtualBoxTestCase(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.TEST_LIST_VMS = get_data_string('virtualbox/list_vms.txt')
        cls.TEST_SHOW_VM = get_data_string('virtualbox/show_vm.txt')

    def setUp(self):
        patch_popen = mock.patch('subprocess.Popen', autospec=True)

        self.addCleanup(patch_popen.stop)

        self.mock_popen = patch_popen.start()

        self.mock_cmd = mock.Mock()
        self.mock_cmd.wait.return_value = 0
        self.mock_cmd.communicate.return_value = 'stdoutdata', 'stderrdata'

        self.mock_popen.return_value = self.mock_cmd

        self.virt = VirtualBox(TEST_VIRTUALIZER_PATH)

    def test_get_virtualizer(self):
        virt = get_virtualizer('VirtualBox', None)

        self.assertIs(type(virt), VirtualBox)

    def test_list_vms(self):
        self.mock_cmd.communicate.return_value = VirtualBoxTestCase.TEST_LIST_VMS, 'stderrdata'

        vms = self.virt.list_vms()

        self.mock_popen.assert_called_once_with([TEST_VIRTUALIZER_PATH, 'list', 'vms'],
                                                stderr=mock.ANY, stdout=mock.ANY, universal_newlines=True)

        self.assertListEqual(vms, [('Test Machine 1', 'b243d621-8003-43e5-9ff4-3e87b2c1b303'),
                                   ('Test Machine 2', 'd208316d-e977-4e56-8933-c942ff7848d4')])

    def test_start_vm(self):
        self.virt.start_vm(TEST_UID)

        self.mock_popen.assert_called_once_with([TEST_VIRTUALIZER_PATH, 'startvm', TEST_UID,
                                                 '--type', 'headless'], stderr=mock.ANY, stdout=mock.ANY,
                                                universal_newlines=True)

        self.mock_cmd.wait.assert_called_once_with()

    def test_stop_vm(self):
        self.virt.stop_vm(TEST_UID)

        self.mock_popen.assert_called_once_with([TEST_VIRTUALIZER_PATH, 'controlvm',
                                                 TEST_UID, 'poweroff'],
                                                stderr=mock.ANY, stdout=mock.ANY, universal_newlines=True)

        self.mock_cmd.wait.assert_called_once_with()

    def test_get_vm_status(self):
        status = self.virt.get_vm_status(TEST_UID)

        self.mock_popen.assert_called_once_with([TEST_VIRTUALIZER_PATH, 'showvminfo',
                                                 TEST_UID], stderr=mock.ANY, stdout=mock.ANY, universal_newlines=True)

        self._test_get_vm_status('powered off', VM_STOPPED)
        self._test_get_vm_status('aborted', VM_STOPPED)
        self._test_get_vm_status('running', VM_RUNNING)
        self._test_get_vm_status('saved', VM_SUSPENDED)
        self._test_get_vm_status('paused', VM_PAUSED)
        self._test_get_vm_status('random test state', VM_UNKNOWN)

    def test_get_vm_os(self):
        status = self.virt.get_vm_status(TEST_UID)

        self.mock_popen.assert_called_once_with([TEST_VIRTUALIZER_PATH, 'showvminfo',
                                                 TEST_UID], stderr=mock.ANY, stdout=mock.ANY, universal_newlines=True)

        self._test_get_vm_os('Windows', OS_WINDOWS)
        self._test_get_vm_os('Windows XP', OS_WINDOWS)
        self._test_get_vm_os('Windows Vista', OS_WINDOWS)
        self._test_get_vm_os('Other Windows', OS_WINDOWS)
        self._test_get_vm_os('Mac OS X', OS_MAC_OS_X)
        self._test_get_vm_os('Linux', OS_LINUX)
        self._test_get_vm_os('Arch Linux', OS_ARCH)
        self._test_get_vm_os('Ubuntu', OS_UBUNTU)
        self._test_get_vm_os('Red Hat', OS_REDHAT)
        self._test_get_vm_os('Debian', OS_DEBIAN)
        self._test_get_vm_os('Fedora', OS_FEDORA)
        self._test_get_vm_os('Gentoo', OS_GENTOO)
        self._test_get_vm_os('openSUSE', OS_OPENSUSE)
        self._test_get_vm_os('Mandriva', OS_MANDRIVA)
        self._test_get_vm_os('Turbolinux', OS_TURBOLINUX)
        self._test_get_vm_os('Xandros', OS_XANDROS)
        self._test_get_vm_os('Oracle', OS_ORACLE)
        self._test_get_vm_os('Random OS', OS_UNKNOWN)

    def test_get_ssh_info(self):
        self.mock_cmd.communicate.return_value = VirtualBoxTestCase.TEST_SHOW_VM, 'stderrdata'

        ssh_info = self.virt.get_ssh_info(TEST_UID, 0)

        self.mock_popen.assert_called_once_with([TEST_VIRTUALIZER_PATH, 'showvminfo',
                                                 TEST_UID], stderr=mock.ANY, stdout=mock.ANY, universal_newlines=True)

        self.assertTupleEqual(ssh_info, (None, None))

        host, port = self.virt.get_ssh_info(TEST_UID, 22)

        self.assertEqual(host, '127.0.0.1')
        self.assertEqual(port, 49152)

    def test_enable_ssh(self):
        test_host_port = 22
        test_guest_port = 49152

        self.virt.enable_ssh(TEST_UID, test_host_port, test_guest_port)

        self.mock_popen.assert_called_once_with([TEST_VIRTUALIZER_PATH, 'modifyvm', TEST_UID,
                                                 '--natpf1', 'ssh,tcp,,{0},,{1}'.format(
                                                    test_host_port, test_guest_port)],
                                                stderr=mock.ANY, stdout=mock.ANY, universal_newlines=True)

        self.mock_cmd.wait.assert_called_once_with()

    def _test_get_vm_status(self, status, expected):
        stdout = 'State:           %s (since 2016-03-25T23:11:46.252000000)' % status

        self.mock_cmd.communicate.return_value = stdout, 'stderrdata'

        vm_status = self.virt.get_vm_status(TEST_UID)

        self.assertEqual(vm_status, expected)

    def _test_get_vm_os(self, os, expected):
        stdout = 'Guest OS:        %s (64-bit)' % os

        self.mock_cmd.communicate.return_value = stdout, 'stderrdata'

        vm_os = self.virt.get_vm_os(TEST_UID)

        self.assertEqual(vm_os, expected)
<|endoftext|>


<commit_before>"""
Testing the wind_turbine module.

SPDX-FileCopyrightText: 2019 oemof developer group <contact@oemof.org>
SPDX-License-Identifier: MIT
"""

import pytest
import os
from windpowerlib.tools import WindpowerlibUserWarning

from windpowerlib.wind_turbine import (get_turbine_data_from_file, WindTurbine,
                                       get_turbine_types, WindTurbineGroup,
                                       load_turbine_data_from_oedb)

class TestWindTurbine:

    @classmethod
    def setup_class(cls):
        """Setup default values"""
        cls.source = os.path.join(os.path.dirname(__file__), '../example/data')

    def test_warning(self, recwarn):
        test_turbine_data = {'hub_height': 100,
                             'rotor_diameter': 80,
                             'turbine_type': 'turbine_not_in_file',
                             'path': self.source}
        assert(WindTurbine(**test_turbine_data).power_curve is None)
        assert recwarn.pop(WindpowerlibUserWarning)

    def test_get_turbine_data_from_file(self):
        # Raise FileNotFoundError due to missing
        with pytest.raises(FileNotFoundError):
            get_turbine_data_from_file(turbine_type='...',
                                       path='not_existent')

    def test_get_turbine_types(self, capsys):
        get_turbine_types()
        captured = capsys.readouterr()
        assert 'Enercon' in captured.out
        get_turbine_types('oedb', print_out=False, filter_=False)
        msg = "`turbine_library` is 'wrong' but must be 'local' or 'oedb'."
        with pytest.raises(ValueError, match=msg):
            get_turbine_types('wrong')

    def test_wrong_url_load_turbine_data(self):
        """Load turbine data from oedb."""

        with pytest.raises(ConnectionError,
                           match="Database connection not successful"):
            load_turbine_data_from_oedb('wrong_schema')

    @pytest.mark.filterwarnings("ignore:The WindTurbine")
    def test_string_representation_of_wind_turbine(self):
        assert "Wind turbine: ['hub height=120 m'" in repr(WindTurbine(120))

    def test_power_curve_is_of_wrong_type(self):
        """Error raising due to wrong type of WindTurbine.power_curve."""
        test_turbine_data = {'hub_height': 100,
                             'rotor_diameter': 80,
                             'turbine_type': 'test_type',
                             'power_curve': 'string'}
        with pytest.raises(TypeError):
            WindTurbine(**test_turbine_data)

    def test_power_coefficient_curve_is_of_wrong_type(self):
        """Error raising due to wrong type of
        WindTurbine.power_coefficient_curve."""
        test_turbine_data = {'hub_height': 100,
                             'rotor_diameter': 80,
                             'turbine_type': 'test_type',
                             'power_coefficient_curve': 'string'}
        with pytest.raises(TypeError):
            WindTurbine(**test_turbine_data)

    def test_to_group_method(self):
        example_turbine = {
           'hub_height': 100,
           'rotor_diameter': 70,
           'turbine_type': 'DUMMY 3',
           'path': self.source}
        e_t_1 = WindTurbine(**example_turbine)
        assert(isinstance(e_t_1.to_group(), WindTurbineGroup))
        assert(e_t_1.to_group(5).number_of_turbines == 5)
        assert(e_t_1.to_group(number_turbines=5).number_of_turbines == 5)
        assert(e_t_1.to_group(total_capacity=3e6).number_of_turbines == 2.0)

    def test_wrongly_defined_to_group_method(self):
        example_turbine = {
           'hub_height': 100,
           'rotor_diameter': 70,
           'turbine_type': 'DUMMY 3',
           'path': self.source}
        e_t_1 = WindTurbine(**example_turbine)
        with pytest.raises(ValueError,
                           match="The 'number' and the 'total_capacity' "
                                 "parameter are mutually exclusive."):
            e_t_1.to_group(5, 3000)
<commit_msg>Test creation of unphysical WindTurbine<commit_after>"""
Testing the wind_turbine module.

SPDX-FileCopyrightText: 2019 oemof developer group <contact@oemof.org>
SPDX-License-Identifier: MIT
"""

import pytest
import os
from windpowerlib.tools import WindpowerlibUserWarning

from windpowerlib.wind_turbine import (get_turbine_data_from_file, WindTurbine,
                                       get_turbine_types, WindTurbineGroup,
                                       load_turbine_data_from_oedb)

class TestWindTurbine:

    @classmethod
    def setup_class(cls):
        """Setup default values"""
        cls.source = os.path.join(os.path.dirname(__file__), '../example/data')

    def test_warning(self, recwarn):
        test_turbine_data = {'hub_height': 100,
                             'rotor_diameter': 80,
                             'turbine_type': 'turbine_not_in_file',
                             'path': self.source}
        assert(WindTurbine(**test_turbine_data).power_curve is None)
        assert recwarn.pop(WindpowerlibUserWarning)

    def test_get_turbine_data_from_file(self):
        # Raise FileNotFoundError due to missing
        with pytest.raises(FileNotFoundError):
            get_turbine_data_from_file(turbine_type='...',
                                       path='not_existent')

    def test_get_turbine_types(self, capsys):
        get_turbine_types()
        captured = capsys.readouterr()
        assert 'Enercon' in captured.out
        get_turbine_types('oedb', print_out=False, filter_=False)
        msg = "`turbine_library` is 'wrong' but must be 'local' or 'oedb'."
        with pytest.raises(ValueError, match=msg):
            get_turbine_types('wrong')

    def test_wrong_url_load_turbine_data(self):
        """Load turbine data from oedb."""

        with pytest.raises(ConnectionError,
                           match="Database connection not successful"):
            load_turbine_data_from_oedb('wrong_schema')

    @pytest.mark.filterwarnings("ignore:The WindTurbine")
    def test_string_representation_of_wind_turbine(self):
        assert "Wind turbine: ['hub height=120 m'" in repr(WindTurbine(120))

    def test_power_curve_is_of_wrong_type(self):
        """Error raising due to wrong type of WindTurbine.power_curve."""
        test_turbine_data = {'hub_height': 100,
                             'rotor_diameter': 80,
                             'turbine_type': 'test_type',
                             'power_curve': 'string'}
        with pytest.raises(TypeError):
            WindTurbine(**test_turbine_data)

    def test_power_coefficient_curve_is_of_wrong_type(self):
        """Error raising due to wrong type of
        WindTurbine.power_coefficient_curve."""
        test_turbine_data = {'hub_height': 100,
                             'rotor_diameter': 80,
                             'turbine_type': 'test_type',
                             'power_coefficient_curve': 'string'}
        with pytest.raises(TypeError):
            WindTurbine(**test_turbine_data)

    def test_to_group_method(self):
        example_turbine = {
           'hub_height': 100,
           'rotor_diameter': 70,
           'turbine_type': 'DUMMY 3',
           'path': self.source}
        e_t_1 = WindTurbine(**example_turbine)
        assert(isinstance(e_t_1.to_group(), WindTurbineGroup))
        assert(e_t_1.to_group(5).number_of_turbines == 5)
        assert(e_t_1.to_group(number_turbines=5).number_of_turbines == 5)
        assert(e_t_1.to_group(total_capacity=3e6).number_of_turbines == 2.0)

    def test_wrongly_defined_to_group_method(self):
        example_turbine = {
           'hub_height': 100,
           'rotor_diameter': 70,
           'turbine_type': 'DUMMY 3',
           'path': self.source}
        e_t_1 = WindTurbine(**example_turbine)
        with pytest.raises(ValueError,
                           match="The 'number' and the 'total_capacity' "
                                 "parameter are mutually exclusive."):
            e_t_1.to_group(5, 3000)

    def test_create_unphysical_turbine(self):
        char = {
           'hub_height': 80,
           'rotor_diameter': 160,
           'turbine_type': 'DUMMY 3',
           'path': self.source
        }
        with pytest.raises(ValueError):
            WindTurbine(**char)
<|endoftext|>


<commit_before><commit_msg>Adds more test on webdav<commit_after><|endoftext|>


<commit_before>from datetime import datetime
from zeit.cms.repository.folder import Folder
from zeit.cms.testcontenttype.testcontenttype import ExampleContentType
from zeit.content.volume.volume import Volume
import zeit.content.cp.centerpage
import lxml.etree
import lxml.objectify
import mock
import pysolr
import pytz
import zeit.cms.content.sources
import zeit.cms.interfaces
import zeit.content.cp.interfaces
import zeit.content.volume.interfaces
import zeit.content.volume.testing
import zeit.content.volume.volume

class TestVolumeCovers(zeit.content.volume.testing.FunctionalTestCase):

    def setUp(self):
        from zeit.content.image.testing import create_image_group
        super(TestVolumeCovers, self).setUp()
        self.repository['imagegroup'] = create_image_group()
        self.cover = self.repository['imagegroup']
        self.volume = Volume()
        self.volume.product = zeit.cms.content.sources.Product(u'ZEI')

    def add_ipad_cover(self, product_id='ZEI'):
        node = lxml.objectify.E.cover(
            href='http://xml.zeit.de/imagegroup/',
            id='ipad',
            product_id=product_id)
        lxml.objectify.deannotate(node[0], cleanup_namespaces=True)
        self.volume.xml.covers.append(node)

    def test_set_raises_for_invalid_product(self):
        with self.assertRaises(ValueError):
            self.volume.set_cover('ipad', 'TEST', self.cover)

    def test_stores_uniqueId_in_xml_of_volume(self):
        self.volume.set_cover('ipad', 'ZEI', self.repository['imagegroup'])
        self.assertEqual(
            '<covers xmlns:py="http://codespeak.net/lxml/objectify/pytype">'
            '<cover href="http://xml.zeit.de/imagegroup/" id="ipad" '
            'product_id="ZEI"/>'
            '</covers>',
            lxml.etree.tostring(self.volume.xml.covers))

    def test_deletes_existing_node_if_value_is_None(self):
        self.volume.set_cover('ipad', 'ZEI', self.repository['imagegroup'])
        self.volume.set_cover('ipad', 'ZEI', None)
        self.assertEqual(
            '<covers xmlns:py="http://codespeak.net/lxml/objectify/pytype"/>',
            lxml.etree.tostring(self.volume.xml.covers))

    def test_raises_if_product_is_not_in_volume(self):
        with self.assertRaises(ValueError):
            self.assertEqual(None, self.volume.get_cover('ipad', 'TEST'))

    def test_resolves_given_product_id(self):
        self.add_ipad_cover('ZMLB')
        self.assertEqual(self.cover, self.volume.get_cover('ipad', 'ZMLB'))

    def test_uses_product_of_volume_if_none_is_given(self):
        self.add_ipad_cover()
        self.assertEqual(self.cover, self.volume.get_cover('ipad'))

    def test_returns_main_product_if_no_dependent_cover_present(self):
        self.add_ipad_cover()
        self.assertEqual(self.cover, self.volume.get_cover('ipad', 'ZMLB'))

class TestReference(zeit.content.volume.testing.FunctionalTestCase):

    def setUp(self):
        super(TestReference, self).setUp()
        volume = Volume()
        volume.year = 2015
        volume.volume = 1
        zeit.cms.content.add.find_or_create_folder('2015', '01')
        self.repository['2015']['01']['ausgabe'] = volume

    def test_content_with_missing_values_does_not_adapt_to_IVolume(self):
        with self.assertRaises(TypeError):
            zeit.content.volume.interfaces.IVolume(ExampleContentType())

    def test_content_with_year_and_volume_and_product_adapts_to_IVolume(self):
        content = ExampleContentType()
        content.year = 2015
        content.volume = 1
        content.product = zeit.cms.content.sources.Product(u'ZEI')
        volume = zeit.content.volume.interfaces.IVolume(content)
        self.assertEqual(
            volume,
            self.repository['2015']['01']['ausgabe'])

    def test_content_online_product_has_no_IVolume(self):
        # *All* content that is added in vivi gets year and volume from
        # zeit.cms.settings.interfaces.IGlobalSettings, so we need to ensure
        # that "online content" has no IVolume, only print content.
        # In addition we want only handle products with a location template
        # configured.
        content = ExampleContentType()
        content.year = 2015
        content.volume = 1
        content.product = zeit.cms.content.sources.Product(u'ZEDE')
        with self.assertRaises(TypeError):
            zeit.content.volume.interfaces.IVolume(content)

    def test_contents_with_defined_dependency_adapt_to_same_volume(self):
        zei_content = ExampleContentType()
        zei_content.year = 2015
        zei_content.volume = 1
        zei_content.product = zeit.cms.content.sources.Product(u'ZEI')
        zmlb_content = ExampleContentType()
        zmlb_content.year = 2015
        zmlb_content.volume = 1
        zmlb_content.product = zeit.cms.content.sources.Product(u'ZMLB')
        self.assertEqual(zeit.content.volume.interfaces.IVolume(zei_content),
                         zeit.content.volume.interfaces.IVolume(zmlb_content))

    def test_cant_adapt_content_with_dependency_defined_to_a_non_volume(
            self):
        zecw_content = ExampleContentType()
        zecw_content.year = 2015
        zecw_content.volume = 1
        zecw_content.product = zeit.cms.content.sources.Product(
            u'BADDEPENDENCY')
        with self.assertRaises(TypeError):
            zeit.content.volume.interfaces.IVolume(zecw_content)

class TestVolume(zeit.content.volume.testing.FunctionalTestCase):

    def setUp(self):
        super(TestVolume, self).setUp()
        volume = Volume()
        volume.year = 2015
        volume.volume = 1
        volume.product = zeit.cms.content.sources.Product(u'ZEI')
        self.repository['2015'] = Folder()
        self.repository['2015']['01'] = Folder()
        self.repository['2015']['01']['ausgabe'] = volume

    def test_looks_up_centerpage_from_product_setting(self):
        self.repository['2015']['01']['index'] = zeit.content.cp.centerpage\
            .CenterPage()
        volume = zeit.cms.interfaces.ICMSContent(
            'http://xml.zeit.de/2015/01/ausgabe')
        cp = zeit.content.cp.interfaces.ICenterPage(volume)
        self.assertEqual('http://xml.zeit.de/2015/01/index', cp.uniqueId)

    def test_non_cp_not_looked_up_for_centerpage(self):
        self.repository['2015']['01']['index'] = ExampleContentType()
        volume = zeit.cms.interfaces.ICMSContent(
            'http://xml.zeit.de/2015/01/ausgabe')
        cp = zeit.content.cp.interfaces.ICenterPage(volume, None)
        self.assertEqual(None, cp)

    def test_looks_up_centerpage_for_depent_product_content(self):
        content = ExampleContentType()
        content.product = zeit.cms.content.sources.Product(u'ZMLB')
        self.repository['2015']['01']['index'] = content
        volume = zeit.cms.interfaces.ICMSContent(
            'http://xml.zeit.de/2015/01/ausgabe')
        self.repository['2015']['01']['index'] = zeit.content.cp.centerpage \
            .CenterPage()
        cp = zeit.content.cp.interfaces.ICenterPage(volume)
        self.assertEqual('http://xml.zeit.de/2015/01/index', cp.uniqueId)

class TestVolumeSolrQuerries(zeit.content.volume.testing.FunctionalTestCase):

    def setUp(self):
        super(TestVolumeSolrQuerries, self).setUp()
        self.create_volume(2015, 1)
        self.create_volume(2015, 2)
        self.solr = mock.Mock()
        self.zca.patch_utility(self.solr, zeit.solr.interfaces.ISolr)

    def create_volume(self, year, name):
        volume = Volume()
        volume.year = year
        volume.volume = name
        volume.product = zeit.cms.content.sources.Product(u'ZEI')
        volume.date_digital_published = datetime(
            year, name, 1, tzinfo=pytz.UTC)
        year = str(year)
        name = '%02d' % name
        self.repository[year] = Folder()
        self.repository[year][name] = Folder()
        self.repository[year][name]['ausgabe'] = volume

    def test_resolves_solr_result(self):
        self.solr.search.return_value = pysolr.Results(
            [{'uniqueId': 'http://xml.zeit.de/2015/02/ausgabe'}], 1)
        vol1 = zeit.cms.interfaces.ICMSContent(
            'http://xml.zeit.de/2015/01/ausgabe')
        vol2 = zeit.cms.interfaces.ICMSContent(
            'http://xml.zeit.de/2015/02/ausgabe')
        self.assertEqual(vol2, vol1.next)

    def test_no_solr_result_returns_None(self):
        self.solr.search.return_value = pysolr.Results([], 0)
        vol1 = zeit.cms.interfaces.ICMSContent(
            'http://xml.zeit.de/2015/01/ausgabe')
        self.assertEqual(None, vol1.next)
        self.assertEqual(None, vol1.previous)

    def test_no_publish_date_returns_None(self):
        volume = Volume()
        year = 2015
        name = 1
        volume.year = year
        volume.volume = name
        volume.product = zeit.cms.content.sources.Product(u'ZEI')
        self.solr.search.return_value = pysolr.Results(
            [{'uniqueId': 'http://xml.zeit.de/2015/02/ausgabe'}], 1)
        self.assertEqual(None, volume.next)
        self.assertEqual(None, volume.previous)

    def test_all_content_via_solr_returns_empty_list_if_no_content(self):
        volume = zeit.cms.interfaces.ICMSContent(
            'http://xml.zeit.de/2015/01/ausgabe')
        self.solr.search.return_value = pysolr.Results(
            [], 0)
        self.assertEqual([], volume.all_content_via_solr(
            additional_query_contstraints=['foo:bar']))

    def test_all_content_via_solr_returns_ICMS_content(self):
        volume = zeit.cms.interfaces.ICMSContent(
            'http://xml.zeit.de/2015/01/ausgabe')
        content = ExampleContentType()
        self.repository['2015']['01']['article'] = content
        self.solr.search.return_value = pysolr.Results(
                [{'uniqueId': 'http://xml.zeit.de/2015/01/article'}], 1)
        self.assertListEqual([content], volume.all_content_via_solr(
            additional_query_contstraints=['foo:bar']))
<commit_msg>ZON-3384: test<commit_after>from datetime import datetime
from zeit.cms.repository.folder import Folder
from zeit.cms.testcontenttype.testcontenttype import ExampleContentType
from zeit.content.volume.volume import Volume
import zeit.content.cp.centerpage
import lxml.etree
import lxml.objectify
import mock
import pysolr
import pytz
import zeit.cms.content.sources
import zeit.cms.interfaces
import zeit.content.cp.interfaces
import zeit.content.volume.interfaces
import zeit.content.volume.testing
import zeit.content.volume.volume

class TestVolumeCovers(zeit.content.volume.testing.FunctionalTestCase):

    def setUp(self):
        from zeit.content.image.testing import create_image_group
        super(TestVolumeCovers, self).setUp()
        self.repository['imagegroup'] = create_image_group()
        self.cover = self.repository['imagegroup']
        self.volume = Volume()
        self.volume.product = zeit.cms.content.sources.Product(u'ZEI')

    def add_ipad_cover(self, product_id='ZEI'):
        node = lxml.objectify.E.cover(
            href='http://xml.zeit.de/imagegroup/',
            id='ipad',
            product_id=product_id)
        lxml.objectify.deannotate(node[0], cleanup_namespaces=True)
        self.volume.xml.covers.append(node)

    def test_set_raises_for_invalid_product(self):
        with self.assertRaises(ValueError):
            self.volume.set_cover('ipad', 'TEST', self.cover)

    def test_stores_uniqueId_in_xml_of_volume(self):
        self.volume.set_cover('ipad', 'ZEI', self.repository['imagegroup'])
        self.assertEqual(
            '<covers xmlns:py="http://codespeak.net/lxml/objectify/pytype">'
            '<cover href="http://xml.zeit.de/imagegroup/" id="ipad" '
            'product_id="ZEI"/>'
            '</covers>',
            lxml.etree.tostring(self.volume.xml.covers))

    def test_deletes_existing_node_if_value_is_None(self):
        self.volume.set_cover('ipad', 'ZEI', self.repository['imagegroup'])
        self.volume.set_cover('ipad', 'ZEI', None)
        self.assertEqual(
            '<covers xmlns:py="http://codespeak.net/lxml/objectify/pytype"/>',
            lxml.etree.tostring(self.volume.xml.covers))

    def test_raises_if_product_is_not_in_volume(self):
        with self.assertRaises(ValueError):
            self.assertEqual(None, self.volume.get_cover('ipad', 'TEST'))

    def test_resolves_given_product_id(self):
        self.add_ipad_cover('ZMLB')
        self.assertEqual(self.cover, self.volume.get_cover('ipad', 'ZMLB'))

    def test_uses_product_of_volume_if_none_is_given(self):
        self.add_ipad_cover()
        self.assertEqual(self.cover, self.volume.get_cover('ipad'))

    def test_returns_main_product_if_no_dependent_cover_present(self):
        self.add_ipad_cover()
        self.assertEqual(self.cover, self.volume.get_cover('ipad', 'ZMLB'))

class TestReference(zeit.content.volume.testing.FunctionalTestCase):

    def setUp(self):
        super(TestReference, self).setUp()
        volume = Volume()
        volume.year = 2015
        volume.volume = 1
        zeit.cms.content.add.find_or_create_folder('2015', '01')
        self.repository['2015']['01']['ausgabe'] = volume

    def test_content_with_missing_values_does_not_adapt_to_IVolume(self):
        with self.assertRaises(TypeError):
            zeit.content.volume.interfaces.IVolume(ExampleContentType())

    def test_content_with_year_and_volume_and_product_adapts_to_IVolume(self):
        content = ExampleContentType()
        content.year = 2015
        content.volume = 1
        content.product = zeit.cms.content.sources.Product(u'ZEI')
        volume = zeit.content.volume.interfaces.IVolume(content)
        self.assertEqual(
            volume,
            self.repository['2015']['01']['ausgabe'])

    def test_content_online_product_has_no_IVolume(self):
        # *All* content that is added in vivi gets year and volume from
        # zeit.cms.settings.interfaces.IGlobalSettings, so we need to ensure
        # that "online content" has no IVolume, only print content.
        # In addition we want only handle products with a location template
        # configured.
        content = ExampleContentType()
        content.year = 2015
        content.volume = 1
        content.product = zeit.cms.content.sources.Product(u'ZEDE')
        with self.assertRaises(TypeError):
            zeit.content.volume.interfaces.IVolume(content)

    def test_contents_with_defined_dependency_adapt_to_same_volume(self):
        zei_content = ExampleContentType()
        zei_content.year = 2015
        zei_content.volume = 1
        zei_content.product = zeit.cms.content.sources.Product(u'ZEI')
        zmlb_content = ExampleContentType()
        zmlb_content.year = 2015
        zmlb_content.volume = 1
        zmlb_content.product = zeit.cms.content.sources.Product(u'ZMLB')
        self.assertEqual(zeit.content.volume.interfaces.IVolume(zei_content),
                         zeit.content.volume.interfaces.IVolume(zmlb_content))

    def test_cant_adapt_content_with_dependency_defined_to_a_non_volume(
            self):
        zecw_content = ExampleContentType()
        zecw_content.year = 2015
        zecw_content.volume = 1
        zecw_content.product = zeit.cms.content.sources.Product(
            u'BADDEPENDENCY')
        with self.assertRaises(TypeError):
            zeit.content.volume.interfaces.IVolume(zecw_content)

class TestVolume(zeit.content.volume.testing.FunctionalTestCase):

    def setUp(self):
        super(TestVolume, self).setUp()
        volume = Volume()
        volume.year = 2015
        volume.volume = 1
        volume.product = zeit.cms.content.sources.Product(u'ZEI')
        self.repository['2015'] = Folder()
        self.repository['2015']['01'] = Folder()
        self.repository['2015']['01']['ausgabe'] = volume

    def test_looks_up_centerpage_from_product_setting(self):
        self.repository['2015']['01']['index'] = zeit.content.cp.centerpage\
            .CenterPage()
        volume = zeit.cms.interfaces.ICMSContent(
            'http://xml.zeit.de/2015/01/ausgabe')
        cp = zeit.content.cp.interfaces.ICenterPage(volume)
        self.assertEqual('http://xml.zeit.de/2015/01/index', cp.uniqueId)

    def test_non_cp_not_looked_up_for_centerpage(self):
        self.repository['2015']['01']['index'] = ExampleContentType()
        volume = zeit.cms.interfaces.ICMSContent(
            'http://xml.zeit.de/2015/01/ausgabe')
        cp = zeit.content.cp.interfaces.ICenterPage(volume, None)
        self.assertEqual(None, cp)

    def test_looks_up_centerpage_for_depent_product_content(self):
        content = ExampleContentType()
        content.product = zeit.cms.content.sources.Product(u'ZMLB')
        self.repository['2015']['01']['index'] = content
        volume = zeit.cms.interfaces.ICMSContent(
            'http://xml.zeit.de/2015/01/ausgabe')
        self.repository['2015']['01']['index'] = zeit.content.cp.centerpage \
            .CenterPage()
        cp = zeit.content.cp.interfaces.ICenterPage(volume)
        self.assertEqual('http://xml.zeit.de/2015/01/index', cp.uniqueId)

class TestVolumeSolrQuerries(zeit.content.volume.testing.FunctionalTestCase):

    def setUp(self):
        super(TestVolumeSolrQuerries, self).setUp()
        self.create_volume(2015, 1)
        self.create_volume(2015, 2)
        self.solr = mock.Mock()
        self.zca.patch_utility(self.solr, zeit.solr.interfaces.ISolr)

    def create_volume(self, year, name):
        volume = Volume()
        volume.year = year
        volume.volume = name
        volume.product = zeit.cms.content.sources.Product(u'ZEI')
        volume.date_digital_published = datetime(
            year, name, 1, tzinfo=pytz.UTC)
        year = str(year)
        name = '%02d' % name
        self.repository[year] = Folder()
        self.repository[year][name] = Folder()
        self.repository[year][name]['ausgabe'] = volume

    def test_resolves_solr_result(self):
        self.solr.search.return_value = pysolr.Results(
            [{'uniqueId': 'http://xml.zeit.de/2015/02/ausgabe'}], 1)
        vol1 = zeit.cms.interfaces.ICMSContent(
            'http://xml.zeit.de/2015/01/ausgabe')
        vol2 = zeit.cms.interfaces.ICMSContent(
            'http://xml.zeit.de/2015/02/ausgabe')
        self.assertEqual(vol2, vol1.next)

    def test_no_solr_result_returns_None(self):
        self.solr.search.return_value = pysolr.Results([], 0)
        vol1 = zeit.cms.interfaces.ICMSContent(
            'http://xml.zeit.de/2015/01/ausgabe')
        self.assertEqual(None, vol1.next)
        self.assertEqual(None, vol1.previous)

    def test_no_publish_date_returns_None(self):
        volume = Volume()
        year = 2015
        name = 1
        volume.year = year
        volume.volume = name
        volume.product = zeit.cms.content.sources.Product(u'ZEI')
        self.solr.search.return_value = pysolr.Results(
            [{'uniqueId': 'http://xml.zeit.de/2015/02/ausgabe'}], 1)
        self.assertEqual(None, volume.next)
        self.assertEqual(None, volume.previous)

    def test_all_content_via_solr_returns_empty_list_if_no_content(self):
        volume = zeit.cms.interfaces.ICMSContent(
            'http://xml.zeit.de/2015/01/ausgabe')
        self.solr.search.return_value = pysolr.Results(
            [], 0)
        self.assertEqual([], volume.all_content_via_solr(
            additional_query_contstraints=['foo:bar']))

    def test_all_content_via_solr_returns_ICMS_content(self):
        volume = zeit.cms.interfaces.ICMSContent(
            'http://xml.zeit.de/2015/01/ausgabe')
        content = ExampleContentType()
        self.repository['2015']['01']['article'] = content
        self.solr.search.return_value = pysolr.Results(
            [{'uniqueId': 'http://xml.zeit.de/2015/01/article'}], 1)
        self.assertListEqual([content], volume.all_content_via_solr(
            additional_query_contstraints=['foo:bar']))

    def test_all_volume_contents_should_change_access_value(self):
        volume = zeit.cms.interfaces.ICMSContent(
            'http://xml.zeit.de/2015/01/ausgabe')
        repo = self.repository['2015']['01']
        content01 = ExampleContentType()
        content02 = ExampleContentType()
        repo['article01'] = content01
        repo['article02'] = content02

        self.solr.search.return_value = pysolr.Results(
            [{'uniqueId': 'http://xml.zeit.de/2015/01/article01'},
             {'uniqueId': 'http://xml.zeit.de/2015/01/article02'}],
            2)

        cnt = volume.all_content_via_solr(
            additional_query_contstraints=['foo:bar'])
        for c in cnt:
            self.assertEqual('free', c.access)

        volume.set_contents_access('abo')

        for c in cnt:
            self.assertEqual('abopflichtig', c.access)
<|endoftext|>


<commit_before>from __future__ import unicode_literals

from django.core.urlresolvers import reverse_lazy
from django.utils.text import format_lazy

DEBUG = True

ROOT_URLCONF = 'testshop.urls'

SECRET_KEY = 'test'

SITE_ID = 1

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': ':memory:',
    }
}

STATIC_URL = '/static/'

MEDIA_URL = '/media/'

TEMPLATES = [{
    'BACKEND': 'django.template.backends.django.DjangoTemplates',
    'APP_DIRS': True,
    'DIRS': [],
    'OPTIONS': {
        'context_processors': [
            'django.contrib.auth.context_processors.auth',
            'django.template.context_processors.debug',
            'django.template.context_processors.i18n',
            'django.template.context_processors.media',
            'django.template.context_processors.static',
            'django.template.context_processors.tz',
            'django.template.context_processors.csrf',
            'django.template.context_processors.request',
            'django.contrib.messages.context_processors.messages',
            'sekizai.context_processors.sekizai',
            'cms.context_processors.cms_settings',
        ]
    }
}, {
    'BACKEND': 'html_email.template.backends.html_email.EmailTemplates',
    'APP_DIRS': True,
    'DIRS': [],
    'OPTIONS': {
        'context_processors': [
            'django.contrib.auth.context_processors.auth',
            'django.template.context_processors.debug',
            'django.template.context_processors.i18n',
            'django.template.context_processors.media',
            'django.template.context_processors.static',
            'django.template.context_processors.tz',
            'django.template.context_processors.request',
        ]
    }
}]

MIDDLEWARE_CLASSES = [
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'shop.middleware.CustomerMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.locale.LocaleMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'cms.middleware.user.CurrentUserMiddleware',
    'cms.middleware.page.CurrentPageMiddleware',
    'cms.middleware.toolbar.ToolbarMiddleware',
    'cms.middleware.language.LanguageCookieMiddleware',
]

INSTALLED_APPS = [
    'django.contrib.auth',
    'email_auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.admin',
    'django.contrib.staticfiles',
    'jsonfield',
    'rest_framework',
    'rest_framework.authtoken',
    'rest_auth',
    'django_fsm',
    'fsm_admin',
    'filer',
    'easy_thumbnails',
    'treebeard',
    'menus',
    'sekizai',
    'cms',
    'adminsortable2',
    'djangocms_text_ckeditor',
    'cmsplugin_cascade',
    'cmsplugin_cascade.clipboard',
    'cmsplugin_cascade.extra_fields',
    'cmsplugin_cascade.icon',
    'cmsplugin_cascade.sharable',
    'cmsplugin_cascade.segmentation',
    'post_office',
    'html_email',
    'shop',
    'testshop',
]

USE_I18N = False

USE_L10N = True

USE_TZ = True

LANGUAGES = [
    ('en', 'English'),
]

LANGUAGE_CODE = 'en'

SESSION_ENGINE = 'django.contrib.sessions.backends.cache'

CMS_TEMPLATES = [
    ('page.html', "Default Page"),
]

CMS_PLACEHOLDER_CONF = {
    'Main Content': {
        'plugins': ['BootstrapContainerPlugin'],
    },
}

CMSPLUGIN_CASCADE_PLUGINS = [
    'cmsplugin_cascade.bootstrap4',
    'cmsplugin_cascade.segmentation',
    'cmsplugin_cascade.generic',
    'cmsplugin_cascade.icon',
    'cmsplugin_cascade.leaflet',
    'cmsplugin_cascade.link',
    'shop.cascade',
]

CMSPLUGIN_CASCADE = {
    'link_plugin_classes': [
        'shop.cascade.plugin_base.CatalogLinkPluginBase',
        'cmsplugin_cascade.link.plugin_base.LinkElementMixin',
        'shop.cascade.plugin_base.CatalogLinkForm',
    ],
    'alien_plugins': ['TextPlugin', 'TextLinkPlugin', 'AcceptConditionPlugin'],
    'bootstrap4': {
        'template_basedir': 'angular-ui',
    },
    'segmentation_mixins': [
        ('shop.cascade.segmentation.EmulateCustomerModelMixin', 'shop.cascade.segmentation.EmulateCustomerAdminMixin'),
    ],
}

THUMBNAIL_PROCESSORS = (
    'easy_thumbnails.processors.colorspace',
    'easy_thumbnails.processors.autocrop',
    'filer.thumbnail_processors.scale_and_crop_with_subject_location',
    'easy_thumbnails.processors.filters',
)

THUMBNAIL_PRESERVE_EXTENSIONS = True,

CKEDITOR_SETTINGS = {
    'language': '{{ language }}',
    'skin': 'moono',
    'toolbar': 'CMS',
    'toolbar_HTMLField': [
        ['Undo', 'Redo'],
        ['cmsplugins', '-', 'ShowBlocks'],
        ['Format', 'Styles'],
        ['TextColor', 'BGColor', '-', 'PasteText', 'PasteFromWord'],
        ['Maximize', ''],
        '/',
        ['Bold', 'Italic', 'Underline', '-', 'Subscript', 'Superscript', '-', 'RemoveFormat'],
        ['JustifyLeft', 'JustifyCenter', 'JustifyRight'],
        ['HorizontalRule'],
        ['NumberedList', 'BulletedList', '-', 'Outdent', 'Indent', '-', 'Table'],
        ['Source']
    ],
    'stylesSet': format_lazy('default:{}', reverse_lazy('admin:cascade_texticon_wysiwig_config')),
}

SILENCED_SYSTEM_CHECKS = ['auth.W004']

SHOP_APP_LABEL = 'testshop'

SHOP_CART_MODIFIERS = [
    'shop.modifiers.defaults.DefaultCartModifier',
    'shop.modifiers.taxes.CartIncludeTaxModifier',
    'shop.payment.modifiers.PayInAdvanceModifier',
    'shop.shipping.modifiers.SelfCollectionModifier',
]

SHOP_ORDER_WORKFLOWS = [
    'shop.payment.workflows.ManualPaymentWorkflowMixin',
    'shop.payment.workflows.CancelOrderWorkflowMixin',
    'shop.shipping.workflows.PartialDeliveryWorkflowMixin',
]

AUTH_USER_MODEL = 'email_auth.User'

AUTHENTICATION_BACKENDS = [
    'django.contrib.auth.backends.ModelBackend',
    'allauth.account.auth_backends.AuthenticationBackend',
]

REST_AUTH_SERIALIZERS = {
    'LOGIN_SERIALIZER': 'shop.serializers.auth.LoginSerializer',
}
<commit_msg>submodule ‘html_email’ is gone<commit_after>from __future__ import unicode_literals

from django.core.urlresolvers import reverse_lazy
from django.utils.text import format_lazy

DEBUG = True

ROOT_URLCONF = 'testshop.urls'

SECRET_KEY = 'test'

SITE_ID = 1

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': ':memory:',
    }
}

STATIC_URL = '/static/'

MEDIA_URL = '/media/'

TEMPLATES = [{
    'BACKEND': 'django.template.backends.django.DjangoTemplates',
    'APP_DIRS': True,
    'DIRS': [],
    'OPTIONS': {
        'context_processors': [
            'django.contrib.auth.context_processors.auth',
            'django.template.context_processors.debug',
            'django.template.context_processors.i18n',
            'django.template.context_processors.media',
            'django.template.context_processors.static',
            'django.template.context_processors.tz',
            'django.template.context_processors.csrf',
            'django.template.context_processors.request',
            'django.contrib.messages.context_processors.messages',
            'sekizai.context_processors.sekizai',
            'cms.context_processors.cms_settings',
        ]
    }
}, {
    'BACKEND': 'html_email.template.backends.html_email.EmailTemplates',
    'APP_DIRS': True,
    'DIRS': [],
    'OPTIONS': {
        'context_processors': [
            'django.contrib.auth.context_processors.auth',
            'django.template.context_processors.debug',
            'django.template.context_processors.i18n',
            'django.template.context_processors.media',
            'django.template.context_processors.static',
            'django.template.context_processors.tz',
            'django.template.context_processors.request',
        ]
    }
}]

MIDDLEWARE_CLASSES = [
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'shop.middleware.CustomerMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.locale.LocaleMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
    'cms.middleware.user.CurrentUserMiddleware',
    'cms.middleware.page.CurrentPageMiddleware',
    'cms.middleware.toolbar.ToolbarMiddleware',
    'cms.middleware.language.LanguageCookieMiddleware',
]

INSTALLED_APPS = [
    'django.contrib.auth',
    'email_auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.admin',
    'django.contrib.staticfiles',
    'jsonfield',
    'rest_framework',
    'rest_framework.authtoken',
    'rest_auth',
    'django_fsm',
    'fsm_admin',
    'filer',
    'easy_thumbnails',
    'treebeard',
    'menus',
    'sekizai',
    'cms',
    'adminsortable2',
    'djangocms_text_ckeditor',
    'cmsplugin_cascade',
    'cmsplugin_cascade.clipboard',
    'cmsplugin_cascade.extra_fields',
    'cmsplugin_cascade.icon',
    'cmsplugin_cascade.sharable',
    'cmsplugin_cascade.segmentation',
    'post_office',
    'shop',
    'testshop',
]

USE_I18N = False

USE_L10N = True

USE_TZ = True

LANGUAGES = [
    ('en', 'English'),
]

LANGUAGE_CODE = 'en'

SESSION_ENGINE = 'django.contrib.sessions.backends.cache'

CMS_TEMPLATES = [
    ('page.html', "Default Page"),
]

CMS_PLACEHOLDER_CONF = {
    'Main Content': {
        'plugins': ['BootstrapContainerPlugin'],
    },
}

CMSPLUGIN_CASCADE_PLUGINS = [
    'cmsplugin_cascade.bootstrap4',
    'cmsplugin_cascade.segmentation',
    'cmsplugin_cascade.generic',
    'cmsplugin_cascade.icon',
    'cmsplugin_cascade.leaflet',
    'cmsplugin_cascade.link',
    'shop.cascade',
]

CMSPLUGIN_CASCADE = {
    'link_plugin_classes': [
        'shop.cascade.plugin_base.CatalogLinkPluginBase',
        'cmsplugin_cascade.link.plugin_base.LinkElementMixin',
        'shop.cascade.plugin_base.CatalogLinkForm',
    ],
    'alien_plugins': ['TextPlugin', 'TextLinkPlugin', 'AcceptConditionPlugin'],
    'bootstrap4': {
        'template_basedir': 'angular-ui',
    },
    'segmentation_mixins': [
        ('shop.cascade.segmentation.EmulateCustomerModelMixin', 'shop.cascade.segmentation.EmulateCustomerAdminMixin'),
    ],
}

THUMBNAIL_PROCESSORS = (
    'easy_thumbnails.processors.colorspace',
    'easy_thumbnails.processors.autocrop',
    'filer.thumbnail_processors.scale_and_crop_with_subject_location',
    'easy_thumbnails.processors.filters',
)

THUMBNAIL_PRESERVE_EXTENSIONS = True,

CKEDITOR_SETTINGS = {
    'language': '{{ language }}',
    'skin': 'moono',
    'toolbar': 'CMS',
    'toolbar_HTMLField': [
        ['Undo', 'Redo'],
        ['cmsplugins', '-', 'ShowBlocks'],
        ['Format', 'Styles'],
        ['TextColor', 'BGColor', '-', 'PasteText', 'PasteFromWord'],
        ['Maximize', ''],
        '/',
        ['Bold', 'Italic', 'Underline', '-', 'Subscript', 'Superscript', '-', 'RemoveFormat'],
        ['JustifyLeft', 'JustifyCenter', 'JustifyRight'],
        ['HorizontalRule'],
        ['NumberedList', 'BulletedList', '-', 'Outdent', 'Indent', '-', 'Table'],
        ['Source']
    ],
    'stylesSet': format_lazy('default:{}', reverse_lazy('admin:cascade_texticon_wysiwig_config')),
}

SILENCED_SYSTEM_CHECKS = ['auth.W004']

SHOP_APP_LABEL = 'testshop'

SHOP_CART_MODIFIERS = [
    'shop.modifiers.defaults.DefaultCartModifier',
    'shop.modifiers.taxes.CartIncludeTaxModifier',
    'shop.payment.modifiers.PayInAdvanceModifier',
    'shop.shipping.modifiers.SelfCollectionModifier',
]

SHOP_ORDER_WORKFLOWS = [
    'shop.payment.workflows.ManualPaymentWorkflowMixin',
    'shop.payment.workflows.CancelOrderWorkflowMixin',
    'shop.shipping.workflows.PartialDeliveryWorkflowMixin',
]

AUTH_USER_MODEL = 'email_auth.User'

AUTHENTICATION_BACKENDS = [
    'django.contrib.auth.backends.ModelBackend',
    'allauth.account.auth_backends.AuthenticationBackend',
]

REST_AUTH_SERIALIZERS = {
    'LOGIN_SERIALIZER': 'shop.serializers.auth.LoginSerializer',
}
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
#  $Id:$
#

import os
import sys
import optparse

a = os.path.join("../")
sys.path.append(a)
a = os.path.join("../../")
sys.path.append(a)
a = os.path.join("../../..")
sys.path.append(a)

from zoni.version import version, revision
from zoni.extra.util import createKey

def main():
	''' This file sets up the database for Zoni '''

	ver = version.split(" ")[0]
	rev = revision

	parser = optparse.OptionParser(usage="%prog -k keyname", version="%prog " + ver + " " + rev)
	parser.add_option("-k", "--keyName", "--keyname", dest="keyName", help="Key name")
	#parser.add_option("-v", "--verbose", dest="verbosity", help="Be verbose", action="store_true", default=False)
	(options, __args) = parser.parse_args()

	if not options.keyName:
		parser.print_help()
		exit(1)

	#(configs, configFiles) = getConfig()

	key = createKey(options.keyName)
	print "##################  DHCP  #####################"
	print "#  Put the following lines in your dhcpd.conf file\n\n"
	print "key %s {" % options.keyName
	print "    algorithm hmac-md5;"
	print "    secret %s" % key
	print "};"
	print ""
	print "omapi-key %s" % options.keyName
	print "omapi-port 7911;\n\n"

	print "##################  DNS  #####################"
	print "#  Put the following in your dns configuration files"
	print "#  E.g. /etc/bind/named.conf.local\n\n"
	print "key %s { algorithm hmac-md5; secret \"%s\"; };" % (options.keyName, key)
	print "\n\nDon't forget to add the following to your zone configs to allow updates\n\n"
	print "allow-update { key %s; };" % options.keyName
	print "\n\n"

if __name__ == "__main__":
	main()

<commit_msg>Adding a semicolon to the end of the key<commit_after>#!/usr/bin/env python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
#  $Id:$
#

import os
import sys
import optparse

a = os.path.join("../")
sys.path.append(a)
a = os.path.join("../../")
sys.path.append(a)
a = os.path.join("../../..")
sys.path.append(a)

from zoni.version import version, revision
from zoni.extra.util import createKey

def main():
	''' This file sets up the database for Zoni '''

	ver = version.split(" ")[0]
	rev = revision

	parser = optparse.OptionParser(usage="%prog -k keyname", version="%prog " + ver + " " + rev)
	parser.add_option("-k", "--keyName", "--keyname", dest="keyName", help="Key name")
	#parser.add_option("-v", "--verbose", dest="verbosity", help="Be verbose", action="store_true", default=False)
	(options, __args) = parser.parse_args()

	if not options.keyName:
		parser.print_help()
		exit(1)

	#(configs, configFiles) = getConfig()

	key = createKey(options.keyName)
	print "##################  DHCP  #####################"
	print "#  Put the following lines in your dhcpd.conf file\n\n"
	print "key %s {" % options.keyName
	print "    algorithm hmac-md5;"
	print "    secret %s;" % key
	print "};"
	print ""
	print "omapi-key %s" % options.keyName
	print "omapi-port 7911;\n\n"

	print "##################  DNS  #####################"
	print "#  Put the following in your dns configuration files"
	print "#  E.g. /etc/bind/named.conf.local\n\n"
	print "key %s { algorithm hmac-md5; secret \"%s\"; };" % (options.keyName, key)
	print "\n\nDon't forget to add the following to your zone configs to allow updates\n\n"
	print "allow-update { key %s; };" % options.keyName
	print "\n\n"

if __name__ == "__main__":
	main()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Copyright [2009-2017] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import os
import re
import csv
import json
import logging
from glob import glob
import functools as ft
from xml.etree import cElementTree as ET

import six

try:
    import dbm
except:
    from six.moves import dbm_gnu as dbm

import attr
from attr.validators import instance_of as is_a

import typing

try:
    from pathlib import Path
except ImportError:
    from pathlib2 import Path

from contextlib import contextmanager
try:
    from contextlib import ExitStack
except:
    from contextlib2 import ExitStack

from .utils import clean_title
from .utils import pretty_location
from .utils import write_lookup
from .fetch import lookup

from rnacentral_pipeline.databases.data import Reference
from rnacentral_pipeline.databases.data import IdReference
from rnacentral_pipeline.databases.data import KnownServices
from rnacentral_pipeline.databases.helpers.publications import reference

LOGGER = logging.getLogger(__name__)

class UncachedReference(Exception):
    """
    This is created when the requested reference was never indexed. This may be
    because it is not an open access publication, or it does not exist at all.
    """
    pass

@attr.s()
class Storage(object):
    namespace = attr.ib(validator=is_a(KnownServices))
    path = attr.ib(validator=is_a(Path))

    @classmethod
    def build(cls, service, base_path):
        return Storage(service, base_path / service.name)

    def store(self, id_ref, json_data):
        assert id_ref.namespace is self.namespace
        key = self.__normalize_key__(id_ref.external_id)
        self.db[key] = json_data

    def get(self, id_ref):
        assert id_ref.namespace is self.namespace
        key = self.__normalize_key__(id_ref.external_id)
        data = self.db.get(key, None)
        if not data:
            return None
        return Reference(**json.loads(data))

    @contextmanager
    def open(self, mode='r'):
        self.db = dbm.open(str(self.path), mode)
        yield self
        self.db.close()

    def __normalize_key__(self, raw):
        encoded = str(raw)
        if isinstance(raw, six.string_types):
            encoded = raw.encode('ascii', 'ignore')
        assert encoded
        return encoded

@attr.s()
class Cache(object):
    _path = attr.ib(type=Path, validator=is_a(Path))
    _pmid = attr.ib(validator=is_a(Storage))
    _doi = attr.ib(validator=is_a(Storage))
    _pmcid = attr.ib(validator=is_a(Storage))

    @classmethod
    def build(cls, base_path):
        path = Path(base_path)
        if not path.exists():
            path.mkdir()
        kwargs = {}
        for service in KnownServices:
            kwargs[service.name] = Storage.build(service, path)
        return cls(path=path, **kwargs)

    @classmethod
    def populate_with(cls, base_path, references):
        cache = cls.build(base_path)
        with cache.open(mode='c') as cache:
            for reference in references:
                cache.store(reference)

    @contextmanager
    def open(self, mode='r'):
        with ExitStack() as stack:
            for service in KnownServices:
                storage = self.__storage__(service)
                db = stack.enter_context(storage.open(mode))
            yield self

    def get(self, id_reference, allow_fallback=False):
        storage = self.__storage__(id_reference.namespace)
        data = storage.get(id_reference)
        if data:
            return data
        elif allow_fallback:
            return lookup(id_reference)
        raise UncachedReference(id_reference)

    def store(self, reference):
        data = json.dumps(attr.asdict(reference))
        for id_reference in reference.id_references:
            storage = self.__storage__(id_reference)
            storage.store(id_reference, data)

    def __storage__(self, ref):
        if isinstance(ref, KnownServices):
            name = ref.name
        elif isinstance(ref, IdReference):
            name = ref.namespace.name
        else:
            raise ValueError("Unknown type: %s" % ref)

        key = '_' + name
        if not hasattr(self, key):
            raise ValueError("Never cached references for: %s" % name)
        return getattr(self, key)

def xml_text(tag, node, missing=None, fn=None):
    found = node.find(tag)
    if found is None or not found.text:
        return missing
    if fn:
        return fn(found.text)
    return found.text

def node_to_reference(node):
    pmid = xml_text('pmid', node, fn=int)
    doi = xml_text('DOI', node)
    if not pmid and not doi:
        return None
    pmcid = six.text_type(xml_text('pmcid', node))

    authors = []
    for author in node.findall('./AuthorList/Author'):
        last = xml_text('LastName', author)
        initials = xml_text('Initials', author)
        authors.append('%s %s' % (last, initials))
    authors = ', '.join(authors) + '.'

    data = {
        'journalTitle': xml_text('journalTitle', node, missing=''),
        'issue': xml_text('issue', node, missing=''),
        'journalVolume': xml_text('journalVolume', node, missing=''),
        'pageInfo': xml_text('pageInfo', node, missing=''),
        'pubYear': xml_text('pubYear', node, missing=''),
    }

    return Reference(
        authors=six.text_type(authors),
        location=six.text_type(pretty_location(data)),
        title=six.text_type(xml_text('title', node, fn=clean_title)),
        pmid=pmid,
        doi=six.text_type(doi),
        pmcid=pmcid,
    )

def parse(xml_file):
    tree = ET.parse(xml_file)
    root = tree.getroot()
    for node in root.findall('./PMC_ARTICLE'):
        ref = node_to_reference(node)
        if not ref:
            continue
        yield ref

def parse_directory(directory):
    for filename in glob(os.path.join(directory, '*.xml')):
        with open(filename, 'r') as xml_file:
            for ref in list(parse(xml_file)):
                yield ref

def index_directory(directory, output):
    Cache.populate_with(output, parse_directory(directory))

def id_refs_from_handle(handle, column=0):
    for row in csv.reader(handle):
        raw = row[column]
        rest = [d for i, d in enumerate(row) if i != column]
        yield (reference(raw), rest)

def write_file_lookup(cache_path, handle, output, column=0, allow_fallback=False,
                      ignore_missing=True):
    with Cache.build(cache_path).open() as db:
        write_lookup(
            ft.partial(db.get, allow_fallback=allow_fallback),
            handle,
            output,
            column=column,
            ignore_errors=ignore_missing,
        )
<commit_msg>Use absolute import<commit_after># -*- coding: utf-8 -*-

from __future__ import absolute_import

"""
Copyright [2009-2017] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import os
import re
import csv
import json
import logging
from glob import glob
import functools as ft
from xml.etree import cElementTree as ET

import six

try:
    import dbm
except:
    from six.moves import dbm_gnu as dbm

import attr
from attr.validators import instance_of as is_a

import typing

try:
    from pathlib import Path
except ImportError:
    from pathlib2 import Path

from contextlib import contextmanager
try:
    from contextlib import ExitStack
except:
    from contextlib2 import ExitStack

from rnacentral_pipeline.databases.data import Reference
from rnacentral_pipeline.databases.data import IdReference
from rnacentral_pipeline.databases.data import KnownServices
from rnacentral_pipeline.databases.helpers.publications import reference

from rnacentral_pipeline.databases.europepmc.utils import clean_title
from rnacentral_pipeline.databases.europepmc.utils import pretty_location
from rnacentral_pipeline.databases.europepmc.utils import write_lookup
from rnacentral_pipeline.databases.europepmc.fetch import lookup

LOGGER = logging.getLogger(__name__)

class UncachedReference(Exception):
    """
    This is created when the requested reference was never indexed. This may be
    because it is not an open access publication, or it does not exist at all.
    """
    pass

@attr.s()
class Storage(object):
    namespace = attr.ib(validator=is_a(KnownServices))
    path = attr.ib(validator=is_a(Path))

    @classmethod
    def build(cls, service, base_path):
        return Storage(service, base_path / service.name)

    def store(self, id_ref, json_data):
        assert id_ref.namespace is self.namespace
        key = self.__normalize_key__(id_ref.external_id)
        self.db[key] = json_data

    def get(self, id_ref):
        assert id_ref.namespace is self.namespace
        key = self.__normalize_key__(id_ref.external_id)
        data = self.db.get(key, None)
        if not data:
            return None
        return Reference(**json.loads(data))

    @contextmanager
    def open(self, mode='r'):
        self.db = dbm.open(str(self.path), mode)
        yield self
        self.db.close()

    def __normalize_key__(self, raw):
        encoded = str(raw)
        if isinstance(raw, six.string_types):
            encoded = raw.encode('ascii', 'ignore')
        assert encoded
        return encoded

@attr.s()
class Cache(object):
    _path = attr.ib(type=Path, validator=is_a(Path))
    _pmid = attr.ib(validator=is_a(Storage))
    _doi = attr.ib(validator=is_a(Storage))
    _pmcid = attr.ib(validator=is_a(Storage))

    @classmethod
    def build(cls, base_path):
        path = Path(base_path)
        if not path.exists():
            path.mkdir()
        kwargs = {}
        for service in KnownServices:
            kwargs[service.name] = Storage.build(service, path)
        return cls(path=path, **kwargs)

    @classmethod
    def populate_with(cls, base_path, references):
        cache = cls.build(base_path)
        with cache.open(mode='c') as cache:
            for reference in references:
                cache.store(reference)

    @contextmanager
    def open(self, mode='r'):
        with ExitStack() as stack:
            for service in KnownServices:
                storage = self.__storage__(service)
                db = stack.enter_context(storage.open(mode))
            yield self

    def get(self, id_reference, allow_fallback=False):
        storage = self.__storage__(id_reference.namespace)
        data = storage.get(id_reference)
        if data:
            return data
        elif allow_fallback:
            return lookup(id_reference)
        raise UncachedReference(id_reference)

    def store(self, reference):
        data = json.dumps(attr.asdict(reference))
        for id_reference in reference.id_references:
            storage = self.__storage__(id_reference)
            storage.store(id_reference, data)

    def __storage__(self, ref):
        if isinstance(ref, KnownServices):
            name = ref.name
        elif isinstance(ref, IdReference):
            name = ref.namespace.name
        else:
            raise ValueError("Unknown type: %s" % ref)

        key = '_' + name
        if not hasattr(self, key):
            raise ValueError("Never cached references for: %s" % name)
        return getattr(self, key)

def xml_text(tag, node, missing=None, fn=None):
    found = node.find(tag)
    if found is None or not found.text:
        return missing
    if fn:
        return fn(found.text)
    return found.text

def node_to_reference(node):
    pmid = xml_text('pmid', node, fn=int)
    doi = xml_text('DOI', node)
    if not pmid and not doi:
        return None
    pmcid = six.text_type(xml_text('pmcid', node))

    authors = []
    for author in node.findall('./AuthorList/Author'):
        last = xml_text('LastName', author)
        initials = xml_text('Initials', author)
        authors.append('%s %s' % (last, initials))
    authors = ', '.join(authors) + '.'

    data = {
        'journalTitle': xml_text('journalTitle', node, missing=''),
        'issue': xml_text('issue', node, missing=''),
        'journalVolume': xml_text('journalVolume', node, missing=''),
        'pageInfo': xml_text('pageInfo', node, missing=''),
        'pubYear': xml_text('pubYear', node, missing=''),
    }

    return Reference(
        authors=six.text_type(authors),
        location=six.text_type(pretty_location(data)),
        title=six.text_type(xml_text('title', node, fn=clean_title)),
        pmid=pmid,
        doi=six.text_type(doi),
        pmcid=pmcid,
    )

def parse(xml_file):
    tree = ET.parse(xml_file)
    root = tree.getroot()
    for node in root.findall('./PMC_ARTICLE'):
        ref = node_to_reference(node)
        if not ref:
            continue
        yield ref

def parse_directory(directory):
    for filename in glob(os.path.join(directory, '*.xml')):
        with open(filename, 'r') as xml_file:
            for ref in list(parse(xml_file)):
                yield ref

def index_directory(directory, output):
    Cache.populate_with(output, parse_directory(directory))

def id_refs_from_handle(handle, column=0):
    for row in csv.reader(handle):
        raw = row[column]
        rest = [d for i, d in enumerate(row) if i != column]
        yield (reference(raw), rest)

def write_file_lookup(cache_path, handle, output, column=0, allow_fallback=False,
                      ignore_missing=True):
    with Cache.build(cache_path).open() as db:
        write_lookup(
            ft.partial(db.get, allow_fallback=allow_fallback),
            handle,
            output,
            column=column,
            ignore_errors=ignore_missing,
        )
<|endoftext|>


<commit_before>import _path_pyblosxom
import time
from StringIO import StringIO

from nose.tools import eq_, raises

from Pyblosxom.tools import STANDARD_FILTERS
from Pyblosxom.pyblosxom import Request
from Pyblosxom.entries.base import EntryBase, generate_entry

from helpers import req_

TIME1 = (2008, 7, 21, 12, 51, 47, 0, 203, 1)

class TestEntryBase:
    def test_data(self):
        e = EntryBase(req_())
        yield eq_, e.getData(), ""

        s1 = "la la la la la"
        e.setData(s1)
        yield eq_, e.getData(), s1
        yield eq_, type(e.getData()), str

        s2 = u"foo foo foo foo foo"
        e.setData(s2)
        yield eq_, e.getData(), s2
        yield eq_, type(e.getData()), str

        s3 = "foo bar"
        e.setData(StringIO(s3))
        yield eq_, e.getData(), s3

    def test_metadata(self):
        e = EntryBase(req_())
        yield eq_, e.getMetadataKeys(), STANDARD_FILTERS.keys()
        yield eq_, e.getMetadata("foo"), None
        yield eq_, e.getMetadata("foo", "bar"), "bar"
        e.setMetadata("foo", "bar")
        yield eq_, e.getMetadata("foo"), "bar"

    def test_time(self):
        # FIXME - these tests are locale dependent
        e = EntryBase(req_())
        e.setTime(TIME1)
        for mem in (("timetuple", TIME1),
                    ("mtime", 1216659107.0),
                    ("ti", "12:51"),
                    ("mo", "Jul"),
                    ("mo_num", "07"),
                    ("da", "21"),
                    ("dw", "Monday"),
                    ("yr", "2008"),
                    ("fulltime", "20080721125147"),
                    ("date", "Mon, 21 Jul 2008"),
                    ("w3cdate", "2008-07-21T16:51:47Z"),
                    ("rfc822date", "Mon, 21 Jul 2008 16:51 GMT")):
            yield eq_, e[mem[0]], mem[1]

    def test_dictlike(self):
        e = EntryBase(req_())
        e["foo"] = "bar"
        e["body"] = "entry body"

        def sortlist(l):
            l.sort()
            return l

        yield eq_, sortlist(e.keys()), sortlist(STANDARD_FILTERS.keys() + ["foo", "body"])

        yield eq_, e["foo"], "bar"
        yield eq_, e.get("foo"), "bar"
        yield eq_, e.get("foo", "fickle"), "bar"
        yield eq_, e.getMetadata("foo"), "bar"
        yield eq_, e.getMetadata("foo", "fickle"), "bar"

        yield eq_, e["body"], "entry body", "e[\"body\"]"
        yield eq_, e.get("body"), "entry body", "e.get(\"body\")"
        yield eq_, e.getData(), "entry body", "e.getData()"

        yield eq_, e.get("missing_key", "default"), "default"
        yield eq_, e.get("missing_key"), None

        e.set("faz", "baz")
        yield eq_, e.get("faz"), "baz"

        yield eq_, e.has_key("foo"), True
        yield eq_, e.has_key("foo2"), False
        yield eq_, e.has_key("body"), True
        yield eq_, "foo" in e, True
        yield eq_, "foo2" in e, False
        yield eq_, "foo2" not in e, True
        yield eq_, "body" in e, True

        e.update({"foo": "bah", "faux": "pearls"})
        yield eq_, e["foo"], "bah"
        yield eq_, e["faux"], "pearls"

        e.update({"body": "new body data"})
        yield eq_, e["body"], "new body data"
        yield eq_, e.getData(), "new body data"

        del e["foo"]
        yield eq_, e.get("foo"), None

    @raises(KeyError)
    def test_delitem_keyerror(self):
        e = EntryBase(req_())
        del e["missing_key"]

    @raises(ValueError)
    def test_delitem_valueerror(self):
        e = EntryBase(req_())
        del e["body"]

    def test_generate_entry(self):
        e = generate_entry(req_(), {"foo": "bar"}, "entry body", TIME1)

        yield eq_, e["foo"], "bar"
        yield eq_, e["body"], "entry body"
        yield eq_, e["rfc822date"], "Mon, 21 Jul 2008 16:51 GMT"

        e = generate_entry(req_(), {"foo": "bar"}, "entry body")

    def test_repr(self):
        # it doesn't really matter what __repr__ sends back--it's only used
        # for logging/debugging.  so this test adds coverage for that line to
        # make sure it doesn't error out.
        e = EntryBase(req_())
        repr(e)
<commit_msg>Cosmetic changes to test_entries.py.<commit_after>import _path_pyblosxom
import time
from StringIO import StringIO

from nose.tools import eq_, raises

from helpers import req_

from Pyblosxom.tools import STANDARD_FILTERS
from Pyblosxom.pyblosxom import Request
from Pyblosxom.entries.base import EntryBase, generate_entry

TIME1 = (2008, 7, 21, 12, 51, 47, 0, 203, 1)

class TestEntryBase:
    def test_data(self):
        e = EntryBase(req_())
        yield eq_, e.getData(), ""

        s1 = "la la la la la"
        e.setData(s1)
        yield eq_, e.getData(), s1
        yield eq_, type(e.getData()), str

        s2 = u"foo foo foo foo foo"
        e.setData(s2)
        yield eq_, e.getData(), s2
        yield eq_, type(e.getData()), str

        s3 = "foo bar"
        e.setData(StringIO(s3))
        yield eq_, e.getData(), s3

    def test_metadata(self):
        e = EntryBase(req_())
        yield eq_, e.getMetadataKeys(), STANDARD_FILTERS.keys()
        yield eq_, e.getMetadata("foo"), None
        yield eq_, e.getMetadata("foo", "bar"), "bar"
        e.setMetadata("foo", "bar")
        yield eq_, e.getMetadata("foo"), "bar"

    def test_time(self):
        # FIXME - these tests are locale dependent
        e = EntryBase(req_())
        e.setTime(TIME1)
        for mem in (("timetuple", TIME1),
                    ("mtime", 1216659107.0),
                    ("ti", "12:51"),
                    ("mo", "Jul"),
                    ("mo_num", "07"),
                    ("da", "21"),
                    ("dw", "Monday"),
                    ("yr", "2008"),
                    ("fulltime", "20080721125147"),
                    ("date", "Mon, 21 Jul 2008"),
                    ("w3cdate", "2008-07-21T16:51:47Z"),
                    ("rfc822date", "Mon, 21 Jul 2008 16:51 GMT")):
            yield eq_, e[mem[0]], mem[1], \
                  "%s != %s (note: this is a locale dependent test)" % (mem[0], mem[1])

    def test_dictlike(self):
        e = EntryBase(req_())
        e["foo"] = "bar"
        e["body"] = "entry body"

        def sortlist(l):
            l.sort()
            return l

        yield eq_, sortlist(e.keys()), sortlist(STANDARD_FILTERS.keys() + ["foo", "body"])

        yield eq_, e["foo"], "bar"
        yield eq_, e.get("foo"), "bar"
        yield eq_, e.get("foo", "fickle"), "bar"
        yield eq_, e.getMetadata("foo"), "bar"
        yield eq_, e.getMetadata("foo", "fickle"), "bar"

        yield eq_, e["body"], "entry body", "e[\"body\"]"
        yield eq_, e.get("body"), "entry body", "e.get(\"body\")"
        yield eq_, e.getData(), "entry body", "e.getData()"

        yield eq_, e.get("missing_key", "default"), "default"
        yield eq_, e.get("missing_key"), None

        e.set("faz", "baz")
        yield eq_, e.get("faz"), "baz"

        yield eq_, e.has_key("foo"), True
        yield eq_, e.has_key("foo2"), False
        yield eq_, e.has_key("body"), True
        yield eq_, "foo" in e, True
        yield eq_, "foo2" in e, False
        yield eq_, "foo2" not in e, True
        yield eq_, "body" in e, True

        e.update({"foo": "bah", "faux": "pearls"})
        yield eq_, e["foo"], "bah"
        yield eq_, e["faux"], "pearls"

        e.update({"body": "new body data"})
        yield eq_, e["body"], "new body data"
        yield eq_, e.getData(), "new body data"

        del e["foo"]
        yield eq_, e.get("foo"), None

    @raises(KeyError)
    def test_delitem_keyerror(self):
        e = EntryBase(req_())
        del e["missing_key"]

    @raises(ValueError)
    def test_delitem_valueerror(self):
        e = EntryBase(req_())
        del e["body"]

    def test_generate_entry(self):
        e = generate_entry(req_(), {"foo": "bar"}, "entry body", TIME1)

        yield eq_, e["foo"], "bar"
        yield eq_, e["body"], "entry body"
        yield eq_, e["rfc822date"], "Mon, 21 Jul 2008 16:51 GMT"

        e = generate_entry(req_(), {"foo": "bar"}, "entry body")

    def test_repr(self):
        # it doesn't really matter what __repr__ sends back--it's only used
        # for logging/debugging.  so this test adds coverage for that line to
        # make sure it doesn't error out.
        e = EntryBase(req_())
        repr(e)
<|endoftext|>


<commit_before>from __future__ import division
from os.path import join, dirname
from subprocess import check_call
import argparse
import json
import numpy as np
from fabric.api import local
from fabric.context_managers import warn_only

from engine.db import DB
from engine.util import proj_root, hdfs_prefix, SMConfig

# def sm_config():
#     with open(join(proj_root(), 'conf/config.json')) as f:
#         return json.load(f)

SMConfig.set_path(join(proj_root(), 'conf/config.json'))

ds_name = 'spheroid_12h'
data_dir_path = join(SMConfig.get_conf()['fs']['data_dir'], ds_name)
input_dir_path = join(proj_root(), 'test/data/sci_test_search_job_spheroid_dataset')

SEARCH_RES_SELECT = ("select sf, adduct, stats "
                     "from iso_image_metrics s "
                     "join formula_db sm_db on sm_db.id = s.db_id "
                     "join agg_formula f on f.id = s.sf_id "
                     "join job j on j.id = s.job_id "
                     "join dataset ds on ds.id = j.ds_id "
                     "where ds.name = %s and sm_db.name = %s "
                     "ORDER BY sf, adduct ")

def compare_search_results(base_search_res, search_res):
    missed_sf_adduct = set(base_search_res.keys()).difference(set(search_res.keys()))
    print 'MISSED FORMULAS: {:.1f}%'.format(len(missed_sf_adduct) / len(base_search_res) * 100)
    print list(missed_sf_adduct)

    new_sf_adduct = set(search_res.keys()).difference(set(base_search_res.keys()))
    print 'FALSE DISCOVERY: {:.1f}%'.format(len(new_sf_adduct) / len(base_search_res) * 100)
    print list(new_sf_adduct)

    print 'DIFFERENCE IN METRICS'
    for b_sf_add, b_metr in base_search_res.iteritems():
        if b_sf_add in search_res.keys():
            metr = search_res[b_sf_add]
            diff = np.abs(b_metr - metr)
            if np.any(diff > 1e-6):
                print '{} metrics diff = {}'.format(b_sf_add, diff)

def zip_engine():
    local('cd {}; zip -rq engine.zip engine'.format(proj_root()))

def run_search():
    cmd = ['python',
           join(proj_root(), 'scripts/run_molecule_search.py'),
           ds_name,
           input_dir_path]
    check_call(cmd)

def clear_data_dirs():
    with warn_only():
        local('rm -rf {}'.format(data_dir_path))
        local(hdfs_prefix() + '-rm -r {}'.format(data_dir_path))

class SciTester(object):

    def __init__(self):
        self.db = DB(SMConfig.get_conf()['db'])
        self.base_search_res_path = join(proj_root(), 'test/reports', 'spheroid_12h_search_res.csv')
        self.metrics = ['chaos', 'img_corr', 'pat_match']

    def metr_dict_to_array(self, metr_d):
        return np.array([metr_d[m] for m in self.metrics])

    def read_base_search_res(self):
        with open(self.base_search_res_path) as f:
            rows = map(lambda line: line.strip('\n').split('\t'), f.readlines()[1:])
            return {(r[0], r[1]): np.array(r[2:], dtype=float) for r in rows}

    def fetch_search_res(self):
        rows = self.db.select(SEARCH_RES_SELECT, (ds_name, 'HMDB'))
        return {(r[0], r[1]): self.metr_dict_to_array(r[2]) for r in rows}

    def run_sci_test(self):
        compare_search_results(self.read_base_search_res(), self.fetch_search_res())

    def save_sci_test_report(self):
        with open(self.base_search_res_path, 'w') as f:
            f.write('\t'.join(['sf', 'adduct'] + self.metrics) + '\n')
            for (sf, adduct), metrics in sorted(self.fetch_search_res().iteritems()):
                f.write('\t'.join([sf, adduct] + metrics.astype(str).tolist()) + '\n')

        print 'Successfully saved sample dataset search report'

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Scientific test runner')
    parser.add_argument('-r', '--run', action='store_true', help='compare current search results with previous')
    parser.add_argument('-s', '--save', action='store_true', help='store current search results')
    args = parser.parse_args()

    sci_tester = SciTester()

    if args.run:
        try:
            zip_engine()
            run_search()
            sci_tester.run_sci_test()
        finally:
            clear_data_dirs()
    elif args.save:
        resp = raw_input('You are going to replace the reference values. Are you sure? (y/n): ')
        if resp == 'y':
            sci_tester.save_sci_test_report()
    else:
        parser.print_help()
<commit_msg>Updated sci test output<commit_after>from __future__ import division
from os.path import join, dirname
from subprocess import check_call
import argparse
import json
import numpy as np
from fabric.api import local
from fabric.context_managers import warn_only

from engine.db import DB
from engine.util import proj_root, hdfs_prefix, SMConfig

# def sm_config():
#     with open(join(proj_root(), 'conf/config.json')) as f:
#         return json.load(f)

SMConfig.set_path(join(proj_root(), 'conf/config.json'))
sm_config = SMConfig.get_conf()

ds_name = 'sci_test_spheroid_12h'
data_dir_path = join(SMConfig.get_conf()['fs']['data_dir'], ds_name)
input_dir_path = join(proj_root(), 'test/data/sci_test_search_job_spheroid_dataset')

SEARCH_RES_SELECT = ("select sf, adduct, stats "
                     "from iso_image_metrics s "
                     "join formula_db sm_db on sm_db.id = s.db_id "
                     "join agg_formula f on f.id = s.sf_id "
                     "join job j on j.id = s.job_id "
                     "join dataset ds on ds.id = j.ds_id "
                     "where ds.name = %s and sm_db.name = %s "
                     "ORDER BY sf, adduct ")

def compare_search_results(base_search_res, search_res):
    missed_sf_adduct = set(base_search_res.keys()).difference(set(search_res.keys()))
    print 'MISSED FORMULAS: {:.1f}%'.format(len(missed_sf_adduct) / len(base_search_res) * 100)
    if missed_sf_adduct:
        print list(missed_sf_adduct)

    new_sf_adduct = set(search_res.keys()).difference(set(base_search_res.keys()))
    print 'FALSE DISCOVERY: {:.1f}%'.format(len(new_sf_adduct) / len(base_search_res) * 100)

    for sf_adduct in new_sf_adduct:
        metrics = search_res[sf_adduct]
        print '{} metrics = {}'.format(sf_adduct, metrics)

    print 'DIFFERENCE IN METRICS'
    for b_sf_add, b_metr in base_search_res.iteritems():
        if b_sf_add in search_res.keys():
            metr = search_res[b_sf_add]
            diff = np.abs(b_metr - metr)
            if np.any(diff > 1e-6):
                print '{} metrics diff = {}'.format(b_sf_add, diff)

def zip_engine():
    local('cd {}; zip -rq engine.zip engine'.format(proj_root()))

def run_search():
    cmd = ['python',
           join(proj_root(), 'scripts/run_molecule_search.py'),
           ds_name,
           input_dir_path]
    check_call(cmd)

def clear_data_dirs():
    with warn_only():
        local('rm -rf {}'.format(data_dir_path))
        if not sm_config['fs']['local']:
            local(hdfs_prefix() + '-rm -r {}'.format(data_dir_path))

class SciTester(object):

    def __init__(self):
        self.db = DB(sm_config['db'])
        self.base_search_res_path = join(proj_root(), 'test/reports', 'spheroid_12h_search_res.csv')
        self.metrics = ['chaos', 'img_corr', 'pat_match']

    def metr_dict_to_array(self, metr_d):
        return np.array([metr_d[m] for m in self.metrics])

    def read_base_search_res(self):
        with open(self.base_search_res_path) as f:
            rows = map(lambda line: line.strip('\n').split('\t'), f.readlines()[1:])
            return {(r[0], r[1]): np.array(r[2:], dtype=float) for r in rows}

    def fetch_search_res(self):
        rows = self.db.select(SEARCH_RES_SELECT, (ds_name, 'HMDB'))
        return {(r[0], r[1]): self.metr_dict_to_array(r[2]) for r in rows}

    def run_sci_test(self):
        compare_search_results(self.read_base_search_res(), self.fetch_search_res())

    def save_sci_test_report(self):
        with open(self.base_search_res_path, 'w') as f:
            f.write('\t'.join(['sf', 'adduct'] + self.metrics) + '\n')
            for (sf, adduct), metrics in sorted(self.fetch_search_res().iteritems()):
                f.write('\t'.join([sf, adduct] + metrics.astype(str).tolist()) + '\n')

        print 'Successfully saved sample dataset search report'

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Scientific test runner')
    parser.add_argument('-r', '--run', action='store_true', help='compare current search results with previous')
    parser.add_argument('-s', '--save', action='store_true', help='store current search results')
    args = parser.parse_args()

    sci_tester = SciTester()

    if args.run:
        try:
            zip_engine()
            run_search()
            sci_tester.run_sci_test()
        finally:
            clear_data_dirs()
    elif args.save:
        resp = raw_input('You are going to replace the reference values. Are you sure? (y/n): ')
        if resp == 'y':
            sci_tester.save_sci_test_report()
    else:
        parser.print_help()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import argparse
import json
import os
import csv
import sys
from numpy import *

#offset by 3
met_index = {'PSNR': 0, 'PSNRHVS': 1, 'SSIM': 2, 'FASTSSIM': 3, 'CIEDE2000': 4,
             'PSNR Cb': 5, 'PSNR Cr': 6, 'APSNR': 7, 'APSNR Cb': 8, 'APSNR Cr':9,
             'MSSSIM':10, 'Encoding Time':11, 'VMAF_old':12, 'Decoding Time': 13,
             "PSNR Y (libvmaf)": 14, "PSNR Cb (libvmaf)": 15, "PSNR Cr (libvmaf)": 16,
             "CIEDE2000 (libvmaf)": 17, "SSIM (libvmaf)": 18, "MS-SSIM (libvmaf)": 19,
             "PSNR-HVS Y (libvmaf)": 20, "PSNR-HVS Cb (libvmaf)": 21, "PSNR-HVS Cr (libvmaf)": 22,
             "PSNR-HVS (libvmaf)": 23, "VMAF": 24, "VMAF-NEG": 25,
             "APSNR Y (libvmaf)": 26, "APSNR Cb (libvmaf)": 27, "APSNR Cr (libvmaf)": 28}

parser = argparse.ArgumentParser(description='Generate CTC CSV version of .out files')
parser.add_argument('run',nargs=1,help='Run folder')
args = parser.parse_args()

info_data = json.load(open(args.run[0]+'/info.json'))
task = info_data['task']
sets = json.load(open(os.path.join(os.getenv("CONFIG_DIR", "rd_tool"), "sets.json")))
videos = sets[task]["sources"]

w = csv.writer(sys.stdout, dialect='excel')
w.writerow(['Video', 'QP', 'Filesize', 'PSNR Y', 'PSNR U', 'PSNR V',
            'SSIM', 'MS-SSIM', 'VMAF', 'nVMAF', 'PSNR-HVS Y', 'DE2K',
            'APSNR Y', 'APSNR U', 'APSNR V', 'Enc T [s]', 'Dec T [s]'])
for video in videos:
    a = loadtxt(os.path.join(args.run[0],task,video+'-daala.out'))
    for row in a:
        w.writerow([video,
                    row[0], #qp
                    row[2],# bitrate
                    row[met_index['PSNR Y (libvmaf)']+3],
                    row[met_index['PSNR Cb (libvmaf)']+3],
                    row[met_index['PSNR Cr (libvmaf)']+3],
                    row[met_index['SSIM (libvmaf)']+3],
                    row[met_index['MS-SSIM (libvmaf)']+3],
                    row[met_index['VMAF']+3],
                    row[met_index['VMAF-NEG']+3],
                    row[met_index['PSNR-HVS Y (libvmaf)']+3],
                    row[met_index['CIEDE2000 (libvmaf)']+3],
                    row[met_index['APSNR Y (libvmaf)']+3],
                    row[met_index['APSNR Cb (libvmaf)']+3],
                    row[met_index['APSNR Cr (libvmaf)']+3],
                    row[met_index['Encoding Time']+3],
                    row[met_index['Decoding Time']+3],
                    ])
<commit_msg>Use kilobits per second in csv_export.py.<commit_after>#!/usr/bin/env python

import argparse
import json
import os
import csv
import sys
import re
from numpy import *

#offset by 3
met_index = {'PSNR': 0, 'PSNRHVS': 1, 'SSIM': 2, 'FASTSSIM': 3, 'CIEDE2000': 4,
             'PSNR Cb': 5, 'PSNR Cr': 6, 'APSNR': 7, 'APSNR Cb': 8, 'APSNR Cr':9,
             'MSSSIM':10, 'Encoding Time':11, 'VMAF_old':12, 'Decoding Time': 13,
             "PSNR Y (libvmaf)": 14, "PSNR Cb (libvmaf)": 15, "PSNR Cr (libvmaf)": 16,
             "CIEDE2000 (libvmaf)": 17, "SSIM (libvmaf)": 18, "MS-SSIM (libvmaf)": 19,
             "PSNR-HVS Y (libvmaf)": 20, "PSNR-HVS Cb (libvmaf)": 21, "PSNR-HVS Cr (libvmaf)": 22,
             "PSNR-HVS (libvmaf)": 23, "VMAF": 24, "VMAF-NEG": 25,
             "APSNR Y (libvmaf)": 26, "APSNR Cb (libvmaf)": 27, "APSNR Cr (libvmaf)": 28}

parser = argparse.ArgumentParser(description='Generate CTC CSV version of .out files')
parser.add_argument('run',nargs=1,help='Run folder')
args = parser.parse_args()

info_data = json.load(open(args.run[0]+'/info.json'))
task = info_data['task']
sets = json.load(open(os.path.join(os.getenv("CONFIG_DIR", "rd_tool"), "sets.json")))
videos = sets[task]["sources"]
videos_dir = os.path.join(os.getenv("MEDIA_DIR", "/mnt/runs/sets"), task) # for getting framerate

w = csv.writer(sys.stdout, dialect='excel')
w.writerow(['Video', 'QP', 'Bitrate (kbps)', 'PSNR Y', 'PSNR U', 'PSNR V',
            'SSIM', 'MS-SSIM', 'VMAF', 'nVMAF', 'PSNR-HVS Y', 'DE2K',
            'APSNR Y', 'APSNR U', 'APSNR V', 'Enc T [s]', 'Dec T [s]'])
for video in videos:
    v = open(os.path.join(videos_dir, video), 'rb')
    line = v.readline().decode('utf-8')
    fps_n, fps_d = re.search(r'F([0-9]*)\:([0-9]*)', line).group(1,2)
    width = re.search(r'W([0-9]*)', line).group(1)
    height = re.search(r'H([0-9]*)', line).group(1)
    a = loadtxt(os.path.join(args.run[0],task,video+'-daala.out'))
    for row in a:
        frames = int(row[1]) / int(width) / int(height)
        w.writerow([video,
                    row[0], #qp
                    int(row[2])*8.0*float(fps_n)/float(fps_d)/frames/1000.0,# bitrate
                    row[met_index['PSNR Y (libvmaf)']+3],
                    row[met_index['PSNR Cb (libvmaf)']+3],
                    row[met_index['PSNR Cr (libvmaf)']+3],
                    row[met_index['SSIM (libvmaf)']+3],
                    row[met_index['MS-SSIM (libvmaf)']+3],
                    row[met_index['VMAF']+3],
                    row[met_index['VMAF-NEG']+3],
                    row[met_index['PSNR-HVS Y (libvmaf)']+3],
                    row[met_index['CIEDE2000 (libvmaf)']+3],
                    row[met_index['APSNR Y (libvmaf)']+3],
                    row[met_index['APSNR Cb (libvmaf)']+3],
                    row[met_index['APSNR Cr (libvmaf)']+3],
                    row[met_index['Encoding Time']+3],
                    row[met_index['Decoding Time']+3],
                    ])
<|endoftext|>