<commit_before>"""Tests for build_package script."""

from mock import (
    Mock,
    patch
)
import os
from textwrap import dedent
import unittest

from build_package import (
    build_binary,
    BUILD_DEB_TEMPLATE,
    build_in_lxc,
    CREATE_LXC_TEMPLATE,
    get_args,
    main,
    parse_dsc,
    setup_local,
    setup_lxc,
    SourceFile,
    teardown_lxc,
)
from utils import temp_dir

DSC_CONTENT = dedent("""\
    Format: 3.0 (quilt)
    Source: juju-core
    Binary: juju-core, juju, juju-local, juju-local-kvm
    Architecture: any all
    Version: 1.24-beta1-0ubuntu1~14.04.1~juju1
    Maintainer: Curtis Hovey <curtis.hovey@canonical.com>
    Homepage: http://launchpad.net/juju-core
    Standards-Version: 3.9.5
    Build-Depends: debhelper (>= 7.0.50~), golang-go, lsb-release, python
    Package-List:
     juju deb devel extra
     juju-core deb devel extra
     juju-local deb devel extra
     juju-local-kvm deb devel extra
    Checksums-Sha1:
     1234 9876 juju-core_1.24-beta1.orig.tar.gz
     2345 4321 juju-core_1.24-beta1-0ubuntu1~14.04.1~juju1.debian.tar.gz
    Checksums-Sha256:
     3456 9876 juju-core_1.24-beta1.orig.tar.gz
     4567 4321 juju-core_1.24-beta1-0ubuntu1~14.04.1~juju1.debian.tar.gz
    Files:
     5678 9876 juju-core_1.24-beta1.orig.tar.gz
     6789 4321 juju-core_1.24-beta1-0ubuntu1~14.04.1~juju1.debian.tar.gz
    Testsuite: autopkgtest
    """)

def make_source_files(workspace, dsc_name):
    dsc_path = os.path.join(workspace, dsc_name)
    dsc_file = SourceFile(None, None, 'my.dsc', dsc_path)
    orig_file = SourceFile(
        '3456', '9876', 'juju-core_1.24-beta1.orig.tar.gz',
        '%s/juju-core_1.24-beta1.orig.tar.gz' % workspace)
    deb_file = SourceFile(
        '4567', '4321',
        'juju-core_1.24-beta1-0ubuntu1~14.04.1~juju1.debian.tar.gz',
        '%s/juju-core_1.24-beta1-0ubuntu1~14.04.1~juju1.debian.tar.gz' %
        workspace)
    source_files = [dsc_file, orig_file, deb_file]
    for sf in source_files:
        with open(sf.path, 'w') as f:
            if '.dsc' in sf.name:
                f.write(DSC_CONTENT)
            else:
                f.write(sf.name)
    return source_files

class BuildPackageTestCase(unittest.TestCase):

    def test_get_args_binary(self):
        args = get_args(
            ['prog', 'binary', 'my.dsc', '~/workspace', 'trusty', 'i386'])
        self.assertEqual('binary', args.command)
        self.assertEqual('my.dsc', args.dsc)
        self.assertEqual('~/workspace', args.location)
        self.assertFalse(args.verbose)

    def test_main_binary(self):
        with patch('build_package.build_binary', autospec=True,
                   return_value=0) as bb_mock:
            code = main(
                ['prog', 'binary', 'my.dsc', '~/workspace', 'trusty', 'i386'])
        self.assertEqual(0, code)
        bb_mock.assert_called_with(
            'my.dsc', '~/workspace', 'trusty', 'i386', verbose=False)

    def test_build_binary(self):
        with patch('build_package.parse_dsc', autospec=True,
                   return_value=['orig', 'debian']) as pd_mock:
            with patch('build_package.setup_local', autospec=True,
                       return_value='build_dir') as sl_mock:
                with patch('build_package.setup_lxc', autospec=True,
                           return_value='trusty-i386') as l_mock:
                    with patch('build_package.build_in_lxc',
                               autospec=True) as bl_mock:
                        with patch('build_package.teardown_lxc', autospec=True,
                                   return_value=True) as tl_mock:
                            code = build_binary(
                                'my.dsc', '~/workspace', 'trusty', 'i386',
                                verbose=False)
        self.assertEqual(0, code)
        pd_mock.assert_called_with('my.dsc', verbose=False)
        sl_mock.assert_called_with(
            '~/workspace', 'trusty', 'i386', ['orig', 'debian'], verbose=False)
        l_mock.assert_called_with('trusty', 'i386', 'build_dir', verbose=False)
        bl_mock.assert_called_with('trusty-i386', verbose=False)
        tl_mock.assert_called_with('trusty-i386', verbose=False)

    def test_parse_dsc(self):
        with temp_dir() as workspace:
            expected_files = make_source_files(workspace, 'my.dsc')
            dsc_path = os.path.join(workspace, 'my.dsc')
            source_files = parse_dsc(dsc_path, verbose=False)
        self.assertEqual(expected_files, source_files)

    def test_setup_local(self):
        with temp_dir() as workspace:
            source_files = make_source_files(workspace, 'my.dsc')
            build_dir = setup_local(
                workspace, 'trusty', 'i386', source_files, verbose=False)
            self.assertEqual(
                os.path.join(workspace, 'juju-build-trusty-i386'),
                build_dir)
            self.assertTrue(os.path.isdir(build_dir))

    def test_setup_lxc(self):
        with patch('subprocess.check_output') as co_mock:
            container = setup_lxc(
                'trusty', 'i386', '/build-dir', verbose=False)
        self.assertEqual('trusty-i386', container)
        lxc_script = CREATE_LXC_TEMPLATE.format(
            container='trusty-i386', series='trusty', arch='i386',
            build_dir='/build-dir')
        co_mock.assert_called_with(['bash', '-c', lxc_script])

    def test_build_in_lxc(self):
        with patch('subprocess.check_call') as cc_mock:
            proc = Mock(returncode=0)
            with patch('subprocess.Popen', return_value=proc) as p_mock:
                code = build_in_lxc('trusty-i386', verbose=False)
        self.assertEqual(0, code)
        proc.communicate.assert_called_with()
        cc_mock.assert_any_call(
            ['sudo', 'lxc-start', '-d', '-n', 'trusty-i386'])
        build_script = BUILD_DEB_TEMPLATE.format(container='trusty-i386')
        p_mock.assert_called_with(['bash', '-c', build_script])
        cc_mock.assert_any_call(['sudo', 'lxc-stop', '-n', 'trusty-i386'])

    def test_build_in_lxc_stop_lxc(self):
        with patch('subprocess.check_call') as cc_mock:
            with patch('subprocess.Popen', side_effect=Exception):
                with self.assertRaises(Exception):
                    build_in_lxc('trusty-i386', verbose=False)
        cc_mock.assert_any_call(['sudo', 'lxc-stop', '-n', 'trusty-i386'])

    def test_teardown_lxc(self):
        with patch('subprocess.check_call') as cc_mock:
            teardown_lxc('trusty-i386', verbose=False)
        cc_mock.assert_called_with(
            ['sudo', 'lxc-destroy', '-n', 'trusty-i386'])
<commit_msg>Added test to verify lxcs are destroyed.<commit_after>"""Tests for build_package script."""

from mock import (
    Mock,
    patch
)
import os
from textwrap import dedent
import unittest

from build_package import (
    build_binary,
    BUILD_DEB_TEMPLATE,
    build_in_lxc,
    CREATE_LXC_TEMPLATE,
    get_args,
    main,
    parse_dsc,
    setup_local,
    setup_lxc,
    SourceFile,
    teardown_lxc,
)
from utils import temp_dir

DSC_CONTENT = dedent("""\
    Format: 3.0 (quilt)
    Source: juju-core
    Binary: juju-core, juju, juju-local, juju-local-kvm
    Architecture: any all
    Version: 1.24-beta1-0ubuntu1~14.04.1~juju1
    Maintainer: Curtis Hovey <curtis.hovey@canonical.com>
    Homepage: http://launchpad.net/juju-core
    Standards-Version: 3.9.5
    Build-Depends: debhelper (>= 7.0.50~), golang-go, lsb-release, python
    Package-List:
     juju deb devel extra
     juju-core deb devel extra
     juju-local deb devel extra
     juju-local-kvm deb devel extra
    Checksums-Sha1:
     1234 9876 juju-core_1.24-beta1.orig.tar.gz
     2345 4321 juju-core_1.24-beta1-0ubuntu1~14.04.1~juju1.debian.tar.gz
    Checksums-Sha256:
     3456 9876 juju-core_1.24-beta1.orig.tar.gz
     4567 4321 juju-core_1.24-beta1-0ubuntu1~14.04.1~juju1.debian.tar.gz
    Files:
     5678 9876 juju-core_1.24-beta1.orig.tar.gz
     6789 4321 juju-core_1.24-beta1-0ubuntu1~14.04.1~juju1.debian.tar.gz
    Testsuite: autopkgtest
    """)

def make_source_files(workspace, dsc_name):
    dsc_path = os.path.join(workspace, dsc_name)
    dsc_file = SourceFile(None, None, 'my.dsc', dsc_path)
    orig_file = SourceFile(
        '3456', '9876', 'juju-core_1.24-beta1.orig.tar.gz',
        '%s/juju-core_1.24-beta1.orig.tar.gz' % workspace)
    deb_file = SourceFile(
        '4567', '4321',
        'juju-core_1.24-beta1-0ubuntu1~14.04.1~juju1.debian.tar.gz',
        '%s/juju-core_1.24-beta1-0ubuntu1~14.04.1~juju1.debian.tar.gz' %
        workspace)
    source_files = [dsc_file, orig_file, deb_file]
    for sf in source_files:
        with open(sf.path, 'w') as f:
            if '.dsc' in sf.name:
                f.write(DSC_CONTENT)
            else:
                f.write(sf.name)
    return source_files

class BuildPackageTestCase(unittest.TestCase):

    def test_get_args_binary(self):
        args = get_args(
            ['prog', 'binary', 'my.dsc', '~/workspace', 'trusty', 'i386'])
        self.assertEqual('binary', args.command)
        self.assertEqual('my.dsc', args.dsc)
        self.assertEqual('~/workspace', args.location)
        self.assertFalse(args.verbose)

    def test_main_binary(self):
        with patch('build_package.build_binary', autospec=True,
                   return_value=0) as bb_mock:
            code = main(
                ['prog', 'binary', 'my.dsc', '~/workspace', 'trusty', 'i386'])
        self.assertEqual(0, code)
        bb_mock.assert_called_with(
            'my.dsc', '~/workspace', 'trusty', 'i386', verbose=False)

    def test_build_binary(self):
        with patch('build_package.parse_dsc', autospec=True,
                   return_value=['orig', 'debian']) as pd_mock:
            with patch('build_package.setup_local', autospec=True,
                       return_value='build_dir') as sl_mock:
                with patch('build_package.setup_lxc', autospec=True,
                           return_value='trusty-i386') as l_mock:
                    with patch('build_package.build_in_lxc',
                               autospec=True) as bl_mock:
                        with patch('build_package.teardown_lxc', autospec=True,
                                   return_value=True) as tl_mock:
                            code = build_binary(
                                'my.dsc', '~/workspace', 'trusty', 'i386',
                                verbose=False)
        self.assertEqual(0, code)
        pd_mock.assert_called_with('my.dsc', verbose=False)
        sl_mock.assert_called_with(
            '~/workspace', 'trusty', 'i386', ['orig', 'debian'], verbose=False)
        l_mock.assert_called_with('trusty', 'i386', 'build_dir', verbose=False)
        bl_mock.assert_called_with('trusty-i386', verbose=False)
        tl_mock.assert_called_with('trusty-i386', verbose=False)

    def test_build_binary_teardown_lxc(self):
        with patch('build_package.parse_dsc', autospec=True,
                   return_value=['orig', 'debian']):
            with patch('build_package.setup_local', autospec=True,
                       return_value='build_dir'):
                with patch('build_package.setup_lxc', autospec=True,
                           return_value='trusty-i386'):
                    with patch('build_package.build_in_lxc', autospec=True,
                               side_effect=Exception):
                        with patch('build_package.teardown_lxc', autospec=True,
                                   return_value=True) as tl_mock:
                            with self.assertRaises(Exception):
                                build_binary(
                                    'my.dsc', '~/workspace', 'trusty', 'i386',
                                    verbose=False)
        tl_mock.assert_called_with('trusty-i386', verbose=False)

    def test_parse_dsc(self):
        with temp_dir() as workspace:
            expected_files = make_source_files(workspace, 'my.dsc')
            dsc_path = os.path.join(workspace, 'my.dsc')
            source_files = parse_dsc(dsc_path, verbose=False)
        self.assertEqual(expected_files, source_files)

    def test_setup_local(self):
        with temp_dir() as workspace:
            source_files = make_source_files(workspace, 'my.dsc')
            build_dir = setup_local(
                workspace, 'trusty', 'i386', source_files, verbose=False)
            self.assertEqual(
                os.path.join(workspace, 'juju-build-trusty-i386'),
                build_dir)
            self.assertTrue(os.path.isdir(build_dir))

    def test_setup_lxc(self):
        with patch('subprocess.check_output') as co_mock:
            container = setup_lxc(
                'trusty', 'i386', '/build-dir', verbose=False)
        self.assertEqual('trusty-i386', container)
        lxc_script = CREATE_LXC_TEMPLATE.format(
            container='trusty-i386', series='trusty', arch='i386',
            build_dir='/build-dir')
        co_mock.assert_called_with(['bash', '-c', lxc_script])

    def test_build_in_lxc(self):
        with patch('subprocess.check_call') as cc_mock:
            proc = Mock(returncode=0)
            with patch('subprocess.Popen', return_value=proc) as p_mock:
                code = build_in_lxc('trusty-i386', verbose=False)
        self.assertEqual(0, code)
        proc.communicate.assert_called_with()
        cc_mock.assert_any_call(
            ['sudo', 'lxc-start', '-d', '-n', 'trusty-i386'])
        build_script = BUILD_DEB_TEMPLATE.format(container='trusty-i386')
        p_mock.assert_called_with(['bash', '-c', build_script])
        cc_mock.assert_any_call(['sudo', 'lxc-stop', '-n', 'trusty-i386'])

    def test_build_in_lxc_stop_lxc(self):
        with patch('subprocess.check_call') as cc_mock:
            with patch('subprocess.Popen', side_effect=Exception):
                with self.assertRaises(Exception):
                    build_in_lxc('trusty-i386', verbose=False)
        cc_mock.assert_any_call(['sudo', 'lxc-stop', '-n', 'trusty-i386'])

    def test_teardown_lxc(self):
        with patch('subprocess.check_call') as cc_mock:
            teardown_lxc('trusty-i386', verbose=False)
        cc_mock.assert_called_with(
            ['sudo', 'lxc-destroy', '-n', 'trusty-i386'])
<|endoftext|>


<commit_before>from examples.checkpointing.checkpointing_example import CheckpointingExample
from examples.checkpointing.checkpoint import DevitoCheckpoint, CheckpointOperator
from examples.seismic.acoustic.acoustic_example import acoustic_setup
from pyrevolve import Revolver
import numpy as np
from conftest import skipif_yask
import pytest
from functools import reduce

from devito import Grid, TimeFunction, Operator, Backward, Function, Eq, silencio

@silencio(log_level='WARNING')
@skipif_yask
def test_segmented_incremment():
    """
    Test for segmented operator execution of a one-sided first order
    function (increment). The corresponding set of stencil offsets in
    the time domain is (1, 0)
    """
    grid = Grid(shape=(5, 5))
    x, y = grid.dimensions
    t = grid.stepping_dim
    f = TimeFunction(name='f', grid=grid, time_order=1)
    fi = f.indexed
    op = Operator(Eq(fi[t, x, y], fi[t-1, x, y] + 1.))

    # Reference solution with a single invocation, 20 timesteps.
    # ==========================================================
    # Developer note: With the current absolute indexing scheme
    # the final time dimension index is 21, and the "write range"
    # is [1 - 20] or [1, 21).
    f_ref = TimeFunction(name='f', grid=grid, time_order=1)
    op(f=f_ref, time=21)
    assert (f_ref.data[19] == 19.).all()
    assert (f_ref.data[20] == 20.).all()

    # Now run with 5 invocations of 4 timesteps each
    nsteps = 4
    for i in range(5):
        op(f=f, time_s=i*nsteps, time_e=(i+1)*nsteps)
    assert (f.data[19] == 19.).all()
    assert (f.data[20] == 20.).all()

@silencio(log_level='WARNING')
@skipif_yask
def test_segmented_fibonacci():
    """
    Test for segmented operator execution of a one-sided second order
    function (fibonacci). The corresponding set of stencil offsets in
    the time domain is (2, 0)
    """
    # Reference Fibonacci solution from:
    # https://stackoverflow.com/questions/4935957/fibonacci-numbers-with-an-one-liner-in-python-3
    fib = lambda n: reduce(lambda x, n: [x[1], x[0] + x[1]], range(n), [0, 1])[0]

    grid = Grid(shape=(5, 5))
    x, y = grid.dimensions
    t = grid.stepping_dim
    f = TimeFunction(name='f', grid=grid, time_order=2)
    fi = f.indexed
    op = Operator(Eq(fi[t, x, y], fi[t-1, x, y] + fi[t-2, x, y]))

    # Reference solution with a single invocation, 12 timesteps.
    # ==========================================================
    # Developer note: the 13th Fibonacci number resides at logical
    # index 12, but we need to give a final index of 13 due to the
    # current convention of computing [t_s+offset(t), t_end).
    f_ref = TimeFunction(name='f', grid=grid, time_order=2)
    f_ref.data[:] = 1.
    op(f=f_ref, time=13)
    assert (f_ref.data[11] == fib(12)).all()
    assert (f_ref.data[12] == fib(13)).all()

    # Now run with 5 invocations of 4 timesteps each
    nsteps = 4
    f.data[:] = 1.
    for i in range(3):
        op(f=f, time_s=i*nsteps, time_e=(i+1)*nsteps)
    assert (f.data[11] == fib(12)).all()
    assert (f.data[12] == fib(13)).all()

@silencio(log_level='WARNING')
@skipif_yask
def test_segmented_averaging():
    """
    Test for segmented operator execution of a two-sided, second order
    function (averaging) in space. The corresponding set of stencil
    offsets in the x domain are (1, 1).
    """
    grid = Grid(shape=(20, 20))
    x, y = grid.dimensions
    t = grid.stepping_dim
    f = TimeFunction(name='f', grid=grid)
    fi = f.indexed
    op = Operator(Eq(f, f.backward + (fi[t-1, x+1, y] + fi[t-1, x-1, y]) / 2.))

    # We add the average to the point itself, so the grid "interior"
    # (domain) is updated only.
    f_ref = TimeFunction(name='f', grid=grid)
    f_ref.data[:] = 1.
    op(f=f_ref, time=2)
    assert (f_ref.data[1, 0, :] == 1.).all()
    assert (f_ref.data[1, 19, :] == 1.).all()
    assert (f_ref.data[1, 1:19, :] == 2.).all()

    # Now we sweep the x direction in 3 segmented steps
    nsteps = 6
    f.data[:] = 1.
    for i in range(3):
        op(f=f, time=2, x_s=i*nsteps, x_e=(i+1)*nsteps)
    assert (f.data[1, 0, :] == 1.).all()
    assert (f.data[1, 19, :] == 1.).all()
    assert (f.data[1, 1:19, :] == 2.).all()

@silencio(log_level='WARNING')
@skipif_yask
@pytest.mark.parametrize('space_order', [4])
@pytest.mark.parametrize('time_order', [2])
@pytest.mark.parametrize('shape', [(70, 80), (50, 50, 50)])
def test_forward_with_breaks(shape, time_order, space_order):
    """ Test running forward in one go and "with breaks"
    and ensure they produce the same result
    """
    spacing = tuple([15.0 for _ in shape])
    tn = 500.
    example = CheckpointingExample(shape, spacing, tn, time_order, space_order)
    m0, dm = example.initial_estimate()

    cp = DevitoCheckpoint([example.forward_field])
    wrap_fw = CheckpointOperator(example.forward_operator, u=example.forward_field,
                                 rec=example.rec, m=m0, src=example.src, dt=example.dt)
    wrap_rev = CheckpointOperator(example.gradient_operator, u=example.forward_field,
                                  v=example.adjoint_field, m=m0, rec=example.rec_g,
                                  grad=example.grad, dt=example.dt)
    wrp = Revolver(cp, wrap_fw, wrap_rev, None, example.nt-time_order)
    example.forward_operator.apply(u=example.forward_field, rec=example.rec, m=m0,
                                   src=example.src, dt=example.dt)
    u_temp = np.copy(example.forward_field.data)
    rec_temp = np.copy(example.rec.data)
    example.forward_field.data[:] = 0
    wrp.apply_forward()
    assert(np.allclose(u_temp, example.forward_field.data))
    assert(np.allclose(rec_temp, example.rec.data))

@silencio(log_level='WARNING')
@skipif_yask
def test_acoustic_save_and_nosave(shape=(50, 50), spacing=(15.0, 15.0), tn=500.,
                                  time_order=2, space_order=4, nbpml=10):
    """ Run the acoustic example with and without save=True. Make sure the result is the
    same
    """
    solver = acoustic_setup(shape=shape, spacing=spacing, nbpml=nbpml, tn=tn,
                            space_order=space_order, time_order=time_order)
    rec, u, summary = solver.forward(save=True)
    last_time_step = solver.source.nt-1
    field_last_time_step = np.copy(u.data[last_time_step, :, :])
    rec_bk = np.copy(rec.data)
    rec, u, summary = solver.forward(save=False)
    last_time_step = (last_time_step) % (time_order + 1)
    assert(np.allclose(u.data[last_time_step, :, :], field_last_time_step))
    assert(np.allclose(rec.data, rec_bk))

@silencio(log_level='WARNING')
@skipif_yask
@pytest.mark.parametrize('space_order', [4])
@pytest.mark.parametrize('time_order', [2])
@pytest.mark.parametrize('shape', [(70, 80), (50, 50, 50)])
def test_checkpointed_gradient_test(shape, time_order, space_order):
    """ Run the gradient test but with checkpointing """
    spacing = tuple([15.0 for _ in shape])
    tn = 500.
    example = CheckpointingExample(shape, spacing, tn, time_order, space_order)
    m0, dm = example.initial_estimate()
    gradient, rec_data = example.gradient(m0)
    example.verify(m0, gradient, rec_data, dm)

@skipif_yask
def test_index_alignment(const):
    """ A much simpler test meant to ensure that the forward and reverse indices are
    correctly aligned (i.e. u * v , where u is the forward field and v the reverse field
    corresponds to the correct timesteps in u and v). The forward operator does u = u + 1
    which means that the field a will be equal to nt (0 -> 1 -> 2 -> 3), the number of
    timesteps this operator is run for. The field at the last time step of the forward is
    used to initialise the field v for the reverse pass. The reverse operator does
    v = v - 1, which means that if the reverse operator is run for the same number of
    timesteps as the forward operator, v should be 0 at the last time step
    (3 -> 2 -> 1 -> 0). There is also a grad = grad + u * v accumulator in the reverse
    operator. If the alignment is correct, u and v should have the same value at every
    time step:
    0 -> 1 -> 2 -> 3 u
    0 <- 1 <- 2 <- 3 v
    and hence grad = 0*0 + 1*1 + 2*2 + 3*3 = sum(n^2) where n -> [0, nt]
    If the test fails, the resulting number can tell you how the fields are misaligned
    """
    nt = 10
    grid = Grid(shape=(3, 5))
    order_of_eqn = 1
    modulo_factor = order_of_eqn + 1
    last_time_step_u = nt - order_of_eqn
    u = TimeFunction(name='u', grid=grid, save=nt)
    # Increment one in the forward pass 0 -> 1 -> 2 -> 3
    fwd_op = Operator(Eq(u.forward, u + 1.*const))

    # Invocation 1
    fwd_op(time=nt, constant=1)

    last_time_step_v = (last_time_step_u) % modulo_factor
    # Last time step should be equal to the number of timesteps we ran
    assert(np.allclose(u.data[last_time_step_u, :, :], nt - order_of_eqn))

    v = TimeFunction(name='v', grid=grid, save=None)
    v.data[last_time_step_v, :, :] = u.data[last_time_step_u, :, :]
    # Decrement one in the reverse pass 3 -> 2 -> 1 -> 0
    adj_eqn = Eq(v.backward, v - 1.*const)
    adj_op = Operator(adj_eqn, time_axis=Backward)

    # Invocation 2
    adj_op(t=nt, constant=1)
    # Last time step should be back to 0
    assert(np.allclose(v.data[0, :, :], 0))

    # Reset v to run the backward again
    v.data[last_time_step_v, :, :] = u.data[last_time_step_u, :, :]
    prod = Function(name="prod", grid=grid)
    # Multiply u and v and add them
    # = 3*3 + 2*2 + 1*1 + 0*0
    prod_eqn = Eq(prod, prod + u * v)
    comb_op = Operator([adj_eqn, prod_eqn], time_axis=Backward)

    # Invocation 3
    comb_op(time=nt, constant=1)
    final_value = sum([n**2 for n in range(nt)])
    # Final value should be sum of squares of first nt natural numbers
    assert(np.allclose(prod.data, final_value))

    # Now reset to repeat all the above tests with checkpointing
    prod.data[:] = 0
    v.data[last_time_step_v, :, :] = u.data[last_time_step_u, :, :]
    # Checkpointed version doesn't require to save u
    u_nosave = TimeFunction(name='u_n', grid=grid)
    # change equations to use new symbols
    fwd_eqn_2 = Eq(u_nosave.forward, u_nosave + 1.*const)
    fwd_op_2 = Operator(fwd_eqn_2)
    cp = DevitoCheckpoint([u_nosave])
    wrap_fw = CheckpointOperator(fwd_op_2, time=nt, constant=1)

    prod_eqn_2 = Eq(prod, prod + u_nosave * v)
    comb_op_2 = Operator([adj_eqn, prod_eqn_2], time_axis=Backward)
    wrap_rev = CheckpointOperator(comb_op_2, constant=1)
    wrp = Revolver(cp, wrap_fw, wrap_rev, None, nt-order_of_eqn)

    # Invocation 4
    wrp.apply_forward()
    assert(np.allclose(u_nosave.data[last_time_step_v, :, :], nt - order_of_eqn))

    # Invocation 5
    wrp.apply_reverse()
    assert(np.allclose(v.data[0, :, :], 0))
    assert(np.allclose(prod.data, final_value))
<commit_msg>checkpint test fix<commit_after>from examples.checkpointing.checkpointing_example import CheckpointingExample
from examples.checkpointing.checkpoint import DevitoCheckpoint, CheckpointOperator
from examples.seismic.acoustic.acoustic_example import acoustic_setup
from pyrevolve import Revolver
import numpy as np
from conftest import skipif_yask
import pytest
from functools import reduce

from devito import Grid, TimeFunction, Operator, Backward, Function, Eq, silencio

@silencio(log_level='WARNING')
@skipif_yask
def test_segmented_incremment():
    """
    Test for segmented operator execution of a one-sided first order
    function (increment). The corresponding set of stencil offsets in
    the time domain is (1, 0)
    """
    grid = Grid(shape=(5, 5))
    x, y = grid.dimensions
    t = grid.stepping_dim
    f = TimeFunction(name='f', grid=grid, time_order=1)
    fi = f.indexed
    op = Operator(Eq(fi[t, x, y], fi[t-1, x, y] + 1.))

    # Reference solution with a single invocation, 20 timesteps.
    # ==========================================================
    # Developer note: With the current absolute indexing scheme
    # the final time dimension index is 21, and the "write range"
    # is [1 - 20] or [1, 21).
    f_ref = TimeFunction(name='f', grid=grid, time_order=1)
    op(f=f_ref, time=21)
    assert (f_ref.data[19] == 19.).all()
    assert (f_ref.data[20] == 20.).all()

    # Now run with 5 invocations of 4 timesteps each
    nsteps = 4
    for i in range(5):
        op(f=f, time_s=i*nsteps, time_e=(i+1)*nsteps)
    assert (f.data[19] == 19.).all()
    assert (f.data[20] == 20.).all()

@silencio(log_level='WARNING')
@skipif_yask
def test_segmented_fibonacci():
    """
    Test for segmented operator execution of a one-sided second order
    function (fibonacci). The corresponding set of stencil offsets in
    the time domain is (2, 0)
    """
    # Reference Fibonacci solution from:
    # https://stackoverflow.com/questions/4935957/fibonacci-numbers-with-an-one-liner-in-python-3
    fib = lambda n: reduce(lambda x, n: [x[1], x[0] + x[1]], range(n), [0, 1])[0]

    grid = Grid(shape=(5, 5))
    x, y = grid.dimensions
    t = grid.stepping_dim
    f = TimeFunction(name='f', grid=grid, time_order=2)
    fi = f.indexed
    op = Operator(Eq(fi[t, x, y], fi[t-1, x, y] + fi[t-2, x, y]))

    # Reference solution with a single invocation, 12 timesteps.
    # ==========================================================
    # Developer note: the 13th Fibonacci number resides at logical
    # index 12, but we need to give a final index of 13 due to the
    # current convention of computing [t_s+offset(t), t_end).
    f_ref = TimeFunction(name='f', grid=grid, time_order=2)
    f_ref.data[:] = 1.
    op(f=f_ref, time=13)
    assert (f_ref.data[11] == fib(12)).all()
    assert (f_ref.data[12] == fib(13)).all()

    # Now run with 5 invocations of 4 timesteps each
    nsteps = 4
    f.data[:] = 1.
    for i in range(3):
        op(f=f, time_s=i*nsteps, time_e=(i+1)*nsteps)
    assert (f.data[11] == fib(12)).all()
    assert (f.data[12] == fib(13)).all()

@silencio(log_level='WARNING')
@skipif_yask
def test_segmented_averaging():
    """
    Test for segmented operator execution of a two-sided, second order
    function (averaging) in space. The corresponding set of stencil
    offsets in the x domain are (1, 1).
    """
    grid = Grid(shape=(20, 20))
    x, y = grid.dimensions
    t = grid.stepping_dim
    f = TimeFunction(name='f', grid=grid)
    fi = f.indexed
    op = Operator(Eq(f, f.backward + (fi[t-1, x+1, y] + fi[t-1, x-1, y]) / 2.))

    # We add the average to the point itself, so the grid "interior"
    # (domain) is updated only.
    f_ref = TimeFunction(name='f', grid=grid)
    f_ref.data[:] = 1.
    op(f=f_ref, time=2)
    assert (f_ref.data[1, 0, :] == 1.).all()
    assert (f_ref.data[1, 19, :] == 1.).all()
    assert (f_ref.data[1, 1:19, :] == 2.).all()

    # Now we sweep the x direction in 3 segmented steps
    nsteps = 6
    f.data[:] = 1.
    for i in range(3):
        op(f=f, time=2, x_s=i*nsteps, x_e=(i+1)*nsteps)
    assert (f.data[1, 0, :] == 1.).all()
    assert (f.data[1, 19, :] == 1.).all()
    assert (f.data[1, 1:19, :] == 2.).all()

@silencio(log_level='WARNING')
@skipif_yask
@pytest.mark.parametrize('space_order', [4])
@pytest.mark.parametrize('kernel', ['OT2'])
@pytest.mark.parametrize('shape', [(70, 80), (50, 50, 50)])
def test_forward_with_breaks(shape, kernel, space_order):
    """ Test running forward in one go and "with breaks"
    and ensure they produce the same result
    """
    spacing = tuple([15.0 for _ in shape])
    tn = 500.
    example = CheckpointingExample(shape, spacing, tn, kernel, space_order)
    m0, dm = example.initial_estimate()

    cp = DevitoCheckpoint([example.forward_field])
    wrap_fw = CheckpointOperator(example.forward_operator, u=example.forward_field,
                                 rec=example.rec, m=m0, src=example.src, dt=example.dt)
    wrap_rev = CheckpointOperator(example.gradient_operator, u=example.forward_field,
                                  v=example.adjoint_field, m=m0, rec=example.rec_g,
                                  grad=example.grad, dt=example.dt)
    wrp = Revolver(cp, wrap_fw, wrap_rev, None, example.nt-2)
    example.forward_operator.apply(u=example.forward_field, rec=example.rec, m=m0,
                                   src=example.src, dt=example.dt)
    u_temp = np.copy(example.forward_field.data)
    rec_temp = np.copy(example.rec.data)
    example.forward_field.data[:] = 0
    wrp.apply_forward()
    assert(np.allclose(u_temp, example.forward_field.data))
    assert(np.allclose(rec_temp, example.rec.data))

@silencio(log_level='WARNING')
@skipif_yask
def test_acoustic_save_and_nosave(shape=(50, 50), spacing=(15.0, 15.0), tn=500.,
                                  time_order=2, space_order=4, nbpml=10):
    """ Run the acoustic example with and without save=True. Make sure the result is the
    same
    """
    solver = acoustic_setup(shape=shape, spacing=spacing, nbpml=nbpml, tn=tn,
                            space_order=space_order, time_order=time_order)
    rec, u, summary = solver.forward(save=True)
    last_time_step = solver.source.nt-1
    field_last_time_step = np.copy(u.data[last_time_step, :, :])
    rec_bk = np.copy(rec.data)
    rec, u, summary = solver.forward(save=False)
    last_time_step = (last_time_step) % (time_order + 1)
    assert(np.allclose(u.data[last_time_step, :, :], field_last_time_step))
    assert(np.allclose(rec.data, rec_bk))

@silencio(log_level='WARNING')
@skipif_yask
@pytest.mark.parametrize('space_order', [4])
@pytest.mark.parametrize('kernel', ['OT2'])
@pytest.mark.parametrize('shape', [(70, 80), (50, 50, 50)])
def test_checkpointed_gradient_test(shape, kernel, space_order):
    """ Run the gradient test but with checkpointing """
    spacing = tuple([15.0 for _ in shape])
    tn = 500.
    example = CheckpointingExample(shape, spacing, tn, kernel, space_order)
    m0, dm = example.initial_estimate()
    gradient, rec_data = example.gradient(m0)
    example.verify(m0, gradient, rec_data, dm)

@skipif_yask
def test_index_alignment(const):
    """ A much simpler test meant to ensure that the forward and reverse indices are
    correctly aligned (i.e. u * v , where u is the forward field and v the reverse field
    corresponds to the correct timesteps in u and v). The forward operator does u = u + 1
    which means that the field a will be equal to nt (0 -> 1 -> 2 -> 3), the number of
    timesteps this operator is run for. The field at the last time step of the forward is
    used to initialise the field v for the reverse pass. The reverse operator does
    v = v - 1, which means that if the reverse operator is run for the same number of
    timesteps as the forward operator, v should be 0 at the last time step
    (3 -> 2 -> 1 -> 0). There is also a grad = grad + u * v accumulator in the reverse
    operator. If the alignment is correct, u and v should have the same value at every
    time step:
    0 -> 1 -> 2 -> 3 u
    0 <- 1 <- 2 <- 3 v
    and hence grad = 0*0 + 1*1 + 2*2 + 3*3 = sum(n^2) where n -> [0, nt]
    If the test fails, the resulting number can tell you how the fields are misaligned
    """
    nt = 10
    grid = Grid(shape=(3, 5))
    order_of_eqn = 1
    modulo_factor = order_of_eqn + 1
    last_time_step_u = nt - order_of_eqn
    u = TimeFunction(name='u', grid=grid, save=nt)
    # Increment one in the forward pass 0 -> 1 -> 2 -> 3
    fwd_op = Operator(Eq(u.forward, u + 1.*const))

    # Invocation 1
    fwd_op(time=nt, constant=1)

    last_time_step_v = (last_time_step_u) % modulo_factor
    # Last time step should be equal to the number of timesteps we ran
    assert(np.allclose(u.data[last_time_step_u, :, :], nt - order_of_eqn))

    v = TimeFunction(name='v', grid=grid, save=None)
    v.data[last_time_step_v, :, :] = u.data[last_time_step_u, :, :]
    # Decrement one in the reverse pass 3 -> 2 -> 1 -> 0
    adj_eqn = Eq(v.backward, v - 1.*const)
    adj_op = Operator(adj_eqn, time_axis=Backward)

    # Invocation 2
    adj_op(t=nt, constant=1)
    # Last time step should be back to 0
    assert(np.allclose(v.data[0, :, :], 0))

    # Reset v to run the backward again
    v.data[last_time_step_v, :, :] = u.data[last_time_step_u, :, :]
    prod = Function(name="prod", grid=grid)
    # Multiply u and v and add them
    # = 3*3 + 2*2 + 1*1 + 0*0
    prod_eqn = Eq(prod, prod + u * v)
    comb_op = Operator([adj_eqn, prod_eqn], time_axis=Backward)

    # Invocation 3
    comb_op(time=nt, constant=1)
    final_value = sum([n**2 for n in range(nt)])
    # Final value should be sum of squares of first nt natural numbers
    assert(np.allclose(prod.data, final_value))

    # Now reset to repeat all the above tests with checkpointing
    prod.data[:] = 0
    v.data[last_time_step_v, :, :] = u.data[last_time_step_u, :, :]
    # Checkpointed version doesn't require to save u
    u_nosave = TimeFunction(name='u_n', grid=grid)
    # change equations to use new symbols
    fwd_eqn_2 = Eq(u_nosave.forward, u_nosave + 1.*const)
    fwd_op_2 = Operator(fwd_eqn_2)
    cp = DevitoCheckpoint([u_nosave])
    wrap_fw = CheckpointOperator(fwd_op_2, time=nt, constant=1)

    prod_eqn_2 = Eq(prod, prod + u_nosave * v)
    comb_op_2 = Operator([adj_eqn, prod_eqn_2], time_axis=Backward)
    wrap_rev = CheckpointOperator(comb_op_2, constant=1)
    wrp = Revolver(cp, wrap_fw, wrap_rev, None, nt-order_of_eqn)

    # Invocation 4
    wrp.apply_forward()
    assert(np.allclose(u_nosave.data[last_time_step_v, :, :], nt - order_of_eqn))

    # Invocation 5
    wrp.apply_reverse()
    assert(np.allclose(v.data[0, :, :], 0))
    assert(np.allclose(prod.data, final_value))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import

import numpy as np
import scipy.spatial.distance as spdist

import dask.array as da

import pytest

import dask_distance

@pytest.mark.parametrize(
    "funcname, kw", [
        ("braycurtis", {}),
        ("canberra", {}),
        ("chebyshev", {}),
        ("cityblock", {}),
        ("correlation", {}),
        ("cosine", {}),
        ("euclidean", {}),
        ("mahalanobis", {"VI": 1}),
        ("minkowski", {"p": 3}),
        ("minkowski", {"p": 1.4}),
        ("seuclidean", {"V": 1}),
        ("sqeuclidean", {}),
        ("wminkowski", {"p": 3, "w": 1}),
    ]
)
@pytest.mark.parametrize("et, u, v", [
    (ValueError, np.zeros((2,), dtype=bool), np.zeros((3,), dtype=bool)),
    (ValueError, np.zeros((1, 2, 1,), dtype=bool), np.zeros((3,), dtype=bool)),
    (ValueError, np.zeros((2,), dtype=bool), np.zeros((1, 3, 1,), dtype=bool)),
])
def test_1d_dist_err(funcname, kw, et, u, v):
    da_func = getattr(dask_distance, funcname)

    with pytest.raises(et):
        da_func(u, v, **kw)

@pytest.mark.parametrize(
    "funcname, kw", [
        ("braycurtis", {}),
        ("canberra", {}),
        ("chebyshev", {}),
        ("cityblock", {}),
        ("correlation", {}),
        ("cosine", {}),
        ("euclidean", {}),
        ("mahalanobis", {}),
        ("minkowski", {"p": 3}),
        ("minkowski", {"p": 1.4}),
        ("seuclidean", {}),
        ("sqeuclidean", {}),
        ("wminkowski", {"p": 4}),
        ("wminkowski", {"p": 1.6}),
    ]
)
@pytest.mark.parametrize(
    "seed", [
        0,
        137,
        34,
    ]
)
@pytest.mark.parametrize(
    "size, chunks", [
        (10, 5),
    ]
)
def test_1d_dist(funcname, kw, seed, size, chunks):
    np.random.seed(seed)

    a_u = 2 * np.random.random((size,)) - 1
    a_v = 2 * np.random.random((size,)) - 1

    d_u = da.from_array(a_u, chunks=chunks)
    d_v = da.from_array(a_v, chunks=chunks)

    sp_func = getattr(spdist, funcname)
    da_func = getattr(dask_distance, funcname)

    if funcname == "mahalanobis":
        kw["VI"] = 2 * np.random.random((size, size)) - 1
    elif funcname == "seuclidean":
        kw["V"] = 2 * np.random.random((size,)) - 1
    elif funcname == "wminkowski":
        kw["w"] = 2 * np.random.random((size,)) - 1

    a_r = sp_func(a_u, a_v, **kw)
    d_r = da_func(d_u, d_v, **kw)

    assert np.allclose(np.array(d_r)[()], a_r, equal_nan=True)

@pytest.mark.parametrize(
    "metric, kw", [
        ("braycurtis", {}),
        ("canberra", {}),
        ("chebyshev", {}),
        ("cityblock", {}),
        ("correlation", {}),
        ("cosine", {}),
        ("euclidean", {}),
        ("mahalanobis", {"VI": None}),
        ("mahalanobis", {}),
        ("minkowski", {}),
        ("minkowski", {"p": 3}),
        ("seuclidean", {"V": None}),
        ("seuclidean", {}),
        ("sqeuclidean", {}),
        ("wminkowski", {}),
        ("wminkowski", {"p": 1.6}),
        (lambda u, v: (abs(u - v) ** 3).sum() ** (1.0 / 3.0), {}),
    ]
)
@pytest.mark.parametrize(
    "seed", [
        0,
        137,
        34,
    ]
)
@pytest.mark.parametrize(
    "u_shape, u_chunks, v_shape, v_chunks", [
        ((7, 6), (2, 2), (3, 6), (2, 2)),
    ]
)
def test_2d_cdist(metric, kw, seed, u_shape, u_chunks, v_shape, v_chunks):
    np.random.seed(seed)

    a_u = 2 * np.random.random(u_shape) - 1
    a_v = 2 * np.random.random(v_shape) - 1

    d_u = da.from_array(a_u, chunks=u_chunks)
    d_v = da.from_array(a_v, chunks=v_chunks)

    if metric == "mahalanobis":
        if "VI" not in kw:
            kw["VI"] = 2 * np.random.random(2 * u_shape[-1:]) - 1
        elif kw["VI"] is None:
            kw.pop("VI")
    elif metric == "seuclidean":
        if "V" not in kw:
            kw["V"] = 2 * np.random.random(u_shape[-1:]) - 1
        elif kw["V"] is None:
            kw.pop("V")
    elif metric == "wminkowski":
        kw["w"] = np.random.random(u_shape[-1:])

    a_r = spdist.cdist(a_u, a_v, metric, **kw)
    d_r = dask_distance.cdist(d_u, d_v, metric, **kw)

    assert np.allclose(np.array(d_r)[()], a_r, equal_nan=True)

@pytest.mark.parametrize(
    "metric, kw", [
        ("braycurtis", {}),
        ("canberra", {}),
        ("chebyshev", {}),
        ("cityblock", {}),
        ("correlation", {}),
        ("cosine", {}),
        ("euclidean", {}),
        ("mahalanobis", {"VI": None}),
        ("mahalanobis", {}),
        ("minkowski", {}),
        ("minkowski", {"p": 3}),
        ("seuclidean", {"V": None}),
        ("seuclidean", {}),
        ("sqeuclidean", {}),
        ("wminkowski", {}),
        ("wminkowski", {"p": 1.6}),
        (lambda u, v: (abs(u - v) ** 3).sum() ** (1.0 / 3.0), {}),
    ]
)
@pytest.mark.parametrize(
    "seed", [
        0,
        137,
        34,
    ]
)
@pytest.mark.parametrize(
    "u_shape, u_chunks", [
        ((7, 6), (2, 3)),
    ]
)
def test_2d_pdist(metric, kw, seed, u_shape, u_chunks):
    np.random.seed(seed)

    a_u = 2 * np.random.random(u_shape) - 1
    d_u = da.from_array(a_u, chunks=u_chunks)

    if metric == "mahalanobis":
        if "VI" not in kw:
            kw["VI"] = 2 * np.random.random(2 * u_shape[-1:]) - 1
        elif kw["VI"] is None:
            kw.pop("VI")
    elif metric == "seuclidean":
        if "V" not in kw:
            kw["V"] = 2 * np.random.random(u_shape[-1:]) - 1
        elif kw["V"] is None:
            kw.pop("V")
    elif metric == "wminkowski":
        kw["w"] = np.random.random(u_shape[-1:])

    a_r = spdist.pdist(a_u, metric, **kw)
    d_r = dask_distance.pdist(d_u, metric, **kw)

    assert np.allclose(np.array(d_r)[()], a_r, equal_nan=True)

@pytest.mark.parametrize(
    "funcname", [
        "dice",
        "hamming",
        "jaccard",
        "kulsinski",
        "rogerstanimoto",
        "russellrao",
        "sokalmichener",
        "sokalsneath",
        "yule",
    ]
)
@pytest.mark.parametrize("et, u, v", [
    (ValueError, np.zeros((2,), dtype=bool), np.zeros((3,), dtype=bool)),
    (ValueError, np.zeros((1, 2, 1,), dtype=bool), np.zeros((3,), dtype=bool)),
    (ValueError, np.zeros((2,), dtype=bool), np.zeros((1, 3, 1,), dtype=bool)),
])
def test_1d_bool_dist_err(funcname, et, u, v):
    da_func = getattr(dask_distance, funcname)

    with pytest.raises(et):
        da_func(u, v)

@pytest.mark.parametrize(
    "funcname", [
        "dice",
        "hamming",
        "jaccard",
        "kulsinski",
        "rogerstanimoto",
        "russellrao",
        "sokalmichener",
        "sokalsneath",
        "yule",
    ]
)
@pytest.mark.parametrize(
    "seed", [
        0,
        137,
        34,
    ]
)
@pytest.mark.parametrize(
    "size, chunks", [
        (10, 5),
    ]
)
def test_1d_bool_dist(funcname, seed, size, chunks):
    np.random.seed(seed)

    a_u = np.random.randint(0, 2, (size,), dtype=bool)
    a_v = np.random.randint(0, 2, (size,), dtype=bool)

    d_u = da.from_array(a_u, chunks=chunks)
    d_v = da.from_array(a_v, chunks=chunks)

    sp_func = getattr(spdist, funcname)
    da_func = getattr(dask_distance, funcname)

    a_r = sp_func(a_u, a_v)
    d_r = da_func(d_u, d_v)

    assert np.allclose(np.array(d_r)[()], a_r, equal_nan=True)

@pytest.mark.parametrize(
    "metric", [
        "dice",
        "hamming",
        "jaccard",
        "kulsinski",
        "rogerstanimoto",
        "russellrao",
        "sokalmichener",
        "sokalsneath",
        "yule",
        lambda u, v: (abs(u - v) ** 3).sum() ** (1.0 / 3.0),
    ]
)
@pytest.mark.parametrize(
    "seed", [
        0,
        137,
        34,
    ]
)
@pytest.mark.parametrize(
    "u_shape, u_chunks, v_shape, v_chunks", [
        ((2, 10), (1, 5), (3, 10), (1, 5)),
    ]
)
def test_2d_bool_cdist(metric, seed, u_shape, u_chunks, v_shape, v_chunks):
    np.random.seed(seed)

    a_u = np.random.randint(0, 2, u_shape, dtype=bool)
    a_v = np.random.randint(0, 2, v_shape, dtype=bool)

    d_u = da.from_array(a_u, chunks=u_chunks)
    d_v = da.from_array(a_v, chunks=v_chunks)

    a_r = spdist.cdist(a_u, a_v, metric)
    d_r = dask_distance.cdist(d_u, d_v, metric)

    assert np.allclose(np.array(d_r)[()], a_r, equal_nan=True)

@pytest.mark.parametrize(
    "metric", [
        "dice",
        "hamming",
        "jaccard",
        "kulsinski",
        "rogerstanimoto",
        "russellrao",
        "sokalmichener",
        "sokalsneath",
        "yule",
        lambda u, v: (abs(u - v) ** 3).sum() ** (1.0 / 3.0),
    ]
)
@pytest.mark.parametrize(
    "seed", [
        0,
        137,
        34,
    ]
)
@pytest.mark.parametrize(
    "u_shape, u_chunks", [
        ((3, 10), (1, 5)),
    ]
)
def test_2d_bool_pdist(metric, seed, u_shape, u_chunks):
    np.random.seed(seed)

    a_u = np.random.randint(0, 2, u_shape, dtype=bool)
    d_u = da.from_array(a_u, chunks=u_chunks)

    a_r = spdist.pdist(a_u, metric)
    d_r = dask_distance.pdist(d_u, metric)

    assert np.allclose(np.array(d_r)[()], a_r, equal_nan=True)
<commit_msg>Test cdist's result shapes match<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import

import numpy as np
import scipy.spatial.distance as spdist

import dask.array as da

import pytest

import dask_distance

@pytest.mark.parametrize(
    "funcname, kw", [
        ("braycurtis", {}),
        ("canberra", {}),
        ("chebyshev", {}),
        ("cityblock", {}),
        ("correlation", {}),
        ("cosine", {}),
        ("euclidean", {}),
        ("mahalanobis", {"VI": 1}),
        ("minkowski", {"p": 3}),
        ("minkowski", {"p": 1.4}),
        ("seuclidean", {"V": 1}),
        ("sqeuclidean", {}),
        ("wminkowski", {"p": 3, "w": 1}),
    ]
)
@pytest.mark.parametrize("et, u, v", [
    (ValueError, np.zeros((2,), dtype=bool), np.zeros((3,), dtype=bool)),
    (ValueError, np.zeros((1, 2, 1,), dtype=bool), np.zeros((3,), dtype=bool)),
    (ValueError, np.zeros((2,), dtype=bool), np.zeros((1, 3, 1,), dtype=bool)),
])
def test_1d_dist_err(funcname, kw, et, u, v):
    da_func = getattr(dask_distance, funcname)

    with pytest.raises(et):
        da_func(u, v, **kw)

@pytest.mark.parametrize(
    "funcname, kw", [
        ("braycurtis", {}),
        ("canberra", {}),
        ("chebyshev", {}),
        ("cityblock", {}),
        ("correlation", {}),
        ("cosine", {}),
        ("euclidean", {}),
        ("mahalanobis", {}),
        ("minkowski", {"p": 3}),
        ("minkowski", {"p": 1.4}),
        ("seuclidean", {}),
        ("sqeuclidean", {}),
        ("wminkowski", {"p": 4}),
        ("wminkowski", {"p": 1.6}),
    ]
)
@pytest.mark.parametrize(
    "seed", [
        0,
        137,
        34,
    ]
)
@pytest.mark.parametrize(
    "size, chunks", [
        (10, 5),
    ]
)
def test_1d_dist(funcname, kw, seed, size, chunks):
    np.random.seed(seed)

    a_u = 2 * np.random.random((size,)) - 1
    a_v = 2 * np.random.random((size,)) - 1

    d_u = da.from_array(a_u, chunks=chunks)
    d_v = da.from_array(a_v, chunks=chunks)

    sp_func = getattr(spdist, funcname)
    da_func = getattr(dask_distance, funcname)

    if funcname == "mahalanobis":
        kw["VI"] = 2 * np.random.random((size, size)) - 1
    elif funcname == "seuclidean":
        kw["V"] = 2 * np.random.random((size,)) - 1
    elif funcname == "wminkowski":
        kw["w"] = 2 * np.random.random((size,)) - 1

    a_r = sp_func(a_u, a_v, **kw)
    d_r = da_func(d_u, d_v, **kw)

    assert np.allclose(np.array(d_r)[()], a_r, equal_nan=True)

@pytest.mark.parametrize(
    "metric, kw", [
        ("braycurtis", {}),
        ("canberra", {}),
        ("chebyshev", {}),
        ("cityblock", {}),
        ("correlation", {}),
        ("cosine", {}),
        ("euclidean", {}),
        ("mahalanobis", {"VI": None}),
        ("mahalanobis", {}),
        ("minkowski", {}),
        ("minkowski", {"p": 3}),
        ("seuclidean", {"V": None}),
        ("seuclidean", {}),
        ("sqeuclidean", {}),
        ("wminkowski", {}),
        ("wminkowski", {"p": 1.6}),
        (lambda u, v: (abs(u - v) ** 3).sum() ** (1.0 / 3.0), {}),
    ]
)
@pytest.mark.parametrize(
    "seed", [
        0,
        137,
        34,
    ]
)
@pytest.mark.parametrize(
    "u_shape, u_chunks, v_shape, v_chunks", [
        ((7, 6), (2, 2), (3, 6), (2, 2)),
    ]
)
def test_2d_cdist(metric, kw, seed, u_shape, u_chunks, v_shape, v_chunks):
    np.random.seed(seed)

    a_u = 2 * np.random.random(u_shape) - 1
    a_v = 2 * np.random.random(v_shape) - 1

    d_u = da.from_array(a_u, chunks=u_chunks)
    d_v = da.from_array(a_v, chunks=v_chunks)

    if metric == "mahalanobis":
        if "VI" not in kw:
            kw["VI"] = 2 * np.random.random(2 * u_shape[-1:]) - 1
        elif kw["VI"] is None:
            kw.pop("VI")
    elif metric == "seuclidean":
        if "V" not in kw:
            kw["V"] = 2 * np.random.random(u_shape[-1:]) - 1
        elif kw["V"] is None:
            kw.pop("V")
    elif metric == "wminkowski":
        kw["w"] = np.random.random(u_shape[-1:])

    a_r = spdist.cdist(a_u, a_v, metric, **kw)
    d_r = dask_distance.cdist(d_u, d_v, metric, **kw)

    assert d_r.shape == a_r.shape
    assert np.allclose(np.array(d_r)[()], a_r, equal_nan=True)

@pytest.mark.parametrize(
    "metric, kw", [
        ("braycurtis", {}),
        ("canberra", {}),
        ("chebyshev", {}),
        ("cityblock", {}),
        ("correlation", {}),
        ("cosine", {}),
        ("euclidean", {}),
        ("mahalanobis", {"VI": None}),
        ("mahalanobis", {}),
        ("minkowski", {}),
        ("minkowski", {"p": 3}),
        ("seuclidean", {"V": None}),
        ("seuclidean", {}),
        ("sqeuclidean", {}),
        ("wminkowski", {}),
        ("wminkowski", {"p": 1.6}),
        (lambda u, v: (abs(u - v) ** 3).sum() ** (1.0 / 3.0), {}),
    ]
)
@pytest.mark.parametrize(
    "seed", [
        0,
        137,
        34,
    ]
)
@pytest.mark.parametrize(
    "u_shape, u_chunks", [
        ((7, 6), (2, 3)),
    ]
)
def test_2d_pdist(metric, kw, seed, u_shape, u_chunks):
    np.random.seed(seed)

    a_u = 2 * np.random.random(u_shape) - 1
    d_u = da.from_array(a_u, chunks=u_chunks)

    if metric == "mahalanobis":
        if "VI" not in kw:
            kw["VI"] = 2 * np.random.random(2 * u_shape[-1:]) - 1
        elif kw["VI"] is None:
            kw.pop("VI")
    elif metric == "seuclidean":
        if "V" not in kw:
            kw["V"] = 2 * np.random.random(u_shape[-1:]) - 1
        elif kw["V"] is None:
            kw.pop("V")
    elif metric == "wminkowski":
        kw["w"] = np.random.random(u_shape[-1:])

    a_r = spdist.pdist(a_u, metric, **kw)
    d_r = dask_distance.pdist(d_u, metric, **kw)

    assert np.allclose(np.array(d_r)[()], a_r, equal_nan=True)

@pytest.mark.parametrize(
    "funcname", [
        "dice",
        "hamming",
        "jaccard",
        "kulsinski",
        "rogerstanimoto",
        "russellrao",
        "sokalmichener",
        "sokalsneath",
        "yule",
    ]
)
@pytest.mark.parametrize("et, u, v", [
    (ValueError, np.zeros((2,), dtype=bool), np.zeros((3,), dtype=bool)),
    (ValueError, np.zeros((1, 2, 1,), dtype=bool), np.zeros((3,), dtype=bool)),
    (ValueError, np.zeros((2,), dtype=bool), np.zeros((1, 3, 1,), dtype=bool)),
])
def test_1d_bool_dist_err(funcname, et, u, v):
    da_func = getattr(dask_distance, funcname)

    with pytest.raises(et):
        da_func(u, v)

@pytest.mark.parametrize(
    "funcname", [
        "dice",
        "hamming",
        "jaccard",
        "kulsinski",
        "rogerstanimoto",
        "russellrao",
        "sokalmichener",
        "sokalsneath",
        "yule",
    ]
)
@pytest.mark.parametrize(
    "seed", [
        0,
        137,
        34,
    ]
)
@pytest.mark.parametrize(
    "size, chunks", [
        (10, 5),
    ]
)
def test_1d_bool_dist(funcname, seed, size, chunks):
    np.random.seed(seed)

    a_u = np.random.randint(0, 2, (size,), dtype=bool)
    a_v = np.random.randint(0, 2, (size,), dtype=bool)

    d_u = da.from_array(a_u, chunks=chunks)
    d_v = da.from_array(a_v, chunks=chunks)

    sp_func = getattr(spdist, funcname)
    da_func = getattr(dask_distance, funcname)

    a_r = sp_func(a_u, a_v)
    d_r = da_func(d_u, d_v)

    assert np.allclose(np.array(d_r)[()], a_r, equal_nan=True)

@pytest.mark.parametrize(
    "metric", [
        "dice",
        "hamming",
        "jaccard",
        "kulsinski",
        "rogerstanimoto",
        "russellrao",
        "sokalmichener",
        "sokalsneath",
        "yule",
        lambda u, v: (abs(u - v) ** 3).sum() ** (1.0 / 3.0),
    ]
)
@pytest.mark.parametrize(
    "seed", [
        0,
        137,
        34,
    ]
)
@pytest.mark.parametrize(
    "u_shape, u_chunks, v_shape, v_chunks", [
        ((2, 10), (1, 5), (3, 10), (1, 5)),
    ]
)
def test_2d_bool_cdist(metric, seed, u_shape, u_chunks, v_shape, v_chunks):
    np.random.seed(seed)

    a_u = np.random.randint(0, 2, u_shape, dtype=bool)
    a_v = np.random.randint(0, 2, v_shape, dtype=bool)

    d_u = da.from_array(a_u, chunks=u_chunks)
    d_v = da.from_array(a_v, chunks=v_chunks)

    a_r = spdist.cdist(a_u, a_v, metric)
    d_r = dask_distance.cdist(d_u, d_v, metric)

    assert d_r.shape == a_r.shape
    assert np.allclose(np.array(d_r)[()], a_r, equal_nan=True)

@pytest.mark.parametrize(
    "metric", [
        "dice",
        "hamming",
        "jaccard",
        "kulsinski",
        "rogerstanimoto",
        "russellrao",
        "sokalmichener",
        "sokalsneath",
        "yule",
        lambda u, v: (abs(u - v) ** 3).sum() ** (1.0 / 3.0),
    ]
)
@pytest.mark.parametrize(
    "seed", [
        0,
        137,
        34,
    ]
)
@pytest.mark.parametrize(
    "u_shape, u_chunks", [
        ((3, 10), (1, 5)),
    ]
)
def test_2d_bool_pdist(metric, seed, u_shape, u_chunks):
    np.random.seed(seed)

    a_u = np.random.randint(0, 2, u_shape, dtype=bool)
    d_u = da.from_array(a_u, chunks=u_chunks)

    a_r = spdist.pdist(a_u, metric)
    d_r = dask_distance.pdist(d_u, metric)

    assert np.allclose(np.array(d_r)[()], a_r, equal_nan=True)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright (c) 2016 - 2020 -- Lars Heuer
# All rights reserved.
#
# License: BSD License
#
"""\
EPC QR Codes.

Test against issue <https://github.com/heuer/segno/issues/55>.
"""
from __future__ import absolute_import, unicode_literals
import decimal
import pytest
from segno.helpers import make_epc_qr, _make_epc_qr_data as make_epc_qr_data

@pytest.mark.parametrize('amount', [12.3,
                                    12.30,
                                    decimal.Decimal('12.3'),
                                    decimal.Decimal('12.30'),
                                    '12.3',
                                    '12.30'])
def test_text_002(amount):
    name = "François D'Alsace S.A."
    iban = 'FR1420041010050500013M02606'
    text = 'Client:Marie Louise La Lune'
    kw = dict(name=name, iban=iban, text=text, amount=amount)
    data = make_epc_qr_data(**kw)
    assert len(data) == 103  # See. EPC069-12 Version 2.1 dtd. 9 February 2012 example 2
    encoding = 'iso-8859-1'
    d = [x.decode(encoding) for x in data.split(b'\n')]
    assert 11 == len(d)
    assert 'BCD' == d[0]
    assert '002' == d[1]
    assert '2' == d[2]
    assert 'SCT' == d[3]
    assert name == d[5]
    assert iban == d[6]
    assert 'EUR12.3' == d[7]
    assert '' == d[8]
    assert '' == d[9]
    assert text == d[10]
    qr = make_epc_qr(**kw)
    assert qr
    assert not qr.is_micro
    assert qr.version <= 13
    assert 'M' == qr.error

@pytest.mark.parametrize('expected_amount, amount', [('EUR1000', 1000),
                                                     ('EUR1000', 1000.0),
                                                     ('EUR2000', decimal.Decimal('2000'))])
def test_trailing_zeros(expected_amount, amount):
    name = "François D'Alsace S.A."
    iban = 'FR1420041010050500013M02606'
    text = 'Client:Marie Louise La Lune'
    kw = dict(name=name, iban=iban, text=text, amount=amount)
    data = make_epc_qr_data(**kw)
    assert len(data) == 103  # See. EPC069-12 Version 2.1 dtd. 9 February 2012 example 2
    encoding = 'iso-8859-1'
    d = [x.decode(encoding) for x in data.split(b'\n')]
    assert expected_amount == d[7]

@pytest.mark.parametrize('amount', [5.0, 5, '5.00', decimal.Decimal('5.00000')])
def test_remove_dot(amount):
    kw = _make_valid_kw()
    kw['amount'] = amount
    d = make_epc_qr_data(**kw).split(b'\n')
    assert b'EUR5' == d[7]

@pytest.mark.parametrize('amount', [12.3,
                                    12.30,
                                    decimal.Decimal('12.3'),
                                    decimal.Decimal('12.30'),
                                    '12.3',
                                    '12.30'])
def test_reference_002(amount):
    name = 'Franz Mustermänn'
    iban = 'DE71110220330123456789'
    reference = 'RF18539007547034'
    purpose = 'GDDS'
    bic = 'BHBLDEHHXXX'
    kw = dict(name=name,
              iban=iban,
              reference=reference,
              bic=bic,
              purpose=purpose,
              amount=amount,
              encoding=1)
    data = make_epc_qr_data(**kw)
    assert len(data) == 96  # See. EPC069-12 Version 2.1 dtd. 9 February 2012 example 1
    encoding = 'utf-8'
    d = [x.decode(encoding) for x in data.split(b'\n')]
    assert 10 == len(d)
    assert 'BCD' == d[0]
    assert '002' == d[1]
    assert '1' == d[2]
    assert 'SCT' == d[3]
    assert name == d[5]
    assert iban == d[6]
    assert 'EUR12.3' == d[7]
    assert purpose == d[8]
    assert reference == d[9]
    qr = make_epc_qr(**kw)
    assert qr
    assert not qr.is_micro
    assert qr.version <= 13
    assert 'M' == qr.error

def _make_valid_kw():
    return dict(name="François D'Alsace S.A.",
                iban='FR1420041010050500013M02606',
                text='Client:Marie Louise La Lune',
                amount=12.3)

@pytest.mark.parametrize('amount', [0,
                                    0.004,
                                    '0.001',
                                    '999999999.999',
                                    9999999990.99])
def test_invalid_amount(amount):
    kw = _make_valid_kw()
    kw['amount'] = amount
    with pytest.raises(ValueError) as ex:
        make_epc_qr_data(**kw)
    assert 'amount' in str(ex.value)
    with pytest.raises(ValueError) as ex:
        make_epc_qr(**kw)
    assert 'amount' in str(ex.value)

@pytest.mark.parametrize('bic', ['BHBLDE',  # Too short
                                 'BHBLDEHHXXXX',  # Too long
                                 'BHBLDEHHXX',  # Too short (either 8 or 11) not 8 <= bic <= 11
                                 'BHBLDEH ',  # Too short after removing trailing WS
                                 ])
def test_invalid_bic(bic):
    kw = _make_valid_kw()
    kw['bic'] = bic
    with pytest.raises(ValueError) as ex:
        make_epc_qr_data(**kw)
    assert 'BIC' in str(ex.value)
    with pytest.raises(ValueError) as ex:
        make_epc_qr(**kw)
    assert 'BIC' in str(ex.value)

def test_utf8_required():
    kw = _make_valid_kw()
    kw['name'] = 'Funny 😃 name'
    d = make_epc_qr_data(**kw).split(b'\n')
    assert b'1' == d[2]

@pytest.mark.parametrize('encoding', range(1, 9))
def test_valid_encoding(encoding):
    kw = _make_valid_kw()
    kw['name'] = 'Simple name'
    kw['encoding'] = encoding
    d = make_epc_qr_data(**kw).split(b'\n')
    assert str(encoding).encode() == d[2]
    qr = make_epc_qr(**kw)
    assert qr

@pytest.mark.parametrize('encoding', [0, 9, '1', b'8', 1.0])
def test_illegal_encoding(encoding):
    kw = _make_valid_kw()
    kw['encoding'] = encoding
    with pytest.raises(ValueError) as ex:
        make_epc_qr_data(**kw)
    assert 'encoding' in str(ex.value)
    with pytest.raises(ValueError) as ex:
        make_epc_qr(**kw)
    assert 'encoding' in str(ex.value)

@pytest.mark.parametrize('text,reference', [('', ''), (' ', '    '),
                                            ('', None), (None, None),
                                            (None, ' '),
                                            ])
def test_no_text_no_reference(text, reference):
    kw = _make_valid_kw()
    kw['text'] = text
    kw['reference'] = reference
    with pytest.raises(ValueError) as ex:
        make_epc_qr_data(**kw)
    assert 'reference' in str(ex.value)
    with pytest.raises(ValueError) as ex:
        make_epc_qr(**kw)
    assert 'reference' in str(ex.value)

@pytest.mark.parametrize('iban', ['DE1' + '1' * 34,
                                  '',
                                  None])
def test_illegal_iban(iban):
    kw = _make_valid_kw()
    kw['iban'] = iban
    with pytest.raises(ValueError) as ex:
        make_epc_qr_data(**kw)
    assert 'IBAN' in str(ex.value)
    with pytest.raises(ValueError) as ex:
        make_epc_qr(**kw)
    assert 'IBAN' in str(ex.value)

@pytest.mark.parametrize('purpose', ['DE1', 'x', 'CDCBC'])
def test_illegal_purpose(purpose):
    kw = _make_valid_kw()
    kw['purpose'] = purpose
    with pytest.raises(ValueError) as ex:
        make_epc_qr_data(**kw)
    assert 'purpose' in str(ex.value)
    with pytest.raises(ValueError) as ex:
        make_epc_qr(**kw)
    assert 'purpose' in str(ex.value)

@pytest.mark.parametrize('name', [None, '',
                                  'a' * 71,  # too long
                                  ])
def test_illegal_name(name):
    kw = _make_valid_kw()
    kw['name'] = name
    with pytest.raises(ValueError) as ex:
        make_epc_qr_data(**kw)
    assert 'name' in str(ex.value)
    with pytest.raises(ValueError) as ex:
        make_epc_qr(**kw)
    assert 'name' in str(ex.value)

def test_text_too_long():
    kw = _make_valid_kw()
    kw['text'] = 'a' * 141
    kw['reference'] = None
    with pytest.raises(ValueError) as ex:
        make_epc_qr_data(**kw)
    assert 'text' in str(ex.value)

def test_refernce_too_long():
    kw = _make_valid_kw()
    kw['text'] = None
    kw['reference'] = 'r' * 36
    with pytest.raises(ValueError) as ex:
        make_epc_qr_data(**kw)
    assert 'reference' in str(ex.value)

if __name__ == '__main__':
    pytest.main([__file__])
<commit_msg>Fixed typo<commit_after># -*- coding: utf-8 -*-
#
# Copyright (c) 2016 - 2020 -- Lars Heuer
# All rights reserved.
#
# License: BSD License
#
"""\
EPC QR Codes.

Test against issue <https://github.com/heuer/segno/issues/55>.
"""
from __future__ import absolute_import, unicode_literals
import decimal
import pytest
from segno.helpers import make_epc_qr, _make_epc_qr_data as make_epc_qr_data

@pytest.mark.parametrize('amount', [12.3,
                                    12.30,
                                    decimal.Decimal('12.3'),
                                    decimal.Decimal('12.30'),
                                    '12.3',
                                    '12.30'])
def test_text_002(amount):
    name = "François D'Alsace S.A."
    iban = 'FR1420041010050500013M02606'
    text = 'Client:Marie Louise La Lune'
    kw = dict(name=name, iban=iban, text=text, amount=amount)
    data = make_epc_qr_data(**kw)
    assert len(data) == 103  # See. EPC069-12 Version 2.1 dtd. 9 February 2012 example 2
    encoding = 'iso-8859-1'
    d = [x.decode(encoding) for x in data.split(b'\n')]
    assert 11 == len(d)
    assert 'BCD' == d[0]
    assert '002' == d[1]
    assert '2' == d[2]
    assert 'SCT' == d[3]
    assert name == d[5]
    assert iban == d[6]
    assert 'EUR12.3' == d[7]
    assert '' == d[8]
    assert '' == d[9]
    assert text == d[10]
    qr = make_epc_qr(**kw)
    assert qr
    assert not qr.is_micro
    assert qr.version <= 13
    assert 'M' == qr.error

@pytest.mark.parametrize('expected_amount, amount', [('EUR1000', 1000),
                                                     ('EUR1000', 1000.0),
                                                     ('EUR2000', decimal.Decimal('2000'))])
def test_trailing_zeros(expected_amount, amount):
    name = "François D'Alsace S.A."
    iban = 'FR1420041010050500013M02606'
    text = 'Client:Marie Louise La Lune'
    kw = dict(name=name, iban=iban, text=text, amount=amount)
    data = make_epc_qr_data(**kw)
    assert len(data) == 103  # See. EPC069-12 Version 2.1 dtd. 9 February 2012 example 2
    encoding = 'iso-8859-1'
    d = [x.decode(encoding) for x in data.split(b'\n')]
    assert expected_amount == d[7]

@pytest.mark.parametrize('amount', [5.0, 5, '5.00', decimal.Decimal('5.00000')])
def test_remove_dot(amount):
    kw = _make_valid_kw()
    kw['amount'] = amount
    d = make_epc_qr_data(**kw).split(b'\n')
    assert b'EUR5' == d[7]

@pytest.mark.parametrize('amount', [12.3,
                                    12.30,
                                    decimal.Decimal('12.3'),
                                    decimal.Decimal('12.30'),
                                    '12.3',
                                    '12.30'])
def test_reference_002(amount):
    name = 'Franz Mustermänn'
    iban = 'DE71110220330123456789'
    reference = 'RF18539007547034'
    purpose = 'GDDS'
    bic = 'BHBLDEHHXXX'
    kw = dict(name=name,
              iban=iban,
              reference=reference,
              bic=bic,
              purpose=purpose,
              amount=amount,
              encoding=1)
    data = make_epc_qr_data(**kw)
    assert len(data) == 96  # See. EPC069-12 Version 2.1 dtd. 9 February 2012 example 1
    encoding = 'utf-8'
    d = [x.decode(encoding) for x in data.split(b'\n')]
    assert 10 == len(d)
    assert 'BCD' == d[0]
    assert '002' == d[1]
    assert '1' == d[2]
    assert 'SCT' == d[3]
    assert name == d[5]
    assert iban == d[6]
    assert 'EUR12.3' == d[7]
    assert purpose == d[8]
    assert reference == d[9]
    qr = make_epc_qr(**kw)
    assert qr
    assert not qr.is_micro
    assert qr.version <= 13
    assert 'M' == qr.error

def _make_valid_kw():
    return dict(name="François D'Alsace S.A.",
                iban='FR1420041010050500013M02606',
                text='Client:Marie Louise La Lune',
                amount=12.3)

@pytest.mark.parametrize('amount', [0,
                                    0.004,
                                    '0.001',
                                    '999999999.999',
                                    9999999990.99])
def test_invalid_amount(amount):
    kw = _make_valid_kw()
    kw['amount'] = amount
    with pytest.raises(ValueError) as ex:
        make_epc_qr_data(**kw)
    assert 'amount' in str(ex.value)
    with pytest.raises(ValueError) as ex:
        make_epc_qr(**kw)
    assert 'amount' in str(ex.value)

@pytest.mark.parametrize('bic', ['BHBLDE',  # Too short
                                 'BHBLDEHHXXXX',  # Too long
                                 'BHBLDEHHXX',  # Too short (either 8 or 11) not 8 <= bic <= 11
                                 'BHBLDEH ',  # Too short after removing trailing WS
                                 ])
def test_invalid_bic(bic):
    kw = _make_valid_kw()
    kw['bic'] = bic
    with pytest.raises(ValueError) as ex:
        make_epc_qr_data(**kw)
    assert 'BIC' in str(ex.value)
    with pytest.raises(ValueError) as ex:
        make_epc_qr(**kw)
    assert 'BIC' in str(ex.value)

def test_utf8_required():
    kw = _make_valid_kw()
    kw['name'] = 'Funny 😃 name'
    d = make_epc_qr_data(**kw).split(b'\n')
    assert b'1' == d[2]

@pytest.mark.parametrize('encoding', range(1, 9))
def test_valid_encoding(encoding):
    kw = _make_valid_kw()
    kw['name'] = 'Simple name'
    kw['encoding'] = encoding
    d = make_epc_qr_data(**kw).split(b'\n')
    assert str(encoding).encode() == d[2]
    qr = make_epc_qr(**kw)
    assert qr

@pytest.mark.parametrize('encoding', [0, 9, '1', b'8', 1.0])
def test_illegal_encoding(encoding):
    kw = _make_valid_kw()
    kw['encoding'] = encoding
    with pytest.raises(ValueError) as ex:
        make_epc_qr_data(**kw)
    assert 'encoding' in str(ex.value)
    with pytest.raises(ValueError) as ex:
        make_epc_qr(**kw)
    assert 'encoding' in str(ex.value)

@pytest.mark.parametrize('text,reference', [('', ''), (' ', '    '),
                                            ('', None), (None, None),
                                            (None, ' '),
                                            ])
def test_no_text_no_reference(text, reference):
    kw = _make_valid_kw()
    kw['text'] = text
    kw['reference'] = reference
    with pytest.raises(ValueError) as ex:
        make_epc_qr_data(**kw)
    assert 'reference' in str(ex.value)
    with pytest.raises(ValueError) as ex:
        make_epc_qr(**kw)
    assert 'reference' in str(ex.value)

@pytest.mark.parametrize('iban', ['DE1' + '1' * 34,
                                  '',
                                  None])
def test_illegal_iban(iban):
    kw = _make_valid_kw()
    kw['iban'] = iban
    with pytest.raises(ValueError) as ex:
        make_epc_qr_data(**kw)
    assert 'IBAN' in str(ex.value)
    with pytest.raises(ValueError) as ex:
        make_epc_qr(**kw)
    assert 'IBAN' in str(ex.value)

@pytest.mark.parametrize('purpose', ['DE1', 'x', 'CDCBC'])
def test_illegal_purpose(purpose):
    kw = _make_valid_kw()
    kw['purpose'] = purpose
    with pytest.raises(ValueError) as ex:
        make_epc_qr_data(**kw)
    assert 'purpose' in str(ex.value)
    with pytest.raises(ValueError) as ex:
        make_epc_qr(**kw)
    assert 'purpose' in str(ex.value)

@pytest.mark.parametrize('name', [None, '',
                                  'a' * 71,  # too long
                                  ])
def test_illegal_name(name):
    kw = _make_valid_kw()
    kw['name'] = name
    with pytest.raises(ValueError) as ex:
        make_epc_qr_data(**kw)
    assert 'name' in str(ex.value)
    with pytest.raises(ValueError) as ex:
        make_epc_qr(**kw)
    assert 'name' in str(ex.value)

def test_text_too_long():
    kw = _make_valid_kw()
    kw['text'] = 'a' * 141
    kw['reference'] = None
    with pytest.raises(ValueError) as ex:
        make_epc_qr_data(**kw)
    assert 'text' in str(ex.value)

def test_reference_too_long():
    kw = _make_valid_kw()
    kw['text'] = None
    kw['reference'] = 'r' * 36
    with pytest.raises(ValueError) as ex:
        make_epc_qr_data(**kw)
    assert 'reference' in str(ex.value)

if __name__ == '__main__':
    pytest.main([__file__])
<|endoftext|>


<commit_before>import pytest

import pandas as pd
import numpy as np
import itertools

from vivarium.interpolation import Interpolation, validate_parameters

def test_1d_interpolation():
    df = pd.DataFrame({'a': np.arange(100), 'b': np.arange(100), 'c': np.arange(100, 0, -1)})
    df = df.sample(frac=1)  # Shuffle table to assure interpolation works given unsorted input

    i = Interpolation(df, (), ('a',), 1)

    query = pd.DataFrame({'a': np.arange(100, step=0.01)})

    assert np.allclose(query.a, i(query).b)
    assert np.allclose(100-query.a, i(query).c)

def test_age_year_interpolation():
    years = list(range(1990, 2010))
    ages = list(range(0, 90))
    pops = np.array(ages)*11.1
    data = []
    for age, pop in zip(ages, pops):
        for year in years:
            for sex in ['Male', 'Female']:
                data.append({'age': age, 'sex': sex, 'year': year, 'pop': pop})
    df = pd.DataFrame(data)

    df = df.sample(frac=1)  # Shuffle table to assure interpolation works given unsorted input

    i = Interpolation(df, ('sex', 'age'), ('year',), 1)
    query = pd.DataFrame({'year': [1990, 1990], 'age': [35, 35], 'sex': ['Male', 'Female']})
    assert np.allclose(i(query), 388.5)

def test_interpolation_called_missing_key_col():
    a = [range(1990, 1995), range(25, 30), ['Male', 'Female']]
    df = pd.DataFrame(list(itertools.product(*a)), columns=['year', 'age', 'sex'])
    df['pop'] = df.age * 11.1
    df = df.sample(frac=1)  # Shuffle table to assure interpolation works given unsorted input
    i = Interpolation(df, ['sex',], ['year','age'], 1)
    query = pd.DataFrame({'year': [1990, 1990], 'age': [35, 35]})
    with pytest.raises(ValueError):
        i(query)

def test_interpolation_called_missing_param_col():
    a = [range(1990, 1995), range(25, 30), ['Male', 'Female']]
    df = pd.DataFrame(list(itertools.product(*a)), columns=['year', 'age', 'sex'])
    df['pop'] = df.age * 11.1
    df = df.sample(frac=1)  # Shuffle table to assure interpolation works given unsorted input
    i = Interpolation(df, ['sex',], ['year','age'], 1)
    query = pd.DataFrame({'year': [1990, 1990], 'sex': ['Male', 'Female']})
    with pytest.raises(ValueError):
        i(query)

def test_2d_interpolation():
    a = np.mgrid[0:5, 0:5][0].reshape(25)
    b = np.mgrid[0:5, 0:5][1].reshape(25)
    df = pd.DataFrame({'a': a, 'b': b, 'c': b, 'd': a})
    df = df.sample(frac=1)  # Shuffle table to assure interpolation works given unsorted input

    i = Interpolation(df, (), ('a', 'b'), 1)

    query = pd.DataFrame({'a': np.arange(4, step=0.01), 'b': np.arange(4, step=0.01)})

    assert np.allclose(query.b, i(query).c)
    assert np.allclose(query.a, i(query).d)

def test_interpolation_with_categorical_parameters():
    a = ['one']*100 + ['two']*100
    b = np.append(np.arange(100), np.arange(100))
    c = np.append(np.arange(100), np.arange(100, 0, -1))
    df = pd.DataFrame({'a': a, 'b': b, 'c': c})
    df = df.sample(frac=1)  # Shuffle table to assure interpolation works given unsorted input

    i = Interpolation(df, ('a',), ('b',), 1)

    query_one = pd.DataFrame({'a': 'one', 'b': np.arange(100, step=0.01)})
    query_two = pd.DataFrame({'a': 'two', 'b': np.arange(100, step=0.01)})

    assert np.allclose(np.arange(100, step=0.01), i(query_one).c)

    assert np.allclose(np.arange(100, 0, step=-0.01), i(query_two).c)

def test_order_zero_2d():
    a = np.mgrid[0:5, 0:5][0].reshape(25)
    b = np.mgrid[0:5, 0:5][1].reshape(25)
    df = pd.DataFrame({'a': a + 0.5, 'b': b + 0.5, 'c': b*3, 'garbage': ['test']*len(a)})
    df = df.sample(frac=1)  # Shuffle table to assure interpolation works given unsorted input

    i = Interpolation(df, ('garbage',), ('a', 'b'), order=0)

    column = np.arange(4, step=0.011)
    query = pd.DataFrame({'a': column, 'b': column, 'garbage': ['test']*(len(column))})

    assert np.allclose(query.b.astype(int) * 3, i(query).c)

def test_order_zero_1d():
    s = pd.Series({0: 0, 1: 1}).reset_index()
    f = Interpolation(s, tuple(), ('index', ), order=0)

    assert f(pd.DataFrame({'index': [0]}))[0][0] == 0, 'should be precise at index values'
    assert f(pd.DataFrame({'index': [1]}))[0][0] == 1
    assert f(pd.DataFrame({'index': [2]}))[0][0] == 1, 'should be constant extrapolation outside of input range'
    assert f(pd.DataFrame({'index': [-1]}))[0][0] == 0

def test_validate_parameters__empty_data():
    with pytest.warns(UserWarning) as record:
        out, data = validate_parameters(pd.DataFrame(columns=["age", "sex", "year", "value"]), ["age", "year"], 1)
    assert len(record) == 2
    message = record[0].message.args[0] + " " + record[1].message.args[0]
    assert "age" in message and "year" in message

    assert set(data.columns) == {"sex", "value"}

<commit_msg>updating validate parameters test to match validation of all params<commit_after>import pytest

import pandas as pd
import numpy as np
import itertools

from vivarium.interpolation import Interpolation, validate_parameters

def test_1d_interpolation():
    df = pd.DataFrame({'a': np.arange(100), 'b': np.arange(100), 'c': np.arange(100, 0, -1)})
    df = df.sample(frac=1)  # Shuffle table to assure interpolation works given unsorted input

    i = Interpolation(df, (), ('a',), 1)

    query = pd.DataFrame({'a': np.arange(100, step=0.01)})

    assert np.allclose(query.a, i(query).b)
    assert np.allclose(100-query.a, i(query).c)

def test_age_year_interpolation():
    years = list(range(1990, 2010))
    ages = list(range(0, 90))
    pops = np.array(ages)*11.1
    data = []
    for age, pop in zip(ages, pops):
        for year in years:
            for sex in ['Male', 'Female']:
                data.append({'age': age, 'sex': sex, 'year': year, 'pop': pop})
    df = pd.DataFrame(data)

    df = df.sample(frac=1)  # Shuffle table to assure interpolation works given unsorted input

    i = Interpolation(df, ('sex', 'age'), ('year',), 1)
    query = pd.DataFrame({'year': [1990, 1990], 'age': [35, 35], 'sex': ['Male', 'Female']})
    assert np.allclose(i(query), 388.5)

def test_interpolation_called_missing_key_col():
    a = [range(1990, 1995), range(25, 30), ['Male', 'Female']]
    df = pd.DataFrame(list(itertools.product(*a)), columns=['year', 'age', 'sex'])
    df['pop'] = df.age * 11.1
    df = df.sample(frac=1)  # Shuffle table to assure interpolation works given unsorted input
    i = Interpolation(df, ['sex',], ['year','age'], 1)
    query = pd.DataFrame({'year': [1990, 1990], 'age': [35, 35]})
    with pytest.raises(ValueError):
        i(query)

def test_interpolation_called_missing_param_col():
    a = [range(1990, 1995), range(25, 30), ['Male', 'Female']]
    df = pd.DataFrame(list(itertools.product(*a)), columns=['year', 'age', 'sex'])
    df['pop'] = df.age * 11.1
    df = df.sample(frac=1)  # Shuffle table to assure interpolation works given unsorted input
    i = Interpolation(df, ['sex',], ['year','age'], 1)
    query = pd.DataFrame({'year': [1990, 1990], 'sex': ['Male', 'Female']})
    with pytest.raises(ValueError):
        i(query)

def test_2d_interpolation():
    a = np.mgrid[0:5, 0:5][0].reshape(25)
    b = np.mgrid[0:5, 0:5][1].reshape(25)
    df = pd.DataFrame({'a': a, 'b': b, 'c': b, 'd': a})
    df = df.sample(frac=1)  # Shuffle table to assure interpolation works given unsorted input

    i = Interpolation(df, (), ('a', 'b'), 1)

    query = pd.DataFrame({'a': np.arange(4, step=0.01), 'b': np.arange(4, step=0.01)})

    assert np.allclose(query.b, i(query).c)
    assert np.allclose(query.a, i(query).d)

def test_interpolation_with_categorical_parameters():
    a = ['one']*100 + ['two']*100
    b = np.append(np.arange(100), np.arange(100))
    c = np.append(np.arange(100), np.arange(100, 0, -1))
    df = pd.DataFrame({'a': a, 'b': b, 'c': c})
    df = df.sample(frac=1)  # Shuffle table to assure interpolation works given unsorted input

    i = Interpolation(df, ('a',), ('b',), 1)

    query_one = pd.DataFrame({'a': 'one', 'b': np.arange(100, step=0.01)})
    query_two = pd.DataFrame({'a': 'two', 'b': np.arange(100, step=0.01)})

    assert np.allclose(np.arange(100, step=0.01), i(query_one).c)

    assert np.allclose(np.arange(100, 0, step=-0.01), i(query_two).c)

def test_order_zero_2d():
    a = np.mgrid[0:5, 0:5][0].reshape(25)
    b = np.mgrid[0:5, 0:5][1].reshape(25)
    df = pd.DataFrame({'a': a + 0.5, 'b': b + 0.5, 'c': b*3, 'garbage': ['test']*len(a)})
    df = df.sample(frac=1)  # Shuffle table to assure interpolation works given unsorted input

    i = Interpolation(df, ('garbage',), ('a', 'b'), order=0)

    column = np.arange(4, step=0.011)
    query = pd.DataFrame({'a': column, 'b': column, 'garbage': ['test']*(len(column))})

    assert np.allclose(query.b.astype(int) * 3, i(query).c)

def test_order_zero_1d():
    s = pd.Series({0: 0, 1: 1}).reset_index()
    f = Interpolation(s, tuple(), ('index', ), order=0)

    assert f(pd.DataFrame({'index': [0]}))[0][0] == 0, 'should be precise at index values'
    assert f(pd.DataFrame({'index': [1]}))[0][0] == 1
    assert f(pd.DataFrame({'index': [2]}))[0][0] == 1, 'should be constant extrapolation outside of input range'
    assert f(pd.DataFrame({'index': [-1]}))[0][0] == 0

def test_validate_parameters__empty_data():
    with pytest.warns(UserWarning) as record:
        out, data, _ = validate_parameters(pd.DataFrame(columns=["age", "sex", "year", "value"]), ["sex"],
                                        ["age", "year"], 1)
    assert len(record) == 2
    message = record[0].message.args[0] + " " + record[1].message.args[0]
    assert "age" in message and "year" in message

    assert set(data.columns) == {"sex", "value"}

<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Hive Netius System
# Copyright (C) 2008-2012 Hive Solutions Lda.
#
# This file is part of Hive Netius System.
#
# Hive Netius System is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Hive Netius System is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Hive Netius System. If not, see <http://www.gnu.org/licenses/>.

__author__ = "João Magalhães joamag@hive.pt>"
""" The author(s) of the module """

__version__ = "1.0.0"
""" The version of the module """

__revision__ = "$LastChangedRevision$"
""" The revision number of the module """

__date__ = "$LastChangedDate$"
""" The last change date of the module """

__copyright__ = "Copyright (c) 2008-2012 Hive Solutions Lda."
""" The copyright for the module """

__license__ = "GNU General Public License (GPL), Version 3"
""" The license for the module """

import os
import urllib
import mimetypes

import netius.servers

BUFFER_SIZE = 4096
""" The size of the buffer that is going to be used when
sending the file to the client, this should not be neither
to big nor to small (as both situations would create problems) """

class FileServer(netius.servers.HTTPServer):

    def __init__(self, base_path = "", *args, **kwargs):
        netius.servers.HTTPServer.__init__(self, *args, **kwargs)
        self.base_path = base_path

    def on_connection_d(self, connection):
        netius.servers.HTTPServer.on_connection_d(self, connection)

        file = hasattr(connection, "file") and connection.file
        if file: file.close()
        setattr(connection, "file", None)

    def on_serve(self):
        netius.servers.HTTPServer.on_serve(self)
        if self.env: self.base_path = os.environ.get("BASE_PATH", self.base_path)
        self.info("Defining '%s' as the root of the file server ..." % (self.base_path or "."))

    def on_data_http(self, connection, parser):
        netius.servers.HTTPServer.on_data_http(self, connection, parser)

        # retrieves the requested path from the parser and the constructs
        # the correct file name/path to be used in the reading from the
        # current file system, so that it's possible to handle the data
        path = parser.get_path()
        path = urllib.unquote(path)
        path = path.lstrip("/")
        path_f = os.path.join(self.base_path, path)
        path_f = os.path.abspath(path_f)
        path_f = os.path.normpath(path_f)

        # verifies if the requested file exists in case it does not
        # raises an error indicating the problem so that the user is
        # notified about the failure to find the appropriate file
        if not os.path.exists(path_f): return self.on_no_file(connection)

        try:
            # verifies if the currently resolved path refers an directory or
            # instead a normal file and handles each of the cases properly by
            # redirecting the request to the proper handlers
            is_dir = os.path.isdir(path_f)
            if is_dir: return self.on_dir_file(connection, path_f)
            else: return self.on_normal_file(connection, path_f)
        except BaseException, exception:
            # handles the exception gracefully by sending the contents of
            # it to the client and identifying the problem correctly
            return self.on_exception_file(connection, exception)

    def on_dir_file(self, connection, path):
        parser = connection.parser
        path_v = parser.get_path()
        path_v = urllib.unquote(path_v)

        items = os.listdir(path)

        is_root = path_v == "" or path_v == "/"
        if not is_root: items.insert(0, "..")

        buffer = list()
        buffer.append("")
        buffer.append("<title>Directory listing for %s</title>" % path_v)
        buffer.append("")
        buffer.append("
Directory listing for %s
" % path_v)
        buffer.append("
")
        buffer.append("
")
        for item in items:
            path_f = os.path.join(path, item)
            is_dir = os.path.isdir(path_f)
            item_s = item + "/" if is_dir else item
            buffer.append("
%s" % (item_s, item_s))
        buffer.append("
")
        buffer.append("
")
        buffer.append("")
        buffer.append("")

        data = "".join(buffer)

        return connection.send_response(
            data = data,
            code = 200,
            apply = True,
            callback = self._file_check_close
        )

    def on_normal_file(self, connection, path):
        # prints a debug message about the file that is going to be read
        # from the current file system to be sent to the connection
        self.debug("Reading file '%s' from file system" % path)

        # tries to guess the mime type of the file present in the target
        # file path that is going to be returned, this may fails as it's not
        # always possible to determine the correct mime type for a file
        type, _encoding = mimetypes.guess_type(path)

        # retrieves the size of the file that has just be resolved using
        # the currently provided path value and then associates the file
        # with the current connection
        file_size = os.path.getsize(path)
        file = open(path, "rb")
        connection.file = file

        # creates the map that will hold the various header values for the
        # the current message to be sent it may contain both the length
        # of the file that is going to be returned and the type of it
        headers = dict()
        headers["content-length"] = "%d" % file_size
        if type: headers["content-type"] = type

        # sends the initial part of the file response containing the headers
        # and the description of the file (includes size) the callback to this
        # operation is the initial sending of the file contents so that the
        # sending of the proper file contents starts with success
        return connection.send_response(
            headers = headers,
            code = 200,
            apply = True,
            callback = self._file_send
        )

    def on_no_file(self, connection):
        return connection.send_response(
            data = "File not found",
            headers = dict(
                connection = "close"
            ),
            code = 404,
            apply = True,
            callback = self._file_close
        )

    def on_exception_file(self, connection, exception):
        return connection.send_response(
            data = "Problem handling request - %s" % str(exception),
            headers = dict(
                connection = "close"
            ),
            code = 500,
            apply = True,
            callback = self._file_close
        )

    def _file_send(self, connection):
        file = connection.file
        data = file.read(BUFFER_SIZE)
        if data: connection.send(
            data,
            delay = True,
            callback = self._file_send
        )
        else: self._file_finish(connection)

    def _file_finish(self, connection):
        connection.file.close()
        connection.file = None
        if connection.parser.keep_alive: return
        connection.close()

    def _file_close(self, connection):
        connection.close()

    def _file_check_close(self, connection):
        if connection.parser.keep_alive: return
        connection.close()

if __name__ == "__main__":
    import logging

    server = FileServer(level = logging.INFO)
    server.serve(env = True)
<commit_msg>small code fix<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Hive Netius System
# Copyright (C) 2008-2012 Hive Solutions Lda.
#
# This file is part of Hive Netius System.
#
# Hive Netius System is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Hive Netius System is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Hive Netius System. If not, see <http://www.gnu.org/licenses/>.

__author__ = "João Magalhães joamag@hive.pt>"
""" The author(s) of the module """

__version__ = "1.0.0"
""" The version of the module """

__revision__ = "$LastChangedRevision$"
""" The revision number of the module """

__date__ = "$LastChangedDate$"
""" The last change date of the module """

__copyright__ = "Copyright (c) 2008-2012 Hive Solutions Lda."
""" The copyright for the module """

__license__ = "GNU General Public License (GPL), Version 3"
""" The license for the module """

import os
import urllib
import mimetypes

import netius.servers

BUFFER_SIZE = 4096
""" The size of the buffer that is going to be used when
sending the file to the client, this should not be neither
to big nor to small (as both situations would create problems) """

class FileServer(netius.servers.HTTPServer):

    def __init__(self, base_path = "", *args, **kwargs):
        netius.servers.HTTPServer.__init__(self, *args, **kwargs)
        self.base_path = base_path

    def on_connection_d(self, connection):
        netius.servers.HTTPServer.on_connection_d(self, connection)

        file = hasattr(connection, "file") and connection.file
        if file: file.close()
        setattr(connection, "file", None)

    def on_serve(self):
        netius.servers.HTTPServer.on_serve(self)
        if self.env: self.base_path = os.environ.get("BASE_PATH", self.base_path)
        self.info("Defining '%s' as the root of the file server ..." % (self.base_path or "."))

    def on_data_http(self, connection, parser):
        netius.servers.HTTPServer.on_data_http(self, connection, parser)

        # retrieves the requested path from the parser and the constructs
        # the correct file name/path to be used in the reading from the
        # current file system, so that it's possible to handle the data
        path = parser.get_path()
        path = urllib.unquote(path)
        path = path.lstrip("/")
        path_f = os.path.join(self.base_path, path)
        path_f = os.path.abspath(path_f)
        path_f = os.path.normpath(path_f)

        # verifies if the requested file exists in case it does not
        # raises an error indicating the problem so that the user is
        # notified about the failure to find the appropriate file
        if not os.path.exists(path_f): return self.on_no_file(connection)

        try:
            # verifies if the currently resolved path refers an directory or
            # instead a normal file and handles each of the cases properly by
            # redirecting the request to the proper handlers
            is_dir = os.path.isdir(path_f)
            if is_dir: return self.on_dir_file(connection, path_f)
            else: return self.on_normal_file(connection, path_f)
        except BaseException, exception:
            # handles the exception gracefully by sending the contents of
            # it to the client and identifying the problem correctly
            return self.on_exception_file(connection, exception)

    def on_dir_file(self, connection, path):
        parser = connection.parser
        path_v = parser.get_path()
        path_v = urllib.unquote(path_v)

        items = os.listdir(path)

        is_root = path_v == "" or path_v == "/"
        if not is_root: items.insert(0, "..")

        buffer = list()
        buffer.append("")
        buffer.append("<title>Directory listing for %s</title>" % path_v)
        buffer.append("")
        buffer.append("
Directory listing for %s
" % path_v)
        buffer.append("
")
        buffer.append("
")
        for item in items:
            path_f = os.path.join(path, item)
            is_dir = os.path.isdir(path_f)
            item_s = item + "/" if is_dir else item
            buffer.append("
%s" % (item_s, item_s))
        buffer.append("
")
        buffer.append("
")
        buffer.append("")
        buffer.append("")

        data = "".join(buffer)

        return connection.send_response(
            data = data,
            code = 200,
            apply = True,
            callback = self._file_check_close
        )

    def on_normal_file(self, connection, path):
        # prints a debug message about the file that is going to be read
        # from the current file system to be sent to the connection
        self.debug("Reading file '%s' from file system" % path)

        # tries to guess the mime type of the file present in the target
        # file path that is going to be returned, this may fails as it's not
        # always possible to determine the correct mime type for a file
        type, _encoding = mimetypes.guess_type(path, strict = True)

        # retrieves the size of the file that has just be resolved using
        # the currently provided path value and then associates the file
        # with the current connection
        file_size = os.path.getsize(path)
        file = open(path, "rb")
        connection.file = file

        # creates the map that will hold the various header values for the
        # the current message to be sent it may contain both the length
        # of the file that is going to be returned and the type of it
        headers = dict()
        headers["content-length"] = "%d" % file_size
        if type: headers["content-type"] = type

        # sends the initial part of the file response containing the headers
        # and the description of the file (includes size) the callback to this
        # operation is the initial sending of the file contents so that the
        # sending of the proper file contents starts with success
        return connection.send_response(
            headers = headers,
            code = 200,
            apply = True,
            callback = self._file_send
        )

    def on_no_file(self, connection):
        return connection.send_response(
            data = "File not found",
            headers = dict(
                connection = "close"
            ),
            code = 404,
            apply = True,
            callback = self._file_close
        )

    def on_exception_file(self, connection, exception):
        return connection.send_response(
            data = "Problem handling request - %s" % str(exception),
            headers = dict(
                connection = "close"
            ),
            code = 500,
            apply = True,
            callback = self._file_close
        )

    def _file_send(self, connection):
        file = connection.file
        data = file.read(BUFFER_SIZE)
        if data: connection.send(
            data,
            delay = True,
            callback = self._file_send
        )
        else: self._file_finish(connection)

    def _file_finish(self, connection):
        connection.file.close()
        connection.file = None
        if connection.parser.keep_alive: return
        connection.close()

    def _file_close(self, connection):
        connection.close()

    def _file_check_close(self, connection):
        if connection.parser.keep_alive: return
        connection.close()

if __name__ == "__main__":
    import logging

    server = FileServer(level = logging.INFO)
    server.serve(env = True)
<|endoftext|>