<commit_before>import os
import shutil
import tempfile
multiprocessing = None
try:
    import multiprocessing
    from ..pool import MemmapingPool
    from ..pool import has_shareable_memory
    from ..pool import ArrayMemmapReducer
    from ..pool import reduce_memmap
except ImportError:
    pass

from nose import SkipTest
from nose.tools import with_setup
from nose.tools import assert_equal
from nose.tools import assert_raises
from nose.tools import assert_false
from nose.tools import assert_true
from .common import with_numpy, np
from .common import setup_autokill
from .common import teardown_autokill

TEMP_FOLDER = None

def setup_module():
    setup_autokill(__name__, timeout=5)

def teardown_module():
    teardown_autokill(__name__)

def check_multiprocessing():
    if multiprocessing is None:
        raise SkipTest('Need multiprocessing to run')

with_multiprocessing = with_setup(check_multiprocessing)

def setup_temp_folder():
    global TEMP_FOLDER
    TEMP_FOLDER = tempfile.mkdtemp(prefix='joblib_test_pool_')

def teardown_temp_folder():
    global TEMP_FOLDER
    if TEMP_FOLDER is not None:
        shutil.rmtree(TEMP_FOLDER)
        TEMP_FOLDER = None

with_temp_folder = with_setup(setup_temp_folder, teardown_temp_folder)

def double(input):
    """Dummy helper function to be executed in subprocesses"""
    assert_array_equal = np.testing.assert_array_equal

    data, position, expected = input
    if expected is not None:
        assert_equal(data[position], expected)
    data[position] *= 2
    if expected is not None:
        assert_array_equal(data[position], 2 * expected)

@with_numpy
@with_multiprocessing
@with_temp_folder
def test_memmap_based_array_reducing():
    """Check that it is possible to reduce a memmap backed array"""
    assert_array_equal = np.testing.assert_array_equal
    filename = os.path.join(TEMP_FOLDER, 'test.mmap')

    # Create a file larger than what will be used by a
    buffer = np.memmap(filename, dtype=np.float64, shape=500, mode='w+')

    # Fill the original buffer with negative markers to detect over of
    # underflow in case of test failures
    buffer[:] = - 1.0 * np.arange(buffer.shape[0], dtype=buffer.dtype)
    buffer.flush()

    # Memmap a 2D fortran array on a offseted subsection of the previous
    # buffer
    a = np.memmap(filename, dtype=np.float64, shape=(3, 5, 4),
                  mode='r+', order='F', offset=4)
    a[:] = np.arange(60).reshape(a.shape)

    # Build various views that share the buffer with the original memmap

    # b is an memmap sliced view on an memmap instance
    b = a[1:-1, 2:-1, 2:4]

    # c and d are array views
    c = np.asarray(b)
    d = c.T

    # Array reducer with auto dumping disabled
    reducer = ArrayMemmapReducer(None, TEMP_FOLDER, 'c')
    def reconstruct_array(x):
        cons, args = reducer(x)
        return cons(*args)

    def reconstruct_memmap(x):
        cons, args = reduce_memmap(x)
        return cons(*args)

    # Reconstruct original memmap
    a_reconstructed = reconstruct_memmap(a)
    assert_true(has_shareable_memory(a_reconstructed))
    assert_true(isinstance(a_reconstructed, np.memmap))
    assert_array_equal(a_reconstructed, a)

    # Reconstruct strided memmap view
    b_reconstructed = reconstruct_memmap(b)
    assert_true(has_shareable_memory(b_reconstructed))
    assert_array_equal(b_reconstructed, b)

    # Reconstruct arrays views on memmap base
    c_reconstructed = reconstruct_array(c)
    assert_false(isinstance(c_reconstructed, np.memmap))
    assert_true(has_shareable_memory(c_reconstructed))
    assert_array_equal(c_reconstructed, c)

    d_reconstructed = reconstruct_array(d)
    assert_false(isinstance(d_reconstructed, np.memmap))
    assert_true(has_shareable_memory(d_reconstructed))
    assert_array_equal(d_reconstructed, d)

    # Test graceful degradation on fake memmap instances with in-memory
    # buffers
    a3 = a * 3
    assert_false(has_shareable_memory(a3))
    a3_reconstructed = reconstruct_memmap(a3)
    assert_false(has_shareable_memory(a3_reconstructed))
    assert_false(isinstance(a3_reconstructed, np.memmap))
    assert_array_equal(a3_reconstructed, a * 3)

    # Test graceful degradation on arrays derived from fake memmap instances
    b3 = np.asarray(a3)
    assert_false(has_shareable_memory(b3))

    b3_reconstructed = reconstruct_array(b3)
    assert_true(isinstance(b3_reconstructed, np.ndarray))
    assert_false(has_shareable_memory(b3_reconstructed))
    assert_array_equal(b3_reconstructed, b3)

@with_numpy
@with_multiprocessing
@with_temp_folder
def test_pool_with_memmap():
    """Check that subprocess can access and update shared memory memmap"""
    assert_array_equal = np.testing.assert_array_equal

    # Fork the subprocess before allocating the objects to be passed
    pool_temp_folder = os.path.join(TEMP_FOLDER, 'pool')
    os.makedirs(pool_temp_folder)
    p = MemmapingPool(10, max_nbytes=2, temp_folder=pool_temp_folder)
    try:
        filename = os.path.join(TEMP_FOLDER, 'test.mmap')
        a = np.memmap(filename, dtype=np.float32, shape=(3, 5), mode='w+')
        a.fill(1.0)

        p.map(double, [(a, (i, j), 1.0)
                       for i in range(a.shape[0])
                       for j in range(a.shape[1])])

        assert_array_equal(a, 2 * np.ones(a.shape))

        # Open a copy-on-write view on the previous data
        b = np.memmap(filename, dtype=np.float32, shape=(5, 3), mode='c')

        p.map(double, [(b, (i, j), 2.0)
                       for i in range(b.shape[0])
                       for j in range(b.shape[1])])

        # Passing memmap instances to the pool should not trigger the creation
        # of new files on the FS
        assert_equal(os.listdir(pool_temp_folder), [])

        # the original data is untouched
        assert_array_equal(a, 2 * np.ones(a.shape))
        assert_array_equal(b, 2 * np.ones(b.shape))

        # readonly maps can be read but not updated
        c = np.memmap(filename, dtype=np.float32, shape=(10,), mode='r',
                      offset=5 * 4)

        assert_raises(AssertionError, p.map, double,
                      [(c, i, 3.0) for i in range(c.shape[0])])

        # depending on the version of numpy one can either get a RuntimeError
        # or a ValueError
        assert_raises((RuntimeError, ValueError), p.map, double,
                      [(c, i, 2.0) for i in range(c.shape[0])])
    finally:
        # Clean all filehandlers held by the pool
        p.terminate()
        del p

@with_numpy
@with_multiprocessing
@with_temp_folder
def test_pool_with_memmap_array_view():
    """Check that subprocess can access and update shared memory array"""
    assert_array_equal = np.testing.assert_array_equal

    # Fork the subprocess before allocating the objects to be passed
    pool_temp_folder = os.path.join(TEMP_FOLDER, 'pool')
    os.makedirs(pool_temp_folder)
    p = MemmapingPool(10, max_nbytes=2, temp_folder=pool_temp_folder)
    try:

        filename = os.path.join(TEMP_FOLDER, 'test.mmap')
        a = np.memmap(filename, dtype=np.float32, shape=(3, 5), mode='w+')
        a.fill(1.0)

        # Create an ndarray view on the memmap instance
        a_view = np.asarray(a)
        assert_false(isinstance(a_view, np.memmap))
        assert_true(has_shareable_memory(a_view))

        p.map(double, [(a_view, (i, j), 1.0)
                       for i in range(a.shape[0])
                       for j in range(a.shape[1])])

        # Both a and the a_view have been updated
        assert_array_equal(a, 2 * np.ones(a.shape))
        assert_array_equal(a_view, 2 * np.ones(a.shape))

        # Passing memmap array view to the pool should not trigger the
        # creation of new files on the FS
        assert_equal(os.listdir(pool_temp_folder), [])

    finally:
        p.terminate()
        del p

@with_numpy
@with_multiprocessing
@with_temp_folder
def test_memmaping_pool_for_large_arrays():
    """Check that large arrays are not copied in memory"""
    assert_array_equal = np.testing.assert_array_equal

    # Check that the tempfolder is empty
    assert_equal(os.listdir(TEMP_FOLDER), [])

    # Build an array reducers that automaticaly dump large array content
    # to filesystem backed memmap instances to avoid memory explosion
    p = MemmapingPool(3, max_nbytes=40, temp_folder=TEMP_FOLDER)
    try:
        # The tempory folder for the pool is not provisioned in advance
        assert_equal(os.listdir(TEMP_FOLDER), [])
        assert_false(os.path.exists(p._temp_folder))

        small = np.ones(5, dtype=np.float32)
        assert_equal(small.nbytes, 20)
        p.map(double, [(small, i, 1.0) for i in range(small.shape[0])])

        # Memory has been copied, the pool filesystem folder is unused
        assert_equal(os.listdir(TEMP_FOLDER), [])

        # Try with a file larger than the memmap threshold of 40 bytes
        large = np.ones(100, dtype=np.float64)
        assert_equal(large.nbytes, 800)
        p.map(double, [(large, i, 1.0) for i in range(large.shape[0])])

        # By defaul, the mmap_mode is copy-on-write to make the pool
        # process able to modify their view individually as if they would have
        # received their own copy of the original array. The original array
        # (which is not a shared memmap instance is untouched)
        assert_false(has_shareable_memory(large))
        assert_array_equal(large, np.ones(100))

        # The data has been dump in a temp folder for subprocess to share it
        # without per-child memory copies
        assert_true(os.path.isdir(p._temp_folder))
        dumped_filenames = os.listdir(p._temp_folder)
        assert_equal(len(dumped_filenames), 2)

    finally:
        # check FS garbage upon pool termination
        p.terminate()
        assert_false(os.path.exists(p._temp_folder))
        del p

@with_numpy
@with_multiprocessing
@with_temp_folder
def test_memmaping_pool_for_large_arrays_disabled():
    """Check that large arrays memmaping can be disabled"""
    # Set max_nbytes to None to disable the auto memmaping feature
    p = MemmapingPool(3, max_nbytes=None, temp_folder=TEMP_FOLDER)
    try:

        # Check that the tempfolder is empty
        assert_equal(os.listdir(TEMP_FOLDER), [])

        # Try with a file largish than the memmap threshold of 40 bytes
        large = np.ones(100, dtype=np.float64)
        assert_equal(large.nbytes, 800)
        p.map(double, [(large, i, 1.0) for i in range(large.shape[0])])

        # Check that the tempfolder is still empty
        assert_equal(os.listdir(TEMP_FOLDER), [])

    finally:
        # Cleanup open file descriptors
        p.terminate()
        del p

@with_numpy
@with_multiprocessing
@with_temp_folder
def test_memmaping_pool_for_large_arrays_in_return():
    """Check that large arrays are not copied in memory in return"""
    assert_array_equal = np.testing.assert_array_equal

    # Build an array reducers that automaticaly dump large array content
    # but check that the returned datastructure are regular arrays to avoid
    # passing a memmap array pointing to a pool controlled temp folder that
    # might be confusing to the user

    # The MemmapingPool user can always return numpy.memmap object explicitly
    # to avoid memory copy
    p = MemmapingPool(3, max_nbytes=10, temp_folder=TEMP_FOLDER)
    try:
        res = p.apply_async(np.ones, args=(1000,))
        large = res.get()
        assert_false(has_shareable_memory(large))
        assert_array_equal(large, np.ones(1000))
    finally:
        p.terminate()
        del p

def _worker_multiply(a, n_times):
    """Multiplication function to be executed by subprocess"""
    assert_true(has_shareable_memory(a))
    return a * n_times

@with_numpy
@with_multiprocessing
@with_temp_folder
def test_workaround_against_bad_memmap_with_copied_buffers():
    """Check that memmaps with a bad buffer are returned as regular arrays

    Unary operations and ufuncs on memmap instances return a new memmap
    instance with an in-memory buffer (probably a numpy bug).
    """
    assert_array_equal = np.testing.assert_array_equal

    p = MemmapingPool(3, max_nbytes=10, temp_folder=TEMP_FOLDER)
    try:
        # Send a complex, large-ish view on a array that will be converted to
        # a memmap in the worker process
        a = np.asarray(np.arange(6000).reshape((1000, 2, 3)),
                       order='F')[:, :1, :]

        # Call a non-inplace multiply operation on the worker and memmap and
        # send it back to the parent.
        b = p.apply_async(_worker_multiply, args=(a, 3)).get()
        assert_false(has_shareable_memory(b))
        assert_array_equal(b, 3 * a)
    finally:
        p.terminate()
        del p
<commit_msg>Protect test_pool against OS than don't support semaphores<commit_after>import os
import shutil
import tempfile
import warnings

multiprocessing = int(os.environ.get('JOBLIB_MULTIPROCESSING', 1)) or None
if multiprocessing:
    try:
        import multiprocessing
    except ImportError:
        multiprocessing = None

# 2nd stage: validate that locking is available on the system and
#            issue a warning if not
if multiprocessing:
    try:
        _sem = multiprocessing.Semaphore()
        del _sem  # cleanup
        from ..pool import MemmapingPool
        from ..pool import has_shareable_memory
        from ..pool import ArrayMemmapReducer
        from ..pool import reduce_memmap
    except (ImportError, OSError) as e:
        multiprocessing = None
        warnings.warn('%s.  joblib will operate in serial mode' % (e,))

from nose import SkipTest
from nose.tools import with_setup
from nose.tools import assert_equal
from nose.tools import assert_raises
from nose.tools import assert_false
from nose.tools import assert_true
from .common import with_numpy, np
from .common import setup_autokill
from .common import teardown_autokill

TEMP_FOLDER = None

def setup_module():
    setup_autokill(__name__, timeout=5)

def teardown_module():
    teardown_autokill(__name__)

def check_multiprocessing():
    if multiprocessing is None:
        raise SkipTest('Need multiprocessing to run')

with_multiprocessing = with_setup(check_multiprocessing)

def setup_temp_folder():
    global TEMP_FOLDER
    TEMP_FOLDER = tempfile.mkdtemp(prefix='joblib_test_pool_')

def teardown_temp_folder():
    global TEMP_FOLDER
    if TEMP_FOLDER is not None:
        shutil.rmtree(TEMP_FOLDER)
        TEMP_FOLDER = None

with_temp_folder = with_setup(setup_temp_folder, teardown_temp_folder)

def double(input):
    """Dummy helper function to be executed in subprocesses"""
    assert_array_equal = np.testing.assert_array_equal

    data, position, expected = input
    if expected is not None:
        assert_equal(data[position], expected)
    data[position] *= 2
    if expected is not None:
        assert_array_equal(data[position], 2 * expected)

@with_numpy
@with_multiprocessing
@with_temp_folder
def test_memmap_based_array_reducing():
    """Check that it is possible to reduce a memmap backed array"""
    assert_array_equal = np.testing.assert_array_equal
    filename = os.path.join(TEMP_FOLDER, 'test.mmap')

    # Create a file larger than what will be used by a
    buffer = np.memmap(filename, dtype=np.float64, shape=500, mode='w+')

    # Fill the original buffer with negative markers to detect over of
    # underflow in case of test failures
    buffer[:] = - 1.0 * np.arange(buffer.shape[0], dtype=buffer.dtype)
    buffer.flush()

    # Memmap a 2D fortran array on a offseted subsection of the previous
    # buffer
    a = np.memmap(filename, dtype=np.float64, shape=(3, 5, 4),
                  mode='r+', order='F', offset=4)
    a[:] = np.arange(60).reshape(a.shape)

    # Build various views that share the buffer with the original memmap

    # b is an memmap sliced view on an memmap instance
    b = a[1:-1, 2:-1, 2:4]

    # c and d are array views
    c = np.asarray(b)
    d = c.T

    # Array reducer with auto dumping disabled
    reducer = ArrayMemmapReducer(None, TEMP_FOLDER, 'c')

    def reconstruct_array(x):
        cons, args = reducer(x)
        return cons(*args)

    def reconstruct_memmap(x):
        cons, args = reduce_memmap(x)
        return cons(*args)

    # Reconstruct original memmap
    a_reconstructed = reconstruct_memmap(a)
    assert_true(has_shareable_memory(a_reconstructed))
    assert_true(isinstance(a_reconstructed, np.memmap))
    assert_array_equal(a_reconstructed, a)

    # Reconstruct strided memmap view
    b_reconstructed = reconstruct_memmap(b)
    assert_true(has_shareable_memory(b_reconstructed))
    assert_array_equal(b_reconstructed, b)

    # Reconstruct arrays views on memmap base
    c_reconstructed = reconstruct_array(c)
    assert_false(isinstance(c_reconstructed, np.memmap))
    assert_true(has_shareable_memory(c_reconstructed))
    assert_array_equal(c_reconstructed, c)

    d_reconstructed = reconstruct_array(d)
    assert_false(isinstance(d_reconstructed, np.memmap))
    assert_true(has_shareable_memory(d_reconstructed))
    assert_array_equal(d_reconstructed, d)

    # Test graceful degradation on fake memmap instances with in-memory
    # buffers
    a3 = a * 3
    assert_false(has_shareable_memory(a3))
    a3_reconstructed = reconstruct_memmap(a3)
    assert_false(has_shareable_memory(a3_reconstructed))
    assert_false(isinstance(a3_reconstructed, np.memmap))
    assert_array_equal(a3_reconstructed, a * 3)

    # Test graceful degradation on arrays derived from fake memmap instances
    b3 = np.asarray(a3)
    assert_false(has_shareable_memory(b3))

    b3_reconstructed = reconstruct_array(b3)
    assert_true(isinstance(b3_reconstructed, np.ndarray))
    assert_false(has_shareable_memory(b3_reconstructed))
    assert_array_equal(b3_reconstructed, b3)

@with_numpy
@with_multiprocessing
@with_temp_folder
def test_pool_with_memmap():
    """Check that subprocess can access and update shared memory memmap"""
    assert_array_equal = np.testing.assert_array_equal

    # Fork the subprocess before allocating the objects to be passed
    pool_temp_folder = os.path.join(TEMP_FOLDER, 'pool')
    os.makedirs(pool_temp_folder)
    p = MemmapingPool(10, max_nbytes=2, temp_folder=pool_temp_folder)
    try:
        filename = os.path.join(TEMP_FOLDER, 'test.mmap')
        a = np.memmap(filename, dtype=np.float32, shape=(3, 5), mode='w+')
        a.fill(1.0)

        p.map(double, [(a, (i, j), 1.0)
                       for i in range(a.shape[0])
                       for j in range(a.shape[1])])

        assert_array_equal(a, 2 * np.ones(a.shape))

        # Open a copy-on-write view on the previous data
        b = np.memmap(filename, dtype=np.float32, shape=(5, 3), mode='c')

        p.map(double, [(b, (i, j), 2.0)
                       for i in range(b.shape[0])
                       for j in range(b.shape[1])])

        # Passing memmap instances to the pool should not trigger the creation
        # of new files on the FS
        assert_equal(os.listdir(pool_temp_folder), [])

        # the original data is untouched
        assert_array_equal(a, 2 * np.ones(a.shape))
        assert_array_equal(b, 2 * np.ones(b.shape))

        # readonly maps can be read but not updated
        c = np.memmap(filename, dtype=np.float32, shape=(10,), mode='r',
                      offset=5 * 4)

        assert_raises(AssertionError, p.map, double,
                      [(c, i, 3.0) for i in range(c.shape[0])])

        # depending on the version of numpy one can either get a RuntimeError
        # or a ValueError
        assert_raises((RuntimeError, ValueError), p.map, double,
                      [(c, i, 2.0) for i in range(c.shape[0])])
    finally:
        # Clean all filehandlers held by the pool
        p.terminate()
        del p

@with_numpy
@with_multiprocessing
@with_temp_folder
def test_pool_with_memmap_array_view():
    """Check that subprocess can access and update shared memory array"""
    assert_array_equal = np.testing.assert_array_equal

    # Fork the subprocess before allocating the objects to be passed
    pool_temp_folder = os.path.join(TEMP_FOLDER, 'pool')
    os.makedirs(pool_temp_folder)
    p = MemmapingPool(10, max_nbytes=2, temp_folder=pool_temp_folder)
    try:

        filename = os.path.join(TEMP_FOLDER, 'test.mmap')
        a = np.memmap(filename, dtype=np.float32, shape=(3, 5), mode='w+')
        a.fill(1.0)

        # Create an ndarray view on the memmap instance
        a_view = np.asarray(a)
        assert_false(isinstance(a_view, np.memmap))
        assert_true(has_shareable_memory(a_view))

        p.map(double, [(a_view, (i, j), 1.0)
                       for i in range(a.shape[0])
                       for j in range(a.shape[1])])

        # Both a and the a_view have been updated
        assert_array_equal(a, 2 * np.ones(a.shape))
        assert_array_equal(a_view, 2 * np.ones(a.shape))

        # Passing memmap array view to the pool should not trigger the
        # creation of new files on the FS
        assert_equal(os.listdir(pool_temp_folder), [])

    finally:
        p.terminate()
        del p

@with_numpy
@with_multiprocessing
@with_temp_folder
def test_memmaping_pool_for_large_arrays():
    """Check that large arrays are not copied in memory"""
    assert_array_equal = np.testing.assert_array_equal

    # Check that the tempfolder is empty
    assert_equal(os.listdir(TEMP_FOLDER), [])

    # Build an array reducers that automaticaly dump large array content
    # to filesystem backed memmap instances to avoid memory explosion
    p = MemmapingPool(3, max_nbytes=40, temp_folder=TEMP_FOLDER)
    try:
        # The tempory folder for the pool is not provisioned in advance
        assert_equal(os.listdir(TEMP_FOLDER), [])
        assert_false(os.path.exists(p._temp_folder))

        small = np.ones(5, dtype=np.float32)
        assert_equal(small.nbytes, 20)
        p.map(double, [(small, i, 1.0) for i in range(small.shape[0])])

        # Memory has been copied, the pool filesystem folder is unused
        assert_equal(os.listdir(TEMP_FOLDER), [])

        # Try with a file larger than the memmap threshold of 40 bytes
        large = np.ones(100, dtype=np.float64)
        assert_equal(large.nbytes, 800)
        p.map(double, [(large, i, 1.0) for i in range(large.shape[0])])

        # By defaul, the mmap_mode is copy-on-write to make the pool
        # process able to modify their view individually as if they would have
        # received their own copy of the original array. The original array
        # (which is not a shared memmap instance is untouched)
        assert_false(has_shareable_memory(large))
        assert_array_equal(large, np.ones(100))

        # The data has been dump in a temp folder for subprocess to share it
        # without per-child memory copies
        assert_true(os.path.isdir(p._temp_folder))
        dumped_filenames = os.listdir(p._temp_folder)
        assert_equal(len(dumped_filenames), 2)

    finally:
        # check FS garbage upon pool termination
        p.terminate()
        assert_false(os.path.exists(p._temp_folder))
        del p

@with_numpy
@with_multiprocessing
@with_temp_folder
def test_memmaping_pool_for_large_arrays_disabled():
    """Check that large arrays memmaping can be disabled"""
    # Set max_nbytes to None to disable the auto memmaping feature
    p = MemmapingPool(3, max_nbytes=None, temp_folder=TEMP_FOLDER)
    try:

        # Check that the tempfolder is empty
        assert_equal(os.listdir(TEMP_FOLDER), [])

        # Try with a file largish than the memmap threshold of 40 bytes
        large = np.ones(100, dtype=np.float64)
        assert_equal(large.nbytes, 800)
        p.map(double, [(large, i, 1.0) for i in range(large.shape[0])])

        # Check that the tempfolder is still empty
        assert_equal(os.listdir(TEMP_FOLDER), [])

    finally:
        # Cleanup open file descriptors
        p.terminate()
        del p

@with_numpy
@with_multiprocessing
@with_temp_folder
def test_memmaping_pool_for_large_arrays_in_return():
    """Check that large arrays are not copied in memory in return"""
    assert_array_equal = np.testing.assert_array_equal

    # Build an array reducers that automaticaly dump large array content
    # but check that the returned datastructure are regular arrays to avoid
    # passing a memmap array pointing to a pool controlled temp folder that
    # might be confusing to the user

    # The MemmapingPool user can always return numpy.memmap object explicitly
    # to avoid memory copy
    p = MemmapingPool(3, max_nbytes=10, temp_folder=TEMP_FOLDER)
    try:
        res = p.apply_async(np.ones, args=(1000,))
        large = res.get()
        assert_false(has_shareable_memory(large))
        assert_array_equal(large, np.ones(1000))
    finally:
        p.terminate()
        del p

def _worker_multiply(a, n_times):
    """Multiplication function to be executed by subprocess"""
    assert_true(has_shareable_memory(a))
    return a * n_times

@with_numpy
@with_multiprocessing
@with_temp_folder
def test_workaround_against_bad_memmap_with_copied_buffers():
    """Check that memmaps with a bad buffer are returned as regular arrays

    Unary operations and ufuncs on memmap instances return a new memmap
    instance with an in-memory buffer (probably a numpy bug).
    """
    assert_array_equal = np.testing.assert_array_equal

    p = MemmapingPool(3, max_nbytes=10, temp_folder=TEMP_FOLDER)
    try:
        # Send a complex, large-ish view on a array that will be converted to
        # a memmap in the worker process
        a = np.asarray(np.arange(6000).reshape((1000, 2, 3)),
                       order='F')[:, :1, :]

        # Call a non-inplace multiply operation on the worker and memmap and
        # send it back to the parent.
        b = p.apply_async(_worker_multiply, args=(a, 3)).get()
        assert_false(has_shareable_memory(b))
        assert_array_equal(b, 3 * a)
    finally:
        p.terminate()
        del p
<|endoftext|>


<commit_before>import collections
import operator
import functools
import sys
from collections import OrderedDict

__version__ = '0.1.0'

class immutabledict(collections.Mapping):
    """
    An immutable wrapper around dictionaries that implements the complete :py:class:`collections.Mapping`
    interface. It can be used as a drop-in replacement for dictionaries where immutability is desired.
    """

    dict_cls = dict

    def __init__(self, *args, **kwargs):
        self._dict = self.dict_cls(*args, **kwargs)
        self._hash = None

    def __getitem__(self, key):
        return self._dict[key]

    def __contains__(self, key):
        return key in self._dict

    def copy(self, **add_or_replace):
        return self.__class__(self, **add_or_replace)

    def __iter__(self):
        return iter(self._dict)

    def __len__(self):
        return len(self._dict)

    def __repr__(self):
        return '<%s %r>' % (self.__class__.__name__, self._dict)

    def __hash__(self):
        if self._hash is None:
            h = 0
            for key, value in self.items():
                h ^= hash((key, value))
            self._hash = h
        return self._hash

class ImmutableOrderedDict(immutabledict):
    """
    A frozendict subclass that maintains key order
    """

    dict_cls = OrderedDict
<commit_msg>Added fromkeys<commit_after>import collections
import operator
import functools
import sys
from collections import OrderedDict

__version__ = '0.1.0'

class immutabledict(collections.Mapping):
    """
    An immutable wrapper around dictionaries that implements the complete :py:class:`collections.Mapping`
    interface. It can be used as a drop-in replacement for dictionaries where immutability is desired.
    """

    dict_cls = dict

    @classmethod
    def fromkeys(cls, seq, value=None, *args, **kwargs):
        return cls(dict.fromkeys(seq, value, *args, **kwargs))

    def __init__(self, *args, **kwargs):
        self._dict = self.dict_cls(*args, **kwargs)
        self._hash = None

    def __getitem__(self, key):
        return self._dict[key]

    def __contains__(self, key):
        return key in self._dict

    def copy(self, **add_or_replace):
        return self.__class__(self, **add_or_replace)

    def __iter__(self):
        return iter(self._dict)

    def __len__(self):
        return len(self._dict)

    def __repr__(self):
        return '<%s %r>' % (self.__class__.__name__, self._dict)

    def __hash__(self):
        if self._hash is None:
            h = 0
            for key, value in self.items():
                h ^= hash((key, value))
            self._hash = h
        return self._hash

class ImmutableOrderedDict(immutabledict):
    """
    A frozendict subclass that maintains key order
    """

    dict_cls = OrderedDict
<|endoftext|>


<commit_before>import os

import openbabel
import cclib.parser
import logging
from subprocess import Popen, PIPE

from qmdata import CCLibData
from molecule import QMMolecule, TDPropertiesCalculator

class Mopac:

    directory = 'QMfiles'
    inputFileExtension = '.mop'
    outputFileExtension = '.out'
    executablePath = os.path.join(os.getenv('MOPAC_DIR', default="/opt/mopac") , 'MOPAC2009.exe')

    usePolar = False#use polar keyword in MOPAC

    "Keywords for the multiplicity"
    multiplicityKeywords = {}
    multiplicityKeywords[1] = ''
    multiplicityKeywords[2] = 'uhf doublet'
    multiplicityKeywords[3] = 'uhf triplet'
    multiplicityKeywords[4] = 'uhf quartet'
    multiplicityKeywords[5] = 'uhf quintet'
    multiplicityKeywords[6] = 'uhf sextet'
    multiplicityKeywords[7] = 'uhf septet'
    multiplicityKeywords[8] = 'uhf octet'
    multiplicityKeywords[9] = 'uhf nonet'

    #: List of phrases that indicate failure
    #: NONE of these must be present in a succesful job.
    failureKeys = [
                   'IMAGINARY FREQUENCIES',
                   'EXCESS NUMBER OF OPTIMIZATION CYCLES',
                   'NOT ENOUGH TIME FOR ANOTHER CYCLE',
                   ]
    #: List of phrases to indicate success.
    #: ALL of these must be present in a successful job.
    successKeys = [
                   'DESCRIPTION OF VIBRATIONS',
                  ]

    @property
    def outputFilePath(self):
        """Get the Mopac output file name."""
        return os.path.join(self.directory, self.geometry.uniqueID + self.outputFileExtension)

    @property
    def inputFilePath(self):
        """Get the Mopac input file name."""
        return os.path.join(self.directory, self.geometry.uniqueID + self.inputFileExtension)

    def writeInputFile(self, attempt, top_keys, bottom_keys, polar_keys):
        """
        Using the :class:`Geometry` object, write the input file
        for the `attmept`th attempt.
        """

        obConversion = openbabel.OBConversion()
        obConversion.SetInAndOutFormats("mol", "mop")
        mol = openbabel.OBMol()

        if attempt <= self.scriptAttempts: #use UFF-refined coordinates
            obConversion.ReadFile(mol, self.geometry.getRefinedMolFilePath() )
        else:
            obConversion.ReadFile(mol, self.geometry.getCrudeMolFilePath() )

        mol.SetTitle(self.geometry.uniqueIDlong)
        obConversion.SetOptions('k', openbabel.OBConversion.OUTOPTIONS)

        input_string = obConversion.WriteString(mol)

        with open(self.inputFilePath, 'w') as mopacFile:
            mopacFile.write(top_keys)
            mopacFile.write(input_string)
            mopacFile.write('\n')
            mopacFile.write(bottom_keys)
            if self.usePolar:
                mopacFile.write('\n\n\n')
                mopacFile.write(polar_keys)

    def run(self):
        # submits the input file to mopac
        process = Popen([self.executablePath, self.inputFilePath])
        process.communicate()# necessary to wait for executable termination!

        return self.verifyOutputFile()

    def verifyOutputFile(self):
        """
        Check's that an output file exists and was successful.

        Returns a boolean flag that states whether a successful MOPAC simulation already exists for the molecule with the
        given (augmented) InChI Key.

        The definition of finding a successful simulation is based on these criteria:
        1) finding an output file with the file name equal to the InChI Key
        2) NOT finding any of the keywords that are denote a calculation failure
        3) finding all the keywords that denote a calculation success.
        4) finding a match between the InChI of the given molecule and the InchI found in the calculation files

        If any of the above criteria is not matched, False will be returned and the procedures to start a new calculation
        will be initiated.
        """

        if not os.path.exists(self.outputFilePath):
            logging.info("Output file {0} does not exist.".format(self.outputFilePath))
            return False

        InChIMatch=False #flag (1 or 0) indicating whether the InChI in the file matches InChIaug this can only be 1 if InChIFound is also 1
        InChIFound=False #flag (1 or 0) indicating whether an InChI was found in the log file

        # Initialize dictionary with "False"s
        successKeysFound = dict([(key, False) for key in self.successKeys])

        with open(self.outputFilePath) as outputFile:
            for line in outputFile:
                line = line.strip()

                for element in self.failureKeys: #search for failure keywords
                    if element in line:
                        logging.error("MOPAC output file contains the following error: %s")%element
                        return False

                for element in self.successKeys: #search for success keywords
                    if element in line:
                        successKeysFound[element] = True

                if "InChI=" in line:
                    logFileInChI = line #output files should take up to 240 characters of the name in the input file
                    InChIFound = True
                    if logFileInChI == self.geometry.uniqueIDlong:
                        InChIMatch = True
                    else:
                        logging.info("InChI in log file didn't match that in geometry.")

        # Check that ALL 'success' keywords were found in the file.
        if not all( successKeysFound.values() ):
            logging.error('Not all of the required keywords for sucess were found in the output file!')
            return False

        if not InChIFound:
            logging.error("No InChI was found in the MOPAC output file {0}".format(self.outputFilePath))
            return False

        if InChIMatch:
            logging.info("Successful MOPAC quantum result found in {0}".format(self.outputFilePath))
            # " + self.molfile.name + " ("+self.molfile.InChIAug+") has been found. This log file will be used.")
            return True

        #InChIs do not match (most likely due to limited name length mirrored in log file (240 characters), but possibly due to a collision)
        return self.checkForInChiKeyCollision(logFileInChI) # Not yet implemented!

    def read():
        # reads the output file
        import ipdb; ipdb.set_trace()
        pass

    def calculate():
        # calculators for the parsing
        import ipdb; ipdb.set_trace()
        pass

    def parse(self):
        """
        Parses the results of the Mopac calculation, and returns a CCLibData object.
        """
        parser = cclib.parser.Mopac(self.outputFilePath)
        parser.logger.setLevel(logging.ERROR) #cf. http://cclib.sourceforge.net/wiki/index.php/Using_cclib#Additional_information

        cclibData = parser.parse()
        radicalNumber = sum([i.radicalElectrons for i in self.molecule.atoms])

        qmData = CCLibData(cclibData, radicalNumber+1)

        return qmData

class MopacMol(QMMolecule, Mopac):

    def generateQMData(self):
        """
        Calculate the QM data and return a QMData object.
        """

        self.createGeometry()

        success = False
        method = MopacMolPM3(self)

        if self.verifyOutputFile():
            logging.info("Found a successful output file already; using that.")
        else:
            for attempt in range(1, self.maxAttempts+1):
                top_keys, bottom_keys, polar_keys = method.inputFileKeys( (attempt-1)%self.scriptAttempts+1 )
                self.writeInputFile(attempt, top_keys, bottom_keys, polar_keys)
                success = self.run()
                if success:
                    logging.info('Attempt {0} of {1} on species {2} succeeded.'.format(attempt, self.maxAttempts, self.molecule.toAugmentedInChI()))
                    break
            else:
                raise Exception('QM thermo calculation failed for {0}.'.format(self.molecule.toAugmentedInChI()))
        result = self.parse() # parsed in cclib
        return result

class MopacMolPM3(MopacMol):

    "Keywords that will be added at the top and bottom of the qm input file"
    keywords = [
                ("precise nosym", "oldgeo thermo nosym precise "),
                ("precise nosym gnorm=0.0 nonr", "oldgeo thermo nosym precise "),
                ("precise nosym gnorm=0.0", "oldgeo thermo nosym precise "),
                ("precise nosym gnorm=0.0 bfgs", "oldgeo thermo nosym precise "),
                ("precise nosym recalc=10 dmax=0.10 nonr cycles=2000 t=2000", "oldgeo thermo nosym precise "),
                ]

    scriptAttempts = len(keywords)
    maxAttempts = 2 * scriptAttempts

    def inputFileKeys(self, attempt):
        """
        Inherits the writeInputFile methods from mopac.py
        """
        multiplicity_keys = self.multiplicityKeywords[self.molecule.geometry.multiplicity]

        top_keys = "pm3 {0} {1}".format(
                multiplicity_keys,
                self.keywordsTop[attempt],
                )
        bottom_keys = "{0} pm3 {1}".format(
                self.keywordsBottom[attempt],
                multiplicity_keys,
                )
        polar_keys = "oldgeo {0} nosym precise pm3 {1}".format(
                'polar' if self.molecule.geometry.multiplicity == 1 else 'static',
                multiplicity_keys,
                )

        return top_keys, bottom_keys, polar_keys<commit_msg>Change keywords handling in Mopac.<commit_after>import os

import openbabel
import cclib.parser
import logging
from subprocess import Popen, PIPE

from qmdata import CCLibData
from molecule import QMMolecule, TDPropertiesCalculator

class Mopac:

    directory = 'QMfiles'
    inputFileExtension = '.mop'
    outputFileExtension = '.out'
    executablePath = os.path.join(os.getenv('MOPAC_DIR', default="/opt/mopac") , 'MOPAC2009.exe')

    usePolar = False#use polar keyword in MOPAC

    "Keywords for the multiplicity"
    multiplicityKeywords = {
                            1: '',
                            2: 'uhf doublet',
                            3: 'uhf triplet',
                            4: 'uhf quartet',
                            5: 'uhf quintet',
                            6: 'uhf sextet',
                            7: 'uhf septet',
                            8: 'uhf octet',
                            9: 'uhf nonet',
                           }

    #: List of phrases that indicate failure
    #: NONE of these must be present in a succesful job.
    failureKeys = [
                   'IMAGINARY FREQUENCIES',
                   'EXCESS NUMBER OF OPTIMIZATION CYCLES',
                   'NOT ENOUGH TIME FOR ANOTHER CYCLE',
                   ]
    #: List of phrases to indicate success.
    #: ALL of these must be present in a successful job.
    successKeys = [
                   'DESCRIPTION OF VIBRATIONS',
                  ]

    @property
    def outputFilePath(self):
        """Get the Mopac output file name."""
        return os.path.join(self.directory, self.geometry.uniqueID + self.outputFileExtension)

    @property
    def inputFilePath(self):
        """Get the Mopac input file name."""
        return os.path.join(self.directory, self.geometry.uniqueID + self.inputFileExtension)

    def run(self):
        # submits the input file to mopac
        process = Popen([self.executablePath, self.inputFilePath])
        process.communicate()# necessary to wait for executable termination!

        return self.verifyOutputFile()

    def verifyOutputFile(self):
        """
        Check's that an output file exists and was successful.

        Returns a boolean flag that states whether a successful MOPAC simulation already exists for the molecule with the
        given (augmented) InChI Key.

        The definition of finding a successful simulation is based on these criteria:
        1) finding an output file with the file name equal to the InChI Key
        2) NOT finding any of the keywords that are denote a calculation failure
        3) finding all the keywords that denote a calculation success.
        4) finding a match between the InChI of the given molecule and the InchI found in the calculation files

        If any of the above criteria is not matched, False will be returned and the procedures to start a new calculation
        will be initiated.
        """

        if not os.path.exists(self.outputFilePath):
            logging.info("Output file {0} does not exist.".format(self.outputFilePath))
            return False

        InChIMatch=False #flag (1 or 0) indicating whether the InChI in the file matches InChIaug this can only be 1 if InChIFound is also 1
        InChIFound=False #flag (1 or 0) indicating whether an InChI was found in the log file

        # Initialize dictionary with "False"s
        successKeysFound = dict([(key, False) for key in self.successKeys])

        with open(self.outputFilePath) as outputFile:
            for line in outputFile:
                line = line.strip()

                for element in self.failureKeys: #search for failure keywords
                    if element in line:
                        logging.error("MOPAC output file contains the following error: %s")%element
                        return False

                for element in self.successKeys: #search for success keywords
                    if element in line:
                        successKeysFound[element] = True

                if "InChI=" in line:
                    logFileInChI = line #output files should take up to 240 characters of the name in the input file
                    InChIFound = True
                    if logFileInChI == self.geometry.uniqueIDlong:
                        InChIMatch = True
                    else:
                        logging.info("InChI in log file didn't match that in geometry.")

        # Check that ALL 'success' keywords were found in the file.
        if not all( successKeysFound.values() ):
            logging.error('Not all of the required keywords for sucess were found in the output file!')
            return False

        if not InChIFound:
            logging.error("No InChI was found in the MOPAC output file {0}".format(self.outputFilePath))
            return False

        if InChIMatch:
            logging.info("Successful MOPAC quantum result found in {0}".format(self.outputFilePath))
            # " + self.molfile.name + " ("+self.molfile.InChIAug+") has been found. This log file will be used.")
            return True

        #InChIs do not match (most likely due to limited name length mirrored in log file (240 characters), but possibly due to a collision)
        return self.checkForInChiKeyCollision(logFileInChI) # Not yet implemented!

    def read():
        # reads the output file
        import ipdb; ipdb.set_trace()
        pass

    def calculate():
        # calculators for the parsing
        import ipdb; ipdb.set_trace()
        pass

    def parse(self):
        """
        Parses the results of the Mopac calculation, and returns a CCLibData object.
        """
        parser = cclib.parser.Mopac(self.outputFilePath)
        parser.logger.setLevel(logging.ERROR) #cf. http://cclib.sourceforge.net/wiki/index.php/Using_cclib#Additional_information

        cclibData = parser.parse()
        radicalNumber = sum([i.radicalElectrons for i in self.molecule.atoms])

        qmData = CCLibData(cclibData, radicalNumber+1)

        return qmData

class MopacMol(QMMolecule, Mopac):

    def writeInputFile(self, attempt):
        """
        Using the :class:`Geometry` object, write the input file
        for the `attmept`th attempt.
        """

        obConversion = openbabel.OBConversion()
        obConversion.SetInAndOutFormats("mol", "mop")
        mol = openbabel.OBMol()

        if attempt <= self.scriptAttempts: #use UFF-refined coordinates
            obConversion.ReadFile(mol, self.geometry.getRefinedMolFilePath() )
        else:
            obConversion.ReadFile(mol, self.geometry.getCrudeMolFilePath() )

        mol.SetTitle(self.geometry.uniqueIDlong)
        obConversion.SetOptions('k', openbabel.OBConversion.OUTOPTIONS)
        input_string = obConversion.WriteString(mol)
        top_keys, bottom_keys, polar_keys = self.inputFileKeywords(attempt)
        with open(self.inputFilePath, 'w') as mopacFile:
            mopacFile.write(top_keys)
            mopacFile.write(input_string)
            mopacFile.write('\n')
            mopacFile.write(bottom_keys)
            if self.usePolar:
                mopacFile.write('\n\n\n')
                mopacFile.write(polar_keys)

    def inputFileKeywords(self, attempt):
        """
        Return the top, bottom, and polar keywords.
        """
        raise NotImplementedError("Should be defined by subclass, eg. MopacMolPM3")

    def generateQMData(self):
        """
        Calculate the QM data and return a QMData object.
        """

        self.createGeometry()

        if self.verifyOutputFile():
            logging.info("Found a successful output file already; using that.")
        else:
            success = False
            for attempt in range(1, self.maxAttempts+1):
                self.writeInputFile(attempt)
                success = self.run()
                if success:
                    logging.info('Attempt {0} of {1} on species {2} succeeded.'.format(attempt, self.maxAttempts, self.molecule.toAugmentedInChI()))
                    break
            else:
                raise Exception('QM thermo calculation failed for {0}.'.format(self.molecule.toAugmentedInChI()))
        result = self.parse() # parsed in cclib
        return result

class MopacMolPM3(MopacMol):

    #: Keywords that will be added at the top and bottom of the qm input file
    keywords = [
                {'top':"precise nosym", 'bottom':"oldgeo thermo nosym precise "},
                {'top':"precise nosym gnorm=0.0 nonr", 'bottom':"oldgeo thermo nosym precise "},
                {'top':"precise nosym gnorm=0.0", 'bottom':"oldgeo thermo nosym precise "},
                {'top':"precise nosym gnorm=0.0 bfgs", 'bottom':"oldgeo thermo nosym precise "},
                {'top':"precise nosym recalc=10 dmax=0.10 nonr cycles=2000 t=2000", 'bottom':"oldgeo thermo nosym precise "},
                ]

    scriptAttempts = len(keywords)
    maxAttempts = 2 * scriptAttempts

    def inputFileKeywords(self, attempt):
        """
        Return the top, bottom, and polar keywords.
        """
        assert attempt <= self.maxAttempts

        if attempt > self.scriptAttempts:
            attempt -= self.scriptAttempts

        multiplicity_keys = self.multiplicityKeywords[self.molecule.geometry.multiplicity]

        top_keys = "pm3 {0} {1}".format(
                multiplicity_keys,
                self.keywords[attempt]['top'],
                )
        bottom_keys = "{0} pm3 {1}".format(
                self.keywords[attempt]['bottom'],
                multiplicity_keys,
                )
        polar_keys = "oldgeo {0} nosym precise pm3 {1}".format(
                'polar' if self.molecule.geometry.multiplicity == 1 else 'static',
                multiplicity_keys,
                )

        return top_keys, bottom_keys, polar_keys<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import click
import csv

from collections import defaultdict
from functools import partial

from elasticsearch import Elasticsearch

from .context import pass_context

progressbar = partial(click.progressbar, fill_char='=', empty_char=' ')
PB_LABEL = '%-15s'

@click.group()
@click.option('--es-hosts')
@pass_context
def cli(context, es_hosts):
    if es_hosts:
        context.es_hosts = es_hosts
    context.es = Elasticsearch(context.es_hosts)

@cli.command()
@pass_context
def clear(context):
    if context.es.indices.exists(context.es_index):
        context.es.indices.delete(context.es_index)

@cli.command()
@pass_context
def setup(context):
    body = {
        'school': {
            'properties': {
                'bsn': {'type': 'string', 'index': 'not_analyzed'},
                'meal': {'type': 'string', 'index': 'analyzed'},
                'name': {'type': 'string', 'index': 'analyzed'},
                'phonenumber': {'type': 'string', 'index': 'not_analyzed'},
                'remarks': {'type': 'string', 'index': 'analyzed'},
                'schoolnumber': {'type': 'string', 'index': 'not_analyzed'},
                'wwwaddress': {'type': 'string', 'index': 'no'},
                'public': {'type': 'boolean', 'index': 'not_analyzed'},
                'schooltype': {'type': 'string', 'index': 'not_analyzed'},
                'languages': {'type': 'string', 'index': 'not_analyzed'},
                'branches': {'type': 'string', 'index': 'not_analyzed', 'store': True},
                'accessibility': {
                    'type': 'nested',
                    'properties': {
                        'parking-lot': {'type': 'integer', 'ignore_malformed': True},
                        'elevator': {'type': 'integer', 'ignore_malformed': True},
                        'toilet': {'type': 'integer', 'ignore_malformed': True},
                        'open-for-wcu': {'type': 'integer', 'ignore_malformed': True},
                        'advice-center-hearing': {'type': 'integer', 'ignore_malformed': True},
                        'advice-center-speaking': {'type': 'integer', 'ignore_malformed': True},
                    }
                },
                'address': {
                    'type': 'nested',
                    'properties': {
                        'district': {'type': 'string', 'index': 'not_analyzed'},
                        'name': {'type': 'string', 'index': 'analyzed'},
                        'location': {'type': 'geo_point', 'lat_lon': True},
                        'street': {'type': 'string', 'index': 'analyzed'},
                        'plz': {'type': 'integer'},
                    }
                },
                'equipments': {
                    'type': 'nested',
                    'properties': {
                        'equipment': {'type': 'string', 'index': 'analyzed'},
                        'comment': {'type': 'string', 'index': 'analyzed'},
                        'wood': {'type': 'integer', 'ignore_malformed': True},
                        'textile': {'type': 'integer', 'ignore_malformed': True},
                        'metal': {'type': 'integer', 'ignore_malformed': True},
                        'electronic': {'type': 'integer', 'ignore_malformed': True},
                        'ceramic': {'type': 'integer', 'ignore_malformed': True},
                        'workshop': {'type': 'integer', 'ignore_malformed': True},
                        'workshops': {'type': 'integer', 'ignore_malformed': True},
                        'weaving': {'type': 'integer', 'ignore_malformed': True},
                        'bakery': {'type': 'integer', 'ignore_malformed': True},
                        'kitchen': {'type': 'integer', 'ignore_malformed': True},
                        'learning-workshop': {'type': 'integer', 'ignore_malformed': True},
                        'learning-workshops': {'type': 'integer', 'ignore_malformed': True},
                        'laptops': {'type': 'integer', 'ignore_malformed': True},
                        'whiteboards': {'type': 'integer', 'ignore_malformed': True},
                        'presenter': {'type': 'integer', 'ignore_malformed': True},
                        'pc-pool': {'type': 'integer', 'ignore_malformed': True},
                        'pc-pools': {'type': 'integer', 'ignore_malformed': True},
                        'media-library': {'type': 'integer', 'ignore_malformed': True},
                        'cybercafe': {'type': 'integer', 'ignore_malformed': True},
                    }
                },
                'personell': {
                    'type': 'object',
                    'properties': {
                        'name': {'type': 'string', 'index': 'not_analyzed'},
                        'data': {
                            'type': 'object',
                            'properties': {
                                'year': {'type': 'string', 'index': 'not_analyzed'},
                                'amount_m': {'type': 'integer', 'ignore_malformed': True},
                                'amount_f': {'type': 'integer', 'ignore_malformed': True}
                            }
                        }
                    }
                },
            }
        },
    }
    if not context.es.indices.exists(context.es_index):
        context.es.indices.create(context.es_index)
    context.es.indices.put_mapping(index=context.es_index,
                                   doc_type='school', body=body)

@cli.command()
@click.argument('schools', type=click.File('r'))
@click.argument('accessibility', type=click.File('r'))
@click.argument('addresses', type=click.File('r'))
@click.argument('equipments', type=click.File('r'))
@click.argument('languages', type=click.File('r'))
@click.argument('schools_ext', type=click.File('r'))
@click.argument('personell', type=click.File('r'))
@pass_context
def load_data(context,
              schools,
              accessibility,
              addresses,
              equipments,
              languages,
              schools_ext,
              personell
              ):
    click.secho('Loading schools ... ', fg='green')
    data = SchoolProcessor(schools).process()
    click.echo('Found %d schools' % len(data.keys()))

    click.secho('\nLoading addition data sets ...', fg='green')
    tmp = AccessibilityProcessor(accessibility).process()
    with progressbar(tmp, label=PB_LABEL % 'Accessibility') as bar:
        for bsn in bar:
            data[bsn]['accessibility'] = tmp[bsn]

    tmp = AddressProcessor(addresses).process()
    with progressbar(tmp, label=PB_LABEL % 'Addresses') as bar:
        for bsn in bar:
            data[bsn]['address'] = tmp[bsn]

    tmp = EquipmentProcessor(equipments).process()
    with progressbar(tmp, label=PB_LABEL % 'Equipment') as bar:
        for bsn in bar:
            data[bsn]['equipments'] = tmp[bsn]

    tmp = LanguageProcessor(languages).process()
    with progressbar(tmp, label=PB_LABEL % 'Language') as bar:
        for bsn in bar:
            data[bsn].update(tmp[bsn])

    tmp = SchoolExtProcessor(schools_ext).process()
    with progressbar(tmp, label=PB_LABEL % 'SchoolExt') as bar:
        for bsn in bar:
            data[bsn].update(tmp[bsn])

    tmp = PersonellProcessor(personell).process()
    with progressbar(tmp, label=PB_LABEL % 'Personell') as bar:
        for bsn in bar:
            if bsn in data:
                # We have data about schools that don't exist -.-
                data[bsn]['personell'] = [{'name': k, 'data': v}
                                          for k, v in tmp[bsn].items()]

    click.secho('\nIndexing ...', fg='green')

    with progressbar(data, label=PB_LABEL % 'Schools') as bar:
        for bsn in bar:
            context.es.index(index=context.es_index, doc_type='school',
                             id=bsn, body=data[bsn])

class Processor(object):

    def __init__(self, infile):
        self.infile = infile

    def process(self):
        reader = csv.DictReader(self.infile)
        data = {}
        for row in reader:
            bsn, rowdata = self.process_row(row)
            if bsn is None:
                continue
            data[bsn] = rowdata
        return data

    def process_row(self, row):
        bsn = row.pop('bsn', None)
        return bsn, row

class AccessibilityProcessor(Processor):
    pass

class AddressProcessor(Processor):

    def process_row(self, row):
        bsn, row = super(AddressProcessor, self).process_row(row)
        row.pop('address_id', None)
        lat = row.pop('latitude', None)
        lon = row.pop('longitude', None)
        if lat and lon:
            row['location'] = {'lat': lat, 'lon': lon}
        return bsn, row

class EquipmentProcessor(Processor):

    def process_row(self, row):
        bsn, row = super(EquipmentProcessor, self).process_row(row)
        row.pop('address_id', None)
        row.pop('district_id', None)
        return bsn, row

class LanguageProcessor(Processor):

    def process_row(self, row):
        bsn, row = super(LanguageProcessor, self).process_row(row)
        row['languages'] = row['languages'].split()
        return bsn, row

class SchoolProcessor(Processor):

    def process_row(self, row):
        row.pop('address_id', None)
        return row['bsn'], row

class SchoolExtProcessor(Processor):

    def process(self):
        reader = csv.DictReader(self.infile)
        data = {}
        for row in reader:
            bsn, newrowdata = self.process_row(row)
            if bsn is None:
                continue
            rowdata = data.get(bsn, newrowdata)
            branches = list(set(rowdata['branches']) | newrowdata['branches'])
            rowdata['branches'] = branches
            data[bsn] = rowdata
        return data

    def process_row(self, row):
        bsn, row = super(SchoolExtProcessor, self).process_row(row)
        data = {
            'public': row['Schultraeger'] == "öffentlich",
            'branches': set([row['Schulzweig']]),
            'schooltype': row['Schulart']
        }
        return bsn, data

class PersonellProcessor(Processor):

    def process(self):
        reader = csv.DictReader(self.infile)
        data = {}
        for row in reader:
            bsn, rowdata = self.process_row(row)
            if bsn is None:
                continue
            existingRowData = data.get(bsn, defaultdict(list))
            existingRowData[rowdata['name']].append(rowdata['data'])
            data[bsn] = existingRowData
        return data

    def process_row(self, row):
        bsn, row = super(PersonellProcessor, self).process_row(row)
        data = {
            'name': row['Text'],
            'data': {
                'year': row['Schuljahr'],
                'amount_f': row['Personal_W'],
                'amount_m': row['Personal_M'],
            }
        }
        return bsn, data

if __name__ == '__main__':
    cli()
<commit_msg>Calculate the betreuungsschlüssel...<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import click
import csv

from collections import defaultdict
from functools import partial

from elasticsearch import Elasticsearch

from .context import pass_context

progressbar = partial(click.progressbar, fill_char='=', empty_char=' ')
PB_LABEL = '%-15s'

@click.group()
@click.option('--es-hosts')
@pass_context
def cli(context, es_hosts):
    if es_hosts:
        context.es_hosts = es_hosts
    context.es = Elasticsearch(context.es_hosts)

@cli.command()
@pass_context
def clear(context):
    if context.es.indices.exists(context.es_index):
        context.es.indices.delete(context.es_index)

@cli.command()
@pass_context
def setup(context):
    body = {
        'school': {
            'properties': {
                'bsn': {'type': 'string', 'index': 'not_analyzed'},
                'meal': {'type': 'string', 'index': 'analyzed'},
                'name': {'type': 'string', 'index': 'analyzed'},
                'phonenumber': {'type': 'string', 'index': 'not_analyzed'},
                'remarks': {'type': 'string', 'index': 'analyzed'},
                'schoolnumber': {'type': 'string', 'index': 'not_analyzed'},
                'wwwaddress': {'type': 'string', 'index': 'no'},
                'public': {'type': 'boolean', 'index': 'not_analyzed'},
                'schooltype': {'type': 'string', 'index': 'not_analyzed'},
                'languages': {'type': 'string', 'index': 'not_analyzed'},
                'branches': {'type': 'string', 'index': 'not_analyzed', 'store': True},
                'accessibility': {
                    'type': 'nested',
                    'properties': {
                        'parking-lot': {'type': 'integer', 'ignore_malformed': True},
                        'elevator': {'type': 'integer', 'ignore_malformed': True},
                        'toilet': {'type': 'integer', 'ignore_malformed': True},
                        'open-for-wcu': {'type': 'integer', 'ignore_malformed': True},
                        'advice-center-hearing': {'type': 'integer', 'ignore_malformed': True},
                        'advice-center-speaking': {'type': 'integer', 'ignore_malformed': True},
                    }
                },
                'address': {
                    'type': 'nested',
                    'properties': {
                        'district': {'type': 'string', 'index': 'not_analyzed'},
                        'name': {'type': 'string', 'index': 'analyzed'},
                        'location': {'type': 'geo_point', 'lat_lon': True},
                        'street': {'type': 'string', 'index': 'analyzed'},
                        'plz': {'type': 'integer'},
                    }
                },
                'classes' : {
                    'type': 'object',
                    'properties': {
                        'name': {'type' : 'string', 'index':'not_analyzed'},
                        'type': {'type' : 'string', 'index':'not_analyzed'},
                        'amount': {'type' : 'integer', 'ignore_malformed':True},
                        'totalStudents' : {'type': 'integer', 'ignore_malformed': True},
                        'maleStudents' : {'type': 'integer', 'ignore_malformed': True},
                        'femaleStudents' : {'type': 'integer', 'ignore_malformed': True},
                        'foreignStudents' : {'type': 'integer', 'ignore_malformed': True},
                        'nonNativeStudents' : {'type': 'integer', 'ignore_malformed': True},
                    }
                },
                'equipments': {
                    'type': 'nested',
                    'properties': {
                        'equipment': {'type': 'string', 'index': 'analyzed'},
                        'comment': {'type': 'string', 'index': 'analyzed'},
                        'wood': {'type': 'integer', 'ignore_malformed': True},
                        'textile': {'type': 'integer', 'ignore_malformed': True},
                        'metal': {'type': 'integer', 'ignore_malformed': True},
                        'electronic': {'type': 'integer', 'ignore_malformed': True},
                        'ceramic': {'type': 'integer', 'ignore_malformed': True},
                        'workshop': {'type': 'integer', 'ignore_malformed': True},
                        'workshops': {'type': 'integer', 'ignore_malformed': True},
                        'weaving': {'type': 'integer', 'ignore_malformed': True},
                        'bakery': {'type': 'integer', 'ignore_malformed': True},
                        'kitchen': {'type': 'integer', 'ignore_malformed': True},
                        'learning-workshop': {'type': 'integer', 'ignore_malformed': True},
                        'learning-workshops': {'type': 'integer', 'ignore_malformed': True},
                        'laptops': {'type': 'integer', 'ignore_malformed': True},
                        'whiteboards': {'type': 'integer', 'ignore_malformed': True},
                        'presenter': {'type': 'integer', 'ignore_malformed': True},
                        'pc-pool': {'type': 'integer', 'ignore_malformed': True},
                        'pc-pools': {'type': 'integer', 'ignore_malformed': True},
                        'media-library': {'type': 'integer', 'ignore_malformed': True},
                        'cybercafe': {'type': 'integer', 'ignore_malformed': True},
                    }
                },
                'personell': {
                    'type': 'object',
                    'properties': {
                        'name': {'type': 'string', 'index': 'not_analyzed'},
                        'data': {
                            'type': 'object',
                            'properties': {
                                'year': {'type': 'string', 'index': 'not_analyzed'},
                                'amount_m': {'type': 'integer', 'ignore_malformed': True},
                                'amount_f': {'type': 'integer', 'ignore_malformed': True}
                            }
                        }
                    }
                },
            }
        },
    }
    if not context.es.indices.exists(context.es_index):
        context.es.indices.create(context.es_index)
    context.es.indices.put_mapping(index=context.es_index,
                                   doc_type='school', body=body)

@cli.command()
@click.argument('schools', type=click.File('r'))
@click.argument('accessibility', type=click.File('r'))
@click.argument('addresses', type=click.File('r'))
@click.argument('equipments', type=click.File('r'))
@click.argument('languages', type=click.File('r'))
@click.argument('schools_ext', type=click.File('r'))
@click.argument('personell', type=click.File('r'))
@click.argument('students', type=click.File('r'))
@pass_context
def load_data(context,
              schools,
              accessibility,
              addresses,
              equipments,
              languages,
              schools_ext,
              personell,
              students
              ):
    click.secho('Loading schools ... ', fg='green')
    data = SchoolProcessor(schools).process()
    click.echo('Found %d schools' % len(data.keys()))

    click.secho('\nLoading addition data sets ...', fg='green')
    tmp = AccessibilityProcessor(accessibility).process()
    with progressbar(tmp, label=PB_LABEL % 'Accessibility') as bar:
        for bsn in bar:
            data[bsn]['accessibility'] = tmp[bsn]

    tmp = AddressProcessor(addresses).process()
    with progressbar(tmp, label=PB_LABEL % 'Addresses') as bar:
        for bsn in bar:
            data[bsn]['address'] = tmp[bsn]

    tmp = EquipmentProcessor(equipments).process()
    with progressbar(tmp, label=PB_LABEL % 'Equipment') as bar:
        for bsn in bar:
            data[bsn]['equipments'] = tmp[bsn]

    tmp = LanguageProcessor(languages).process()
    with progressbar(tmp, label=PB_LABEL % 'Language') as bar:
        for bsn in bar:
            data[bsn].update(tmp[bsn])

    tmp = SchoolExtProcessor(schools_ext).process()
    with progressbar(tmp, label=PB_LABEL % 'SchoolExt') as bar:
        for bsn in bar:
            data[bsn].update(tmp[bsn])

    tmp = PersonellProcessor(personell).process()
    with progressbar(tmp, label=PB_LABEL % 'Personell') as bar:
        for bsn in bar:
            if bsn in data:
                # We have data about schools that don't exist -.-
                data[bsn]['personell'] = [{'name': k, 'data': v}
                                          for k, v in tmp[bsn].items()]

    tmp = StudentsProcessor(students).process()
    with progressbar(tmp, label=PB_LABEL % 'Students') as bar:
        for bsn in bar:
            if bsn in data:
                data[bsn]['classes'] = tmp[bsn]

    click.secho('\nIndexing ...', fg='green')

    with progressbar(data, label=PB_LABEL % 'Schools') as bar:
        for bsn in bar:
            context.es.index(index=context.es_index, doc_type='school',
                             id=bsn, body=data[bsn])

    for bsn in data:
        # sum up all the personell
        personell = 0
        if 'personell' in data[bsn]:
            for v in data[bsn]['personell']:
                if v['name'] in ('Lehrkräfte', 'Erzieher(innen)'):
                    d = v['data'][0]
                    personell = personell + float(d['amount_f']) + float(d['amount_m'])

            students = 0
            if 'classes' in data[bsn]:
                for clazz in data[bsn]['classes']:
                    students = students + float(clazz['totalStudents'])

                print (bsn + ': ' + str(students / personell)+ '\n')

class Processor(object):

    def __init__(self, infile):
        self.infile = infile

    def process(self):
        reader = csv.DictReader(self.infile)
        data = {}
        for row in reader:
            bsn, rowdata = self.process_row(row)
            if bsn is None:
                continue
            data[bsn] = rowdata
        return data

    def process_row(self, row):
        bsn = row.pop('bsn', None)
        return bsn, row

class AccessibilityProcessor(Processor):
    pass

class AddressProcessor(Processor):

    def process_row(self, row):
        bsn, row = super(AddressProcessor, self).process_row(row)
        row.pop('address_id', None)
        lat = row.pop('latitude', None)
        lon = row.pop('longitude', None)
        if lat and lon:
            row['location'] = {'lat': lat, 'lon': lon}
        return bsn, row

class EquipmentProcessor(Processor):

    def process_row(self, row):
        bsn, row = super(EquipmentProcessor, self).process_row(row)
        row.pop('address_id', None)
        row.pop('district_id', None)
        return bsn, row

class LanguageProcessor(Processor):

    def process_row(self, row):
        bsn, row = super(LanguageProcessor, self).process_row(row)
        row['languages'] = row['languages'].split()
        return bsn, row

class SchoolProcessor(Processor):

    def process_row(self, row):
        row.pop('address_id', None)
        return row['bsn'], row

class SchoolExtProcessor(Processor):

    def process(self):
        reader = csv.DictReader(self.infile)
        data = {}
        for row in reader:
            bsn, newrowdata = self.process_row(row)
            if bsn is None:
                continue
            rowdata = data.get(bsn, newrowdata)
            branches = list(set(rowdata['branches']) | newrowdata['branches'])
            rowdata['branches'] = branches
            data[bsn] = rowdata
        return data

    def process_row(self, row):
        bsn, row = super(SchoolExtProcessor, self).process_row(row)
        data = {
            'public': row['Schultraeger'] == "öffentlich",
            'branches': set([row['Schulzweig']]),
            'schooltype': row['Schulart']
        }
        return bsn, data

class PersonellProcessor(Processor):

    def process(self):
        reader = csv.DictReader(self.infile)
        data = {}
        for row in reader:
            bsn, rowdata = self.process_row(row)
            if bsn is None:
                continue
            existingRowData = data.get(bsn, defaultdict(list))
            existingRowData[rowdata['name']].append(rowdata['data'])
            data[bsn] = existingRowData
        return data

    def process_row(self, row):
        bsn, row = super(PersonellProcessor, self).process_row(row)
        data = {
            'name': row['Text'],
            'data': {
                'year': row['Schuljahr'],
                'amount_f': row['Personal_W'],
                'amount_m': row['Personal_M'],
            }
        }
        return bsn, data

class StudentsProcessor(Processor):

    def process(self):
        reader = csv.DictReader(self.infile)
        data = {}
        for row in reader:
            bsn, rowdata = self.process_row(row)
            if bsn is None:
                continue
            existingRowData = data.get(bsn, [])
            existingRowData.append(rowdata)
            data[bsn] = existingRowData
        return data

    def process_row(self, row):
        bsn, row = super(StudentsProcessor, self).process_row(row)
        data = {
            'name' : row['Jahrgangsstufe'],
            'type' : row['Klassenart'],
            'amount' : row['K'],
            'totalStudents' : row['S'],
            'maleStudents': row['SM'],
            'femaleStudents':row['SW'],
            'foreignStudents':row['N'],
            'nonNativeStudents':row['H']
        }
        return bsn, data

if __name__ == '__main__':
    cli()
<|endoftext|>


<commit_before>import logging

from django.http import JsonResponse, HttpResponse, HttpRequest
from django.views.generic import View

import rmr

class Json(View):

    http_code = 200

    logger = logging.getLogger('rmr.request')

    def dispatch(self, request: HttpRequest, *args, **kwargs):
        http_code = self.http_code
        try:
            result = super().dispatch(request, *args, **kwargs)
            if isinstance(result, HttpResponse):
                return result
            api_result = dict(
                data=result,
            )
        except rmr.Error as error:

            self.logger.log(
                error.level,
                '%(code)s: %(message)s',
                dict(message=error.message, code=error.code),
            )

            http_code = error.http_code

            api_result = dict(
                error=dict(
                    code=error.code,
                    description=error.message,
                ),
            )

        self.logger.debug(
            'request_path: %(request_path)s, '
            'request_headers: %(request_headers)s, '
            'request_params: %(request_params)s, '
            'request_data: %(request_data)s, '
            'response_code: %(response_code)s, '
            'response_data: %(response_data)s',
            dict(
                request_path=request.path,
                request_headers=request.META,
                request_params=request.GET,
                request_data=request.POST,
                response_code=http_code,
                response_data=api_result,
            ),
        )

        return JsonResponse(api_result, status=http_code)

    @staticmethod
    def get_device_id(request):
        return request.META.get('HTTP_DEVICE_ID')
<commit_msg>added requested method log<commit_after>import logging

from django.http import JsonResponse, HttpResponse, HttpRequest
from django.views.generic import View

import rmr

class Json(View):

    http_code = 200

    logger = logging.getLogger('rmr.request')

    def dispatch(self, request: HttpRequest, *args, **kwargs):
        http_code = self.http_code
        try:
            result = super().dispatch(request, *args, **kwargs)
            if isinstance(result, HttpResponse):
                return result
            api_result = dict(
                data=result,
            )
        except rmr.Error as error:

            self.logger.log(
                error.level,
                '%(code)s: %(message)s',
                dict(message=error.message, code=error.code),
            )

            http_code = error.http_code

            api_result = dict(
                error=dict(
                    code=error.code,
                    description=error.message,
                ),
            )

        self.logger.debug(
            'request_method: %(request_method)s, '
            'request_path: %(request_path)s, '
            'request_headers: %(request_headers)s, '
            'request_params: %(request_params)s, '
            'request_data: %(request_data)s, '
            'response_code: %(response_code)s, '
            'response_data: %(response_data)s',
            dict(
                request_method=request.method,
                request_path=request.path,
                request_headers=request.META,
                request_params=request.GET,
                request_data=request.POST,
                response_code=http_code,
                response_data=api_result,
            ),
        )

        return JsonResponse(api_result, status=http_code)

    @staticmethod
    def get_device_id(request):
        return request.META.get('HTTP_DEVICE_ID')
<|endoftext|>