<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# test/test_pyemd.py
"""Tests for PyEMD"""

import numpy as np
import pytest

from pyemd import emd, emd_samples, emd_with_flow

EMD_PRECISION = 5
FLOW_PRECISION = 4

# `emd()` and `emd_with_flow()` testing
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

def test_case_1():
    first_signature = np.array([0.0, 1.0])
    second_signature = np.array([5.0, 3.0])
    distance_matrix = np.array([[0.0, 0.5],
                                [0.5, 0.0]])
    assert emd(first_signature, second_signature, distance_matrix) == 3.5

def test_case_1_flow():
    first_signature = np.array([0.0, 1.0])
    second_signature = np.array([5.0, 3.0])
    distance_matrix = np.array([[0.0, 0.5],
                                [0.5, 0.0]])
    assert (emd_with_flow(first_signature, second_signature, distance_matrix) ==
            (3.5, [[0.0, 0.0],
                   [0.0, 1.0]]))

def test_case_2():
    first_signature = np.array([1.0, 1.0])
    second_signature = np.array([1.0, 1.0])
    distance_matrix = np.array([[0.0, 1.0],
                                [1.0, 0.0]])
    assert emd(first_signature, second_signature, distance_matrix) == 0.0

def test_case_2_flow():
    first_signature = np.array([1.0, 1.0])
    second_signature = np.array([1.0, 1.0])
    distance_matrix = np.array([[0.0, 1.0],
                                [1.0, 0.0]])

    assert (emd_with_flow(first_signature, second_signature, distance_matrix) ==
            (0.0, [[1.0, 0.0],
                   [0.0, 1.0]]))

def test_case_3():
    first_signature = np.array([6.0, 1.0])
    second_signature = np.array([1.0, 7.0])
    distance_matrix = np.array([[0.0, 0.0],
                                [0.0, 0.0]])
    assert emd(first_signature, second_signature, distance_matrix) == 0.0

def test_case_3_flow():
    first_signature = np.array([6.0, 1.0])
    second_signature = np.array([1.0, 7.0])
    distance_matrix = np.array([[0.0, 0.0],
                                [0.0, 0.0]])
    assert (emd_with_flow(first_signature, second_signature, distance_matrix) ==
            (0.0, [[1.0, 5.0],
                   [0.0, 1.0]]))

def test_case_4_flow():
    first_signature = np.array([1.0, 7.0])
    second_signature = np.array([6.0, 1.0])
    distance_matrix = np.array([[0.0, 0.0],
                                [0.0, 0.0]])
    assert (emd_with_flow(first_signature, second_signature,
                          distance_matrix) == (0.0, [[1.0, 0.0],
                                                     [5.0, 1.0]]))

def test_case_5_flow():
    first_signature = np.array([3.0, 5.0])
    second_signature = np.array([6.0, 2.0])
    distance_matrix = np.array([[0.0, 0.0],
                                [0.0, 0.0]])
    assert (emd_with_flow(first_signature, second_signature, distance_matrix) ==
            (0.0, [[3.0, 0.0],
                   [3.0, 2.0]]))

def test_case_6():
    first_signature = np.array([1.0, 2.0, 1.0, 2.0])
    second_signature = np.array([2.0, 1.0, 2.0, 1.0])
    distance_matrix = np.array([[0.0, 1.0, 1.0, 2.0],
                                [1.0, 0.0, 2.0, 1.0],
                                [1.0, 2.0, 0.0, 1.0],
                                [2.0, 1.0, 1.0, 0.0]])
    emd_value = emd(first_signature, second_signature, distance_matrix)
    assert round(emd_value, EMD_PRECISION) == 2.0

def test_case_6_flow():
    first_signature = np.array([1.0, 2.0, 1.0, 2.0])
    second_signature = np.array([2.0, 1.0, 2.0, 1.0])
    distance_matrix = np.array([[0.0, 1.0, 1.0, 2.0],
                                [1.0, 0.0, 2.0, 1.0],
                                [1.0, 2.0, 0.0, 1.0],
                                [2.0, 1.0, 1.0, 0.0]])
    emd_value, flow = emd_with_flow(first_signature, second_signature,
                                    distance_matrix)
    emd_value = round(emd_value, EMD_PRECISION)
    assert emd_value == 2.0
    flow = np.round(flow, FLOW_PRECISION)
    assert np.array_equal(flow, [[1.0, 0.0, 0.0, 0.0],
                                 [1.0, 1.0, 0.0, 0.0],
                                 [0.0, 0.0, 1.0, 0.0],
                                 [0.0, 0.0, 1.0, 1.0]])

def test_extra_mass_penalty():
    first_signature = np.array([0.0, 2.0, 1.0, 2.0])
    second_signature = np.array([2.0, 1.0, 2.0, 1.0])
    distance_matrix = np.array([[0.0, 1.0, 1.0, 2.0],
                                [1.0, 0.0, 2.0, 1.0],
                                [1.0, 2.0, 0.0, 1.0],
                                [2.0, 1.0, 1.0, 0.0]])
    emd_value = emd(
        first_signature,
        second_signature,
        distance_matrix,
        extra_mass_penalty=2.5)
    assert round(emd_value, EMD_PRECISION) == 4.5

def test_extra_mass_penalty_flow():
    first_signature = np.array([0.0, 2.0, 1.0, 2.0])
    second_signature = np.array([2.0, 1.0, 2.0, 1.0])
    distance_matrix = np.array([[0.0, 1.0, 1.0, 2.0],
                                [1.0, 0.0, 2.0, 1.0],
                                [1.0, 2.0, 0.0, 1.0],
                                [2.0, 1.0, 1.0, 0.0]])
    emd_value, flow = emd_with_flow(
        first_signature,
        second_signature,
        distance_matrix,
        extra_mass_penalty=2.5)
    emd_value = round(emd_value, EMD_PRECISION)
    assert emd_value == 4.5
    flow = np.round(flow, FLOW_PRECISION)
    print(flow)
    assert np.array_equal(flow, [[0.0, 0.0, 0.0, 0.0],
                                 [1.0, 1.0, 0.0, 0.0],
                                 [0.0, 0.0, 1.0, 0.0],
                                 [0.0, 0.0, 1.0, 1.0]])

# `emd_samples()` testing
# ~~~~~~~~~~~~~~~~~~~~~

def test_case_samples_1():
    first_array = [1, 2, 3, 4]
    second_array = [2, 3, 4, 5]
    emd_value = emd_samples(first_array, second_array)
    assert round(emd_value, EMD_PRECISION) == 0.75

def test_case_samples_1_binsize():
    first_array = [1, 2, 3, 4]
    second_array = [2, 3, 4, 5]
    emd_value = emd_samples(first_array, second_array, bins=2)
    assert round(emd_value, EMD_PRECISION) == 0.5

def test_case_samples_1_manual_range():
    first_array = [1, 2, 3, 4]
    second_array = [2, 3, 4, 5]
    emd_value = emd_samples(first_array, second_array, range=(0, 10))
    assert round(emd_value, EMD_PRECISION) == 1.0

def test_case_samples_1_not_normalized():
    first_array = [1, 2, 3, 4]
    second_array = [2, 3, 4, 5]
    emd_value = emd_samples(first_array, second_array, normalized=False)
    assert round(emd_value, EMD_PRECISION) == 3.0

def test_case_samples_1_custom_distance():
    dist = lambda x: np.array([[0. if i == j else 1. for i in x] for j in x])
    first_array = [1, 2, 3, 4]
    second_array = [2, 3, 4, 5]
    emd_value = emd_samples(first_array, second_array, distance=dist)
    assert round(emd_value, EMD_PRECISION) == 0.25

def test_case_samples_2():
    first_array = [1]
    second_array = [2]
    emd_value = emd_samples(first_array, second_array)
    assert round(emd_value, EMD_PRECISION) == 0.5

def test_case_samples_3():
    first_array = [1, 1, 1, 2, 3]
    second_array = [1, 2, 2, 2, 3]
    emd_value = emd_samples(first_array, second_array)
    assert round(emd_value, EMD_PRECISION) == 0.32

def test_case_samples_4():
    first_array = [1, 2, 3, 4, 5]
    second_array = [99, 98, 97, 96, 95]
    emd_value = emd_samples(first_array, second_array)
    assert round(emd_value, EMD_PRECISION) == 78.4

# Validation testing
# ~~~~~~~~~~~~~~~~~~

def test_larger_signatures():
    first_signature = np.array([0.0, 1.0, 2.0])
    second_signature = np.array([5.0, 3.0, 3.0])
    distance_matrix = np.array([[0.0, 0.5],
                                [0.5, 0.0]])
    with pytest.raises(ValueError):
        emd(first_signature, second_signature, distance_matrix)

def test_larger_signatures_flow():
    first_signature = np.array([0.0, 1.0, 2.0])
    second_signature = np.array([5.0, 3.0, 3.0])
    distance_matrix = np.array([[0.0, 0.5],
                                [0.5, 0.0]])
    with pytest.raises(ValueError):
        emd_with_flow(first_signature, second_signature, distance_matrix)

def test_larger_signatures_1():
    first_signature = np.array([0.0, 1.0, 2.0])
    second_signature = np.array([5.0, 3.0])
    distance_matrix = np.array([[0.0, 0.5],
                                [0.5, 0.0]])
    with pytest.raises(ValueError):
        emd_with_flow(first_signature, second_signature, distance_matrix)

def test_larger_signatures_1_flow():
    first_signature = np.array([0.0, 1.0, 2.0])
    second_signature = np.array([5.0, 3.0])
    distance_matrix = np.array([[0.0, 0.5],
                                [0.5, 0.0]])
    with pytest.raises(ValueError):
        emd(first_signature, second_signature, distance_matrix)

def test_larger_signatures_2():
    first_signature = np.array([0.0, 1.0])
    second_signature = np.array([5.0, 3.0, 3.0])
    distance_matrix = np.array([[0.0, 0.5], [0.5, 0.0]])
    with pytest.raises(ValueError):
        emd(first_signature, second_signature, distance_matrix)

def test_larger_signatures_2_flow():
    first_signature = np.array([0.0, 1.0])
    second_signature = np.array([5.0, 3.0, 3.0])
    distance_matrix = np.array([[0.0, 0.5],
                                [0.5, 0.0]])
    with pytest.raises(ValueError):
        emd_with_flow(first_signature, second_signature, distance_matrix)

def test_different_signature_dims():
    first_signature = np.array([0.0, 1.0])
    second_signature = np.array([5.0, 3.0, 3.0])
    distance_matrix = np.array([[0.0, 0.5, 0.0],
                                [0.5, 0.0, 0.0],
                                [0.5, 0.0, 0.0]])
    with pytest.raises(ValueError):
        emd(first_signature, second_signature, distance_matrix)

def test_different_signature_dims_flow():
    first_signature = np.array([0.0, 1.0])
    second_signature = np.array([5.0, 3.0, 3.0])
    distance_matrix = np.array([[0.0, 0.5, 0.0],
                                [0.5, 0.0, 0.0],
                                [0.5, 0.0, 0.0]])
    with pytest.raises(ValueError):
        emd_with_flow(first_signature, second_signature, distance_matrix)

def test_symmetric_distance_matrix():
    first_signature = np.array([0.0, 1.0])
    second_signature = np.array([5.0, 3.0])
    distance_matrix = np.array([[0.0, 0.5, 3.0],
                                [0.5, 0.0]])
    with pytest.raises(ValueError):
        emd(first_signature, second_signature, distance_matrix)

def test_symmetric_distance_matrix_flow():
    first_signature = np.array([0.0, 1.0])
    second_signature = np.array([5.0, 3.0])
    distance_matrix = np.array([[0.0, 0.5, 3.0],
                                [0.5, 0.0]])
    with pytest.raises(ValueError):
        emd_with_flow(first_signature, second_signature, distance_matrix)
<commit_msg>test_pyemd.py: Refactor assert statements<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# test/test_pyemd.py
"""Tests for PyEMD"""

import numpy as np
import pytest

from pyemd import emd, emd_samples, emd_with_flow

EMD_PRECISION = 5
FLOW_PRECISION = 4

def emd_assert(got, expected):
    assert round(got, EMD_PRECISION) == expected

def emd_flow_assert(got, expected):
    got_value, got_flow = got
    expected_value, expected_flow = expected
    assert round(got_value, EMD_PRECISION) == expected_value
    assert np.array_equal(np.round(got_flow, FLOW_PRECISION), expected_flow)

# `emd()` and `emd_with_flow()` testing
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

def test_case_1():
    first_signature = np.array([0.0, 1.0])
    second_signature = np.array([5.0, 3.0])
    distance_matrix = np.array([[0.0, 0.5],
                                [0.5, 0.0]])
    emd_assert(
        emd(first_signature, second_signature, distance_matrix),
        3.5
    )

def test_case_1_flow():
    first_signature = np.array([0.0, 1.0])
    second_signature = np.array([5.0, 3.0])
    distance_matrix = np.array([[0.0, 0.5],
                                [0.5, 0.0]])
    emd_flow_assert(
        emd_with_flow(first_signature, second_signature, distance_matrix),
        (3.5, [[0.0, 0.0],
               [0.0, 1.0]])
    )

def test_case_2():
    first_signature = np.array([1.0, 1.0])
    second_signature = np.array([1.0, 1.0])
    distance_matrix = np.array([[0.0, 1.0],
                                [1.0, 0.0]])
    emd_assert(
        emd(first_signature, second_signature, distance_matrix),
        0.0
    )

def test_case_2_flow():
    first_signature = np.array([1.0, 1.0])
    second_signature = np.array([1.0, 1.0])
    distance_matrix = np.array([[0.0, 1.0],
                                [1.0, 0.0]])

    emd_flow_assert(
        emd_with_flow(first_signature, second_signature, distance_matrix),
        (0.0, [[1.0, 0.0],
               [0.0, 1.0]])
    )

def test_case_3():
    first_signature = np.array([6.0, 1.0])
    second_signature = np.array([1.0, 7.0])
    distance_matrix = np.array([[0.0, 0.0],
                                [0.0, 0.0]])
    emd_assert(
        emd(first_signature, second_signature, distance_matrix),
        0.0
    )

def test_case_3_flow():
    first_signature = np.array([6.0, 1.0])
    second_signature = np.array([1.0, 7.0])
    distance_matrix = np.array([[0.0, 0.0],
                                [0.0, 0.0]])
    emd_flow_assert(
        emd_with_flow(first_signature, second_signature, distance_matrix),
        (0.0, [[1.0, 5.0],
               [0.0, 1.0]])
    )

def test_case_4_flow():
    first_signature = np.array([1.0, 7.0])
    second_signature = np.array([6.0, 1.0])
    distance_matrix = np.array([[0.0, 0.0],
                                [0.0, 0.0]])
    emd_flow_assert(
        emd_with_flow(first_signature, second_signature, distance_matrix),
        (0.0, [[1.0, 0.0],
               [5.0, 1.0]])
    )

def test_case_5_flow():
    first_signature = np.array([3.0, 5.0])
    second_signature = np.array([6.0, 2.0])
    distance_matrix = np.array([[0.0, 0.0],
                                [0.0, 0.0]])
    emd_flow_assert(
        emd_with_flow(first_signature, second_signature, distance_matrix),
        (0.0, [[3.0, 0.0],
               [3.0, 2.0]])
    )

def test_case_6():
    first_signature = np.array([1.0, 2.0, 1.0, 2.0])
    second_signature = np.array([2.0, 1.0, 2.0, 1.0])
    distance_matrix = np.array([[0.0, 1.0, 1.0, 2.0],
                                [1.0, 0.0, 2.0, 1.0],
                                [1.0, 2.0, 0.0, 1.0],
                                [2.0, 1.0, 1.0, 0.0]])
    emd_assert(
        emd(first_signature, second_signature, distance_matrix),
        2.0
    )

def test_case_6_flow():
    first_signature = np.array([1.0, 2.0, 1.0, 2.0])
    second_signature = np.array([2.0, 1.0, 2.0, 1.0])
    distance_matrix = np.array([[0.0, 1.0, 1.0, 2.0],
                                [1.0, 0.0, 2.0, 1.0],
                                [1.0, 2.0, 0.0, 1.0],
                                [2.0, 1.0, 1.0, 0.0]])
    emd_flow_assert(
        emd_with_flow(first_signature, second_signature, distance_matrix),
        (2.0, [[1.0, 0.0, 0.0, 0.0],
               [1.0, 1.0, 0.0, 0.0],
               [0.0, 0.0, 1.0, 0.0],
               [0.0, 0.0, 1.0, 1.0]])
    )

def test_extra_mass_penalty():
    first_signature = np.array([0.0, 2.0, 1.0, 2.0])
    second_signature = np.array([2.0, 1.0, 2.0, 1.0])
    distance_matrix = np.array([[0.0, 1.0, 1.0, 2.0],
                                [1.0, 0.0, 2.0, 1.0],
                                [1.0, 2.0, 0.0, 1.0],
                                [2.0, 1.0, 1.0, 0.0]])
    emd_assert(
        emd(first_signature, second_signature, distance_matrix,
            extra_mass_penalty=2.5),
        4.5
    )

def test_extra_mass_penalty_flow():
    first_signature = np.array([0.0, 2.0, 1.0, 2.0])
    second_signature = np.array([2.0, 1.0, 2.0, 1.0])
    distance_matrix = np.array([[0.0, 1.0, 1.0, 2.0],
                                [1.0, 0.0, 2.0, 1.0],
                                [1.0, 2.0, 0.0, 1.0],
                                [2.0, 1.0, 1.0, 0.0]])
    emd_flow_assert(
        emd_with_flow(first_signature, second_signature, distance_matrix,
                      extra_mass_penalty=2.5),
        (4.5, [[0.0, 0.0, 0.0, 0.0],
               [1.0, 1.0, 0.0, 0.0],
               [0.0, 0.0, 1.0, 0.0],
               [0.0, 0.0, 1.0, 1.0]])
    )

# `emd_samples()` testing
# ~~~~~~~~~~~~~~~~~~~~~

def test_case_samples_1():
    first_array = [1, 2, 3, 4]
    second_array = [2, 3, 4, 5]
    emd_assert(emd_samples(first_array, second_array), 0.75)

def test_case_samples_1_binsize():
    first_array = [1, 2, 3, 4]
    second_array = [2, 3, 4, 5]
    emd_assert(emd_samples(first_array, second_array, bins=2), 0.5)

def test_case_samples_1_manual_range():
    first_array = [1, 2, 3, 4]
    second_array = [2, 3, 4, 5]
    emd_assert(emd_samples(first_array, second_array, range=(0, 10)), 1.0)

def test_case_samples_1_not_normalized():
    first_array = [1, 2, 3, 4]
    second_array = [2, 3, 4, 5]
    emd_assert(emd_samples(first_array, second_array, normalized=False), 3.0)

def test_case_samples_1_custom_distance():
    dist = lambda x: np.array([[0.0 if i == j else 1.0 for i in x] for j in x])
    first_array = [1, 2, 3, 4]
    second_array = [2, 3, 4, 5]
    emd_assert(emd_samples(first_array, second_array, distance=dist), 0.25)

def test_case_samples_2():
    first_array = [1]
    second_array = [2]
    emd_assert(emd_samples(first_array, second_array), 0.5)

def test_case_samples_3():
    first_array = [1, 1, 1, 2, 3]
    second_array = [1, 2, 2, 2, 3]
    emd_assert(emd_samples(first_array, second_array), 0.32)

def test_case_samples_4():
    first_array = [1, 2, 3, 4, 5]
    second_array = [99, 98, 97, 96, 95]
    emd_assert(emd_samples(first_array, second_array), 78.4)

# Validation testing
# ~~~~~~~~~~~~~~~~~~

def test_larger_signatures():
    first_signature = np.array([0.0, 1.0, 2.0])
    second_signature = np.array([5.0, 3.0, 3.0])
    distance_matrix = np.array([[0.0, 0.5],
                                [0.5, 0.0]])
    with pytest.raises(ValueError):
        emd(first_signature, second_signature, distance_matrix)

def test_larger_signatures_flow():
    first_signature = np.array([0.0, 1.0, 2.0])
    second_signature = np.array([5.0, 3.0, 3.0])
    distance_matrix = np.array([[0.0, 0.5],
                                [0.5, 0.0]])
    with pytest.raises(ValueError):
        emd_with_flow(first_signature, second_signature, distance_matrix)

def test_larger_signatures_1():
    first_signature = np.array([0.0, 1.0, 2.0])
    second_signature = np.array([5.0, 3.0])
    distance_matrix = np.array([[0.0, 0.5],
                                [0.5, 0.0]])
    with pytest.raises(ValueError):
        emd_with_flow(first_signature, second_signature, distance_matrix)

def test_larger_signatures_1_flow():
    first_signature = np.array([0.0, 1.0, 2.0])
    second_signature = np.array([5.0, 3.0])
    distance_matrix = np.array([[0.0, 0.5],
                                [0.5, 0.0]])
    with pytest.raises(ValueError):
        emd(first_signature, second_signature, distance_matrix)

def test_larger_signatures_2():
    first_signature = np.array([0.0, 1.0])
    second_signature = np.array([5.0, 3.0, 3.0])
    distance_matrix = np.array([[0.0, 0.5], [0.5, 0.0]])
    with pytest.raises(ValueError):
        emd(first_signature, second_signature, distance_matrix)

def test_larger_signatures_2_flow():
    first_signature = np.array([0.0, 1.0])
    second_signature = np.array([5.0, 3.0, 3.0])
    distance_matrix = np.array([[0.0, 0.5],
                                [0.5, 0.0]])
    with pytest.raises(ValueError):
        emd_with_flow(first_signature, second_signature, distance_matrix)

def test_different_signature_dims():
    first_signature = np.array([0.0, 1.0])
    second_signature = np.array([5.0, 3.0, 3.0])
    distance_matrix = np.array([[0.0, 0.5, 0.0],
                                [0.5, 0.0, 0.0],
                                [0.5, 0.0, 0.0]])
    with pytest.raises(ValueError):
        emd(first_signature, second_signature, distance_matrix)

def test_different_signature_dims_flow():
    first_signature = np.array([0.0, 1.0])
    second_signature = np.array([5.0, 3.0, 3.0])
    distance_matrix = np.array([[0.0, 0.5, 0.0],
                                [0.5, 0.0, 0.0],
                                [0.5, 0.0, 0.0]])
    with pytest.raises(ValueError):
        emd_with_flow(first_signature, second_signature, distance_matrix)

def test_symmetric_distance_matrix():
    first_signature = np.array([0.0, 1.0])
    second_signature = np.array([5.0, 3.0])
    distance_matrix = np.array([[0.0, 0.5, 3.0],
                                [0.5, 0.0]])
    with pytest.raises(ValueError):
        emd(first_signature, second_signature, distance_matrix)

def test_symmetric_distance_matrix_flow():
    first_signature = np.array([0.0, 1.0])
    second_signature = np.array([5.0, 3.0])
    distance_matrix = np.array([[0.0, 0.5, 3.0],
                                [0.5, 0.0]])
    with pytest.raises(ValueError):
        emd_with_flow(first_signature, second_signature, distance_matrix)
<|endoftext|>


<commit_before># Copyright 2015 The Tornado Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from __future__ import absolute_import, division, print_function, with_statement

__all__ = ['Condition', 'Event', 'Semaphore']

import collections

from tornado import gen, ioloop
from tornado.concurrent import Future

class Condition(object):
    """A condition allows one or more coroutines to wait until notified.

    Like a standard `threading.Condition`, but does not need an underlying lock
    that is acquired and released.
    """

    def __init__(self):
        self.io_loop = ioloop.IOLoop.current()
        self._waiters = collections.deque()  # Futures.
        self._timeouts = 0

    def __str__(self):
        result = '<%s' % (self.__class__.__name__, )
        if self._waiters:
            result += ' waiters[%s]' % len(self._waiters)
        return result + '>'

    def wait(self, timeout=None):
        """Wait for `.notify`.

        Returns a `.Future` that resolves ``True`` if the condition is notified,
        or ``False`` after a timeout.
        """
        waiter = Future()
        self._waiters.append(waiter)
        if timeout:
            def on_timeout():
                waiter.set_result(False)
                self._garbage_collect()
            self.io_loop.add_timeout(timeout, on_timeout)
        return waiter

    def notify(self, n=1):
        """Wake ``n`` waiters."""
        waiters = []  # Waiters we plan to run right now.
        while n and self._waiters:
            waiter = self._waiters.popleft()
            if not waiter.done():  # Might have timed out.
                n -= 1
                waiters.append(waiter)

        for waiter in waiters:
            waiter.set_result(True)

    def notify_all(self):
        """Wake all waiters."""
        self.notify(len(self._waiters))

    def _garbage_collect(self):
        # Occasionally clear timed-out waiters, if many coroutines wait with a
        # timeout but notify is called rarely.
        self._timeouts += 1
        if self._timeouts > 100:
            self._timeouts = 0
            self._waiters = collections.deque(
                w for w in self._waiters if not w.done())

class Event(object):
    """An event blocks coroutines until its internal flag is set to True.

    Similar to `threading.Event`.
    """
    def __init__(self):
        self._future = Future()

    def __str__(self):
        return '<%s %s>' % (
            self.__class__.__name__, 'set' if self.is_set() else 'clear')

    def is_set(self):
        """Return ``True`` if the internal flag is true."""
        return self._future.done()

    def set(self):
        """Set the internal flag to ``True``. All waiters are awakened.

        Calling `.wait` once the flag is set will not block.
        """
        if not self._future.done():
            self._future.set_result(None)

    def clear(self):
        """Reset the internal flag to ``False``.

        Calls to `.wait` will block until `.set` is called.
        """
        if self._future.done():
            self._future = Future()

    def wait(self, timeout=None):
        """Block until the internal flag is true.

        Returns a Future, which raises :exc:`~tornado.gen.TimeoutError` after a
        timeout.
        """
        if timeout is None:
            return self._future
        else:
            return gen.with_timeout(timeout, self._future)

class _ReleasingContextManager(object):
    """Releases a Lock or Semaphore at the end of a "with" statement.

        with (yield semaphore.acquire()):
            pass

        # Now semaphore.release() has been called.
    """
    def __init__(self, obj):
        self._obj = obj

    def __enter__(self):
        pass

    def __exit__(self, exc_type, exc_val, exc_tb):
        self._obj.release()

class Semaphore(object):
    """A lock that can be acquired a fixed number of times before blocking.

    A Semaphore manages a counter representing the number of `.release` calls
    minus the number of `.acquire` calls, plus an initial value. The `.acquire`
    method blocks if necessary until it can return without making the counter
    negative.

    `.acquire` supports the context manager protocol:

    >>> from tornado import gen, locks
    >>> semaphore = locks.Semaphore()
    >>> @gen.coroutine
    ... def f():
    ...    with (yield semaphore.acquire()):
    ...        # Do something holding the semaphore.
    ...        pass
    ...
    ...    # Now the semaphore is released.
    """
    def __init__(self, value=1):
        if value < 0:
            raise ValueError('semaphore initial value must be >= 0')

        self.io_loop = ioloop.IOLoop.current()
        self._value = value
        self._waiters = collections.deque()

    def __repr__(self):
        res = super(Semaphore, self).__repr__()
        extra = 'locked' if self._value == 0 else 'unlocked,value:{0}'.format(
            self._value)
        if self._waiters:
            extra = '{0},waiters:{1}'.format(extra, len(self._waiters))
        return '<{0} [{1}]>'.format(res[1:-1], extra)

    def release(self):
        """Increment the counter and wake one waiter."""
        self._value += 1
        for waiter in self._waiters:
            if not waiter.done():
                self._value -= 1

                # If the waiter is a coroutine paused at
                #
                #     with (yield semaphore.acquire()):
                #
                # then the context manager's __exit__ calls release() at the end
                # of the "with" block.
                waiter.set_result(_ReleasingContextManager(self))
                break

    def acquire(self, timeout=None):
        """Decrement the counter. Returns a Future.

        Block if the counter is zero and wait for a `.release`. The Future
        raises `.TimeoutError` after the deadline.
        """
        if self._value > 0:
            self._value -= 1
            future = Future()
            future.set_result(_ReleasingContextManager(self))
        else:
            waiter = Future()
            self._waiters.append(waiter)
            if timeout:
                future = gen.with_timeout(timeout, waiter, self.io_loop,
                                          quiet_exceptions=gen.TimeoutError)

                # Set waiter's exception after the deadline.
                gen.chain_future(future, waiter)
            else:
                future = waiter
        return future

    def __enter__(self):
        raise RuntimeError(
            "Use Semaphore like 'with (yield semaphore.acquire())', not like"
            " 'with semaphore'")

    __exit__ = __enter__
<commit_msg>Remove Semaphore.io_loop.<commit_after># Copyright 2015 The Tornado Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from __future__ import absolute_import, division, print_function, with_statement

__all__ = ['Condition', 'Event', 'Semaphore']

import collections

from tornado import gen, ioloop
from tornado.concurrent import Future

class Condition(object):
    """A condition allows one or more coroutines to wait until notified.

    Like a standard `threading.Condition`, but does not need an underlying lock
    that is acquired and released.
    """

    def __init__(self):
        self.io_loop = ioloop.IOLoop.current()
        self._waiters = collections.deque()  # Futures.
        self._timeouts = 0

    def __str__(self):
        result = '<%s' % (self.__class__.__name__, )
        if self._waiters:
            result += ' waiters[%s]' % len(self._waiters)
        return result + '>'

    def wait(self, timeout=None):
        """Wait for `.notify`.

        Returns a `.Future` that resolves ``True`` if the condition is notified,
        or ``False`` after a timeout.
        """
        waiter = Future()
        self._waiters.append(waiter)
        if timeout:
            def on_timeout():
                waiter.set_result(False)
                self._garbage_collect()
            self.io_loop.add_timeout(timeout, on_timeout)
        return waiter

    def notify(self, n=1):
        """Wake ``n`` waiters."""
        waiters = []  # Waiters we plan to run right now.
        while n and self._waiters:
            waiter = self._waiters.popleft()
            if not waiter.done():  # Might have timed out.
                n -= 1
                waiters.append(waiter)

        for waiter in waiters:
            waiter.set_result(True)

    def notify_all(self):
        """Wake all waiters."""
        self.notify(len(self._waiters))

    def _garbage_collect(self):
        # Occasionally clear timed-out waiters, if many coroutines wait with a
        # timeout but notify is called rarely.
        self._timeouts += 1
        if self._timeouts > 100:
            self._timeouts = 0
            self._waiters = collections.deque(
                w for w in self._waiters if not w.done())

class Event(object):
    """An event blocks coroutines until its internal flag is set to True.

    Similar to `threading.Event`.
    """
    def __init__(self):
        self._future = Future()

    def __str__(self):
        return '<%s %s>' % (
            self.__class__.__name__, 'set' if self.is_set() else 'clear')

    def is_set(self):
        """Return ``True`` if the internal flag is true."""
        return self._future.done()

    def set(self):
        """Set the internal flag to ``True``. All waiters are awakened.

        Calling `.wait` once the flag is set will not block.
        """
        if not self._future.done():
            self._future.set_result(None)

    def clear(self):
        """Reset the internal flag to ``False``.

        Calls to `.wait` will block until `.set` is called.
        """
        if self._future.done():
            self._future = Future()

    def wait(self, timeout=None):
        """Block until the internal flag is true.

        Returns a Future, which raises :exc:`~tornado.gen.TimeoutError` after a
        timeout.
        """
        if timeout is None:
            return self._future
        else:
            return gen.with_timeout(timeout, self._future)

class _ReleasingContextManager(object):
    """Releases a Lock or Semaphore at the end of a "with" statement.

        with (yield semaphore.acquire()):
            pass

        # Now semaphore.release() has been called.
    """
    def __init__(self, obj):
        self._obj = obj

    def __enter__(self):
        pass

    def __exit__(self, exc_type, exc_val, exc_tb):
        self._obj.release()

class Semaphore(object):
    """A lock that can be acquired a fixed number of times before blocking.

    A Semaphore manages a counter representing the number of `.release` calls
    minus the number of `.acquire` calls, plus an initial value. The `.acquire`
    method blocks if necessary until it can return without making the counter
    negative.

    `.acquire` supports the context manager protocol:

    >>> from tornado import gen, locks
    >>> semaphore = locks.Semaphore()
    >>> @gen.coroutine
    ... def f():
    ...    with (yield semaphore.acquire()):
    ...        # Do something holding the semaphore.
    ...        pass
    ...
    ...    # Now the semaphore is released.
    """
    def __init__(self, value=1):
        if value < 0:
            raise ValueError('semaphore initial value must be >= 0')

        self._value = value
        self._waiters = collections.deque()

    def __repr__(self):
        res = super(Semaphore, self).__repr__()
        extra = 'locked' if self._value == 0 else 'unlocked,value:{0}'.format(
            self._value)
        if self._waiters:
            extra = '{0},waiters:{1}'.format(extra, len(self._waiters))
        return '<{0} [{1}]>'.format(res[1:-1], extra)

    def release(self):
        """Increment the counter and wake one waiter."""
        self._value += 1
        for waiter in self._waiters:
            if not waiter.done():
                self._value -= 1

                # If the waiter is a coroutine paused at
                #
                #     with (yield semaphore.acquire()):
                #
                # then the context manager's __exit__ calls release() at the end
                # of the "with" block.
                waiter.set_result(_ReleasingContextManager(self))
                break

    def acquire(self, timeout=None):
        """Decrement the counter. Returns a Future.

        Block if the counter is zero and wait for a `.release`. The Future
        raises `.TimeoutError` after the deadline.
        """
        if self._value > 0:
            self._value -= 1
            future = Future()
            future.set_result(_ReleasingContextManager(self))
        else:
            waiter = Future()
            self._waiters.append(waiter)
            if timeout:
                future = gen.with_timeout(timeout, waiter,
                                          quiet_exceptions=gen.TimeoutError)

                # Set waiter's exception after the deadline.
                gen.chain_future(future, waiter)
            else:
                future = waiter
        return future

    def __enter__(self):
        raise RuntimeError(
            "Use Semaphore like 'with (yield semaphore.acquire())', not like"
            " 'with semaphore'")

    __exit__ = __enter__
<|endoftext|>


<commit_before>import pytest

from nex.state import GlobalState, Mode
from nex.tex_parameters import Parameters
from nex.fonts import GlobalFontState
from nex import box
from nex.box_writer import write_to_file

from test_banisher import test_char_to_cat

class DummyCodes:
    def __init__(self, char_to_cat):
        if char_to_cat is None:
            self.char_to_cat = test_char_to_cat.copy()
        else:
            self.char_to_cat = char_to_cat

    def get_cat_code(self, char):
        return self.char_to_cat[char]

    def get_lower_case_code(self, c):
        return c.lower()

    def get_upper_case_code(self, c):
        return c.upper()

class DummyRouter:

    def __init__(self, cs_map):
        self.cs_map = cs_map

    def lookup_control_sequence(self, name, *args, **kwargs):
        canon_token = self.cs_map[name]
        return canon_token.copy(*args, **kwargs)

class DummyFontState:

    def __init__(self):
        self._current_font_id = 0

    def set_current_font(self, is_global, font_id):
        self._current_font_id = font_id

    @property
    def current_font_id(self):
        return self._current_font_id

class DummyParameters:

    def __init__(self, param_map):
        self.param_map = param_map

    def get(self, name, *args, **kwargs):
        return self.param_map[name]

class DummyFontInfo:

    def __init__(self, file_name, file_path, at_clause):
        self.font_name = file_name
        self.file_name = file_name
        self.file_path = file_path
        self.at_clause = at_clause

class DummyGlobalFontState(GlobalFontState):

    FontInfo = DummyFontInfo

    def define_new_font(self, file_name, at_clause):
        font_info = DummyFontInfo(file_name=file_name,
                                  file_path=f'/dummy/font/path/{file_name}',
                                  at_clause=at_clause)
        font_id = max(self.fonts.keys()) + 1
        self.fonts[font_id] = font_info
        # Return new font id.
        return font_id

font_path = '/Users/ejm/projects/nex/example/fonts'

def get_state(char_to_cat, cs_map, param_map):
    router = DummyRouter(cs_map)
    parameters = DummyParameters(param_map)
    codes = DummyCodes(char_to_cat)
    font = DummyFontState()
    global_font_state = DummyGlobalFontState()
    return GlobalState(global_font_state=global_font_state,
                       codes=codes, registers=None,
                       scoped_font_state=font, router=router,
                       parameters=parameters)

def test():
    def g(d, str, shr):
        return {'dimen': d, 'stretch': str, 'shrink': shr}
    params = {
        Parameters.par_skip: g(10, 0, 0),
        Parameters.par_indent: 100,
        Parameters.par_fill_skip: g(10, 0, 0),
        Parameters.h_size: 10,
        Parameters.base_line_skip: g(10, 0, 0),
        Parameters.mag: 1000
    }
    state = get_state(test_char_to_cat, {}, params)
    font_id = state.define_new_font(file_name='testy', at_clause=None)
    state.select_font(is_global=True, font_id=font_id)
    state.add_character('a')
    state.do_paragraph()
    assert len(state.modes) == 1
    assert state.mode == Mode.vertical
    lst = state._layout_list
    assert len(lst) == 3
    assert isinstance(lst[0], box.FontDefinition)
    assert isinstance(lst[1], box.FontSelection)
    assert isinstance(lst[2], box.Character)
<commit_msg>Optionally write to file in state test<commit_after>import pytest

from nex.state import GlobalState, Mode
from nex.tex_parameters import Parameters
from nex.fonts import GlobalFontState
from nex import box
from nex.box_writer import write_to_file

from test_banisher import test_char_to_cat

do_output = True

class DummyCodes:
    def __init__(self, char_to_cat):
        if char_to_cat is None:
            self.char_to_cat = test_char_to_cat.copy()
        else:
            self.char_to_cat = char_to_cat

    def get_cat_code(self, char):
        return self.char_to_cat[char]

    def get_lower_case_code(self, c):
        return c.lower()

    def get_upper_case_code(self, c):
        return c.upper()

class DummyRouter:

    def __init__(self, cs_map):
        self.cs_map = cs_map

    def lookup_control_sequence(self, name, *args, **kwargs):
        canon_token = self.cs_map[name]
        return canon_token.copy(*args, **kwargs)

class DummyFontState:

    def __init__(self):
        self._current_font_id = 0

    def set_current_font(self, is_global, font_id):
        self._current_font_id = font_id

    @property
    def current_font_id(self):
        return self._current_font_id

class DummyParameters:

    def __init__(self, param_map):
        self.param_map = param_map

    def get(self, name, *args, **kwargs):
        return self.param_map[name]

class DummyFontInfo:

    def __init__(self, file_name, file_path, at_clause):
        self.font_name = file_name
        self.file_name = file_name
        self.file_path = file_path
        self.at_clause = at_clause

class DummyGlobalFontState(GlobalFontState):

    FontInfo = DummyFontInfo

    def define_new_font(self, file_name, at_clause):
        font_info = DummyFontInfo(file_name=file_name,
                                  file_path=f'/dummy/font/path/{file_name}',
                                  at_clause=at_clause)
        font_id = max(self.fonts.keys()) + 1
        self.fonts[font_id] = font_info
        # Return new font id.
        return font_id

font_path = '/Users/ejm/projects/nex/example/fonts'

def get_state(char_to_cat, cs_map, param_map):
    router = DummyRouter(cs_map)
    parameters = DummyParameters(param_map)
    codes = DummyCodes(char_to_cat)
    font = DummyFontState()
    if do_output:
        global_font_state = GlobalFontState(search_paths=['/Users/ejm/projects/nex/example/fonts'])
    else:
        global_font_state = DummyGlobalFontState()
    state = GlobalState(global_font_state=global_font_state,
                        codes=codes, registers=None,
                        scoped_font_state=font, router=router,
                        parameters=parameters)
    font_id = state.define_new_font(file_name='cmr10', at_clause=None)
    state.select_font(is_global=True, font_id=font_id)
    return state

def g(d, str, shr):
    return {'dimen': d, 'stretch': str, 'shrink': shr}

def test_single_letter():
    params = {
        Parameters.par_skip: g(10, 0, 0),
        Parameters.par_indent: 100,
        Parameters.par_fill_skip: g(10, 0, 0),
        Parameters.h_size: 10,
        Parameters.base_line_skip: g(10, 0, 0),
        Parameters.mag: 1000
    }
    state = get_state(test_char_to_cat, {}, params)
    state.add_character('a')
    state.do_paragraph()
    assert len(state.modes) == 1
    assert state.mode == Mode.vertical
    lst = state._layout_list
    assert len(lst) == 3
    assert isinstance(lst[0], box.FontDefinition)
    assert isinstance(lst[1], box.FontSelection)
    assert isinstance(lst[2], box.Character)
    if do_output:
        write_to_file(state, 'test_single_letter.dvi')
<|endoftext|>


<commit_before>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 02-08-2017 16:40
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This module contains models for dealing with PoGo stuff
"""

import os
import abc
import threading
# Application imports
import config_manager as main_app_config_manager
from parallel.models import ParallelRunner, CommandLineRunnerFactory
from parallel.exceptions import CommandLineRunnerException
from . import config_manager as module_config_manager
from .exceptions import PogoRunnerException

class PogoRunResult:
    # TODO - Needs to be extended for abstracting from results files from '-mm' parameter use
    # TODO - This needs refactoring to make it pythonist compliant
    def __init__(self,
                 pogo_runner):
        """
        Just the constructor, I had this implemented as syntactic sugar, but I was wrong
        :param ncbi_taxonomy_id: ncbi taxonomy id for this PoGo run results
        :param pogo_source_file_path: path of the source file used to run PoGo
        :param protein_sequence_file_path: FASTA file
        :param gtf_file_path: GTF file
        """
        # Logging
        self.__logger = main_app_config_manager.get_app_config_manager().get_logger_for(
            "{}.{}".format(__name__, type(self).__name__))
        # Map<pogo_result_file_extension, pogo_result_file_path>
        self.__pogo_result_file_paths = {}
        self.__pogo_source_file_path = None
        self.__pogo_runner = pogo_runner

    @property
    def pogo_runner(self):
        return self.__pogo_runner

    @pogo_runner.setter
    def pogo_runner(self, pogo_runner):
        if not self.__pogo_runner or (self.__pogo_runner != pogo_runner):
            self.__pogo_runner = pogo_runner
            # Regenerate pogo result file paths
            self.__generate_pogo_result_file_paths(pogo_runner.pogo_input_file)

    @property
    def ncbi_taxonomy_id(self):
        return self.pogo_runner.ncbi_taxonomy_id

    @property
    def protein_sequence_file_path(self):
        return self.pogo_runner.protein_sequence_file_path

    @property
    def gtf_file_path(self):
        return self.pogo_runner.gtf_file_path

    def __generate_pogo_result_file_paths(self, pogo_source_file_path):
        """
        Given a full path to a source file, e.g. /nfs/fullpath/PXD001110-9606.pogo, and the fact that PoGo generates
        result files by appending to the source file name different extensions, e.g.
            PXD001110-9606.pogo.bed
            PXD001110-9606.pogo.gct
            PXD001110-9606.pogo_no-ptm.bed
            PXD001110-9606.pogo_out.gtf
            PXD001110-9606.pogo_patch_hapl_scaff.bed
            PXD001110-9606.pogo_patch_hapl_scaff.gct
            PXD001110-9606.pogo_patch_hapl_scaff_no-ptm.bed
            PXD001110-9606.pogo_patch_hapl_scaff_out.gtf
            PXD001110-9606.pogo_patch_hapl_scaff_ptm.bed
            PXD001110-9606.pogo_ptm.bed
            PXD001110-9606.pogo_ptm_sorted.bed
            PXD001110-9606.pogo_sorted.bed
            PXD001110-9606.pogo_unmapped.txt
        being the extensions: .bed, .gct, _no-ptm.bed, _out.gtf, _patch_hapl_scaff.bed ... etc.
        This method collects all the result files based on being them files that start with the source file name
        'PXD001110-9606.pogo', and it will create a map (file_suffix, file_full_path), e.g.
            {
                ...
                '_patch_hapl_scaff.bed': '/nfs/fullpath/PXD001110-9606.pogo_patch_hapl_scaff.bed'
                ...
            }
        This means, for the current use case where we could be running PoGo specifying parameters like '-mm' that will
        produce files with different (extended) suffixes, e.g. _1MM_patch_hapl_scaff.bed, in the same folder as other
        PoGo runners with different parameters, this method will also map the other runners result files available at
        the time of performing the file system scan. THIS IS PERFECTLY OK, DON'T PANIC, because whe you use the getters
        for full paths to the result files for a particular runner, only the ones belonging to that runner will be
        picked up from the map, e.g. if this is a runner with '-mm' parameter active, only those '_1MM_*" files will be
        returned as result.
        :param pogo_source_file_path:
        :return:
        """
        self.__pogo_result_file_paths = {}
        pogo_results_folder_path = os.path.dirname(pogo_source_file_path)
        pogo_source_file_name = os.path.basename(pogo_source_file_path)
        for root, dirs, files in os.walk(pogo_results_folder_path):
            for pogo_result_file_path in files:
                pogo_result_file_name = os.path.basename(pogo_result_file_path)
                if pogo_result_file_name.startswith(pogo_source_file_name) and (
                            pogo_result_file_name != pogo_source_file_name):
                    # It is file generated by PoGo
                    pogo_result_file_extension = pogo_result_file_name.split(pogo_source_file_name)[1]
                    # self.__pogo_result_file_paths[pogo_result_file_extension] = pogo_result_file_path
                    # I need the full path for the pogo result file path
                    self.__pogo_result_file_paths[pogo_result_file_extension] = os.path.join(pogo_results_folder_path,
                                                                                pogo_result_file_path)
                    self._get_logger().info(
                        "PoGo run result '{}' ---> '{}', at full path '{}'".format(pogo_result_file_extension,
                                                                                pogo_result_file_path,
                                                                                self.__pogo_result_file_paths[
                                                                                pogo_result_file_extension]))

    def __get_pogo_result_file_path(self, file_extension):
        if file_extension in self.__pogo_result_file_paths:
            return self.__pogo_result_file_paths[file_extension]
        return None

    def _get_logger(self):
        return self.__logger

    def set_ncbi_taxonomy_id(self, taxonomy_id):
        self.__ncbi_taxonomy_id = taxonomy_id

    def set_protein_sequence_file_path(self, protein_sequence_file_path):
        self.__protein_sequence_file_path = protein_sequence_file_path

    def set_gtf_file_path(self, gtf_file_path):
        self.__gtf_file_path = gtf_file_path

    def get_ncbi_taxonomy_id(self):
        return self.__ncbi_taxonomy_id

    def get_pogo_result_main_bed_file_path(self):
        return self.__get_pogo_result_file_path(
            module_config_manager.get_configuration_service().get_pogo_result_file_extension_for_main_bed_file())

    def get_pogo_result_main_ptm_bed_file_path(self):
        return self.__get_pogo_result_file_path(
            module_config_manager.get_configuration_service().get_pogo_result_file_extension_for_main_ptm_bed_file()
        )

# PoGo Runners
class PogoRunnerFactory:
    @staticmethod
    def get_pogo_runner(ncbi_taxonomy_id, pogo_input_file, protein_sequence_file_path, gtf_file_path, mm_gap=None):
        # TODO - In the future, more PoGo runners will be implemented
        return PogoRunnerLocalThread(ncbi_taxonomy_id,
                                     pogo_input_file,
                                     protein_sequence_file_path,
                                     gtf_file_path,
                                     mm_gap)

class PogoRunner(ParallelRunner):
    def __init__(self,
                 ncbi_taxonomy_id=None,
                 pogo_input_file=None,
                 protein_sequence_file_path=None,
                 gtf_file_path=None,
                 mm_gap=None):
        super().__init__()
        self.ncbi_taxonomy_id = ncbi_taxonomy_id
        self.pogo_input_file = pogo_input_file
        self.protein_sequence_file_path = protein_sequence_file_path
        self.gtf_file_path = gtf_file_path
        self.mm_gap = mm_gap
        self._stdout = b' '
        self._stderr = b' '
        self.__success = True
        self.__pogo_run_result = None

    def _set_failed_to_run_pogo(self):
        self.__success = False

    def _set_success_to_run_pogo(self):
        self.__success = self.__success and True

    def _validate_environment_for_running_pogo(self):
        validation_ok = True
        if not self.ncbi_taxonomy_id:
            self._logger.warning("NO NCBI TAXONOMY ID has been specified for running PoGo with input file '{}'"
                                 .format(self.pogo_input_file))
        if not os.path.isfile(self.pogo_input_file):
            self._logger.error("PoGo input file '{}' IS NOT VALID".format(self.pogo_input_file))
            validation_ok = False
        if not os.path.isfile(self.protein_sequence_file_path):
            self._logger.error("Protein sequence file '{}' IS NOT VALID".format(self.protein_sequence_file_path))
            validation_ok = False
        if not os.path.isfile(self.gtf_file_path):
            self._logger.error("GTF file '{}' IS NOT VALID".format(self.gtf_file_path))
            validation_ok = False
        if not os.path.isfile(module_config_manager.get_configuration_service().get_pogo_binary_file_path()):
            self._logger.error("PoGo binary file '{}' IS NOT VALID"
                               .format(module_config_manager.get_configuration_service().get_pogo_binary_file_path()))
            validation_ok = False
        return validation_ok

    def get_stdout(self):
        # Never give it back until the runner is done with whatever it is doing
        if not self._done:
            raise PogoRunnerException("PoGo Runner is NOT DONE doing its job, thus 'stdout' is NOT AVAILABLE")
        return self._stdout

    def get_stderr(self):
        # Never give it back until the runner is done with whatever it is doing
        if not self._done:
            raise PogoRunnerException("PoGo Runner is NOT DONE doing its job, thus 'stderr' is NOT AVAILABLE")
        return self._stderr

    def get_pogo_run_command(self):
        pogo_parameter_species = ''
        if self.ncbi_taxonomy_id:
            pogo_parameter_species = " -species {} ".format(self.ncbi_taxonomy_id)
        pogo_parameter_mm_gap = ''
        if self.mm_gap:
            pogo_parameter_mm_gap = " -mm {}".format(self.mm_gap)
        pogo_command = "time {}{}{} -fasta {} -gtf {} -in {}" \
            .format(module_config_manager.get_configuration_service().get_pogo_binary_file_path(),
                    pogo_parameter_species,
                    pogo_parameter_mm_gap,
                    self.protein_sequence_file_path,
                    self.gtf_file_path,
                    self.pogo_input_file)
        self._logger.debug("Command for running PoGo - '{}'".format(pogo_command))
        return pogo_command

    @abc.abstractmethod
    def _get_command_line_runner(self):
        """
        Although a factory is used to get a command line runner, its initialization differs depending on the final
        PogoRunner, i.e. we delegate to subclasses the provisioning of an initialized CommandLineRunner, how we use
        it, is the same for all subclasses.
        It will prepare the command line runner completely, including the command it has to run.
        :return: a CommandLineRunner for running PoGo
        """
        ...

    def _run(self):
        if not self._validate_environment_for_running_pogo():
            self._logger.error("Can't run PoGo for file '{}'".format(self.pogo_input_file))
            self._set_failed_to_run_pogo()
        else:
            command_line_runner = self._get_command_line_runner()
            try:
                command_line_runner.start()
                self._logger.debug("PoGo --- STARTED --- command '{}'".format(command_line_runner.command))
                command_line_runner.wait()
            except CommandLineRunnerException as e:
                self._logger.error("ERROR RUNNING PoGo - {}, STDOUT ---> '{}', STDERR ---> '{}'"
                                   .format(e.value,
                                           command_line_runner.get_stdout().decode('utf8'),
                                           command_line_runner.get_stderr().decode('utf8')))
                self._set_failed_to_run_pogo()
            finally:
                self._stdout = command_line_runner.get_stdout()
                self._stderr = command_line_runner.get_stderr()
            if not command_line_runner.command_success:
                self._set_failed_to_run_pogo()
                self._logger.error("PoGo FAILED on input file '{}', command '{}'"
                                   .format(self.pogo_input_file, command_line_runner.command))
            else:
                # Successful run of PoGo
                self._set_success_to_run_pogo()
                self._logger.info("PoGo SUCCESS on input file '{}', command '{}'"
                                  .format(self.pogo_input_file, command_line_runner.command))

    def get_pogo_run_result(self):
        if not self.is_done():
            raise PogoRunnerException("PoGo runner IS NOT FINISHED YET, thus, results can't be retrieved")
        if not self.__pogo_run_result:
            self.__pogo_run_result = PogoRunResult(self)
        return self.__pogo_run_result

    def is_success(self):
        return self.__success

class PogoRunnerLocalThread(PogoRunner):
    def __init__(self,
                 ncbi_taxonomy_id=None,
                 pogo_input_file=None,
                 protein_sequence_file_path=None,
                 gtf_file_path=None,
                 mm_gap=None):
        super().__init__(ncbi_taxonomy_id, pogo_input_file, protein_sequence_file_path, gtf_file_path, mm_gap)
        self._logger = main_app_config_manager \
            .get_app_config_manager() \
            .get_logger_for("{}.{}-{}".format(__name__, type(self).__name__, threading.current_thread().getName()))

    def _get_command_line_runner(self):
        command_line_runner = CommandLineRunnerFactory.get_multithread_command_line_runner()
        command_line_runner.command = self.get_pogo_run_command()
        command_line_runner.timeout = module_config_manager.get_configuration_service().get_pogo_run_timeout()
        return command_line_runner

class PogoRunnerHpc(PogoRunner):
    pass

if __name__ == '__main__':
    print("ERROR: This script is part of a pipeline collection and it is not meant to be run in stand alone mode")
<commit_msg>refactor, un-needed setters removed<commit_after>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 02-08-2017 16:40
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This module contains models for dealing with PoGo stuff
"""

import os
import abc
import threading
# Application imports
import config_manager as main_app_config_manager
from parallel.models import ParallelRunner, CommandLineRunnerFactory
from parallel.exceptions import CommandLineRunnerException
from . import config_manager as module_config_manager
from .exceptions import PogoRunnerException

class PogoRunResult:
    # TODO - Needs to be extended for abstracting from results files from '-mm' parameter use
    # TODO - This needs refactoring to make it pythonist compliant
    def __init__(self,
                 pogo_runner):
        """
        Just the constructor, I had this implemented as syntactic sugar, but I was wrong
        :param ncbi_taxonomy_id: ncbi taxonomy id for this PoGo run results
        :param pogo_source_file_path: path of the source file used to run PoGo
        :param protein_sequence_file_path: FASTA file
        :param gtf_file_path: GTF file
        """
        # Logging
        self.__logger = main_app_config_manager.get_app_config_manager().get_logger_for(
            "{}.{}".format(__name__, type(self).__name__))
        # Map<pogo_result_file_extension, pogo_result_file_path>
        self.__pogo_result_file_paths = {}
        self.__pogo_source_file_path = None
        self.__pogo_runner = pogo_runner

    @property
    def pogo_runner(self):
        return self.__pogo_runner

    @pogo_runner.setter
    def pogo_runner(self, pogo_runner):
        if not self.__pogo_runner or (self.__pogo_runner != pogo_runner):
            self.__pogo_runner = pogo_runner
            # Regenerate pogo result file paths
            self.__generate_pogo_result_file_paths(pogo_runner.pogo_input_file)

    @property
    def ncbi_taxonomy_id(self):
        return self.pogo_runner.ncbi_taxonomy_id

    @property
    def protein_sequence_file_path(self):
        return self.pogo_runner.protein_sequence_file_path

    @property
    def gtf_file_path(self):
        return self.pogo_runner.gtf_file_path

    def __generate_pogo_result_file_paths(self, pogo_source_file_path):
        """
        Given a full path to a source file, e.g. /nfs/fullpath/PXD001110-9606.pogo, and the fact that PoGo generates
        result files by appending to the source file name different extensions, e.g.
            PXD001110-9606.pogo.bed
            PXD001110-9606.pogo.gct
            PXD001110-9606.pogo_no-ptm.bed
            PXD001110-9606.pogo_out.gtf
            PXD001110-9606.pogo_patch_hapl_scaff.bed
            PXD001110-9606.pogo_patch_hapl_scaff.gct
            PXD001110-9606.pogo_patch_hapl_scaff_no-ptm.bed
            PXD001110-9606.pogo_patch_hapl_scaff_out.gtf
            PXD001110-9606.pogo_patch_hapl_scaff_ptm.bed
            PXD001110-9606.pogo_ptm.bed
            PXD001110-9606.pogo_ptm_sorted.bed
            PXD001110-9606.pogo_sorted.bed
            PXD001110-9606.pogo_unmapped.txt
        being the extensions: .bed, .gct, _no-ptm.bed, _out.gtf, _patch_hapl_scaff.bed ... etc.
        This method collects all the result files based on being them files that start with the source file name
        'PXD001110-9606.pogo', and it will create a map (file_suffix, file_full_path), e.g.
            {
                ...
                '_patch_hapl_scaff.bed': '/nfs/fullpath/PXD001110-9606.pogo_patch_hapl_scaff.bed'
                ...
            }
        This means, for the current use case where we could be running PoGo specifying parameters like '-mm' that will
        produce files with different (extended) suffixes, e.g. _1MM_patch_hapl_scaff.bed, in the same folder as other
        PoGo runners with different parameters, this method will also map the other runners result files available at
        the time of performing the file system scan. THIS IS PERFECTLY OK, DON'T PANIC, because whe you use the getters
        for full paths to the result files for a particular runner, only the ones belonging to that runner will be
        picked up from the map, e.g. if this is a runner with '-mm' parameter active, only those '_1MM_*" files will be
        returned as result.
        :param pogo_source_file_path:
        :return:
        """
        self.__pogo_result_file_paths = {}
        pogo_results_folder_path = os.path.dirname(pogo_source_file_path)
        pogo_source_file_name = os.path.basename(pogo_source_file_path)
        for root, dirs, files in os.walk(pogo_results_folder_path):
            for pogo_result_file_path in files:
                pogo_result_file_name = os.path.basename(pogo_result_file_path)
                if pogo_result_file_name.startswith(pogo_source_file_name) and (
                            pogo_result_file_name != pogo_source_file_name):
                    # It is file generated by PoGo
                    pogo_result_file_extension = pogo_result_file_name.split(pogo_source_file_name)[1]
                    # self.__pogo_result_file_paths[pogo_result_file_extension] = pogo_result_file_path
                    # I need the full path for the pogo result file path
                    self.__pogo_result_file_paths[pogo_result_file_extension] = os.path.join(pogo_results_folder_path,
                                                                                pogo_result_file_path)
                    self._get_logger().info(
                        "PoGo run result '{}' ---> '{}', at full path '{}'".format(pogo_result_file_extension,
                                                                                pogo_result_file_path,
                                                                                self.__pogo_result_file_paths[
                                                                                pogo_result_file_extension]))

    def __get_pogo_result_file_path(self, file_extension):
        if file_extension in self.__pogo_result_file_paths:
            return self.__pogo_result_file_paths[file_extension]
        return None

    def _get_logger(self):
        return self.__logger

    def get_pogo_result_main_bed_file_path(self):
        return self.__get_pogo_result_file_path(
            module_config_manager.get_configuration_service().get_pogo_result_file_extension_for_main_bed_file())

    def get_pogo_result_main_ptm_bed_file_path(self):
        return self.__get_pogo_result_file_path(
            module_config_manager.get_configuration_service().get_pogo_result_file_extension_for_main_ptm_bed_file()
        )

# PoGo Runners
class PogoRunnerFactory:
    @staticmethod
    def get_pogo_runner(ncbi_taxonomy_id, pogo_input_file, protein_sequence_file_path, gtf_file_path, mm_gap=None):
        # TODO - In the future, more PoGo runners will be implemented
        return PogoRunnerLocalThread(ncbi_taxonomy_id,
                                     pogo_input_file,
                                     protein_sequence_file_path,
                                     gtf_file_path,
                                     mm_gap)

class PogoRunner(ParallelRunner):
    def __init__(self,
                 ncbi_taxonomy_id=None,
                 pogo_input_file=None,
                 protein_sequence_file_path=None,
                 gtf_file_path=None,
                 mm_gap=None):
        super().__init__()
        self.ncbi_taxonomy_id = ncbi_taxonomy_id
        self.pogo_input_file = pogo_input_file
        self.protein_sequence_file_path = protein_sequence_file_path
        self.gtf_file_path = gtf_file_path
        self.mm_gap = mm_gap
        self._stdout = b' '
        self._stderr = b' '
        self.__success = True
        self.__pogo_run_result = None

    def _set_failed_to_run_pogo(self):
        self.__success = False

    def _set_success_to_run_pogo(self):
        self.__success = self.__success and True

    def _validate_environment_for_running_pogo(self):
        validation_ok = True
        if not self.ncbi_taxonomy_id:
            self._logger.warning("NO NCBI TAXONOMY ID has been specified for running PoGo with input file '{}'"
                                 .format(self.pogo_input_file))
        if not os.path.isfile(self.pogo_input_file):
            self._logger.error("PoGo input file '{}' IS NOT VALID".format(self.pogo_input_file))
            validation_ok = False
        if not os.path.isfile(self.protein_sequence_file_path):
            self._logger.error("Protein sequence file '{}' IS NOT VALID".format(self.protein_sequence_file_path))
            validation_ok = False
        if not os.path.isfile(self.gtf_file_path):
            self._logger.error("GTF file '{}' IS NOT VALID".format(self.gtf_file_path))
            validation_ok = False
        if not os.path.isfile(module_config_manager.get_configuration_service().get_pogo_binary_file_path()):
            self._logger.error("PoGo binary file '{}' IS NOT VALID"
                               .format(module_config_manager.get_configuration_service().get_pogo_binary_file_path()))
            validation_ok = False
        return validation_ok

    def get_stdout(self):
        # Never give it back until the runner is done with whatever it is doing
        if not self._done:
            raise PogoRunnerException("PoGo Runner is NOT DONE doing its job, thus 'stdout' is NOT AVAILABLE")
        return self._stdout

    def get_stderr(self):
        # Never give it back until the runner is done with whatever it is doing
        if not self._done:
            raise PogoRunnerException("PoGo Runner is NOT DONE doing its job, thus 'stderr' is NOT AVAILABLE")
        return self._stderr

    def get_pogo_run_command(self):
        pogo_parameter_species = ''
        if self.ncbi_taxonomy_id:
            pogo_parameter_species = " -species {} ".format(self.ncbi_taxonomy_id)
        pogo_parameter_mm_gap = ''
        if self.mm_gap:
            pogo_parameter_mm_gap = " -mm {}".format(self.mm_gap)
        pogo_command = "time {}{}{} -fasta {} -gtf {} -in {}" \
            .format(module_config_manager.get_configuration_service().get_pogo_binary_file_path(),
                    pogo_parameter_species,
                    pogo_parameter_mm_gap,
                    self.protein_sequence_file_path,
                    self.gtf_file_path,
                    self.pogo_input_file)
        self._logger.debug("Command for running PoGo - '{}'".format(pogo_command))
        return pogo_command

    @abc.abstractmethod
    def _get_command_line_runner(self):
        """
        Although a factory is used to get a command line runner, its initialization differs depending on the final
        PogoRunner, i.e. we delegate to subclasses the provisioning of an initialized CommandLineRunner, how we use
        it, is the same for all subclasses.
        It will prepare the command line runner completely, including the command it has to run.
        :return: a CommandLineRunner for running PoGo
        """
        ...

    def _run(self):
        if not self._validate_environment_for_running_pogo():
            self._logger.error("Can't run PoGo for file '{}'".format(self.pogo_input_file))
            self._set_failed_to_run_pogo()
        else:
            command_line_runner = self._get_command_line_runner()
            try:
                command_line_runner.start()
                self._logger.debug("PoGo --- STARTED --- command '{}'".format(command_line_runner.command))
                command_line_runner.wait()
            except CommandLineRunnerException as e:
                self._logger.error("ERROR RUNNING PoGo - {}, STDOUT ---> '{}', STDERR ---> '{}'"
                                   .format(e.value,
                                           command_line_runner.get_stdout().decode('utf8'),
                                           command_line_runner.get_stderr().decode('utf8')))
                self._set_failed_to_run_pogo()
            finally:
                self._stdout = command_line_runner.get_stdout()
                self._stderr = command_line_runner.get_stderr()
            if not command_line_runner.command_success:
                self._set_failed_to_run_pogo()
                self._logger.error("PoGo FAILED on input file '{}', command '{}'"
                                   .format(self.pogo_input_file, command_line_runner.command))
            else:
                # Successful run of PoGo
                self._set_success_to_run_pogo()
                self._logger.info("PoGo SUCCESS on input file '{}', command '{}'"
                                  .format(self.pogo_input_file, command_line_runner.command))

    def get_pogo_run_result(self):
        if not self.is_done():
            raise PogoRunnerException("PoGo runner IS NOT FINISHED YET, thus, results can't be retrieved")
        if not self.__pogo_run_result:
            self.__pogo_run_result = PogoRunResult(self)
        return self.__pogo_run_result

    def is_success(self):
        return self.__success

class PogoRunnerLocalThread(PogoRunner):
    def __init__(self,
                 ncbi_taxonomy_id=None,
                 pogo_input_file=None,
                 protein_sequence_file_path=None,
                 gtf_file_path=None,
                 mm_gap=None):
        super().__init__(ncbi_taxonomy_id, pogo_input_file, protein_sequence_file_path, gtf_file_path, mm_gap)
        self._logger = main_app_config_manager \
            .get_app_config_manager() \
            .get_logger_for("{}.{}-{}".format(__name__, type(self).__name__, threading.current_thread().getName()))

    def _get_command_line_runner(self):
        command_line_runner = CommandLineRunnerFactory.get_multithread_command_line_runner()
        command_line_runner.command = self.get_pogo_run_command()
        command_line_runner.timeout = module_config_manager.get_configuration_service().get_pogo_run_timeout()
        return command_line_runner

class PogoRunnerHpc(PogoRunner):
    pass

if __name__ == '__main__':
    print("ERROR: This script is part of a pipeline collection and it is not meant to be run in stand alone mode")
<|endoftext|>