<commit_before># Author: Leland McInnes <leland.mcinnes@gmail.com>
#
# License: BSD 2 clause

import numba
import numpy as np

@numba.njit('void(i8[:], i8)')
def seed(rng_state, seed):
    """Seed the random number generator with a given seed."""
    rng_state.fill(seed + 0xffff)

@numba.njit('i4(i8[:])')
def tau_rand_int(state):
    """A fast (pseudo)-random number generator.

    Parameters
    ----------
    state: array of int64, shape (3,)
        The internal state of the rng

    Returns
    -------
    A (pseudo)-random int32 value
    """
    state[0] = (((state[0] & 4294967294) << 12) & 0xffffffff) ^ \
               ((((state[0] << 13) & 0xffffffff) ^ state[0]) >> 19)
    state[1] = (((state[1] & 4294967288) << 4) & 0xffffffff) ^ \
               ((((state[1] << 2) & 0xffffffff) ^ state[1]) >> 25)
    state[2] = (((state[2] & 4294967280) << 17) & 0xffffffff) ^ \
               ((((state[2] << 3) & 0xffffffff) ^ state[2]) >> 11)

    return state[0] ^ state[1] ^ state[2]

@numba.njit('f4(i8[:])')
def tau_rand(state):
    """A fast (pseudo)-random number generator for floats in the range [0,1]

    Parameters
    ----------
    state: array of int64, shape (3,)
        The internal state of the rng

    Returns
    -------
    A (pseudo)-random float32 in the interval [0, 1]
    """
    integer = tau_rand_int(state)
    return abs(float(integer) / 0x7fffffff)

@numba.njit(fastmath=True)
def norm(vec):
    """Compute the (standard l2) norm of a vector.

    Parameters
    ----------
    vec: array of shape (dim,)

    Returns
    -------
    The l2 norm of vec.
    """
    result = 0.0
    for i in range(vec.shape[0]):
        result += vec[i] ** 2
    return np.sqrt(result)

@numba.njit(parallel=True)
def rejection_sample(n_samples, pool_size, rng_state):
    """Generate n_samples many integers from 0 to pool_size such that no
    integer is selected twice. The duplication constraint is achieved via
    rejection sampling.

    Parameters
    ----------
    n_samples: int
        The number of random samples to select from the pool

    pool_size: int
        The size of the total pool of candidates to sample from

    rng_state: array of int64, shape (3,)
        Internal state of the random number generator

    Returns
    -------
    sample: array of shape(n_samples,)
        The ``n_samples`` randomly selected elements from the pool.
    """
    result = np.empty(n_samples, dtype=np.int64)
    for i in range(n_samples):
        reject_sample = True
        while reject_sample:
            j = tau_rand_int(rng_state) % pool_size
            for k in range(i):
                if j == result[k]:
                    break
            else:
                reject_sample = False
        result[i] = j
    return result

@numba.njit('f8[:, :, :](i8,i8)')
def make_heap(n_points, size):
    """Constructor for the numba enabled heap objects. The heaps are used
    for approximate nearest neighbor search, maintaining a list of potential
    neighbors sorted by their distance. We also flag if potential neighbors
    are newly added to the list or not. Internally this is stored as
    a single ndarray; the first axis determines whether we are looking at the
    array of candidate indices, the array of distances, or the flag array for
    whether elements are new or not. Each of these arrays are of shape
    (``n_points``, ``size``)

    Parameters
    ----------
    n_points: int
        The number of data points to track in the heap.

    size: int
        The number of items to keep on the heap for each data point.

    Returns
    -------
    heap: An ndarray suitable for passing to other numba enabled heap functions.
    """
    result = np.zeros((3, int(n_points), int(size)), dtype=np.float64)
    result[0] = -1
    result[1] = np.infty
    result[2] = 0

    return result

@numba.jit('i8(f8[:,:,:],i8,f8,i8,i8)')
def heap_push(heap, row, weight, index, flag):
    """Push a new element onto the heap. The heap stores potential neighbors
    for each data point. The ``row`` parameter determines which data point we
    are addressing, the ``weight`` determines the distance (for heap sorting),
    the ``index`` is the element to add, and the flag determines whether this
    is to be considered a new addition.

    Parameters
    ----------
    heap: ndarray generated by ``make_heap``
        The heap object to push into

    row: int
        Which actual heap within the heap object to push to

    weight: float
        The priority value of the element to push onto the heap

    index: int
        The actual value to be pushed

    flag: int
        Whether to flag the newly added element or not.

    Returns
    -------
    success: The number of new elements successfully pushed into the heap.
    """
    indices = heap[0, row]
    weights = heap[1, row]
    is_new = heap[2, row]

    if weight >= weights[0]:
        return 0

    # break if we already have this element.
    for i in range(indices.shape[0]):
        if index == indices[i]:
            return 0

    # insert val at position zero
    weights[0] = weight
    indices[0] = index
    is_new[0] = flag

    # descend the heap, swapping values until the max heap criterion is met
    i = 0
    while True:
        ic1 = 2 * i + 1
        ic2 = ic1 + 1

        if ic1 >= heap.shape[2]:
            break
        elif ic2 >= heap.shape[2]:
            if weights[ic1] > weight:
                i_swap = ic1
            else:
                break
        elif weights[ic1] >= weights[ic2]:
            if weight < weights[ic1]:
                i_swap = ic1
            else:
                break
        else:
            if weight < weights[ic2]:
                i_swap = ic2
            else:
                break

        weights[i] = weights[i_swap]
        indices[i] = indices[i_swap]
        is_new[i] = is_new[i_swap]

        i = i_swap

    weights[i] = weight
    indices[i] = index
    is_new[i] = flag

    return 1

@numba.jit('i8(f8[:,:,:],i8,f8,i8,i8)')
def unchecked_heap_push(heap, row, weight, index, flag):
    """Push a new element onto the heap. The heap stores potential neighbors
    for each data point. The ``row`` parameter determines which data point we
    are addressing, the ``weight`` determines the distance (for heap sorting),
    the ``index`` is the element to add, and the flag determines whether this
    is to be considered a new addition.

    Parameters
    ----------
    heap: ndarray generated by ``make_heap``
        The heap object to push into

    row: int
        Which actual heap within the heap object to push to

    weight: float
        The priority value of the element to push onto the heap

    index: int
        The actual value to be pushed

    flag: int
        Whether to flag the newly added element or not.

    Returns
    -------
    success: The number of new elements successfully pushed into the heap.
    """
    indices = heap[0, row]
    weights = heap[1, row]
    is_new = heap[2, row]

    if weight >= weights[0]:
        return 0

    # insert val at position zero
    weights[0] = weight
    indices[0] = index
    is_new[0] = flag

    # descend the heap, swapping values until the max heap criterion is met
    i = 0
    while True:
        ic1 = 2 * i + 1
        ic2 = ic1 + 1

        if ic1 >= heap.shape[2]:
            break
        elif ic2 >= heap.shape[2]:
            if weights[ic1] > weight:
                i_swap = ic1
            else:
                break
        elif weights[ic1] >= weights[ic2]:
            if weight < weights[ic1]:
                i_swap = ic1
            else:
                break
        else:
            if weight < weights[ic2]:
                i_swap = ic2
            else:
                break

        weights[i] = weights[i_swap]
        indices[i] = indices[i_swap]
        is_new[i] = is_new[i_swap]

        i = i_swap

    weights[i] = weight
    indices[i] = index
    is_new[i] = flag

    return 1

@numba.njit()
def siftdown(heap1, heap2, elt):
    """Restore the heap property for a heap with an out of place element
    at position ``elt``. This works with a heap pair where heap1 carries
    the weights and heap2 holds the corresponding elements."""
    while elt * 2 + 1 < heap1.shape[0]:
        left_child = elt * 2 + 1
        right_child = left_child + 1
        swap = elt

        if heap1[swap] < heap1[left_child]:
            swap = left_child

        if right_child < heap1.shape[0] and heap1[swap] < heap1[right_child]:
            swap = right_child

        if swap == elt:
            break
        else:
            heap1[elt], heap1[swap] = heap1[swap], heap1[elt]
            heap2[elt], heap2[swap] = heap2[swap], heap2[elt]
            elt = swap

@numba.njit()
def deheap_sort(heap):
    """Given an array of heaps (of indices and weights), unpack the heap
    out to give and array of sorted lists of indices and weights by increasing
    weight. This is effectively just the second half of heap sort (the first
    half not being required since we already have the data in a heap).

    Parameters
    ----------
    heap : array of shape (3, n_samples, n_neighbors)
        The heap to turn into sorted lists.

    Returns
    -------
    indices, weights: arrays of shape (n_samples, n_neighbors)
        The indices and weights sorted by increasing weight.
    """
    indices = heap[0]
    weights = heap[1]

    for i in range(indices.shape[0]):

        ind_heap = indices[i]
        dist_heap = weights[i]

        for j in range(ind_heap.shape[0] - 1):
            ind_heap[0], ind_heap[ind_heap.shape[0] - j - 1] = \
                ind_heap[ind_heap.shape[0] - j - 1], ind_heap[0]
            dist_heap[0], dist_heap[dist_heap.shape[0] - j - 1] = \
                dist_heap[dist_heap.shape[0] - j - 1], dist_heap[0]

            siftdown(dist_heap[:dist_heap.shape[0] - j - 1],
                     ind_heap[:ind_heap.shape[0] - j - 1], 0)

    return indices.astype(np.int64), weights

@numba.njit('i8(f8[:, :, :],i8)')
def smallest_flagged(heap, row):
    """Search the heap for the smallest element that is
    still flagged.

    Parameters
    ----------
    heap: array of shape (3, n_samples, n_neighbors)
        The heaps to search

    row: int
        Which of the heaps to search

    Returns
    -------
    index: int
        The index of the smallest flagged element
        of the ``row``th heap, or -1 if no flagged
        elements remain in the heap.
    """
    ind = heap[0, row]
    dist = heap[1, row]
    flag = heap[2, row]

    min_dist = np.inf
    result_index = -1

    for i in range(ind.shape[0]):
        if flag[i] == 1 and dist[i] < min_dist:
            min_dist = dist[i]
            result_index = i

    if result_index >= 0:
        flag[result_index] = 0.0
        return int(ind[result_index])
    else:
        return -1

@numba.njit(parallel=True)
def build_candidates(current_graph, n_vertices, n_neighbors, max_candidates,
                     rng_state, rho=0.5, seed_per_row=False):
    """Build a heap of candidate neighbors for nearest neighbor descent. For
    each vertex the candidate neighbors are any current neighbors, and any
    vertices that have the vertex as one of their nearest neighbors.

    Parameters
    ----------
    current_graph: heap
        The current state of the graph for nearest neighbor descent.

    n_vertices: int
        The total number of vertices in the graph.

    n_neighbors: int
        The number of neighbor edges per node in the current graph.

    max_candidates: int
        The maximum number of new candidate neighbors.

    rng_state: array of int64, shape (3,)
        The internal state of the rng

    Returns
    -------
    candidate_neighbors: A heap with an array of (randomly sorted) candidate
    neighbors for each vertex in the graph.
    """
    new_candidate_neighbors = make_heap(n_vertices, max_candidates)
    old_candidate_neighbors = make_heap(n_vertices, max_candidates)

    for i in range(n_vertices):
        if seed_per_row:
            seed(rng_state, i)
        for j in range(n_neighbors):
            if current_graph[0, i, j] < 0:
                continue
            idx = current_graph[0, i, j]
            isn = current_graph[2, i, j]
            d = tau_rand(rng_state)
            if tau_rand(rng_state) < rho:
                c = 0
                if isn:
                    c += heap_push(new_candidate_neighbors, i, d, idx, isn)
                    c += heap_push(new_candidate_neighbors, idx, d, i, isn)
                else:
                    heap_push(old_candidate_neighbors, i, d, idx, isn)
                    heap_push(old_candidate_neighbors, idx, d, i, isn)

                if c > 0 :
                    current_graph[2, i, j] = 0

    return new_candidate_neighbors, old_candidate_neighbors

<commit_msg>Ensure indexing by ints<commit_after># Author: Leland McInnes <leland.mcinnes@gmail.com>
#
# License: BSD 2 clause

import numba
import numpy as np

@numba.njit('void(i8[:], i8)')
def seed(rng_state, seed):
    """Seed the random number generator with a given seed."""
    rng_state.fill(seed + 0xffff)

@numba.njit('i4(i8[:])')
def tau_rand_int(state):
    """A fast (pseudo)-random number generator.

    Parameters
    ----------
    state: array of int64, shape (3,)
        The internal state of the rng

    Returns
    -------
    A (pseudo)-random int32 value
    """
    state[0] = (((state[0] & 4294967294) << 12) & 0xffffffff) ^ \
               ((((state[0] << 13) & 0xffffffff) ^ state[0]) >> 19)
    state[1] = (((state[1] & 4294967288) << 4) & 0xffffffff) ^ \
               ((((state[1] << 2) & 0xffffffff) ^ state[1]) >> 25)
    state[2] = (((state[2] & 4294967280) << 17) & 0xffffffff) ^ \
               ((((state[2] << 3) & 0xffffffff) ^ state[2]) >> 11)

    return state[0] ^ state[1] ^ state[2]

@numba.njit('f4(i8[:])')
def tau_rand(state):
    """A fast (pseudo)-random number generator for floats in the range [0,1]

    Parameters
    ----------
    state: array of int64, shape (3,)
        The internal state of the rng

    Returns
    -------
    A (pseudo)-random float32 in the interval [0, 1]
    """
    integer = tau_rand_int(state)
    return abs(float(integer) / 0x7fffffff)

@numba.njit(fastmath=True)
def norm(vec):
    """Compute the (standard l2) norm of a vector.

    Parameters
    ----------
    vec: array of shape (dim,)

    Returns
    -------
    The l2 norm of vec.
    """
    result = 0.0
    for i in range(vec.shape[0]):
        result += vec[i] ** 2
    return np.sqrt(result)

@numba.njit(parallel=True)
def rejection_sample(n_samples, pool_size, rng_state):
    """Generate n_samples many integers from 0 to pool_size such that no
    integer is selected twice. The duplication constraint is achieved via
    rejection sampling.

    Parameters
    ----------
    n_samples: int
        The number of random samples to select from the pool

    pool_size: int
        The size of the total pool of candidates to sample from

    rng_state: array of int64, shape (3,)
        Internal state of the random number generator

    Returns
    -------
    sample: array of shape(n_samples,)
        The ``n_samples`` randomly selected elements from the pool.
    """
    result = np.empty(n_samples, dtype=np.int64)
    for i in range(n_samples):
        reject_sample = True
        while reject_sample:
            j = tau_rand_int(rng_state) % pool_size
            for k in range(i):
                if j == result[k]:
                    break
            else:
                reject_sample = False
        result[i] = j
    return result

@numba.njit('f8[:, :, :](i8,i8)')
def make_heap(n_points, size):
    """Constructor for the numba enabled heap objects. The heaps are used
    for approximate nearest neighbor search, maintaining a list of potential
    neighbors sorted by their distance. We also flag if potential neighbors
    are newly added to the list or not. Internally this is stored as
    a single ndarray; the first axis determines whether we are looking at the
    array of candidate indices, the array of distances, or the flag array for
    whether elements are new or not. Each of these arrays are of shape
    (``n_points``, ``size``)

    Parameters
    ----------
    n_points: int
        The number of data points to track in the heap.

    size: int
        The number of items to keep on the heap for each data point.

    Returns
    -------
    heap: An ndarray suitable for passing to other numba enabled heap functions.
    """
    result = np.zeros((3, int(n_points), int(size)), dtype=np.float64)
    result[0] = -1
    result[1] = np.infty
    result[2] = 0

    return result

@numba.jit('i8(f8[:,:,:],i8,f8,i8,i8)')
def heap_push(heap, row, weight, index, flag):
    """Push a new element onto the heap. The heap stores potential neighbors
    for each data point. The ``row`` parameter determines which data point we
    are addressing, the ``weight`` determines the distance (for heap sorting),
    the ``index`` is the element to add, and the flag determines whether this
    is to be considered a new addition.

    Parameters
    ----------
    heap: ndarray generated by ``make_heap``
        The heap object to push into

    row: int
        Which actual heap within the heap object to push to

    weight: float
        The priority value of the element to push onto the heap

    index: int
        The actual value to be pushed

    flag: int
        Whether to flag the newly added element or not.

    Returns
    -------
    success: The number of new elements successfully pushed into the heap.
    """
    row = int(row)
    indices = heap[0, row]
    weights = heap[1, row]
    is_new = heap[2, row]

    if weight >= weights[0]:
        return 0

    # break if we already have this element.
    for i in range(indices.shape[0]):
        if index == indices[i]:
            return 0

    # insert val at position zero
    weights[0] = weight
    indices[0] = index
    is_new[0] = flag

    # descend the heap, swapping values until the max heap criterion is met
    i = 0
    while True:
        ic1 = 2 * i + 1
        ic2 = ic1 + 1

        if ic1 >= heap.shape[2]:
            break
        elif ic2 >= heap.shape[2]:
            if weights[ic1] > weight:
                i_swap = ic1
            else:
                break
        elif weights[ic1] >= weights[ic2]:
            if weight < weights[ic1]:
                i_swap = ic1
            else:
                break
        else:
            if weight < weights[ic2]:
                i_swap = ic2
            else:
                break

        weights[i] = weights[i_swap]
        indices[i] = indices[i_swap]
        is_new[i] = is_new[i_swap]

        i = i_swap

    weights[i] = weight
    indices[i] = index
    is_new[i] = flag

    return 1

@numba.jit('i8(f8[:,:,:],i8,f8,i8,i8)')
def unchecked_heap_push(heap, row, weight, index, flag):
    """Push a new element onto the heap. The heap stores potential neighbors
    for each data point. The ``row`` parameter determines which data point we
    are addressing, the ``weight`` determines the distance (for heap sorting),
    the ``index`` is the element to add, and the flag determines whether this
    is to be considered a new addition.

    Parameters
    ----------
    heap: ndarray generated by ``make_heap``
        The heap object to push into

    row: int
        Which actual heap within the heap object to push to

    weight: float
        The priority value of the element to push onto the heap

    index: int
        The actual value to be pushed

    flag: int
        Whether to flag the newly added element or not.

    Returns
    -------
    success: The number of new elements successfully pushed into the heap.
    """
    indices = heap[0, row]
    weights = heap[1, row]
    is_new = heap[2, row]

    if weight >= weights[0]:
        return 0

    # insert val at position zero
    weights[0] = weight
    indices[0] = index
    is_new[0] = flag

    # descend the heap, swapping values until the max heap criterion is met
    i = 0
    while True:
        ic1 = 2 * i + 1
        ic2 = ic1 + 1

        if ic1 >= heap.shape[2]:
            break
        elif ic2 >= heap.shape[2]:
            if weights[ic1] > weight:
                i_swap = ic1
            else:
                break
        elif weights[ic1] >= weights[ic2]:
            if weight < weights[ic1]:
                i_swap = ic1
            else:
                break
        else:
            if weight < weights[ic2]:
                i_swap = ic2
            else:
                break

        weights[i] = weights[i_swap]
        indices[i] = indices[i_swap]
        is_new[i] = is_new[i_swap]

        i = i_swap

    weights[i] = weight
    indices[i] = index
    is_new[i] = flag

    return 1

@numba.njit()
def siftdown(heap1, heap2, elt):
    """Restore the heap property for a heap with an out of place element
    at position ``elt``. This works with a heap pair where heap1 carries
    the weights and heap2 holds the corresponding elements."""
    while elt * 2 + 1 < heap1.shape[0]:
        left_child = elt * 2 + 1
        right_child = left_child + 1
        swap = elt

        if heap1[swap] < heap1[left_child]:
            swap = left_child

        if right_child < heap1.shape[0] and heap1[swap] < heap1[right_child]:
            swap = right_child

        if swap == elt:
            break
        else:
            heap1[elt], heap1[swap] = heap1[swap], heap1[elt]
            heap2[elt], heap2[swap] = heap2[swap], heap2[elt]
            elt = swap

@numba.njit()
def deheap_sort(heap):
    """Given an array of heaps (of indices and weights), unpack the heap
    out to give and array of sorted lists of indices and weights by increasing
    weight. This is effectively just the second half of heap sort (the first
    half not being required since we already have the data in a heap).

    Parameters
    ----------
    heap : array of shape (3, n_samples, n_neighbors)
        The heap to turn into sorted lists.

    Returns
    -------
    indices, weights: arrays of shape (n_samples, n_neighbors)
        The indices and weights sorted by increasing weight.
    """
    indices = heap[0]
    weights = heap[1]

    for i in range(indices.shape[0]):

        ind_heap = indices[i]
        dist_heap = weights[i]

        for j in range(ind_heap.shape[0] - 1):
            ind_heap[0], ind_heap[ind_heap.shape[0] - j - 1] = \
                ind_heap[ind_heap.shape[0] - j - 1], ind_heap[0]
            dist_heap[0], dist_heap[dist_heap.shape[0] - j - 1] = \
                dist_heap[dist_heap.shape[0] - j - 1], dist_heap[0]

            siftdown(dist_heap[:dist_heap.shape[0] - j - 1],
                     ind_heap[:ind_heap.shape[0] - j - 1], 0)

    return indices.astype(np.int64), weights

@numba.njit('i8(f8[:, :, :],i8)')
def smallest_flagged(heap, row):
    """Search the heap for the smallest element that is
    still flagged.

    Parameters
    ----------
    heap: array of shape (3, n_samples, n_neighbors)
        The heaps to search

    row: int
        Which of the heaps to search

    Returns
    -------
    index: int
        The index of the smallest flagged element
        of the ``row``th heap, or -1 if no flagged
        elements remain in the heap.
    """
    ind = heap[0, row]
    dist = heap[1, row]
    flag = heap[2, row]

    min_dist = np.inf
    result_index = -1

    for i in range(ind.shape[0]):
        if flag[i] == 1 and dist[i] < min_dist:
            min_dist = dist[i]
            result_index = i

    if result_index >= 0:
        flag[result_index] = 0.0
        return int(ind[result_index])
    else:
        return -1

@numba.njit(parallel=True)
def build_candidates(current_graph, n_vertices, n_neighbors, max_candidates,
                     rng_state, rho=0.5, seed_per_row=False):
    """Build a heap of candidate neighbors for nearest neighbor descent. For
    each vertex the candidate neighbors are any current neighbors, and any
    vertices that have the vertex as one of their nearest neighbors.

    Parameters
    ----------
    current_graph: heap
        The current state of the graph for nearest neighbor descent.

    n_vertices: int
        The total number of vertices in the graph.

    n_neighbors: int
        The number of neighbor edges per node in the current graph.

    max_candidates: int
        The maximum number of new candidate neighbors.

    rng_state: array of int64, shape (3,)
        The internal state of the rng

    Returns
    -------
    candidate_neighbors: A heap with an array of (randomly sorted) candidate
    neighbors for each vertex in the graph.
    """
    new_candidate_neighbors = make_heap(n_vertices, max_candidates)
    old_candidate_neighbors = make_heap(n_vertices, max_candidates)

    for i in range(n_vertices):
        if seed_per_row:
            seed(rng_state, i)
        for j in range(n_neighbors):
            if current_graph[0, i, j] < 0:
                continue
            idx = current_graph[0, i, j]
            isn = current_graph[2, i, j]
            d = tau_rand(rng_state)
            if tau_rand(rng_state) < rho:
                c = 0
                if isn:
                    c += heap_push(new_candidate_neighbors, i, d, idx, isn)
                    c += heap_push(new_candidate_neighbors, idx, d, i, isn)
                else:
                    heap_push(old_candidate_neighbors, i, d, idx, isn)
                    heap_push(old_candidate_neighbors, idx, d, i, isn)

                if c > 0 :
                    current_graph[2, i, j] = 0

    return new_candidate_neighbors, old_candidate_neighbors

<|endoftext|>


<commit_before>"""Main entry point for untz."""
from concurrent.futures import ThreadPoolExecutor
import logging
import os

from .encoder import apply_gain, encode_file
from .utils import get_args, preflight_checks, recursive_file_search

ARGS = get_args()
LOGGER = logging.getLogger(__name__)

def _encode_on_filter(file_entry):
    LOGGER.info('Encoding "%s"', file_entry)

    if file_entry.lower().endswith('.flac'):
        encode_file(file_entry,
                    ARGS.base_dir.rstrip('/'),
                    ARGS.pattern,
                    ARGS.quality,
                    ARGS.ogg_cli_parameters)

def main():
    """Main logic for untz."""
    if ARGS.verbose:
        logging.basicConfig(level=logging.DEBUG)

    preflight_checks()

    LOGGER.info('Starting %d threads.', ARGS.threads)
    with ThreadPoolExecutor(max_workers=ARGS.threads) as tpe:
        for i in ARGS.inputs:
            if os.path.isdir(i):
                tpe.map(_encode_on_filter, recursive_file_search(i))
            else:
                tpe.submit(_encode_on_filter, i)

    if ARGS.replaygain:
        apply_gain(ARGS.base_dir)

    LOGGER.info('Program exiting now.')
<commit_msg>Move logging message<commit_after>"""Main entry point for untz."""
from concurrent.futures import ThreadPoolExecutor
import logging
import os

from .encoder import apply_gain, encode_file
from .utils import get_args, preflight_checks, recursive_file_search

ARGS = get_args()
LOGGER = logging.getLogger(__name__)

def _encode_on_filter(file_entry):
    if file_entry.lower().endswith('.flac'):
        LOGGER.info('Encoding "%s"', file_entry)
        encode_file(file_entry,
                    ARGS.base_dir.rstrip('/'),
                    ARGS.pattern,
                    ARGS.quality,
                    ARGS.ogg_cli_parameters)

def main():
    """Main logic for untz."""
    if ARGS.verbose:
        logging.basicConfig(level=logging.DEBUG)

    preflight_checks()

    LOGGER.info('Starting %d threads.', ARGS.threads)
    with ThreadPoolExecutor(max_workers=ARGS.threads) as tpe:
        for i in ARGS.inputs:
            if os.path.isdir(i):
                tpe.map(_encode_on_filter, recursive_file_search(i))
            else:
                tpe.submit(_encode_on_filter, i)

    if ARGS.replaygain:
        apply_gain(ARGS.base_dir)

    LOGGER.info('Program exiting now.')
<|endoftext|>


<commit_before>import re

from datetime import datetime

import arrow
import requests

from bs4 import BeautifulSoup
import jsondate as json
from twython import Twython

import settings

TFL_NAME_CORRECTIONS = {
    'King\'s Cross': 'King\'s Cross St. Pancras',
    'Cutty Sark': 'Cutty Sark for Maritime Greenwich'
}

# Getter and Creator for Station List
def create_station_list():
    # TODO: Save this and replace once a day
    station_status_URI = "http://cloud.tfl.gov.uk/TrackerNet/StationStatus"

    r = requests.get(station_status_URI)
    xml = r.text
    # CHECK RESPONSE CODE 200!
    soup = BeautifulSoup(xml)

    return list(reversed(sorted([station['name'] for station in soup.find_all('station')], key=len)))

def get_station_list():

    station_list = create_station_list()

    return station_list

def find_station_name(possible_name):

    # Correlate a name from a tweet/trackernet with one we expect to find
    for station_name in get_station_list():
        if station_name.lower() in possible_name.lower():
            return station_name

    # If we don't find it in the proper list, we have to look in our corrections list
    for station_name in TFL_NAME_CORRECTIONS.keys():
        if station_name.lower() in possible_name.lower():
            return TFL_NAME_CORRECTIONS[station_name]

    print station_name, 'not found'

def remove_tfl_specifics(text):
    # We always reset this just in case there is an update
    text = re.sub('(Please )?[Cc]all.*0[38]43 ?222 ?1234.*journey\.?', '', text)
    text = re.sub('we ', 'TfL ', text, re.IGNORECASE)
    text = re.sub('member of staff', 'member of TfL staff', text, re.IGNORECASE)

    return text

def fix_additional_info_grammar(text):
    # It appears that full stops and spaces aren't always used...
    text = re.sub(r'\.([A-Z<])', r'. \1', text)
    text = re.sub(r'[^ \.](Please allow extra time for your journey.)', r'. \1', text)

    return text

def parse_date(text):
    possible_formats = [
        'D MMMM YYYY', 'D MMMM', 'MMMM YYYY', 'MMMM'
    ]

    try:
        date_ = arrow.get(text, possible_formats)
        if 'early' in text:
            date_ = date_.replace(day=10)
        if 'mid' in text:
            date_ = date_.replace(day=20)
        if 'late' in text:
            date_ = date_.ceil('month')

        if date_.year == 1:
            date_ = date_.replace(year=arrow.utcnow().year)

        date_ = date_.naive

    except:
        date_ = None

    return date_

def find_dates(text):
    start_and_end = re.search('From (.*?) until (.*?)(,|there|due)', text, flags=re.I)

    start_date = None
    end_date = None

    if start_and_end:
        start_date = parse_date(start_and_end.groups()[0])
        end_date = parse_date(start_and_end.groups()[1])

        while start_date and end_date and end_date < start_date:
            start_date = start_date.replace(year=start_date.year - 1)

    return start_date, end_date

def get_problem_stations(problems_dict):
    problems = []

    for problem in problems_dict:
        if problem and problem[0:1] != '_':
            if not problems_dict[problem]['resolved'] and \
                    not problems_dict[problem]['information']:
                problems.append(problem)

    return sorted(problems)

def send_tweet(tweet_text):
    """ Send a tweet """
    if settings.PRODUCTION:
        try:
            twitter = Twython(
                settings.settings.TWITTER_KEY, settings.settings.TWITTER_SECRET,
                settings.TUBELIFTS_OAUTH_TOKEN, settings.TUBELIFTS_OAUTH_TOKEN_SECRET
            )

            twitter.update_status(status=tweet_text)
        # Except everything. TODO: Look into some of twitters annoying foibles
        except:
            pass
    else:
        print "Should have tweeted: " + tweet_text

def get_problems_dict():
    try:
        with open(settings.TEMPLATE_FILE_LOCATION + 'problems.json', 'r') as f:
            problems_dict = json.loads(f.read())
            if '_last_updated' in problems_dict:
                del problems_dict['_last_updated']
    except IOError:
        with open(settings.TEMPLATE_FILE_LOCATION + 'problems.json', 'w') as f:
            problems_dict = {}
            f.write(json.dumps(problems_dict))

    return problems_dict

def set_problems_dict(problems):

    problems['_last_updated'] = datetime.now()

    with open(settings.TEMPLATE_FILE_LOCATION + 'problems.json', 'w') as f:
        f.write(json.dumps(problems))

    return True
<commit_msg>Double settings<commit_after>import re

from datetime import datetime

import arrow
import requests

from bs4 import BeautifulSoup
import jsondate as json
from twython import Twython

import settings

TFL_NAME_CORRECTIONS = {
    'King\'s Cross': 'King\'s Cross St. Pancras',
    'Cutty Sark': 'Cutty Sark for Maritime Greenwich'
}

# Getter and Creator for Station List
def create_station_list():
    # TODO: Save this and replace once a day
    station_status_URI = "http://cloud.tfl.gov.uk/TrackerNet/StationStatus"

    r = requests.get(station_status_URI)
    xml = r.text
    # CHECK RESPONSE CODE 200!
    soup = BeautifulSoup(xml)

    return list(reversed(sorted([station['name'] for station in soup.find_all('station')], key=len)))

def get_station_list():

    station_list = create_station_list()

    return station_list

def find_station_name(possible_name):

    # Correlate a name from a tweet/trackernet with one we expect to find
    for station_name in get_station_list():
        if station_name.lower() in possible_name.lower():
            return station_name

    # If we don't find it in the proper list, we have to look in our corrections list
    for station_name in TFL_NAME_CORRECTIONS.keys():
        if station_name.lower() in possible_name.lower():
            return TFL_NAME_CORRECTIONS[station_name]

    print station_name, 'not found'

def remove_tfl_specifics(text):
    # We always reset this just in case there is an update
    text = re.sub('(Please )?[Cc]all.*0[38]43 ?222 ?1234.*journey\.?', '', text)
    text = re.sub('we ', 'TfL ', text, re.IGNORECASE)
    text = re.sub('member of staff', 'member of TfL staff', text, re.IGNORECASE)

    return text

def fix_additional_info_grammar(text):
    # It appears that full stops and spaces aren't always used...
    text = re.sub(r'\.([A-Z<])', r'. \1', text)
    text = re.sub(r'[^ \.](Please allow extra time for your journey.)', r'. \1', text)

    return text

def parse_date(text):
    possible_formats = [
        'D MMMM YYYY', 'D MMMM', 'MMMM YYYY', 'MMMM'
    ]

    try:
        date_ = arrow.get(text, possible_formats)
        if 'early' in text:
            date_ = date_.replace(day=10)
        if 'mid' in text:
            date_ = date_.replace(day=20)
        if 'late' in text:
            date_ = date_.ceil('month')

        if date_.year == 1:
            date_ = date_.replace(year=arrow.utcnow().year)

        date_ = date_.naive

    except:
        date_ = None

    return date_

def find_dates(text):
    start_and_end = re.search('From (.*?) until (.*?)(,|there|due)', text, flags=re.I)

    start_date = None
    end_date = None

    if start_and_end:
        start_date = parse_date(start_and_end.groups()[0])
        end_date = parse_date(start_and_end.groups()[1])

        while start_date and end_date and end_date < start_date:
            start_date = start_date.replace(year=start_date.year - 1)

    return start_date, end_date

def get_problem_stations(problems_dict):
    problems = []

    for problem in problems_dict:
        if problem and problem[0:1] != '_':
            if not problems_dict[problem]['resolved'] and \
                    not problems_dict[problem]['information']:
                problems.append(problem)

    return sorted(problems)

def send_tweet(tweet_text):
    """ Send a tweet """
    if settings.PRODUCTION:
        try:
            twitter = Twython(
                settings.TWITTER_KEY, settings.TWITTER_SECRET,
                settings.TUBELIFTS_OAUTH_TOKEN, settings.TUBELIFTS_OAUTH_TOKEN_SECRET
            )

            twitter.update_status(status=tweet_text)
        # Except everything. TODO: Look into some of twitters annoying foibles
        except:
            pass
    else:
        print "Should have tweeted: " + tweet_text

def get_problems_dict():
    try:
        with open(settings.TEMPLATE_FILE_LOCATION + 'problems.json', 'r') as f:
            problems_dict = json.loads(f.read())
            if '_last_updated' in problems_dict:
                del problems_dict['_last_updated']
    except IOError:
        with open(settings.TEMPLATE_FILE_LOCATION + 'problems.json', 'w') as f:
            problems_dict = {}
            f.write(json.dumps(problems_dict))

    return problems_dict

def set_problems_dict(problems):

    problems['_last_updated'] = datetime.now()

    with open(settings.TEMPLATE_FILE_LOCATION + 'problems.json', 'w') as f:
        f.write(json.dumps(problems))

    return True
<|endoftext|>


<commit_before># hashrng.py
# created: 6/28/2016
# Contributors: Matthew O'Brien
# Initial MD5 Hashfunction ideas for RNG
# -------------------------------------------------------------------------
# Initial file for use of the MD5 hash function to generate a random number
# from a given input.
# Currently reading in a nonsense string to output a hex number. Which is
# converted into a decimal number.

import hashlib
import math
import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
from scipy import stats

def create_test_list():
    bit_list = []
    for i in range(1028):
        if i % 3 == 0:
            bit_list.append(1)
        else:
            bit_list.append(0)

    return bit_list

# Calculate the distributions of 1s and 0s in total string.
def dist_calculations(bit_list):

    # For testing of the distribution. Comment out later
    bit_string = int("".join(str(x) for x in bit_list))
    bit_string = str(bit_string)
    one_count = bit_string.count('1')
    zero_count = bit_string.count('0')
    total_length = len(bit_string)

    percent_one = one_count / float(total_length)
    percent_zero = zero_count / float(total_length)

    return percent_one, percent_zero

# Entropy calculations
def entropy_calculations(percent_one, percent_zero):

    # calculating the bits of entropy
    entropy = (-percent_one*math.log(percent_one,2))+(-percent_zero*math.log(percent_zero,2))
    # entropy = stats.entropy(percent_one) + stats.entropy(percent_zero)

    return entropy

# Find the corrected length of bits given entropy calculations
def entropy_correction(entropy):
    corrected_bits = math.ceil(128 * entropy)
    corrected_bits = int(corrected_bits)
    corrected_bits = 128 + (128-corrected_bits)
    return corrected_bits

# Grab the needed amounts of bits to ensure 128 bits of entropy
def alter_bit_length(bit_list, corrected_bits):
    new_bit_list = []
    for i in range(corrected_bits):
        new_bit_list.append(bit_list.pop())
    return new_bit_list

# Whitener for the 128 bit list to generate a random number
def whiten_numbers(min_value,max_value, bit_list):

    # Use SHA1 to hash the string.
    bit_string = int("".join(str(x) for x in bit_list))
    bit_string = str(bit_string)
    hash_number = hashlib.sha1(bit_string.encode('utf-8')).hexdigest()
    hash_number = int(hash_number,32)
    random_number = hash_number % max_value
    return random_number

# Main function for testing
def main():
    # Make the testing list
    bit_list = create_test_list()

    # Calculate the percents of 1s and 0s of test list
    percent_one, percent_zero = dist_calculations(bit_list)

    # Calculate Entropy
    entropy = entropy_calculations(percent_one,percent_zero)

    # Find the corrected bits we need for 128 bits of entropy.
    corrected_bits = entropy_correction(entropy)
    new_bit_list = alter_bit_length(bit_list,corrected_bits)

    # Whiten the number and set parameters for the number.
    random_number = whiten_numbers(0,100,new_bit_list)

    # Reporting all the information
    print(len(new_bit_list))
    print "Percent of 1s:", percent_one
    print "Percent of 0s:", percent_zero
    print "Bits of Entropy:", entropy
    print "Random number is:", random_number

main()
<commit_msg>Added XOR and changed to numpy<commit_after># hashrng.py
# created: 6/28/2016
# Contributors: Matthew O'Brien
# Initial MD5 Hashfunction ideas for RNG
# -------------------------------------------------------------------------
# Initial file for use of the MD5 hash function to generate a random number
# from a given input.
# Currently reading in a nonsense string to output a hex number. Which is
# converted into a decimal number.

import hashlib
import math
import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
from scipy import stats

def create_test_list(num):
    bit_list = np.zeros(1028, dtype=int)
    for i in range(len(bit_list)):
        if i % num == 0:
            bit_list[i] = 1

    return bit_list

# Stir functions
def stir_pool(entropy_one, entropy_two):
    new_entropy_pool = np.logical_xor(entropy_one,entropy_two)

    return new_entropy_pool

# Calculate the distributions of 1s and 0s in total string.
def dist_calculations(bit_list):

    one_count = np.count_nonzero(bit_list)
    total_length = len(bit_list)
    zero_count = total_length - one_count

    percent_one = one_count / float(total_length)
    percent_zero = zero_count / float(total_length)

    return percent_one, percent_zero

# Entropy calculations
def entropy_calculations(percent_one, percent_zero):

    # calculating the bits of entropy
    entropy = (-percent_one*math.log(percent_one,2))+(-percent_zero*math.log(percent_zero,2))
    # entropy = stats.entropy(percent_one) + stats.entropy(percent_zero)

    return entropy

# Find the corrected length of bits given entropy calculations
def entropy_correction(entropy):
    corrected_bits = math.ceil(128 * entropy)
    corrected_bits = int(corrected_bits)
    corrected_bits = 128 + (128-corrected_bits)
    return corrected_bits

# Grab the needed amounts of bits to ensure 128 bits of entropy
def alter_bit_length(bit_list, corrected_bits):
    new_bit_list = np.zeros(len(bit_list))
    for i in range(corrected_bits):
        new_bit_list[i] = bit_list[i]
    return new_bit_list

# Whitener for the 128 bit list to generate a random number
def whiten_numbers(min_value,max_value, bit_list):

    # Use SHA1 to hash the string.
    #bit_string = int("".join(str(x) for x in bit_list))
    #bit_string = str(bit_string)
    bit_string = np.array2string(bit_list)
    hash_number = hashlib.sha1(bit_string.encode('utf-8')).hexdigest()
    hash_number = int(hash_number,32)
    random_number = hash_number % max_value
    return random_number

# Main function for testing
def main():
    # Make the testing list
    bit_list = create_test_list(3)
    second_bit_list = create_test_list(2)

    # Calculate the percents of 1s and 0s of test list
    percent_one, percent_zero = dist_calculations(bit_list)

    # Calculate Entropy
    entropy = entropy_calculations(percent_one,percent_zero)

    # Find the corrected bits we need for 128 bits of entropy.
    corrected_bits = entropy_correction(entropy)
    new_bit_list = alter_bit_length(bit_list,corrected_bits)

    # Whiten the number and set parameters for the number.
    random_number = whiten_numbers(0,100,new_bit_list)

    # Reporting all the information
    print(len(new_bit_list))
    print "Percent of 1s:", percent_one
    print "Percent of 0s:", percent_zero
    print "Bits of Entropy:", entropy
    print "Random number is:", random_number

    new_entropy_pool = stir_pool(new_bit_list,second_bit_list)
    new_entropy_pool = stir_pool(new_entropy_pool, bit_list)

    new_random_number = whiten_numbers(0,100,new_entropy_pool)

    print new_random_number

main()
<|endoftext|>


<commit_before>
import websocket
from fee import Fee

import thread
import simplejson as json
import threading

from aplus import Promise

#-------------------------------------------------------------------------------

def _handle_error(self, msg_json):

	tid = msg_json['id']
	if tid in self.requests:

		if 'error_message' in msg_json:
			error_msg = msg_json['error_message']
		else:
			error_msg = msg_json['error']

		request = self.requests[tid]
		promise = request['promise']
		command = request['command']
		del self.requests[tid]
		del self.promises[command]

		promise.reject(Exception(error_msg))

		if msg_json['error'] == 'noNetwork':
			return True
		else:
			return False

def _handle_find_path(self, msg_json):
	self.path_callback(msg_json)
	return True

def _handle_ledger_closed(self, msg_json):

	self.fee.set_fee_scale(msg_json)
	p = Promise()
	p.fulfill(self.fee.calculate_fee())
	self.fee_promise = p
	return True

# These commands always return success, no matter if the stellard
# is synchronized or not, so don't set status to synced when one
# of these occur.
_skiplist = {'ping', 'subscribe', 'unsubscribe'}

def _handle_response(self, msg_json):
	""" Handle messages that arrive as a direct response to a sent command """

	tid = msg_json['id']
	if tid in self.requests:
		request = self.requests[tid]
		promise = request['promise']
		command_key = request['command']

		command_dict = json.loads(command_key)
		command = command_dict['command']

		del self.requests[tid]
		del self.promises[command_key]

		promise.fulfill(msg_json)
		return command not in _skiplist

	else:
		return True

def _handle_server_status(self, msg_json):

	self.fee.set_load_scale(msg_json)
	p = Promise()
	p.fulfill(self.fee.calculate_fee())
	self.fee_promise = p
	return True

def _handle_transaction(self, msg_json):

	if msg_json['status'] != 'closed':
		return True

	if msg_json['engine_result_code'] != 0:
		return True

	tx_type = msg_json['transaction']['TransactionType']
	if tx_type in self.tx_callbacks:
		for callback in self.tx_callbacks[tx_type]:
			callback(msg_json)

	return True

_msg_handlers = {
	'find_path':	_handle_find_path,			# stellar
	'path_find':	_handle_find_path,			# ripple
	'ledgerClosed':	_handle_ledger_closed,
	'response':		_handle_response,
	'serverStatus':	_handle_server_status,
	'transaction':	_handle_transaction,
}

def _on_message(self, message):

	self.event.set()
	msg_json = json.loads(message)

	res = False
	if 'error' in msg_json:
		res = _handle_error(self, msg_json)
		status = False

	else:
		msg_type = msg_json['type']
		if msg_type in _msg_handlers:
			res = _msg_handlers[msg_type](self, msg_json)
			status = True
	#	else:
	#		print msg_type

	if res:
		self._set_sync_status(status)

def _on_error(self, error):
#	print error
	pass

def _on_close(self):
	print "websocket closed"
#	pass

def _on_open(self):
	self.is_open = True
	for tx in self.queue:
		self.send(tx)

class Server(websocket.WebSocketApp):

	def __init__(self, url, callback):
		super(Server, self).__init__(
			url,
			on_open		= _on_open,
			on_message	= _on_message,
			on_error	= _on_error,
			on_close	= _on_close,
		)

		self.fee = Fee()
		self.queue = []
		self.is_open = False

		self.requests = {}
		self.promises = {}
		self.last_id = -1

		self.set_path_callback(None)
		self.set_sync_callback(None)
		self.sync_flag = None

		self.tx_callbacks = {
			'AccountMerge':		[],
			'AccountSet':		[],
			'OfferCancel':		[],
			'OfferCreate':		[],
			'Payment': 			[],
			'SetRegularKey':	[],
			'TrustSet':			[]
		}

		self.__clear_subscriptions()
		self.subscribe_fee()

		self.event = threading.Event()
		self.event.clear()

	def set_path_callback(self, callback):
		self.path_callback = callback if callback else lambda x: None

	def set_sync_callback(self, callback):
		self.sync_callback = callback if callback else lambda x: None

	def _set_sync_status(self, flag):

		if flag != self.sync_flag:
			self.sync_flag = flag
			self.sync_callback(flag)

	def __get_id(self):
		self.last_id +=1
		return self.last_id

	def cancel(self, promise):

		for tid, req in self.requests.items():
			if req['promise'] == promise:
				command_key = req['command']
				del self.requests[tid]
				del self.promises[command_key]

	def __send_request(self, command_key, **kwargs):
		tid = self.__get_id()
		kwargs['id'] = tid
		js = json.dumps(kwargs)

		if self.is_open:
			self.send(js)
		else:
			self.queue.append(js)

		p = Promise()
		request = {
			'command':	command_key,
			'promise':	p
		}

		self.requests[tid] = request
		self.promises[command_key] = p
		return p

	def request(self, command, **kwargs):

		kwargs['command'] = command
		command_key = json.dumps(kwargs, sort_keys=True)
		if command_key not in self.promises:
			return self.__send_request(command_key, **kwargs)
		else:
			return self.promises[command_key]

	def run(self):

		def thread_target():

			while True:
				self.__start_ping_thread()
				self.run_forever()
				self.__stop_ping_thread()
				self.__resubscribe()

		thread.start_new_thread(thread_target, ())

	#
	#	fees
	#

	def subscribe_fee(self):
		""" Start subscribing to network fee updates """
		p = self.subscribe(streams=['ledger', 'server'])\
			.then(self.fee.set_initial_fee)
		self.fee_promise = p

	#
	#	subscription management
	#

	def add_callback(self, tx_type, callback):
		self.tx_callbacks[tx_type].append(callback)

	def __clear_subscriptions(self):

		self.subscriptions = {
			'streams':		[],
			'accounts':		[],
			'accounts_rt':	[],
			'books':		[]
		}

	def __resubscribe(self):

		kwargs = {}
		for key, value in self.subscriptions.iteritems():
			if value:
				kwargs[key] = [
					json.loads(item)
					for item in value
				]

		self.request('subscribe', **kwargs)

	def subscribe(self, **kwargs):

		for key, value in kwargs.iteritems():

			value = [
				json.dumps(item, sort_keys=True)
				for item in value
			]

			t = self.subscriptions[key]
			t = list(set(t).union(value))
			self.subscriptions[key] = t

		return self.request('subscribe', **kwargs)

	def unsubscribe(self, **kwargs):

		for key, value in kwargs.iteritems():

			value = [
				json.dumps(item, sort_keys=True)
				for item in value
			]

			t = self.subscriptions[key]
			t = list(set(t).difference(value))
			self.subscriptions[key] = t

		return self.request('unsubscribe', **kwargs)

	#
	#	ping thread management
	#

	def __ping_thread_target(self):
		while not self.end_ping_thread:
			if not self.event.wait(30):
				self.request('ping')
			self.event.clear()

	def __start_ping_thread(self):
		self.event = threading.Event()
		self.end_ping_thread = False
		self.ping_thread = threading.Thread(target=self.__ping_thread_target)
		self.ping_thread.setDaemon(True)
		self.ping_thread.start()

	def __stop_ping_thread(self):
		self.end_ping_thread = True
		self.event.set()
		self.ping_thread.join()
<commit_msg>- Changed the web socket from a super class to a member, for easier restarting<commit_after>
import websocket
from fee import Fee

import thread
import simplejson as json
import threading

from aplus import Promise

#-------------------------------------------------------------------------------

def _handle_error(self, msg_json):

	tid = msg_json['id']
	if tid in self.requests:

		if 'error_message' in msg_json:
			error_msg = msg_json['error_message']
		else:
			error_msg = msg_json['error']

		request = self.requests[tid]
		promise = request['promise']
		command = request['command']
		del self.requests[tid]
		del self.promises[command]

		promise.reject(Exception(error_msg))

		if msg_json['error'] == 'noNetwork':
			return True
		else:
			return False

def _handle_find_path(self, msg_json):
	self.path_callback(msg_json)
	return True

def _handle_ledger_closed(self, msg_json):

	self.fee.set_fee_scale(msg_json)
	p = Promise()
	p.fulfill(self.fee.calculate_fee())
	self.fee_promise = p
	return True

# These commands always return success, no matter if the stellard
# is synchronized or not, so don't set status to synced when one
# of these occur.
_skiplist = {'ping', 'subscribe', 'unsubscribe'}

def _handle_response(self, msg_json):
	""" Handle messages that arrive as a direct response to a sent command """

	tid = msg_json['id']
	if tid in self.requests:
		request = self.requests[tid]
		promise = request['promise']
		command_key = request['command']

		command_dict = json.loads(command_key)
		command = command_dict['command']

		del self.requests[tid]
		del self.promises[command_key]

		promise.fulfill(msg_json)
		return command not in _skiplist

	else:
		return True

def _handle_server_status(self, msg_json):

	self.fee.set_load_scale(msg_json)
	p = Promise()
	p.fulfill(self.fee.calculate_fee())
	self.fee_promise = p
	return True

def _handle_transaction(self, msg_json):

	if msg_json['status'] != 'closed':
		return True

	if msg_json['engine_result_code'] != 0:
		return True

	tx_type = msg_json['transaction']['TransactionType']
	if tx_type in self.tx_callbacks:
		for callback in self.tx_callbacks[tx_type]:
			callback(msg_json)

	return True

_msg_handlers = {
	'find_path':	_handle_find_path,			# stellar
	'path_find':	_handle_find_path,			# ripple
	'ledgerClosed':	_handle_ledger_closed,
	'response':		_handle_response,
	'serverStatus':	_handle_server_status,
	'transaction':	_handle_transaction,
}

def _on_message(ws, message):

	ws.self.event.set()
	msg_json = json.loads(message)

	res = False
	if 'error' in msg_json:
		res = _handle_error(ws.self, msg_json)
		status = False

	else:
		msg_type = msg_json['type']
		if msg_type in _msg_handlers:
			res = _msg_handlers[msg_type](ws.self, msg_json)
			status = True
	#	else:
	#		print msg_type

	if res:
		ws.self._set_sync_status(status)

def _on_error(ws, error):
#	print error
	pass

def _on_close(ws):
	print "websocket closed"

def _on_open(ws):
	ws.self.is_open = True
	for tx in ws.self.queue:
		ws.send(tx)

class Server(object):

	def __open_websocket(self):

		print "<open_websocket>"
		self.is_open = False
		self.ws = websocket.WebSocketApp(
			self.url,
			on_open		= _on_open,
			on_message	= _on_message,
			on_error	= _on_error,
			on_close	= _on_close,
		)
		self.ws.self = self

	def send(self, data):
		self.ws.send(data)

	def __init__(self, url, callback):

		self.url = url
		self.__open_websocket()

		self.fee = Fee()
		self.queue = []

		self.requests = {}
		self.promises = {}
		self.last_id = -1

		self.set_path_callback(None)
		self.set_sync_callback(None)
		self.sync_flag = None

		self.tx_callbacks = {
			'AccountMerge':		[],
			'AccountSet':		[],
			'OfferCancel':		[],
			'OfferCreate':		[],
			'Payment': 			[],
			'SetRegularKey':	[],
			'TrustSet':			[]
		}

		self.__clear_subscriptions()
		self.subscribe_fee()

		self.event = threading.Event()
		self.event.clear()

	def set_path_callback(self, callback):
		self.path_callback = callback if callback else lambda x: None

	def set_sync_callback(self, callback):
		self.sync_callback = callback if callback else lambda x: None

	def _set_sync_status(self, flag):

		if flag != self.sync_flag:
			self.sync_flag = flag
			self.sync_callback(flag)

	def __get_id(self):
		self.last_id +=1
		return self.last_id

	def cancel(self, promise):

		for tid, req in self.requests.items():
			if req['promise'] == promise:
				command_key = req['command']
				del self.requests[tid]
				del self.promises[command_key]

	def __send_request(self, command_key, **kwargs):
		tid = self.__get_id()
		kwargs['id'] = tid
		js = json.dumps(kwargs)

		if self.is_open:
			self.send(js)
		else:
			self.queue.append(js)

		p = Promise()
		request = {
			'command':	command_key,
			'promise':	p
		}

		self.requests[tid] = request
		self.promises[command_key] = p
		return p

	def request(self, command, **kwargs):

		kwargs['command'] = command
		command_key = json.dumps(kwargs, sort_keys=True)
		if command_key not in self.promises:
			return self.__send_request(command_key, **kwargs)
		else:
			return self.promises[command_key]

	def run(self):

		def thread_target():

			while True:
				self.__start_ping_thread()
				self.ws.run_forever()
				self.__stop_ping_thread()
				self.__open_websocket()
				self.__resubscribe()

		thread.start_new_thread(thread_target, ())

	#
	#	fees
	#

	def subscribe_fee(self):
		""" Start subscribing to network fee updates """
		p = self.subscribe(streams=['ledger', 'server'])\
			.then(self.fee.set_initial_fee)
		self.fee_promise = p

	#
	#	subscription management
	#

	def add_callback(self, tx_type, callback):
		self.tx_callbacks[tx_type].append(callback)

	def __clear_subscriptions(self):

		self.subscriptions = {
			'streams':		[],
			'accounts':		[],
			'accounts_rt':	[],
			'books':		[]
		}

	def __resubscribe(self):

		kwargs = {}
		for key, value in self.subscriptions.iteritems():
			if value:
				kwargs[key] = [
					json.loads(item)
					for item in value
				]

		self.request('subscribe', **kwargs)

	def subscribe(self, **kwargs):

		for key, value in kwargs.iteritems():

			value = [
				json.dumps(item, sort_keys=True)
				for item in value
			]

			t = self.subscriptions[key]
			t = list(set(t).union(value))
			self.subscriptions[key] = t

		return self.request('subscribe', **kwargs)

	def unsubscribe(self, **kwargs):

		for key, value in kwargs.iteritems():

			value = [
				json.dumps(item, sort_keys=True)
				for item in value
			]

			t = self.subscriptions[key]
			t = list(set(t).difference(value))
			self.subscriptions[key] = t

		return self.request('unsubscribe', **kwargs)

	#
	#	ping thread management
	#

	def __ping_thread_target(self):
		while not self.end_ping_thread:
			if not self.event.wait(30):
				self.request('ping')
			self.event.clear()

	def __start_ping_thread(self):
		self.event = threading.Event()
		self.end_ping_thread = False
		self.ping_thread = threading.Thread(target=self.__ping_thread_target)
		self.ping_thread.setDaemon(True)
		self.ping_thread.start()

	def __stop_ping_thread(self):
		self.end_ping_thread = True
		self.event.set()
		self.ping_thread.join()
<|endoftext|>


<commit_before>"""Main entry point for untz."""
import argparse
import logging
import multiprocessing
import os

from .encoder import apply_gain, encode_file
from .utils import preflight_checks, recursive_file_search

LOGGER = logging.getLogger(__name__)

def get_args():
    """Parse and return arguments."""
    parser = argparse.ArgumentParser(description='Convert FLACs to OGGs and '
                                                 'sort into a sensible folder hierarchy.')
    parser.add_argument('-b', '--base',
                        dest='base_dir',
                        help='Base directory to store output files to.', required=True)
    parser.add_argument('-n', '--names',
                        default='%n - %t',
                        dest='pattern',
                        help='Produce filenames as this string, with %%g, %%a, %%l, %%n, %%t, %%d '
                             'replaced by genre, artist, album, track number, title, '
                             'and date, respectively. Also, %%%% gives a literal %%. '
                             'Defaults to %%n - %%t.')
    parser.add_argument('-p', '--passthrough',
                        dest='ogg_cli_parameters',
                        help='Additional CLI arguments to pass through to oggenc.',
                        default='')
    parser.add_argument('-q', '--quality',
                        dest='quality',
                        type=float,
                        default=10,
                        help='Sets encoding quality to n, between -1 (low) and 10 (high). '
                             'Fractional quality levels such as 2.5 are permitted.')
    parser.add_argument('-r', '--replaygain',
                        dest='replygain',
                        action='store_true',
                        help='Apply replaygain tags with vorbisgain.')
    parser.add_argument('-t', '--threads',
                        dest='threads',
                        type=int,
                        default=multiprocessing.cpu_count(),
                        help='Thread pool size to spawn.')
    parser.add_argument('-v', '--verbose',
                        action='store_true',
                        dest='verbose',
                        help='Set verbose logging.')
    parser.add_argument(dest='inputs', help='List of folder/file inputs.', nargs='+')

    return parser.parse_args()

def _encode_on_filter(args, queue):
    while True:
        file_entry = queue.get()

        # FIFO queue so we can do this
        if file_entry is None:
            LOGGER.info("PID %d received poison signal. Breaking.", os.getpid())
            break

        if file_entry.lower().endswith('.flac'):
            encode_file(file_entry,
                        args.base_dir.rstrip('/'),
                        args.pattern,
                        args.quality,
                        args.ogg_cli_parameters)

def main():
    """Main logic for untz."""
    args = get_args()

    if args.verbose:
        LOGGER.setLevel(logging.DEBUG)

    preflight_checks()

    queue = multiprocessing.Queue()

    LOGGER.info('Starting %d threads.', args.threads)
    processes = [multiprocessing.Process(target=_encode_on_filter,
                                         args=(args, queue)) for i in range(args.threads)]

    for process in processes:
        process.start()

    for i in args.inputs:
        if os.path.isdir(i):
            for file_entry in recursive_file_search(i):
                LOGGER.debug('Encoding "%s"', file_entry)
                queue.put_nowait(file_entry)
        else:
            LOGGER.debug('Encoding "%s"', i)
            queue.put(i)

    # Send stop signal
    LOGGER.debug('Sending poison signal.')
    for _ in range(args.threads):
        queue.put(None)

    LOGGER.info('Waiting for processes to die.')
    for process in processes:
        process.join()

    if args.replaygain:
        apply_gain(args.base_dir)

    LOGGER.warning('Program exiting now.')
<commit_msg>Fix typo<commit_after>"""Main entry point for untz."""
import argparse
import logging
import multiprocessing
import os

from .encoder import apply_gain, encode_file
from .utils import preflight_checks, recursive_file_search

LOGGER = logging.getLogger(__name__)

def get_args():
    """Parse and return arguments."""
    parser = argparse.ArgumentParser(description='Convert FLACs to OGGs and '
                                                 'sort into a sensible folder hierarchy.')
    parser.add_argument('-b', '--base',
                        dest='base_dir',
                        help='Base directory to store output files to.', required=True)
    parser.add_argument('-n', '--names',
                        default='%n - %t',
                        dest='pattern',
                        help='Produce filenames as this string, with %%g, %%a, %%l, %%n, %%t, %%d '
                             'replaced by genre, artist, album, track number, title, '
                             'and date, respectively. Also, %%%% gives a literal %%. '
                             'Defaults to %%n - %%t.')
    parser.add_argument('-p', '--passthrough',
                        dest='ogg_cli_parameters',
                        help='Additional CLI arguments to pass through to oggenc.',
                        default='')
    parser.add_argument('-q', '--quality',
                        dest='quality',
                        type=float,
                        default=10,
                        help='Sets encoding quality to n, between -1 (low) and 10 (high). '
                             'Fractional quality levels such as 2.5 are permitted.')
    parser.add_argument('-r', '--replaygain',
                        dest='replaygain',
                        action='store_true',
                        help='Apply replaygain tags with vorbisgain.')
    parser.add_argument('-t', '--threads',
                        dest='threads',
                        type=int,
                        default=multiprocessing.cpu_count(),
                        help='Thread pool size to spawn.')
    parser.add_argument('-v', '--verbose',
                        action='store_true',
                        dest='verbose',
                        help='Set verbose logging.')
    parser.add_argument(dest='inputs', help='List of folder/file inputs.', nargs='+')

    return parser.parse_args()

def _encode_on_filter(args, queue):
    while True:
        file_entry = queue.get()

        # FIFO queue so we can do this
        if file_entry is None:
            LOGGER.info("PID %d received poison signal. Breaking.", os.getpid())
            break

        if file_entry.lower().endswith('.flac'):
            encode_file(file_entry,
                        args.base_dir.rstrip('/'),
                        args.pattern,
                        args.quality,
                        args.ogg_cli_parameters)

def main():
    """Main logic for untz."""
    args = get_args()

    if args.verbose:
        LOGGER.setLevel(logging.DEBUG)

    preflight_checks()

    queue = multiprocessing.Queue()

    LOGGER.info('Starting %d threads.', args.threads)
    processes = [multiprocessing.Process(target=_encode_on_filter,
                                         args=(args, queue)) for i in range(args.threads)]

    for process in processes:
        process.start()

    for i in args.inputs:
        if os.path.isdir(i):
            for file_entry in recursive_file_search(i):
                LOGGER.debug('Encoding "%s"', file_entry)
                queue.put_nowait(file_entry)
        else:
            LOGGER.debug('Encoding "%s"', i)
            queue.put(i)

    # Send stop signal
    LOGGER.debug('Sending poison signal.')
    for _ in range(args.threads):
        queue.put(None)

    LOGGER.info('Waiting for processes to die.')
    for process in processes:
        process.join()

    if args.replaygain:
        apply_gain(args.base_dir)

    LOGGER.warning('Program exiting now.')
<|endoftext|>


<commit_before># Copyright (c) 2013 - Network Box
#
# This file is part of Uptrack.
#
# Uptrack is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Uptrack is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Uptrack.  If not, see <http://www.gnu.org/licenses/>.

import logging

from sqlalchemy.sql import and_

import transaction

from uptrack.kojibase import KojiBase
from uptrack.models import DBSession, Package, Distro
from uptrack.yumbase import YumBase, YumError

class SyncError(Exception):
    pass

class Sync(object):
    def __init__(self, settings):
        self.log = logging.getLogger("uptrack")

        self.kojibase = KojiBase(settings["kojihub_url"])
        self.yumbase = YumBase(settings["yum_dir"])

        self.latest = {}

    def get_latest_builds(self, distro):
        return self.kojibase.get_latest_builds(distro.koji_tag)

    def get_upstream(self, pkg):
        if pkg.distro.upstream:
            # Packages from this distro all come from the same upstream
            return pkg.distro.upstream

        # TODO: Handle the other case
        raise SyncError("Could not find upstream for %s" % pkg)

    def get_upstream_evr(self, pkg):
        if not pkg.upstream.name in self.latest:
            self.latest[pkg.upstream.name] = {}

        pkgname = pkg.name

        if not pkgname in self.latest[pkg.upstream.name]:
            evr = self.yumbase.get_srpm_evr(pkgname, pkg.upstream.name,
                                            pkg.upstream.base_urls)
            self.latest[pkg.upstream.name][pkgname] = evr

        else:
            evr = self.latest[pkg.upstream.name][pkgname]

        return evr

    def run(self):
        """Run the sync"""
        pkgs = DBSession.query(Package)
        distros = DBSession.query(Distro)

        for distro in distros:
            self.log.info("Synchronizing %s..." % distro.name)
            builds = self.get_latest_builds(distro)

            for build in builds:
                self.log.info("Processing %s..." % build.name)
                pkg = pkgs.filter(and_(Package.name==build.name,
                                       Package.distro==distro)).first()

                if not pkg and build.blocked:
                    self.log.debug("Ignoring %s: it was never sync-ed and is "
                                   "now blocked" % build.name)
                    continue

                elif pkg and build.blocked:
                    self.log.warning("%s has been blocked from %s since the "
                                     "las sync, deleting it"
                                     % (pkg.name, distro.name))
                    DBSession.delete(pkg)
                    continue

                elif not pkg and not build.blocked:
                    pkg = Package(name=build.name, distro=distro,
                                  evr=build.evr)
                    DBSession.add(pkg)
                    self.log.debug("%s is a newly sync-ed package" % pkg.name)

                elif pkg and not build.blocked:
                    if pkg.evr != build.evr:
                        self.log.debug("%s was updated to %s" % (pkg, build.evr))
                        pkg.evr = build.evr

                if pkg.downstream:
                    self.log.debug("Ignoring %s: it is a downstream-only "
                                   "package" % pkg.name)
                    continue

                try:
                    pkg.upstream = self.get_upstream(pkg)
                    self.log.debug("Found package upstream: %s"
                                   % pkg.upstream)

                except SyncError as e:
                    self.log.error(e)
                    pkg.upstream = None
                    continue

                try:
                    pkg.upstream_evr = self.get_upstream_evr(pkg)
                    self.log.debug("Found package upstream evr: %s"
                                   % pkg.upstream_evr)

                except YumError as e:
                    self.log.error(e)
                    pkg.upstream_evr = None
                    continue

        transaction.commit()
<commit_msg>sync: Properly handle renamed packages<commit_after># Copyright (c) 2013 - Network Box
#
# This file is part of Uptrack.
#
# Uptrack is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Uptrack is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Uptrack.  If not, see <http://www.gnu.org/licenses/>.

import logging

from sqlalchemy.sql import and_

import transaction

from uptrack.kojibase import KojiBase
from uptrack.models import DBSession, Package, Distro
from uptrack.yumbase import YumBase, YumError

class SyncError(Exception):
    pass

class Sync(object):
    def __init__(self, settings):
        self.log = logging.getLogger("uptrack")

        self.kojibase = KojiBase(settings["kojihub_url"])
        self.yumbase = YumBase(settings["yum_dir"])

        self.latest = {}

    def get_latest_builds(self, distro):
        return self.kojibase.get_latest_builds(distro.koji_tag)

    def get_upstream(self, pkg):
        if pkg.distro.upstream:
            # Packages from this distro all come from the same upstream
            return pkg.distro.upstream

        # TODO: Handle the other case
        raise SyncError("Could not find upstream for %s" % pkg)

    def get_upstream_evr(self, pkg):
        if not pkg.upstream.name in self.latest:
            self.latest[pkg.upstream.name] = {}

        pkgname = pkg.upstream_pkgname if pkg.upstream_pkgname else pkg.name

        if not pkgname in self.latest[pkg.upstream.name]:
            evr = self.yumbase.get_srpm_evr(pkgname, pkg.upstream.name,
                                            pkg.upstream.base_urls)
            self.latest[pkg.upstream.name][pkgname] = evr

        else:
            evr = self.latest[pkg.upstream.name][pkgname]

        return evr

    def run(self):
        """Run the sync"""
        pkgs = DBSession.query(Package)
        distros = DBSession.query(Distro)

        for distro in distros:
            self.log.info("Synchronizing %s..." % distro.name)
            builds = self.get_latest_builds(distro)

            for build in builds:
                self.log.info("Processing %s..." % build.name)
                pkg = pkgs.filter(and_(Package.name==build.name,
                                       Package.distro==distro)).first()

                if not pkg and build.blocked:
                    self.log.debug("Ignoring %s: it was never sync-ed and is "
                                   "now blocked" % build.name)
                    continue

                elif pkg and build.blocked:
                    self.log.warning("%s has been blocked from %s since the "
                                     "las sync, deleting it"
                                     % (pkg.name, distro.name))
                    DBSession.delete(pkg)
                    continue

                elif not pkg and not build.blocked:
                    pkg = Package(name=build.name, distro=distro,
                                  evr=build.evr)
                    DBSession.add(pkg)
                    self.log.debug("%s is a newly sync-ed package" % pkg.name)

                elif pkg and not build.blocked:
                    if pkg.evr != build.evr:
                        self.log.debug("%s was updated to %s" % (pkg, build.evr))
                        pkg.evr = build.evr

                if pkg.downstream:
                    self.log.debug("Ignoring %s: it is a downstream-only "
                                   "package" % pkg.name)
                    continue

                try:
                    pkg.upstream = self.get_upstream(pkg)
                    self.log.debug("Found package upstream: %s"
                                   % pkg.upstream)

                except SyncError as e:
                    self.log.error(e)
                    pkg.upstream = None
                    continue

                try:
                    pkg.upstream_evr = self.get_upstream_evr(pkg)
                    self.log.debug("Found package upstream evr: %s"
                                   % pkg.upstream_evr)

                except YumError as e:
                    self.log.error(e)
                    pkg.upstream_evr = None
                    continue

        transaction.commit()
<|endoftext|>


<commit_before>import json
import re
import socket

class Board:
    SEND_TIMEOUT_SECS = 2
    RECV_BUFFER_BYTES = 2048

    def __init__(self, socket_path):
        self.sock_path = socket_path
        self.sock = None
        self.data = b''
        self._connect(socket_path)

    def _greeting_response(self, data):
        """
        Handle the response to the greeting command
        NOTE: This is called on reconnect in addition to first connection
        """
        pass

    def _connect(self, socket_path):
        """
        (re)connect to a new socket
        :param socket_path: Path for the unix socket
        """
        self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET)
        self.sock.settimeout(Board.SEND_TIMEOUT_SECS)
        self.sock.connect(socket_path)
        greeting = self._recv()
        self._greeting_response(greeting)

    def _get_lc_error(self):
        """
        :return: The text for a lost connection error
        """
        return "Lost Connection to {} at {}".format(str(self.__class__.__name__), self.sock_path)

    def _send_recv(self, message):
        self._send(message)
        return self._recv()

    def _send_recv_data(self, data):
            return json.loads(self._send_recv(json.dumps(data).encode('utf-8')))

    def _send(self, message, _is_retry=False):
        """
        Send a message to robotd
        :param retry: used internally
        :param message: message to send
        """
        try:
            self.sock.send(message)
        except (socket.timeout, BrokenPipeError, OSError):
            if _is_retry:
                raise ConnectionError(self._get_lc_error())
            else:
                try:
                    self._connect(self.sock_path)  # Reconnect
                except FileNotFoundError:
                    raise ConnectionError(self._get_lc_error())
                self._send(message, _is_retry=True)  # Retry Recursively

    def _socket_with_single_retry(self, handler):
        retryable_errors = (socket.timeout, BrokenPipeError, OSError)

        try:
            return handler(self.sock)
        except retryable_errors:
            pass

        # Retry once, need to reconnect first
        try:
            self._connect(self.sock_path)
        except FileNotFoundError:
            raise ConnectionError(self._get_lc_error())

        try:
            return handler(self.sock)
        except retryable_errors:
            raise ConnectionError(self._get_lc_error())

    def _receive_raw_from_socket_with_single_retry(self):
        return self._socket_with_single_retry(
            lambda s: s.recv(Board.RECV_BUFFER_BYTES),
        )

    def _recv(self, should_retry=True):
        while b'\n' not in self.data:
            if should_retry:
                message = self._receive_raw_from_socket_with_single_retry()
            else:
                message = self.sock.recv()
            if message == b'':
                # Received blank, return blank
                return b''

            self.data += message
        line = re.search(b'.*\n', self.data).group(0)
        self.data = self.data[len(line):]
        return line

    def _clean_up(self):
        self.sock.detach()
<commit_msg>simplify serial receive<commit_after>import json
import socket

class Board:
    SEND_TIMEOUT_SECS = 2
    RECV_BUFFER_BYTES = 2048

    def __init__(self, socket_path):
        self.sock_path = socket_path
        self.sock = None
        self.data = b''
        self._connect(socket_path)

    def _greeting_response(self, data):
        """
        Handle the response to the greeting command
        NOTE: This is called on reconnect in addition to first connection
        """
        pass

    def _connect(self, socket_path):
        """
        (re)connect to a new socket
        :param socket_path: Path for the unix socket
        """
        self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET)
        self.sock.settimeout(Board.SEND_TIMEOUT_SECS)
        self.sock.connect(socket_path)
        greeting = self._recv()
        self._greeting_response(greeting)

    def _get_lc_error(self):
        """
        :return: The text for a lost connection error
        """
        return "Lost Connection to {} at {}".format(str(self.__class__.__name__), self.sock_path)

    def _send_recv(self, message):
        self._send(message)
        return self._recv()

    def _send_recv_data(self, data):
            return json.loads(self._send_recv(json.dumps(data).encode('utf-8')))

    def _send(self, message, _is_retry=False):
        """
        Send a message to robotd
        :param retry: used internally
        :param message: message to send
        """
        try:
            self.sock.send(message)
        except (socket.timeout, BrokenPipeError, OSError):
            if _is_retry:
                raise ConnectionError(self._get_lc_error())
            else:
                try:
                    self._connect(self.sock_path)  # Reconnect
                except FileNotFoundError:
                    raise ConnectionError(self._get_lc_error())
                self._send(message, _is_retry=True)  # Retry Recursively

    def _socket_with_single_retry(self, handler):
        retryable_errors = (socket.timeout, BrokenPipeError, OSError)

        try:
            return handler(self.sock)
        except retryable_errors:
            pass

        # Retry once, need to reconnect first
        try:
            self._connect(self.sock_path)
        except FileNotFoundError:
            raise ConnectionError(self._get_lc_error())

        try:
            return handler(self.sock)
        except retryable_errors:
            raise ConnectionError(self._get_lc_error())

    def _receive_raw_from_socket_with_single_retry(self):
        return self._socket_with_single_retry(
            lambda s: s.recv(Board.RECV_BUFFER_BYTES),
        )

    def _recv(self, should_retry=True):
        while b'\n' not in self.data:
            if should_retry:
                message = self._receive_raw_from_socket_with_single_retry()
            else:
                message = self.sock.recv()
            if message == b'':
                # Received blank, return blank
                return b''

            self.data += message
        line = self.data.split(b'\n',1)
        self.data = self.data[len(line):]
        return line

    def _clean_up(self):
        self.sock.detach()
<|endoftext|>