<commit_before>import unittest

import itertools
import numpy

import chainer
from chainer.backends import cuda
from chainer import initializers
from chainer import links
from chainer import testing
from chainer.testing import attr

@testing.parameterize(*testing.product({
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
    'initial_cost': ['random', None],
}))
class TestCRF1d(unittest.TestCase):
    def _calc_score(self, batch, ys):
        cost = self.link.cost.array
        return sum(x[batch, y] for x, y in zip(self.xs, ys)) + \
            sum(cost[y1, y2] for y1, y2 in zip(ys[:-1], ys[1:]))

    def _crf1d(self, cost_data, xs_data, ys_data):
        z = numpy.zeros((self.batches[0],), numpy.float32)
        for b, length in enumerate(self.lengths):
            for ys in itertools.product(range(self.n_label), repeat=length):
                z[b] += numpy.exp(self._calc_score(b, ys))

        score = numpy.zeros((self.batches[0],), numpy.float32)
        for b, length in enumerate(self.lengths):
            ys = [self.ys[i][b] for i in range(length)]
            score[b] = self._calc_score(b, ys)

        loss = -(score - numpy.log(z))
        return numpy.sum(loss) / self.batches[0]

    def setUp(self):
        self._config_user = chainer.using_config('dtype', self.dtype)
        self._config_user.__enter__()
        self.n_label = 3

        self.lengths = [3, 3]
        self.batches = [2, 2, 2]

        self.xs = [numpy.random.uniform(-1, 1, (b, 3)).astype(self.dtype)
                   for b in self.batches]
        self.ys = [numpy.random.randint(
            0, self.n_label, (b,)).astype(numpy.int32)
            for b in self.batches]

        self.link = links.CRF1d(n_label=self.n_label)
        self.cost_shape = (self.n_label, self.n_label)

        if self.dtype == numpy.float16:
            self.check_forward_options = {'atol': 5e-3}
        else:
            self.check_forward_options = {'atol': 1e-4}

    def tearDown(self):
        self._config_user.__exit__(None, None, None)

    def check_forward(self, x_data, t_data):
        x = self.link(x_data, t_data)
        t = self._crf1d(self.link.cost.array, x_data, t_data)
        testing.assert_allclose(x.array, t,
                                **self.check_forward_options)

    def test_forward_cpu(self):
        self.check_forward(self.xs, self.ys)

    @attr.gpu
    def test_forward_gpu(self):
        self.link.to_gpu()
        self.check_forward(cuda.to_gpu(self.xs), cuda.to_gpu(self.ys))

@testing.parameterize(*testing.product({
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
    'initializer': ['random', None]
}))
class TestInitialization(unittest.TestCase):

    def setUp(self):
        self.n_label = 3
        self.initial_cost = numpy.empty((self.n_label, self.n_label),
                                        dtype=self.dtype)

        if self.initializer is None:
            initializer = initializers.constant.Zero()

        elif self.initializer == 'random':
            initializer = initializers.GlorotUniform()

        initializer(self.initial_cost)
        with chainer.using_config('dtype', self.dtype):
            self.link = links.CRF1d(self.n_label,
                                    initial_cost=self.initial_cost)

    def check_param(self):
        link = self.link
        dtype = self.dtype
        assert link.cost.dtype == dtype

        if self.initializer is None:
            cost = numpy.empty(
                (self.n_label, self.n_label), dtype=self.dtype)
            testing.assert_allclose(cost, link.cost.array, atol=0, rtol=0)

        elif self.initializer == 'random':
            testing.assert_allclose(link.cost.array, self.initial_cost,
                                    atol=0, rtol=0)

    def test_param_cpu(self):
        self.check_param()

    @attr.gpu
    def test_param_gpu(self):
        self.link.to_gpu()
        self.check_param()

testing.run_module(__name__, __file__)
<commit_msg>Remove the unnecessary branch<commit_after>import unittest

import itertools
import numpy

import chainer
from chainer.backends import cuda
from chainer import initializers
from chainer import links
from chainer import testing
from chainer.testing import attr

@testing.parameterize(*testing.product({
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
    'initial_cost': ['random', None],
}))
class TestCRF1d(unittest.TestCase):
    def _calc_score(self, batch, ys):
        cost = self.link.cost.array
        return sum(x[batch, y] for x, y in zip(self.xs, ys)) + \
            sum(cost[y1, y2] for y1, y2 in zip(ys[:-1], ys[1:]))

    def _crf1d(self, cost_data, xs_data, ys_data):
        z = numpy.zeros((self.batches[0],), numpy.float32)
        for b, length in enumerate(self.lengths):
            for ys in itertools.product(range(self.n_label), repeat=length):
                z[b] += numpy.exp(self._calc_score(b, ys))

        score = numpy.zeros((self.batches[0],), numpy.float32)
        for b, length in enumerate(self.lengths):
            ys = [self.ys[i][b] for i in range(length)]
            score[b] = self._calc_score(b, ys)

        loss = -(score - numpy.log(z))
        return numpy.sum(loss) / self.batches[0]

    def setUp(self):
        self._config_user = chainer.using_config('dtype', self.dtype)
        self._config_user.__enter__()
        self.n_label = 3

        self.lengths = [3, 3]
        self.batches = [2, 2, 2]

        self.xs = [numpy.random.uniform(-1, 1, (b, 3)).astype(self.dtype)
                   for b in self.batches]
        self.ys = [numpy.random.randint(
            0, self.n_label, (b,)).astype(numpy.int32)
            for b in self.batches]

        self.link = links.CRF1d(n_label=self.n_label)
        self.cost_shape = (self.n_label, self.n_label)

        if self.dtype == numpy.float16:
            self.check_forward_options = {'atol': 5e-3}
        else:
            self.check_forward_options = {'atol': 1e-4}

    def tearDown(self):
        self._config_user.__exit__(None, None, None)

    def check_forward(self, x_data, t_data):
        x = self.link(x_data, t_data)
        t = self._crf1d(self.link.cost.array, x_data, t_data)
        testing.assert_allclose(x.array, t,
                                **self.check_forward_options)

    def test_forward_cpu(self):
        self.check_forward(self.xs, self.ys)

    @attr.gpu
    def test_forward_gpu(self):
        self.link.to_gpu()
        self.check_forward(cuda.to_gpu(self.xs), cuda.to_gpu(self.ys))

@testing.parameterize(*testing.product({
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
    'initializer': ['random', None]
}))
class TestInitialization(unittest.TestCase):

    def setUp(self):
        self.n_label = 3
        self.initial_cost = numpy.empty((self.n_label, self.n_label),
                                        dtype=self.dtype)

        if self.initializer is None:
            initializer = initializers.constant.Zero()

        elif self.initializer == 'random':
            initializer = initializers.GlorotUniform()

        initializer(self.initial_cost)
        with chainer.using_config('dtype', self.dtype):
            self.link = links.CRF1d(self.n_label,
                                    initial_cost=self.initial_cost)

    def check_param(self):
        link = self.link
        dtype = self.dtype
        assert link.cost.dtype == dtype
        testing.assert_allclose(link.cost.array,
                                self.initial_cost,
                                atol=0, rtol=0)

    def test_param_cpu(self):
        self.check_param()

    @attr.gpu
    def test_param_gpu(self):
        self.link.to_gpu()
        self.check_param()

testing.run_module(__name__, __file__)
<|endoftext|>


<commit_before>from collections import defaultdict

def main():
    #list demo
    ll = list()
    #append
    ll.append(2)

    #extend
    ll2 = [1,2,3,4,5]
    ll.extend(ll2)

    #print
    print ll

    #pop
    print ll.pop(0)

    #count
    print ll.count(2)

    #insert
    ll.insert(0,222)
    print ll

    """
    dict practice

    """
    dict1 = {3:"bye", 4:"cya"}
    dict1[2]="hello"
    print dict1[2]

    for key,value in dict1.items():
        print key,value

    print dict1.keys()
    print dict1.values()
    print dict1.items()

    "defualt dict"

    dict2 = defaultdict(int)

    for i in range(0,100):
        dict2[i]+=1

    print dict2

    print help('collections')

if __name__=="__main__":
    main()
<commit_msg>custom sort<commit_after>from collections import defaultdict

def func(a,b):
    return 1 if a>b else -1

def main():

    s= "hello"
    ll = [5,4,3,2,1]
    ll.sort(func)
    print ll

if __name__=="__main__":
    main()
<|endoftext|>


<commit_before># Copyright (C) 2006-2020  Music Technology Group - Universitat Pompeu Fabra
#
# This file is part of Essentia
#
# Essentia is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation (FSF), either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the Affero GNU General Public License
# version 3 along with this program. If not, see http://www.gnu.org/licenses/

import argparse
import os

import numpy as np

from essentia import Pool
from essentia import run

from essentia.streaming import (MonoLoader, FrameCutter, Windowing, Spectrum,
                                MelBands, UnaryOperator)

ZERO_PADDING = 0
WINDOW_TYPE = 'hann'
FRAME_SIZE = 1024
HOP_SIZE = 512
NUMBER_BANDS = 24
SAMPLE_RATE = 44100.
LOW_FREQUENCY_BOUND = 0.
HIGH_FREQUENCY_BOUND = 22050.
WARPING_FORMULA = 'htkMel'
WEIGHTING = 'warping'
NORMALIZE = 'unit_sum'
BANDS_TYPE = 'power'
COMPRESSION_TYPE = 'shift_scale_log'

def melspectrogram(filename, npy_file=None, force=False, verbose=False, sample_rate=SAMPLE_RATE, frame_size=FRAME_SIZE,
    hop_size=HOP_SIZE, window_type=WINDOW_TYPE, zero_padding=ZERO_PADDING, low_frequency_bound=LOW_FREQUENCY_BOUND,
    high_frequency_bound=HIGH_FREQUENCY_BOUND, number_bands=NUMBER_BANDS, warping_formula=WARPING_FORMULA,
    weighting=WEIGHTING, normalize=NORMALIZE, bands_type=BANDS_TYPE, compression_type=COMPRESSION_TYPE):
    """Computes the mel spectrogram given the audio filename.
    When the parameter `npy_file` is specified, the data is saved to disk as a numpy array (.npy).
    Use the parameter `force` to overwrite the numpy array in case it already exists.
    The rest of parameters are directly mapped to Essentia algorithms as explained below.

    Note: this functionallity is also avaiable as a command line script.

    Parameters:
        sample_rate:
        real ∈ (0,inf) (default = 44100)
        the desired output sampling rate [Hz]

        frame_size:
        integer ∈ [1,inf) (default = 1024)
        the output frame size

        hop_size:
        integer ∈ [1,inf) (default = 512)
        the hop size between frames

        window_type:
        string ∈ {hamming,hann,hannnsgcq,triangular,square,blackmanharris62,blackmanharris70,blackmanharris74,blackmanharris92} (default = "hann")
        the window type, which can be 'hamming', 'hann', 'triangular', 'square' or 'blackmanharrisXX'

        zero_padding:
        integer ∈ [0,inf) (default = 0)
        the size of the zero-padding

        low_frequency_bound:
        real ∈ [0,inf) (default = 0)
        a lower-bound limit for the frequencies to be included in the bands

        high_frequency_bound:
        real ∈ [0,inf) (default = 22050)
        an upper-bound limit for the frequencies to be included in the bands

        number_bands:
        integer ∈ (1,inf) (default = 24)
        the number of output bands

        warping_formula:
        string ∈ {slaneyMel,htkMel} (default = "htkMel")
        The scale implementation type: 'htkMel' scale from the HTK toolkit [2, 3]
        (default) or 'slaneyMel' scale from the Auditory toolbox [4]

        weighting:
        string ∈ {warping,linear} (default = "warping")
        type of weighting function for determining triangle area

        normalize:
        string ∈ {unit_sum,unit_tri,unit_max} (default = "unit_sum")
        spectrum bin weights to use for each mel band: 'unit_max' to make each mel
        band vertex equal to 1, 'unit_sum' to make each mel band area equal to 1
        summing the actual weights of spectrum bins, 'unit_area' to make each
        triangle mel band area equal to 1 normalizing the weights of each triangle
        by its bandwidth

        bands_type:
        string ∈ {magnitude,power} (default = "power")
        'power' to output squared units, 'magnitude' to keep it as the input

        compression_type:
        string ∈ {dB,shift_scale_log,none} (default = "shift_scale_log")
        the sample rate

    Returns:
        (2D array): The mel-spectrogram.
    """

    padded_size = frame_size + zero_padding
    spectrum_size = (padded_size) // 2 + 1

    # In case we want to save the melbands to a file
    # check if the file already existss
    if npy_file:
        if not npy_file.endswith('.npy'):
            npy_file += '.npy'

        if not force:
            if os.path.exists(npy_file):
                if verbose:
                    print('Skipping "{}"'.format(npy_file))
                return

    pool = Pool()

    loader = MonoLoader(filename=filename,
                        sampleRate=sample_rate)
    frameCutter = FrameCutter(frameSize=frame_size,
                              hopSize=hop_size)
    w = Windowing(zeroPadding=zero_padding,
                  type=window_type,
                  normalized=False)  # None of the mel bands extraction methods
                                     # we have seen requires window-level normalization.
    spec = Spectrum(size=padded_size)
    mels = MelBands(inputSize=spectrum_size,
                    numberBands=number_bands,
                    sampleRate=sample_rate,
                    lowFrequencyBound=low_frequency_bound,
                    highFrequencyBound=high_frequency_bound,
                    warpingFormula=warping_formula,
                    weighting=weighting,
                    normalize=normalize,
                    type=bands_type,
                    log=False)  # Do not compute any compression here.
                                # Use the `UnaryOperator`s methods before
                                # in case a new compression type is required.

    if compression_type.lower() == 'db':
        shift = UnaryOperator(type='identity')
        compressor = UnaryOperator(type='lin2db')

    elif compression_type.lower() == 'shift_scale_log':
        shift = UnaryOperator(type='identity', scale=1e4, shift=1)
        compressor = UnaryOperator(type='log10')

    elif compression_type.lower() == 'none':
        shift = UnaryOperator(type='identity')
        compressor = UnaryOperator(type='identity')

    loader.audio >> frameCutter.signal
    frameCutter.frame >> w.frame >> spec.frame
    spec.spectrum >> mels.spectrum
    mels.bands >> shift.array >> compressor.array >> (pool, 'mel_bands')

    run(loader)

    mel_bands = np.array(pool['mel_bands'])

    if npy_file:
        np.save(npy_file, mel_bands)

    if verbose:
        print('Done for "{}"'.format(npy_file))

    return mel_bands

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Computes the mel spectrogram given the audio filename.')

    parser.add_argument('filename',
                        help='the name of the file from which to read')
    parser.add_argument('npy_file', type=str,
                        help='the name of the output file')
    parser.add_argument('--force', '-f', action='store_true',
                        help='whether to recompute if the output file already exists')
    parser.add_argument('--verbose', '-v', action='store_true',
                        help='whether to print out status to the standard output')
    parser.add_argument('--sample-rate', '-sr', type=float, default=SAMPLE_RATE,
                        help='the sample rate')
    parser.add_argument('--frame-size', '-fs', type=int, default=FRAME_SIZE,
                        help='the output frame size')
    parser.add_argument('--hop-size', '-hs', type=int, default=HOP_SIZE,
                        help='the hop size between frames')
    parser.add_argument('--window-type', '-wt', type=str, default=WINDOW_TYPE,
                        help='window type', choices=('hamming', 'hann', 'hannnsgcq', 'triangular', 'square', 'blackmanharris62',
                                                     'blackmanharris70', 'blackmanharris74', 'blackmanharris92'))
    parser.add_argument('--zero-padding', '-zp', type=int, default=ZERO_PADDING,
                        help='the size of the zero-padding')
    parser.add_argument('--low-frequency-bound', '-lf', type=float, default=LOW_FREQUENCY_BOUND,
                        help='a lower-bound limit for the frequencies to be included in the bands')
    parser.add_argument('--high-frequency-bound', '-hf', type=float, default=HIGH_FREQUENCY_BOUND,
                        help='an upper-bound limit for the frequencies to be included in the bands')
    parser.add_argument('--number-bands', '-nb', type=int, default=NUMBER_BANDS,
                        help='the number of output bands')
    parser.add_argument('--warping-formula', '-wf', type=str, default=WARPING_FORMULA, choices=('slaneyMel','htkMel'),
                        help='the scale implementation type: `htkMel` scale from the HTK toolkit(default) or `slaneyMel` scale from the Auditory toolbox')
    parser.add_argument('--weighting', '-we', type=str, default=WEIGHTING, choices=('warping','linear'),
                        help='type of weighting function for determining triangle area')
    parser.add_argument('--normalize', '-n', type=str, default=NORMALIZE, choices=('unit_sum', 'unit_tri', 'unit_max'),
                        help='spectrum bin weights to use for each mel band: `unit_max` to make each mel band vertex equal to 1, `unit_sum` to make each mel band area equal to 1 summing the actual weights of spectrum bins, `unit_area` to make each triangle mel band area equal to 1 normalizing the weights of each triangle by its bandwidth')
    parser.add_argument('--bands-type', '-bt', type=str, default=BANDS_TYPE, choices=('magnitude','power'),
                        help='`power` to output squared units, `magnitude` to keep it as the input')
    parser.add_argument('--compression-type', '-ct', type=str, default=COMPRESSION_TYPE, choices=('dB', 'shift_scale_log', 'none'),
                        help='dB: 10log10(x). shift_scale_log: log(1 + 10000 * x)')

    melspectrogram(**vars(parser.parse_args()))
<commit_msg>Fix incorrect melspectrogram doc<commit_after># Copyright (C) 2006-2020  Music Technology Group - Universitat Pompeu Fabra
#
# This file is part of Essentia
#
# Essentia is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation (FSF), either version 3 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the Affero GNU General Public License
# version 3 along with this program. If not, see http://www.gnu.org/licenses/

import argparse
import os

import numpy as np

from essentia import Pool
from essentia import run

from essentia.streaming import (MonoLoader, FrameCutter, Windowing, Spectrum,
                                MelBands, UnaryOperator)

ZERO_PADDING = 0
WINDOW_TYPE = 'hann'
FRAME_SIZE = 1024
HOP_SIZE = 512
NUMBER_BANDS = 24
SAMPLE_RATE = 44100.
LOW_FREQUENCY_BOUND = 0.
HIGH_FREQUENCY_BOUND = 22050.
WARPING_FORMULA = 'htkMel'
WEIGHTING = 'warping'
NORMALIZE = 'unit_sum'
BANDS_TYPE = 'power'
COMPRESSION_TYPE = 'shift_scale_log'

def melspectrogram(filename, npy_file=None, force=False, verbose=False, sample_rate=SAMPLE_RATE, frame_size=FRAME_SIZE,
    hop_size=HOP_SIZE, window_type=WINDOW_TYPE, zero_padding=ZERO_PADDING, low_frequency_bound=LOW_FREQUENCY_BOUND,
    high_frequency_bound=HIGH_FREQUENCY_BOUND, number_bands=NUMBER_BANDS, warping_formula=WARPING_FORMULA,
    weighting=WEIGHTING, normalize=NORMALIZE, bands_type=BANDS_TYPE, compression_type=COMPRESSION_TYPE):
    """Computes the mel spectrogram given the audio filename.
    When the parameter `npy_file` is specified, the data is saved to disk as a numpy array (.npy).
    Use the parameter `force` to overwrite the numpy array in case it already exists.
    The rest of parameters are directly mapped to Essentia algorithms as explained below.

    Note: this functionallity is also avaiable as a command line script.

    Parameters:
        sample_rate:
        real ∈ (0,inf) (default = 44100)
        the desired output sampling rate [Hz]

        frame_size:
        integer ∈ [1,inf) (default = 1024)
        the output frame size

        hop_size:
        integer ∈ [1,inf) (default = 512)
        the hop size between frames

        window_type:
        string ∈ {hamming,hann,hannnsgcq,triangular,square,blackmanharris62,blackmanharris70,blackmanharris74,blackmanharris92} (default = "hann")
        the window type, which can be 'hamming', 'hann', 'triangular', 'square' or 'blackmanharrisXX'

        zero_padding:
        integer ∈ [0,inf) (default = 0)
        the size of the zero-padding

        low_frequency_bound:
        real ∈ [0,inf) (default = 0)
        a lower-bound limit for the frequencies to be included in the bands

        high_frequency_bound:
        real ∈ [0,inf) (default = 22050)
        an upper-bound limit for the frequencies to be included in the bands

        number_bands:
        integer ∈ (1,inf) (default = 24)
        the number of output bands

        warping_formula:
        string ∈ {slaneyMel,htkMel} (default = "htkMel")
        The scale implementation type: 'htkMel' scale from the HTK toolkit [2, 3]
        (default) or 'slaneyMel' scale from the Auditory toolbox [4]

        weighting:
        string ∈ {warping,linear} (default = "warping")
        type of weighting function for determining triangle area

        normalize:
        string ∈ {unit_sum,unit_tri,unit_max} (default = "unit_sum")
        spectrum bin weights to use for each mel band: 'unit_max' to make each mel
        band vertex equal to 1, 'unit_sum' to make each mel band area equal to 1
        summing the actual weights of spectrum bins, 'unit_area' to make each
        triangle mel band area equal to 1 normalizing the weights of each triangle
        by its bandwidth

        bands_type:
        string ∈ {magnitude,power} (default = "power")
        'power' to output squared units, 'magnitude' to keep it as the input

        compression_type:
        string ∈ {dB,shift_scale_log,none} (default = "shift_scale_log")
        the compression type to use.
        'shift_scale_log' is log10(10000 * x + 1)
        'dB' is 10 * log10(x)

    Returns:
        (2D array): The mel-spectrogram.
    """

    padded_size = frame_size + zero_padding
    spectrum_size = (padded_size) // 2 + 1

    # In case we want to save the melbands to a file
    # check if the file already existss
    if npy_file:
        if not npy_file.endswith('.npy'):
            npy_file += '.npy'

        if not force:
            if os.path.exists(npy_file):
                if verbose:
                    print('Skipping "{}"'.format(npy_file))
                return

    pool = Pool()

    loader = MonoLoader(filename=filename,
                        sampleRate=sample_rate)
    frameCutter = FrameCutter(frameSize=frame_size,
                              hopSize=hop_size)
    w = Windowing(zeroPadding=zero_padding,
                  type=window_type,
                  normalized=False)  # None of the mel bands extraction methods
                                     # we have seen requires window-level normalization.
    spec = Spectrum(size=padded_size)
    mels = MelBands(inputSize=spectrum_size,
                    numberBands=number_bands,
                    sampleRate=sample_rate,
                    lowFrequencyBound=low_frequency_bound,
                    highFrequencyBound=high_frequency_bound,
                    warpingFormula=warping_formula,
                    weighting=weighting,
                    normalize=normalize,
                    type=bands_type,
                    log=False)  # Do not compute any compression here.
                                # Use the `UnaryOperator`s methods before
                                # in case a new compression type is required.

    if compression_type.lower() == 'db':
        shift = UnaryOperator(type='identity')
        compressor = UnaryOperator(type='lin2db')

    elif compression_type.lower() == 'shift_scale_log':
        shift = UnaryOperator(type='identity', scale=1e4, shift=1)
        compressor = UnaryOperator(type='log10')

    elif compression_type.lower() == 'none':
        shift = UnaryOperator(type='identity')
        compressor = UnaryOperator(type='identity')

    loader.audio >> frameCutter.signal
    frameCutter.frame >> w.frame >> spec.frame
    spec.spectrum >> mels.spectrum
    mels.bands >> shift.array >> compressor.array >> (pool, 'mel_bands')

    run(loader)

    mel_bands = np.array(pool['mel_bands'])

    if npy_file:
        np.save(npy_file, mel_bands)

    if verbose:
        print('Done for "{}"'.format(npy_file))

    return mel_bands

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Computes the mel spectrogram given the audio filename.')

    parser.add_argument('filename',
                        help='the name of the file from which to read')
    parser.add_argument('npy_file', type=str,
                        help='the name of the output file')
    parser.add_argument('--force', '-f', action='store_true',
                        help='whether to recompute if the output file already exists')
    parser.add_argument('--verbose', '-v', action='store_true',
                        help='whether to print out status to the standard output')
    parser.add_argument('--sample-rate', '-sr', type=float, default=SAMPLE_RATE,
                        help='the sample rate')
    parser.add_argument('--frame-size', '-fs', type=int, default=FRAME_SIZE,
                        help='the output frame size')
    parser.add_argument('--hop-size', '-hs', type=int, default=HOP_SIZE,
                        help='the hop size between frames')
    parser.add_argument('--window-type', '-wt', type=str, default=WINDOW_TYPE,
                        help='window type', choices=('hamming', 'hann', 'hannnsgcq', 'triangular', 'square', 'blackmanharris62',
                                                     'blackmanharris70', 'blackmanharris74', 'blackmanharris92'))
    parser.add_argument('--zero-padding', '-zp', type=int, default=ZERO_PADDING,
                        help='the size of the zero-padding')
    parser.add_argument('--low-frequency-bound', '-lf', type=float, default=LOW_FREQUENCY_BOUND,
                        help='a lower-bound limit for the frequencies to be included in the bands')
    parser.add_argument('--high-frequency-bound', '-hf', type=float, default=HIGH_FREQUENCY_BOUND,
                        help='an upper-bound limit for the frequencies to be included in the bands')
    parser.add_argument('--number-bands', '-nb', type=int, default=NUMBER_BANDS,
                        help='the number of output bands')
    parser.add_argument('--warping-formula', '-wf', type=str, default=WARPING_FORMULA, choices=('slaneyMel','htkMel'),
                        help='the scale implementation type: `htkMel` scale from the HTK toolkit(default) or `slaneyMel` scale from the Auditory toolbox')
    parser.add_argument('--weighting', '-we', type=str, default=WEIGHTING, choices=('warping','linear'),
                        help='type of weighting function for determining triangle area')
    parser.add_argument('--normalize', '-n', type=str, default=NORMALIZE, choices=('unit_sum', 'unit_tri', 'unit_max'),
                        help='spectrum bin weights to use for each mel band: `unit_max` to make each mel band vertex equal to 1, `unit_sum` to make each mel band area equal to 1 summing the actual weights of spectrum bins, `unit_area` to make each triangle mel band area equal to 1 normalizing the weights of each triangle by its bandwidth')
    parser.add_argument('--bands-type', '-bt', type=str, default=BANDS_TYPE, choices=('magnitude','power'),
                        help='`power` to output squared units, `magnitude` to keep it as the input')
    parser.add_argument('--compression-type', '-ct', type=str, default=COMPRESSION_TYPE, choices=('dB', 'shift_scale_log', 'none'),
                        help='dB: 10log10(x). shift_scale_log: log(1 + 10000 * x)')

    melspectrogram(**vars(parser.parse_args()))
<|endoftext|>


<commit_before># Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import json
import signal

from ducktape.utils.util import wait_until
from ducktape.services.background_thread import BackgroundThreadService
from kafkatest.directory_layout.kafka_path import KafkaPathResolverMixin
from ducktape.cluster.remoteaccount import RemoteCommandError

class TransactionalMessageCopier(KafkaPathResolverMixin, BackgroundThreadService):
    """This service wraps org.apache.kafka.tools.TransactionalMessageCopier for
    use in system testing.
    """
    PERSISTENT_ROOT = "/mnt/transactional_message_copier"
    STDOUT_CAPTURE = os.path.join(PERSISTENT_ROOT, "transactional_message_copier.stdout")
    STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "transactional_message_copier.stderr")
    LOG_DIR = os.path.join(PERSISTENT_ROOT, "logs")
    LOG_FILE = os.path.join(LOG_DIR, "transactional_message_copier.log")
    LOG4J_CONFIG = os.path.join(PERSISTENT_ROOT, "tools-log4j.properties")

    logs = {
        "transactional_message_copier_stdout": {
            "path": STDOUT_CAPTURE,
            "collect_default": True},
        "transactional_message_copier_stderr": {
            "path": STDERR_CAPTURE,
            "collect_default": True},
        "transactional_message_copier_log": {
            "path": LOG_FILE,
            "collect_default": True}
    }

    def __init__(self, context, num_nodes, kafka, transactional_id, consumer_group,
                 input_topic, input_partition, output_topic, max_messages = -1,
                 transaction_size = 1000):
        super(TransactionalMessageCopier, self).__init__(context, num_nodes)
        self.kafka = kafka
        self.transactional_id = transactional_id
        self.consumer_group = consumer_group
        self.transaction_size = transaction_size
        self.input_topic = input_topic
        self.input_partition = input_partition
        self.output_topic = output_topic
        self.max_messages = max_messages
        self.message_copy_finished = False
        self.consumed = -1
        self.remaining = -1
        self.stop_timeout_sec = 60
        self.loggers = {
            "org.apache.kafka.clients.producer.internals": "TRACE"
        }

    def _worker(self, idx, node):
        node.account.ssh("mkdir -p %s" % TransactionalMessageCopier.PERSISTENT_ROOT,
                         allow_fail=False)
        # Create and upload log properties
        log_config = self.render('tools_log4j.properties',
                                 log_file=TransactionalMessageCopier.LOG_FILE)
        node.account.create_file(TransactionalMessageCopier.LOG4J_CONFIG, log_config)
        # Configure security
        self.security_config = self.kafka.security_config.client_config(node=node)
        self.security_config.setup_node(node)
        cmd = self.start_cmd(node, idx)
        self.logger.debug("TransactionalMessageCopier %d command: %s" % (idx, cmd))
        try:
            for line in node.account.ssh_capture(cmd):
                line = line.strip()
                data = self.try_parse_json(line)
                if data is not None:
                    with self.lock:
                        self.remaining = int(data["remaining"])
                        self.consumed = int(data["consumed"])
                        self.logger.info("%s: consumed %d, remaining %d" %
                                         (self.transactional_id, self.consumed, self.remaining))
                        if "shutdown_complete" in data:
                           if self.remaining == 0:
                                # We are only finished if the remaining
                                # messages at the time of shutdown is 0.
                                #
                                # Otherwise a clean shutdown would still print
                                # a 'shutdown complete' messages even though
                                # there are unprocessed messages, causing
                                # tests to fail.
                                self.logger.info("%s : Finished message copy" % self.transactional_id)
                                self.message_copy_finished = True
                           else:
                               self.logger.info("%s : Shut down without finishing message copy." %\
                                                self.transactional_id)
        except RemoteCommandError as e:
            self.logger.debug("Got exception while reading output from copier, \
                              probably because it was SIGKILL'd (exit code 137): %s" % str(e))

    def start_cmd(self, node, idx):
        cmd  = "export LOG_DIR=%s;" % TransactionalMessageCopier.LOG_DIR
        cmd += " export KAFKA_OPTS=%s;" % self.security_config.kafka_opts
        cmd += " export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % TransactionalMessageCopier.LOG4J_CONFIG
        cmd += self.path.script("kafka-run-class.sh", node) + " org.apache.kafka.tools." + "TransactionalMessageCopier"
        cmd += " --broker-list %s" % self.kafka.bootstrap_servers(self.security_config.security_protocol)
        cmd += " --transactional-id %s" % self.transactional_id
        cmd += " --consumer-group %s" % self.consumer_group
        cmd += " --input-topic %s" % self.input_topic
        cmd += " --output-topic %s" % self.output_topic
        cmd += " --input-partition %s" % str(self.input_partition)
        cmd += " --transaction-size %s" % str(self.transaction_size)
        if self.max_messages > 0:
            cmd += " --max-messages %s" % str(self.max_messages)
        cmd += " 2>> %s | tee -a %s &" % (TransactionalMessageCopier.STDERR_CAPTURE, TransactionalMessageCopier.STDOUT_CAPTURE)

        return cmd

    def clean_node(self, node, clean_shutdown=True):
        self.kill_node(node, clean_shutdown)
        node.account.ssh("rm -rf " + self.PERSISTENT_ROOT, allow_fail=False)
        self.security_config.clean_node(node)

    def pids(self, node):
        try:
            cmd = "ps ax | grep -i TransactionalMessageCopier | grep java | grep -v grep | awk '{print $1}'"
            pid_arr = [pid for pid in node.account.ssh_capture(cmd, allow_fail=True, callback=int)]
            return pid_arr
        except (RemoteCommandError, ValueError) as e:
            self.logger.error("Could not list pids: %s" % str(e))
            return []

    def alive(self, node):
        return len(self.pids(node)) > 0

    def kill_node(self, node, clean_shutdown=True):
        pids = self.pids(node)
        sig = signal.SIGTERM if clean_shutdown else signal.SIGKILL
        for pid in pids:
            node.account.signal(pid, sig)
            wait_until(lambda: len(self.pids(node)) == 0, timeout_sec=60, err_msg="Message Copier failed to stop")

    def stop_node(self, node, clean_shutdown=True):
        self.kill_node(node, clean_shutdown)
        stopped = self.wait_node(node, timeout_sec=self.stop_timeout_sec)
        assert stopped, "Node %s: did not stop within the specified timeout of %s seconds" % \
            (str(node.account), str(self.stop_timeout_sec))

    def restart(self, clean_shutdown):
        if self.is_done:
            return
        node = self.nodes[0]
        with self.lock:
            self.consumed = -1
            self.remaining = -1
        self.stop_node(node, clean_shutdown)
        self.start_node(node)

    def try_parse_json(self, string):
        """Try to parse a string as json. Return None if not parseable."""
        try:
            record = json.loads(string)
            return record
        except ValueError:
            self.logger.debug("Could not parse as json: %s" % str(string))
            return None

    @property
    def is_done(self):
        return self.message_copy_finished

    def progress_percent(self):
        with self.lock:
            if self.remaining < 0:
                return 0
            if self.consumed + self.remaining == 0:
                return 100
            return (float(self.consumed)/float(self.consumed + self.remaining)) * 100
<commit_msg>KAFKA-5437; Always send a sig_kill when cleaning the message copier<commit_after># Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import json
import signal

from ducktape.utils.util import wait_until
from ducktape.services.background_thread import BackgroundThreadService
from kafkatest.directory_layout.kafka_path import KafkaPathResolverMixin
from ducktape.cluster.remoteaccount import RemoteCommandError

class TransactionalMessageCopier(KafkaPathResolverMixin, BackgroundThreadService):
    """This service wraps org.apache.kafka.tools.TransactionalMessageCopier for
    use in system testing.
    """
    PERSISTENT_ROOT = "/mnt/transactional_message_copier"
    STDOUT_CAPTURE = os.path.join(PERSISTENT_ROOT, "transactional_message_copier.stdout")
    STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "transactional_message_copier.stderr")
    LOG_DIR = os.path.join(PERSISTENT_ROOT, "logs")
    LOG_FILE = os.path.join(LOG_DIR, "transactional_message_copier.log")
    LOG4J_CONFIG = os.path.join(PERSISTENT_ROOT, "tools-log4j.properties")

    logs = {
        "transactional_message_copier_stdout": {
            "path": STDOUT_CAPTURE,
            "collect_default": True},
        "transactional_message_copier_stderr": {
            "path": STDERR_CAPTURE,
            "collect_default": True},
        "transactional_message_copier_log": {
            "path": LOG_FILE,
            "collect_default": True}
    }

    def __init__(self, context, num_nodes, kafka, transactional_id, consumer_group,
                 input_topic, input_partition, output_topic, max_messages = -1,
                 transaction_size = 1000):
        super(TransactionalMessageCopier, self).__init__(context, num_nodes)
        self.kafka = kafka
        self.transactional_id = transactional_id
        self.consumer_group = consumer_group
        self.transaction_size = transaction_size
        self.input_topic = input_topic
        self.input_partition = input_partition
        self.output_topic = output_topic
        self.max_messages = max_messages
        self.message_copy_finished = False
        self.consumed = -1
        self.remaining = -1
        self.stop_timeout_sec = 60
        self.loggers = {
            "org.apache.kafka.clients.producer.internals": "TRACE"
        }

    def _worker(self, idx, node):
        node.account.ssh("mkdir -p %s" % TransactionalMessageCopier.PERSISTENT_ROOT,
                         allow_fail=False)
        # Create and upload log properties
        log_config = self.render('tools_log4j.properties',
                                 log_file=TransactionalMessageCopier.LOG_FILE)
        node.account.create_file(TransactionalMessageCopier.LOG4J_CONFIG, log_config)
        # Configure security
        self.security_config = self.kafka.security_config.client_config(node=node)
        self.security_config.setup_node(node)
        cmd = self.start_cmd(node, idx)
        self.logger.debug("TransactionalMessageCopier %d command: %s" % (idx, cmd))
        try:
            for line in node.account.ssh_capture(cmd):
                line = line.strip()
                data = self.try_parse_json(line)
                if data is not None:
                    with self.lock:
                        self.remaining = int(data["remaining"])
                        self.consumed = int(data["consumed"])
                        self.logger.info("%s: consumed %d, remaining %d" %
                                         (self.transactional_id, self.consumed, self.remaining))
                        if "shutdown_complete" in data:
                           if self.remaining == 0:
                                # We are only finished if the remaining
                                # messages at the time of shutdown is 0.
                                #
                                # Otherwise a clean shutdown would still print
                                # a 'shutdown complete' messages even though
                                # there are unprocessed messages, causing
                                # tests to fail.
                                self.logger.info("%s : Finished message copy" % self.transactional_id)
                                self.message_copy_finished = True
                           else:
                               self.logger.info("%s : Shut down without finishing message copy." %\
                                                self.transactional_id)
        except RemoteCommandError as e:
            self.logger.debug("Got exception while reading output from copier, \
                              probably because it was SIGKILL'd (exit code 137): %s" % str(e))

    def start_cmd(self, node, idx):
        cmd  = "export LOG_DIR=%s;" % TransactionalMessageCopier.LOG_DIR
        cmd += " export KAFKA_OPTS=%s;" % self.security_config.kafka_opts
        cmd += " export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % TransactionalMessageCopier.LOG4J_CONFIG
        cmd += self.path.script("kafka-run-class.sh", node) + " org.apache.kafka.tools." + "TransactionalMessageCopier"
        cmd += " --broker-list %s" % self.kafka.bootstrap_servers(self.security_config.security_protocol)
        cmd += " --transactional-id %s" % self.transactional_id
        cmd += " --consumer-group %s" % self.consumer_group
        cmd += " --input-topic %s" % self.input_topic
        cmd += " --output-topic %s" % self.output_topic
        cmd += " --input-partition %s" % str(self.input_partition)
        cmd += " --transaction-size %s" % str(self.transaction_size)
        if self.max_messages > 0:
            cmd += " --max-messages %s" % str(self.max_messages)
        cmd += " 2>> %s | tee -a %s &" % (TransactionalMessageCopier.STDERR_CAPTURE, TransactionalMessageCopier.STDOUT_CAPTURE)

        return cmd

    def clean_node(self, node):
        self.kill_node(node, clean_shutdown=False)
        node.account.ssh("rm -rf " + self.PERSISTENT_ROOT, allow_fail=False)
        self.security_config.clean_node(node)

    def pids(self, node):
        try:
            cmd = "ps ax | grep -i TransactionalMessageCopier | grep java | grep -v grep | awk '{print $1}'"
            pid_arr = [pid for pid in node.account.ssh_capture(cmd, allow_fail=True, callback=int)]
            return pid_arr
        except (RemoteCommandError, ValueError) as e:
            self.logger.error("Could not list pids: %s" % str(e))
            return []

    def alive(self, node):
        return len(self.pids(node)) > 0

    def kill_node(self, node, clean_shutdown=True):
        pids = self.pids(node)
        sig = signal.SIGTERM if clean_shutdown else signal.SIGKILL
        for pid in pids:
            node.account.signal(pid, sig)
            wait_until(lambda: len(self.pids(node)) == 0, timeout_sec=60, err_msg="Message Copier failed to stop")

    def stop_node(self, node, clean_shutdown=True):
        self.kill_node(node, clean_shutdown)
        stopped = self.wait_node(node, timeout_sec=self.stop_timeout_sec)
        assert stopped, "Node %s: did not stop within the specified timeout of %s seconds" % \
            (str(node.account), str(self.stop_timeout_sec))

    def restart(self, clean_shutdown):
        if self.is_done:
            return
        node = self.nodes[0]
        with self.lock:
            self.consumed = -1
            self.remaining = -1
        self.stop_node(node, clean_shutdown)
        self.start_node(node)

    def try_parse_json(self, string):
        """Try to parse a string as json. Return None if not parseable."""
        try:
            record = json.loads(string)
            return record
        except ValueError:
            self.logger.debug("Could not parse as json: %s" % str(string))
            return None

    @property
    def is_done(self):
        return self.message_copy_finished

    def progress_percent(self):
        with self.lock:
            if self.remaining < 0:
                return 0
            if self.consumed + self.remaining == 0:
                return 100
            return (float(self.consumed)/float(self.consumed + self.remaining)) * 100
<|endoftext|>


<commit_before>import asyncio, atexit

from .position import *
from .drive import DriveMotors, TURN_TIME
from .room import RoomEntryDetection
from .status import LED
from .firefighting import *

from collections import defaultdict

L = "L"
R = "R"
S = "S"
X = "X"

DELAY_TIME = 1.2

VISION_NOTHING = 1
VISION_POSSIBLE = 3
VISION_LIKELY = 4

class Navigator:
    def __init__(self, start_in_room=True):
        self.loop = asyncio.get_event_loop()
        self.ir = IRPositioning()
        self.driver = DriveMotors()
        self.room = RoomEntryDetection()
        self.flamedetect = FlameDetect()
        self.flamedetect.on = True
        self.extinguisher = Extinguisher()

        self.flameled = LED(0)
        self.videoled = LED(1)
        self.readyled = LED(2)
        self.roomled = LED(3)
        self.flameled.on = 0

        self.__turning = False
        self.__turn_handles = defaultdict(bool)

        self.__inroom = not start_in_room
        self.__initial_room = start_in_room
        self.__toggle_room(True)
        self.__turning = None # "INITIAL_ORIENT"

        self.__steps = [X]
        self.__going_backwards = False
        self.__fourway = None
        self.__orientation = 0 # N, E, S, W
        self.__initial_orient = 0

        for i in (L_OPENING, R_OPENING, FRONT_WALL_DETECT, FOUR_WAY_DETECT):
            self.ir.register(i, self.perform_turn)
        self.ir.register(NOT_PARALLEL, self.parallel_correct)
        self.ir.register(OFF_CENTER, self.center_correct)
        self.flamedetect.register(self.vision)
        self.room.register(self.toggle_room)

        atexit.register(self.driver.stop)
        self.loop.call_soon(self.driver.forward) #self.driver.turnleft(None)
        #self.loop.call_later(0.1, self.initial_orient)

    def initial_orient(self):
        print(abs(self.ir.data[5] - self.ir.data[4]))
        turn_time = TURN_TIME / 5
        if abs(self.ir.data[5] - self.ir.data[4]) < 6 and self.ir.data[5]:
            self.__initial_orient += 1
            turn_time = 0.5
            self.driver.stop()
            print("STOPPED")
        else:
            self.__initial_orient = 0
            self.driver.turnleft(None)
            print("TURNING")
        if self.__initial_orient < 4:
            self.loop.call_later(turn_time, self.initial_orient)
        else:
            print("DONE ORIENTING")
            self.driver.cancel()
            self.loop.call_soon(self.driver.forward)
            self.loop.call_later(0.1, self.end_turn)
            print(self.driver.left, self.driver.right)

    def scan_room(self):
        self.__scan_start = self.loop.time()
        self.__giving_up = self.loop.call_later(TURN_TIME * 15, self.give_up)
        self.driver.right = 0.04
        self.driver.left = 0.04

    def give_up(self):
        print("GIVING UP")
        self.driver.forward()

    def vision(self, state, deg):
        print(repr(state), VISION_LIKELY, deg)
        if state == VISION_LIKELY:
            if deg < 0:
                self.driver.right = -0.03
                self.driver.left = -0.03
            elif deg > 0:
                self.driver.right = 0.03
                self.driver.left = 0.03
            if abs(deg) < 15:
                self.flameled.on = True
                self.driver.stop()
                self.__back_time = self.loop.time()
                self.extinguisher.go()
                self.__giving_up.cancel()
                self.loop.call_soon(self.approach)
                self.loop.call_later(1, self.extinguish)
                #self.loop.call_later(1, self.go_home)

    def approach(self):
        print((self.ir.data[4] + self.ir.data[5]) / 2)
        if (self.ir.data[4] + self.ir.data[5]) / 2 > 25:
            self.driver.forward()
            self.loop.call_later(0.1, self.approach)
        else:
            self.driver.stop()

    def extinguish(self):
        if self.flamedetect.laststate != 1:
            self.extinguisher.go()
            self.loop.call_later(1, self.extinguish)
        else:
            exit()

    def parallel_correct(self, name, state):
        self.readyled.on = state
        if self.__turning:
            return False
        if not (self.ir.state[NOT_PARALLEL] or
                self.ir.state[OFF_CENTER]):
            if not self.__inroom:
                self.driver.forward()
        elif state:
            direction = [0, 0] # left, right
            print(self.ir.data)
            if self.ir.data[0] < 30 and self.ir.data[1] < 30:
                direction[int(self.ir.data[0] < self.ir.data[1])] += 1
            if self.ir.data[2] < 30 and self.ir.data[3] < 30:
                direction[int(self.ir.data[2] < self.ir.data[3])] += 1
            if self.ir.data[5] < 30 and self.ir.data[4] < 30:
                direction[int(self.ir.data[5] < self.ir.data[4])] += 1
            print(direction, abs(self.ir.data[0] - self.ir.data[1]),
                  abs(self.ir.data[2] - self.ir.data[3]),
                  abs(self.ir.data[5] - self.ir.data[4]))
            if sum(direction) > 0 and sum(direction) != 2:
                if direction[0] < direction[1]:
                    if self.__initial_room or not self.__inroom:
                        self.driver.slightleft(False)
                else:
                    if self.__initial_room or not self.__inroom:
                        self.driver.slightright(False)
            #self.loop.call_later(1, self.parallel_correct, name, state)

    def center_correct(self, name, state):
        if self.__turning or self.__inroom:
            return False
        if not (self.ir.state[NOT_PARALLEL] or
                self.ir.state[OFF_CENTER]):
            self.driver.forward()
        elif state:
            if (self.ir.state[L_OPENING_START] or
                self.ir.state[R_OPENING_START]):
                if self.ir.data[1] > self.ir.data[2]:
                    self.driver.slightright(False)
                else:
                    self.driver.slightleft(False)
            else:
                if self.ir.data[0] > self.ir.data[3]:
                    self.driver.slightright(False)
                else:
                    self.driver.slightleft(False)
            #self.loop.call_later(1, self.center_correct, name, state)

    def end_turn(self):
        self.__turning = False

    def perform_turn(self, name, state, GO=False):
        if not GO and name == FRONT_WALL_DETECT:
            self.videoled.on = state
        elif not GO:
            if self.__turn_handles[name]:
                if not state:
                    pass
                    #print("CANCELLING TURN")
                    #self.__turn_handles[name].cancel()
            self.__turn_handles[name] = self.loop.call_later(
                0.3, self.perform_turn, name, state, True)
            return False
        #else:
        #   print("Executing", name, state)
        if self.__initial_room and name == FRONT_WALL_DETECT and not self.__turning:
            print("LEFT TURN")
            self.__turning = "LEFT"
            self.driver.turnleft()
            self.loop.call_later(TURN_TIME, self.end_turn)
            return False
        if (self.__turning and state) or self.__inroom:
            return False
        elif not state:
            if name == self.__turning:
                print("End TURN")
                self.loop.call_later(TURN_TIME + DELAY_TIME, self.end_turn)
            return False
        if self.ir.state[FOUR_WAY_DETECT]:
            print("4WAY")
            self.__turning = FOUR_WAY_DETECT
            """if self.__fourway:
                if self.__orientation % 2 == self.__fourway[1] % 2:
                    pass
                else:
                    d = L
            else:
                d = R"""
            d = L
            self.__fourway = (len(self.__steps), self.__orientation)
        elif (self.ir.state[L_OPENING] and not (self.__going_backwards and
                                                self.__steps[-1] == R)):
            self.__turning = name
            d = L
        elif (self.ir.state[R_OPENING] and not (self.__going_backwards and
                                                self.__steps[-1] == L)):
            self.__turning = name
            d = R
        elif not self.ir.state[FRONT_WALL_DETECT]:
            self.__turning = name
            print(self.ir.state)
            d = S
        else:
            if self.__going_backwards:
                self.__turning = name
                d = R if self.__steps[-1] == L else L
            else:
                print("TURNING AROUND")
                self.__turning = "AROUND"
                self.turn_around()
                self.loop.call_later(TURN_TIME * 2, self.end_turn)
        self.decision(d)

    def decision(self, d):
        print(d)
        if self.__going_backwards:
            if ((self.__steps[-1] == R and d == L) or
                (self.__steps[-1] == L and d == R)):
                self.__steps.pop()
            else:
                self.__going_backwards = False
        if not self.__going_backwards:
            self.__steps.append(d)
        if d == R:
            self.__orientation += 1
            self.driver.turnright()
        elif d == L:
            self.__orientation -= 1
            self.driver.turnleft()
        else:
            self.driver.forward()
        self.__orientation %= 4

    def __toggle_room(self, manual=False):
        self.__inroom = not self.__inroom
        self.flamedetect.on = self.__inroom and not self.__initial_room
        print(self.flamedetect.on)
        if self.__inroom and not manual:
            self.driver.turnleft(False)
            self.loop.call_later(TURN_TIME * 1.25, self.scan_room)
        self.roomled.on = self.__inroom

    def toggle_room(self, state):
        self.__initial_room = False
        if state:
            self.loop.call_later(DELAY_TIME, self.__toggle_room)

    def turn_around(self):
        self.driver.turnright()
        self.loop.call_later(TURN_TIME, self.driver.turnright)
        self.__orientation += 2
        self.__orientation %= 4
        self.__going_backwards = True
<commit_msg>Better fourway handling<commit_after>import asyncio, atexit

from .position import *
from .drive import DriveMotors, TURN_TIME
from .room import RoomEntryDetection
from .status import LED
from .firefighting import *

from collections import defaultdict

L = "L"
R = "R"
S = "S"
X = "X"

DELAY_TIME = 1.2

VISION_NOTHING = 1
VISION_POSSIBLE = 3
VISION_LIKELY = 4

class Navigator:
    def __init__(self, start_in_room=True):
        self.loop = asyncio.get_event_loop()
        self.ir = IRPositioning()
        self.driver = DriveMotors()
        self.room = RoomEntryDetection()
        self.flamedetect = FlameDetect()
        self.flamedetect.on = True
        self.extinguisher = Extinguisher()

        self.flameled = LED(0)
        self.videoled = LED(1)
        self.readyled = LED(2)
        self.roomled = LED(3)
        self.flameled.on = 0

        self.__turning = False
        self.__turn_handles = defaultdict(bool)

        self.__inroom = not start_in_room
        self.__initial_room = start_in_room
        self.__toggle_room(True)
        self.__turning = None # "INITIAL_ORIENT"

        self.__steps = [X]
        self.__going_backwards = False
        self.__fourway = None
        self.__orientation = 0 # N, E, S, W
        self.__initial_orient = 0

        for i in (L_OPENING, R_OPENING, FRONT_WALL_DETECT, FOUR_WAY_DETECT):
            self.ir.register(i, self.perform_turn)
        self.ir.register(NOT_PARALLEL, self.parallel_correct)
        self.ir.register(OFF_CENTER, self.center_correct)
        self.flamedetect.register(self.vision)
        self.room.register(self.toggle_room)

        atexit.register(self.driver.stop)
        self.loop.call_soon(self.driver.forward) #self.driver.turnleft(None)
        #self.loop.call_later(0.1, self.initial_orient)

    def initial_orient(self):
        print(abs(self.ir.data[5] - self.ir.data[4]))
        turn_time = TURN_TIME / 5
        if abs(self.ir.data[5] - self.ir.data[4]) < 6 and self.ir.data[5]:
            self.__initial_orient += 1
            turn_time = 0.5
            self.driver.stop()
            print("STOPPED")
        else:
            self.__initial_orient = 0
            self.driver.turnleft(None)
            print("TURNING")
        if self.__initial_orient < 4:
            self.loop.call_later(turn_time, self.initial_orient)
        else:
            print("DONE ORIENTING")
            self.driver.cancel()
            self.loop.call_soon(self.driver.forward)
            self.loop.call_later(0.1, self.end_turn)
            print(self.driver.left, self.driver.right)

    def scan_room(self):
        self.__scan_start = self.loop.time()
        self.__giving_up = self.loop.call_later(TURN_TIME * 15, self.give_up)
        self.driver.right = 0.04
        self.driver.left = 0.04

    def give_up(self):
        print("GIVING UP")
        self.__orientation += 2
        self.__orientation %= 4
        self.__going_backwards = True
        self.driver.forward()

    def vision(self, state, deg):
        print(repr(state), VISION_LIKELY, deg)
        if state == VISION_LIKELY:
            if deg < 0:
                self.driver.right = -0.03
                self.driver.left = -0.03
            elif deg > 0:
                self.driver.right = 0.03
                self.driver.left = 0.03
            if abs(deg) < 15:
                self.flameled.on = True
                self.driver.stop()
                self.__back_time = self.loop.time()
                self.extinguisher.go()
                self.__giving_up.cancel()
                self.loop.call_soon(self.approach)
                self.loop.call_later(1, self.extinguish)
                #self.loop.call_later(1, self.go_home)

    def approach(self):
        print((self.ir.data[4] + self.ir.data[5]) / 2)
        if (self.ir.data[4] + self.ir.data[5]) / 2 > 25:
            self.driver.forward()
            self.loop.call_later(0.1, self.approach)
        else:
            self.driver.stop()

    def extinguish(self):
        if self.flamedetect.laststate != 1:
            self.extinguisher.go()
            self.loop.call_later(1, self.extinguish)
        else:
            exit()

    def parallel_correct(self, name, state):
        self.readyled.on = state
        if self.__turning:
            return False
        if not (self.ir.state[NOT_PARALLEL] or
                self.ir.state[OFF_CENTER]):
            if not self.__inroom:
                self.driver.forward()
        elif state:
            direction = [0, 0] # left, right
            print(self.ir.data)
            if self.ir.data[0] < 30 and self.ir.data[1] < 30:
                direction[int(self.ir.data[0] < self.ir.data[1])] += 1
            if self.ir.data[2] < 30 and self.ir.data[3] < 30:
                direction[int(self.ir.data[2] < self.ir.data[3])] += 1
            if self.ir.data[5] < 30 and self.ir.data[4] < 30:
                direction[int(self.ir.data[5] < self.ir.data[4])] += 1
            print(direction, abs(self.ir.data[0] - self.ir.data[1]),
                  abs(self.ir.data[2] - self.ir.data[3]),
                  abs(self.ir.data[5] - self.ir.data[4]))
            if sum(direction) > 0 and sum(direction) != 2:
                if direction[0] < direction[1]:
                    if self.__initial_room or not self.__inroom:
                        self.driver.slightleft(False)
                else:
                    if self.__initial_room or not self.__inroom:
                        self.driver.slightright(False)
            #self.loop.call_later(1, self.parallel_correct, name, state)

    def center_correct(self, name, state):
        if self.__turning or self.__inroom:
            return False
        if not (self.ir.state[NOT_PARALLEL] or
                self.ir.state[OFF_CENTER]):
            self.driver.forward()
        elif state:
            if (self.ir.state[L_OPENING_START] or
                self.ir.state[R_OPENING_START]):
                if self.ir.data[1] > self.ir.data[2]:
                    self.driver.slightright(False)
                else:
                    self.driver.slightleft(False)
            else:
                if self.ir.data[0] > self.ir.data[3]:
                    self.driver.slightright(False)
                else:
                    self.driver.slightleft(False)
            #self.loop.call_later(1, self.center_correct, name, state)

    def end_turn(self):
        self.__turning = False

    def perform_turn(self, name, state, GO=False):
        if not GO and name == FRONT_WALL_DETECT:
            self.videoled.on = state
        elif not GO:
            if self.__turn_handles[name]:
                if not state:
                    pass
                    #print("CANCELLING TURN")
                    #self.__turn_handles[name].cancel()
            self.__turn_handles[name] = self.loop.call_later(
                0.3, self.perform_turn, name, state, True)
            return False
        #else:
        #   print("Executing", name, state)
        if self.__initial_room and name == FRONT_WALL_DETECT and not self.__turning:
            print("LEFT TURN")
            self.__turning = "LEFT"
            self.driver.turnleft()
            self.loop.call_later(TURN_TIME, self.end_turn)
            return False
        if (self.__turning and state) or self.__inroom:
            return False
        elif not state:
            if name == self.__turning:
                print("End TURN")
                self.loop.call_later(TURN_TIME + DELAY_TIME, self.end_turn)
            return False
        fourway = False
        if self.ir.state[FOUR_WAY_DETECT]:
            print("4WAY")
            self.__turning = FOUR_WAY_DETECT
            """if self.__fourway:
                if self.__orientation % 2 == self.__fourway[1] % 2:
                    pass
                else:
                    d = L
            else:
                d = R"""
            d = L
            if self.__fourway and (
                    (self.orientation + 1) % 4 == self.__fourway[1]):
                while len(self.__steps) > self.__fourway[0]:
                    self.__steps.pop()
                self.__going_backwards = True
                fourway = True
            if not self.__fourway:
                self.__fourway = (len(self.__steps), self.__orientation)
        elif (self.ir.state[L_OPENING]
              and not ((self.__going_backwards and self.__steps[-1] == R)
                       or (self.__steps[-1] == self.__steps[-2] == L))):
            self.__turning = name
            d = L
        elif (self.ir.state[R_OPENING]
              and not ((self.__going_backwards and self.__steps[-1] == L)
                       or (self.__steps[-1] == self.__steps[-2] == R))):
            self.__turning = name
            d = R
        elif not self.ir.state[FRONT_WALL_DETECT]:
            self.__turning = name
            print(self.ir.state)
            d = S
        else:
            if self.__going_backwards:
                self.__turning = name
                d = R if self.__steps[-1] == L else L
            else:
                print("TURNING AROUND")
                self.__turning = "AROUND"
                self.turn_around()
                self.loop.call_later(TURN_TIME * 2, self.end_turn)
        self.decision(d, fourway)

    def decision(self, d, fourway=False):
        print(d)
        if self.__going_backwards:
            if ((self.__steps[-1] == R and d == L) or
                (self.__steps[-1] == L and d == R)):
                self.__steps.pop()
            elif fourway:
                pass
            else:
                self.__going_backwards = False
        if not self.__going_backwards:
            self.__steps.append(d)
        if d == R:
            self.__orientation += 1
            self.driver.turnright()
        elif d == L:
            self.__orientation -= 1
            self.driver.turnleft()
        else:
            self.driver.forward()
        self.__orientation %= 4

    def __toggle_room(self, manual=False):
        self.__inroom = not self.__inroom
        self.flamedetect.on = self.__inroom and not self.__initial_room
        print(self.flamedetect.on)
        if self.__inroom and not manual:
            self.driver.turnleft(False)
            self.loop.call_later(TURN_TIME * 1.25, self.scan_room)
        self.roomled.on = self.__inroom

    def toggle_room(self, state):
        self.__initial_room = False
        if state:
            self.loop.call_later(DELAY_TIME, self.__toggle_room)

    def turn_around(self):
        self.driver.turnright()
        self.loop.call_later(TURN_TIME, self.driver.turnright)
        self.__orientation += 2
        self.__orientation %= 4
        self.__going_backwards = True
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
'''
Created on 07.04.2014

@author: heinz-peterlang
'''
import json
import logging

from lxml import etree
import hashlib

logger = logging.getLogger('weblyzard_api.xml_content.parsers')

class XMLParser(object):
    VERSION = None
    SUPPORTED_NAMESPACE = None
    DOCUMENT_NAMESPACES = None
    ATTR_MAPPING = None
    SENTENCE_MAPPING = None
    ANNOTATION_MAPPING = None
    FEATURE_MAPPING = None
    RELATION_MAPPING = None
    DEFAULT_NAMESPACE = 'wl'

    @classmethod
    def encode_value(cls, value):
        if isinstance(value, basestring):
            return value
        else:
            try:
                return json.dumps(value)
            except Exception, e:
                logger.error('could not encode %s: %s' % (value, e))
                return

    @classmethod
    def decode_value(cls, value):
        try:
            decoded = json.loads(value)
            if decoded in (float('inf'), float('-inf'), float('nan')):
                raise ValueError('deserializing of invalid json values')
            else:
                return decoded
        except ValueError as err:
            return value

    @classmethod
    def is_supported(cls, xml_content):
        return 'xmlns:wl="%s"' % cls.SUPPORTED_NAMESPACE in xml_content

        # if xml_content:
        # return any([ns in xml_content for ns in cls.get_supported_namespaces()])

    @classmethod
    def parse(cls, xml_content, remove_duplicates=True):
        ''' '''
        parser = etree.XMLParser(recover=True, strip_cdata=False)
        root = etree.fromstring(xml_content.replace('encoding="UTF-8"', ''),
                                parser=parser)
        try:
            attributes = cls.load_attributes(root.attrib,
                                             mapping=cls.ATTR_MAPPING)
        except Exception, e:
            attributes = {}

        sentences = cls.load_sentences(root, page_attributes=attributes,
                                       remove_duplicates=remove_duplicates)
        title_sentence_ids = [sentence['md5sum'] for sentence in sentences \
                              if 'is_title' in sentence and sentence['is_title']]

        title_annotations = []
        body_annotations = []
        for annotation in cls.load_annotations(root):
            if 'md5sum' in annotation and annotation['md5sum'] in title_sentence_ids:
                title_annotations.append(annotation)
            else:
                body_annotations.append(annotation)

        features = cls.load_features(root, page_attributes=attributes)
        relations = cls.load_relations(root, page_attributes=attributes)
        return attributes, sentences, title_annotations, body_annotations, features, relations

    @classmethod
    def load_attributes(cls, attributes, mapping):
        new_attributes = {}

        for key, value in attributes.iteritems():
            if mapping and key in mapping:
                key = mapping.get(key, key)

            value = cls.decode_value(value)

            if not value == 'None':
                new_attributes[key] = value

        return new_attributes

    @classmethod
    def load_annotations(cls, root):
        ''' '''
        annotations = []
        for annotation_element in root.iterfind('{%s}annotation' % cls.get_default_ns(),
                                          namespaces=cls.DOCUMENT_NAMESPACES):
            annotations.append(cls.load_attributes(annotation_element.attrib,
                                                  mapping=cls.ANNOTATION_MAPPING))

        return annotations

    @classmethod
    def load_sentences(cls, root, page_attributes=None, remove_duplicates=True):
        ''' '''
        sentences = []
        seen_sentences = []

        for sent_element in root.iterfind('{%s}sentence' % cls.get_default_ns(),
                                          namespaces=cls.DOCUMENT_NAMESPACES):

            sent_attributes = cls.load_attributes(sent_element.attrib,
                                                  mapping=cls.SENTENCE_MAPPING)
            sent_attributes['value'] = sent_element.text.strip()

            if 'md5sum' in sent_attributes:
                sent_id = sent_attributes['md5sum']
            elif 'id' in sent_attributes:
                sent_id = sent_attributes['id']
                sent_attributes['md5sum'] = sent_id
                del sent_attributes['id']
            else:
                sent_id = hashlib.md5(sent_element.text.encode('utf-8')).hexdigest()
                sent_attributes['md5sum'] = sent_id

            if not sent_id in seen_sentences:
                sentences.append(sent_attributes)

                if remove_duplicates:
                    seen_sentences.append(sent_id)

        return sentences

    @classmethod
    def cast_item(cls, item):
        if item.lower()=='true':
            return True
        elif item.lower()=='false':
            return False

        try:
            return int(item)
        except Exception:
            pass

        try:
            return float(item)
        except Exception:
            pass

        return item

    @classmethod
    def load_features(cls, root, page_attributes):
        ''' '''
        features = {}
        for feat_element in root.iterfind('{%s}feature' % cls.get_default_ns(),
                                          namespaces=cls.DOCUMENT_NAMESPACES):
            feat_attributes = cls.load_attributes(feat_element.attrib,
                                                  mapping=cls.FEATURE_MAPPING)
            if 'key' in feat_attributes and feat_attributes['key'] in features:
                if not isinstance(features[feat_attributes['key']], list):
                    features[feat_attributes['key']] = [features[feat_attributes['key']]]
                if feat_element.text is not None:
                    features[feat_attributes['key']].append(cls.cast_item(feat_element.text.strip()))
            elif feat_element.text is not None:
                features[feat_attributes['key']] = cls.cast_item(feat_element.text.strip())
        return features

    @classmethod
    def load_relations(cls, root, page_attributes):
        ''' '''
        relations = {}
        for rel_element in root.iterfind('{%s}relation' % cls.get_default_ns(),
                                          namespaces=cls.DOCUMENT_NAMESPACES):
            rel_attributes = cls.load_attributes(rel_element.attrib,
                                                  mapping=cls.RELATION_MAPPING)
            if 'key' in rel_attributes and rel_attributes['key'] in relations:
                if not isinstance(relations[rel_attributes['key']], list):
                    relations[rel_attributes['key']] = [relations[rel_attributes['key']]]
                if rel_element.text is not None:
                    relations[rel_attributes['key']].append(cls.cast_item(rel_element.text.strip()))
            elif rel_element.text is not None:
                relations[rel_attributes['key']] = cls.cast_item(rel_element.text.strip())
        return relations

    @classmethod
    def dump_xml_attributes(cls, attributes, mapping):
        new_attributes = {}

        for key, value in attributes.iteritems():

            if mapping and key in mapping:
                key = mapping[key]
            elif ':' in key:
                continue
            elif '{' in value:
                continue

            if value and value not in ('None', 'null', '0.0'):
                new_attributes[key] = cls.encode_value(value)

        return new_attributes

    @classmethod
    def dump_xml(cls, titles, attributes, sentences, annotations=[],
                 features={}, relations={}):
        ''' returns a webLyzard XML document '''

        attributes, sentences = cls.pre_xml_dump(titles=titles,
                                                 attributes=attributes,
                                                 sentences=sentences)

        if attributes:
            assert isinstance(attributes, dict), 'dict required'

        if cls.ATTR_MAPPING:
            invert_mapping = dict(zip(cls.ATTR_MAPPING.values(),
                cls.ATTR_MAPPING.keys()))
        else:
            invert_mapping = None

        attributes = cls.dump_xml_attributes(attributes=attributes,
                                             mapping=invert_mapping)
        root = etree.Element('{%s}page' % cls.get_default_ns(),
                             attrib=attributes,
                             nsmap=cls.DOCUMENT_NAMESPACES)

        if cls.SENTENCE_MAPPING:
            sent_mapping = dict(zip(cls.SENTENCE_MAPPING.values(),
                cls.SENTENCE_MAPPING.keys()))
        else:
            sent_mapping = None

        for sent in sentences:
            sent = sent.as_dict()
            assert isinstance(sent, dict), 'dict required'
            value = sent['value']
            del sent['value']

            if not value:
                continue

            sent_attributes = cls.dump_xml_attributes(sent,
                                                      mapping=sent_mapping)
            sent_elem = etree.SubElement(root,
                                         '{%s}sentence' % cls.get_default_ns(),
                                         attrib=sent_attributes,
                                         nsmap=cls.DOCUMENT_NAMESPACES)
            if isinstance(value, int):
                value = str(value)

            try:
                sent_elem.text = etree.CDATA(value)
            except Exception, e:
                print('Skipping bad cdata: %s (%s)' % (value, e))
                continue

        if cls.ANNOTATION_MAPPING:
            annotation_mapping = dict(zip(cls.ANNOTATION_MAPPING.values(),
                                    cls.ANNOTATION_MAPPING.keys()))
        else:
            annotation_mapping = None

        if annotations:
            # add all annotations as body annotations
            for a_type, a_items in annotations.iteritems():
                for annotation in a_items:
                    if not isinstance(annotation, dict):
                        continue
                    assert isinstance(annotation, dict), 'dict required'
                    if 'entities' in annotation:
                        for entity in annotation['entities']:
                            entity['annotation_type'] = a_type
                            entity['key'] = annotation['key']
                            preferred_name = annotation['preferredName']
                            if not isinstance(preferred_name, unicode):
                                preferred_name = preferred_name.decode('utf-8')
                            entity['preferredName'] = preferred_name

                            annotation_attributes = cls.dump_xml_attributes(entity,
                                                                            mapping=annotation_mapping)

                            try:
                                etree.SubElement(root,
                                                 '{%s}annotation' % cls.get_default_ns(),
                                                 attrib=annotation_attributes,
                                                 nsmap=cls.DOCUMENT_NAMESPACES)
                            except Exception, e:
                                continue

        # featrure mappings if specified
        if cls.FEATURE_MAPPING and len(cls.FEATURE_MAPPING):
            feature_mapping = dict(zip(cls.FEATURE_MAPPING.values(),
                                       cls.FEATURE_MAPPING.keys()))

            for key, items in features.iteritems():
                feature_attributes = cls.dump_xml_attributes({'key': key},
                                                             mapping=feature_mapping)
                if not isinstance(items, list):
                    items = [items]

                for value in items:
                    try:
                        feat_elem = etree.SubElement(root,
                                                     '{%s}feature' % cls.get_default_ns(),
                                                     attrib=feature_attributes,
                                                     nsmap=cls.DOCUMENT_NAMESPACES)
                        if isinstance(value, int) or isinstance(value, list):
                            value = str(value)

                        feat_elem.text = etree.CDATA(value)
                    except Exception, e:
                        print('Skipping bad cdata: %s (%s)' % (value, e))
                        continue

        # relation mappings, if specified
        if cls.RELATION_MAPPING and len(cls.RELATION_MAPPING):
            relation_mapping = dict(zip(cls.RELATION_MAPPING.values(),
                                       cls.RELATION_MAPPING.keys()))

            for key, items in relations.iteritems():
                rel_attributes = cls.dump_xml_attributes({'key': key},
                                                         mapping=relation_mapping)
                if not isinstance(items, list):
                    items = [items]

                for value in items:
                    try:
                        rel_elem = etree.SubElement(root,
                                                    '{%s}relation' % cls.get_default_ns(),
                                                    attrib=rel_attributes,
                                                    nsmap=cls.DOCUMENT_NAMESPACES)
                        if isinstance(value, int) or isinstance(value, list):
                            value = str(value)

                        rel_elem.text = etree.CDATA(value)
                    except Exception, e:
                        print('Skipping bad cdata: %s (%s)' % (value, e))
                        continue

        return etree.tostring(root, encoding='UTF-8', pretty_print=True)

    @classmethod
    def pre_xml_dump(cls, titles, attributes, sentences):
        ''' overriding this functions allows to perform custom cleanup tasks'''
        return attributes, sentences

    @classmethod
    def get_supported_namespaces(cls):
        if not isinstance(cls.SUPPORTED_NAMESPACES, list):
            return [cls.SUPPORTED_NAMESPACES]
        return cls.SUPPORTED_NAMESPACES

    @classmethod
    def get_default_ns(cls):
        return cls.SUPPORTED_NAMESPACE
<commit_msg>fix: error<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
'''
Created on 07.04.2014

@author: heinz-peterlang
'''
import json
import logging

from lxml import etree
import hashlib

logger = logging.getLogger('weblyzard_api.xml_content.parsers')

class XMLParser(object):
    VERSION = None
    SUPPORTED_NAMESPACE = None
    DOCUMENT_NAMESPACES = None
    ATTR_MAPPING = None
    SENTENCE_MAPPING = None
    ANNOTATION_MAPPING = None
    FEATURE_MAPPING = None
    RELATION_MAPPING = None
    DEFAULT_NAMESPACE = 'wl'

    @classmethod
    def encode_value(cls, value):
        if isinstance(value, basestring):
            return value
        else:
            try:
                return json.dumps(value)
            except Exception, e:
                logger.error('could not encode %s: %s' % (value, e))
                return

    @classmethod
    def decode_value(cls, value):
        try:
            decoded = json.loads(value)
            if decoded in (float('inf'), float('-inf'), float('nan')):
                raise ValueError('deserializing of invalid json values')
            else:
                return decoded
        except ValueError as err:
            return value

    @classmethod
    def is_supported(cls, xml_content):
        return 'xmlns:wl="%s"' % cls.SUPPORTED_NAMESPACE in xml_content

        # if xml_content:
        # return any([ns in xml_content for ns in cls.get_supported_namespaces()])

    @classmethod
    def parse(cls, xml_content, remove_duplicates=True):
        ''' '''
        parser = etree.XMLParser(recover=True, strip_cdata=False)
        root = etree.fromstring(xml_content.replace('encoding="UTF-8"', ''),
                                parser=parser)
        try:
            attributes = cls.load_attributes(root.attrib,
                                             mapping=cls.ATTR_MAPPING)
        except Exception, e:
            attributes = {}

        sentences = cls.load_sentences(root, page_attributes=attributes,
                                       remove_duplicates=remove_duplicates)
        title_sentence_ids = [sentence['md5sum'] for sentence in sentences \
                              if 'is_title' in sentence and sentence['is_title']]

        title_annotations = []
        body_annotations = []
        for annotation in cls.load_annotations(root):
            if 'md5sum' in annotation and annotation['md5sum'] in title_sentence_ids:
                title_annotations.append(annotation)
            else:
                body_annotations.append(annotation)

        features = cls.load_features(root, page_attributes=attributes)
        relations = cls.load_relations(root, page_attributes=attributes)
        return attributes, sentences, title_annotations, body_annotations, features, relations

    @classmethod
    def load_attributes(cls, attributes, mapping):
        new_attributes = {}

        for key, value in attributes.iteritems():
            if mapping and key in mapping:
                key = mapping.get(key, key)

            value = cls.decode_value(value)

            if not value == 'None':
                new_attributes[key] = value

        return new_attributes

    @classmethod
    def load_annotations(cls, root):
        ''' '''
        annotations = []
        for annotation_element in root.iterfind('{%s}annotation' % cls.get_default_ns(),
                                          namespaces=cls.DOCUMENT_NAMESPACES):
            annotations.append(cls.load_attributes(annotation_element.attrib,
                                                  mapping=cls.ANNOTATION_MAPPING))

        return annotations

    @classmethod
    def load_sentences(cls, root, page_attributes=None, remove_duplicates=True):
        ''' '''
        sentences = []
        seen_sentences = []

        for sent_element in root.iterfind('{%s}sentence' % cls.get_default_ns(),
                                          namespaces=cls.DOCUMENT_NAMESPACES):

            sent_attributes = cls.load_attributes(sent_element.attrib,
                                                  mapping=cls.SENTENCE_MAPPING)
            sent_attributes['value'] = sent_element.text.strip()

            if 'md5sum' in sent_attributes:
                sent_id = sent_attributes['md5sum']
            elif 'id' in sent_attributes:
                sent_id = sent_attributes['id']
                sent_attributes['md5sum'] = sent_id
                del sent_attributes['id']
            else:
                sent_id = hashlib.md5(sent_element.text.encode('utf-8')).hexdigest()
                sent_attributes['md5sum'] = sent_id

            if not sent_id in seen_sentences:
                sentences.append(sent_attributes)

                if remove_duplicates:
                    seen_sentences.append(sent_id)

        return sentences

    @classmethod
    def cast_item(cls, item):
        if item.lower()=='true':
            return True
        elif item.lower()=='false':
            return False

        try:
            return int(item)
        except Exception:
            pass

        try:
            return float(item)
        except Exception:
            pass

        return item

    @classmethod
    def load_features(cls, root, page_attributes):
        ''' '''
        features = {}
        for feat_element in root.iterfind('{%s}feature' % cls.get_default_ns(),
                                          namespaces=cls.DOCUMENT_NAMESPACES):
            feat_attributes = cls.load_attributes(feat_element.attrib,
                                                  mapping=cls.FEATURE_MAPPING)
            if 'key' in feat_attributes and feat_attributes['key'] in features:
                if not isinstance(features[feat_attributes['key']], list):
                    features[feat_attributes['key']] = [features[feat_attributes['key']]]
                if feat_element.text is not None:
                    features[feat_attributes['key']].append(cls.cast_item(feat_element.text.strip()))
            elif feat_element.text is not None:
                features[feat_attributes['key']] = cls.cast_item(feat_element.text.strip())
        return features

    @classmethod
    def load_relations(cls, root, page_attributes):
        ''' '''
        relations = {}
        for rel_element in root.iterfind('{%s}relation' % cls.get_default_ns(),
                                          namespaces=cls.DOCUMENT_NAMESPACES):
            rel_attributes = cls.load_attributes(rel_element.attrib,
                                                  mapping=cls.RELATION_MAPPING)
            if 'key' in rel_attributes and rel_attributes['key'] in relations:
                if not isinstance(relations[rel_attributes['key']], list):
                    relations[rel_attributes['key']] = [relations[rel_attributes['key']]]
                if rel_element.text is not None:
                    relations[rel_attributes['key']].append(cls.cast_item(rel_element.text.strip()))
            elif rel_element.text is not None:
                relations[rel_attributes['key']] = cls.cast_item(rel_element.text.strip())
        return relations

    @classmethod
    def dump_xml_attributes(cls, attributes, mapping):
        new_attributes = {}

        for key, value in attributes.iteritems():

            if mapping and key in mapping:
                key = mapping[key]
            elif ':' in key:
                continue

            if value and value not in ('None', 'null', '0.0'):
                new_attributes[key] = cls.encode_value(value)

        return new_attributes

    @classmethod
    def dump_xml(cls, titles, attributes, sentences, annotations=[],
                 features={}, relations={}):
        ''' returns a webLyzard XML document '''

        attributes, sentences = cls.pre_xml_dump(titles=titles,
                                                 attributes=attributes,
                                                 sentences=sentences)

        if attributes:
            assert isinstance(attributes, dict), 'dict required'

        if cls.ATTR_MAPPING:
            invert_mapping = dict(zip(cls.ATTR_MAPPING.values(),
                cls.ATTR_MAPPING.keys()))
        else:
            invert_mapping = None

        attributes = cls.dump_xml_attributes(attributes=attributes,
                                             mapping=invert_mapping)
        root = etree.Element('{%s}page' % cls.get_default_ns(),
                             attrib=attributes,
                             nsmap=cls.DOCUMENT_NAMESPACES)

        if cls.SENTENCE_MAPPING:
            sent_mapping = dict(zip(cls.SENTENCE_MAPPING.values(),
                cls.SENTENCE_MAPPING.keys()))
        else:
            sent_mapping = None

        for sent in sentences:
            sent = sent.as_dict()
            assert isinstance(sent, dict), 'dict required'
            value = sent['value']
            del sent['value']

            if not value:
                continue

            sent_attributes = cls.dump_xml_attributes(sent,
                                                      mapping=sent_mapping)
            sent_elem = etree.SubElement(root,
                                         '{%s}sentence' % cls.get_default_ns(),
                                         attrib=sent_attributes,
                                         nsmap=cls.DOCUMENT_NAMESPACES)
            if isinstance(value, int):
                value = str(value)

            try:
                sent_elem.text = etree.CDATA(value)
            except Exception, e:
                print('Skipping bad cdata: %s (%s)' % (value, e))
                continue

        if cls.ANNOTATION_MAPPING:
            annotation_mapping = dict(zip(cls.ANNOTATION_MAPPING.values(),
                                    cls.ANNOTATION_MAPPING.keys()))
        else:
            annotation_mapping = None

        if annotations:
            # add all annotations as body annotations
            for a_type, a_items in annotations.iteritems():
                for annotation in a_items:
                    if not isinstance(annotation, dict):
                        continue
                    assert isinstance(annotation, dict), 'dict required'
                    if 'entities' in annotation:
                        for entity in annotation['entities']:
                            entity['annotation_type'] = a_type
                            entity['key'] = annotation['key']
                            preferred_name = annotation['preferredName']
                            if not isinstance(preferred_name, unicode):
                                preferred_name = preferred_name.decode('utf-8')
                            entity['preferredName'] = preferred_name

                            annotation_attributes = cls.dump_xml_attributes(entity,
                                                                            mapping=annotation_mapping)

                            try:
                                etree.SubElement(root,
                                                 '{%s}annotation' % cls.get_default_ns(),
                                                 attrib=annotation_attributes,
                                                 nsmap=cls.DOCUMENT_NAMESPACES)
                            except Exception, e:
                                continue

        # featrure mappings if specified
        if cls.FEATURE_MAPPING and len(cls.FEATURE_MAPPING):
            feature_mapping = dict(zip(cls.FEATURE_MAPPING.values(),
                                       cls.FEATURE_MAPPING.keys()))

            for key, items in features.iteritems():
                feature_attributes = cls.dump_xml_attributes({'key': key},
                                                             mapping=feature_mapping)
                if not isinstance(items, list):
                    items = [items]

                for value in items:
                    try:
                        feat_elem = etree.SubElement(root,
                                                     '{%s}feature' % cls.get_default_ns(),
                                                     attrib=feature_attributes,
                                                     nsmap=cls.DOCUMENT_NAMESPACES)
                        if isinstance(value, int) or isinstance(value, list):
                            value = str(value)

                        feat_elem.text = etree.CDATA(value)
                    except Exception, e:
                        print('Skipping bad cdata: %s (%s)' % (value, e))
                        continue

        # relation mappings, if specified
        if cls.RELATION_MAPPING and len(cls.RELATION_MAPPING):
            relation_mapping = dict(zip(cls.RELATION_MAPPING.values(),
                                       cls.RELATION_MAPPING.keys()))

            for key, items in relations.iteritems():
                rel_attributes = cls.dump_xml_attributes({'key': key},
                                                         mapping=relation_mapping)
                if not isinstance(items, list):
                    items = [items]

                for value in items:
                    try:
                        rel_elem = etree.SubElement(root,
                                                    '{%s}relation' % cls.get_default_ns(),
                                                    attrib=rel_attributes,
                                                    nsmap=cls.DOCUMENT_NAMESPACES)
                        if isinstance(value, int) or isinstance(value, list):
                            value = str(value)

                        rel_elem.text = etree.CDATA(value)
                    except Exception, e:
                        print('Skipping bad cdata: %s (%s)' % (value, e))
                        continue

        return etree.tostring(root, encoding='UTF-8', pretty_print=True)

    @classmethod
    def pre_xml_dump(cls, titles, attributes, sentences):
        ''' overriding this functions allows to perform custom cleanup tasks'''
        return attributes, sentences

    @classmethod
    def get_supported_namespaces(cls):
        if not isinstance(cls.SUPPORTED_NAMESPACES, list):
            return [cls.SUPPORTED_NAMESPACES]
        return cls.SUPPORTED_NAMESPACES

    @classmethod
    def get_default_ns(cls):
        return cls.SUPPORTED_NAMESPACE
<|endoftext|>