<commit_before>from __future__ import print_function
import os
import time
import glob
import numpy as np
import matplotlib.pyplot as plt
from pyAudioAnalysis import utilities
from pyAudioAnalysis import audioBasicIO
from pyAudioAnalysis import ShortTermFeatures
eps = 0.00000001

""" Time-domain audio features """

def beat_extraction(short_features, window_size, plot=False):
    """
    This function extracts an estimate of the beat rate for a musical signal.
    ARGUMENTS:
     - short_features:     a np array (n_feats x numOfShortTermWindows)
     - window_size:        window size in seconds
    RETURNS:
     - bpm:            estimates of beats per minute
     - ratio:          a confidence measure
    """

    # Features that are related to the beat tracking task:
    selected_features = [0, 1, 3, 4, 5, 6, 7, 8, 9, 10,
                         11, 12, 13, 14, 15, 16, 17, 18]

    max_beat_time = int(round(2.0 / window_size))
    hist_all = np.zeros((max_beat_time,))
    # for each feature
    for ii, i in enumerate(selected_features):
        # dif threshold (3 x Mean of Difs)
        dif_threshold = 2.0 * (np.abs(short_features[i, 0:-1] -
                                      short_features[i, 1::])).mean()
        if dif_threshold <= 0:
            dif_threshold = 0.0000000000000001
        # detect local maxima
        [pos1, _] = utilities.peakdet(short_features[i, :], dif_threshold)
        position_diffs = []
        # compute histograms of local maxima changes
        for j in range(len(pos1)-1):
            position_diffs.append(pos1[j+1]-pos1[j])
        histogram_times, histogram_edges = \
            np.histogram(position_diffs, np.arange(0.5, max_beat_time + 1.5))
        hist_centers = (histogram_edges[0:-1] + histogram_edges[1::]) / 2.0
        histogram_times = \
            histogram_times.astype(float) / short_features.shape[1]
        hist_all += histogram_times
        if plot:
            plt.subplot(9, 2, ii + 1)
            plt.plot(short_features[i, :], 'k')
            for k in pos1:
                plt.plot(k, short_features[i, k], 'k*')
            f1 = plt.gca()
            f1.axes.get_xaxis().set_ticks([])
            f1.axes.get_yaxis().set_ticks([])

    if plot:
        plt.show(block=False)
        plt.figure()

    # Get beat as the argmax of the agregated histogram:
    max_indices = np.argmax(hist_all)
    bpms = 60 / (hist_centers * window_size)
    bpm = bpms[max_indices]
    # ... and the beat ratio:
    ratio = hist_all[max_indices] / (hist_all.sum() + eps)

    if plot:
        # filter out >500 beats from plotting:
        hist_all = hist_all[bpms < 500]
        bpms = bpms[bpms < 500]

        plt.plot(bpms, hist_all, 'k')
        plt.xlabel('Beats per minute')
        plt.ylabel('Freq Count')
        plt.show(block=True)

    return bpm, ratio

def mid_feature_extraction(signal, sampling_rate, mid_window, mid_step,
                           short_window, short_step):
    """
    Mid-term feature extraction
    """

    short_features, short_feature_names = \
        ShortTermFeatures.feature_extraction(signal, sampling_rate,
                                             short_window, short_step)

    n_stats = 2
    n_feats = len(short_features)
    #mid_window_ratio = int(round(mid_window / short_step))
    mid_window_ratio = round((mid_window -
                              (short_window - short_step)) / short_step)
    mt_step_ratio = int(round(mid_step / short_step))

    mid_features, mid_feature_names = [], []
    for i in range(n_stats * n_feats):
        mid_features.append([])
        mid_feature_names.append("")

    # for each of the short-term features:
    for i in range(n_feats):
        cur_position = 0
        num_short_features = len(short_features[i])
        mid_feature_names[i] = short_feature_names[i] + "_" + "mean"
        mid_feature_names[i + n_feats] = short_feature_names[i] + "_" + "std"

        while cur_position < num_short_features:
            end = cur_position + mid_window_ratio
            if end > num_short_features:
                end = num_short_features
            cur_st_feats = short_features[i][cur_position:end]

            mid_features[i].append(np.mean(cur_st_feats))
            mid_features[i + n_feats].append(np.std(cur_st_feats))
            cur_position += mt_step_ratio
    mid_features = np.array(mid_features)
    mid_features = np.nan_to_num(mid_features)
    return mid_features, short_features, mid_feature_names

""" Feature Extraction Wrappers
 - The first two feature extraction wrappers are used to extract
   long-term averaged audio features for a list of WAV files stored in a
   given category.
   It is important to note that, one single feature is extracted per WAV
   file (not the whole sequence of feature vectors)

 """

def directory_feature_extraction(folder_path, mid_window, mid_step,
                                 short_window, short_step,
                                 compute_beat=True):
    """
    This function extracts the mid-term features of the WAVE files of a
    particular folder.

    The resulting feature vector is extracted by long-term averaging the
    mid-term features.
    Therefore ONE FEATURE VECTOR is extracted for each WAV file.

    ARGUMENTS:
        - folder_path:        the path of the WAVE directory
        - mid_window, mid_step:    mid-term window and step (in seconds)
        - short_window, short_step:    short-term window and step (in seconds)
    """

    mid_term_features = np.array([])
    process_times = []

    types = ('*.wav', '*.aif',  '*.aiff', '*.mp3', '*.au', '*.ogg')
    wav_file_list = []
    for files in types:
        wav_file_list.extend(glob.glob(os.path.join(folder_path, files)))

    wav_file_list = sorted(wav_file_list)
    wav_file_list2, mid_feature_names = [], []
    for i, file_path in enumerate(wav_file_list):
        print("Analyzing file {0:d} of {1:d}: {2:s}".format(i + 1,
                                                            len(wav_file_list),
                                                            file_path))
        if os.stat(file_path).st_size == 0:
            print("   (EMPTY FILE -- SKIPPING)")
            continue
        sampling_rate, signal = audioBasicIO.read_audio_file(file_path)
        if sampling_rate == 0:
            continue

        t1 = time.time()
        signal = audioBasicIO.stereo_to_mono(signal)
        if signal.shape[0] < float(sampling_rate)/5:
            print("  (AUDIO FILE TOO SMALL - SKIPPING)")
            continue
        wav_file_list2.append(file_path)
        if compute_beat:
            mid_features, short_features, mid_feature_names = \
                mid_feature_extraction(signal, sampling_rate,
                                       round(mid_window * sampling_rate),
                                       round(mid_step * sampling_rate),
                                       round(sampling_rate * short_window),
                                       round(sampling_rate * short_step))
            beat, beat_conf = beat_extraction(short_features, short_step)
        else:
            mid_features, _, mid_feature_names = \
                mid_feature_extraction(signal, sampling_rate,
                                       round(mid_window * sampling_rate),
                                       round(mid_step * sampling_rate),
                                       round(sampling_rate * short_window),
                                       round(sampling_rate * short_step))

        mid_features = np.transpose(mid_features)
        mid_features = mid_features.mean(axis=0)
        # long term averaging of mid-term statistics
        if (not np.isnan(mid_features).any()) and \
                (not np.isinf(mid_features).any()):
            if compute_beat:
                mid_features = np.append(mid_features, beat)
                mid_features = np.append(mid_features, beat_conf)
            if len(mid_term_features) == 0:
                # append feature vector
                mid_term_features = mid_features
            else:
                mid_term_features = np.vstack((mid_term_features, mid_features))
            t2 = time.time()
            duration = float(len(signal)) / sampling_rate
            process_times.append((t2 - t1) / duration)
    if len(process_times) > 0:
        print("Feature extraction complexity ratio: "
              "{0:.1f} x realtime".format((1.0 /
                                           np.mean(np.array(process_times)))))
    return mid_term_features, wav_file_list2, mid_feature_names

def multiple_directory_feature_extraction(path_list, mid_window, mid_step,
                                          short_window, short_step,
                                          compute_beat=False):
    """
    Same as dirWavFeatureExtraction, but instead of a single dir it
    takes a list of paths as input and returns a list of feature matrices.
    EXAMPLE:
    [features, classNames] =
           a.dirsWavFeatureExtraction(['audioData/classSegmentsRec/noise',
                                       'audioData/classSegmentsRec/speech',
                                       'audioData/classSegmentsRec/brush-teeth',
                                       'audioData/classSegmentsRec/shower'], 1,
                                       1, 0.02, 0.02);

    It can be used during the training process of a classification model ,
    in order to get feature matrices from various audio classes (each stored in
    a separate path)
    """

    # feature extraction for each class:
    features = []
    class_names = []
    file_names = []
    for i, d in enumerate(path_list):
        f, fn, feature_names = \
            directory_feature_extraction(d, mid_window, mid_step,
                                         short_window, short_step,
                                         compute_beat=compute_beat)
        if f.shape[0] > 0:
            # if at least one audio file has been found in the provided folder:
            features.append(f)
            file_names.append(fn)
            if d[-1] == os.sep:
                class_names.append(d.split(os.sep)[-2])
            else:
                class_names.append(d.split(os.sep)[-1])
    return features, class_names, file_names

def directory_feature_extraction_no_avg(folder_path, mid_window, mid_step,
                                        short_window, short_step):
    """
    This function extracts the mid-term features of the WAVE
    files of a particular folder without averaging each file.

    ARGUMENTS:
        - folder_path:          the path of the WAVE directory
        - mid_window, mid_step:    mid-term window and step (in seconds)
        - short_window, short_step:    short-term window and step (in seconds)
    RETURNS:
        - X:                A feature matrix
        - Y:                A matrix of file labels
        - filenames:
    """

    wav_file_list = []
    signal_idx = np.array([])
    mid_features = np.array([])
    types = ('*.wav', '*.aif',  '*.aiff', '*.ogg')
    for files in types:
        wav_file_list.extend(glob.glob(os.path.join(folder_path, files)))

    wav_file_list = sorted(wav_file_list)

    for i, file_path in enumerate(wav_file_list):
        sampling_rate, signal = audioBasicIO.read_audio_file(file_path)
        if sampling_rate == 0:
            continue
        signal = audioBasicIO.stereo_to_mono(signal)
        mid_feature_vector, _, _ = \
            mid_feature_extraction(signal, sampling_rate,
                                   round(mid_window * sampling_rate),
                                   round(mid_step * sampling_rate),
                                   round(sampling_rate * short_window),
                                   round(sampling_rate * short_step))

        mid_feature_vector = np.transpose(mid_feature_vector)
        if len(mid_features) == 0:                # append feature vector
            mid_features = mid_feature_vector
            signal_idx = np.zeros((mid_feature_vector.shape[0], ))
        else:
            mid_features = np.vstack((mid_features, mid_feature_vector))
            signal_idx = np.append(signal_idx, i *
                                   np.ones((mid_feature_vector.shape[0], )))

    return mid_features, signal_idx, wav_file_list

"""
The following two feature extraction wrappers extract features for given audio
files, however  NO LONG-TERM AVERAGING is performed. Therefore, the output for
each audio file is NOT A SINGLE FEATURE VECTOR but a whole feature matrix.

Also, another difference between the following two wrappers and the previous
is that they NO LONG-TERM AVERAGING IS PERFORMED. In other words, the WAV
files in these functions are not used as uniform samples that need to be
averaged but as sequences
"""

def mid_feature_extraction_to_file(file_path, mid_window, mid_step,
                                   short_window, short_step, output_file,
                                   store_short_features=False, store_csv=False,
                                   plot=False):
    """
    This function is used as a wrapper to:
    a) read the content of a WAV file
    b) perform mid-term feature extraction on that signal
    c) write the mid-term feature sequences to a np file
    """
    sampling_rate, signal = audioBasicIO.read_audio_file(file_path)
    signal = audioBasicIO.stereo_to_mono(signal)
    if store_short_features:
        mid_features, short_features, _ = \
            mid_feature_extraction(signal, sampling_rate,
                                   round(sampling_rate * mid_window),
                                   round(sampling_rate * mid_step),
                                   round(sampling_rate * short_window),
                                   (sampling_rate * short_step))
        # save st features to np file
        np.save(output_file + "_st", short_features)
        if plot:
            print("Short-term np file: " + output_file + "_st.npy saved")
        if store_csv:
            # store st features to CSV file
            np.savetxt(output_file + "_st.csv", short_features.T, delimiter=",")
            if plot:
                print("Short-term CSV file: " + output_file + "_st.csv saved")
    else:
        mid_features, _, _ = \
            mid_feature_extraction(signal, sampling_rate,
                                   round(sampling_rate * mid_window),
                                   round(sampling_rate * mid_step),
                                   round(sampling_rate * short_window),
                                   round(sampling_rate * short_step))
        # save mt features to np file
        np.save(output_file, mid_features)
        if plot:
            print("Mid-term np file: " + output_file + ".npy saved")
        if store_csv:
            np.savetxt(output_file + ".csv", mid_features.T, delimiter=",")
            if plot:
                print("Mid-term CSV file: " + output_file + ".csv saved")

def mid_feature_extraction_file_dir(folder_path, mid_window, mid_step,
                                    short_window, short_step,
                                    store_short_features=False, store_csv=False,
                                    plot=False):
    types = (folder_path + os.sep + '*.wav',)
    files_list = []
    for t in types:
        files_list.extend(glob.glob(t))
    for f in files_list:
        output_path = f
        mid_feature_extraction_to_file(f, mid_window, mid_step, short_window,
                                       short_step, output_path,
                                       store_short_features, store_csv, plot)
<commit_msg>fix issue #318 Mismatch between wiki and program functionality in Feature extraction - storing to file<commit_after>from __future__ import print_function
import os
import time
import glob
import numpy as np
import matplotlib.pyplot as plt
from pyAudioAnalysis import utilities
from pyAudioAnalysis import audioBasicIO
from pyAudioAnalysis import ShortTermFeatures
eps = 0.00000001

""" Time-domain audio features """

def beat_extraction(short_features, window_size, plot=False):
    """
    This function extracts an estimate of the beat rate for a musical signal.
    ARGUMENTS:
     - short_features:     a np array (n_feats x numOfShortTermWindows)
     - window_size:        window size in seconds
    RETURNS:
     - bpm:            estimates of beats per minute
     - ratio:          a confidence measure
    """

    # Features that are related to the beat tracking task:
    selected_features = [0, 1, 3, 4, 5, 6, 7, 8, 9, 10,
                         11, 12, 13, 14, 15, 16, 17, 18]

    max_beat_time = int(round(2.0 / window_size))
    hist_all = np.zeros((max_beat_time,))
    # for each feature
    for ii, i in enumerate(selected_features):
        # dif threshold (3 x Mean of Difs)
        dif_threshold = 2.0 * (np.abs(short_features[i, 0:-1] -
                                      short_features[i, 1::])).mean()
        if dif_threshold <= 0:
            dif_threshold = 0.0000000000000001
        # detect local maxima
        [pos1, _] = utilities.peakdet(short_features[i, :], dif_threshold)
        position_diffs = []
        # compute histograms of local maxima changes
        for j in range(len(pos1)-1):
            position_diffs.append(pos1[j+1]-pos1[j])
        histogram_times, histogram_edges = \
            np.histogram(position_diffs, np.arange(0.5, max_beat_time + 1.5))
        hist_centers = (histogram_edges[0:-1] + histogram_edges[1::]) / 2.0
        histogram_times = \
            histogram_times.astype(float) / short_features.shape[1]
        hist_all += histogram_times
        if plot:
            plt.subplot(9, 2, ii + 1)
            plt.plot(short_features[i, :], 'k')
            for k in pos1:
                plt.plot(k, short_features[i, k], 'k*')
            f1 = plt.gca()
            f1.axes.get_xaxis().set_ticks([])
            f1.axes.get_yaxis().set_ticks([])

    if plot:
        plt.show(block=False)
        plt.figure()

    # Get beat as the argmax of the agregated histogram:
    max_indices = np.argmax(hist_all)
    bpms = 60 / (hist_centers * window_size)
    bpm = bpms[max_indices]
    # ... and the beat ratio:
    ratio = hist_all[max_indices] / (hist_all.sum() + eps)

    if plot:
        # filter out >500 beats from plotting:
        hist_all = hist_all[bpms < 500]
        bpms = bpms[bpms < 500]

        plt.plot(bpms, hist_all, 'k')
        plt.xlabel('Beats per minute')
        plt.ylabel('Freq Count')
        plt.show(block=True)

    return bpm, ratio

def mid_feature_extraction(signal, sampling_rate, mid_window, mid_step,
                           short_window, short_step):
    """
    Mid-term feature extraction
    """

    short_features, short_feature_names = \
        ShortTermFeatures.feature_extraction(signal, sampling_rate,
                                             short_window, short_step)

    n_stats = 2
    n_feats = len(short_features)
    #mid_window_ratio = int(round(mid_window / short_step))
    mid_window_ratio = round((mid_window -
                              (short_window - short_step)) / short_step)
    mt_step_ratio = int(round(mid_step / short_step))

    mid_features, mid_feature_names = [], []
    for i in range(n_stats * n_feats):
        mid_features.append([])
        mid_feature_names.append("")

    # for each of the short-term features:
    for i in range(n_feats):
        cur_position = 0
        num_short_features = len(short_features[i])
        mid_feature_names[i] = short_feature_names[i] + "_" + "mean"
        mid_feature_names[i + n_feats] = short_feature_names[i] + "_" + "std"

        while cur_position < num_short_features:
            end = cur_position + mid_window_ratio
            if end > num_short_features:
                end = num_short_features
            cur_st_feats = short_features[i][cur_position:end]

            mid_features[i].append(np.mean(cur_st_feats))
            mid_features[i + n_feats].append(np.std(cur_st_feats))
            cur_position += mt_step_ratio
    mid_features = np.array(mid_features)
    mid_features = np.nan_to_num(mid_features)
    return mid_features, short_features, mid_feature_names

""" Feature Extraction Wrappers
 - The first two feature extraction wrappers are used to extract
   long-term averaged audio features for a list of WAV files stored in a
   given category.
   It is important to note that, one single feature is extracted per WAV
   file (not the whole sequence of feature vectors)

 """

def directory_feature_extraction(folder_path, mid_window, mid_step,
                                 short_window, short_step,
                                 compute_beat=True):
    """
    This function extracts the mid-term features of the WAVE files of a
    particular folder.

    The resulting feature vector is extracted by long-term averaging the
    mid-term features.
    Therefore ONE FEATURE VECTOR is extracted for each WAV file.

    ARGUMENTS:
        - folder_path:        the path of the WAVE directory
        - mid_window, mid_step:    mid-term window and step (in seconds)
        - short_window, short_step:    short-term window and step (in seconds)
    """

    mid_term_features = np.array([])
    process_times = []

    types = ('*.wav', '*.aif',  '*.aiff', '*.mp3', '*.au', '*.ogg')
    wav_file_list = []
    for files in types:
        wav_file_list.extend(glob.glob(os.path.join(folder_path, files)))

    wav_file_list = sorted(wav_file_list)
    wav_file_list2, mid_feature_names = [], []
    for i, file_path in enumerate(wav_file_list):
        print("Analyzing file {0:d} of {1:d}: {2:s}".format(i + 1,
                                                            len(wav_file_list),
                                                            file_path))
        if os.stat(file_path).st_size == 0:
            print("   (EMPTY FILE -- SKIPPING)")
            continue
        sampling_rate, signal = audioBasicIO.read_audio_file(file_path)
        if sampling_rate == 0:
            continue

        t1 = time.time()
        signal = audioBasicIO.stereo_to_mono(signal)
        if signal.shape[0] < float(sampling_rate)/5:
            print("  (AUDIO FILE TOO SMALL - SKIPPING)")
            continue
        wav_file_list2.append(file_path)
        if compute_beat:
            mid_features, short_features, mid_feature_names = \
                mid_feature_extraction(signal, sampling_rate,
                                       round(mid_window * sampling_rate),
                                       round(mid_step * sampling_rate),
                                       round(sampling_rate * short_window),
                                       round(sampling_rate * short_step))
            beat, beat_conf = beat_extraction(short_features, short_step)
        else:
            mid_features, _, mid_feature_names = \
                mid_feature_extraction(signal, sampling_rate,
                                       round(mid_window * sampling_rate),
                                       round(mid_step * sampling_rate),
                                       round(sampling_rate * short_window),
                                       round(sampling_rate * short_step))

        mid_features = np.transpose(mid_features)
        mid_features = mid_features.mean(axis=0)
        # long term averaging of mid-term statistics
        if (not np.isnan(mid_features).any()) and \
                (not np.isinf(mid_features).any()):
            if compute_beat:
                mid_features = np.append(mid_features, beat)
                mid_features = np.append(mid_features, beat_conf)
            if len(mid_term_features) == 0:
                # append feature vector
                mid_term_features = mid_features
            else:
                mid_term_features = np.vstack((mid_term_features, mid_features))
            t2 = time.time()
            duration = float(len(signal)) / sampling_rate
            process_times.append((t2 - t1) / duration)
    if len(process_times) > 0:
        print("Feature extraction complexity ratio: "
              "{0:.1f} x realtime".format((1.0 /
                                           np.mean(np.array(process_times)))))
    return mid_term_features, wav_file_list2, mid_feature_names

def multiple_directory_feature_extraction(path_list, mid_window, mid_step,
                                          short_window, short_step,
                                          compute_beat=False):
    """
    Same as dirWavFeatureExtraction, but instead of a single dir it
    takes a list of paths as input and returns a list of feature matrices.
    EXAMPLE:
    [features, classNames] =
           a.dirsWavFeatureExtraction(['audioData/classSegmentsRec/noise',
                                       'audioData/classSegmentsRec/speech',
                                       'audioData/classSegmentsRec/brush-teeth',
                                       'audioData/classSegmentsRec/shower'], 1,
                                       1, 0.02, 0.02);

    It can be used during the training process of a classification model ,
    in order to get feature matrices from various audio classes (each stored in
    a separate path)
    """

    # feature extraction for each class:
    features = []
    class_names = []
    file_names = []
    for i, d in enumerate(path_list):
        f, fn, feature_names = \
            directory_feature_extraction(d, mid_window, mid_step,
                                         short_window, short_step,
                                         compute_beat=compute_beat)
        if f.shape[0] > 0:
            # if at least one audio file has been found in the provided folder:
            features.append(f)
            file_names.append(fn)
            if d[-1] == os.sep:
                class_names.append(d.split(os.sep)[-2])
            else:
                class_names.append(d.split(os.sep)[-1])
    return features, class_names, file_names

def directory_feature_extraction_no_avg(folder_path, mid_window, mid_step,
                                        short_window, short_step):
    """
    This function extracts the mid-term features of the WAVE
    files of a particular folder without averaging each file.

    ARGUMENTS:
        - folder_path:          the path of the WAVE directory
        - mid_window, mid_step:    mid-term window and step (in seconds)
        - short_window, short_step:    short-term window and step (in seconds)
    RETURNS:
        - X:                A feature matrix
        - Y:                A matrix of file labels
        - filenames:
    """

    wav_file_list = []
    signal_idx = np.array([])
    mid_features = np.array([])
    types = ('*.wav', '*.aif',  '*.aiff', '*.ogg')
    for files in types:
        wav_file_list.extend(glob.glob(os.path.join(folder_path, files)))

    wav_file_list = sorted(wav_file_list)

    for i, file_path in enumerate(wav_file_list):
        sampling_rate, signal = audioBasicIO.read_audio_file(file_path)
        if sampling_rate == 0:
            continue
        signal = audioBasicIO.stereo_to_mono(signal)
        mid_feature_vector, _, _ = \
            mid_feature_extraction(signal, sampling_rate,
                                   round(mid_window * sampling_rate),
                                   round(mid_step * sampling_rate),
                                   round(sampling_rate * short_window),
                                   round(sampling_rate * short_step))

        mid_feature_vector = np.transpose(mid_feature_vector)
        if len(mid_features) == 0:                # append feature vector
            mid_features = mid_feature_vector
            signal_idx = np.zeros((mid_feature_vector.shape[0], ))
        else:
            mid_features = np.vstack((mid_features, mid_feature_vector))
            signal_idx = np.append(signal_idx, i *
                                   np.ones((mid_feature_vector.shape[0], )))

    return mid_features, signal_idx, wav_file_list

"""
The following two feature extraction wrappers extract features for given audio
files, however  NO LONG-TERM AVERAGING is performed. Therefore, the output for
each audio file is NOT A SINGLE FEATURE VECTOR but a whole feature matrix.

Also, another difference between the following two wrappers and the previous
is that they NO LONG-TERM AVERAGING IS PERFORMED. In other words, the WAV
files in these functions are not used as uniform samples that need to be
averaged but as sequences
"""

def mid_feature_extraction_to_file(file_path, mid_window, mid_step,
                                   short_window, short_step, output_file,
                                   store_short_features=False, store_csv=False,
                                   plot=False):
    """
    This function is used as a wrapper to:
    a) read the content of a WAV file
    b) perform mid-term feature extraction on that signal
    c) write the mid-term feature sequences to a np file
    d) optionally write contents to csv file as well
    e) optionally write short-term features in csv and np file
    """
    sampling_rate, signal = audioBasicIO.read_audio_file(file_path)
    signal = audioBasicIO.stereo_to_mono(signal)
    mid_features, short_features, _ = \
        mid_feature_extraction(signal, sampling_rate,
                               round(sampling_rate * mid_window),
                               round(sampling_rate * mid_step),
                               round(sampling_rate * short_window),
                               round(sampling_rate * short_step))
    if store_short_features:
        # save st features to np file
        np.save(output_file + "_st", short_features)
        if plot:
            print("Short-term np file: " + output_file + "_st.npy saved")
        if store_csv:
            # store st features to CSV file
            np.savetxt(output_file + "_st.csv", short_features.T, delimiter=",")
            if plot:
                print("Short-term CSV file: " + output_file + "_st.csv saved")

    # save mt features to np file
    np.save(output_file + "_mt", mid_features)
    if plot:
        print("Mid-term np file: " + output_file + "_mt.npy saved")
    if store_csv:
        np.savetxt(output_file + "_mt.csv", mid_features.T, delimiter=",")
        if plot:
            print("Mid-term CSV file: " + output_file + "_mt.csv saved")

def mid_feature_extraction_file_dir(folder_path, mid_window, mid_step,
                                    short_window, short_step,
                                    store_short_features=False, store_csv=False,
                                    plot=False):
    types = (folder_path + os.sep + '*.wav',)
    files_list = []
    for t in types:
        files_list.extend(glob.glob(t))
    for f in files_list:
        output_path = f
        mid_feature_extraction_to_file(f, mid_window, mid_step, short_window,
                                       short_step, output_path,
                                       store_short_features, store_csv, plot)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# © 2009 Jordi Esteve <jesteve@zikzakmedia.com>
# © 2013 Ignacio Ibeas <ignacio@acysos.com>
# © 2015 Sergio Teruel <sergio@incaser.es>
# © 2013-2016 Pedro M. Baeza <pedro.baeza@tecnativa.com>
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

{
    "name": "Adaptación de los clientes, proveedores y bancos para España",
    "version": "8.0.1.5.1",
    "author": "ZikZak,"
              "Acysos,"
              "Tecnativa,"
              "Odoo Community Association (OCA)",
    "website": "https://github.com/OCA/l10n-spain",
    "category": "Localisation/Europe",
    "license": "AGPL-3",
    "external_dependencies": {
        'python': [
            'requests',
        ],
    },
    "depends": [
        "base",
        "base_iban",
        "base_vat",
        "l10n_es_toponyms",
    ],
    "data": [
        "views/l10n_es_partner_view.xml",
        "wizard/l10n_es_partner_wizard.xml",
    ],
    "installable": True,
}
<commit_msg>[MIG] Make modules uninstallable<commit_after># -*- coding: utf-8 -*-
# © 2009 Jordi Esteve <jesteve@zikzakmedia.com>
# © 2013 Ignacio Ibeas <ignacio@acysos.com>
# © 2015 Sergio Teruel <sergio@incaser.es>
# © 2013-2016 Pedro M. Baeza <pedro.baeza@tecnativa.com>
# License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html

{
    "name": "Adaptación de los clientes, proveedores y bancos para España",
    "version": "8.0.1.5.1",
    "author": "ZikZak,"
              "Acysos,"
              "Tecnativa,"
              "Odoo Community Association (OCA)",
    "website": "https://github.com/OCA/l10n-spain",
    "category": "Localisation/Europe",
    "license": "AGPL-3",
    "external_dependencies": {
        'python': [
            'requests',
        ],
    },
    "depends": [
        "base",
        "base_iban",
        "base_vat",
        "l10n_es_toponyms",
    ],
    "data": [
        "views/l10n_es_partner_view.xml",
        "wizard/l10n_es_partner_wizard.xml",
    ],
    'installable': False,
}
<|endoftext|>


<commit_before># --------------------------------------------------------------------------- #
#   list.py                                                                   #
#                                                                             #
#   Copyright © 2015-2021, Rajiv Bakulesh Shah, original author.              #
#                                                                             #
#   Licensed under the Apache License, Version 2.0 (the "License");           #
#   you may not use this file except in compliance with the License.          #
#   You may obtain a copy of the License at:                                  #
#       http://www.apache.org/licenses/LICENSE-2.0                            #
#                                                                             #
#   Unless required by applicable law or agreed to in writing, software       #
#   distributed under the License is distributed on an "AS IS" BASIS,         #
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  #
#   See the License for the specific language governing permissions and       #
#   limitations under the License.                                            #
# --------------------------------------------------------------------------- #

# TODO: Remove the following import after deferred evaluation of annotations
# because the default.
#   1. https://docs.python.org/3/whatsnew/3.7.html#whatsnew37-pep563
#   2. https://www.python.org/dev/peps/pep-0563/
#   3. https://www.python.org/dev/peps/pep-0649/
from __future__ import annotations

import collections.abc
import functools
import itertools
import uuid
import warnings
from typing import Any
from typing import Callable
from typing import Iterable
from typing import List
from typing import Optional
from typing import Union
from typing import cast

from redis import Redis
from redis import ResponseError
from redis.client import Pipeline

from .annotations import F
from .base import Base
from .base import JSONTypes
from .exceptions import InefficientAccessWarning
from .exceptions import KeyExistsError

def _raise_on_error(func: F) -> Callable[[F], F]:
    @functools.wraps(func)
    def wrapper(*args: Any, **kwargs: Any) -> Any:
        try:
            return func(*args, **kwargs)
        except ResponseError as error:
            raise IndexError('list assignment index out of range') from error
    return wrapper

class RedisList(Base, collections.abc.MutableSequence):
    'Redis-backed container compatible with Python lists.'

    def __slice_to_indices(self, slice_or_index: Union[slice, int]) -> range:
        if isinstance(slice_or_index, slice):
            start, stop, step = cast(slice, slice_or_index).indices(len(self))
        elif isinstance(slice_or_index, int):
            start = cast(int, slice_or_index)
            stop = cast(int, slice_or_index) + 1
            step = 1
        else:
            raise TypeError(
                'list indices must be integers or slices, '
                f'not {slice_or_index.__class__.__name__}'
            )
        indices = range(start, stop, step)
        return indices

    def __init__(self,
                 iterable: Iterable[JSONTypes] = tuple(),
                 *,
                 redis: Optional[Redis] = None,
                 key: Optional[str] = None,
                 ) -> None:
        'Initialize the RedisList.  O(n)'
        super().__init__(redis=redis, key=key)
        if iterable:
            with self._watch(iterable) as pipeline:
                if pipeline.exists(self.key):
                    raise KeyExistsError(pipeline, self.key)
                self._populate(pipeline, iterable)

    def _populate(self,
                  pipeline: Pipeline,
                  iterable: Iterable[JSONTypes] = tuple(),
                  ) -> None:
        encoded_values = [self._encode(value) for value in iterable]
        if encoded_values:
            pipeline.multi()
            pipeline.rpush(self.key, *encoded_values)

    # Methods required by collections.abc.MutableSequence:

    def __getitem__(self, index: Union[slice, int]) -> Any:
        'l.__getitem__(index) <==> l[index].  O(n)'
        with self._watch() as pipeline:
            if isinstance(index, slice):
                # Python's list API requires us to get elements by slice (a
                # start index, a stop index, and a step).  Redis supports only
                # getting elements by start and stop (no step).  So our
                # workaround is to get all of the elements between start and
                # stop from Redis, then discard the ones between step in Python
                # code.  More info:
                #   http://redis.io/commands/lrange
                warnings.warn(
                    cast(str, InefficientAccessWarning.__doc__),
                    InefficientAccessWarning,
                )
                indices = self.__slice_to_indices(index)
                if indices.step >= 0:
                    start, stop = indices.start, indices.stop-1
                else:
                    start, stop = indices.stop+1, indices.start
                pipeline.multi()
                pipeline.lrange(self.key, start, stop)
                encoded = pipeline.execute()[0]
                encoded = encoded[::index.step]
                value: Union[List[JSONTypes], JSONTypes] = [
                    self._decode(value) for value in encoded
                ]
            else:
                index = self.__slice_to_indices(index).start
                len_ = cast(int, pipeline.llen(self.key))
                if index not in {-1, 0, len_-1}:
                    warnings.warn(
                        cast(str, InefficientAccessWarning.__doc__),
                        InefficientAccessWarning,
                    )
                encoded = pipeline.lindex(self.key, index)
                if encoded is None:
                    raise IndexError('list index out of range')
                value = self._decode(encoded)
        return value

    @_raise_on_error
    def __setitem__(self, index: int, value: JSONTypes) -> None:  # type: ignore
        'l.__setitem__(index, value) <==> l[index] = value.  O(n)'
        with self._watch() as pipeline:
            if isinstance(index, slice):
                warnings.warn(
                    cast(str, InefficientAccessWarning.__doc__),
                    InefficientAccessWarning,
                )
                encoded_values = [self._encode(value) for value in value]
                indices = self.__slice_to_indices(index)
                pipeline.multi()
                for index, encoded_value in zip(indices, encoded_values):
                    pipeline.lset(self.key, index, encoded_value)
                indices, num = indices[len(encoded_values):], 0
                for index in indices:
                    pipeline.lset(self.key, index, 0)
                    num += 1
                if num:
                    pipeline.lrem(self.key, num, 0)
            else:
                index = self.__slice_to_indices(index).start
                len_ = cast(int, pipeline.llen(self.key))
                if index not in {-1, 0, len_-1}:
                    warnings.warn(
                        cast(str, InefficientAccessWarning.__doc__),
                        InefficientAccessWarning,
                    )
                pipeline.multi()
                pipeline.lset(self.key, index, self._encode(value))

    @_raise_on_error
    def __delitem__(self, index: Union[slice, int]) -> None:  # type: ignore
        'l.__delitem__(index) <==> del l[index].  O(n)'
        with self._watch() as pipeline:
            self.__delete(pipeline, index)

    # Preserve the Open-Closed Principle with name mangling.
    #   https://youtu.be/miGolgp9xq8?t=2086
    #   https://stackoverflow.com/a/38534939
    def __delete(self, pipeline: Pipeline, index: Union[slice, int]) -> None:
        # Python's list API requires us to delete an element by *index.*  Redis
        # supports only deleting an element by *value.*  So our workaround is
        # to set l[index] to a UUID4, then to delete the value UUID4.  More
        # info:
        #   http://redis.io/commands/lrem
        warnings.warn(
            cast(str, InefficientAccessWarning.__doc__),
            InefficientAccessWarning,
        )
        indices, num = self.__slice_to_indices(index), 0
        uuid4 = str(uuid.uuid4())
        pipeline.multi()
        for index in indices:
            pipeline.lset(self.key, index, uuid4)
            num += 1
        if num:
            pipeline.lrem(self.key, num, uuid4)

    def __len__(self) -> int:
        'Return the number of items in the RedisList.  O(1)'
        return self.redis.llen(self.key)

    def insert(self, index: int, value: JSONTypes) -> None:
        'Insert an element into the RedisDeque before the given index.  O(n)'
        with self._watch() as pipeline:
            self.__insert(index, value, pipeline=pipeline)

    def _insert(self,
                index: int,
                value: JSONTypes,
                *,
                pipeline: Pipeline,
                ) -> None:
        encoded_value = self._encode(value)
        current_length = cast(int, pipeline.llen(self.key))
        if 0 < index < current_length:
            # Python's list API requires us to insert an element before the
            # given *index.*  Redis supports only inserting an element before a
            # given (pivot) *value.*  So our workaround is to set the pivot
            # value to a UUID4, then to insert the desired value before the
            # UUID4, then to set the value UUID4 back to the original pivot
            # pivot value.  More info:
            #   http://redis.io/commands/linsert
            warnings.warn(
                cast(str, InefficientAccessWarning.__doc__),
                InefficientAccessWarning,
            )
            uuid4 = str(uuid.uuid4())
            pivot = cast(bytes, pipeline.lindex(self.key, index))
            pipeline.multi()
            pipeline.lset(self.key, index, uuid4)
            pipeline.linsert(self.key, 'BEFORE', uuid4, encoded_value)
            pipeline.lset(self.key, index+1, pivot)
        else:
            pipeline.multi()
            push_method = pipeline.lpush if index <= 0 else pipeline.rpush
            push_method(self.key, encoded_value)

    __insert = _insert

    # Methods required for Raj's sanity:

    def sort(self, *, key: Optional[str] = None, reverse: bool = False) -> None:
        'Sort the RedisList in place.  O(n)'
        if key is not None:
            raise NotImplementedError('sorting by key not implemented')
        warnings.warn(
            cast(str, InefficientAccessWarning.__doc__),
            InefficientAccessWarning,
        )
        self.redis.sort(self.key, desc=reverse, store=self.key)

    def __eq__(self, other: Any) -> bool:
        if self is other:
            return True

        if self._same_redis(other) and self.key == other.key:
            # self and other are both RedisLists on the same Redis database and
            # with the same key.  No need to compare element by element.
            return True

        with self._watch(other) as pipeline:
            len_xs = cast(int, pipeline.llen(self.key))
            if isinstance(other, RedisList):
                len_ys = cast(int, pipeline.llen(other.key))
            else:
                try:
                    len_ys = len(other)
                except TypeError:
                    # TypeError: other has no len()
                    return False
            if len_xs != len_ys:
                # self and other are different lengths.
                return False

            if isinstance(other, collections.abc.MutableSequence):
                # self and other are the same length, and other is a mutable
                # sequence too.  Compare self's and other's elements, pair by
                # pair.
                warnings.warn(
                    cast(str, InefficientAccessWarning.__doc__),
                    InefficientAccessWarning,
                )
                encoded_xs = cast(
                    List[bytes],
                    pipeline.lrange(self.key, 0, -1),
                )
                decoded_xs = (self._decode(x) for x in encoded_xs)
                if isinstance(other, RedisList):
                    encoded_ys = cast(
                        List[bytes],
                        pipeline.lrange(other.key, 0, -1),
                    )
                    decoded_ys: collections.abc.Iterable[Any] = (  # pragma: no cover
                        self._decode(y) for y in encoded_ys
                    )
                else:
                    decoded_ys = other
                return all(x == y for x, y in zip(decoded_xs, decoded_ys))

        # self and other are the same length, but other is an unordered
        # collection.
        return False

    def __add__(self, other: List[JSONTypes]) -> RedisList:
        'Append the items in other to the RedisList.  O(n)'
        warnings.warn(
            cast(str, InefficientAccessWarning.__doc__),
            InefficientAccessWarning,
        )
        with self._watch(other):
            iterable = itertools.chain(self, other)
            return self.__class__(iterable, redis=self.redis)

    def __repr__(self) -> str:
        'Return the string representation of the RedisList.  O(n)'
        warnings.warn(
            cast(str, InefficientAccessWarning.__doc__),
            InefficientAccessWarning,
        )
        return self.__class__.__name__ + str(self.to_list())

    # Method overrides:

    # From collections.abc.MutableSequence:
    def append(self, value: JSONTypes) -> None:
        'Add an element to the right side of the RedisList.  O(1)'
        self.__extend((value,))

    # From collections.abc.MutableSequence:
    def extend(self, values: Iterable[JSONTypes]) -> None:
        'Extend the RedisList by appending elements from the iterable.  O(1)'
        encoded_values = (self._encode(value) for value in values)
        self.redis.rpush(self.key, *encoded_values)

    __extend = extend

    # From collections.abc.MutableSequence:
    def pop(self, index: Optional[int] = None) -> JSONTypes:
        with self._watch() as pipeline:
            len_ = len(self)
            if index and index >= len_:
                raise IndexError('pop index out of range')
            elif index in {0, None, len_-1, -1}:
                pop_method = 'lpop' if index == 0 else 'rpop'
                pipeline.multi()
                getattr(pipeline, pop_method)(self.key)
                encoded_value = pipeline.execute()[0]
                if encoded_value is None:
                    raise IndexError(
                        f'pop from an empty {self.__class__.__name__}'
                    )
                return self._decode(encoded_value)
            else:
                value: JSONTypes = self[cast(int, index)]
                self.__delete(pipeline, cast(int, index))
                return value

    # From collections.abc.MutableSequence:
    def remove(self, value: JSONTypes) -> None:
        'Remove first occurrence of value.  O(n)'
        warnings.warn(
            cast(str, InefficientAccessWarning.__doc__),
            InefficientAccessWarning,
        )
        with self._watch() as pipeline:
            for index, element in enumerate(self):
                if element == value:
                    self.__delete(pipeline, index)
                    return
        class_name = self.__class__.__name__
        raise ValueError(f'{class_name}.remove(x): x not in {class_name}')

    def to_list(self) -> List[JSONTypes]:
        'Convert the RedisList to a Python list.  O(n)'
        warnings.warn(
            cast(str, InefficientAccessWarning.__doc__),
            InefficientAccessWarning,
        )
        encoded = self.redis.lrange(self.key, 0, -1)
        values = [self._decode(value) for value in encoded]
        return values
<commit_msg>Preserve Open-Closed Principle with name mangling (#568)<commit_after># --------------------------------------------------------------------------- #
#   list.py                                                                   #
#                                                                             #
#   Copyright © 2015-2021, Rajiv Bakulesh Shah, original author.              #
#                                                                             #
#   Licensed under the Apache License, Version 2.0 (the "License");           #
#   you may not use this file except in compliance with the License.          #
#   You may obtain a copy of the License at:                                  #
#       http://www.apache.org/licenses/LICENSE-2.0                            #
#                                                                             #
#   Unless required by applicable law or agreed to in writing, software       #
#   distributed under the License is distributed on an "AS IS" BASIS,         #
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  #
#   See the License for the specific language governing permissions and       #
#   limitations under the License.                                            #
# --------------------------------------------------------------------------- #

# TODO: Remove the following import after deferred evaluation of annotations
# because the default.
#   1. https://docs.python.org/3/whatsnew/3.7.html#whatsnew37-pep563
#   2. https://www.python.org/dev/peps/pep-0563/
#   3. https://www.python.org/dev/peps/pep-0649/
from __future__ import annotations

import collections.abc
import functools
import itertools
import uuid
import warnings
from typing import Any
from typing import Callable
from typing import Iterable
from typing import List
from typing import Optional
from typing import Union
from typing import cast

from redis import Redis
from redis import ResponseError
from redis.client import Pipeline

from .annotations import F
from .base import Base
from .base import JSONTypes
from .exceptions import InefficientAccessWarning
from .exceptions import KeyExistsError

def _raise_on_error(func: F) -> Callable[[F], F]:
    @functools.wraps(func)
    def wrapper(*args: Any, **kwargs: Any) -> Any:
        try:
            return func(*args, **kwargs)
        except ResponseError as error:
            raise IndexError('list assignment index out of range') from error
    return wrapper

class RedisList(Base, collections.abc.MutableSequence):
    'Redis-backed container compatible with Python lists.'

    def __slice_to_indices(self, slice_or_index: Union[slice, int]) -> range:
        if isinstance(slice_or_index, slice):
            start, stop, step = cast(slice, slice_or_index).indices(len(self))
        elif isinstance(slice_or_index, int):
            start = cast(int, slice_or_index)
            stop = cast(int, slice_or_index) + 1
            step = 1
        else:
            raise TypeError(
                'list indices must be integers or slices, '
                f'not {slice_or_index.__class__.__name__}'
            )
        indices = range(start, stop, step)
        return indices

    def __init__(self,
                 iterable: Iterable[JSONTypes] = tuple(),
                 *,
                 redis: Optional[Redis] = None,
                 key: Optional[str] = None,
                 ) -> None:
        'Initialize the RedisList.  O(n)'
        super().__init__(redis=redis, key=key)
        if iterable:
            with self._watch(iterable) as pipeline:
                if pipeline.exists(self.key):
                    raise KeyExistsError(pipeline, self.key)
                self._populate(pipeline, iterable)

    def _populate(self,
                  pipeline: Pipeline,
                  iterable: Iterable[JSONTypes] = tuple(),
                  ) -> None:
        encoded_values = [self._encode(value) for value in iterable]
        if encoded_values:
            pipeline.multi()
            pipeline.rpush(self.key, *encoded_values)

    # Methods required by collections.abc.MutableSequence:

    def __getitem__(self, index: Union[slice, int]) -> Any:
        'l.__getitem__(index) <==> l[index].  O(n)'
        with self._watch() as pipeline:
            if isinstance(index, slice):
                # Python's list API requires us to get elements by slice (a
                # start index, a stop index, and a step).  Redis supports only
                # getting elements by start and stop (no step).  So our
                # workaround is to get all of the elements between start and
                # stop from Redis, then discard the ones between step in Python
                # code.  More info:
                #   http://redis.io/commands/lrange
                warnings.warn(
                    cast(str, InefficientAccessWarning.__doc__),
                    InefficientAccessWarning,
                )
                indices = self.__slice_to_indices(index)
                if indices.step >= 0:
                    start, stop = indices.start, indices.stop-1
                else:
                    start, stop = indices.stop+1, indices.start
                pipeline.multi()
                pipeline.lrange(self.key, start, stop)
                encoded = pipeline.execute()[0]
                encoded = encoded[::index.step]
                value: Union[List[JSONTypes], JSONTypes] = [
                    self._decode(value) for value in encoded
                ]
            else:
                index = self.__slice_to_indices(index).start
                len_ = cast(int, pipeline.llen(self.key))
                if index not in {-1, 0, len_-1}:
                    warnings.warn(
                        cast(str, InefficientAccessWarning.__doc__),
                        InefficientAccessWarning,
                    )
                encoded = pipeline.lindex(self.key, index)
                if encoded is None:
                    raise IndexError('list index out of range')
                value = self._decode(encoded)
        return value

    @_raise_on_error
    def __setitem__(self, index: int, value: JSONTypes) -> None:  # type: ignore
        'l.__setitem__(index, value) <==> l[index] = value.  O(n)'
        with self._watch() as pipeline:
            if isinstance(index, slice):
                warnings.warn(
                    cast(str, InefficientAccessWarning.__doc__),
                    InefficientAccessWarning,
                )
                encoded_values = [self._encode(value) for value in value]
                indices = self.__slice_to_indices(index)
                pipeline.multi()
                for index, encoded_value in zip(indices, encoded_values):
                    pipeline.lset(self.key, index, encoded_value)
                indices, num = indices[len(encoded_values):], 0
                for index in indices:
                    pipeline.lset(self.key, index, 0)
                    num += 1
                if num:
                    pipeline.lrem(self.key, num, 0)
            else:
                index = self.__slice_to_indices(index).start
                len_ = cast(int, pipeline.llen(self.key))
                if index not in {-1, 0, len_-1}:
                    warnings.warn(
                        cast(str, InefficientAccessWarning.__doc__),
                        InefficientAccessWarning,
                    )
                pipeline.multi()
                pipeline.lset(self.key, index, self._encode(value))

    @_raise_on_error
    def __delitem__(self, index: Union[slice, int]) -> None:  # type: ignore
        'l.__delitem__(index) <==> del l[index].  O(n)'
        with self._watch() as pipeline:
            self.__delete(pipeline, index)

    # Preserve the Open-Closed Principle with name mangling.
    #   https://youtu.be/miGolgp9xq8?t=2086
    #   https://stackoverflow.com/a/38534939
    def __delete(self, pipeline: Pipeline, index: Union[slice, int]) -> None:
        # Python's list API requires us to delete an element by *index.*  Redis
        # supports only deleting an element by *value.*  So our workaround is
        # to set l[index] to a UUID4, then to delete the value UUID4.  More
        # info:
        #   http://redis.io/commands/lrem
        warnings.warn(
            cast(str, InefficientAccessWarning.__doc__),
            InefficientAccessWarning,
        )
        indices, num = self.__slice_to_indices(index), 0
        uuid4 = str(uuid.uuid4())
        pipeline.multi()
        for index in indices:
            pipeline.lset(self.key, index, uuid4)
            num += 1
        if num:
            pipeline.lrem(self.key, num, uuid4)

    def __len__(self) -> int:
        'Return the number of items in the RedisList.  O(1)'
        return self.redis.llen(self.key)

    def insert(self, index: int, value: JSONTypes) -> None:
        'Insert an element into the RedisDeque before the given index.  O(n)'
        with self._watch() as pipeline:
            self.__insert(index, value, pipeline=pipeline)

    def _insert(self,
                index: int,
                value: JSONTypes,
                *,
                pipeline: Pipeline,
                ) -> None:
        encoded_value = self._encode(value)
        current_length = cast(int, pipeline.llen(self.key))
        if 0 < index < current_length:
            # Python's list API requires us to insert an element before the
            # given *index.*  Redis supports only inserting an element before a
            # given (pivot) *value.*  So our workaround is to set the pivot
            # value to a UUID4, then to insert the desired value before the
            # UUID4, then to set the value UUID4 back to the original pivot
            # pivot value.  More info:
            #   http://redis.io/commands/linsert
            warnings.warn(
                cast(str, InefficientAccessWarning.__doc__),
                InefficientAccessWarning,
            )
            uuid4 = str(uuid.uuid4())
            pivot = cast(bytes, pipeline.lindex(self.key, index))
            pipeline.multi()
            pipeline.lset(self.key, index, uuid4)
            pipeline.linsert(self.key, 'BEFORE', uuid4, encoded_value)
            pipeline.lset(self.key, index+1, pivot)
        else:
            pipeline.multi()
            push_method = pipeline.lpush if index <= 0 else pipeline.rpush
            push_method(self.key, encoded_value)

    __insert = _insert

    # Methods required for Raj's sanity:

    def sort(self, *, key: Optional[str] = None, reverse: bool = False) -> None:
        'Sort the RedisList in place.  O(n)'
        if key is not None:
            raise NotImplementedError('sorting by key not implemented')
        warnings.warn(
            cast(str, InefficientAccessWarning.__doc__),
            InefficientAccessWarning,
        )
        self.redis.sort(self.key, desc=reverse, store=self.key)

    def __eq__(self, other: Any) -> bool:
        if self is other:
            return True

        if self._same_redis(other) and self.key == other.key:
            # self and other are both RedisLists on the same Redis database and
            # with the same key.  No need to compare element by element.
            return True

        with self._watch(other) as pipeline:
            len_xs = cast(int, pipeline.llen(self.key))
            if isinstance(other, RedisList):
                len_ys = cast(int, pipeline.llen(other.key))
            else:
                try:
                    len_ys = len(other)
                except TypeError:
                    # TypeError: other has no len()
                    return False
            if len_xs != len_ys:
                # self and other are different lengths.
                return False

            if isinstance(other, collections.abc.MutableSequence):
                # self and other are the same length, and other is a mutable
                # sequence too.  Compare self's and other's elements, pair by
                # pair.
                warnings.warn(
                    cast(str, InefficientAccessWarning.__doc__),
                    InefficientAccessWarning,
                )
                encoded_xs = cast(
                    List[bytes],
                    pipeline.lrange(self.key, 0, -1),
                )
                decoded_xs = (self._decode(x) for x in encoded_xs)
                if isinstance(other, RedisList):
                    encoded_ys = cast(
                        List[bytes],
                        pipeline.lrange(other.key, 0, -1),
                    )
                    decoded_ys: collections.abc.Iterable[Any] = (  # pragma: no cover
                        self._decode(y) for y in encoded_ys
                    )
                else:
                    decoded_ys = other
                return all(x == y for x, y in zip(decoded_xs, decoded_ys))

        # self and other are the same length, but other is an unordered
        # collection.
        return False

    def __add__(self, other: List[JSONTypes]) -> RedisList:
        'Append the items in other to the RedisList.  O(n)'
        warnings.warn(
            cast(str, InefficientAccessWarning.__doc__),
            InefficientAccessWarning,
        )
        with self._watch(other):
            iterable = itertools.chain(self, other)
            return self.__class__(iterable, redis=self.redis)

    def __repr__(self) -> str:
        'Return the string representation of the RedisList.  O(n)'
        warnings.warn(
            cast(str, InefficientAccessWarning.__doc__),
            InefficientAccessWarning,
        )
        return self.__class__.__name__ + str(self.__to_list())

    # Method overrides:

    # From collections.abc.MutableSequence:
    def append(self, value: JSONTypes) -> None:
        'Add an element to the right side of the RedisList.  O(1)'
        self.__extend((value,))

    # From collections.abc.MutableSequence:
    def extend(self, values: Iterable[JSONTypes]) -> None:
        'Extend the RedisList by appending elements from the iterable.  O(1)'
        encoded_values = (self._encode(value) for value in values)
        self.redis.rpush(self.key, *encoded_values)

    __extend = extend

    # From collections.abc.MutableSequence:
    def pop(self, index: Optional[int] = None) -> JSONTypes:
        with self._watch() as pipeline:
            len_ = len(self)
            if index and index >= len_:
                raise IndexError('pop index out of range')
            elif index in {0, None, len_-1, -1}:
                pop_method = 'lpop' if index == 0 else 'rpop'
                pipeline.multi()
                getattr(pipeline, pop_method)(self.key)
                encoded_value = pipeline.execute()[0]
                if encoded_value is None:
                    raise IndexError(
                        f'pop from an empty {self.__class__.__name__}'
                    )
                return self._decode(encoded_value)
            else:
                value: JSONTypes = self[cast(int, index)]
                self.__delete(pipeline, cast(int, index))
                return value

    # From collections.abc.MutableSequence:
    def remove(self, value: JSONTypes) -> None:
        'Remove first occurrence of value.  O(n)'
        warnings.warn(
            cast(str, InefficientAccessWarning.__doc__),
            InefficientAccessWarning,
        )
        with self._watch() as pipeline:
            for index, element in enumerate(self):
                if element == value:
                    self.__delete(pipeline, index)
                    return
        class_name = self.__class__.__name__
        raise ValueError(f'{class_name}.remove(x): x not in {class_name}')

    def to_list(self) -> List[JSONTypes]:
        'Convert the RedisList to a Python list.  O(n)'
        warnings.warn(
            cast(str, InefficientAccessWarning.__doc__),
            InefficientAccessWarning,
        )
        encoded = self.redis.lrange(self.key, 0, -1)
        values = [self._decode(value) for value in encoded]
        return values

    __to_list = to_list
<|endoftext|>


<commit_before><commit_msg>bump version to 0.0.3<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

# The MIT License (MIT)

# Copyright (c) 2017-2018 CNRS

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr

import torch
from torch.autograd import Variable
import torch.nn as nn
import torch.nn.functional as F

class ClopiNet(nn.Module):
    """ClopiNet sequence embedding

    RNN          ⎤
      » RNN      ⎥ » MLP » Weight » temporal pooling › normalize
           » RNN ⎦

    Parameters
    ----------
    n_features : int
        Input feature dimension.
    rnn : {'LSTM', 'GRU'}, optional
        Defaults to 'LSTM'.
    recurrent : list, optional
        List of hidden dimensions of stacked recurrent layers. Defaults to
        [16, ], i.e. one recurrent layer with hidden dimension of 16.
    bidirectional : bool, optional
        Use bidirectional recurrent layers. Defaults to False, i.e. use
        mono-directional RNNs.
    linear : list, optional
        List of hidden dimensions of linear layers. Defaults to [16, ], i.e.
        one linear layer with hidden dimension of 16.
    weighted : bool, optional
        Add dimension-wise trainable weights. Defaults to False.
    internal : bool, optional
        Return sequence of internal embeddings. Defaults to False.
    normalize : bool, optional
        Set to False to **not** unit-normalize embeddings.
    attention : list of int, optional
        List of hidden dimensions of attention linear layers (e.g. [16, ]).
        Defaults to False (i.e. no attention).
    return_attention : bool, optional
    batch_norm : bool, optional
        Defaults to False. Has not effect when internal is set to True.

    Usage
    -----
    >>> model = ClopiNet(n_features)
    >>> embedding = model(sequence)
    """

    def __init__(self, n_features,
                 rnn='LSTM', recurrent=[16,], bidirectional=False,
                 linear=[16, ], weighted=False, internal=False,
                 normalize=True, attention=False, return_attention=False,
                 batch_norm=False):

        super(ClopiNet, self).__init__()

        self.n_features = n_features
        self.rnn = rnn
        self.recurrent = recurrent
        self.bidirectional = bidirectional
        self.linear = linear
        self.weighted = weighted
        self.internal = internal
        self.normalize = normalize
        self.attention = attention
        self.return_attention = return_attention
        self.batch_norm = batch_norm

        self.num_directions_ = 2 if self.bidirectional else 1

        # create list of recurrent layers
        self.recurrent_layers_ = []
        input_dim = self.n_features
        for i, hidden_dim in enumerate(self.recurrent):
            if self.rnn == 'LSTM':
                recurrent_layer = nn.LSTM(input_dim, hidden_dim,
                                          bidirectional=self.bidirectional)
            elif self.rnn == 'GRU':
                recurrent_layer = nn.GRU(input_dim, hidden_dim,
                                         bidirectional=self.bidirectional)
            else:
                raise ValueError('"rnn" must be one of {"LSTM", "GRU"}.')
            self.add_module('recurrent_{0}'.format(i), recurrent_layer)
            self.recurrent_layers_.append(recurrent_layer)
            input_dim = hidden_dim

        # the output of recurrent layers are concatenated so the input
        # dimension of subsequent linear layers is the sum of their output
        # dimension
        input_dim = sum(self.recurrent)

        # create list of linear layers
        self.linear_layers_ = []
        for i, hidden_dim in enumerate(self.linear):
            linear_layer = nn.Linear(input_dim, hidden_dim, bias=True)
            self.add_module('linear_{0}'.format(i), linear_layer)
            self.linear_layers_.append(linear_layer)
            input_dim = hidden_dim

        if self.weighted:
            self.alphas_ = nn.Parameter(torch.ones(input_dim))

        if self.batch_norm:
            self.batch_norm_ = nn.BatchNorm1d(input_dim, eps=1e-5,
                                              momentum=0.1, affine=False)

        # create attention layers
        self.attention_layers_ = []
        if not self.attention:
            return

        input_dim = self.n_features
        for i, hidden_dim in enumerate(self.attention):
            attention_layer = nn.Linear(input_dim, hidden_dim, bias=True)
            self.add_module('attention_{0}'.format(i), attention_layer)
            self.attention_layers_.append(attention_layer)
            input_dim = hidden_dim
        if input_dim > 1:
            attention_layer = nn.Linear(input_dim, 1, bias=True)
            self.add_module('attention_{0}'.format(len(self.attention)),
                            attention_layer)
            self.attention_layers_.append(attention_layer)

    @property
    def output_dim(self):
        if self.linear:
            return self.linear[-1]
        return sum(self.recurrent)

    def forward(self, sequence):

        # check input feature dimension
        n_samples, batch_size, n_features = sequence.size()
        if n_features != self.n_features:
            msg = 'Wrong feature dimension. Found {0}, should be {1}'
            raise ValueError(msg.format(n_features, self.n_features))

        output = sequence

        gpu = sequence.is_cuda

        outputs = []
        # stack recurrent layers
        for hidden_dim, layer in zip(self.recurrent, self.recurrent_layers_):

            if self.rnn == 'LSTM':

                # initial hidden and cell states
                h = torch.zeros(self.num_directions_, batch_size, hidden_dim)
                c = torch.zeros(self.num_directions_, batch_size, hidden_dim)
                if gpu:
                    h = h.cuda()
                    c = c.cuda()
                hidden = (Variable(h, requires_grad=False),
                          Variable(c, requires_grad=False))

            elif self.rnn == 'GRU':
                # initial hidden state
                h = torch.zeros(self.num_directions_, batch_size, hidden_dim)
                if gpu:
                    h = h.cuda()
                hidden = Variable(h, requires_grad=False)

            # apply current recurrent layer and get output sequence
            output, _ = layer(output, hidden)

            # average both directions in case of bidirectional layers
            if self.bidirectional:
                output = .5 * (output[:, :, :hidden_dim] + \
                               output[:, :, hidden_dim:])

            outputs.append(output)

        # concatenate outputs
        output = torch.cat(outputs, dim=2)
        # n_samples, batch_size, dimension

        # stack linear layers
        for hidden_dim, layer in zip(self.linear, self.linear_layers_):

            # apply current linear layer
            output = layer(output)

            # apply non-linear activation function
            output = F.tanh(output)

        # n_samples, batch_size, dimension

        if self.weighted:
            if gpu:
                self.alphas_ = self.alphas_.cuda()
            output = output * self.alphas_

        if self.internal:
            if self.normalize:
                output = output / torch.norm(output, 2, 2, keepdim=True)

            return output

        if self.attention_layers_:
            attn = sequence
            for layer, hidden_dim in zip(self.attention_layers_,
                                         self.attention + [1]):
                attn = layer(attn)
                attn = F.tanh(attn)
            attn = F.softmax(attn, dim=0)
            output = output * attn

        # average temporal pooling
        output = output.sum(dim=0)
        # batch_size, dimension

        # batch normalization
        if self.batch_norm:
            output = self.batch_norm_(output)

        # L2 normalization
        if self.normalize:
            output = output / torch.norm(output, 2, 1, keepdim=True)
            # batch_size, dimension

        if self.return_attention:
            return output, attn

        return output
<commit_msg>wip: update ClopiNet default values<commit_after>#!/usr/bin/env python
# encoding: utf-8

# The MIT License (MIT)

# Copyright (c) 2017-2018 CNRS

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr

import torch
from torch.autograd import Variable
import torch.nn as nn
import torch.nn.functional as F

class ClopiNet(nn.Module):
    """ClopiNet sequence embedding

    RNN          ⎤
      » RNN      ⎥ » MLP » Weight » temporal pooling › normalize
           » RNN ⎦

    Parameters
    ----------
    n_features : int
        Input feature dimension.
    rnn : {'LSTM', 'GRU'}, optional
        Defaults to 'LSTM'.
    recurrent : list, optional
        List of hidden dimensions of stacked recurrent layers. Defaults to
        [64, 64, 64], i.e. three recurrent layers with hidden dimension of 64.
    bidirectional : bool, optional
        Use bidirectional recurrent layers. Defaults to False, i.e. use
        mono-directional RNNs.
    normalize : bool, optional
        Unit-normalize embeddings.
    weighted : bool, optional
        Add dimension-wise trainable weights. Defaults to False.
    linear : list, optional
        List of hidden dimensions of linear layers. Defaults to none.
    internal : bool, optional
        Return sequence of internal embeddings. Defaults to False.
    attention : list of int, optional
        List of hidden dimensions of attention linear layers (e.g. [16, ]).
        Defaults to no attention.
    return_attention : bool, optional

    Usage
    -----
    >>> model = ClopiNet(n_features)
    >>> embedding = model(sequence)
    """

    def __init__(self, n_features,
                 rnn='LSTM', recurrent=[64, 64, 64], bidirectional=False,
                 normalize=False, weighted=False, linear=None,
                 internal=False, attention=None, return_attention=False):

        super(ClopiNet, self).__init__()

        self.n_features = n_features
        self.rnn = rnn
        self.recurrent = recurrent
        self.bidirectional = bidirectional
        self.normalize = normalize
        self.weighted = weighted
        self.linear = [] if linear is None else linear
        self.internal = internal
        self.attention = [] if attention is None else attention
        self.return_attention = return_attention

        self.num_directions_ = 2 if self.bidirectional else 1

        # create list of recurrent layers
        self.recurrent_layers_ = []
        input_dim = self.n_features
        for i, hidden_dim in enumerate(self.recurrent):
            if self.rnn == 'LSTM':
                recurrent_layer = nn.LSTM(input_dim, hidden_dim,
                                          bidirectional=self.bidirectional)
            elif self.rnn == 'GRU':
                recurrent_layer = nn.GRU(input_dim, hidden_dim,
                                         bidirectional=self.bidirectional)
            else:
                raise ValueError('"rnn" must be one of {"LSTM", "GRU"}.')
            self.add_module('recurrent_{0}'.format(i), recurrent_layer)
            self.recurrent_layers_.append(recurrent_layer)
            input_dim = hidden_dim

        # the output of recurrent layers are concatenated so the input
        # dimension of subsequent linear layers is the sum of their output
        # dimension
        input_dim = sum(self.recurrent)

        if self.weighted:
            self.alphas_ = nn.Parameter(torch.ones(input_dim))

        # create list of linear layers
        self.linear_layers_ = []
        for i, hidden_dim in enumerate(self.linear):
            linear_layer = nn.Linear(input_dim, hidden_dim, bias=True)
            self.add_module('linear_{0}'.format(i), linear_layer)
            self.linear_layers_.append(linear_layer)
            input_dim = hidden_dim

        # batch normalization: embeddings will have "0" mean and "1" variance
        self.batch_norm_ = nn.BatchNorm1d(input_dim, eps=1e-5,
                                          momentum=0.1, affine=False)

        # create attention layers
        self.attention_layers_ = []
        if not self.attention:
            return

        input_dim = self.n_features
        for i, hidden_dim in enumerate(self.attention):
            attention_layer = nn.Linear(input_dim, hidden_dim, bias=True)
            self.add_module('attention_{0}'.format(i), attention_layer)
            self.attention_layers_.append(attention_layer)
            input_dim = hidden_dim
        if input_dim > 1:
            attention_layer = nn.Linear(input_dim, 1, bias=True)
            self.add_module('attention_{0}'.format(len(self.attention)),
                            attention_layer)
            self.attention_layers_.append(attention_layer)

    @property
    def output_dim(self):
        if self.linear:
            return self.linear[-1]
        return sum(self.recurrent)

    def forward(self, sequence):

        # check input feature dimension
        n_samples, batch_size, n_features = sequence.size()
        if n_features != self.n_features:
            msg = 'Wrong feature dimension. Found {0}, should be {1}'
            raise ValueError(msg.format(n_features, self.n_features))

        output = sequence

        gpu = sequence.is_cuda

        outputs = []
        # stack recurrent layers
        for hidden_dim, layer in zip(self.recurrent, self.recurrent_layers_):

            if self.rnn == 'LSTM':

                # initial hidden and cell states
                h = torch.zeros(self.num_directions_, batch_size, hidden_dim)
                c = torch.zeros(self.num_directions_, batch_size, hidden_dim)
                if gpu:
                    h = h.cuda()
                    c = c.cuda()
                hidden = (Variable(h, requires_grad=False),
                          Variable(c, requires_grad=False))

            elif self.rnn == 'GRU':
                # initial hidden state
                h = torch.zeros(self.num_directions_, batch_size, hidden_dim)
                if gpu:
                    h = h.cuda()
                hidden = Variable(h, requires_grad=False)

            # apply current recurrent layer and get output sequence
            output, _ = layer(output, hidden)

            # average both directions in case of bidirectional layers
            if self.bidirectional:
                output = .5 * (output[:, :, :hidden_dim] + \
                               output[:, :, hidden_dim:])

            outputs.append(output)

        # concatenate outputs
        output = torch.cat(outputs, dim=2)
        # n_samples, batch_size, dimension

        if self.weighted:
            if gpu:
                self.alphas_ = self.alphas_.cuda()
            output = output * self.alphas_

        # stack linear layers
        for hidden_dim, layer in zip(self.linear, self.linear_layers_):

            # apply current linear layer
            output = layer(output)

            # apply non-linear activation function
            output = F.tanh(output)

        # n_samples, batch_size, dimension

        if self.internal:
            if self.normalize:
                output = output / torch.norm(output, 2, 2, keepdim=True)

            return output

        if self.attention_layers_:
            attn = sequence
            for layer, hidden_dim in zip(self.attention_layers_,
                                         self.attention + [1]):
                attn = layer(attn)
                attn = F.tanh(attn)
            attn = F.softmax(attn, dim=0)
            output = output * attn

        # average temporal pooling
        output = output.sum(dim=0)
        # batch_size, dimension

        # batch normalization
        output = self.batch_norm_(output)

        # L2 normalization
        if self.normalize:
            output = output / torch.norm(output, 2, 1, keepdim=True)
            # batch_size, dimension

        if self.return_attention:
            return output, attn

        return output
<|endoftext|>