<commit_before>#!/usr/bin/python
#
# learnthings.py (C) 2012 Yrjo Kari-Koskinen <ykk@peruna.fi>
#
# Program for small kids to learn recognizing e.g. animals
#
# See README and LICENSE.txt

import os, sys, pygame, random, time
from pygame import mixer
from pygame.locals import *

class NoneSound:
    def play(self): pass

class Thing:
    """A thing that contains an image and a sound"""
    def __init__(self, category, name, width, height, lang):
        self.name = name
        self.category = category
        self.screensize = (width, height)
        self.lang = lang
        self.sound = False
        self.load()

    def load(self):
        self.loadsounds()
        self.loadimage()

    def loadimage(self):
        imagepath = os.path.join("media", self.category, "photos", self.name + ".jpg")
        try:
            image = pygame.image.load(imagepath).convert()
            self.image = pygame.transform.smoothscale(image, self.targetsize(image.get_size()))
        except pygame.error, message:
            raise SystemExit, message

    def loadsounds(self):
        if not mixer:
            self.sound = NoneSound()
        else:
            soundpath = os.path.join("media", self.category, "sounds", self.name + ".ogg")
            if (os.path.isfile(soundpath)):
                self.sound = mixer.Sound(soundpath)
            if self.lang != "":
                namepath = os.path.join("media", self.category, "names", self.lang, self.name + ".ogg")
                self.namesound = mixer.Sound(namepath)

    def show(self, screen):
        imagesize = self.image.get_size()
        center = (self.screensize[0] - imagesize[0]) / 2
        for y in range(-self.screensize[1], 1, 25):
            screen.fill((0,0,0), (0, 0, self.screensize[0], imagesize[1] + y))
            screen.blit(self.image, (center,y))
            pygame.display.flip()
        screen.blit(self.image, (center,0))
        pygame.display.flip()
        if self.lang != "":
            self.namesound.play()
            time.sleep(self.namesound.get_length() - 0.5)
        if self.sound:
            self.sound.play()

    def targetsize(self, size):
        if ((size[0] / size[1]) > (self.screensize[0] / self.screensize[1])):
            return (self.screensize[0], size[1] * self.screensize[0] / size[0])
        else:
            return (size[0] * self.screensize[1] / size[1], self.screensize[1])

class Random:
    """Produce random numbers between 0-limit that do not repeat"""
    def __init__(self, limit):
        self.limit = limit-1;
        self.last = -1;

    def next(self):
        nextRandom = random.randint(0, self.limit)
        if nextRandom == self.last:
            return self.next()
        else:
            self.last = nextRandom
            return nextRandom

def mainLoop(things, screen):
    rnd = Random(len(things))
    # Show first random thing
    things[rnd.next()].show(screen)
    thingShowTime = time.time()

    keypresses = 0
    thingsShown = 1
    # Exit sequence: q, u, i,t
    exitSeq = [K_q, K_u, K_i, K_t]
    exitIndex = 0

    # Event loop
    while exitIndex != len(exitSeq):
        if keypresses > keypressTreshold:
            keypresses = 0
            things[rnd.next()].show(screen)
            thingsShown = thingsShown + 1
            thingShowTime = time.time()
        for event in pygame.event.get():
            if event.type == QUIT:
                sys.exit()
            elif event.type == KEYDOWN:
                if (time.time() - thingShowTime) > timeTreshold:
                    keypresses = keypresses + 1
                if event.key == exitSeq[exitIndex]:
                    exitIndex = exitIndex + 1
                else:
                    exitIndex = 0

    return thingsShown

def main(timeTreshold, keypressTreshold):
    # Parse arguments
    if len(sys.argv) < 2:
        print "Usage: ", sys.argv[0], "category", "[language]"
        sys.exit()

    category = sys.argv[1]
    lang = ""
    if len(sys.argv) > 2:
        lang = sys.argv[2]
        if not os.access(os.path.join("media", category, "names", lang), os.R_OK):
            print lang, "language not found"
            sys.exit()

    if not os.access(os.path.join("media", category), os.R_OK):
        print category, "category not found"
        sys.exit()

    # Init pygame display
    print >> sys.stderr, "Loading", category, "..."
    pygame.init()
    screen = pygame.display.set_mode((0,0), pygame.FULLSCREEN)
    pygame.mouse.set_visible(False)
    info = pygame.display.Info()
    width = info.current_w
    height = info.current_h

    # Load things
    thingNames = map(
        lambda s: s[:s.rindex(".jpg")],
        os.listdir(os.path.join("media", category, "photos")))
    things = map(lambda n: Thing(category, n, width, height, lang), thingNames)

    startTime = time.time()
    thingsShown = mainLoop(things, screen)

    delta = time.time() - startTime
    timePerThing = int(delta/thingsShown)
    print >> sys.stderr, thingsShown, "things shown in", int(delta), "seconds, ", timePerThing, "s/thing"

if __name__ == "__main__":
    # At least how many seconds each image is showed
    timeTreshold = 2
    # How many times any key must be pressed after timeTreshold seconds
    keypressTreshold = 3

    main(timeTreshold, keypressTreshold)
<commit_msg>center low images and clear empty space blow<commit_after>#!/usr/bin/python
#
# learnthings.py (C) 2012 Yrjo Kari-Koskinen <ykk@peruna.fi>
#
# Program for small kids to learn recognizing e.g. animals
#
# See README and LICENSE.txt

import os, sys, pygame, random, time
from pygame import mixer
from pygame.locals import *

class NoneSound:
    def play(self): pass

class Thing:
    """A thing that contains an image and a sound"""
    def __init__(self, category, name, width, height, lang):
        self.name = name
        self.category = category
        self.screensize = (width, height)
        self.lang = lang
        self.sound = False
        self.namesound = False
        self.load()

    def load(self):
        self.loadsounds()
        self.loadimage()

    def loadimage(self):
        imagepath = os.path.join("media", self.category, "photos", self.name + ".jpg")
        try:
            image = pygame.image.load(imagepath).convert()
            self.image = pygame.transform.smoothscale(image, self.targetsize(image.get_size()))
        except pygame.error, message:
            raise SystemExit, message

    def loadsounds(self):
        if not mixer:
            self.sound = NoneSound()
        else:
            soundpath = os.path.join("media", self.category, "sounds", self.name + ".ogg")
            if (os.path.isfile(soundpath)):
                self.sound = mixer.Sound(soundpath)
            if self.lang != "":
                namepath = os.path.join("media", self.category, "names", self.lang, self.name + ".ogg")
                if (os.path.isfile(namepath)):
                    self.namesound = mixer.Sound(namepath)

    def show(self, screen):
        imagesize = self.image.get_size()
        center = ((self.screensize[0] - imagesize[0]) / 2, (self.screensize[1] - imagesize[1]) / 2)

        for y in range(-self.screensize[1], center[1], 25):
            screen.fill((0,0,0), (0, 0, self.screensize[0], imagesize[1] + y))
            screen.blit(self.image, (center[0],y))
            pygame.display.flip()
        screen.blit(self.image, (center[0],center[1]))
        pygame.display.flip()

        # clear space below photo
        for y in range(center[1] + imagesize[1], self.screensize[1]+1, 25):
            screen.fill((0,0,0), (0, y, self.screensize[0], y + 25))
            pygame.display.flip()

        # play sounds
        if self.namesound:
            self.namesound.play()
            time.sleep(self.namesound.get_length() - 0.5)
        if self.sound:
            self.sound.play()

    def targetsize(self, size):
        if ((size[0] / size[1]) > (self.screensize[0] / self.screensize[1])):
            return (self.screensize[0], size[1] * self.screensize[0] / size[0])
        else:
            return (size[0] * self.screensize[1] / size[1], self.screensize[1])

class Random:
    """Produce random numbers between 0-limit that do not repeat"""
    def __init__(self, limit):
        self.limit = limit-1;
        self.last = -1;

    def next(self):
        nextRandom = random.randint(0, self.limit)
        if nextRandom == self.last:
            return self.next()
        else:
            self.last = nextRandom
            return nextRandom

def mainLoop(things, screen):
    rnd = Random(len(things))
    # Show first random thing
    things[rnd.next()].show(screen)
    thingShowTime = time.time()

    keypresses = 0
    thingsShown = 1
    # Exit sequence: q, u, i,t
    exitSeq = [K_q, K_u, K_i, K_t]
    exitIndex = 0

    # Event loop
    while exitIndex != len(exitSeq):
        if keypresses > keypressTreshold:
            keypresses = 0
            things[rnd.next()].show(screen)
            thingsShown = thingsShown + 1
            thingShowTime = time.time()
        for event in pygame.event.get():
            if event.type == QUIT:
                sys.exit()
            elif event.type == KEYDOWN:
                if (time.time() - thingShowTime) > timeTreshold:
                    keypresses = keypresses + 1
                if event.key == exitSeq[exitIndex]:
                    exitIndex = exitIndex + 1
                else:
                    exitIndex = 0

    return thingsShown

def main(timeTreshold, keypressTreshold):
    # Parse arguments
    if len(sys.argv) < 2:
        print "Usage: ", sys.argv[0], "category", "[language]"
        sys.exit()

    category = sys.argv[1]
    lang = ""
    if len(sys.argv) > 2:
        lang = sys.argv[2]
        if not os.access(os.path.join("media", category, "names", lang), os.R_OK):
            print lang, "language not found"
            sys.exit()

    if not os.access(os.path.join("media", category), os.R_OK):
        print category, "category not found"
        sys.exit()

    # Init pygame display
    print >> sys.stderr, "Loading", category, "..."
    pygame.init()
    screen = pygame.display.set_mode((0,0), pygame.FULLSCREEN)
    pygame.mouse.set_visible(False)
    info = pygame.display.Info()
    width = info.current_w
    height = info.current_h

    # Load things
    thingNames = map(
        lambda s: s[:s.rindex(".jpg")],
        os.listdir(os.path.join("media", category, "photos")))
    things = map(lambda n: Thing(category, n, width, height, lang), thingNames)

    startTime = time.time()
    thingsShown = mainLoop(things, screen)

    delta = time.time() - startTime
    timePerThing = int(delta/thingsShown)
    print >> sys.stderr, thingsShown, "things shown in", int(delta), "seconds, ", timePerThing, "s/thing"

if __name__ == "__main__":
    # At least how many seconds each image is showed
    timeTreshold = 2
    # How many times any key must be pressed after timeTreshold seconds
    keypressTreshold = 3

    main(timeTreshold, keypressTreshold)
<|endoftext|>


<commit_before>"""
feature extraction
"""

# from standard library
import sys
import os
import glob

# from dependencies
import numpy as np
from sklearn.externals import joblib

# from hvc
from .parseconfig import parse_config
from . import features
from .utils import timestamp

SELECT_TEMPLATE = """select:
  num_replicates: None
  num_train_samples:
    start : None
    stop : None
    step : None
  num_test_samples: None

    models:"""

MODELS_TEMPLATE = """
    -
      model: {0}
      feature_list_indices: {1}
      hyperparameters:
        {2}
"""

SVM_HYPERPARAMS = """C : None
        gamma : None
"""

KNN_HYPERPARAMS = """k : None
"""

TODO_TEMPLATE = """  todo_list:
    -
      feature_file : {0}
      output_dir: {1}"""

def write_select_config(summary_ftr_file_dict,
                       summary_filename,
                       output_dir):
    """writes summary output dict from extract to a config file for select
    Only called when feature_group is a list, to save user from figuring
    out manually which features belong to which group.

    Parameters
    ----------
    summary_ftr_file_dict : dictionary
        as defined in featureextract.extract
    summary_filename : string
        name of summary feature file
    output_dir : string
        name of output directory -- assumes it will be the same as it was for extract.yml

    Returns
    -------
    None

    Doesn't return anything, just saves .yml file
    """

    select_config_filename = 'select.config.from_' + summary_filename + '.yml'
    with open(select_config_filename, 'w') as yml_outfile:
        yml_outfile.write(SELECT_TEMPLATE)
        for model_name, model_ID in summary_ftr_file_dict['feature_group_ID_dict'].items():
            inds = [ftr_list_ind for ftr_list_ind, grp_ID in
                    enumerate(summary_ftr_file_dict['feature_list_group_ID'])
                    if grp_ID == model_ID]
            if model_name == 'svm':
                hyperparams = SVM_HYPERPARAMS
            elif model_name == 'knn':
                hyperparams = KNN_HYPERPARAMS
            yml_outfile.write(MODELS_TEMPLATE.format(model_name,
                                                     inds,
                                                     hyperparams))
        yml_outfile.write(TODO_TEMPLATE.format(summary_filename,
                                               output_dir))

def _extract(extract_params, make_summary_file=True):
    """helper function that loops through data dirs,
    extracts features, saves feature files, and then
    makes summary feature file.
    Put into a separate helper function so that it can
    also be used by labelpredict.predict

    Parameters
    ----------
    extract_params : dict with following key, value pairs
        data_dirs : list
            list of directories with data files
        home_dir : str
        output_dir :
        file_format : str
            {'evtaf', 'koumura'}
        feature_list : list
            list of features generated by hvc.parse.extract
        spect_params : dict
            as defined for hvc.audiofileIO.Spectrogram class
        labelset : str
            labels for syllables as a string, e.g., 'iabcdefg'
        segment_params : dict
            as defined for hvc.audiofileIO.Song and hvc.audiofileIO.segment_song

    Returns
    -------
    None
    Saves output files in output dir

    """

    for data_dir in extract_params['data_dirs']:
        print('Changing to data directory: {}'.format(data_dir))
        os.chdir(extract_params['home_dir'])
        os.chdir(data_dir)

        if 'features_from_all_files' in locals():
            # from last time through loop
            # (need to re-initialize for each directory)
            del features_from_all_files

        if 'neuralnet_inputs_dict' in locals():
            del neuralnet_inputs_dict

        if extract_params['file_format'] == 'evtaf':
            songfiles_list = glob.glob('*.cbin')
        elif extract_params['file_format'] == 'koumura':
            songfiles_list = glob.glob('*.wav')

        num_songfiles = len(songfiles_list)
        all_labels = []
        song_IDs = []
        song_ID_counter = 0
        for file_num, songfile in enumerate(songfiles_list):
            print('Processing audio file {} of {}.'.format(file_num + 1, num_songfiles))
            # segment_params defined for todo_list item takes precedence over any default
            # defined for `extract` config
            extract_dict = features.extract.from_file(songfile,
                                                      extract_params['file_format'],
                                                      extract_params['feature_list'],
                                                      extract_params['spect_params'],
                                                      extract_params['labelset'],
                                                      extract_params['segment_params'])

            if extract_dict is None:
                # because no labels from labels_to_use were found in songfile
                continue

            all_labels.extend(extract_dict['labels'])
            song_IDs.extend([song_ID_counter] * len(extract_dict['labels']))
            song_ID_counter += 1

            if 'features_arr' in extract_dict:
                if 'features_from_all_files' in locals():
                    features_from_all_files = np.concatenate((features_from_all_files,
                                                              extract_dict['features_arr']),
                                                             axis=0)
                else:
                    features_from_all_files = extract_dict['features_arr']

            if 'neuralnet_inputs_dict' in extract_dict:
                if 'neuralnet_inputs_all_files' in locals():
                    for key, val in neuralnet_inputs_all_files.items():
                        new_val = np.concatenate((neuralnet_inputs_all_files[key],
                                                  extract_dict['neuralnet_inputs_dict'][key]))
                        neuralnet_inputs_all_files[key] = new_val
                else:
                    neuralnet_inputs_all_files = extract_dict['neuralnet_inputs_dict']

        # get dir name without the rest of path so it doesn't have separators in the name
        # because those can't be in filename
        just_dir_name = os.getcwd().split(os.path.sep)[-1]
        feature_file = os.path.join(extract_params['output_dir'],
                                    'features_from_' + just_dir_name + '_created_' + timestamp())
        feature_file_dict = {
            'labels': all_labels,
            'feature_list': extract_params['feature_list'],
            'spect_params': extract_params['spect_params'],
            'segment_params': extract_params['segment_params'],
            'labelset': extract_params['labelset'],
            'file_format': extract_params['file_format'],
            'bird_ID': extract_params['bird_ID'],
            'song_IDs': song_IDs
        }

        if 'features_from_all_files' in locals():
            feature_file_dict['features'] = features_from_all_files
            feature_file_dict['features_arr_column_IDs'] = extract_dict['feature_inds']

            if 'feature_list_group_ID' in extract_params:
                feature_file_dict['feature_list_group_ID'] = extract_params['feature_list_group_ID']
                feature_file_dict['feature_group_ID_dict'] = extract_params['feature_group_ID_dict']

        if 'neuralnet_inputs_all_files' in locals():
            feature_file_dict['neuralnet_inputs'] = neuralnet_inputs_all_files

        joblib.dump(feature_file_dict,
                    feature_file,
                    compress=3)

    ##########################################################
    # after looping through all data_dirs for this todo_item #
    ##########################################################
    if make_summary_file:
        print('making summary file')
        os.chdir(extract_params['output_dir'])
        summary_filename = 'summary_feature_file_created_' + timestamp()
        summary_filename_with_path = os.path.join(extract_params['output_dir'],
                                                  summary_filename)
        ftr_output_files = glob.glob('*features_from_*')
        if len(ftr_output_files) > 1:
            # make a 'summary' data file
            list_of_output_dicts = []
            summary_ftr_file_dict = {}
            for feature_file in ftr_output_files:
                feature_file_dict = joblib.load(feature_file)

                if 'labels' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['labels'] = feature_file_dict['labels']
                else:
                    summary_ftr_file_dict['labels'] = \
                        summary_ftr_file_dict['labels'] + feature_file_dict['labels']

                if 'spect_params' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['spect_params'] = feature_file_dict['spect_params']
                else:
                    if feature_file_dict['spect_params'] != summary_ftr_file_dict['spect_params']:
                        raise ValueError('mismatch between spect_params in {} '
                                         'and other feature files'.format(feature_file))

                if 'segment_params' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['segment_params'] = feature_file_dict['segment_params']
                else:
                    if feature_file_dict['segment_params'] != summary_ftr_file_dict['segment_params']:
                        raise ValueError('mismatch between segment_params in {} '
                                         'and other feature files'.format(feature_file))

                if 'labelset' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['labelset'] = feature_file_dict['labelset']
                else:
                    if feature_file_dict['labelset'] != summary_ftr_file_dict['labelset']:
                        raise ValueError('mismatch between labelset in {} '
                                         'and other feature files'.format(feature_file))

                if 'file_format' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['file_format'] = feature_file_dict['file_format']
                else:
                    if feature_file_dict['file_format'] != summary_ftr_file_dict['file_format']:
                        raise ValueError('mismatch between file_format in {} '
                                         'and other feature files'.format(feature_file))

                if 'bird_ID' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['bird_ID'] = feature_file_dict['bird_ID']
                else:
                    if feature_file_dict['bird_ID'] != summary_ftr_file_dict['bird_ID']:
                        raise ValueError('mismatch between bird_ID in {} '
                                         'and other feature files'.format(feature_file))

                if 'song_IDs' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['song_IDs'] = feature_file_dict['song_IDs']
                else:
                    curr_last_ID = summary_ftr_file_dict['song_IDs'][-1]
                    tmp_song_IDs = [el + curr_last_ID + 1 for el in feature_file_dict['song_IDs']]
                    summary_ftr_file_dict['song_IDs'].extend(tmp_song_IDs)

                if 'feature_list' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['feature_list'] = feature_file_dict['feature_list']
                else:
                    if feature_file_dict['feature_list'] != summary_ftr_file_dict['feature_list']:
                        raise ValueError('mismatch between feature_list in {} '
                                         'and other feature files'.format(feature_file))

                # only if not-neuralnet features were used
                if 'features' in feature_file_dict:
                    if 'features' not in summary_ftr_file_dict:
                        summary_ftr_file_dict['features'] = feature_file_dict['features']
                    else:
                        summary_ftr_file_dict['features'] = np.concatenate((summary_ftr_file_dict['features'],
                                                                            feature_file_dict['features']))

                    if 'features_arr_column_IDs' not in summary_ftr_file_dict:
                        summary_ftr_file_dict['features_arr_column_IDs'] = feature_file_dict['features_arr_column_IDs']
                    else:
                        if any(feature_file_dict['features_arr_column_IDs'] !=
                                       summary_ftr_file_dict['features_arr_column_IDs']):
                            raise ValueError('mismatch between features_arr_column_IDs in {} '
                                             'and other feature files'.format(feature_file))

                    if 'feature_list_group_ID' in extract_params:
                        if 'feature_list_group_ID' not in summary_ftr_file_dict:
                            summary_ftr_file_dict['feature_list_group_ID'] = feature_file_dict['feature_list_group_ID']
                        else:
                            if feature_file_dict['feature_list_group_ID'] != \
                                    summary_ftr_file_dict['feature_list_group_ID']:
                                raise ValueError('mismatch between feature_list_group_ID in {} '
                                                 'and other feature files'.format(feature_file))

                        if 'feature_group_ID_dict' not in summary_ftr_file_dict:
                            summary_ftr_file_dict['feature_group_ID_dict'] = \
                                feature_file_dict['feature_group_ID_dict']
                        else:
                            if feature_file_dict['feature_group_ID_dict'] != \
                                    summary_ftr_file_dict['feature_group_ID_dict']:
                                raise ValueError('mismatch between feature_group_ID_dict in {} '
                                                 'and other feature files'.format(feature_file))

                # if extracting inputs for neuralnets
                if 'neuralnet_inputs' in feature_file_dict:
                    if 'neuralnet_inputs' not in summary_ftr_file_dict:
                        summary_ftr_file_dict['neuralnet_inputs'] = \
                            feature_file_dict['neuralnet_inputs']
                    else:
                        for key, val in summary_ftr_file_dict['neuralnet_inputs'].items():
                            newval = np.concatenate((summary_ftr_file_dict['neuralnet_inputs'][key],
                                                     feature_file_dict['neuralnet_inputs'][key]))
                            summary_ftr_file_dict['neuralnet_inputs'][key] = newval

            joblib.dump(summary_ftr_file_dict,
                        summary_filename)

        else:  # if only one feature_file
            os.rename(ftr_output_files[0],
                      summary_filename)
            summary_ftr_file_dict = joblib.load(summary_filename)

        if 'feature_list_group_ID' in summary_ftr_file_dict:
            write_select_config(summary_ftr_file_dict,
                                summary_filename,
                                extract_params['output_dir'])

def extract(config_file):
    """main function that runs feature extraction.
    Does not return anything, just runs through directories specified in config_file
    and extracts features.

    Parameters
    ----------
    config_file : string
        filename of YAML file that configures feature extraction
    """

    extract_config = parse_config(config_file, 'extract')
    print('Parsed extract config.')

    home_dir = os.getcwd()

    todo_list = extract_config['todo_list']
    for ind, todo in enumerate(todo_list):

        print('Completing item {} of {} in to-do list'.format(ind+1, len(todo_list)))
        file_format = todo['file_format']
        if file_format == 'evtaf':
            if 'evfuncs' not in sys.modules:
                from . import evfuncs
        elif file_format == 'koumura':
            if 'koumura' not in sys.modules:
                from . import koumura

        output_dir = 'extract_output_' + timestamp()
        output_dir_with_path = os.path.join(todo['output_dir'], output_dir)
        if not os.path.isdir(output_dir_with_path):
            os.mkdir(output_dir_with_path)

        extract_params = {
            'bird_ID': todo['bird_ID'],
            'feature_list': todo['feature_list'],
            'output_dir': output_dir_with_path,
            'home_dir': home_dir,
            'data_dirs': todo['data_dirs'],
            'labelset': todo['labelset'],
            'file_format': todo['file_format']
        }

        # segment_params defined for todo_list item takes precedence over any default
        # defined for `extract` config
        if 'segment_params' in todo:
            extract_params['segment_params'] = todo['segment_params']
        else:
            extract_params['segment_params'] = extract_config['segment_params']

        if 'spect_params' in todo:
            extract_params['spect_params'] = todo['spect_params']
        else:
            extract_params['spect_params'] = extract_config['spect_params']

        if 'feature_list_group_ID' in todo:
            extract_params['feature_list_group_ID'] = todo['feature_list_group_ID']
            extract_params['feature_group_ID_dict'] = todo['feature_group_ID_dict']

        _extract(extract_params)

        os.chdir(home_dir)
<commit_msg>bugfix -- default Koumura freq_cutoffs causing crash<commit_after>"""
feature extraction
"""

# from standard library
import sys
import os
import glob

# from dependencies
import numpy as np
from sklearn.externals import joblib

# from hvc
from .parseconfig import parse_config
from . import features
from .utils import timestamp

SELECT_TEMPLATE = """select:
  num_replicates: None
  num_train_samples:
    start : None
    stop : None
    step : None
  num_test_samples: None

    models:"""

MODELS_TEMPLATE = """
    -
      model: {0}
      feature_list_indices: {1}
      hyperparameters:
        {2}
"""

SVM_HYPERPARAMS = """C : None
        gamma : None
"""

KNN_HYPERPARAMS = """k : None
"""

TODO_TEMPLATE = """  todo_list:
    -
      feature_file : {0}
      output_dir: {1}"""

def write_select_config(summary_ftr_file_dict,
                       summary_filename,
                       output_dir):
    """writes summary output dict from extract to a config file for select
    Only called when feature_group is a list, to save user from figuring
    out manually which features belong to which group.

    Parameters
    ----------
    summary_ftr_file_dict : dictionary
        as defined in featureextract.extract
    summary_filename : string
        name of summary feature file
    output_dir : string
        name of output directory -- assumes it will be the same as it was for extract.yml

    Returns
    -------
    None

    Doesn't return anything, just saves .yml file
    """

    select_config_filename = 'select.config.from_' + summary_filename + '.yml'
    with open(select_config_filename, 'w') as yml_outfile:
        yml_outfile.write(SELECT_TEMPLATE)
        for model_name, model_ID in summary_ftr_file_dict['feature_group_ID_dict'].items():
            inds = [ftr_list_ind for ftr_list_ind, grp_ID in
                    enumerate(summary_ftr_file_dict['feature_list_group_ID'])
                    if grp_ID == model_ID]
            if model_name == 'svm':
                hyperparams = SVM_HYPERPARAMS
            elif model_name == 'knn':
                hyperparams = KNN_HYPERPARAMS
            yml_outfile.write(MODELS_TEMPLATE.format(model_name,
                                                     inds,
                                                     hyperparams))
        yml_outfile.write(TODO_TEMPLATE.format(summary_filename,
                                               output_dir))

def _extract(extract_params, make_summary_file=True):
    """helper function that loops through data dirs,
    extracts features, saves feature files, and then
    makes summary feature file.
    Put into a separate helper function so that it can
    also be used by labelpredict.predict

    Parameters
    ----------
    extract_params : dict with following key, value pairs
        data_dirs : list
            list of directories with data files
        home_dir : str
        output_dir :
        file_format : str
            {'evtaf', 'koumura'}
        feature_list : list
            list of features generated by hvc.parse.extract
        spect_params : dict
            as defined for hvc.audiofileIO.Spectrogram class
        labelset : str
            labels for syllables as a string, e.g., 'iabcdefg'
        segment_params : dict
            as defined for hvc.audiofileIO.Song and hvc.audiofileIO.segment_song

    Returns
    -------
    None
    Saves output files in output dir

    """

    for data_dir in extract_params['data_dirs']:
        print('Changing to data directory: {}'.format(data_dir))
        os.chdir(extract_params['home_dir'])
        os.chdir(data_dir)

        if 'features_from_all_files' in locals():
            # from last time through loop
            # (need to re-initialize for each directory)
            del features_from_all_files

        if 'neuralnet_inputs_dict' in locals():
            del neuralnet_inputs_dict

        if extract_params['file_format'] == 'evtaf':
            songfiles_list = glob.glob('*.cbin')
        elif extract_params['file_format'] == 'koumura':
            songfiles_list = glob.glob('*.wav')

        num_songfiles = len(songfiles_list)
        all_labels = []
        song_IDs = []
        song_ID_counter = 0
        for file_num, songfile in enumerate(songfiles_list):
            print('Processing audio file {} of {}.'.format(file_num + 1, num_songfiles))
            # segment_params defined for todo_list item takes precedence over any default
            # defined for `extract` config
            extract_dict = features.extract.from_file(songfile,
                                                      extract_params['file_format'],
                                                      extract_params['feature_list'],
                                                      extract_params['spect_params'],
                                                      extract_params['labelset'],
                                                      extract_params['segment_params'])

            if extract_dict is None:
                # because no labels from labels_to_use were found in songfile
                continue

            all_labels.extend(extract_dict['labels'])
            song_IDs.extend([song_ID_counter] * len(extract_dict['labels']))
            song_ID_counter += 1

            if 'features_arr' in extract_dict:
                if 'features_from_all_files' in locals():
                    features_from_all_files = np.concatenate((features_from_all_files,
                                                              extract_dict['features_arr']),
                                                             axis=0)
                else:
                    features_from_all_files = extract_dict['features_arr']

            if 'neuralnet_inputs_dict' in extract_dict:
                if 'neuralnet_inputs_all_files' in locals():
                    for key, val in neuralnet_inputs_all_files.items():
                        new_val = np.concatenate((neuralnet_inputs_all_files[key],
                                                  extract_dict['neuralnet_inputs_dict'][key]))
                        neuralnet_inputs_all_files[key] = new_val
                else:
                    neuralnet_inputs_all_files = extract_dict['neuralnet_inputs_dict']

        # get dir name without the rest of path so it doesn't have separators in the name
        # because those can't be in filename
        just_dir_name = os.getcwd().split(os.path.sep)[-1]
        feature_file = os.path.join(extract_params['output_dir'],
                                    'features_from_' + just_dir_name + '_created_' + timestamp())
        feature_file_dict = {
            'labels': all_labels,
            'feature_list': extract_params['feature_list'],
            'spect_params': extract_params['spect_params'],
            'segment_params': extract_params['segment_params'],
            'labelset': extract_params['labelset'],
            'file_format': extract_params['file_format'],
            'bird_ID': extract_params['bird_ID'],
            'song_IDs': song_IDs
        }

        if 'features_from_all_files' in locals():
            feature_file_dict['features'] = features_from_all_files
            feature_file_dict['features_arr_column_IDs'] = extract_dict['feature_inds']

            if 'feature_list_group_ID' in extract_params:
                feature_file_dict['feature_list_group_ID'] = extract_params['feature_list_group_ID']
                feature_file_dict['feature_group_ID_dict'] = extract_params['feature_group_ID_dict']

        if 'neuralnet_inputs_all_files' in locals():
            feature_file_dict['neuralnet_inputs'] = neuralnet_inputs_all_files

        joblib.dump(feature_file_dict,
                    feature_file,
                    compress=3)

    ##########################################################
    # after looping through all data_dirs for this todo_item #
    ##########################################################

    if make_summary_file:
        print('making summary file')
        os.chdir(extract_params['output_dir'])
        summary_filename = 'summary_feature_file_created_' + timestamp()
        summary_filename_with_path = os.path.join(extract_params['output_dir'],
                                                  summary_filename)
        ftr_output_files = glob.glob('*features_from_*')
        if len(ftr_output_files) > 1:
            # make a 'summary' data file
            list_of_output_dicts = []
            summary_ftr_file_dict = {}
            for feature_file in ftr_output_files:
                feature_file_dict = joblib.load(feature_file)

                if 'labels' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['labels'] = feature_file_dict['labels']
                else:
                    summary_ftr_file_dict['labels'] = \
                        summary_ftr_file_dict['labels'] + feature_file_dict['labels']

                if 'spect_params' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['spect_params'] = feature_file_dict['spect_params']
                else:
                    if feature_file_dict['spect_params'] != summary_ftr_file_dict['spect_params']:
                        raise ValueError('mismatch between spect_params in {} '
                                         'and other feature files'.format(feature_file))

                if 'segment_params' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['segment_params'] = feature_file_dict['segment_params']
                else:
                    if feature_file_dict['segment_params'] != summary_ftr_file_dict['segment_params']:
                        raise ValueError('mismatch between segment_params in {} '
                                         'and other feature files'.format(feature_file))

                if 'labelset' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['labelset'] = feature_file_dict['labelset']
                else:
                    if feature_file_dict['labelset'] != summary_ftr_file_dict['labelset']:
                        raise ValueError('mismatch between labelset in {} '
                                         'and other feature files'.format(feature_file))

                if 'file_format' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['file_format'] = feature_file_dict['file_format']
                else:
                    if feature_file_dict['file_format'] != summary_ftr_file_dict['file_format']:
                        raise ValueError('mismatch between file_format in {} '
                                         'and other feature files'.format(feature_file))

                if 'bird_ID' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['bird_ID'] = feature_file_dict['bird_ID']
                else:
                    if feature_file_dict['bird_ID'] != summary_ftr_file_dict['bird_ID']:
                        raise ValueError('mismatch between bird_ID in {} '
                                         'and other feature files'.format(feature_file))

                if 'song_IDs' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['song_IDs'] = feature_file_dict['song_IDs']
                else:
                    curr_last_ID = summary_ftr_file_dict['song_IDs'][-1]
                    tmp_song_IDs = [el + curr_last_ID + 1 for el in feature_file_dict['song_IDs']]
                    summary_ftr_file_dict['song_IDs'].extend(tmp_song_IDs)

                if 'feature_list' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['feature_list'] = feature_file_dict['feature_list']
                else:
                    if feature_file_dict['feature_list'] != summary_ftr_file_dict['feature_list']:
                        raise ValueError('mismatch between feature_list in {} '
                                         'and other feature files'.format(feature_file))

                # only if not-neuralnet features were used
                if 'features' in feature_file_dict:
                    if 'features' not in summary_ftr_file_dict:
                        summary_ftr_file_dict['features'] = feature_file_dict['features']
                    else:
                        summary_ftr_file_dict['features'] = np.concatenate((summary_ftr_file_dict['features'],
                                                                            feature_file_dict['features']))

                    if 'features_arr_column_IDs' not in summary_ftr_file_dict:
                        summary_ftr_file_dict['features_arr_column_IDs'] = feature_file_dict['features_arr_column_IDs']
                    else:
                        if any(feature_file_dict['features_arr_column_IDs'] !=
                                       summary_ftr_file_dict['features_arr_column_IDs']):
                            raise ValueError('mismatch between features_arr_column_IDs in {} '
                                             'and other feature files'.format(feature_file))

                    if 'feature_list_group_ID' in extract_params:
                        if 'feature_list_group_ID' not in summary_ftr_file_dict:
                            summary_ftr_file_dict['feature_list_group_ID'] = feature_file_dict['feature_list_group_ID']
                        else:
                            if feature_file_dict['feature_list_group_ID'] != \
                                    summary_ftr_file_dict['feature_list_group_ID']:
                                raise ValueError('mismatch between feature_list_group_ID in {} '
                                                 'and other feature files'.format(feature_file))

                        if 'feature_group_ID_dict' not in summary_ftr_file_dict:
                            summary_ftr_file_dict['feature_group_ID_dict'] = \
                                feature_file_dict['feature_group_ID_dict']
                        else:
                            if feature_file_dict['feature_group_ID_dict'] != \
                                    summary_ftr_file_dict['feature_group_ID_dict']:
                                raise ValueError('mismatch between feature_group_ID_dict in {} '
                                                 'and other feature files'.format(feature_file))

                # if extracting inputs for neuralnets
                if 'neuralnet_inputs' in feature_file_dict:
                    if 'neuralnet_inputs' not in summary_ftr_file_dict:
                        summary_ftr_file_dict['neuralnet_inputs'] = \
                            feature_file_dict['neuralnet_inputs']
                    else:
                        for key, val in summary_ftr_file_dict['neuralnet_inputs'].items():
                            newval = np.concatenate((summary_ftr_file_dict['neuralnet_inputs'][key],
                                                     feature_file_dict['neuralnet_inputs'][key]))
                            summary_ftr_file_dict['neuralnet_inputs'][key] = newval

            joblib.dump(summary_ftr_file_dict,
                        summary_filename)

        else:  # if only one feature_file
            os.rename(ftr_output_files[0],
                      summary_filename)
            summary_ftr_file_dict = joblib.load(summary_filename)

        if 'feature_list_group_ID' in summary_ftr_file_dict:
            write_select_config(summary_ftr_file_dict,
                                summary_filename,
                                extract_params['output_dir'])

def extract(config_file):
    """main function that runs feature extraction.
    Does not return anything, just runs through directories specified in config_file
    and extracts features.

    Parameters
    ----------
    config_file : string
        filename of YAML file that configures feature extraction
    """

    extract_config = parse_config(config_file, 'extract')
    print('Parsed extract config.')

    home_dir = os.getcwd()

    todo_list = extract_config['todo_list']
    for ind, todo in enumerate(todo_list):

        print('Completing item {} of {} in to-do list'.format(ind+1, len(todo_list)))
        file_format = todo['file_format']
        if file_format == 'evtaf':
            if 'evfuncs' not in sys.modules:
                from . import evfuncs
        elif file_format == 'koumura':
            if 'koumura' not in sys.modules:
                from . import koumura

        output_dir = 'extract_output_' + timestamp()
        output_dir_with_path = os.path.join(todo['output_dir'], output_dir)
        if not os.path.isdir(output_dir_with_path):
            os.mkdir(output_dir_with_path)

        extract_params = {
            'bird_ID': todo['bird_ID'],
            'feature_list': todo['feature_list'],
            'output_dir': output_dir_with_path,
            'home_dir': home_dir,
            'data_dirs': todo['data_dirs'],
            'labelset': todo['labelset'],
            'file_format': todo['file_format']
        }

        # segment_params defined for todo_list item takes precedence over any default
        # defined for `extract` config
        if 'segment_params' in todo:
            extract_params['segment_params'] = todo['segment_params']
        else:
            extract_params['segment_params'] = extract_config['segment_params']

        if 'spect_params' in todo:
            extract_params['spect_params'] = todo['spect_params']
        else:
            extract_params['spect_params'] = extract_config['spect_params']

        if 'feature_list_group_ID' in todo:
            extract_params['feature_list_group_ID'] = todo['feature_list_group_ID']
            extract_params['feature_group_ID_dict'] = todo['feature_group_ID_dict']

        _extract(extract_params)

        os.chdir(home_dir)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
import codecs
import shutil
import logging

from chardet.universaldetector import UniversalDetector

from .i18n import LocalizedFile

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
logger.addHandler(logging.StreamHandler())

STRINGS_FILE = 'Localizable.strings'

class Merger(object):

    def __init__(self, path, excluded_paths, logging_level=logging.INFO):
        super(Merger, self).__init__()
        self.path = path
        self.excluded_paths = excluded_paths
        logger.level = logging_level

    def _find_encoding(self, filename):
        detector = UniversalDetector()
        for line in open(filename, 'rb'):
            detector.feed(line)
            if detector.done: break
        detector.close()
        best_guess_encoding = detector.result.get('encoding')
        return best_guess_encoding

    def _merge(self, merged_fname, old_fname, new_fname, encoding='utf8'):
        old = LocalizedFile()
        old.read(old_fname, encoding=encoding)
        new = LocalizedFile()
        new.read(new_fname, encoding=encoding)
        merged = old.merge(new)
        merged.save(merged_fname, encoding=encoding)

    def _get_languages(self):
        languages = []
        for root, dirs, files in os.walk(self.path):
            logger.debug(dirs)
            for name in dirs:
                if name.endswith('.lproj'):
                    languages.append(os.path.join(root, name))
        return languages

    def _convert_encoding(self, source_filename, souce_encoding, target_filename, target_encoding):
        block_size = 1024
        with codecs.open(source_filename, "rb", souce_encoding) as sourceFile:
            with codecs.open(target_filename, "wb", target_encoding) as targetFile:
                while True:
                    contents = sourceFile.read(block_size)
                    if not contents:
                        break
                    targetFile.write(contents)

    def _run_genstrings(self, lang):
        if self.excluded_paths:
            exclusion_options = ' '.join('-not -path "{}"'.format(excl_path) for excl_path in self.excluded_paths)
        else:
            exclusion_options = ''
        cmd = 'find {} {} -name \*.m -or -name \*.mm -or -name \*.swift | xargs genstrings -q -o "{}"'.format(
            self.path,
            exclusion_options,
            lang)
        os.system(cmd)

        # convert the generated utf-16 to utf-8
        # cause now Apple is recommending UTF-8
        final_filename = os.path.join(lang, STRINGS_FILE)
        utf8_final_filename = "{}.encoded"
        self._convert_encoding(final_filename, 'utf-16', utf8_final_filename, 'utf-8')
        os.remove(final_filename)
        os.rename(utf8_final_filename, final_filename)

    def _merge_locale(self, lang):
        final_filename = os.path.join(lang, STRINGS_FILE)
        old_filename = '{}.old'.format(final_filename)
        new_filename = '{}.new'.format(final_filename)

        if not os.path.exists(final_filename):
            # create the initial localized strings
            self._run_genstrings(lang)
            return

        encoding = self._find_encoding(final_filename)
        if encoding is None:
            logger.warn("No encoding found for your file '{}'. You should use UTF-8 encoding as recommended by Apple.")
            encoding = 'utf-8'  # fallback
        if encoding.lower().startswith('utf-16'):
            raise ValueError("igenstrings now only supports UTF-8 encoded files how recommended by Apple. "
                             "We found your file to be encoded with `{}`. "
                             "Please convert your file to proceed, with iconv for instance".format(encoding))
        if not encoding.lower().startswith('utf-8'):
            logger.warn("It's recommended to encode your file '{}' with the UTF-8 encoding, "
                        "instead we found '{}', behavior is unexpected".format(final_filename, encoding))

        os.rename(final_filename, old_filename)
        self._run_genstrings(lang)
        shutil.copy(final_filename, new_filename)

        # merge
        self._merge(final_filename, old_filename, new_filename, encoding=encoding)
        logger.info("Job done for lang: %s" % lang)
        os.remove(old_filename)
        os.remove(new_filename)

    def merge_localized_strings(self):
        languages = self._get_languages()
        logger.info("languages found : {}".format(languages))
        for lang in languages:
            self._merge_locale(lang)
<commit_msg>Improve the error message for utf16 encoding<commit_after># -*- coding: utf-8 -*-

import os
import codecs
import shutil
import logging

from chardet.universaldetector import UniversalDetector

from .i18n import LocalizedFile

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
logger.addHandler(logging.StreamHandler())

STRINGS_FILE = 'Localizable.strings'

class Merger(object):

    def __init__(self, path, excluded_paths, logging_level=logging.INFO):
        super(Merger, self).__init__()
        self.path = path
        self.excluded_paths = excluded_paths
        logger.level = logging_level

    def _find_encoding(self, filename):
        detector = UniversalDetector()
        for line in open(filename, 'rb'):
            detector.feed(line)
            if detector.done: break
        detector.close()
        best_guess_encoding = detector.result.get('encoding')
        return best_guess_encoding

    def _merge(self, merged_fname, old_fname, new_fname, encoding='utf8'):
        old = LocalizedFile()
        old.read(old_fname, encoding=encoding)
        new = LocalizedFile()
        new.read(new_fname, encoding=encoding)
        merged = old.merge(new)
        merged.save(merged_fname, encoding=encoding)

    def _get_languages(self):
        languages = []
        for root, dirs, files in os.walk(self.path):
            logger.debug(dirs)
            for name in dirs:
                if name.endswith('.lproj'):
                    languages.append(os.path.join(root, name))
        return languages

    def _convert_encoding(self, source_filename, souce_encoding, target_filename, target_encoding):
        block_size = 1024
        with codecs.open(source_filename, "rb", souce_encoding) as sourceFile:
            with codecs.open(target_filename, "wb", target_encoding) as targetFile:
                while True:
                    contents = sourceFile.read(block_size)
                    if not contents:
                        break
                    targetFile.write(contents)

    def _run_genstrings(self, lang):
        if self.excluded_paths:
            exclusion_options = ' '.join('-not -path "{}"'.format(excl_path) for excl_path in self.excluded_paths)
        else:
            exclusion_options = ''
        cmd = 'find {} {} -name \*.m -or -name \*.mm -or -name \*.swift | xargs genstrings -q -o "{}"'.format(
            self.path,
            exclusion_options,
            lang)
        os.system(cmd)

        # convert the generated utf-16 to utf-8
        # cause now Apple is recommending UTF-8
        final_filename = os.path.join(lang, STRINGS_FILE)
        utf8_final_filename = "{}.encoded"
        self._convert_encoding(final_filename, 'utf-16', utf8_final_filename, 'utf-8')
        os.remove(final_filename)
        os.rename(utf8_final_filename, final_filename)

    def _merge_locale(self, lang):
        final_filename = os.path.join(lang, STRINGS_FILE)
        old_filename = '{}.old'.format(final_filename)
        new_filename = '{}.new'.format(final_filename)

        if not os.path.exists(final_filename):
            # create the initial localized strings
            self._run_genstrings(lang)
            return

        encoding = self._find_encoding(final_filename)
        if encoding is None:
            logger.warn("No encoding found for your file '{}'. You should use UTF-8 encoding as recommended by Apple.")
            encoding = 'utf-8'  # fallback
        if encoding.lower().startswith('utf-16'):
            raise ValueError("igenstrings now only supports UTF-8 encoded files, as recommended by Apple. "
                             "We found that your file is `{}` encoded. "
                             "Please convert your file to proceed, like with this command: "
                             "`$ iconv -f {} -t utf-8 {}`".format(encoding, encoding.lower(), final_filename))
        if not encoding.lower().startswith('utf-8'):
            logger.warn("It's recommended to encode your file '{}' with the UTF-8 encoding, "
                        "instead we found '{}', behavior is unexpected".format(final_filename, encoding))

        os.rename(final_filename, old_filename)
        self._run_genstrings(lang)
        shutil.copy(final_filename, new_filename)

        # merge
        self._merge(final_filename, old_filename, new_filename, encoding=encoding)
        logger.info("Job done for lang: %s" % lang)
        os.remove(old_filename)
        os.remove(new_filename)

    def merge_localized_strings(self):
        languages = self._get_languages()
        logger.info("languages found : {}".format(languages))
        for lang in languages:
            self._merge_locale(lang)
<|endoftext|>


<commit_before># Copyright (c) 2015 Rackspace, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pyrax

retry_exceptions = [
    pyrax.exc.DNSCallTimedOut,
    pyrax.exc.OverLimit
]
<commit_msg>fix: add exception classes on which retry operations need to be done<commit_after># Copyright (c) 2015 Rackspace, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from pyrax import exceptions

retry_exceptions = [
    exceptions.DNSCallTimedOut,
    exceptions.OverLimit,
    exceptions.DomainCreationFailed,
    exceptions.DomainDeletionFailed,
    exceptions.DomainRecordUpdateFailed,
    exceptions.DomainRecordAdditionFailed,
    exceptions.DomainRecordDeletionFailed,
    exceptions.DomainRecordUpdateFailed,
]
<|endoftext|>


<commit_before>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Request (Model) query functions.
"""

__authors__ = [
  '"Sverre Rabbelier" <sverre@rabbelier.nl>',
  '"Lennard de Rijk" <ljvderijk@gmail.com>'
  ]

import soc.models.request

from soc.logic.helper import notifications
from soc.logic.models import base
from soc.logic.models import linkable as linkable_logic

class Logic(base.Logic):
  """Logic methods for the Request model.
  """

  def __init__(self, model=soc.models.request.Request,
               base_model=None, scope_logic=linkable_logic):
    """Defines the name, key_name and model for this entity.
    """

    super(Logic, self).__init__(model, base_model=base_model,
                                scope_logic=scope_logic)

  def getKeyValuesFromEntity(self, entity):
    """See base.Logic.getKeyNameValues.
    """

    return [entity.scope.link_id, entity.role, entity.link_id]

  def getKeyValuesFromFields(self, fields):
    """See base.Logic.getKeyValuesFromFields.
    """

    return [fields['scope_path'], fields['role'], fields['link_id']]

  def getKeyFieldNames(self):
    """See base.Logic.getKeyFieldNames.
    """

    return ['scope_path', 'role', 'link_id']

  def _onCreate(self, entity):
    """Sends out a message notifying users about the new invite/request.
    """

    if entity.status == 'group_accepted':
      # this is an invite
      notifications.sendInviteNotification(entity)
    elif entity.status == 'new':
      # this is a request
      # TODO(Lennard) Create a new request message
      pass

    super(Logic, self)._onCreate(entity)

  def _updateField(self, entity, entity_properties, name):
    """Called when the fields of the request are updated.

      - Sends out a message depending on the change of status
    """

    value = entity_properties[name]

    if name == 'status' and entity.status != value:
      if value == 'group_accepted':
       # this is an invite
        notifications.sendInviteNotification(entity)
      elif value == 'new':
        # this is a request
        # TODO(Lennard) Create a new request message
        pass

    return super(Logic, self)._updateField(entity, entity_properties, name)

logic = Logic()
<commit_msg>Minor style fix.<commit_after>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Request (Model) query functions.
"""

__authors__ = [
  '"Sverre Rabbelier" <sverre@rabbelier.nl>',
  '"Lennard de Rijk" <ljvderijk@gmail.com>'
  ]

import soc.models.request

from soc.logic.helper import notifications
from soc.logic.models import base
from soc.logic.models import linkable as linkable_logic

class Logic(base.Logic):
  """Logic methods for the Request model.
  """

  def __init__(self, model=soc.models.request.Request,
               base_model=None, scope_logic=linkable_logic):
    """Defines the name, key_name and model for this entity.
    """

    super(Logic, self).__init__(model, base_model=base_model,
                                scope_logic=scope_logic)

  def getKeyValuesFromEntity(self, entity):
    """See base.Logic.getKeyNameValues.
    """

    return [entity.scope.link_id, entity.role, entity.link_id]

  def getKeyValuesFromFields(self, fields):
    """See base.Logic.getKeyValuesFromFields.
    """

    return [fields['scope_path'], fields['role'], fields['link_id']]

  def getKeyFieldNames(self):
    """See base.Logic.getKeyFieldNames.
    """

    return ['scope_path', 'role', 'link_id']

  def _onCreate(self, entity):
    """Sends out a message notifying users about the new invite/request.
    """

    if entity.status == 'group_accepted':
      # this is an invite
      notifications.sendInviteNotification(entity)
    elif entity.status == 'new':
      # this is a request
      # TODO(Lennard) Create a new request message
      pass

    super(Logic, self)._onCreate(entity)

  def _updateField(self, entity, entity_properties, name):
    """Called when the fields of the request are updated.

      Sends out a message depending on the change of status.
    """

    value = entity_properties[name]

    if name == 'status' and entity.status != value:
      if value == 'group_accepted':
       # this is an invite
        notifications.sendInviteNotification(entity)
      elif value == 'new':
        # this is a request
        # TODO(Lennard) Create a new request message
        pass

    return super(Logic, self)._updateField(entity, entity_properties, name)

logic = Logic()
<|endoftext|>


<commit_before>### THIS IS A SKETCH

import os
import re
from math import log1p

from cheshire3.document import StringDocument
from cheshire3.internal import cheshire3Root
from cheshire3.server import SimpleServer
from cheshire.baseObjects import Session

cheshirePath = os.path.join('HOME', '/home/cheshire')

class Keywords(object):

    def __init__(self):
        self.session = Session()
        session.database = 'db_dickens'
        serv = SimpleServer(session,
                            os.path.join(cheshire3Root, 'configs', 'serverConfig.xml')
                            )
        self.db = serv.get_object(session, session.database)
        self.qf = self.db.get_object(session, 'defaultQueryFactory')
        self.resultSetStore = self.db.get_object(session, 'resultSetStore')
        self.idxStore = self.db.get_object(session, 'indexStore')
        self.logger = self.db.get_object(session, 'keywordLogger')

    def list_keywords(self, testIdxName, testMaterials, refIdxName, refMaterials):
        self.logger.log('CREATING KEYWORDS FOR RS: {0}'.format(id))
        session = self.session
        db = self.db

        # Test results
        clauses = []
        for testMaterial in testMaterials:
            if testMaterial in ['dickens', 'ntc']:
                testMatIdx = 'subCorpus-idx'
            else:
                testMatIdx = 'novel-idx'
            clauses.append('c3.{0} = "{1}"'.format(testMatIdx, testMaterial))

        test_query = self.qf.get_query(session,
                                       ' or '.join(clauses)
                                       )
        test_results = db.search(session, test_query)
        test_idx = db.get_object(session, testIdxName)
        test_facets = test_idx.facets(session, test_results)
        ## create dictionary containing word/cluster and number of occurrences
        test_dict = {x[0]: x[1][2] for x in test_facets}

        # [("term", (termId, totalRecords, totalOccurrences)), ("tern2", (...))]
        # TODO: Repeat for reference
        # Reference results
        clauses_ref = []
        for refMaterial in refMaterials:
            if refMaterial in ['dickens', 'ntc']:
                refMatIdx = 'subCorpus-idx'
            else:
                refMatIdx = 'novel-idx'
            clauses_ref.append('c3.{0} = "{1}"'.format(refMatIdx, refMaterial))

        ref_query = self.qf.get_query(session,
                                       ' or '.join(clauses_ref)
                                       )
        ref_results = db.search(session, ref_query)
        ref_idx = db.get_object(session, refIdxName)
        ref_facets = ref_idx.facets(session, ref_results)
        ref_dict = {x[0]: x[1][2] for x in ref_facets}

        ## get test length
        testLength = sum(test_dict.values())
        refLength = sum(ref_dict.values())
#
#         refLength = 0
#         for i in referenceIndex:
#             refLength += i.totalOccs
#         ## refLength is the difference between reference and test
#         refLength = refLength - testLength

#         KW_list = ""
#         for i in testIndex:
#             freqTest = i.totalOccs
#             q = qf.get_query(session, 'c3.%s-idx = "%s"' % (idx, i.queryTerm))
#             ## search for query in sentences
#             ## (not sure what 1, = does)
#             ref = referenceIndex.scan(session, q, 1, '=')
#             if len(ref):
#                 ## We want to know how many observations of a given word is found in ref corpus but not in test corpus
#                 ## Subtract number of occurrences in testIndex from number of occurrences in sentences
#                 freqRef = float(ref[0][1][2] - freqTest)
#                 if freqRef <= 0:
#                     freqRef = 5.0e-324
#             else:
#                 freqRef = 5.0e-324

        kw_list = []
        for term, freqTest in test_dict.iteritems():
            try:
                ## We want to know how many observations of a given word is found in ref corpus but not in test corpus
                ## Subtract number of occurrences in testIndex from number of occurrences in sentences
                freqRef = float(ref_dict[term] - freqTest)
            except KeyError:
                freqRef = 5.0e-324
            else:
                if freqRef <= 0:
                    freqRef = 5.0e-324

            ## following Paul Ryson formula for log likelihood (http://ucrel.lancs.ac.uk/llwizard.html)
            ## 1. Expected occurrence within corpus
            ## 1a. Expected reference value: based on sentence index
            ## - Get the total N from corpus 1 (reference corpus)
            ## - Multiply by the sum of observations only found in corpus 1 and those only found in corpus 2 (test corpus)
            ## - Divide by the sum of total N in test corpus and reference corpus
            expectedRef = refLength*(freqTest+freqRef)/(testLength+refLength)
            ## 1b. Expected test value: based on quotes index
            ## Equivalent steps to 1a, but multiply by test N
            expectedTest = testLength*(freqTest+freqRef)/(testLength+refLength)

            ## 2. Log Likelihood
            ## Compare actual observations with expected ocurrence for both test and ref, and add these values
            ## Use log1p() (for natural logarithm - ln) instead of log()
            try:
                LL = 2*((freqTest*log1p(freqTest/expectedTest)) + (freqRef*log1p(freqRef/expectedRef)))
            except:
                LL = 909090

            #print '%s\t%d\t%d\t%.2f' % (i.queryTerm, freqTest, freqRef, LL)
            ## only print if occurence > 5
            if freqTest > 5:
#                 KW = '%s\t%d\t%d\t%.2f\t%.2f' % (i.queryTerm, freqTest, freqRef, LL, refTestRatio)
#                 KW_list = ''.join(KW_list + '\n' + KW)
                kw_list.append([term, freqTest, freqRef, LL])

            return kw_list

        <commit_msg>first setup of keywords app<commit_after>### THIS IS A SKETCH

import os
import re
from math import log1p

from cheshire3.document import StringDocument
from cheshire3.internal import cheshire3Root
from cheshire3.server import SimpleServer
from cheshire.baseObjects import Session

cheshirePath = os.path.join('HOME', '/home/cheshire')

class Keywords(object):

    def __init__(self):
        self.session = Session()
        session.database = 'db_dickens'
        serv = SimpleServer(session,
                            os.path.join(cheshire3Root, 'configs', 'serverConfig.xml')
                            )
        self.db = serv.get_object(session, session.database)
        self.qf = self.db.get_object(session, 'defaultQueryFactory')
        self.resultSetStore = self.db.get_object(session, 'resultSetStore')
        self.idxStore = self.db.get_object(session, 'indexStore')
        self.logger = self.db.get_object(session, 'keywordLogger')

    def list_keywords(self, testIdxName, testMaterials, refIdxName, refMaterials):
        self.logger.log('CREATING KEYWORDS FOR RS: {0}'.format(id))
        session = self.session
        db = self.db

        # Test results
        clauses = []
        for testMaterial in testMaterials:
            if testMaterial in ['dickens', 'ntc']:
                testMatIdx = 'subCorpus-idx'
            else:
                testMatIdx = 'book-idx'
            clauses.append('c3.{0} = "{1}"'.format(testMatIdx, testMaterial))

        test_query = self.qf.get_query(session,
                                       ' or '.join(clauses)
                                       )
        test_results = db.search(session, test_query)
        test_idx = db.get_object(session, testIdxName)
        test_facets = test_idx.facets(session, test_results)
        ## create dictionary containing word/cluster and number of occurrences
        test_dict = {x[0]: x[1][2] for x in test_facets}

        # [("term", (termId, totalRecords, totalOccurrences)), ("tern2", (...))]
        # Reference results
        clauses_ref = []
        for refMaterial in refMaterials:
            if refMaterial in ['dickens', 'ntc']:
                refMatIdx = 'subCorpus-idx'
            else:
                refMatIdx = 'book-idx'
            clauses_ref.append('c3.{0} = "{1}"'.format(refMatIdx, refMaterial))

        ref_query = self.qf.get_query(session,
                                       ' or '.join(clauses_ref)
                                       )
        ref_results = db.search(session, ref_query)
        ref_idx = db.get_object(session, refIdxName)
        ref_facets = ref_idx.facets(session, ref_results)
        ref_dict = {x[0]: x[1][2] for x in ref_facets}

        ## get test length
        testLength = sum(test_dict.values())
        refLength = sum(ref_dict.values())
#
#         refLength = 0
#         for i in referenceIndex:
#             refLength += i.totalOccs
#         ## refLength is the difference between reference and test
#         refLength = refLength - testLength

#         KW_list = ""
#         for i in testIndex:
#             freqTest = i.totalOccs
#             q = qf.get_query(session, 'c3.%s-idx = "%s"' % (idx, i.queryTerm))
#             ## search for query in sentences
#             ## (not sure what 1, = does)
#             ref = referenceIndex.scan(session, q, 1, '=')
#             if len(ref):
#                 ## We want to know how many observations of a given word is found in ref corpus but not in test corpus
#                 ## Subtract number of occurrences in testIndex from number of occurrences in sentences
#                 freqRef = float(ref[0][1][2] - freqTest)
#                 if freqRef <= 0:
#                     freqRef = 5.0e-324
#             else:
#                 freqRef = 5.0e-324

        kw_list = []
        for term, freqTest in test_dict.iteritems():
            try:
                ## We want to know how many observations of a given word is found in ref corpus but not in test corpus
                ## Subtract number of occurrences in testIndex from number of occurrences in sentences
                freqRef = float(ref_dict[term] - freqTest)
            except KeyError:
                freqRef = 5.0e-324
            else:
                if freqRef <= 0:
                    freqRef = 5.0e-324

            ## following Paul Ryson formula for log likelihood (http://ucrel.lancs.ac.uk/llwizard.html)
            ## 1. Expected occurrence within corpus
            ## 1a. Expected reference value: based on sentence index
            ## - Get the total N from corpus 1 (reference corpus)
            ## - Multiply by the sum of observations only found in corpus 1 and those only found in corpus 2 (test corpus)
            ## - Divide by the sum of total N in test corpus and reference corpus
            expectedRef = refLength*(freqTest+freqRef)/(testLength+refLength)
            ## 1b. Expected test value: based on quotes index
            ## Equivalent steps to 1a, but multiply by test N
            expectedTest = testLength*(freqTest+freqRef)/(testLength+refLength)

            ## 2. Log Likelihood
            ## Compare actual observations with expected ocurrence for both test and ref, and add these values
            ## Use log1p() (for natural logarithm - ln) instead of log()
            try:
                LL = 2*((freqTest*log1p(freqTest/expectedTest)) + (freqRef*log1p(freqRef/expectedRef)))
            except:
                LL = 909090

            #print '%s\t%d\t%d\t%.2f' % (i.queryTerm, freqTest, freqRef, LL)
            ## only print if occurence > 5
            if freqTest > 5:
#                 KW = '%s\t%d\t%d\t%.2f\t%.2f' % (i.queryTerm, freqTest, freqRef, LL, refTestRatio)
#                 KW_list = ''.join(KW_list + '\n' + KW)
                kw_list.append([term, freqTest, freqRef, LL])

            return kw_list

        <|endoftext|>


<commit_before>import itertools
import math
from functools import lru_cache
from typing import Tuple, Iterator

import numpy as np
import numpy.ma as ma
from tqdm import tqdm

from .pose import Pose

class PoseVisualizer:
    def __init__(self, pose: Pose, thickness=None):
        self.pose = pose
        self.thickness = thickness
        self.pose_fps = float(self.pose.body.fps)

        try:
            import cv2
            self.cv2 = cv2
        except ImportError:
            raise ImportError(
                "Please install OpenCV with: pip install opencv-python"
            )

    def _draw_frame(self, frame: ma.MaskedArray, frame_confidence: np.ndarray, img) -> np.ndarray:
        background_color = img[0][0]  # Estimation of background color for opacity. `mean` is slow

        thickness = self.thickness if self.thickness is not None else round(
            math.sqrt(img.shape[0] * img.shape[1]) / 150)
        radius = round(thickness / 2)

        for person, person_confidence in zip(frame, frame_confidence):
            c = person_confidence.tolist()
            idx = 0
            for component in self.pose.header.components:
                colors = [np.array(c[::-1]) for c in component.colors]

                @lru_cache(maxsize=None)
                def _point_color(p_i: int):
                    opacity = c[p_i + idx]
                    np_color = colors[p_i % len(component.colors)] * opacity + (1 - opacity) * background_color
                    return tuple([int(c) for c in np_color])

                # Draw Points
                for i, point_name in enumerate(component.points):
                    if c[i + idx] > 0:
                        self.cv2.circle(img=img, center=tuple(person[i + idx][:2]), radius=radius,
                                        color=_point_color(i), thickness=-1, lineType=16)

                if self.pose.header.is_bbox:
                    point1 = tuple(person[0 + idx].tolist())
                    point2 = tuple(person[1 + idx].tolist())
                    color = tuple(np.mean([_point_color(0), _point_color(1)], axis=0))

                    self.cv2.rectangle(img=img, pt1=point1, pt2=point2, color=color, thickness=thickness)
                else:
                    int_person = person.astype(np.int32)
                    # Draw Limbs
                    for (p1, p2) in component.limbs:
                        if c[p1 + idx] > 0 and c[p2 + idx] > 0:
                            point1 = tuple(int_person[p1 + idx].tolist()[:2])
                            point2 = tuple(int_person[p2 + idx].tolist()[:2])

                            # length = ((point1[0] - point2[0]) ** 2 + (point1[1] - point2[1]) ** 2) ** 0.5

                            color = tuple(np.mean([_point_color(p1), _point_color(p2)], axis=0))

                            self.cv2.line(img, point1, point2, color, thickness, lineType=self.cv2.LINE_AA)

                            # deg = math.degrees(math.atan2(point1[1] - point2[1], point1[0] - point2[0]))
                            # polygon = cv2.ellipse2Poly(
                            #   (int((point1[0] + point2[0]) / 2), int((point1[1] + point2[1]) / 2)),
                            #   (int(length / 2), thickness),
                            #   int(deg),
                            #   0, 360, 1)
                            # cv2.fillConvexPoly(img=img, points=polygon, color=color)

                idx += len(component.points)

        return img

    def draw(self, background_color: Tuple[int, int, int] = (255, 255, 255), max_frames: int = None):
        int_data = np.array(np.around(self.pose.body.data.data), dtype="int32")
        background = np.full((self.pose.header.dimensions.height, self.pose.header.dimensions.width, 3),
                             fill_value=background_color, dtype="uint8")
        for frame, confidence in itertools.islice(zip(int_data, self.pose.body.confidence), max_frames):
            yield self._draw_frame(frame, confidence, img=background.copy())

    def draw_on_video(self, background_video, max_frames: int = None, blur=False):
        int_data = np.array(np.around(self.pose.body.data.data), dtype="int32")

        if max_frames is None:
            max_frames = len(int_data)

        def get_frames(video_path):

            cap = self.cv2.VideoCapture(video_path)
            video_fps = cap.get(self.cv2.CAP_PROP_FPS)

            assert math.isclose(video_fps, self.pose_fps, abs_tol=0.5), \
                "Fps of pose and video do not match: %f != %f" % (self.pose_fps, video_fps)

            while True:
                ret, vf = cap.read()
                if not ret:
                    break
                yield vf
            cap.release()

        if isinstance(background_video, str):
            background_video = iter(get_frames(background_video))

        for frame, confidence, background in itertools.islice(
                zip(int_data, self.pose.body.confidence, background_video),
                max_frames):
            background = self.cv2.resize(background,
                                         (self.pose.header.dimensions.width, self.pose.header.dimensions.height))

            if blur:
                background = self.cv2.blur(background, (20, 20))

            yield self._draw_frame(frame, confidence, background)

    def save_frame(self, f_name: str, frame: np.ndarray):
        self.cv2.imwrite(f_name, frame)

    def save_video(self, f_name: str, frames: Iterator, custom_ffmpeg=None):
        try:
            from vidgear.gears import WriteGear
        except ImportError:
            raise ImportError(
                "Please install vidgear with: pip install vidgear"
            )

        # image_size = (self.pose.header.dimensions.width, self.pose.header.dimensions.height)

        output_params = {
            "-vcodec": "libx264",
            "-preset": "fast",
            "-input_framerate": self.pose.body.fps,
            "-pix_fmt": "yuv420p",
        }

        # Define writer with defined parameters and suitable output filename for e.g. `Output.mp4`
        out = WriteGear(output_filename=f_name, logging=False, custom_ffmpeg=custom_ffmpeg, **output_params)
        # out = cv2.VideoWriter(f_name, cv2.VideoWriter_fourcc(*'MP4V'), self.pose.body.fps, image_size)
        for frame in tqdm(frames):
            out.write(frame)

        out.close()
<commit_msg>feat(pose visualizer): add gif generation<commit_after>import itertools
import math
from functools import lru_cache
from typing import Tuple, Iterable

import numpy as np
import numpy.ma as ma
from tqdm import tqdm

from .pose import Pose

class PoseVisualizer:
    def __init__(self, pose: Pose, thickness=None):
        self.pose = pose
        self.thickness = thickness
        self.pose_fps = float(self.pose.body.fps)

        try:
            import cv2
            self.cv2 = cv2
        except ImportError:
            raise ImportError(
                "Please install OpenCV with: pip install opencv-python"
            )

    def _draw_frame(self, frame: ma.MaskedArray, frame_confidence: np.ndarray, img) -> np.ndarray:
        background_color = img[0][0]  # Estimation of background color for opacity. `mean` is slow

        thickness = self.thickness if self.thickness is not None else round(
            math.sqrt(img.shape[0] * img.shape[1]) / 150)
        radius = round(thickness / 2)

        for person, person_confidence in zip(frame, frame_confidence):
            c = person_confidence.tolist()
            idx = 0
            for component in self.pose.header.components:
                colors = [np.array(c[::-1]) for c in component.colors]

                @lru_cache(maxsize=None)
                def _point_color(p_i: int):
                    opacity = c[p_i + idx]
                    np_color = colors[p_i % len(component.colors)] * opacity + (1 - opacity) * background_color
                    return tuple([int(c) for c in np_color])

                # Draw Points
                for i, point_name in enumerate(component.points):
                    if c[i + idx] > 0:
                        self.cv2.circle(img=img, center=tuple(person[i + idx][:2]), radius=radius,
                                        color=_point_color(i), thickness=-1, lineType=16)

                if self.pose.header.is_bbox:
                    point1 = tuple(person[0 + idx].tolist())
                    point2 = tuple(person[1 + idx].tolist())
                    color = tuple(np.mean([_point_color(0), _point_color(1)], axis=0))

                    self.cv2.rectangle(img=img, pt1=point1, pt2=point2, color=color, thickness=thickness)
                else:
                    int_person = person.astype(np.int32)
                    # Draw Limbs
                    for (p1, p2) in component.limbs:
                        if c[p1 + idx] > 0 and c[p2 + idx] > 0:
                            point1 = tuple(int_person[p1 + idx].tolist()[:2])
                            point2 = tuple(int_person[p2 + idx].tolist()[:2])

                            # length = ((point1[0] - point2[0]) ** 2 + (point1[1] - point2[1]) ** 2) ** 0.5

                            color = tuple(np.mean([_point_color(p1), _point_color(p2)], axis=0))

                            self.cv2.line(img, point1, point2, color, thickness, lineType=self.cv2.LINE_AA)

                            # deg = math.degrees(math.atan2(point1[1] - point2[1], point1[0] - point2[0]))
                            # polygon = cv2.ellipse2Poly(
                            #   (int((point1[0] + point2[0]) / 2), int((point1[1] + point2[1]) / 2)),
                            #   (int(length / 2), thickness),
                            #   int(deg),
                            #   0, 360, 1)
                            # cv2.fillConvexPoly(img=img, points=polygon, color=color)

                idx += len(component.points)

        return img

    def draw(self, background_color: Tuple[int, int, int] = (255, 255, 255), max_frames: int = None):
        int_data = np.array(np.around(self.pose.body.data.data), dtype="int32")
        background = np.full((self.pose.header.dimensions.height, self.pose.header.dimensions.width, 3),
                             fill_value=background_color, dtype="uint8")
        for frame, confidence in itertools.islice(zip(int_data, self.pose.body.confidence), max_frames):
            yield self._draw_frame(frame, confidence, img=background.copy())

    def draw_on_video(self, background_video, max_frames: int = None, blur=False):
        int_data = np.array(np.around(self.pose.body.data.data), dtype="int32")

        if max_frames is None:
            max_frames = len(int_data)

        def get_frames(video_path):

            cap = self.cv2.VideoCapture(video_path)
            video_fps = cap.get(self.cv2.CAP_PROP_FPS)

            assert math.isclose(video_fps, self.pose_fps, abs_tol=0.5), \
                "Fps of pose and video do not match: %f != %f" % (self.pose_fps, video_fps)

            while True:
                ret, vf = cap.read()
                if not ret:
                    break
                yield vf
            cap.release()

        if isinstance(background_video, str):
            background_video = iter(get_frames(background_video))

        for frame, confidence, background in itertools.islice(
                zip(int_data, self.pose.body.confidence, background_video),
                max_frames):
            background = self.cv2.resize(background,
                                         (self.pose.header.dimensions.width, self.pose.header.dimensions.height))

            if blur:
                background = self.cv2.blur(background, (20, 20))

            yield self._draw_frame(frame, confidence, background)

    def save_frame(self, f_name: str, frame: np.ndarray):
        self.cv2.imwrite(f_name, frame)

    def save_gif(self, f_name: str, frames: Iterable[np.ndarray]):
        try:
            from PIL import Image
        except ImportError:
            raise ImportError(
                "Please install Pillow with: pip install Pillow"
            )

        images = [Image.fromarray(self.cv2.cvtColor(frame, self.cv2.COLOR_BGR2RGB)) for frame in frames]
        images[0].save(f_name, format="GIF", append_images=images,
                       save_all=True, duration=1000 / self.pose.body.fps, loop=0)

    def save_video(self, f_name: str, frames: Iterable[np.ndarray], custom_ffmpeg=None):
        try:
            from vidgear.gears import WriteGear
        except ImportError:
            raise ImportError(
                "Please install vidgear with: pip install vidgear"
            )

        # image_size = (self.pose.header.dimensions.width, self.pose.header.dimensions.height)

        output_params = {
            "-vcodec": "libx264",
            "-preset": "fast",
            "-input_framerate": self.pose.body.fps,
            "-pix_fmt": "yuv420p",
        }

        # Define writer with defined parameters and suitable output filename for e.g. `Output.mp4`
        out = WriteGear(output_filename=f_name, logging=False, custom_ffmpeg=custom_ffmpeg, **output_params)
        # out = cv2.VideoWriter(f_name, cv2.VideoWriter_fourcc(*'MP4V'), self.pose.body.fps, image_size)
        for frame in tqdm(frames):
            out.write(frame)

        out.close()
<|endoftext|>