<commit_before>#!/usr/bin/env python3

"""
python3 concat.py [-h] [--generate] [--merge] [--both] [--max-gap MAX_GAP]
                 [-e EXT]
                 [TARGET_DIR]

Generates concat files for merging. Creates separate videos for separate
broadcasts and incompatible resolutions, ignores very broken videos.

positional arguments:
  TARGET_DIR         Optional. defaults to the current working directory.

optional arguments:
  -h, --help         show this help message and exit
  --generate         generates concat files in TARGET_DIR, runs by default
  --merge            merges videos in TARGET_DIR according to existing concat
                     files
  --both             both generates concat files and merges videos
  --max-gap MAX_GAP  maximum gap between merged videos, in seconds. anything
                     larger is treated as a separate broadcast
  -e EXT             extension to merge, defaults to mp4

When merging, watch the output for "Non-monotonous DTS in output stream" -- A
few of these are harmless but a wall of them means that video is probably
corrupted.

"""

import os
import glob
import argparse
from subprocess import check_output, run, CalledProcessError
import json
from math import floor

# known resolutions:
# 352x198
# 640x360
# 704x396
# 1280x720 (a single kimi dare episode)
GOOD_HEIGHTS = (198, 360, 396, 720)

# old version
"""
def create_concat_files(target_dir, target_ext):
    oldcwd = os.getcwd()
    os.chdir(target_dir)

    # TODO: use ffprobe to separate files with incompatible resolutions and those with a gap greater than ~10 minutes

    files = sorted(glob.glob('{}/*.{}'.format(target_dir, target_ext)))

    member_dict = {}
    for file in files:
        member_name = file.rsplit(' ', 1)[0]
        if member_name not in member_dict:
            member_dict[member_name] = []
        member_dict[member_name].append(file)

    concat_files = {}
    for key in member_dict.keys():
        filename = key +' ' + member_dict[key][0].rsplit(' ', 1)[1][:4] + '.mp4.concat'
        text = ""
        for item in member_dict[key]:
            text += "file '" + item + "'\n"
        concat_files.update({filename:text})

    for key in concat_files.keys():
        with open(key, 'w', encoding='utf8') as outfp:
            _ = outfp.write(concat_files[key])

    os.chdir(oldcwd)
"""

"""
{
    "member1" : [
        video1,
        video2,
        video3,
        video4
    ],
    "member2" : [

    ]
}

video: {
    "start_time" : parsed from file name; in seconds,
    "duration"   : parsed from ffprobe,
    "height"     : parsed from ffprobe,
    "valid"      : true or false (false for stuff with no video content),
    "file"       : location of file
}
"duration"
"height"

sample ffprobe output:
{
    "programs": [

    ],
    "streams": [
        {
            "height": 198,
            "duration": "499.654000"
        }
    ]
}
for member in members:

"""

def generate_concat_files(target_dir, target_ext, max_gap):
    oldcwd = os.getcwd()
    os.chdir(target_dir)

    max_gap = float(max_gap)

    # TODO: deal with leftovers (from after 24:00)
    files = sorted(glob.glob('*.{}'.format(target_ext)))

    def get_start_seconds(file):
        time_str = file.rsplit(' ', 1)[1].split('.')[0]
        hours, minutes, seconds = int(time_str[:2]), int(time_str[2:4]), int(time_str[4:6])
        return float(hours*60*60 + minutes*60 + seconds)

    def get_start_hhmm(seconds):
        hours = seconds/(60*60)
        minutes = (hours - floor(hours))*60
        return '{:02d}{:02d}'.format(floor(hours), floor(minutes))

    member_dict = {}
    for file in files:
        try:
            results = check_output([
                    "ffprobe",
                    '-loglevel', '16',
                    '-show_entries', 'stream=duration,height',
                    '-select_streams', 'v',
                    '-i', file,
                    '-of', 'json'
                ],
                universal_newlines=True
                )
        except CalledProcessError:
            continue

        member_name = file.rsplit(' ', 1)[0]
        if member_name not in member_dict:
            member_dict[member_name] = []
        new_video = {"start_time": get_start_seconds(file)}
        try:
            stream = json.loads(results)['streams'][0]
        except IndexError:
            new_video['valid']  = False
        else:
            new_video['file']     = file
            new_video['duration'] = float(stream['duration'])
            new_video['height']   = int(stream['height'])
            if new_video['duration'] >= 0.001 and (new_video['height'] in GOOD_HEIGHTS):
                new_video['valid']  = True
            else:
                new_video['valid']  = False

        if new_video['valid']:
            member_dict[member_name].append(new_video)

    concat_files = {}

    def new_concat_file(member, first_video):
        filename = '{} {}.{}.concat'.format(member, get_start_hhmm(first_video['start_time']), target_ext)
        info = {'files': []}
        info['height'] = first_video['height']
        info['last_time'] = first_video['start_time'] + first_video['duration']
        info['files'].append(first_video['file'])
        return filename, info

    for member in member_dict.keys():
        """
        file_specifier (name + hhmm) : {
            height : 360 or 198,
            last_time : start_time + duration of most recently processed video
            files: [
                list of files
            ]
        }
        """
        try:
            filename, working = new_concat_file(member, member_dict[member][0])
        except IndexError:
            # no valid videos
            print('Failed to read videos for {}'.format(member))
            continue
        for item in member_dict[member][1:]:
            if (item['start_time'] >= working['last_time'] + max_gap
                    or item['height'] != working['height']):
                if filename in working:
                    # This needs to be dealt with by hand for now
                    print('Tried to add duplicate concat file name: {}'.format(filename))
                    raise FileExistsError
                concat_files[filename] = working
                filename, working = new_concat_file(member, item)
            else:
                if item['start_time'] < working['last_time'] - 5.0:
                    print('{} overlaps {}'.format(item['file'], working['files'][-1]))
                    # these have to be dealt with manually
                working['files'].append(item['file'])
                working['last_time'] = item['start_time'] + item['duration']
        concat_files[filename] = working

    for file in concat_files.keys():
        # skip singleton videos
        # if len(concat_files[file]['files']) == 1:
        #    continue
        text = ""
        for item in concat_files[file]['files']:
            text += "file '" + item + "'\n"
        with open(file, 'w', encoding='utf8') as outfp:
            outfp.write(text)

    os.chdir(oldcwd)

"""
#!/bin/bash

# for f in ./*.mp4; do echo "file '$f'" >> mylist.txt; done
# for f in ./*.concat; do echo "$( basename $f )"; done
# for f in ./*.concat; do g="\"$( basename "$f" .mp4)\""; echo $f; echo $g; done
# echo "\"$( basename ./160612\ Showroom\ -\ AKB48\ Team\ K\ Tano\ Yuka\ 124028.mp4 .mp4)\""
for f in ./*.concat; do
    g=$( basename "$f" .concat);
    #ffmpeg -copytb 1 -f concat -i "$f" -vf "pad=width=640:height=360:(ow-iw)/2:(oh-ih)/2:color=black" -movflags +faststart "$g";
    ffmpeg -copytb 1 -f concat -i "$f" -movflags +faststart -c copy "$g";
done
"""

def merge_videos(target_dir, output_dir, copytb=1):
    oldcwd = os.getcwd()
    os.chdir(target_dir)

    os.makedirs(output_dir, exist_ok=True)

    for concat_file in glob.glob('*.concat'):
        outfile = '{}/{}'.format(output_dir, os.path.splitext(concat_file)[0])

        instructions = ['-copytb', str(copytb)]

        # wish i didn't have to read it twice...
        with open(concat_file, encoding='utf8') as infp:
            data = infp.read()
        if data.count('file \'') == 1:
            src = data[5:].strip('\'\n./')
            instructions.extend(['-i', src])
        elif data.count('file \'') > 1:
            instructions.extend(['-f', 'concat', '-safe', '0', '-i', concat_file])
        else:
            print("Empty concat file")
            raise FileNotFoundError

        run(['ffmpeg',
            *instructions,
            '-movflags', '+faststart',
            '-c', 'copy', outfile])

    os.chdir(oldcwd)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description="Generates concat files for merging. Creates separate videos for separate broadcasts and \
        incompatible resolutions, ignores very broken videos.",
        epilog="When merging, watch the output for \"Non-monotonous DTS in output stream\" -- A few of these are \
        harmless but a wall of them means that video is probably corrupted.")
    parser.add_argument("--generate", action='store_true', help='generates concat files in TARGET_DIR, runs by default')
    parser.add_argument("--merge", action='store_true',
                        help='merges videos in TARGET_DIR according to existing concat files')
    parser.add_argument("--both", action='store_true', help='both generates concat files and merges videos')
    parser.add_argument("target_dir", nargs='?', default='.',
                        help='Optional. defaults to the current working directory.', metavar='TARGET_DIR')
    parser.add_argument("--max-gap", type=float, default=300.0,
                        help='maximum gap between merged videos, in seconds. anything larger is treated as a separate \
                        broadcast. default = 300.0')
    parser.add_argument("-e", dest='ext', default='mp4', help='extension to merge, defaults to mp4')
    parser.add_argument("--copytb", type=int, choices=[-1, 0, 1], default=1,
                        help='it may be useful to try setting this to 0 or -1 if a video has timing issues')
    parser.add_argument("--output-dir", "-o", dest='output_dir', type=str, default='.',
                        help='Optional, defaults to target directory. Note that relative paths will be relative to \
                        the target directory, not the current working directory', metavar='OUTPUT_DIR')
    args = parser.parse_args()

    if not args.merge:
        generate_concat_files(args.target_dir, args.ext, args.max_gap)
    if args.merge or args.both:
        merge_videos(args.target_dir, args.output_dir, args.copytb)

<commit_msg>Option to resize during concat (#5)<commit_after>#!/usr/bin/env python3

"""
python3 concat.py [-h] [--generate] [--merge] [--both] [--max-gap MAX_GAP]
                 [-e EXT]
                 [TARGET_DIR]

Generates concat files for merging. Creates separate videos for separate
broadcasts and incompatible resolutions, ignores very broken videos.

positional arguments:
  TARGET_DIR         Optional. defaults to the current working directory.

optional arguments:
  -h, --help         show this help message and exit
  --generate         generates concat files in TARGET_DIR, runs by default
  --merge            merges videos in TARGET_DIR according to existing concat
                     files
  --both             both generates concat files and merges videos
  --max-gap MAX_GAP  maximum gap between merged videos, in seconds. anything
                     larger is treated as a separate broadcast
  -e EXT             extension to merge, defaults to mp4

When merging, watch the output for "Non-monotonous DTS in output stream" -- A
few of these are harmless but a wall of them means that video is probably
corrupted.

"""

import os
import glob
import argparse
from subprocess import check_output, run, CalledProcessError
import json
from math import floor

# known resolutions:
# 352x198
# 640x360
# 704x396
# 1280x720 (a single kimi dare episode)
GOOD_HEIGHTS = (198, 360, 396, 720)

DEF_BITRATE = '300k'

# old version
"""
def create_concat_files(target_dir, target_ext):
    oldcwd = os.getcwd()
    os.chdir(target_dir)

    # TODO: use ffprobe to separate files with incompatible resolutions and those with a gap greater than ~10 minutes

    files = sorted(glob.glob('{}/*.{}'.format(target_dir, target_ext)))

    member_dict = {}
    for file in files:
        member_name = file.rsplit(' ', 1)[0]
        if member_name not in member_dict:
            member_dict[member_name] = []
        member_dict[member_name].append(file)

    concat_files = {}
    for key in member_dict.keys():
        filename = key +' ' + member_dict[key][0].rsplit(' ', 1)[1][:4] + '.mp4.concat'
        text = ""
        for item in member_dict[key]:
            text += "file '" + item + "'\n"
        concat_files.update({filename:text})

    for key in concat_files.keys():
        with open(key, 'w', encoding='utf8') as outfp:
            _ = outfp.write(concat_files[key])

    os.chdir(oldcwd)
"""

"""
{
    "member1" : [
        video1,
        video2,
        video3,
        video4
    ],
    "member2" : [

    ]
}

video: {
    "start_time" : parsed from file name; in seconds,
    "duration"   : parsed from ffprobe,
    "height"     : parsed from ffprobe,
    "valid"      : true or false (false for stuff with no video content),
    "file"       : location of file
}
"duration"
"height"

sample ffprobe output:
{
    "programs": [

    ],
    "streams": [
        {
            "height": 198,
            "duration": "499.654000"
        }
    ]
}
for member in members:

"""

def probe_file(filename, streams):
    try:
        results = check_output([
            "ffprobe",
            '-loglevel', '16',
            '-show_entries', 'stream={}'.format(','.join(streams)),
            '-select_streams', 'v',
            '-i', filename,
            '-of', 'json'
        ],
            universal_newlines=True
        )
    except CalledProcessError:
        return None
    else:
        return results

def resize_videos(target_dir, target_ext, copytb=1, target_bitrate='300k'):

    oldcwd = os.getcwd()
    os.chdir(target_dir)
    files = sorted(glob.glob('*.{}'.format(target_ext)))

    to_resize=[]
    for file in files:
        results = probe_file(file, streams=('duration', 'height'))
        if results:
            try:
                stream = json.loads(results)['streams'][0]
            except IndexError:
                continue
            if float(stream['duration']) >= 0.001 and int(stream['height']) == 198:
                to_resize.append(file)

    if len(to_resize) > 0:
        os.makedirs('resized', exist_ok=True)
    else:
        return
    codecs = {'mp4': 'libx264', 'webm': 'libvpx'}

    video_codec = codecs[target_ext]

    for file in to_resize:
        low_res_file = 'resized/' + file.replace('.' + target_ext, '_198p.' + target_ext)
        os.replace(file, low_res_file)
        run(['ffmpeg',
             '-copytb', str(copytb),
             '-i', low_res_file,
             '-c:v', video_codec,
             # '-maxrate', str(target_bitrate),
             # '-bufsize', BUFSIZE,
             # '-crf', '18',
             '-b:v', target_bitrate,
             '-vf', 'scale=-1:360',
             '-c:a', 'copy', file])

    os.chdir(oldcwd)

def generate_concat_files(target_dir, target_ext, max_gap):
    oldcwd = os.getcwd()
    os.chdir(target_dir)

    max_gap = float(max_gap)

    # TODO: deal with leftovers (from after 24:00)
    files = sorted(glob.glob('*.{}'.format(target_ext)))

    def get_start_seconds(file):
        time_str = file.rsplit(' ', 1)[1].split('.')[0]
        hours, minutes, seconds = int(time_str[:2]), int(time_str[2:4]), int(time_str[4:6])
        return float(hours*60*60 + minutes*60 + seconds)

    def get_start_hhmm(seconds):
        hours = seconds/(60*60)
        minutes = (hours - floor(hours))*60
        return '{:02d}{:02d}'.format(floor(hours), floor(minutes))

    member_dict = {}
    for file in files:
        results = probe_file(file, streams=('duration', 'height'))
        if not results:
            continue

        member_name = file.rsplit(' ', 1)[0]
        if member_name not in member_dict:
            member_dict[member_name] = []
        new_video = {"start_time": get_start_seconds(file)}
        try:
            stream = json.loads(results)['streams'][0]
        except IndexError:
            new_video['valid']  = False
        else:
            new_video['file']     = file
            new_video['duration'] = float(stream['duration'])
            new_video['height']   = int(stream['height'])
            if new_video['duration'] >= 0.001 and (new_video['height'] in GOOD_HEIGHTS):
                new_video['valid']  = True
            else:
                new_video['valid']  = False

        if new_video['valid']:
            member_dict[member_name].append(new_video)

    concat_files = {}

    def new_concat_file(member, first_video):
        filename = '{} {}.{}.concat'.format(member, get_start_hhmm(first_video['start_time']), target_ext)
        info = {'files': []}
        info['height'] = first_video['height']
        info['last_time'] = first_video['start_time'] + first_video['duration']
        info['files'].append(first_video['file'])
        return filename, info

    for member in member_dict.keys():
        """
        file_specifier (name + hhmm) : {
            height : 360 or 198,
            last_time : start_time + duration of most recently processed video
            files: [
                list of files
            ]
        }
        """
        try:
            filename, working = new_concat_file(member, member_dict[member][0])
        except IndexError:
            # no valid videos
            print('Failed to read videos for {}'.format(member))
            continue
        for item in member_dict[member][1:]:
            if (item['start_time'] >= working['last_time'] + max_gap
                    or item['height'] != working['height']):
                if filename in working:
                    # This needs to be dealt with by hand for now
                    print('Tried to add duplicate concat file name: {}'.format(filename))
                    raise FileExistsError
                concat_files[filename] = working
                filename, working = new_concat_file(member, item)
            else:
                if item['start_time'] < working['last_time'] - 5.0:
                    print('{} overlaps {}'.format(item['file'], working['files'][-1]))
                    # these have to be dealt with manually
                working['files'].append(item['file'])
                working['last_time'] = item['start_time'] + item['duration']
        concat_files[filename] = working

    for file in concat_files.keys():
        # skip singleton videos
        # if len(concat_files[file]['files']) == 1:
        #    continue
        text = ""
        for item in concat_files[file]['files']:
            text += "file '" + item + "'\n"
        with open(file, 'w', encoding='utf8') as outfp:
            outfp.write(text)

    os.chdir(oldcwd)

"""
#!/bin/bash

# for f in ./*.mp4; do echo "file '$f'" >> mylist.txt; done
# for f in ./*.concat; do echo "$( basename $f )"; done
# for f in ./*.concat; do g="\"$( basename "$f" .mp4)\""; echo $f; echo $g; done
# echo "\"$( basename ./160612\ Showroom\ -\ AKB48\ Team\ K\ Tano\ Yuka\ 124028.mp4 .mp4)\""
for f in ./*.concat; do
    g=$( basename "$f" .concat);
    #ffmpeg -copytb 1 -f concat -i "$f" -vf "pad=width=640:height=360:(ow-iw)/2:(oh-ih)/2:color=black" -movflags +faststart "$g";
    ffmpeg -copytb 1 -f concat -i "$f" -movflags +faststart -c copy "$g";
done
"""

def merge_videos(target_dir, output_dir, copytb=1):
    oldcwd = os.getcwd()
    os.chdir(target_dir)

    os.makedirs(output_dir, exist_ok=True)

    for concat_file in glob.glob('*.concat'):
        outfile = '{}/{}'.format(output_dir, os.path.splitext(concat_file)[0])

        instructions = ['-copytb', str(copytb)]

        # wish i didn't have to read it twice...
        with open(concat_file, encoding='utf8') as infp:
            data = infp.read()
        if data.count('file \'') == 1:
            src = data[5:].strip('\'\n./')
            instructions.extend(['-i', src])
        elif data.count('file \'') > 1:
            instructions.extend(['-f', 'concat', '-safe', '0', '-i', concat_file])
        else:
            print("Empty concat file")
            raise FileNotFoundError

        run(['ffmpeg',
            *instructions,
            '-movflags', '+faststart',
            '-c', 'copy', outfile])

    os.chdir(oldcwd)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description="Generates concat files for merging. Creates separate videos for separate broadcasts and \
        incompatible resolutions, ignores very broken videos.",
        epilog="When merging, watch the output for \"Non-monotonous DTS in output stream\" -- A few of these are \
        harmless but a wall of them means that video is probably corrupted.")
    parser.add_argument("--resize", action='store_true', help='resizes 198p videos in TARGET_DIR to 360p, '
                        'saves the old videos in a new "resized" subdirectory. Only supports h264 (MP4) and vpx (WEBM)')
    parser.add_argument("--generate", action='store_true', help='generates concat files in TARGET_DIR, runs by default')
    parser.add_argument("--merge", action='store_true',
                        help='merges videos in TARGET_DIR according to existing concat files')
    parser.add_argument("--both", action='store_true', help='both generates concat files and merges videos')
    parser.add_argument("--aggressive", action='store_true', help='resizes, generates, and merges')
    parser.add_argument("target_dir", nargs='?', default='.',
                        help='Optional. defaults to the current working directory.', metavar='TARGET_DIR')
    parser.add_argument("--max-gap", type=float, default=300.0,
                        help='maximum gap between merged videos, in seconds. anything larger is treated as a separate \
                        broadcast. default = 300.0')
    parser.add_argument("-e", dest='ext', default='mp4', help='extension to merge, defaults to mp4')
    parser.add_argument("--copytb", type=int, choices=[-1, 0, 1], default=1,
                        help='it may be useful to try setting this to 0 or -1 if a video has timing issues.'
                        'Defaults to %(default)s')
    parser.add_argument("--output-dir", "-o", dest='output_dir', type=str, default='.',
                        help='Optional, defaults to target directory. Note that relative paths will be relative to \
                        the target directory, not the current working directory', metavar='OUTPUT_DIR')
    parser.add_argument("--bitrate", "-b", type=str, default=DEF_BITRATE,
                        help='Bitrate for resizing. Defaults to %(default)s')
    args = parser.parse_args()

    if args.resize or args.aggressive:
        resize_videos(target_dir=args.target_dir, target_ext=args.ext,
                      copytb=args.copytb, target_bitrate=args.bitrate)
    if args.generate or args.both or args.aggressive:
        generate_concat_files(target_dir=args.target_dir, target_ext=args.ext,
                              max_gap=args.max_gap)
    if args.merge or args.both or args.aggressive:
        merge_videos(target_dir=args.target_dir, output_dir=args.output_dir,
                     copytb=args.copytb)

<|endoftext|>


<commit_before>#!/usr/bin/env python
'''
Concatenates video files using ffmpeg stream copy
'''
import sys
import subprocess
import os
import argparse
import time
import sipcreator
import ififuncs

def parse_args(args_):
    '''
    Parse command line arguments.
    '''
    parser = argparse.ArgumentParser(
        description='Concatenate video files using ffmpeg stream copy'
        ' Written by Kieran O\'Leary.'
    )
    parser.add_argument(
        '-i', nargs='+',
        help='full path of input directory', required=True
    )
    parser.add_argument(
        '-o', '-output',
        help='full path of output directory', required=True
    )
    parser.add_argument(
        '-r', '-recursive',
        help='recursively process all files in subdirectories. This could be potentially a disaster - so use with caution or with XDCAM', action='store_true'
    )
    parser.add_argument(
        '-s', '-sip',
        help='Run sipcreator.py on the resulting file.', action='store_true'
    )
    parsed_args = parser.parse_args(args_)
    return parsed_args

def ffmpeg_concat(concat_file, args, uuid):
    '''
    Launch the actual ffmpeg concatenation command
    '''
    cmd = [
        'ffmpeg', '-f', 'concat', '-safe', '0',
        '-i', concat_file,
        '-c', 'copy', '-map', '0:a?', '-map', '0:v',
        os.path.join(args.o, '%s.mkv' % uuid),
        '-f', 'md5','-map', '0:a?', '-map', '0:v',  '-'
    ]
    print cmd
    source_bitstream_md5 = subprocess.check_output(
        cmd
    )
    return source_bitstream_md5

def recursive_file_list(video_files):
    '''
    Recursively search through directories for AV files and add to list.
    '''
    # check if all inputs are actually directories
    recursive_list = []
    for directory in video_files:
        if not os.path.isdir(directory):
            print 'You have selected the recursive option, but not all of your inputs are directories.'
            sys.exit()
        else:
            for root, _, filenames in os.walk(directory):
                for filename in filenames:
                    if filename.endswith(('.MP4', '.mp4', '.mov', '.mkv', '.mxf')):
                        recursive_list.append(os.path.join(root, filename))
    print recursive_list
    return recursive_list

def main(args_):
    '''
    Launches the functions that prepare and execute the concatenation.
    '''
    uuid = ififuncs.create_uuid()
    args = parse_args(args_)

    log_name_source = os.path.join(args.o, '%s_concat_log.log' % time.strftime("_%Y_%m_%dT%H_%M_%S"))
    ififuncs.generate_log(log_name_source, 'concat.py started.')
    ififuncs.generate_log(
        log_name_source,
        'eventDetail=concat.py %s' % ififuncs.get_script_version('concat.py'))
    user = ififuncs.get_user()
    ififuncs.generate_log(
        log_name_source,
        'EVENT = agentName=%s' % user
    )
    source_uuid = ififuncs.get_source_uuid()
    ififuncs.generate_log(
        log_name_source,
        'Relationship, derivation, has source=%s' % source_uuid
    )
    video_files = args.i
    concat_file = ififuncs.get_temp_concat('concat_stuff')
    if args.r:
        video_files = recursive_file_list(video_files)
    video_files = ififuncs.sanitise_filenames(video_files)
    ififuncs.concat_textfile(video_files, concat_file)
    source_bitstream_md5 = ffmpeg_concat(concat_file, args, uuid)
    output_file = os.path.join(args.o, '%s.mkv' % uuid)
    output_bitstream_md5 = subprocess.check_output([
        'ffmpeg',
        '-i', output_file,
        '-f', 'md5','-map', '0:a?', '-map', '0:v',  '-'
    ])
    if source_bitstream_md5 == output_bitstream_md5:
        print 'process appears to be lossless'
        print source_bitstream_md5, output_bitstream_md5
    else:
        print 'something went wrong - not lossless!'
    print uuid
    with open(log_name_source, 'r') as concat_log:
        concat_lines = concat_log.readlines()
    if args.s:
        sipcreator_log = sipcreator.main(['-i', output_file, '-u', uuid, '-user', user, '-o', args.o])
        with open(sipcreator_log, 'r') as sipcreator_log_object:
            sipcreator_lines = sipcreator_log_object.readlines()
        with open(sipcreator_log, 'wb') as fo:
            for lines in concat_lines:
                fo.write(lines)
            for remaining_lines in sipcreator_lines:
                fo.write(remaining_lines)

if __name__ == '__main__':
    main(sys.argv[1:])
<commit_msg>concat - more reliable lossless checks<commit_after>#!/usr/bin/env python
'''
Concatenates video files using ffmpeg stream copy
'''
import sys
import subprocess
import os
import argparse
import time
import sipcreator
import ififuncs

def parse_args(args_):
    '''
    Parse command line arguments.
    '''
    parser = argparse.ArgumentParser(
        description='Concatenate video files using ffmpeg stream copy'
        ' Written by Kieran O\'Leary.'
    )
    parser.add_argument(
        '-i', nargs='+',
        help='full path of input directory', required=True
    )
    parser.add_argument(
        '-o', '-output',
        help='full path of output directory', required=True
    )
    parser.add_argument(
        '-r', '-recursive',
        help='recursively process all files in subdirectories. This could be potentially a disaster - so use with caution or with XDCAM', action='store_true'
    )
    parser.add_argument(
        '-s', '-sip',
        help='Run sipcreator.py on the resulting file.', action='store_true'
    )
    parsed_args = parser.parse_args(args_)
    return parsed_args

def ffmpeg_concat(concat_file, args, uuid):
    '''
    Launch the actual ffmpeg concatenation command
    '''
    cmd = [
        'ffmpeg', '-f', 'concat', '-safe', '0',
        '-i', concat_file,
        '-c', 'copy', '-map', '0:v', '-map', '0:a?',
        os.path.join(args.o, '%s.mkv' % uuid),
        '-f', 'md5', '-map', '0:v', '-map', '0:a?','-c', 'copy',  '-'
    ]
    print cmd
    source_bitstream_md5 = subprocess.check_output(
        cmd
    )
    return source_bitstream_md5.rstrip()

def recursive_file_list(video_files):
    '''
    Recursively search through directories for AV files and add to list.
    '''
    # check if all inputs are actually directories
    recursive_list = []
    for directory in video_files:
        if not os.path.isdir(directory):
            print 'You have selected the recursive option, but not all of your inputs are directories.'
            sys.exit()
        else:
            for root, _, filenames in os.walk(directory):
                for filename in filenames:
                    if filename.endswith(('.MP4', '.mp4', '.mov', '.mkv', '.mxf')):
                        recursive_list.append(os.path.join(root, filename))
    print recursive_list
    return recursive_list

def main(args_):
    '''
    Launches the functions that prepare and execute the concatenation.
    '''
    uuid = ififuncs.create_uuid()
    args = parse_args(args_)
    log_name_source = os.path.join(args.o, '%s_concat_log.log' % time.strftime("_%Y_%m_%dT%H_%M_%S"))
    ififuncs.generate_log(log_name_source, 'concat.py started.')
    ififuncs.generate_log(
        log_name_source,
        'eventDetail=concat.py %s' % ififuncs.get_script_version('concat.py'))
    user = ififuncs.get_user()
    ififuncs.generate_log(
        log_name_source,
        'EVENT = agentName=%s' % user
    )
    source_uuid = ififuncs.get_source_uuid()
    ififuncs.generate_log(
        log_name_source,
        'Relationship, derivation, has source=%s' % source_uuid
    )
    video_files = args.i
    concat_file = ififuncs.get_temp_concat('concat_stuff')
    if args.r:
        video_files = recursive_file_list(video_files)
    video_files = ififuncs.sanitise_filenames(video_files)
    ififuncs.concat_textfile(video_files, concat_file)
    source_bitstream_md5 = ffmpeg_concat(concat_file, args, uuid)
    output_file = os.path.join(args.o, '%s.mkv' % uuid)
    output_bitstream_md5 = subprocess.check_output([
        'ffmpeg',
        '-i', output_file,
        '-f', 'md5', '-map', '0:v', '-map', '0:a?', '-c', 'copy', '-'
    ]).rstrip()
    if source_bitstream_md5 == output_bitstream_md5:
        print 'process appears to be lossless'
        print source_bitstream_md5, output_bitstream_md5
    else:
        print 'something went wrong - not lossless!'
        print source_bitstream_md5,output_bitstream_md5
    print uuid
    with open(log_name_source, 'r') as concat_log:
        concat_lines = concat_log.readlines()
    if args.s:
        sipcreator_log = sipcreator.main(['-i', output_file, '-u', uuid, '-user', user, '-o', args.o])
        with open(sipcreator_log, 'r') as sipcreator_log_object:
            sipcreator_lines = sipcreator_log_object.readlines()
        with open(sipcreator_log, 'wb') as fo:
            for lines in concat_lines:
                fo.write(lines)
            for remaining_lines in sipcreator_lines:
                fo.write(remaining_lines)

if __name__ == '__main__':
    main(sys.argv[1:])
<|endoftext|>


<commit_before>r"""
variational formulation described at https://www.authorea.com/users/23640/articles/61529
Inspired from http://sfepy.org/doc-devel/examples/diffusion/poisson_field_dependent_material.html
"""

from sfepy import data_dir
from sfepy.base.base import output
import numpy as np

filename_mesh = data_dir + '/meshes/3d/cube_big_tetra.mesh'

# Constitutive relationships
class VanGenuchten(object):
    """
    Returns the water retention curve and hydraulic conductivity function
    according to van Genuchten (1980)'s functions
    """
    def __init__(self, thR, thS, aVG, nVG, mVG, ksat, psi, lVG = 0.5):
        self.aVG = aVG
        self.nVG = nVG
        self.mVG = mVG
        self.lVG = lVG
        self.ksat = ksat
        self.thR = thR
        self.thS = thS
        self.psi = psi

    def wrc(self):
        if np.all(self.psi > 0):
            self.th = self.thS
        elif np.all(self.psi <= 0):
            self.th = self.thR + (self.thS - self.thR) * (1+(self.aVG * self.psi) ** self.nVG) ** (-self.mVG)
        else:
            self.th = np.zeros(len(self.psi))
            self.th[self.psi <= 0] = self.thS
            self.th[self.psi > 0] = self.thR + (self.thS - self.thR) * (1+(self.aVG * self.psi[self.psi > 0]) ** self.nVG) ** (-self.mVG)
        return([self.psi, self.th])

    def hcf(self):
        if np.all(self.psi > 0):
            self.k = self.ksat
        elif np.all(self.psi <= 0):
            self.k = self.ksat*((1-((self.aVG*self.psi)**(self.nVG*self.mVG))* \
            ((1+((self.aVG*self.psi)**self.nVG))**(-self.mVG)))**2) / \
            ((1+((self.aVG*self.psi)**self.nVG))**(self.mVG*self.lVG))
        else:
            self.k = np.zeros(len(self.psi))
            self.k[self.psi > 0] = self.ksat
            self.k[self.psi <= 0] = self.ksat*((1-((self.aVG*self.psi[self.psi <= 0])**(self.nVG*self.mVG))*      ((1+((self.aVG*self.psi[self.psi <= 0])**self.nVG))**(-self.mVG)))**2) / \
            ((1+((self.aVG*self.psi[self.psi <= 0])**self.nVG))**(self.mVG*self.lVG))
        return([self.psi, self.k])

# Material parameters
cbl_thR = 0.045
cbl_thS = 0.430
cbl_aVG = 14.5
cbl_nVG = 2.68
cbl_mVG = 1-1/2.68
cbl_lVG = 0.5
cbl_ksat = 8.25e-5

def get_conductivity(ts, coors, problem, equations = None, mode = None, **kwargs):
    """
    Calculates the conductivity with VanGenuchten returns it.
    This relation results in larger h gradients where h is small.
    """
    if mode == 'qp':
        h_values = problem.evaluate('ev_volume_integrate.i.Omega(h)',
                                    mode = 'qp', verbose = False)
        val = VanGenuchten(thR = cbl_thR, thS = cbl_thS, aVG = cbl_aVG, nVG = cbl_nVG, mVG = 1-1/cbl_nVG,
                       lVG = cbl_lVG, ksat = cbl_ksat, psi = -h_values).hcf()[0]
        val.shape = (val.shape[0] * val.shape[1], 1, 1)
        return {'val' : val}

materials = {
    'coef' : 'get_conductivity',
    'flux' : ({'val' : -1E-6},),
}

fields = {
    'pressure' : ('real', 1, 'Omega', 1),
}

variables = {
    'h' : ('unknown field', 'pressure', 0),
    'v' : ('test field',    'pressure', 'h'),
}

regions = {
    'Omega' : 'all',
    'Gamma_Bottom' : ('vertices in (z < 0.00001)', 'facet'),
    'Gamma_Top' : ('vertices in (z > 0.49999)', 'facet'),
}

ebcs = {
    'P1' : ('Gamma_Bottom', {'h.0' : 0}),
}

functions = {
    'get_conductivity' : (get_conductivity,),
}

integrals = {
    'i' : 1,
}

equations = {
    'Pressure' : """dw_laplace.i.Omega(coef.val, v, h) - dw_surface_integrate.2.Gamma_Top(flux.val, v) = 0"""
}

solvers = {
    'ls' : ('ls.scipy_direct', {}),
    'newton' : ('nls.newton', {
        'i_max' : 1,
        'eps_a' : 1e-10,
        'eps_r' : 1.0,
    }),
}

options = {
    'nls' : 'newton',
    'ls' : 'ls',
}
<commit_msg>Adjusting flux to a realistic value<commit_after>r"""
variational formulation described at https://www.authorea.com/users/23640/articles/61529
Inspired from http://sfepy.org/doc-devel/examples/diffusion/poisson_field_dependent_material.html
"""

from sfepy import data_dir
from sfepy.base.base import output
import numpy as np

filename_mesh = data_dir + '/meshes/3d/cube_big_tetra.mesh'

# Constitutive relationships
class VanGenuchten(object):
    """
    Returns the water retention curve and hydraulic conductivity function
    according to van Genuchten (1980)'s functions
    """
    def __init__(self, thR, thS, aVG, nVG, mVG, ksat, psi, lVG = 0.5):
        self.aVG = aVG
        self.nVG = nVG
        self.mVG = mVG
        self.lVG = lVG
        self.ksat = ksat
        self.thR = thR
        self.thS = thS
        self.psi = psi

    def wrc(self):
        if np.all(self.psi > 0):
            self.th = self.thS
        elif np.all(self.psi <= 0):
            self.th = self.thR + (self.thS - self.thR) * (1+(self.aVG * self.psi) ** self.nVG) ** (-self.mVG)
        else:
            self.th = np.zeros(len(self.psi))
            self.th[self.psi <= 0] = self.thS
            self.th[self.psi > 0] = self.thR + (self.thS - self.thR) * (1+(self.aVG * self.psi[self.psi > 0]) ** self.nVG) ** (-self.mVG)
        return([self.psi, self.th])

    def hcf(self):
        if np.all(self.psi > 0):
            self.k = self.ksat
        elif np.all(self.psi <= 0):
            self.k = self.ksat*((1-((self.aVG*self.psi)**(self.nVG*self.mVG))* \
            ((1+((self.aVG*self.psi)**self.nVG))**(-self.mVG)))**2) / \
            ((1+((self.aVG*self.psi)**self.nVG))**(self.mVG*self.lVG))
        else:
            self.k = np.zeros(len(self.psi))
            self.k[self.psi > 0] = self.ksat
            self.k[self.psi <= 0] = self.ksat*((1-((self.aVG*self.psi[self.psi <= 0])**(self.nVG*self.mVG))*      ((1+((self.aVG*self.psi[self.psi <= 0])**self.nVG))**(-self.mVG)))**2) / \
            ((1+((self.aVG*self.psi[self.psi <= 0])**self.nVG))**(self.mVG*self.lVG))
        return([self.psi, self.k])

# Material parameters
cbl_thR = 0.045
cbl_thS = 0.430
cbl_aVG = 14.5
cbl_nVG = 2.68
cbl_mVG = 1-1/2.68
cbl_lVG = 0.5
cbl_ksat = 8.25e-5

def get_conductivity(ts, coors, problem, equations = None, mode = None, **kwargs):
    """
    Calculates the conductivity with VanGenuchten returns it.
    This relation results in larger h gradients where h is small.
    """
    if mode == 'qp':
        h_values = problem.evaluate('ev_volume_integrate.i.Omega(h)',
                                    mode = 'qp', verbose = False)
        val = VanGenuchten(thR = cbl_thR, thS = cbl_thS, aVG = cbl_aVG, nVG = cbl_nVG, mVG = 1-1/cbl_nVG,
                       lVG = cbl_lVG, ksat = cbl_ksat, psi = -h_values).hcf()[0]
        val.shape = (val.shape[0] * val.shape[1], 1, 1)
        return {'val' : val}

materials = {
    'coef' : 'get_conductivity',
    'flux' : ({'val' : 1E-7},),
}

fields = {
    'pressure' : ('real', 1, 'Omega', 1),
}

variables = {
    'h' : ('unknown field', 'pressure', 0),
    'v' : ('test field',    'pressure', 'h'),
}

regions = {
    'Omega' : 'all',
    'Gamma_Bottom' : ('vertices in (z < 0.00001)', 'facet'),
    'Gamma_Top' : ('vertices in (z > 0.49999)', 'facet'),
}

ebcs = {
    'P1' : ('Gamma_Bottom', {'h.0' : 0}),
}

functions = {
    'get_conductivity' : (get_conductivity,),
}

integrals = {
    'i' : 1,
}

equations = {
    'Pressure' : """dw_laplace.i.Omega(coef.val, v, h) - dw_surface_integrate.2.Gamma_Top(flux.val, v) = 0"""
}

solvers = {
    'ls' : ('ls.scipy_direct', {}),
    'newton' : ('nls.newton', {
        'i_max' : 1,
        'eps_a' : 1e-10,
        'eps_r' : 1.0,
    }),
}

options = {
    'nls' : 'newton',
    'ls' : 'ls',
}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from .rapid import RAPID
from .dataset import RAPIDDataset<commit_msg>added line at end of file<commit_after># -*- coding: utf-8 -*-
from .rapid import RAPID
from .dataset import RAPIDDataset
<|endoftext|>


<commit_before># Takes lists of objects returned by the zkclient module, and
# consolidates the information for display.

import logging
import simplejson as json
from datetime import datetime
from dateutil.relativedelta import relativedelta

class NullHandler(logging.Handler):
    def emit(self, record):
        pass

logger = logging.getLogger('kafka.codec').addHandler(NullHandler())

import socket
from collections import namedtuple
from kafka.client import KafkaClient
from kafka.common import OffsetRequest, FetchRequest

class ProcessorError(Exception):
    def __init__(self, msg):
        self.msg = msg

    def __str__(self):
        return self.msg

PartitionState = namedtuple('PartitionState',
    [
        'broker',           # Broker host
        'topic',            # Topic on broker
        'partition',        # The partition
        'earliest',         # Earliest offset within partition on broker
        'latest',           # Current offset within partition on broker
        'depth',            # Depth of partition on broker.
        'spout',            # The Spout consuming this partition
        'current',          # Current offset for Spout
        'delta',            # Difference between latest and current
        'timestamp',        # Current offset datetime
        'lag',              # Time lag between Storm and Kafka latest
    ])
PartitionsSummary = namedtuple('PartitionsSummary',
    [
        'total_depth',      # Total queue depth.
        'total_delta',      # Total delta across all spout tasks.
        'total_lag',        # Total lag between Storm and Kafka
        'num_partitions',   # Number of partitions.
        'num_brokers',      # Number of Kafka Brokers.
        'partitions'        # Tuple of PartitionStates
    ])

def get_timestamp(k, p, current):
    buffer_size = 1024
    responses = k.send_fetch_request([FetchRequest(p['topic'], p['partition'], current, buffer_size)])
    for resp in responses:
        for message in resp.messages:
            return json.loads(message.message.value)['s']

def now():
    return int((datetime.utcnow() - datetime(1970, 1, 1)).total_seconds() * 1000)

def max_relativedate(a, b):
    attrs = ['years', 'months', 'days', 'hours', 'minutes', 'seconds']
    for attr in attrs:
        if getattr(a, attr) > getattr(b, attr):
            return a
        if getattr(b, attr) > getattr(a, attr):
            return b
    return a

def process(spouts):
    '''
    Returns a named tuple of type PartitionsSummary.
    '''
    results = []
    total_depth = 0
    total_delta = 0
    total_lag = relativedelta()
    brokers = []
    seen_hosts, seen_topics, seen_partitions = set(), set(), set()
    for s in spouts:
        for p in s.partitions:
            if p['broker']['host'] in seen_hosts and p['topic'] in seen_topics and p['partition'] in seen_partitions:
                continue
            else:
                seen_hosts.add(p['broker']['host'])
                seen_topics.add(p['topic'])
                seen_partitions.add(p['partition'])

            try:
                k = KafkaClient('%s:%s' % (p['broker']['host'], p['broker']['port']))
            except socket.gaierror, e:
                raise ProcessorError('Failed to contact Kafka broker %s (%s)' %
                                     (p['broker']['host'], str(e)))
            earliest_off = OffsetRequest(p['topic'], p['partition'], -2, 1)
            latest_off = OffsetRequest(p['topic'], p['partition'], -1, 1)

            earliest = k.send_offset_request([earliest_off])[0].offsets[0]
            latest = k.send_offset_request([latest_off])[0].offsets[0]
            current = p['offset']

            brokers.append(p['broker']['host'])
            total_depth = total_depth + (latest - earliest)
            total_delta = total_delta + (latest - current)

            timestamp = get_timestamp(k, p, current) or 0
            lag = relativedelta(datetime.now(), datetime.fromtimestamp(timestamp/1000.0))
            total_lag = max_relativedate(total_lag, lag)

            results.append(PartitionState._make([
                p['broker']['host'],
                p['topic'],
                p['partition'],
                earliest,
                latest,
                latest - earliest,
                s.id,
                current,
                latest - current,
                timestamp,
                lag]))
    return PartitionsSummary(total_depth=total_depth,
                             total_delta=total_delta,
                             total_lag=total_lag,
                             num_partitions=len(results),
                             num_brokers=len(set(brokers)),
                             partitions=tuple(results))
<commit_msg>Don't show incorrect lag when we're all caught up<commit_after># Takes lists of objects returned by the zkclient module, and
# consolidates the information for display.

import logging
import simplejson as json
from datetime import datetime
from dateutil.relativedelta import relativedelta

class NullHandler(logging.Handler):
    def emit(self, record):
        pass

logger = logging.getLogger('kafka.codec').addHandler(NullHandler())

import socket
from collections import namedtuple
from kafka.client import KafkaClient
from kafka.common import OffsetRequest, FetchRequest

class ProcessorError(Exception):
    def __init__(self, msg):
        self.msg = msg

    def __str__(self):
        return self.msg

PartitionState = namedtuple('PartitionState',
    [
        'broker',           # Broker host
        'topic',            # Topic on broker
        'partition',        # The partition
        'earliest',         # Earliest offset within partition on broker
        'latest',           # Current offset within partition on broker
        'depth',            # Depth of partition on broker.
        'spout',            # The Spout consuming this partition
        'current',          # Current offset for Spout
        'delta',            # Difference between latest and current
        'timestamp',        # Current offset datetime
        'lag',              # Time lag between Storm and Kafka latest
    ])
PartitionsSummary = namedtuple('PartitionsSummary',
    [
        'total_depth',      # Total queue depth.
        'total_delta',      # Total delta across all spout tasks.
        'total_lag',        # Total lag between Storm and Kafka
        'num_partitions',   # Number of partitions.
        'num_brokers',      # Number of Kafka Brokers.
        'partitions'        # Tuple of PartitionStates
    ])

def get_timestamp(k, p, current):
    buffer_size = 1024
    responses = k.send_fetch_request([FetchRequest(p['topic'], p['partition'], current, buffer_size)])
    for resp in responses:
        for message in resp.messages:
            return json.loads(message.message.value)['s']

def now():
    return int((datetime.utcnow() - datetime(1970, 1, 1)).total_seconds() * 1000)

def max_relativedate(a, b):
    attrs = ['years', 'months', 'days', 'hours', 'minutes', 'seconds']
    for attr in attrs:
        if getattr(a, attr) > getattr(b, attr):
            return a
        if getattr(b, attr) > getattr(a, attr):
            return b
    return a

def process(spouts):
    '''
    Returns a named tuple of type PartitionsSummary.
    '''
    results = []
    total_depth = 0
    total_delta = 0
    total_lag = relativedelta()
    brokers = []
    seen_hosts, seen_topics, seen_partitions = set(), set(), set()
    for s in spouts:
        for p in s.partitions:
            if p['broker']['host'] in seen_hosts and p['topic'] in seen_topics and p['partition'] in seen_partitions:
                continue
            else:
                seen_hosts.add(p['broker']['host'])
                seen_topics.add(p['topic'])
                seen_partitions.add(p['partition'])

            try:
                k = KafkaClient('%s:%s' % (p['broker']['host'], p['broker']['port']))
            except socket.gaierror, e:
                raise ProcessorError('Failed to contact Kafka broker %s (%s)' %
                                     (p['broker']['host'], str(e)))
            earliest_off = OffsetRequest(p['topic'], p['partition'], -2, 1)
            latest_off = OffsetRequest(p['topic'], p['partition'], -1, 1)

            earliest = k.send_offset_request([earliest_off])[0].offsets[0]
            latest = k.send_offset_request([latest_off])[0].offsets[0]
            current = p['offset']

            brokers.append(p['broker']['host'])
            depth = (latest - earliest)
            delta = (latest - current)
            total_depth = total_depth + depth
            total_delta = total_delta + delta

            if delta == 0:
                timestamp = now()
                lag = relativedelta(0, 0)
            else:
                timestamp = get_timestamp(k, p, current)
                lag = relativedelta(datetime.now(), datetime.fromtimestamp(timestamp/1000.0))
            total_lag = max_relativedate(total_lag, lag)

            results.append(PartitionState._make([
                p['broker']['host'],
                p['topic'],
                p['partition'],
                earliest,
                latest,
                depth,
                s.id,
                current,
                delta,
                timestamp,
                lag]))
    return PartitionsSummary(total_depth=total_depth,
                             total_delta=total_delta,
                             total_lag=total_lag,
                             num_partitions=len(results),
                             num_brokers=len(set(brokers)),
                             partitions=tuple(results))
<|endoftext|>


<commit_before>from oslo_log import log as logging
from taskflow import exceptions as exc
import taskflow.engines
import flow_utils
from taskflow.engines.helpers import load_from_detail
import contextlib
from taskflow.persistence.backends import impl_sqlalchemy
from oslo_config import cfg
import sql_exec
LOG = logging.getLogger(__name__)

CONF = dict(connection=cfg.CONF.database.connection)

def check_and_run():
    backend = impl_sqlalchemy.SQLAlchemyBackend(CONF)
    if sql_exec.flowdetails[5] != 'SUCCESS':
        book_id = sql_exec.flowdetails[2]
        flow_id = sql_exec.flowdetails[6]

    if all([book_id,flow_id]):
        with contextlib.closing(backend.get_connection()) as conn:
        try:
            lb = conn.get_logbook(book_id)
            flow_detail = lb.find(flow_id)
        except exc.NotFound:
            pass

    flow_engine = taskflow.engines.load_from_detail(flow_detail,
                                                    backend=backend,
                                                    engine='serial')

    with flow_utils.DynamicLogListener(flow_engine, logger=LOG):
        flow_engine.run()
        LOG.info("The previously partially completed flow is finished.")

<commit_msg>Update check_and_run.py<commit_after>from oslo_log import log as logging
from taskflow import exceptions as exc
import taskflow.engines
import flow_utils
from taskflow.engines.helpers import load_from_detail
import contextlib
from taskflow.persistence.backends import impl_sqlalchemy
from oslo_config import cfg
import sql_exec
LOG = logging.getLogger(__name__)

CONF = dict(connection=cfg.CONF.database.connection)

def check_and_run():
    backend = impl_sqlalchemy.SQLAlchemyBackend(CONF)
    if sql_exec.flowdetails[5] != 'SUCCESS':
        book_id = sql_exec.flowdetails[2]
        flow_id = sql_exec.flowdetails[6]

    if all([book_id,flow_id]):
        with contextlib.closing(backend.get_connection()) as conn:
            try:
                lb = conn.get_logbook(book_id)
                flow_detail = lb.find(flow_id)
            except exc.NotFound:
                pass

    flow_engine = taskflow.engines.load_from_detail(flow_detail,
                                                    backend=backend,
                                                    engine='serial')

    with flow_utils.DynamicLogListener(flow_engine, logger=LOG):
        flow_engine.run()
        LOG.info("The previously partially completed flow is finished.")

<|endoftext|>


<commit_before># Copyright (c) 2015 Ultimaker B.V.
# Cura is released under the terms of the AGPLv3 or higher.

import numpy
from string import Formatter
from enum import IntEnum

from UM.Job import Job
from UM.Application import Application
from UM.Logger import Logger

from UM.Scene.SceneNode import SceneNode
from UM.Scene.Iterator.DepthFirstIterator import DepthFirstIterator

from UM.Settings.Validator import ValidatorState
from UM.Settings.SettingRelation import RelationType

from cura.OneAtATimeIterator import OneAtATimeIterator

import cura.Settings

class StartJobResult(IntEnum):
    Finished = 1
    Error = 2
    SettingError = 3
    NothingToSlice = 4
    MaterialIncompatible = 5
    BuildPlateError = 6

##  Formatter class that handles token expansion in start/end gcod
class GcodeStartEndFormatter(Formatter):
    def get_value(self, key, args, kwargs):  # [CodeStyle: get_value is an overridden function from the Formatter class]
        if isinstance(key, str):
            try:
                return kwargs[key]
            except KeyError:
                Logger.log("w", "Unable to replace '%s' placeholder in start/end gcode", key)
                return "{" + key + "}"
        else:
            Logger.log("w", "Incorrectly formatted placeholder '%s' in start/end gcode", key)
            return "{" + str(key) + "}"

##  Job class that builds up the message of scene data to send to CuraEngine.
class StartSliceJob(Job):
    def __init__(self, slice_message):
        super().__init__()

        self._scene = Application.getInstance().getController().getScene()
        self._slice_message = slice_message
        self._is_cancelled = False

    def getSliceMessage(self):
        return self._slice_message

    ##  Check if a stack has any errors.
    ##  returns true if it has errors, false otherwise.
    def _checkStackForErrors(self, stack):
        if stack is None:
            return False

        for key in stack.getAllKeys():
            validation_state = stack.getProperty(key, "validationState")
            if validation_state in (ValidatorState.Exception, ValidatorState.MaximumError, ValidatorState.MinimumError):
                Logger.log("w", "Setting %s is not valid, but %s. Aborting slicing.", key, validation_state)
                return True
            Job.yieldThread()
        return False

    ##  Runs the job that initiates the slicing.
    def run(self):
        stack = Application.getInstance().getGlobalContainerStack()
        if not stack:
            self.setResult(StartJobResult.Error)
            return

        # Don't slice if there is a setting with an error value.
        if Application.getInstance().getMachineManager().stacksHaveErrors:
            self.setResult(StartJobResult.SettingError)
            return

        if Application.getInstance().getBuildVolume().hasErrors():
            self.setResult(StartJobResult.BuildPlateError)
            return

        for extruder_stack in cura.Settings.ExtruderManager.getInstance().getMachineExtruders(stack.getId()):
            material = extruder_stack.findContainer({"type": "material"})
            if material:
                if material.getMetaDataEntry("compatible") == False:
                    self.setResult(StartJobResult.MaterialIncompatible)
                    return

        # Don't slice if there is a per object setting with an error value.
        for node in DepthFirstIterator(self._scene.getRoot()):
            if type(node) is not SceneNode or not node.isSelectable():
                continue

            if self._checkStackForErrors(node.callDecoration("getStack")):
                self.setResult(StartJobResult.SettingError)
                return

        with self._scene.getSceneLock():
            # Remove old layer data.
            for node in DepthFirstIterator(self._scene.getRoot()):
                if node.callDecoration("getLayerData"):
                    node.getParent().removeChild(node)
                    break

            # Get the objects in their groups to print.
            object_groups = []
            if stack.getProperty("print_sequence", "value") == "one_at_a_time":
                for node in OneAtATimeIterator(self._scene.getRoot()):
                    temp_list = []

                    # Node can't be printed, so don't bother sending it.
                    if getattr(node, "_outside_buildarea", False):
                        continue

                    children = node.getAllChildren()
                    children.append(node)
                    for child_node in children:
                        if type(child_node) is SceneNode and child_node.getMeshData() and child_node.getMeshData().getVertices() is not None:
                            temp_list.append(child_node)

                    if temp_list:
                        object_groups.append(temp_list)
                    Job.yieldThread()
                if len(object_groups) == 0:
                    Logger.log("w", "No objects suitable for one at a time found, or no correct order found")
            else:
                temp_list = []
                for node in DepthFirstIterator(self._scene.getRoot()):
                    if type(node) is SceneNode and node.getMeshData() and node.getMeshData().getVertices() is not None:
                        if not getattr(node, "_outside_buildarea", False):
                            temp_list.append(node)
                    Job.yieldThread()

                if temp_list:
                    object_groups.append(temp_list)

            # There are cases when there is nothing to slice. This can happen due to one at a time slicing not being
            # able to find a possible sequence or because there are no objects on the build plate (or they are outside
            # the build volume)
            if not object_groups:
                self.setResult(StartJobResult.NothingToSlice)
                return

            self._buildGlobalSettingsMessage(stack)
            self._buildGlobalInheritsStackMessage(stack)

            for extruder_stack in cura.Settings.ExtruderManager.getInstance().getMachineExtruders(stack.getId()):
                self._buildExtruderMessage(extruder_stack)

            for group in object_groups:
                group_message = self._slice_message.addRepeatedMessage("object_lists")
                if group[0].getParent().callDecoration("isGroup"):
                    self._handlePerObjectSettings(group[0].getParent(), group_message)
                for object in group:
                    mesh_data = object.getMeshData().getTransformed(object.getWorldTransformation())

                    obj = group_message.addRepeatedMessage("objects")
                    obj.id = id(object)
                    verts = mesh_data.getVertices()
                    indices = mesh_data.getIndices()

                    if indices is not None:
                        flat_verts = numpy.take(verts, indices.flatten(), axis=0)
                    else:
                        flat_verts = numpy.array(verts)

                    # Convert from Y up axes to Z up axes. Equals a 90 degree rotation.
                    flat_verts[:, [1, 2]] = flat_verts[:, [2, 1]]
                    flat_verts[:, 1] *= -1

                    obj.vertices = flat_verts

                    self._handlePerObjectSettings(object, obj)

                    Job.yieldThread()

        self.setResult(StartJobResult.Finished)

    def cancel(self):
        super().cancel()
        self._is_cancelled = True

    def isCancelled(self):
        return self._is_cancelled

    def _expandGcodeTokens(self, key, value, settings):
        try:
            # any setting can be used as a token
            fmt = GcodeStartEndFormatter()
            return str(fmt.format(value, **settings)).encode("utf-8")
        except:
            Logger.logException("w", "Unable to do token replacement on start/end gcode")
            return str(value).encode("utf-8")

    ##  Create extruder message from stack
    def _buildExtruderMessage(self, stack):
        message = self._slice_message.addRepeatedMessage("extruders")
        message.id = int(stack.getMetaDataEntry("position"))

        material_instance_container = stack.findContainer({"type": "material"})

        for key in stack.getAllKeys():
            # Do not send settings that are not settable_per_extruder.
            if stack.getProperty(key, "settable_per_extruder") == False:
                continue
            setting = message.getMessage("settings").addRepeatedMessage("settings")
            setting.name = key
            if key == "material_guid" and material_instance_container:
                # Also send the material GUID. This is a setting in fdmprinter, but we have no interface for it.
                setting.value = str(material_instance_container.getMetaDataEntry("GUID", "")).encode("utf-8")
            else:
                setting.value = str(stack.getProperty(key, "value")).encode("utf-8")
            Job.yieldThread()

    ##  Sends all global settings to the engine.
    #
    #   The settings are taken from the global stack. This does not include any
    #   per-extruder settings or per-object settings.
    def _buildGlobalSettingsMessage(self, stack):
        keys = stack.getAllKeys()
        settings = {}
        for key in keys:
            # Use resolvement value if available, or take the value
            resolved_value = stack.getProperty(key, "resolve")
            if resolved_value is not None:
                # There is a resolvement value. Check if we need to use it.
                user_container = stack.findContainer({"type": "user"})
                quality_changes_container = stack.findContainer({"type": "quality_changes"})
                if user_container.hasProperty(key,"value") or quality_changes_container.hasProperty(key,"value"):
                    # Normal case
                    settings[key] = stack.getProperty(key, "value")
                else:
                    settings[key] = resolved_value
            else:
                # Normal case
                settings[key] = stack.getProperty(key, "value")
            Job.yieldThread()

        start_gcode = settings["machine_start_gcode"]
        settings["material_bed_temp_prepend"] = "{material_bed_temperature}" not in start_gcode #Pre-compute material material_bed_temp_prepend and material_print_temp_prepend
        settings["material_print_temp_prepend"] = "{material_print_temperature}" not in start_gcode

        for key, value in settings.items(): #Add all submessages for each individual setting.
            setting_message = self._slice_message.getMessage("global_settings").addRepeatedMessage("settings")
            setting_message.name = key
            if key == "machine_start_gcode" or key == "machine_end_gcode": #If it's a g-code message, use special formatting.
                setting_message.value = self._expandGcodeTokens(key, value, settings)
            else:
                setting_message.value = str(value).encode("utf-8")
            Job.yieldThread()

    ##  Sends for some settings which extruder they should fallback to if not
    #   set.
    #
    #   This is only set for settings that have the limit_to_extruder
    #   property.
    #
    #   \param stack The global stack with all settings, from which to read the
    #   limit_to_extruder property.
    def _buildGlobalInheritsStackMessage(self, stack):
        for key in stack.getAllKeys():
            extruder = int(round(float(stack.getProperty(key, "limit_to_extruder"))))
            if extruder >= 0: #Set to a specific extruder.
                setting_extruder = self._slice_message.addRepeatedMessage("limit_to_extruder")
                setting_extruder.name = key
                setting_extruder.extruder = extruder
            Job.yieldThread()

    ##  Check if a node has per object settings and ensure that they are set correctly in the message
    #   \param node \type{SceneNode} Node to check.
    #   \param message object_lists message to put the per object settings in
    def _handlePerObjectSettings(self, node, message):
        stack = node.callDecoration("getStack")
        # Check if the node has a stack attached to it and the stack has any settings in the top container.
        if stack:
            # Check all settings for relations, so we can also calculate the correct values for dependant settings.
            changed_setting_keys = set(stack.getTop().getAllKeys())
            for key in stack.getTop().getAllKeys():
                instance = stack.getTop().getInstance(key)
                self._addRelations(changed_setting_keys, instance.definition.relations)
                Job.yieldThread()

            # Ensure that the engine is aware what the build extruder is
            if stack.getProperty("machine_extruder_count", "value") > 1:
                changed_setting_keys.add("extruder_nr")

            # Get values for all changed settings
            for key in changed_setting_keys:
                setting = message.addRepeatedMessage("settings")
                setting.name = key
                setting.value = str(stack.getProperty(key, "value")).encode("utf-8")
                Job.yieldThread()

    ##  Recursive function to put all settings that require eachother for value changes in a list
    #   \param relations_set \type{set} Set of keys (strings) of settings that are influenced
    #   \param relations list of relation objects that need to be checked.
    def _addRelations(self, relations_set, relations):
        for relation in filter(lambda r: r.role == "value", relations):
            if relation.type == RelationType.RequiresTarget:
                continue

            relations_set.add(relation.target.key)
            self._addRelations(relations_set, relation.target.relations)<commit_msg>Reduce time spent in "Preparing to slice" by optimizing vertex data sending<commit_after># Copyright (c) 2015 Ultimaker B.V.
# Cura is released under the terms of the AGPLv3 or higher.

import numpy
from string import Formatter
from enum import IntEnum

from UM.Job import Job
from UM.Application import Application
from UM.Logger import Logger

from UM.Scene.SceneNode import SceneNode
from UM.Scene.Iterator.DepthFirstIterator import DepthFirstIterator

from UM.Settings.Validator import ValidatorState
from UM.Settings.SettingRelation import RelationType

from cura.OneAtATimeIterator import OneAtATimeIterator

import cura.Settings

class StartJobResult(IntEnum):
    Finished = 1
    Error = 2
    SettingError = 3
    NothingToSlice = 4
    MaterialIncompatible = 5
    BuildPlateError = 6

##  Formatter class that handles token expansion in start/end gcod
class GcodeStartEndFormatter(Formatter):
    def get_value(self, key, args, kwargs):  # [CodeStyle: get_value is an overridden function from the Formatter class]
        if isinstance(key, str):
            try:
                return kwargs[key]
            except KeyError:
                Logger.log("w", "Unable to replace '%s' placeholder in start/end gcode", key)
                return "{" + key + "}"
        else:
            Logger.log("w", "Incorrectly formatted placeholder '%s' in start/end gcode", key)
            return "{" + str(key) + "}"

##  Job class that builds up the message of scene data to send to CuraEngine.
class StartSliceJob(Job):
    def __init__(self, slice_message):
        super().__init__()

        self._scene = Application.getInstance().getController().getScene()
        self._slice_message = slice_message
        self._is_cancelled = False

    def getSliceMessage(self):
        return self._slice_message

    ##  Check if a stack has any errors.
    ##  returns true if it has errors, false otherwise.
    def _checkStackForErrors(self, stack):
        if stack is None:
            return False

        for key in stack.getAllKeys():
            validation_state = stack.getProperty(key, "validationState")
            if validation_state in (ValidatorState.Exception, ValidatorState.MaximumError, ValidatorState.MinimumError):
                Logger.log("w", "Setting %s is not valid, but %s. Aborting slicing.", key, validation_state)
                return True
            Job.yieldThread()
        return False

    ##  Runs the job that initiates the slicing.
    def run(self):
        stack = Application.getInstance().getGlobalContainerStack()
        if not stack:
            self.setResult(StartJobResult.Error)
            return

        # Don't slice if there is a setting with an error value.
        if Application.getInstance().getMachineManager().stacksHaveErrors:
            self.setResult(StartJobResult.SettingError)
            return

        if Application.getInstance().getBuildVolume().hasErrors():
            self.setResult(StartJobResult.BuildPlateError)
            return

        for extruder_stack in cura.Settings.ExtruderManager.getInstance().getMachineExtruders(stack.getId()):
            material = extruder_stack.findContainer({"type": "material"})
            if material:
                if material.getMetaDataEntry("compatible") == False:
                    self.setResult(StartJobResult.MaterialIncompatible)
                    return

        # Don't slice if there is a per object setting with an error value.
        for node in DepthFirstIterator(self._scene.getRoot()):
            if type(node) is not SceneNode or not node.isSelectable():
                continue

            if self._checkStackForErrors(node.callDecoration("getStack")):
                self.setResult(StartJobResult.SettingError)
                return

        with self._scene.getSceneLock():
            # Remove old layer data.
            for node in DepthFirstIterator(self._scene.getRoot()):
                if node.callDecoration("getLayerData"):
                    node.getParent().removeChild(node)
                    break

            # Get the objects in their groups to print.
            object_groups = []
            if stack.getProperty("print_sequence", "value") == "one_at_a_time":
                for node in OneAtATimeIterator(self._scene.getRoot()):
                    temp_list = []

                    # Node can't be printed, so don't bother sending it.
                    if getattr(node, "_outside_buildarea", False):
                        continue

                    children = node.getAllChildren()
                    children.append(node)
                    for child_node in children:
                        if type(child_node) is SceneNode and child_node.getMeshData() and child_node.getMeshData().getVertices() is not None:
                            temp_list.append(child_node)

                    if temp_list:
                        object_groups.append(temp_list)
                    Job.yieldThread()
                if len(object_groups) == 0:
                    Logger.log("w", "No objects suitable for one at a time found, or no correct order found")
            else:
                temp_list = []
                for node in DepthFirstIterator(self._scene.getRoot()):
                    if type(node) is SceneNode and node.getMeshData() and node.getMeshData().getVertices() is not None:
                        if not getattr(node, "_outside_buildarea", False):
                            temp_list.append(node)
                    Job.yieldThread()

                if temp_list:
                    object_groups.append(temp_list)

            # There are cases when there is nothing to slice. This can happen due to one at a time slicing not being
            # able to find a possible sequence or because there are no objects on the build plate (or they are outside
            # the build volume)
            if not object_groups:
                self.setResult(StartJobResult.NothingToSlice)
                return

            self._buildGlobalSettingsMessage(stack)
            self._buildGlobalInheritsStackMessage(stack)

            for extruder_stack in cura.Settings.ExtruderManager.getInstance().getMachineExtruders(stack.getId()):
                self._buildExtruderMessage(extruder_stack)

            for group in object_groups:
                group_message = self._slice_message.addRepeatedMessage("object_lists")
                if group[0].getParent().callDecoration("isGroup"):
                    self._handlePerObjectSettings(group[0].getParent(), group_message)
                for object in group:
                    mesh_data = object.getMeshData()
                    rot_scale = object.getWorldTransformation().getTransposed().getData()[0:3, 0:3]
                    translate = object.getWorldTransformation().getData()[:3, 3]

                    # This effectively performs a limited form of MeshData.getTransformed that ignores normals.
                    verts = mesh_data.getVertices()
                    verts = verts.dot(rot_scale)
                    verts += translate

                    # Convert from Y up axes to Z up axes. Equals a 90 degree rotation.
                    verts[:, [1, 2]] = verts[:, [2, 1]]
                    verts[:, 1] *= -1

                    obj = group_message.addRepeatedMessage("objects")
                    obj.id = id(object)

                    indices = mesh_data.getIndices()
                    if indices is not None:
                        flat_verts = numpy.take(verts, indices.flatten(), axis=0)
                    else:
                        flat_verts = numpy.array(verts)

                    obj.vertices = flat_verts

                    self._handlePerObjectSettings(object, obj)

                    Job.yieldThread()

        self.setResult(StartJobResult.Finished)

    def cancel(self):
        super().cancel()
        self._is_cancelled = True

    def isCancelled(self):
        return self._is_cancelled

    def _expandGcodeTokens(self, key, value, settings):
        try:
            # any setting can be used as a token
            fmt = GcodeStartEndFormatter()
            return str(fmt.format(value, **settings)).encode("utf-8")
        except:
            Logger.logException("w", "Unable to do token replacement on start/end gcode")
            return str(value).encode("utf-8")

    ##  Create extruder message from stack
    def _buildExtruderMessage(self, stack):
        message = self._slice_message.addRepeatedMessage("extruders")
        message.id = int(stack.getMetaDataEntry("position"))

        material_instance_container = stack.findContainer({"type": "material"})

        for key in stack.getAllKeys():
            # Do not send settings that are not settable_per_extruder.
            if stack.getProperty(key, "settable_per_extruder") == False:
                continue
            setting = message.getMessage("settings").addRepeatedMessage("settings")
            setting.name = key
            if key == "material_guid" and material_instance_container:
                # Also send the material GUID. This is a setting in fdmprinter, but we have no interface for it.
                setting.value = str(material_instance_container.getMetaDataEntry("GUID", "")).encode("utf-8")
            else:
                setting.value = str(stack.getProperty(key, "value")).encode("utf-8")
            Job.yieldThread()

    ##  Sends all global settings to the engine.
    #
    #   The settings are taken from the global stack. This does not include any
    #   per-extruder settings or per-object settings.
    def _buildGlobalSettingsMessage(self, stack):
        keys = stack.getAllKeys()
        settings = {}
        for key in keys:
            # Use resolvement value if available, or take the value
            resolved_value = stack.getProperty(key, "resolve")
            if resolved_value is not None:
                # There is a resolvement value. Check if we need to use it.
                user_container = stack.findContainer({"type": "user"})
                quality_changes_container = stack.findContainer({"type": "quality_changes"})
                if user_container.hasProperty(key,"value") or quality_changes_container.hasProperty(key,"value"):
                    # Normal case
                    settings[key] = stack.getProperty(key, "value")
                else:
                    settings[key] = resolved_value
            else:
                # Normal case
                settings[key] = stack.getProperty(key, "value")
            Job.yieldThread()

        start_gcode = settings["machine_start_gcode"]
        settings["material_bed_temp_prepend"] = "{material_bed_temperature}" not in start_gcode #Pre-compute material material_bed_temp_prepend and material_print_temp_prepend
        settings["material_print_temp_prepend"] = "{material_print_temperature}" not in start_gcode

        for key, value in settings.items(): #Add all submessages for each individual setting.
            setting_message = self._slice_message.getMessage("global_settings").addRepeatedMessage("settings")
            setting_message.name = key
            if key == "machine_start_gcode" or key == "machine_end_gcode": #If it's a g-code message, use special formatting.
                setting_message.value = self._expandGcodeTokens(key, value, settings)
            else:
                setting_message.value = str(value).encode("utf-8")
            Job.yieldThread()

    ##  Sends for some settings which extruder they should fallback to if not
    #   set.
    #
    #   This is only set for settings that have the limit_to_extruder
    #   property.
    #
    #   \param stack The global stack with all settings, from which to read the
    #   limit_to_extruder property.
    def _buildGlobalInheritsStackMessage(self, stack):
        for key in stack.getAllKeys():
            extruder = int(round(float(stack.getProperty(key, "limit_to_extruder"))))
            if extruder >= 0: #Set to a specific extruder.
                setting_extruder = self._slice_message.addRepeatedMessage("limit_to_extruder")
                setting_extruder.name = key
                setting_extruder.extruder = extruder
            Job.yieldThread()

    ##  Check if a node has per object settings and ensure that they are set correctly in the message
    #   \param node \type{SceneNode} Node to check.
    #   \param message object_lists message to put the per object settings in
    def _handlePerObjectSettings(self, node, message):
        stack = node.callDecoration("getStack")
        # Check if the node has a stack attached to it and the stack has any settings in the top container.
        if stack:
            # Check all settings for relations, so we can also calculate the correct values for dependant settings.
            changed_setting_keys = set(stack.getTop().getAllKeys())
            for key in stack.getTop().getAllKeys():
                instance = stack.getTop().getInstance(key)
                self._addRelations(changed_setting_keys, instance.definition.relations)
                Job.yieldThread()

            # Ensure that the engine is aware what the build extruder is
            if stack.getProperty("machine_extruder_count", "value") > 1:
                changed_setting_keys.add("extruder_nr")

            # Get values for all changed settings
            for key in changed_setting_keys:
                setting = message.addRepeatedMessage("settings")
                setting.name = key
                setting.value = str(stack.getProperty(key, "value")).encode("utf-8")
                Job.yieldThread()

    ##  Recursive function to put all settings that require eachother for value changes in a list
    #   \param relations_set \type{set} Set of keys (strings) of settings that are influenced
    #   \param relations list of relation objects that need to be checked.
    def _addRelations(self, relations_set, relations):
        for relation in filter(lambda r: r.role == "value", relations):
            if relation.type == RelationType.RequiresTarget:
                continue

            relations_set.add(relation.target.key)
            self._addRelations(relations_set, relation.target.relations)
<|endoftext|>


<commit_before>import pytest
from model.contact import Contact
import random
import string

def random_string(prefix, maxlen):
    symbols = string.ascii_letters + string.digits + " "*10
    return prefix + "".join([random.choice(symbols) for i in range(random.randrange(maxlen))])

def random_phone_numbers():
    return random.randint(100000000000, 999999999999)

def random_fax():
    return random.randint(100000, 999999)

def random_year():
    return random.choice(range(1950, 2001))

#####################
# generating emails
#####################

domains = ["hotmail.com", "gmail.com", "aol.com", "mail.com", "mail.kz", "yahoo.com"]
letters = string.ascii_lowercase[:12]

def get_random_domain(domains):
    return random.choice(domains)

def get_random_name(letters, length):
    return ''.join(random.choice(letters) for i in range(length))

def random_emails(length):
    return get_random_name(letters, length) + '@' + get_random_domain(domains)

testdata = [Contact(firstname="", middlename="", lastname="", nickname="",
                                   title="", company="", address="",
                                   home="", mobile="", work="", fax="",
                                   email="", email2="", email3="", byear="",
                                   address2="", phone2="", notes="")] + [

    Contact(firstname=random_string("firstname", 10), middlename=random_string("middlename", 10), lastname=random_string("lastname", 20), nickname=random_string("nickname", 10),
            title=random_string("title", 20), company=random_string("company", 20), address=random_string("address", 10),
            home=random_phone_numbers(), mobile=random_phone_numbers(), work=random_phone_numbers(), fax=random_fax(),
            email=random_emails(7), email2=random_emails(7), email3=random_emails(7), byear=random_year(),
            address2=random_string("address2: ", 10), phone2=random_phone_numbers(), notes=random_string("notes: ", 10))
    for i in range(5)
]

@pytest.mark.parametrize("contact", testdata, ids=[repr(x) for x in testdata])
def test_add_contact(app, contact):
    old_contacts = app.contact.get_contact_list()
    app.contact.create(contact)
    new_contacts = app.contact.get_contact_list()
    assert len(old_contacts) + 1 == app.contact.count()
    old_contacts.append(contact)
    assert sorted(old_contacts, key=Contact.id_or_max) == sorted(new_contacts, key=Contact.id_or_max)
<commit_msg>Lesson 6. Exercise no.19 Loading test data from files<commit_after>from model.contact import Contact

def test_add_contact(app, json_contacts):
    contact = json_contacts
    old_contacts = app.contact.get_contact_list()
    app.contact.create(contact)
    new_contacts = app.contact.get_contact_list()
    assert len(old_contacts) + 1 == app.contact.count()
    old_contacts.append(contact)
    assert sorted(old_contacts, key=Contact.id_or_max) == sorted(new_contacts, key=Contact.id_or_max)
<|endoftext|>


<commit_before>#
# Copyright (C) 2012, 2013 Satoru SATOH <ssato @ redhat.com>
# License: MIT
#
import anyconfig.globals as G
import anyconfig.mergeabledict as M
import anyconfig.backend.backends as Backends
import anyconfig.backend.json_ as BJ
import anyconfig.parser as P
import anyconfig.utils as U

# pylint: disable=W0611
# Import some global constants will be re-exported:
from anyconfig.mergeabledict import MS_REPLACE, MS_NO_REPLACE, \
    MS_DICTS, MS_DICTS_AND_LISTS, MERGE_STRATEGIES
# pylint: enable=W0611

# pylint: disable=C0103
# Re-export:
list_types = Backends.list_types
getLogger = G.getLogger

# aliases:
container = M.MergeableDict
logging = G.LOGGER
# pylint: enable=W0611

def set_loglevel(level):
    """
    :param level: Log level, e.g. logging.INFO and logging.WARN.
    """
    logging.setLevel(level)

def find_loader(config_path, forced_type=None):
    """
    :param config_path: Configuration file path
    :param forced_type: Forced configuration parser type
    :return: ConfigParser-inherited class object
    """
    if forced_type is not None:
        cparser = Backends.find_by_type(forced_type)
        if not cparser:
            logging.error(
                "No parser found for given type: " + forced_type
            )
            return None
    else:
        cparser = Backends.find_by_file(config_path)
        if not cparser:
            logging.error(
                "No parser found for given file: " + config_path
            )
            return None

    logging.debug("Using config parser of type: " + cparser.type())
    return cparser

def single_load(config_path, forced_type=None, **kwargs):
    """
    Load single config file.

    :param config_path: Configuration file path
    :param forced_type: Forced configuration parser type
    :param kwargs: Backend specific optional arguments, e.g. {"indent": 2} for
        JSON loader/dumper backend
    :return: Dict-like object (instance of
        anyconfig.mergeabledict.MergeableDict by default) supports merge
        operations.
    """
    cparser = find_loader(config_path, forced_type)
    if cparser is None:
        return None

    logging.info("Loading: " + config_path)
    return cparser.load(config_path, **kwargs)

def multi_load(paths, forced_type=None, merge=MS_DICTS, marker='*', **kwargs):
    """
    Load multiple config files.

    The first argument `paths` may be a list of config file paths or
    a glob pattern specifying that. That is, if a.yml, b.yml and c.yml are in
    the dir /etc/foo/conf.d/, the followings give same results::

      multi_load(["/etc/foo/conf.d/a.yml", "/etc/foo/conf.d/b.yml",
                  "/etc/foo/conf.d/c.yml", ])

      multi_load("/etc/foo/conf.d/*.yml")

    :param paths: List of config file paths or a glob pattern to list paths
    :param forced_type: Forced configuration parser type
    :param merge: Strategy to merge config results of multiple config files
        loaded. see also: anyconfig.mergeabledict.MergeableDict.update()
    :param marker: Globbing markerer to detect paths patterns
    :param kwargs: Backend specific optional arguments, e.g. {"indent": 2} for
        JSON loader/dumper backend
    :return: Dict-like object (instance of
        anyconfig.mergeabledict.MergeableDict by default) supports merge
        operations.
    """
    assert merge in MERGE_STRATEGIES, "Invalid merge strategy: " + merge

    if marker in paths:
        paths = U.sglob(paths)

    config = container()
    for path in paths:
        if marker in path:  # Nested patterns like ['*.yml', '/a/b/c.yml'].
            conf_updates = multi_load(path, forced_type, merge, marker,
                                      **kwargs)
        else:
            conf_updates = single_load(path, forced_type, **kwargs)

        config.update(conf_updates, merge)

    return config

def load(path_specs, forced_type=None, merge=MS_DICTS, marker='*', **kwargs):
    """
    Load single or multiple config files or multiple config files specified in
    given paths pattern.

    :param path_specs:
        Configuration file path or paths or its pattern such as '/a/b/*.json'
    :param forced_type: Forced configuration parser type
    :param merge: Merging strategy to use
    :param marker: Globbing marker to detect paths patterns
    :param kwargs: Backend specific optional arguments, e.g. {"indent": 2} for
        JSON loader/dumper backend
    :return: Dict-like object (instance of
        anyconfig.mergeabledict.MergeableDict by default) supports merge
        operations.
    """
    if marker in path_specs or U.is_iterable(path_specs):
        return multi_load(path_specs, forced_type, merge, marker, **kwargs)
    else:
        return single_load(path_specs, forced_type, **kwargs)

def loads(config_content, forced_type=None, **kwargs):
    """
    :param config_content: Configuration file's content
    :param forced_type: Forced configuration parser type
    :param kwargs: Backend specific optional arguments, e.g. {"indent": 2} for
        JSON loader/dumper backend
    :return: Dict-like object (instance of
        anyconfig.mergeabledict.MergeableDict by default) supports merge
        operations.
    """
    if forced_type is None:
        return P.parse(config_content)

    cparser = find_loader(None, forced_type)
    if cparser is None:
        return P.parse(config_content)

    return cparser.loads(config_content, **kwargs)

def _find_dumper(config_path, forced_type=None):
    """
    Find configuration parser to dump data.

    :param config_path: Output filename
    :param forced_type: Forced configuration parser type
    :return: ConfigParser-inherited class object
    """
    cparser = find_loader(config_path, forced_type)

    if cparser is None or not getattr(cparser, "dump", False):
        logging.warn(
            "Dump method not implemented. Fallback to JsonConfigParser"
        )
        cparser = BJ.JsonConfigParser()

    return cparser

def dump(data, config_path, forced_type=None, **kwargs):
    """
    Save `data` as `config_path`.

    :param data: Config data object to dump ::
        anyconfig.mergeabledict.MergeableDict by default
    :param config_path: Output filename
    :param forced_type: Forced configuration parser type
    :param kwargs: Backend specific optional arguments, e.g. {"indent": 2} for
        JSON loader/dumper backend
    """
    dumper = _find_dumper(config_path, forced_type)

    logging.info("Dumping: " + config_path)
    dumper.dump(data, config_path, **kwargs)

def dumps(data, forced_type, **kwargs):
    """
    Return string representation of `data` in forced type format.

    :param data: Config data object to dump ::
        anyconfig.mergeabledict.MergeableDict by default
    :param forced_type: Forced configuration parser type
    :param kwargs: Backend specific optional arguments, e.g. {"indent": 2} for
        JSON loader/dumper backend
    :return: Backend-specific string representation for the given data
    """
    return _find_dumper(None, forced_type).dumps(data, **kwargs)

# vim:sw=4:ts=4:et:
<commit_msg>follow changes in anyconfig.globals, s/getLogger/get_logger/g<commit_after>#
# Copyright (C) 2012, 2013 Satoru SATOH <ssato @ redhat.com>
# License: MIT
#
import anyconfig.globals as G
import anyconfig.mergeabledict as M
import anyconfig.backend.backends as Backends
import anyconfig.backend.json_ as BJ
import anyconfig.parser as P
import anyconfig.utils as U

# pylint: disable=W0611
# Import some global constants will be re-exported:
from anyconfig.mergeabledict import MS_REPLACE, MS_NO_REPLACE, \
    MS_DICTS, MS_DICTS_AND_LISTS, MERGE_STRATEGIES
# pylint: enable=W0611

# pylint: disable=C0103
# Re-export:
list_types = Backends.list_types
getLogger = G.get_logger

# aliases:
container = M.MergeableDict
logging = G.LOGGER
# pylint: enable=W0611

def set_loglevel(level):
    """
    :param level: Log level, e.g. logging.INFO and logging.WARN.
    """
    logging.setLevel(level)

def find_loader(config_path, forced_type=None):
    """
    :param config_path: Configuration file path
    :param forced_type: Forced configuration parser type
    :return: ConfigParser-inherited class object
    """
    if forced_type is not None:
        cparser = Backends.find_by_type(forced_type)
        if not cparser:
            logging.error(
                "No parser found for given type: " + forced_type
            )
            return None
    else:
        cparser = Backends.find_by_file(config_path)
        if not cparser:
            logging.error(
                "No parser found for given file: " + config_path
            )
            return None

    logging.debug("Using config parser of type: " + cparser.type())
    return cparser

def single_load(config_path, forced_type=None, **kwargs):
    """
    Load single config file.

    :param config_path: Configuration file path
    :param forced_type: Forced configuration parser type
    :param kwargs: Backend specific optional arguments, e.g. {"indent": 2} for
        JSON loader/dumper backend
    :return: Dict-like object (instance of
        anyconfig.mergeabledict.MergeableDict by default) supports merge
        operations.
    """
    cparser = find_loader(config_path, forced_type)
    if cparser is None:
        return None

    logging.info("Loading: " + config_path)
    return cparser.load(config_path, **kwargs)

def multi_load(paths, forced_type=None, merge=MS_DICTS, marker='*', **kwargs):
    """
    Load multiple config files.

    The first argument `paths` may be a list of config file paths or
    a glob pattern specifying that. That is, if a.yml, b.yml and c.yml are in
    the dir /etc/foo/conf.d/, the followings give same results::

      multi_load(["/etc/foo/conf.d/a.yml", "/etc/foo/conf.d/b.yml",
                  "/etc/foo/conf.d/c.yml", ])

      multi_load("/etc/foo/conf.d/*.yml")

    :param paths: List of config file paths or a glob pattern to list paths
    :param forced_type: Forced configuration parser type
    :param merge: Strategy to merge config results of multiple config files
        loaded. see also: anyconfig.mergeabledict.MergeableDict.update()
    :param marker: Globbing markerer to detect paths patterns
    :param kwargs: Backend specific optional arguments, e.g. {"indent": 2} for
        JSON loader/dumper backend
    :return: Dict-like object (instance of
        anyconfig.mergeabledict.MergeableDict by default) supports merge
        operations.
    """
    assert merge in MERGE_STRATEGIES, "Invalid merge strategy: " + merge

    if marker in paths:
        paths = U.sglob(paths)

    config = container()
    for path in paths:
        if marker in path:  # Nested patterns like ['*.yml', '/a/b/c.yml'].
            conf_updates = multi_load(path, forced_type, merge, marker,
                                      **kwargs)
        else:
            conf_updates = single_load(path, forced_type, **kwargs)

        config.update(conf_updates, merge)

    return config

def load(path_specs, forced_type=None, merge=MS_DICTS, marker='*', **kwargs):
    """
    Load single or multiple config files or multiple config files specified in
    given paths pattern.

    :param path_specs:
        Configuration file path or paths or its pattern such as '/a/b/*.json'
    :param forced_type: Forced configuration parser type
    :param merge: Merging strategy to use
    :param marker: Globbing marker to detect paths patterns
    :param kwargs: Backend specific optional arguments, e.g. {"indent": 2} for
        JSON loader/dumper backend
    :return: Dict-like object (instance of
        anyconfig.mergeabledict.MergeableDict by default) supports merge
        operations.
    """
    if marker in path_specs or U.is_iterable(path_specs):
        return multi_load(path_specs, forced_type, merge, marker, **kwargs)
    else:
        return single_load(path_specs, forced_type, **kwargs)

def loads(config_content, forced_type=None, **kwargs):
    """
    :param config_content: Configuration file's content
    :param forced_type: Forced configuration parser type
    :param kwargs: Backend specific optional arguments, e.g. {"indent": 2} for
        JSON loader/dumper backend
    :return: Dict-like object (instance of
        anyconfig.mergeabledict.MergeableDict by default) supports merge
        operations.
    """
    if forced_type is None:
        return P.parse(config_content)

    cparser = find_loader(None, forced_type)
    if cparser is None:
        return P.parse(config_content)

    return cparser.loads(config_content, **kwargs)

def _find_dumper(config_path, forced_type=None):
    """
    Find configuration parser to dump data.

    :param config_path: Output filename
    :param forced_type: Forced configuration parser type
    :return: ConfigParser-inherited class object
    """
    cparser = find_loader(config_path, forced_type)

    if cparser is None or not getattr(cparser, "dump", False):
        logging.warn(
            "Dump method not implemented. Fallback to JsonConfigParser"
        )
        cparser = BJ.JsonConfigParser()

    return cparser

def dump(data, config_path, forced_type=None, **kwargs):
    """
    Save `data` as `config_path`.

    :param data: Config data object to dump ::
        anyconfig.mergeabledict.MergeableDict by default
    :param config_path: Output filename
    :param forced_type: Forced configuration parser type
    :param kwargs: Backend specific optional arguments, e.g. {"indent": 2} for
        JSON loader/dumper backend
    """
    dumper = _find_dumper(config_path, forced_type)

    logging.info("Dumping: " + config_path)
    dumper.dump(data, config_path, **kwargs)

def dumps(data, forced_type, **kwargs):
    """
    Return string representation of `data` in forced type format.

    :param data: Config data object to dump ::
        anyconfig.mergeabledict.MergeableDict by default
    :param forced_type: Forced configuration parser type
    :param kwargs: Backend specific optional arguments, e.g. {"indent": 2} for
        JSON loader/dumper backend
    :return: Backend-specific string representation for the given data
    """
    return _find_dumper(None, forced_type).dumps(data, **kwargs)

# vim:sw=4:ts=4:et:
<|endoftext|>


<commit_before>from data_collection.management.commands import BaseXpressWebLookupCsvImporter

class Command(BaseXpressWebLookupCsvImporter):
    council_id       = 'E07000226'
    addresses_name   = 'May 2017 crawley property and polling station.csv'
    stations_name    = 'May 2017 crawley property and polling station.csv'
    elections        = [
        'local.west-sussex.2017-05-04',
        'parl.2017-06-08'
    ]
    station_id_field = 'pollingplace_uprn'
<commit_msg>update Crawley import script for parl.2017-06-08 (closes #958)<commit_after>from data_collection.management.commands import BaseXpressDemocracyClubCsvImporter

class Command(BaseXpressDemocracyClubCsvImporter):
    council_id       = 'E07000226'
    addresses_name   = 'parl.2017-06-08/Version 1/Crawley Democracy_Club__08June2017.tsv'
    stations_name    = 'parl.2017-06-08/Version 1/Crawley Democracy_Club__08June2017.tsv'
    elections        = ['parl.2017-06-08']
    csv_delimiter    = '\t'
<|endoftext|>