<commit_before>#!/usr/bin/env python
# encoding: utf-8
"""
auditok.auditok -- Audio Activity Detection tool

auditok.auditok is a program that can be used for Audio/Acoustic
activity detection. It can read audio data from audio files as well
as from built-in device(s) or standard input.

@author:     Mohamed El Amine SEHILI
@copyright:  2015-2019 Mohamed El Amine SEHILI
@license:    GPL v3
@contact:    amine.sehili@gmail.com
@deffield    updated: 01 Dec 2018
"""

import sys
import os
from argparse import ArgumentParser
import time
import threading

from auditok import __version__, AudioRegion
from .util import AudioDataSource
from .exceptions import EndOfProcessing, AudioEncodingWarning
from .io import player_for
from .cmdline_util import make_logger, make_kwargs, initialize_workers
from . import workers

__all__ = []
version = __version__
__date__ = "2015-11-23"
__updated__ = "2018-12-01"

def main(argv=None):
    program_name = os.path.basename(sys.argv[0])
    if argv is None:
        argv = sys.argv[1:]
    try:
        parser = ArgumentParser(
            prog=program_name, description="An Audio Tokenization tool"
        )
        parser.add_argument(
            "--version", "-v", action="version", version=version
        )
        group = parser.add_argument_group("Input-Output options")
        group.add_argument(
            "-i",
            "--input",
            dest="input",
            help="Input audio or video file. Use '-' for stdin "
            "[default: read from microphone using pyaudio]",
            metavar="FILE",
        )
        group.add_argument(
            "-I",
            "--input-device-index",
            dest="input_device_index",
            help="Audio device index [default: %(default)s]. "
            "Optional and only effective when using PyAudio",
            type=int,
            default=None,
            metavar="INT",
        )
        group.add_argument(
            "-F",
            "--audio-frame-per-buffer",
            dest="frame_per_buffer",
            help="Audio frame per buffer [default: %(default)s]. "
            "Optional and only effective when using PyAudio",
            type=int,
            default=1024,
            metavar="INT",
        )
        group.add_argument(
            "-f",
            "--input-format",
            dest="input_format",
            type=str,
            default=None,
            help="Input audio file format. If not given, guess format from "
            "extension. If output file name has no extension, guess format "
            "from file header (requires pydub). If none of the previous is "
            "true, raise an error",
            metavar="STRING",
        )
        group.add_argument(
            "-M",
            "--max-read",
            dest="max_read",
            type=float,
            default=None,
            help="Maximum data (in seconds) to read from microphone or file "
            "[default: read until the end of file/stream]",
            metavar="FLOAT",
        )
        group.add_argument(
            "-L",
            "--large-file",
            dest="large_file",
            action="store_true",
            default=False,
            help="Whether input file should be treated as a large file. "
            "If True, data will be read from file on demand, otherwise all "
            "audio data is loaded to memory before tokenization.",
        )
        group.add_argument(
            "-O",
            "--save-stream",
            dest="save_stream",
            type=str,
            default=None,
            help="Save acquired audio data (from file or microphone) to disk."
            " If omitted no data will be saved. [default: omitted]",
            metavar="FILE",
        )
        group.add_argument(
            "-o",
            "--save-detections-as",
            dest="save_detections_as",
            type=str,
            default=None,
            help="File name format for detections."
            "The following placeholders can be used to build output file name "
            "for each detection: {id} (sequential, starts from 1), {start}, "
            "{end} and {duration}. Time placeholders are in seconds. "
            "Example: 'Event_{id}_{start}-{end}_{duration:.3f}.wav'",
            metavar="STRING",
        )
        group.add_argument(
            "-T",
            "--output-format",
            dest="output_format",
            type=str,
            default=None,
            help="Audio format used to save detections and/or main stream. "
            "If not supplied, then it will: (1. be guessed from extension or (2. "
            "use raw format",
            metavar="STRING",
        )
        group.add_argument(
            "-u",
            "--use-channel",
            dest="use_channel",
            type=str,
            default=None,
            help="Which channel to use for tokenization when input stream is "
            "multi-channel (0 is the first channel). Default is None, meaning "
            "that all channels will be considered for tokenization (i.e., get "
            "any valid audio event regardless of the channel it occurs in). "
            "This value can also be 'mix' (alias 'avg' or 'average') and "
            "means mix down all audio channels into one channel (i.e. compute "
            "average channel) and use the resulting channel for tokenization. "
            "Whatever option is used, saved audio events will contain the same"
            " number of channels as input stream. "
            "[Default: None, use all channels]",
            metavar="INT/STRING",
        )

        group = parser.add_argument_group(
            "Tokenization options", "Set tokenizer options."
        )
        group.add_argument(
            "-a",
            "--analysis-window",
            dest="analysis_window",
            default=0.01,
            type=float,
            help="Size of analysis window in seconds [default: %(default)s (10ms)]",
            metavar="FLOAT",
        )
        group.add_argument(
            "-n",
            "--min-duration",
            dest="min_duration",
            type=float,
            default=0.2,
            help="Min duration of a valid audio event in seconds [default: %(default)s]",
            metavar="FLOAT",
        )
        group.add_argument(
            "-m",
            "--max-duration",
            dest="max_duration",
            type=float,
            default=5,
            help="Max duration of a valid audio event in seconds [default: %(default)s]",
            metavar="FLOAT",
        )
        group.add_argument(
            "-s",
            "--max-silence",
            dest="max_silence",
            type=float,
            default=0.3,
            help="Max duration of a consecutive silence within a valid audio event "
            "in seconds [default: %(default)s]",
            metavar="FLOAT",
        )
        group.add_argument(
            "-d",
            "--drop-trailing-silence",
            dest="drop_trailing_silence",
            action="store_true",
            default=False,
            help="Drop trailing silence from a detection [default: keep "
            "trailing silence]",
        )
        group.add_argument(
            "-R",
            "--strict-min-duration",
            dest="strict_min_duration",
            action="store_true",
            default=False,
            help="Reject an event shorter than --min-duration even if it's "
            "adjacent to the latest valid event that reached max-duration "
            "[default: keep such events]",
        )
        group.add_argument(
            "-e",
            "--energy-threshold",
            dest="energy_threshold",
            type=float,
            default=50,
            help="Log energy threshold for detection [default: %(default)s]",
            metavar="FLOAT",
        )

        group = parser.add_argument_group(
            "Audio parameters",
            "Define audio parameters if data is read from a "
            "headerless file (raw or stdin) or you want to use "
            "different microphone parameters.",
        )
        group.add_argument(
            "-r",
            "--rate",
            dest="sampling_rate",
            type=int,
            default=16000,
            help="Sampling rate of audio data [default: %(default)s]",
            metavar="INT",
        )
        group.add_argument(
            "-c",
            "--channels",
            dest="channels",
            type=int,
            default=1,
            help="Number of channels of audio data [default: %(default)s]",
            metavar="INT",
        )
        group.add_argument(
            "-w",
            "--width",
            dest="sample_width",
            type=int,
            default=2,
            help="Number of bytes per audio sample [default: %(default)s]",
            metavar="INT",
        )

        group = parser.add_argument_group(
            "Do something with audio events",
            "Use these options to print, play or plot detections.",
        )
        group.add_argument(
            "-C",
            "--command",
            dest="command",
            type=str,
            help="Command to call when an audio detection occurs. Use $ to "
            "represent the file name to use with the command (e.g. -C "
            "'du -h $')",
            metavar="STRING",
        )
        group.add_argument(
            "-E",
            "--echo",
            dest="echo",
            action="store_true",
            default=False,
            help="Play back each detection immediately using pyaudio",
        )
        group.add_argument(
            "-B",
            "--progress-bar",
            dest="progress_bar",
            action="store_true",
            default=False,
            help="Show a progress bar when playing audio",
        )
        group.add_argument(
            "-p",
            "--plot",
            dest="plot",
            action="store_true",
            default=False,
            help="Plot and show audio signal and detections (requires matplotlib)",
        )
        group.add_argument(
            "--save-image",
            dest="save_image",
            type=str,
            help="Save plotted audio signal and detections as a picture or a PDF "
            "file (requires matplotlib)",
            metavar="FILE",
        )
        group.add_argument(
            "--printf",
            dest="printf",
            type=str,
            default="{id} {start} {end}",
            help="Print audio events information, one per line, using this "
            "format. Format can contain text with the following placeholders: "
            "{id} (sequential, starts from 1), {start}, {end}, {duration} and "
            "{timestamp}. The first 3 time placeholders are in seconds and "
            "their format can be set using --time-format argument. "
            "{timestamp} is the system timestamp (date and time) of the event "
            "and can be set using --timestamp-format argument.\n"
            "Example: '[{id}]: {start} -> {end} -- {timestamp}'",
            metavar="STRING",
        )
        group.add_argument(
            "--time-format",
            dest="time_format",
            type=str,
            default="%S",
            help="Format used to print {start}, {end} and {duration} "
            "placeholders used with --printf [default= %(default)s]. The "
            "following formats are accepted:\n"
            "%%S: absolute time in seconds. %%I: absolute time in ms. If at "
            "least one of (%%h, %%m, %%s, %%i) is used, convert time into hours, "
            "minutes, seconds and millis (e.g. %%h:%%m:%%s.%%i). Only supplied "
            "fields are printed. Note that %%S and %%I can only be used alone",
            metavar="STRING",
        )
        group.add_argument(
            "--timestamp-format",
            dest="timestamp_format",
            type=str,
            default="%Y/%m/%d %H:%M:%S",
            help="Format used to print {timestamp}. Should be a format "
            "accepted by datetime Default %%Y/%%m/%%d %%H:%%M:%%S",
        )
        parser.add_argument(
            "-q",
            "--quiet",
            dest="quiet",
            action="store_true",
            default=False,
            help="Do not print any information about detections [default: print "
            "'id', 'start' and 'end' of each detection]",
        )
        parser.add_argument(
            "-D",
            "--debug",
            dest="debug",
            action="store_true",
            default=False,
            help="Print processing operations to STDOUT",
        )
        parser.add_argument(
            "--debug-file",
            dest="debug_file",
            type=str,
            default=None,
            help="Print processing operations to FILE",
            metavar="FILE",
        )

        args = parser.parse_args(argv)
        logger = make_logger(args.debug, args.debug_file)
        kwargs = make_kwargs(args)
        reader, observers = initialize_workers(
            logger=logger, **kwargs.io, **kwargs.miscellaneous
        )
        tokenizer_worker = workers.TokenizerWorker(
            reader, observers, logger=logger, **kwargs.split
        )
        tokenizer_worker.start_all()

        while True:
            time.sleep(1)
            if len(threading.enumerate()) == 1:
                raise EndOfProcessing

    except (KeyboardInterrupt, EndOfProcessing):
        if tokenizer_worker is not None:
            tokenizer_worker.stop_all()

            if isinstance(reader, workers.StreamSaverWorker):
                reader.join()
                try:
                    reader.save_stream()
                except AudioEncodingWarning as ae_warn:
                    print(str(ae_warn), file=sys.stderr)

            if args.plot or args.save_image is not None:
                from .plotting import plot_detections

                reader.rewind()
                record = AudioRegion(
                    reader.data, reader.sr, reader.sw, reader.ch
                )
                detections = (
                    (det.start, det.end) for det in tokenizer_worker.detections
                )
                plot_detections(
                    record,
                    record.sr,
                    detections,
                    show=True,
                    save_as=args.save_image,
                )
        return 0

if __name__ == "__main__":
    sys.exit(main(None))
<commit_msg>Update call to plotting code in main<commit_after>#!/usr/bin/env python
# encoding: utf-8
"""
auditok.auditok -- Audio Activity Detection tool

auditok.auditok is a program that can be used for Audio/Acoustic
activity detection. It can read audio data from audio files as well
as from built-in device(s) or standard input.

@author:     Mohamed El Amine SEHILI
@copyright:  2015-2019 Mohamed El Amine SEHILI
@license:    GPL v3
@contact:    amine.sehili@gmail.com
@deffield    updated: 01 Dec 2018
"""

import sys
import os
from argparse import ArgumentParser
import time
import threading

from auditok import __version__, AudioRegion
from .util import AudioDataSource
from .exceptions import EndOfProcessing, AudioEncodingWarning
from .io import player_for
from .cmdline_util import make_logger, make_kwargs, initialize_workers
from . import workers

__all__ = []
version = __version__
__date__ = "2015-11-23"
__updated__ = "2018-12-01"

def main(argv=None):
    program_name = os.path.basename(sys.argv[0])
    if argv is None:
        argv = sys.argv[1:]
    try:
        parser = ArgumentParser(
            prog=program_name, description="An Audio Tokenization tool"
        )
        parser.add_argument(
            "--version", "-v", action="version", version=version
        )
        group = parser.add_argument_group("Input-Output options")
        group.add_argument(
            "-i",
            "--input",
            dest="input",
            help="Input audio or video file. Use '-' for stdin "
            "[default: read from microphone using pyaudio]",
            metavar="FILE",
        )
        group.add_argument(
            "-I",
            "--input-device-index",
            dest="input_device_index",
            help="Audio device index [default: %(default)s]. "
            "Optional and only effective when using PyAudio",
            type=int,
            default=None,
            metavar="INT",
        )
        group.add_argument(
            "-F",
            "--audio-frame-per-buffer",
            dest="frame_per_buffer",
            help="Audio frame per buffer [default: %(default)s]. "
            "Optional and only effective when using PyAudio",
            type=int,
            default=1024,
            metavar="INT",
        )
        group.add_argument(
            "-f",
            "--input-format",
            dest="input_format",
            type=str,
            default=None,
            help="Input audio file format. If not given, guess format from "
            "extension. If output file name has no extension, guess format "
            "from file header (requires pydub). If none of the previous is "
            "true, raise an error",
            metavar="STRING",
        )
        group.add_argument(
            "-M",
            "--max-read",
            dest="max_read",
            type=float,
            default=None,
            help="Maximum data (in seconds) to read from microphone or file "
            "[default: read until the end of file/stream]",
            metavar="FLOAT",
        )
        group.add_argument(
            "-L",
            "--large-file",
            dest="large_file",
            action="store_true",
            default=False,
            help="Whether input file should be treated as a large file. "
            "If True, data will be read from file on demand, otherwise all "
            "audio data is loaded to memory before tokenization.",
        )
        group.add_argument(
            "-O",
            "--save-stream",
            dest="save_stream",
            type=str,
            default=None,
            help="Save acquired audio data (from file or microphone) to disk."
            " If omitted no data will be saved. [default: omitted]",
            metavar="FILE",
        )
        group.add_argument(
            "-o",
            "--save-detections-as",
            dest="save_detections_as",
            type=str,
            default=None,
            help="File name format for detections."
            "The following placeholders can be used to build output file name "
            "for each detection: {id} (sequential, starts from 1), {start}, "
            "{end} and {duration}. Time placeholders are in seconds. "
            "Example: 'Event_{id}_{start}-{end}_{duration:.3f}.wav'",
            metavar="STRING",
        )
        group.add_argument(
            "-T",
            "--output-format",
            dest="output_format",
            type=str,
            default=None,
            help="Audio format used to save detections and/or main stream. "
            "If not supplied, then it will: (1. be guessed from extension or (2. "
            "use raw format",
            metavar="STRING",
        )
        group.add_argument(
            "-u",
            "--use-channel",
            dest="use_channel",
            type=str,
            default=None,
            help="Which channel to use for tokenization when input stream is "
            "multi-channel (0 is the first channel). Default is None, meaning "
            "that all channels will be considered for tokenization (i.e., get "
            "any valid audio event regardless of the channel it occurs in). "
            "This value can also be 'mix' (alias 'avg' or 'average') and "
            "means mix down all audio channels into one channel (i.e. compute "
            "average channel) and use the resulting channel for tokenization. "
            "Whatever option is used, saved audio events will contain the same"
            " number of channels as input stream. "
            "[Default: None, use all channels]",
            metavar="INT/STRING",
        )

        group = parser.add_argument_group(
            "Tokenization options", "Set tokenizer options."
        )
        group.add_argument(
            "-a",
            "--analysis-window",
            dest="analysis_window",
            default=0.01,
            type=float,
            help="Size of analysis window in seconds [default: %(default)s (10ms)]",
            metavar="FLOAT",
        )
        group.add_argument(
            "-n",
            "--min-duration",
            dest="min_duration",
            type=float,
            default=0.2,
            help="Min duration of a valid audio event in seconds [default: %(default)s]",
            metavar="FLOAT",
        )
        group.add_argument(
            "-m",
            "--max-duration",
            dest="max_duration",
            type=float,
            default=5,
            help="Max duration of a valid audio event in seconds [default: %(default)s]",
            metavar="FLOAT",
        )
        group.add_argument(
            "-s",
            "--max-silence",
            dest="max_silence",
            type=float,
            default=0.3,
            help="Max duration of a consecutive silence within a valid audio event "
            "in seconds [default: %(default)s]",
            metavar="FLOAT",
        )
        group.add_argument(
            "-d",
            "--drop-trailing-silence",
            dest="drop_trailing_silence",
            action="store_true",
            default=False,
            help="Drop trailing silence from a detection [default: keep "
            "trailing silence]",
        )
        group.add_argument(
            "-R",
            "--strict-min-duration",
            dest="strict_min_duration",
            action="store_true",
            default=False,
            help="Reject an event shorter than --min-duration even if it's "
            "adjacent to the latest valid event that reached max-duration "
            "[default: keep such events]",
        )
        group.add_argument(
            "-e",
            "--energy-threshold",
            dest="energy_threshold",
            type=float,
            default=50,
            help="Log energy threshold for detection [default: %(default)s]",
            metavar="FLOAT",
        )

        group = parser.add_argument_group(
            "Audio parameters",
            "Define audio parameters if data is read from a "
            "headerless file (raw or stdin) or you want to use "
            "different microphone parameters.",
        )
        group.add_argument(
            "-r",
            "--rate",
            dest="sampling_rate",
            type=int,
            default=16000,
            help="Sampling rate of audio data [default: %(default)s]",
            metavar="INT",
        )
        group.add_argument(
            "-c",
            "--channels",
            dest="channels",
            type=int,
            default=1,
            help="Number of channels of audio data [default: %(default)s]",
            metavar="INT",
        )
        group.add_argument(
            "-w",
            "--width",
            dest="sample_width",
            type=int,
            default=2,
            help="Number of bytes per audio sample [default: %(default)s]",
            metavar="INT",
        )

        group = parser.add_argument_group(
            "Do something with audio events",
            "Use these options to print, play or plot detections.",
        )
        group.add_argument(
            "-C",
            "--command",
            dest="command",
            type=str,
            help="Command to call when an audio detection occurs. Use $ to "
            "represent the file name to use with the command (e.g. -C "
            "'du -h $')",
            metavar="STRING",
        )
        group.add_argument(
            "-E",
            "--echo",
            dest="echo",
            action="store_true",
            default=False,
            help="Play back each detection immediately using pyaudio",
        )
        group.add_argument(
            "-B",
            "--progress-bar",
            dest="progress_bar",
            action="store_true",
            default=False,
            help="Show a progress bar when playing audio",
        )
        group.add_argument(
            "-p",
            "--plot",
            dest="plot",
            action="store_true",
            default=False,
            help="Plot and show audio signal and detections (requires matplotlib)",
        )
        group.add_argument(
            "--save-image",
            dest="save_image",
            type=str,
            help="Save plotted audio signal and detections as a picture or a PDF "
            "file (requires matplotlib)",
            metavar="FILE",
        )
        group.add_argument(
            "--printf",
            dest="printf",
            type=str,
            default="{id} {start} {end}",
            help="Print audio events information, one per line, using this "
            "format. Format can contain text with the following placeholders: "
            "{id} (sequential, starts from 1), {start}, {end}, {duration} and "
            "{timestamp}. The first 3 time placeholders are in seconds and "
            "their format can be set using --time-format argument. "
            "{timestamp} is the system timestamp (date and time) of the event "
            "and can be set using --timestamp-format argument.\n"
            "Example: '[{id}]: {start} -> {end} -- {timestamp}'",
            metavar="STRING",
        )
        group.add_argument(
            "--time-format",
            dest="time_format",
            type=str,
            default="%S",
            help="Format used to print {start}, {end} and {duration} "
            "placeholders used with --printf [default= %(default)s]. The "
            "following formats are accepted:\n"
            "%%S: absolute time in seconds. %%I: absolute time in ms. If at "
            "least one of (%%h, %%m, %%s, %%i) is used, convert time into hours, "
            "minutes, seconds and millis (e.g. %%h:%%m:%%s.%%i). Only supplied "
            "fields are printed. Note that %%S and %%I can only be used alone",
            metavar="STRING",
        )
        group.add_argument(
            "--timestamp-format",
            dest="timestamp_format",
            type=str,
            default="%Y/%m/%d %H:%M:%S",
            help="Format used to print {timestamp}. Should be a format "
            "accepted by datetime Default %%Y/%%m/%%d %%H:%%M:%%S",
        )
        parser.add_argument(
            "-q",
            "--quiet",
            dest="quiet",
            action="store_true",
            default=False,
            help="Do not print any information about detections [default: print "
            "'id', 'start' and 'end' of each detection]",
        )
        parser.add_argument(
            "-D",
            "--debug",
            dest="debug",
            action="store_true",
            default=False,
            help="Print processing operations to STDOUT",
        )
        parser.add_argument(
            "--debug-file",
            dest="debug_file",
            type=str,
            default=None,
            help="Print processing operations to FILE",
            metavar="FILE",
        )

        args = parser.parse_args(argv)
        logger = make_logger(args.debug, args.debug_file)
        kwargs = make_kwargs(args)
        reader, observers = initialize_workers(
            logger=logger, **kwargs.io, **kwargs.miscellaneous
        )
        tokenizer_worker = workers.TokenizerWorker(
            reader, observers, logger=logger, **kwargs.split
        )
        tokenizer_worker.start_all()

        while True:
            time.sleep(1)
            if len(threading.enumerate()) == 1:
                raise EndOfProcessing

    except (KeyboardInterrupt, EndOfProcessing):
        if tokenizer_worker is not None:
            tokenizer_worker.stop_all()

            if isinstance(reader, workers.StreamSaverWorker):
                reader.join()
                try:
                    reader.save_stream()
                except AudioEncodingWarning as ae_warn:
                    print(str(ae_warn), file=sys.stderr)

            if args.plot or args.save_image is not None:
                from .plotting import plot

                reader.rewind()
                record = AudioRegion(
                    reader.data, reader.sr, reader.sw, reader.ch
                )
                detections = (
                    (det.start, det.end) for det in tokenizer_worker.detections
                )
                plot(
                    record,
                    detections=detections,
                    show=True,
                    save_as=args.save_image,
                )
        return 0

if __name__ == "__main__":
    sys.exit(main(None))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function

import argparse
import logging
import os
import subprocess

from .test_suite import TestCase, TestSuite, testcase_path

SOCKET_OPERATIONS = set(['bind', 'listen', 'accept', 'connect', 'getsockopt', 'shutdown', 'close'])
PROCESS_SYSCALLS = set(['close', 'execve', 'fork', 'clone'])

def main():
    if os.geteuid() != 0:
        print("You have to be root to run the test driver. Please use sudo.")
        exit(1)

    parser = argparse.ArgumentParser(description="Test driver for client-server networking applications.")
    parser.add_argument("--debug", "-d", action="store_true", help="Print debug messages.")
    parser.add_argument("--list-testcases", "-l", action="store_true", help="List testcases and scenarios.")
    parser.add_argument("--list-scenarios", action="store_true", help="List testcases and scenarios.")
    parser.add_argument("--deps", action="store_true", help="List dependencies.")
    parser.add_argument("--outdir", default="./json-output/", help="List dependencies.")
    parser.add_argument("testcases", nargs="?")
    parser.add_argument("scenarios", nargs="?")
    options = parser.parse_args()

    if options.debug:
        logging.getLogger().setLevel(logging.DEBUG)

    testcases = options.testcases and options.testcases.split(',')
    scenarios = options.scenarios and options.scenarios.split(',')

    suite = TestSuite(testcases, scenarios)
    if options.list_testcases:
        for testcase in suite.testcases:
            print(testcase.name)
        exit(0)
    elif options.list_scenarios:
        for scenario in TestCase.scenario_classes:
            print(scenario.name)
        exit(0)
    elif options.deps:
        for testcase in suite.testcases:
            path = os.path.join(testcase_path, testcase.name, "deps")
            if os.path.isfile(path):
                subprocess.check_call(['cat', path])
        exit(0)
    else:
        suite.run()
        suite.save(options.outdir)
        suite.report()

    exit(0 if suite.result else 1)

if __name__ == '__main__':
    main()
<commit_msg>Removed unused constants from client_server<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function

import argparse
import logging
import os
import subprocess

from .test_suite import TestCase, TestSuite, testcase_path

def main():
    if os.geteuid() != 0:
        print("You have to be root to run the test driver. Please use sudo.")
        exit(1)

    parser = argparse.ArgumentParser(description="Test driver for client-server networking applications.")
    parser.add_argument("--debug", "-d", action="store_true", help="Print debug messages.")
    parser.add_argument("--list-testcases", "-l", action="store_true", help="List testcases and scenarios.")
    parser.add_argument("--list-scenarios", action="store_true", help="List testcases and scenarios.")
    parser.add_argument("--deps", action="store_true", help="List dependencies.")
    parser.add_argument("--outdir", default="./json-output/", help="List dependencies.")
    parser.add_argument("testcases", nargs="?")
    parser.add_argument("scenarios", nargs="?")
    options = parser.parse_args()

    if options.debug:
        logging.getLogger().setLevel(logging.DEBUG)

    testcases = options.testcases and options.testcases.split(',')
    scenarios = options.scenarios and options.scenarios.split(',')

    suite = TestSuite(testcases, scenarios)
    if options.list_testcases:
        for testcase in suite.testcases:
            print(testcase.name)
        exit(0)
    elif options.list_scenarios:
        for scenario in TestCase.scenario_classes:
            print(scenario.name)
        exit(0)
    elif options.deps:
        for testcase in suite.testcases:
            path = os.path.join(testcase_path, testcase.name, "deps")
            if os.path.isfile(path):
                subprocess.check_call(['cat', path])
        exit(0)
    else:
        suite.run()
        suite.save(options.outdir)
        suite.report()

    exit(0 if suite.result else 1)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import mock

from piper.core import Piper
from piper.utils import DotDict

from test.utils import builtin

class PiperTestBase(object):
    def setup_method(self, method):
        self.piper = Piper()

class TestPiperSetup(PiperTestBase):
    def setup_method(self, method):
        self.methods = (
            'load_config',
            'validate_config',
        )

        super(TestPiperSetup, self).setup_method(method)

    def test_setup_calls(self):
        for method in self.methods:
            setattr(self.piper, method, mock.MagicMock())

        self.piper.setup()

        for method in self.methods:
            getattr(self.piper, method).assert_called_once_with()

class TestPiperConfigLoader(PiperTestBase):
    def setup_method(self, method):
        self.data = 'lel: 10\ntest: wizard\n\n'
        super(TestPiperConfigLoader, self).setup_method(method)

    @mock.patch('sys.exit')
    @mock.patch('os.path.isfile')
    def test_load_config_no_file(self, isfile, exit):
        isfile.return_value = False
        self.piper.load_config()

        exit.assert_called_once_with(127)

    @mock.patch('sys.exit')
    @mock.patch('os.path.isfile')
    def test_load_config_invalid_yaml(self, isfile, exit):
        isfile.return_value = True
        fake = mock.mock_open(read_data='{')

        with mock.patch(builtin('open'), fake):
            self.piper.load_config()

        exit.assert_called_once_with(126)

    @mock.patch('yaml.safe_load')
    @mock.patch('os.path.isfile')
    def test_load_config(self, isfile, sl):
        isfile.return_value = True
        fake = mock.mock_open(read_data=str(self.data))

        with mock.patch(builtin('open'), fake):
            self.piper.load_config()

        sl.assert_called_once_with(fake.return_value.read.return_value)
        assert self.piper.raw_config == sl.return_value
        assert isinstance(self.piper.config, DotDict)
        assert self.piper.config.data == sl.return_value
<commit_msg>Rename test<commit_after>import mock

from piper.core import Piper
from piper.utils import DotDict

from test.utils import builtin

class PiperTestBase(object):
    def setup_method(self, method):
        self.piper = Piper()

class TestPiperSetup(PiperTestBase):
    def setup_method(self, method):
        self.methods = (
            'load_config',
            'validate_config',
        )

        super(TestPiperSetup, self).setup_method(method)

    def test_setup_calls(self):
        for method in self.methods:
            setattr(self.piper, method, mock.MagicMock())

        self.piper.setup()

        for method in self.methods:
            getattr(self.piper, method).assert_called_once_with()

class TestPiperConfigLoader(PiperTestBase):
    def setup_method(self, method):
        self.data = 'lel: 10\ntest: wizard\n\n'
        super(TestPiperConfigLoader, self).setup_method(method)

    @mock.patch('sys.exit')
    @mock.patch('os.path.isfile')
    def test_load_config_no_file(self, isfile, exit):
        isfile.return_value = False
        self.piper.load_config()

        exit.assert_called_once_with(127)

    @mock.patch('sys.exit')
    @mock.patch('os.path.isfile')
    def test_load_config_invalid_yaml(self, isfile, exit):
        isfile.return_value = True
        fake = mock.mock_open(read_data='{')

        with mock.patch(builtin('open'), fake):
            self.piper.load_config()

        exit.assert_called_once_with(126)

    @mock.patch('yaml.safe_load')
    @mock.patch('os.path.isfile')
    def test_load_config_actual_load(self, isfile, sl):
        isfile.return_value = True
        fake = mock.mock_open(read_data=str(self.data))

        with mock.patch(builtin('open'), fake):
            self.piper.load_config()

        sl.assert_called_once_with(fake.return_value.read.return_value)
        assert self.piper.raw_config == sl.return_value
        assert isinstance(self.piper.config, DotDict)
        assert self.piper.config.data == sl.return_value
<|endoftext|>


<commit_before>#!/usr/bin/env python2
# Copyright 2016 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import ast
import matplotlib.pyplot as plt
import os
import re

layer_regex_pattern = re.compile(r"^(\d)sl(\d)tl$")
def writable_dir(directory):
  if not os.path.isdir(directory) or not os.access(directory, os.W_OK):
    raise argparse.ArgumentTypeError("'%s' is either not a directory or cannot be opened for writing.\n" % directory)
  return directory

def formats(formats_list):
  formats = formats_list.split(',')
  for extension in formats:
    if extension not in ['svg', 'png']:
      raise argparse.ArgumentTypeError("'%s' is not a valid file format.\n" % extension)
  return formats

parser = argparse.ArgumentParser(description='Generate graphs from data files.')
parser.add_argument('graph_files', nargs='+', metavar='graph_file.txt', type=argparse.FileType('r'))
parser.add_argument('--out_dir', required=True, type=writable_dir)
parser.add_argument('--formats', type=formats, metavar='png,svg', help='comma-separated list of output formats', default=['svg'])

def split_data(graph_data, attribute):
  groups = {}
  for element in graph_data:
    value = element[attribute]
    if value not in groups:
      groups[value] = []
    groups[value].append(element)
  return groups.values()

def normalize_bitrate_config_string(config):
  return ":".join([str(int(x * 100.0 / config[-1])) for x in config])

def generate_graphs(output_dict, graph_data, target_metric, bitrate_config_string):
  lines = {}
  for encoder in split_data(graph_data, 'encoder'):
    for codec in split_data(encoder, 'codec'):
      for layer in split_data(codec, 'temporal-layer'):
        metric_data = []
        for data in layer:
          metric_data.append((data['target-bitrate-bps']/1000, data[target_metric], data['bitrate-utilization']))
        line_name = '%s-%s-tl%d' % (layer[0]['encoder'], layer[0]['codec'], layer[0]['temporal-layer'])
        # Sort points on target bitrate.
        lines[line_name] = sorted(metric_data, key=lambda point: point[0])

  graph_name = "%s-%s-%s:%s" % (graph_data[0]['input-file'], graph_data[0]['layer-pattern'], bitrate_config_string, target_metric)
  output_dict[graph_name] = lines

def main():
  args = parser.parse_args()
  graph_data = []
  for f in args.graph_files:
    graph_data += ast.literal_eval(f.read())

  graph_dict = {}
  for input_files in split_data(graph_data, 'input-file'):
    for layer_pattern in split_data(input_files, 'layer-pattern'):
      for data in layer_pattern:
        metrics = [
          'vpx-ssim',
          'ssim',
          'ssim-y',
          'ssim-u',
          'ssim-v',
          'avg-psnr',
          'avg-psnr-y',
          'avg-psnr-u',
          'avg-psnr-v',
          'glb-psnr',
          'glb-psnr-y',
          'glb-psnr-u',
          'glb-psnr-v',
          'encode-time-utilization',
        ]
        for metric in metrics:
          generate_graphs(graph_dict, layer_pattern, metric, normalize_bitrate_config_string(data['bitrate-config-kbps']))

  for point in graph_data:
    line_name = '%s-%s' % (point['encoder'], point['codec'])
    pattern_match = layer_regex_pattern.match(point['layer-pattern'])
    num_temporal_layers = int(pattern_match.group(2))
    temporal_divide = 2 ** (num_temporal_layers - 1 - point['temporal-layer'])
    frame_metrics = [
      'frame-ssim',
      'frame-ssim-y',
      'frame-ssim-u',
      'frame-ssim-v',
      'frame-psnr',
      'frame-psnr-y',
      'frame-psnr-u',
      'frame-psnr-v',
      'frame-qp',
    ]
    for target_metric in frame_metrics:
      # TODO(pbos): Add frame QP to AV1 decoding.
      if target_metric == 'frame-qp' and not 'frame-qp' in point:
        continue
      split_on_codecs = target_metric == 'frame-qp'

      if split_on_codecs:
        graph_name = "%s-%s-%s-%s-%dkbps-tl%d:%s" % (point['input-file'], point['codec'], point['layer-pattern'], normalize_bitrate_config_string(point['bitrate-config-kbps']), point['bitrate-config-kbps'][-1], point['temporal-layer'], target_metric)
        line_name = '%s' % point['encoder']
      else:
        graph_name = "%s-%s-%s-%dkbps-tl%d:%s" % (point['input-file'], point['layer-pattern'], normalize_bitrate_config_string(point['bitrate-config-kbps']), point['bitrate-config-kbps'][-1], point['temporal-layer'], target_metric)
      if not graph_name in graph_dict:
        graph_dict[graph_name] = {}
      line = []
      for idx, val in enumerate(point[target_metric]):
        if 'frame-bytes' in point:
          line.append((temporal_divide * idx + 1, val, point['frame-bytes'][idx]))
        else:
          # TODO(pbos): Add frame size to AV1 decoding.
          line.append((temporal_divide * idx + 1, val, 0))
      graph_dict[graph_name][line_name] = line

  current_graph = 1
  total_graphs = len(graph_dict)
  for graph_name, lines in graph_dict.iteritems():
    print "[%d/%d] %s" % (current_graph, total_graphs, graph_name)
    current_graph += 1
    metric = graph_name.split(':')[-1]
    fig, ax = plt.subplots()
    ax.set_title(graph_name)
    frame_data = 'frame-' in metric
    ax2 = None
    ax2_bitrate_utilization = False
    linestyle = 'o--'
    ax2_linestyle = 'x-'

    if frame_data:
      ax.set_xlabel('Frame')
      ax.set_ylabel(metric.replace('frame-', '').upper())
      ax2 = ax.twinx()
      ax2.set_ylabel('Frame Size (bytes / frame)')
      linestyle = '-'
      ax2_linestyle = '-'
    elif metric == 'encode-time-utilization':
      ax.set_xlabel('Layer Target Bitrate (kbps)')
      ax.set_ylabel('Encode Time (fraction)')
      # Draw a reference line for realtime.
      ax.axhline(1.0, color='k', alpha=0.2, linestyle='--')
    else:
      ax.set_xlabel('Layer Target Bitrate (kbps)')
      ax.set_ylabel(metric.upper())
      ax2 = ax.twinx()
      ax2.set_ylabel('Bitrate Utilization (actual / target)')
      ax2_bitrate_utilization = True

    for title in sorted(lines.keys()):
      points = lines[title]
      x = []
      y = []
      y2 = []
      for bitrate_kbps, value, utilization in points:
          x.append(bitrate_kbps)
          y.append(value)
          y2.append(utilization)
      ax.plot(x, y, linestyle, linewidth=1, label=title)
      if ax2:
        ax2.plot(x, y2, ax2_linestyle, alpha=0.2)
      ax.legend(loc='best', fancybox=True, framealpha=0.5)

    if metric == 'encode-time-utilization':
      # Make sure the horizontal reference line at 1.0 can be seen.
      (lower, upper) = ax.get_ylim()
      if upper < 1.10:
        ax.set_ylim(top=1.10)

    if ax2_bitrate_utilization:
      # Set bitrate limit axes to +/- 20%.
      ax2.set_ylim(bottom=0.80, top=1.20)

    for extension in args.formats:
      plt.savefig(os.path.join(args.out_dir, "%s.%s" % (graph_name.replace(":", "-"), extension)))
    plt.close()

if __name__ == '__main__':
  main()

<commit_msg>Rename frame-qp graph names.<commit_after>#!/usr/bin/env python2
# Copyright 2016 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import ast
import matplotlib.pyplot as plt
import os
import re

layer_regex_pattern = re.compile(r"^(\d)sl(\d)tl$")
def writable_dir(directory):
  if not os.path.isdir(directory) or not os.access(directory, os.W_OK):
    raise argparse.ArgumentTypeError("'%s' is either not a directory or cannot be opened for writing.\n" % directory)
  return directory

def formats(formats_list):
  formats = formats_list.split(',')
  for extension in formats:
    if extension not in ['svg', 'png']:
      raise argparse.ArgumentTypeError("'%s' is not a valid file format.\n" % extension)
  return formats

parser = argparse.ArgumentParser(description='Generate graphs from data files.')
parser.add_argument('graph_files', nargs='+', metavar='graph_file.txt', type=argparse.FileType('r'))
parser.add_argument('--out_dir', required=True, type=writable_dir)
parser.add_argument('--formats', type=formats, metavar='png,svg', help='comma-separated list of output formats', default=['svg'])

def split_data(graph_data, attribute):
  groups = {}
  for element in graph_data:
    value = element[attribute]
    if value not in groups:
      groups[value] = []
    groups[value].append(element)
  return groups.values()

def normalize_bitrate_config_string(config):
  return ":".join([str(int(x * 100.0 / config[-1])) for x in config])

def generate_graphs(output_dict, graph_data, target_metric, bitrate_config_string):
  lines = {}
  for encoder in split_data(graph_data, 'encoder'):
    for codec in split_data(encoder, 'codec'):
      for layer in split_data(codec, 'temporal-layer'):
        metric_data = []
        for data in layer:
          metric_data.append((data['target-bitrate-bps']/1000, data[target_metric], data['bitrate-utilization']))
        line_name = '%s-%s-tl%d' % (layer[0]['encoder'], layer[0]['codec'], layer[0]['temporal-layer'])
        # Sort points on target bitrate.
        lines[line_name] = sorted(metric_data, key=lambda point: point[0])

  graph_name = "%s-%s-%s:%s" % (graph_data[0]['input-file'], graph_data[0]['layer-pattern'], bitrate_config_string, target_metric)
  output_dict[graph_name] = lines

def main():
  args = parser.parse_args()
  graph_data = []
  for f in args.graph_files:
    graph_data += ast.literal_eval(f.read())

  graph_dict = {}
  for input_files in split_data(graph_data, 'input-file'):
    for layer_pattern in split_data(input_files, 'layer-pattern'):
      for data in layer_pattern:
        metrics = [
          'vpx-ssim',
          'ssim',
          'ssim-y',
          'ssim-u',
          'ssim-v',
          'avg-psnr',
          'avg-psnr-y',
          'avg-psnr-u',
          'avg-psnr-v',
          'glb-psnr',
          'glb-psnr-y',
          'glb-psnr-u',
          'glb-psnr-v',
          'encode-time-utilization',
        ]
        for metric in metrics:
          generate_graphs(graph_dict, layer_pattern, metric, normalize_bitrate_config_string(data['bitrate-config-kbps']))

  for point in graph_data:
    line_name = '%s-%s' % (point['encoder'], point['codec'])
    pattern_match = layer_regex_pattern.match(point['layer-pattern'])
    num_temporal_layers = int(pattern_match.group(2))
    temporal_divide = 2 ** (num_temporal_layers - 1 - point['temporal-layer'])
    frame_metrics = [
      'frame-ssim',
      'frame-ssim-y',
      'frame-ssim-u',
      'frame-ssim-v',
      'frame-psnr',
      'frame-psnr-y',
      'frame-psnr-u',
      'frame-psnr-v',
      'frame-qp',
    ]
    for target_metric in frame_metrics:
      # TODO(pbos): Add frame QP to AV1 decoding.
      if target_metric == 'frame-qp' and not 'frame-qp' in point:
        continue
      split_on_codecs = target_metric == 'frame-qp'

      if split_on_codecs:
        graph_name = "%s-%s-%s-%dkbps-tl%d-%s:%s" % (point['input-file'], point['layer-pattern'], normalize_bitrate_config_string(point['bitrate-config-kbps']), point['bitrate-config-kbps'][-1], point['temporal-layer'], point['codec'], target_metric)
        line_name = '%s' % point['encoder']
      else:
        graph_name = "%s-%s-%s-%dkbps-tl%d:%s" % (point['input-file'], point['layer-pattern'], normalize_bitrate_config_string(point['bitrate-config-kbps']), point['bitrate-config-kbps'][-1], point['temporal-layer'], target_metric)
      if not graph_name in graph_dict:
        graph_dict[graph_name] = {}
      line = []
      for idx, val in enumerate(point[target_metric]):
        if 'frame-bytes' in point:
          line.append((temporal_divide * idx + 1, val, point['frame-bytes'][idx]))
        else:
          # TODO(pbos): Add frame size to AV1 decoding.
          line.append((temporal_divide * idx + 1, val, 0))
      graph_dict[graph_name][line_name] = line

  current_graph = 1
  total_graphs = len(graph_dict)
  for graph_name, lines in graph_dict.iteritems():
    print "[%d/%d] %s" % (current_graph, total_graphs, graph_name)
    current_graph += 1
    metric = graph_name.split(':')[-1]
    fig, ax = plt.subplots()
    ax.set_title(graph_name)
    frame_data = 'frame-' in metric
    ax2 = None
    ax2_bitrate_utilization = False
    linestyle = 'o--'
    ax2_linestyle = 'x-'

    if frame_data:
      ax.set_xlabel('Frame')
      ax.set_ylabel(metric.replace('frame-', '').upper())
      ax2 = ax.twinx()
      ax2.set_ylabel('Frame Size (bytes / frame)')
      linestyle = '-'
      ax2_linestyle = '-'
    elif metric == 'encode-time-utilization':
      ax.set_xlabel('Layer Target Bitrate (kbps)')
      ax.set_ylabel('Encode Time (fraction)')
      # Draw a reference line for realtime.
      ax.axhline(1.0, color='k', alpha=0.2, linestyle='--')
    else:
      ax.set_xlabel('Layer Target Bitrate (kbps)')
      ax.set_ylabel(metric.upper())
      ax2 = ax.twinx()
      ax2.set_ylabel('Bitrate Utilization (actual / target)')
      ax2_bitrate_utilization = True

    for title in sorted(lines.keys()):
      points = lines[title]
      x = []
      y = []
      y2 = []
      for bitrate_kbps, value, utilization in points:
          x.append(bitrate_kbps)
          y.append(value)
          y2.append(utilization)
      ax.plot(x, y, linestyle, linewidth=1, label=title)
      if ax2:
        ax2.plot(x, y2, ax2_linestyle, alpha=0.2)
      ax.legend(loc='best', fancybox=True, framealpha=0.5)

    if metric == 'encode-time-utilization':
      # Make sure the horizontal reference line at 1.0 can be seen.
      (lower, upper) = ax.get_ylim()
      if upper < 1.10:
        ax.set_ylim(top=1.10)

    if ax2_bitrate_utilization:
      # Set bitrate limit axes to +/- 20%.
      ax2.set_ylim(bottom=0.80, top=1.20)

    for extension in args.formats:
      plt.savefig(os.path.join(args.out_dir, "%s.%s" % (graph_name.replace(":", "-"), extension)))
    plt.close()

if __name__ == '__main__':
  main()

<|endoftext|>


<commit_before>import json
import re
from typing import Callable
from typing import Dict
from typing import List
from typing import Union

import numpy as np
import tensorflow as tf
from packaging.version import parse
from tensorflow.keras.losses import Loss
from tensorflow.keras.metrics import Metric
from tensorflow.python.util import nest

# some types
AcceptableLoss = Union[str, Callable, Loss, None]
AcceptableMetric = Union[str, Callable, Metric, None]
AcceptableMetrics = Union[List[AcceptableMetric],
                          List[List[AcceptableMetric]],
                          Dict[str, AcceptableMetric],
                          None]

def get_global_average_pooling(shape):
    return [tf.keras.layers.GlobalAveragePooling1D,
            tf.keras.layers.GlobalAveragePooling2D,
            tf.keras.layers.GlobalAveragePooling3D][len(shape) - 3]

def get_global_max_pooling(shape):
    return [tf.keras.layers.GlobalMaxPool1D,
            tf.keras.layers.GlobalMaxPool2D,
            tf.keras.layers.GlobalMaxPool3D][len(shape) - 3]

def get_max_pooling(shape):
    return [tf.keras.layers.MaxPool1D,
            tf.keras.layers.MaxPool2D,
            tf.keras.layers.MaxPool3D][len(shape) - 3]

def get_conv(shape):
    return [tf.keras.layers.Conv1D,
            tf.keras.layers.Conv2D,
            tf.keras.layers.Conv3D][len(shape) - 3]

def get_sep_conv(shape):
    return [tf.keras.layers.SeparableConv1D,
            tf.keras.layers.SeparableConv2D,
            tf.keras.layers.Conv3D][len(shape) - 3]

def get_dropout(shape):
    return [tf.keras.layers.SpatialDropout1D,
            tf.keras.layers.SpatialDropout2D,
            tf.keras.layers.SpatialDropout3D][len(shape) - 3]

def validate_num_inputs(inputs, num):
    inputs = nest.flatten(inputs)
    if not len(inputs) == num:
        raise ValueError('Expected {num} elements in the inputs list '
                         'but received {len} inputs.'.format(num=num,
                                                             len=len(inputs)))

def split_dataset(dataset, validation_split):
    """Split dataset into training and validation.

    # Arguments
        dataset: tf.data.Dataset. The entire dataset to be split.
        validation_split: Float. The split ratio for the validation set.

    # Raises
        ValueError: If the dataset provided is too small to be split.

    # Returns
        A tuple of two tf.data.Dataset. The training set and the validation set.
    """
    num_instances = dataset.reduce(np.int64(0), lambda x, _: x + 1).numpy()
    if num_instances < 2:
        raise ValueError('The dataset should at least contain 2 '
                         'instances to be split.')
    validation_set_size = min(
        max(int(num_instances * validation_split), 1),
        num_instances - 1)
    train_set_size = num_instances - validation_set_size
    train_dataset = dataset.take(train_set_size)
    validation_dataset = dataset.skip(train_set_size)
    return train_dataset, validation_dataset

def get_name_scope():
    with tf.name_scope('a') as scope:
        name_scope = scope[:-2]
    return name_scope

def dataset_shape(dataset):
    return tf.compat.v1.data.get_output_shapes(dataset)

def is_label(y):
    """Check if the targets are one-hot encoded or plain labels.

    # Arguments
        y: numpy.ndarray. The targets.

    # Returns
        Boolean. Whether the targets are plain label, not encoded.
    """
    return len(y.flatten()) == len(y)

def to_snake_case(name):
    intermediate = re.sub('(.)([A-Z][a-z0-9]+)', r'\1_\2', name)
    insecure = re.sub('([a-z])([A-Z])', r'\1_\2', intermediate).lower()
    # If the class is private the name starts with "_" which is not secure
    # for creating scopes. We prefix the name with "private" in this case.
    if insecure[0] != '_':
        return insecure
    return 'private' + insecure

def to_type_key(dictionary, convert_func):
    """Convert the keys of a dictionary to a different type.

    # Arguments
        dictionary: Dictionary. The dictionary to be converted.
        convert_func: Function. The function to convert a key.
    """
    return {convert_func(key): value
            for key, value in dictionary.items()}

def check_tf_version() -> None:
    if parse(tf.__version__) < parse('2.1.0'):
        raise ImportError(
            'The Tensorflow package version needs to be at least v2.1.0 \n'
            'for AutoKeras to run. Currently, your TensorFlow version is \n'
            'v{version}. Please upgrade with \n'
            '`$ pip install --upgrade tensorflow` -> GPU version \n'
            'or \n'
            '`$ pip install --upgrade tensorflow-cpu` -> CPU version. \n'
            'You can use `pip freeze` to check afterwards that everything is '
            'ok.'.format(version=tf.__version__)
        )

def save_json(path, obj):
    obj = json.dumps(obj)
    with tf.io.gfile.GFile(path, 'w') as f:
        f.write(obj)

def load_json(path):
    with tf.io.gfile.GFile(path, 'r') as f:
        obj = f.read()
    return json.loads(obj)
<commit_msg>Update utils.py (#1029)<commit_after>import json
import re
from typing import Callable
from typing import Dict
from typing import List
from typing import Union

import numpy as np
import tensorflow as tf
from packaging.version import parse
from tensorflow.keras.losses import Loss
from tensorflow.keras.metrics import Metric
from tensorflow.python.util import nest

# some types
AcceptableLoss = Union[str, Callable, Loss]
AcceptableMetric = Union[str, Callable, Metric]
AcceptableMetrics = Union[List[AcceptableMetric],
                          List[List[AcceptableMetric]],
                          Dict[str, AcceptableMetric],
                          None]

def get_global_average_pooling(shape):
    return [tf.keras.layers.GlobalAveragePooling1D,
            tf.keras.layers.GlobalAveragePooling2D,
            tf.keras.layers.GlobalAveragePooling3D][len(shape) - 3]

def get_global_max_pooling(shape):
    return [tf.keras.layers.GlobalMaxPool1D,
            tf.keras.layers.GlobalMaxPool2D,
            tf.keras.layers.GlobalMaxPool3D][len(shape) - 3]

def get_max_pooling(shape):
    return [tf.keras.layers.MaxPool1D,
            tf.keras.layers.MaxPool2D,
            tf.keras.layers.MaxPool3D][len(shape) - 3]

def get_conv(shape):
    return [tf.keras.layers.Conv1D,
            tf.keras.layers.Conv2D,
            tf.keras.layers.Conv3D][len(shape) - 3]

def get_sep_conv(shape):
    return [tf.keras.layers.SeparableConv1D,
            tf.keras.layers.SeparableConv2D,
            tf.keras.layers.Conv3D][len(shape) - 3]

def get_dropout(shape):
    return [tf.keras.layers.SpatialDropout1D,
            tf.keras.layers.SpatialDropout2D,
            tf.keras.layers.SpatialDropout3D][len(shape) - 3]

def validate_num_inputs(inputs, num):
    inputs = nest.flatten(inputs)
    if not len(inputs) == num:
        raise ValueError('Expected {num} elements in the inputs list '
                         'but received {len} inputs.'.format(num=num,
                                                             len=len(inputs)))

def split_dataset(dataset, validation_split):
    """Split dataset into training and validation.

    # Arguments
        dataset: tf.data.Dataset. The entire dataset to be split.
        validation_split: Float. The split ratio for the validation set.

    # Raises
        ValueError: If the dataset provided is too small to be split.

    # Returns
        A tuple of two tf.data.Dataset. The training set and the validation set.
    """
    num_instances = dataset.reduce(np.int64(0), lambda x, _: x + 1).numpy()
    if num_instances < 2:
        raise ValueError('The dataset should at least contain 2 '
                         'instances to be split.')
    validation_set_size = min(
        max(int(num_instances * validation_split), 1),
        num_instances - 1)
    train_set_size = num_instances - validation_set_size
    train_dataset = dataset.take(train_set_size)
    validation_dataset = dataset.skip(train_set_size)
    return train_dataset, validation_dataset

def get_name_scope():
    with tf.name_scope('a') as scope:
        name_scope = scope[:-2]
    return name_scope

def dataset_shape(dataset):
    return tf.compat.v1.data.get_output_shapes(dataset)

def is_label(y):
    """Check if the targets are one-hot encoded or plain labels.

    # Arguments
        y: numpy.ndarray. The targets.

    # Returns
        Boolean. Whether the targets are plain label, not encoded.
    """
    return len(y.flatten()) == len(y)

def to_snake_case(name):
    intermediate = re.sub('(.)([A-Z][a-z0-9]+)', r'\1_\2', name)
    insecure = re.sub('([a-z])([A-Z])', r'\1_\2', intermediate).lower()
    # If the class is private the name starts with "_" which is not secure
    # for creating scopes. We prefix the name with "private" in this case.
    if insecure[0] != '_':
        return insecure
    return 'private' + insecure

def to_type_key(dictionary, convert_func):
    """Convert the keys of a dictionary to a different type.

    # Arguments
        dictionary: Dictionary. The dictionary to be converted.
        convert_func: Function. The function to convert a key.
    """
    return {convert_func(key): value
            for key, value in dictionary.items()}

def check_tf_version() -> None:
    if parse(tf.__version__) < parse('2.1.0'):
        raise ImportError(
            'The Tensorflow package version needs to be at least v2.1.0 \n'
            'for AutoKeras to run. Currently, your TensorFlow version is \n'
            'v{version}. Please upgrade with \n'
            '`$ pip install --upgrade tensorflow` -> GPU version \n'
            'or \n'
            '`$ pip install --upgrade tensorflow-cpu` -> CPU version. \n'
            'You can use `pip freeze` to check afterwards that everything is '
            'ok.'.format(version=tf.__version__)
        )

def save_json(path, obj):
    obj = json.dumps(obj)
    with tf.io.gfile.GFile(path, 'w') as f:
        f.write(obj)

def load_json(path):
    with tf.io.gfile.GFile(path, 'r') as f:
        obj = f.read()
    return json.loads(obj)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#    Copyright (C) 2014 The Patacrep Team
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""
Functions for song file (.sg) management.
"""
from django.utils.text import slugify
from django.utils.encoding import smart_text
from django.conf.global_settings import LANGUAGES

from patacrep.plastex import parsetex, SongParser

from generator.models import Song, Artist

import re
import os
import sys
import pprint
import logging

from contextlib import contextmanager

LOGGER = logging.getLogger(__name__)

class Renderer(object):
    """Render a PlasTeX-parsed song as HTML"""

    def __init__(self, document):
        self.document = document
        self._render = {
                '#text': self.render_text,
                'verse': self.render_verse,
                'verse*': self.render_verse,
                'chorus': self.render_verse,
                'bridge': self.render_verse,
                'par': self.render_par,
                'chord': self.render_chord,
                'active::\n': self.render_plain_text(u"
"),
                'dots': self.render_plain_text(u"…"),
                'selectlanguage': self.render_silent,
                'songcolumns': self.render_silent,
                'beginsong': self.render_silent,
                'endsong': self.render_silent,
                'cover': self.render_silent,
                'gtab': self.render_gtab,
                }
        self._render_text = {}

    @contextmanager
    def push(self, attr, extension):
        """With statement used to locally update a dictonary

        Arguments:
        - attr: name (as a string) of the attribute to update. `self.attr`
          must be a dictonary.
        - extension: dictionary to use to update the dictonary.
        """
        old = dict(getattr(self, attr))
        getattr(self, attr).update(extension)
        yield
        setattr(self, attr, old)

    @staticmethod
    def render_plain_text(string):
        """Return a `render_*`-like function that return a constant string."""
        def _render_plain_text(__node):
            """Return constant string."""
            return string
        return _render_plain_text

    def render_nodes(self, nodes):
        """Render a list of nodes"""
        return "".join([
            self._render.get(
                node.nodeName,
                self.render_default
                )(node)
            for node in nodes
            ])

    @staticmethod
    def render_default(node):
        """Default rendering of a node"""
        print("TODO Default for", unicode(node), node.nodeName)
        return u""

    @staticmethod
    def render_silent(_):
        """Return nothing

        To be used by LaTeX commands that does not produce any output.
        """
        return u""

    @staticmethod
    def render_gtab(node):
        r"""Render LaTeX `\gtab` command."""
        print "TODO GTAB"
        return u"GTAB({chord}, {diagram})".format(
                diagram=node.attributes["diagram"],
                chord=node.attributes["chord"],
                )

    def render_text(self, node):
        """Render a text node.

        If `unicode(node)` is a key of `self._render_text`, call the
        corresponding method. Otherwise, return unicode(node).
        """
        return self._render_text.get(
                unicode(node),
                self.render_plain_text(unicode(node)),
                )(node)

    def render_verse(self, node):
        """Render a `verse`, `verse*` or `chorus` environment."""
        res = ""
        res += "
\n".format(node.nodeName.replace('*', '_star'))
        res += self.render_nodes(node.childNodes)
        res += "
"
        return res

    @staticmethod
    def render_par(__node):
        """Render a paragraph."""
        # TODO
        return ""

    def render_chord(self, node):
        r"""Render a chord command `\[`."""
        with self.push("_render", {'active::&': self.render_plain_text(u"♭")}):
            with self.push("_render_text", {'#': self.render_plain_text(u"♯")}):
                return u"""

                         {}

                         {}

                         """.format(
                                 self.render_nodes(node.childNodes),
                                 "", # TODO
                                 )

def parse_song(filename):
    """Parse song 'filename', and return the corresponding HTML code."""
    filename = filename.replace("../patacrep/data/examples/songs/", "")
    tex = SongParser.parse(filename)
    return Renderer(tex).render_nodes(tex.childNodes)

def import_song(repo, filepath):
    '''Import a song in the database'''
    data = parsetex(filepath)
    LOGGER.info("Processing " +
                pprint.pformat(data['titles'][0]))

    artist_name = smart_text(data['args']['by'], 'utf-8')
    artist_slug = slugify(artist_name)

    artist_model, created = Artist.objects.get_or_create(
                            slug=artist_slug,
                            defaults={'name': artist_name}
                            )
    if not created:
        if (artist_model.name != artist_name):
            LOGGER.warning(
                "*** Artist name differs though "
                "slugs are equal : "
                + artist_name + " / "
                + artist_model.name)

    if (len(data['languages']) > 1):
        LOGGER.warning("*** Multiple languages "
                        "in song file; we though"
                        " only support one. "
                        "Picking any.")
    if (len(data['languages']) > 0):
        language_name = data["languages"].pop()
        language_code = next(
                    (x for x in LANGUAGES
                     if x[1].lower() == language_name.lower()
                     ),
                    ('', '')
                     )[0]
        if language_code == '':
            LOGGER.warning("*** No code found for "
                    "language : '" + language_name + "'")

    song_title = smart_text(data['titles'][0], 'utf-8')
    song_slug = slugify(song_title)

    object_hash = repo.git.hash_object(filepath)
    SONG_DIR = os.path.join(repo.working_dir, "songs")
    filepath_rel = os.path.relpath(filepath, SONG_DIR)

    import random

    # For some reason - probably after having interrupted
    # the generation - insertion fails because slug is
    # empty, and there is already an empty one.
    # We assign here a random value, that gets overwritten
    # afterwards.
    song_model, created = Song.objects.get_or_create(
                            title=song_title,
                            artist=artist_model,
                            defaults={
                                'title': song_title,
                                'language': language_code,
                                'file_path': filepath_rel,
                                'slug': ('%06x' % random.randrange(16**6))
                            })
    if created:
        if Song.objects.filter(slug=song_slug).exists():
            song_slug += '-' + str(song_model.id)
        song_model.slug = song_slug

    else:
        LOGGER.info("-> Already exists.")

    artist_model.save()
    song_model.object_hash = object_hash
    song_model.save()
<commit_msg>Amélioration du rendu dans les cas non gérés<commit_after># -*- coding: utf-8 -*-
#    Copyright (C) 2014 The Patacrep Team
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""
Functions for song file (.sg) management.
"""
from django.utils.text import slugify
from django.utils.encoding import smart_text
from django.conf.global_settings import LANGUAGES

from patacrep.plastex import parsetex, SongParser

from generator.models import Song, Artist

import re
import os
import sys
import pprint
import logging

from contextlib import contextmanager

LOGGER = logging.getLogger(__name__)

class Renderer(object):
    """Render a PlasTeX-parsed song as HTML"""

    def __init__(self, document):
        self.document = document
        self._render = {
                '#text': self.render_text,
                'verse': self.render_verse,
                'verse*': self.render_verse,
                'chorus': self.render_verse,
                'bridge': self.render_verse,
                'par': self.render_par,
                'chord': self.render_chord,
                'active::\n': self.render_plain_text(u"
"),
                'dots': self.render_plain_text(u"…"),
                'selectlanguage': self.render_silent,
                'songcolumns': self.render_silent,
                'beginsong': self.render_silent,
                'endsong': self.render_silent,
                'cover': self.render_silent,
                'gtab': self.render_gtab,
                'bgroup': self.render_group,
                }
        self._render_text = {}

    @contextmanager
    def push(self, attr, extension):
        """With statement used to locally update a dictonary

        Arguments:
        - attr: name (as a string) of the attribute to update. `self.attr`
          must be a dictonary.
        - extension: dictionary to use to update the dictonary.
        """
        old = dict(getattr(self, attr))
        getattr(self, attr).update(extension)
        yield
        setattr(self, attr, old)

    @staticmethod
    def render_plain_text(string):
        """Return a `render_*`-like function that return a constant string."""
        def _render_plain_text(__node):
            """Return constant string."""
            return string
        return _render_plain_text

    def render_nodes(self, nodes):
        """Render a list of nodes"""
        return "".join([
            self._render.get(
                node.nodeName,
                self.render_default
                )(node)
            for node in nodes
            ])

    @staticmethod
    def render_default(node):
        """Default rendering of a node"""
        print("TODO Default for", unicode(node), node.nodeName)
        return node.textContent

    @staticmethod
    def render_silent(_):
        """Return nothing

        To be used by LaTeX commands that does not produce any output.
        """
        return u""

    def render_group(self, node):
        return """\n{}\n""".format(self.render_nodes(node.childNodes))

    @staticmethod
    def render_gtab(node):
        r"""Render LaTeX `\gtab` command."""
        print "TODO GTAB"
        return u"GTAB({chord}, {diagram})".format(
                diagram=node.attributes["diagram"],
                chord=node.attributes["chord"],
                )

    def render_text(self, node):
        """Render a text node.

        If `unicode(node)` is a key of `self._render_text`, call the
        corresponding method. Otherwise, return unicode(node).
        """
        return self._render_text.get(
                unicode(node),
                self.render_plain_text(unicode(node)),
                )(node)

    def render_verse(self, node):
        """Render a `verse`, `verse*` or `chorus` environment."""
        res = ""
        res += "
\n".format(node.nodeName.replace('*', '_star'))
        res += self.render_nodes(node.childNodes)
        res += "
"
        return res

    @staticmethod
    def render_par(__node):
        """Render a paragraph."""
        # TODO
        return ""

    def render_chord(self, node):
        r"""Render a chord command `\[`."""
        with self.push("_render", {'active::&': self.render_plain_text(u"♭")}):
            with self.push("_render_text", {'#': self.render_plain_text(u"♯")}):
                return u"""

                         {}

                         {}

                         """.format(
                                 self.render_nodes(node.childNodes),
                                 "", # TODO
                                 )

def parse_song(filename):
    """Parse song 'filename', and return the corresponding HTML code."""
    filename = filename.replace("../patacrep/data/examples/songs/", "")
    tex = SongParser.parse(filename)
    return Renderer(tex).render_nodes(tex.childNodes)

def import_song(repo, filepath):
    '''Import a song in the database'''
    data = parsetex(filepath)
    LOGGER.info("Processing " +
                pprint.pformat(data['titles'][0]))

    artist_name = smart_text(data['args']['by'], 'utf-8')
    artist_slug = slugify(artist_name)

    artist_model, created = Artist.objects.get_or_create(
                            slug=artist_slug,
                            defaults={'name': artist_name}
                            )
    if not created:
        if (artist_model.name != artist_name):
            LOGGER.warning(
                "*** Artist name differs though "
                "slugs are equal : "
                + artist_name + " / "
                + artist_model.name)

    if (len(data['languages']) > 1):
        LOGGER.warning("*** Multiple languages "
                        "in song file; we though"
                        " only support one. "
                        "Picking any.")
    if (len(data['languages']) > 0):
        language_name = data["languages"].pop()
        language_code = next(
                    (x for x in LANGUAGES
                     if x[1].lower() == language_name.lower()
                     ),
                    ('', '')
                     )[0]
        if language_code == '':
            LOGGER.warning("*** No code found for "
                    "language : '" + language_name + "'")

    song_title = smart_text(data['titles'][0], 'utf-8')
    song_slug = slugify(song_title)

    object_hash = repo.git.hash_object(filepath)
    SONG_DIR = os.path.join(repo.working_dir, "songs")
    filepath_rel = os.path.relpath(filepath, SONG_DIR)

    import random

    # For some reason - probably after having interrupted
    # the generation - insertion fails because slug is
    # empty, and there is already an empty one.
    # We assign here a random value, that gets overwritten
    # afterwards.
    song_model, created = Song.objects.get_or_create(
                            title=song_title,
                            artist=artist_model,
                            defaults={
                                'title': song_title,
                                'language': language_code,
                                'file_path': filepath_rel,
                                'slug': ('%06x' % random.randrange(16**6))
                            })
    if created:
        if Song.objects.filter(slug=song_slug).exists():
            song_slug += '-' + str(song_model.id)
        song_model.slug = song_slug

    else:
        LOGGER.info("-> Already exists.")

    artist_model.save()
    song_model.object_hash = object_hash
    song_model.save()
<|endoftext|>