<commit_before>"""\
Also run this when the links or keys are changed in the schema.

Example:

    %(prog)s production.ini --app-name app

"""
import logging
import transaction

from copy import deepcopy
from itertools import groupby

from pyramid import paster
from pyramid.traversal import find_resource
from pyramid.view import view_config

from snovault import (
    CONNECTION,
    STORAGE,
    UPGRADER,
)
from snovault.schema_utils import validate

BATCH_UPGRADE_LOG = logging.getLogger('snovault.batchupgrade')
EPILOG = __doc__
testapp = None

def includeme(config):
    config.add_route('batch_upgrade', '/batch_upgrade')
    config.scan(__name__)

def update_item(storage, context):
    target_version = context.type_info.schema_version
    current_version = context.properties.get('schema_version', '')
    update = False
    errors = []
    properties = context.properties
    if target_version is None or current_version == target_version:
        unique_keys = context.unique_keys(properties)
        links = context.links(properties)
        keys_add, keys_remove = storage._update_keys(context.model, unique_keys)
        if keys_add or keys_remove:
            update = True
        rels_add, rels_remove = storage._update_rels(context.model, links)
        if rels_add or rels_remove:
            update = True
    else:
        properties = deepcopy(properties)
        upgrader = context.registry[UPGRADER]
        properties = upgrader.upgrade(
            context.type_info.name, properties, current_version, target_version,
            context=context, registry=context.registry)
        if 'schema_version' in properties:
            del properties['schema_version']
        schema = context.type_info.schema
        properties['uuid'] = str(context.uuid)
        validated, errors = validate(schema, properties, properties)
        # Do not send modification events to skip indexing
        context.update(validated)
        update = True
    return update, errors

@view_config(route_name='batch_upgrade', request_method='POST', permission='import_items')
def batch_upgrade(request):
    request.datastore = 'database'
    transaction.get().setExtendedInfo('upgrade', True)
    batch = request.json['batch']
    root = request.root
    storage = request.registry[STORAGE].write
    session = storage.DBSession()
    results = []
    for uuid in batch:
        item_type = None
        update = False
        error = False
        sp = session.begin_nested()
        try:
            item = find_resource(root, uuid)
            item_type = item.type_info.item_type
            update, errors = update_item(storage, item)
        except Exception as e:
            BATCH_UPGRADE_LOG.error('Error %s updating: /%s/%s' % (e, item_type, uuid))
            sp.rollback()
            error = True
        else:
            if errors:
                # redmine 5161 sometimes error.path has an int
                errortext = [
                    '%s: %s' % ('/'.join([str(x) or '<root>' for x in error.path]), error.message)
                    for error in errors
                ]
                BATCH_UPGRADE_LOG.error(
                    'Validation failure: /%s/%s\n%s', item_type, uuid, '\n'.join(errortext)
                )
                sp.rollback()
                error = True
            else:
                sp.commit()
        results.append((item_type, uuid, update, error))
    return {'results': results}

def _pool_batch_results(iterable, n=1):
    l = len(iterable)
    for ndx in range(0, l, n):
        yield iterable[ndx:min(ndx+n, l)]

def _pool_initializer(*args, **kw):
    global testapp
    testapp = _internal_app(*args, **kw)

def _pool_worker(batch):
    res = testapp.post_json('/batch_upgrade', {'batch': batch})
    return res.json

def _run_pool(uuids, args):
    from multiprocessing import get_context
    from multiprocessing.pool import Pool
    transaction.abort()
    pool = Pool(
        processes=args.processes,
        initializer=_pool_initializer,
        initargs=(args.config_uri, args.app_name, args.username),
        context=get_context('forkserver'),
    )
    all_results = []
    try:
        for result in pool.imap_unordered(
            _pool_worker,
            _pool_batch_results(uuids, args.batchsize),
            chunksize=args.chunksize,
        ):
            results = result['results']
            errors = sum(error for item_type, path, update, error in results)
            updated = sum(update for item_type, path, update, error in results)
            BATCH_UPGRADE_LOG.info(
                'Batch: Updated %d of %d (errors %d)' % (
                    updated, len(results), errors
                )
            )
            all_results.extend(results)
    finally:
        pool.terminate()
        pool.join()
    return all_results

def _summarize_results(all_results, verbose=False):

    def result_item_type(result):
        return result[0] or ''

    error_logs = []
    updated_logs = []
    all_logs = []
    for item_type, results in groupby(
            sorted(all_results, key=result_item_type), key=result_item_type):
        results = list(results)
        errors = sum(error for item_type, path, update, error in results)
        updated = sum(update for item_type, path, update, error in results)
        log_message = 'Collection {}: Updated {} of {} (errors {})' .format(
            item_type,
            updated,
            len(results),
            errors,
        )
        if errors:
            errors_logs.append(log_message)
        if updated:
            updated_logs.append(log_message)
        all_logs.append(log_message)
    BATCH_UPGRADE_LOG.info('Upgrade Summary')
    if verbose:
        for log_msg in all_logs:
            BATCH_UPGRADE_LOG.info(log_msg)
    BATCH_UPGRADE_LOG.info('Sum updated: %d' % len(updated_logs))
    if updated_logs:
        for log_msg in updated_logs:
            BATCH_UPGRADE_LOG.info(log_msg)
    BATCH_UPGRADE_LOG.info('Sum errors: %d' % len(error_logs))
    if error_logs:
        for log_msg in error_logs:
            BATCH_UPGRADE_LOG.error(log_msg)

def _internal_app(configfile, app_name=None, username=None):
    from webtest import TestApp
    from pyramid import paster
    app = paster.get_app(configfile, app_name)
    if not username:
        username = 'UPGRADE'
    environ = {
        'HTTP_ACCEPT': 'application/json',
        'REMOTE_USER': username,
    }
    return TestApp(app, environ)

def main():
    args = _parse_args()
    # Setup Logger
    paster.setup_logging(args.config_uri)
    logging.getLogger('snovault').setLevel(logging.INFO)
    # Get Uuids
    testapp = _internal_app(args.config_uri, app_name=args.app_name, username=args.username)
    connection = testapp.app.registry[CONNECTION]
    uuids = [str(uuid) for uuid in connection.__iter__(*args.item_types)]
    if uuids:
        BATCH_UPGRADE_LOG.info('Start Upgrade with %d items' % (len(uuids)))
        all_results = _run_pool(uuids, args)
        BATCH_UPGRADE_LOG.info('End Upgrade')
        _summarize_results(all_results, verbose=args.verbose)
    else:
        BATCH_UPGRADE_LOG.warning('No uuids to upgrade.')

def _parse_args():
    import argparse
    parser = argparse.ArgumentParser(
        description="Batch upgrade content items.", epilog=EPILOG,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument('config_uri', help="path to configfile")
    parser.add_argument('--app-name', help="Pyramid app name in configfile")
    parser.add_argument('--batchsize', type=int, default=50)
    parser.add_argument('--chunksize', type=int, default=1)
    parser.add_argument('--item-types', action='append', default=[])
    parser.add_argument('--processes', type=int, default=2)
    parser.add_argument('--username')
    parser.add_argument('-v', '--verbose', action='store_true')
    args = parser.parse_args()
    return args

if __name__ == '__main__':
    main()
<commit_msg>SNO-100 Parallelize batchupgrade script<commit_after>"""\
Also run this when the links or keys are changed in the schema.

Example:

    %(prog)s production.ini --app-name app

"""
import logging
import time
import transaction

from copy import deepcopy
from itertools import groupby

from pyramid import paster
from pyramid.traversal import find_resource
from pyramid.view import view_config

from snovault import (
    CONNECTION,
    STORAGE,
    UPGRADER,
)
from snovault.schema_utils import validate

BATCH_UPGRADE_LOG = logging.getLogger('snovault.batchupgrade')
EPILOG = __doc__
testapp = None

def includeme(config):
    config.add_route('batch_upgrade', '/batch_upgrade')
    config.scan(__name__)

def update_item(storage, context):
    target_version = context.type_info.schema_version
    current_version = context.properties.get('schema_version', '')
    update = False
    errors = []
    properties = context.properties
    if target_version is None or current_version == target_version:
        unique_keys = context.unique_keys(properties)
        links = context.links(properties)
        keys_add, keys_remove = storage._update_keys(context.model, unique_keys)
        if keys_add or keys_remove:
            update = True
        rels_add, rels_remove = storage._update_rels(context.model, links)
        if rels_add or rels_remove:
            update = True
    else:
        properties = deepcopy(properties)
        upgrader = context.registry[UPGRADER]
        properties = upgrader.upgrade(
            context.type_info.name, properties, current_version, target_version,
            context=context, registry=context.registry)
        if 'schema_version' in properties:
            del properties['schema_version']
        schema = context.type_info.schema
        properties['uuid'] = str(context.uuid)
        validated, errors = validate(schema, properties, properties)
        # Do not send modification events to skip indexing
        context.update(validated)
        update = True
    return update, errors

@view_config(route_name='batch_upgrade', request_method='POST', permission='import_items')
def batch_upgrade(request):
    request.datastore = 'database'
    transaction.get().setExtendedInfo('upgrade', True)
    batch = request.json['batch']
    root = request.root
    storage = request.registry[STORAGE].write
    session = storage.DBSession()
    results = []
    for uuid in batch:
        item_type = None
        update = False
        error = False
        sp = session.begin_nested()
        try:
            item = find_resource(root, uuid)
            item_type = item.type_info.item_type
            update, errors = update_item(storage, item)
        except Exception as e:
            BATCH_UPGRADE_LOG.error('Error %s updating: /%s/%s' % (e, item_type, uuid))
            sp.rollback()
            error = True
        else:
            if errors:
                # redmine 5161 sometimes error.path has an int
                errortext = [
                    '%s: %s' % ('/'.join([str(x) or '<root>' for x in error.path]), error.message)
                    for error in errors
                ]
                BATCH_UPGRADE_LOG.error(
                    'Validation failure: /%s/%s\n%s', item_type, uuid, '\n'.join(errortext)
                )
                sp.rollback()
                error = True
            else:
                sp.commit()
        results.append((item_type, uuid, update, error))
    return {'results': results}

def _pool_batch_results(iterable, n=1):
    l = len(iterable)
    for ndx in range(0, l, n):
        yield iterable[ndx:min(ndx+n, l)]

def _pool_initializer(*args, **kw):
    global testapp
    testapp = _internal_app(*args, **kw)

def _pool_worker(batch):
    res = testapp.post_json('/batch_upgrade', {'batch': batch})
    return res.json

def _pool_chunksize(uuid_count, chunksize, processes):
    chunkiness = int((uuid_count - 1) / processes) + 1
    if chunkiness > chunksize:
        return chunksize
    return chunkiness

def _run_pool(uuids, args):
    from multiprocessing import get_context
    from multiprocessing.pool import Pool
    transaction.abort()
    pool = Pool(
        processes=args.processes,
        initializer=_pool_initializer,
        initargs=(args.config_uri, args.app_name, args.username),
        context=get_context('forkserver'),
        maxtasksperchild=args.maxtasksperchild,
    )
    est_loops = int(len(uuids) / args.batchsize)
    all_results = []
    try:
        pool_gen = pool.imap_unordered(
            _pool_worker,
            _pool_batch_results(uuids, args.batchsize),
            chunksize=args.chunksize,
        )
        for loop, result in enumerate(pool_gen, 1):
            results = result['results']
            errors = sum(error for item_type, path, update, error in results)
            updated = sum(update for item_type, path, update, error in results)
            log_msg = "{} of ~{} Batch: Updated {} of {} (errors {})".format(
                loop,
                est_loops,
                updated,
                len(results),
                errors,
            )
            BATCH_UPGRADE_LOG.info(log_msg)
            all_results.extend(results)
    finally:
        pool.terminate()
        pool.join()
    return all_results

def _summarize_results(all_results, runtime_str=None, verbose=False):

    def result_item_type(result):
        return result[0] or ''

    error_logs = []
    updated_logs = []
    all_logs = []
    for item_type, results in groupby(
            sorted(all_results, key=result_item_type), key=result_item_type):
        results = list(results)
        errors = sum(error for item_type, path, update, error in results)
        updated = sum(update for item_type, path, update, error in results)
        log_message = 'Collection {}: Updated {} of {} (errors {})' .format(
            item_type,
            updated,
            len(results),
            errors,
        )
        if errors:
            errors_logs.append(log_message)
        if updated:
            updated_logs.append(log_message)
        all_logs.append(log_message)
    BATCH_UPGRADE_LOG.info('Upgrade Summary')
    if verbose:
        for log_msg in all_logs:
            BATCH_UPGRADE_LOG.info(log_msg)
    BATCH_UPGRADE_LOG.info('Sum updated: %d' % len(updated_logs))
    if updated_logs:
        for log_msg in updated_logs:
            BATCH_UPGRADE_LOG.info(log_msg)
    BATCH_UPGRADE_LOG.info('Sum errors: %d' % len(error_logs))
    if error_logs:
        for log_msg in error_logs:
            BATCH_UPGRADE_LOG.error(log_msg)
    if runtime_str:
        BATCH_UPGRADE_LOG.info("Run Time: %s" % runtime_str)

def _internal_app(configfile, app_name=None, username=None):
    from webtest import TestApp
    from pyramid import paster
    app = paster.get_app(configfile, app_name)
    if not username:
        username = 'UPGRADE'
    environ = {
        'HTTP_ACCEPT': 'application/json',
        'REMOTE_USER': username,
    }
    return TestApp(app, environ)

def main():
    args = _parse_args()
    # Setup Logger
    paster.setup_logging(args.config_uri)
    logging.getLogger('snovault').setLevel(logging.INFO)
    # Get Uuids
    testapp = _internal_app(args.config_uri, app_name=args.app_name, username=args.username)
    connection = testapp.app.registry[CONNECTION]
    uuids = [str(uuid) for uuid in connection.__iter__(*args.item_types)]
    if uuids:
        log_msg = "Start Upgrade with {} items: {}, {}, {}, {}".format(
            len(uuids),
            args.batchsize,
            args.chunksize,
            args.processes,
            args.maxtasksperchild,
        )
        BATCH_UPGRADE_LOG.info(log_msg)
        pool_start = time.time()
        all_results = _run_pool(uuids, args)
        runtime_mins_str = "{:0.2f} minutes".format(
            (time.time() - pool_start) / 60
        )
        BATCH_UPGRADE_LOG.info('End Upgrade')
        _summarize_results(all_results, runtime_str=runtime_mins_str, verbose=args.verbose)
    else:
        BATCH_UPGRADE_LOG.warning('No uuids to upgrade.')

def _parse_args():
    import argparse
    parser = argparse.ArgumentParser(
        description="Batch upgrade content items.", epilog=EPILOG,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument('config_uri', help="path to configfile")
    parser.add_argument('--app-name', help="Pyramid app name in configfile")
    parser.add_argument('--batchsize', type=int, default=50)
    parser.add_argument('--chunksize', type=int, default=1)
    parser.add_argument('--item-types', action='append', default=[])
    parser.add_argument('--maxtasksperchild', type=int, default=1)
    parser.add_argument('--processes', type=int, default=2)
    parser.add_argument('--username')
    parser.add_argument('-v', '--verbose', action='store_true')
    args = parser.parse_args()
    return args

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# cardinal_pythonlib/tools/estimate_mysql_memory_usage.py

"""
===============================================================================

    Original code copyright (C) 2009-2019 Rudolf Cardinal (rudolf@pobox.com).

    This file is part of cardinal_pythonlib.

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.

===============================================================================

**Script to check the (approximate) memory usage of a running MySQL instance.**

- From: https://dev.mysql.com/doc/refman/5.0/en/memory-use.html

- However, ``innodb_additional_mem_pool_size`` deprecated in 5.6.3 and removed
  in 5.7.4; http://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html

"""

import argparse
import logging
import subprocess
from typing import Dict, Union

from prettytable import PrettyTable

from cardinal_pythonlib.logs import (
    BraceStyleAdapter,
    main_only_quicksetup_rootlogger,
)

log = BraceStyleAdapter(logging.getLogger(__name__))

MYSQL_DEFAULT_PORT = 3306
MYSQL_DEFAULT_USER = 'root'
UNITS_MB = 'Mb'

def get_mysql_vars(mysql: str,
                   host: str,
                   port: int,
                   user: str) -> Dict[str, str]:
    """
    Asks MySQL for its variables and status.

    Args:
        mysql: ``mysql`` executable filename
        host: host name
        port: TCP/IP port number
        user: username

    Returns:
        dictionary of MySQL variables/values

    """
    cmdargs = [
        mysql,
        "-h", host,
        "-P", str(port),
        "-e", "SHOW VARIABLES; SHOW STATUS",
        "-u", user,
        "-p"  # prompt for password
    ]
    log.info("Connecting to MySQL with user: {}", user)
    log.debug(cmdargs)
    process = subprocess.Popen(cmdargs, stdout=subprocess.PIPE)
    out, err = process.communicate()
    lines = out.decode("utf8").splitlines()
    mysqlvars = {}
    for line in lines:
        var, val = line.split("\t")
        mysqlvars[var] = val
    return mysqlvars

def val_mb(valstr: Union[int, str]) -> str:
    """
    Converts a value in bytes (in string format) to megabytes.
    """
    try:
        return "{:.3f}".format(int(valstr) / (1024 * 1024))
    except (TypeError, ValueError):
        return '?'

def val_int(val: int) -> str:
    """
    Formats an integer value.
    """
    return str(val) + " " * 4

def add_var_mb(table: PrettyTable,
               vardict: Dict[str, str],
               varname: str) -> None:
    """
    Adds a row to ``table`` for ``varname``, in megabytes.
    """
    valstr = vardict.get(varname, None)
    table.add_row([varname, val_mb(valstr), UNITS_MB])

def add_blank_row(table: PrettyTable) -> None:
    """
    Adds a blank row to ``table``.
    """
    table.add_row([''] * 3)

def main():
    """
    Command-line processor. See ``--help`` for details.
    """
    main_only_quicksetup_rootlogger(level=logging.DEBUG)
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--mysql", default="mysql",
        help="MySQL program (default=mysql)")
    parser.add_argument(
        "--host", default="127.0.0.1",
        help="MySQL server/host (prefer '127.0.0.1' to 'localhost')")
    parser.add_argument(
        "--port", type=int, default=MYSQL_DEFAULT_PORT,
        help=f"MySQL port (default={MYSQL_DEFAULT_PORT})")
    parser.add_argument(
        "--user", default=MYSQL_DEFAULT_USER,
        help=f"MySQL user (default={MYSQL_DEFAULT_USER})")
    args = parser.parse_args()

    vardict = get_mysql_vars(
        mysql=args.mysql,
        host=args.host,
        port=args.port,
        user=args.user,
    )
    max_conn = int(vardict["max_connections"])
    max_used_conn = int(vardict["Max_used_connections"])
    base_mem = (
        int(vardict["key_buffer_size"]) +
        int(vardict["query_cache_size"]) +
        int(vardict["innodb_buffer_pool_size"]) +
        # int(vardict["innodb_additional_mem_pool_size"]) +
        int(vardict["innodb_log_buffer_size"])
    )
    mem_per_conn = (
        int(vardict["read_buffer_size"]) +
        int(vardict["read_rnd_buffer_size"]) +
        int(vardict["sort_buffer_size"]) +
        int(vardict["join_buffer_size"]) +
        int(vardict["binlog_cache_size"]) +
        int(vardict["thread_stack"]) +
        int(vardict["tmp_table_size"])
    )
    mem_total_min = base_mem + mem_per_conn * max_used_conn
    mem_total_max = base_mem + mem_per_conn * max_conn

    table = PrettyTable(["Variable", "Value", "Units"])
    table.align["Variable"] = "l"
    table.align["Value"] = "r"
    table.align["Units"] = "l"

    add_var_mb(table, vardict, "key_buffer_size")
    add_var_mb(table, vardict, "query_cache_size")
    add_var_mb(table, vardict, "innodb_buffer_pool_size")
    # print_var_mb(table, vardict, "innodb_additional_mem_pool_size")
    add_var_mb(table, vardict, "innodb_log_buffer_size")
    add_blank_row(table)
    table.add_row(["BASE MEMORY", val_mb(base_mem), UNITS_MB])
    add_blank_row(table)
    add_var_mb(table, vardict, "sort_buffer_size")
    add_var_mb(table, vardict, "read_buffer_size")
    add_var_mb(table, vardict, "read_rnd_buffer_size")
    add_var_mb(table, vardict, "join_buffer_size")
    add_var_mb(table, vardict, "thread_stack")
    add_var_mb(table, vardict, "binlog_cache_size")
    add_var_mb(table, vardict, "tmp_table_size")
    add_blank_row(table)
    table.add_row(["MEMORY PER CONNECTION", val_mb(mem_per_conn), UNITS_MB])
    add_blank_row(table)
    table.add_row(["Max_used_connections", val_int(max_used_conn), ''])
    table.add_row(["max_connections", val_int(max_conn), ''])
    add_blank_row(table)
    table.add_row(["TOTAL (MIN)", val_mb(mem_total_min), UNITS_MB])
    table.add_row(["TOTAL (MAX)", val_mb(mem_total_max), UNITS_MB])

    print(table.get_string())

if __name__ == '__main__':
    main()
<commit_msg>log bugfix in estimate_mysql_memory_usage.py<commit_after>#!/usr/bin/env python
# cardinal_pythonlib/tools/estimate_mysql_memory_usage.py

"""
===============================================================================

    Original code copyright (C) 2009-2019 Rudolf Cardinal (rudolf@pobox.com).

    This file is part of cardinal_pythonlib.

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.

===============================================================================

**Script to check the (approximate) memory usage of a running MySQL instance.**

- From: https://dev.mysql.com/doc/refman/5.0/en/memory-use.html

- However, ``innodb_additional_mem_pool_size`` deprecated in 5.6.3 and removed
  in 5.7.4; http://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html

"""

import argparse
import logging
import subprocess
from typing import Dict, Union

from prettytable import PrettyTable

from cardinal_pythonlib.logs import main_only_quicksetup_rootlogger

log = logging.getLogger(__name__)

MYSQL_DEFAULT_PORT = 3306
MYSQL_DEFAULT_USER = 'root'
UNITS_MB = 'Mb'

def get_mysql_vars(mysql: str,
                   host: str,
                   port: int,
                   user: str) -> Dict[str, str]:
    """
    Asks MySQL for its variables and status.

    Args:
        mysql: ``mysql`` executable filename
        host: host name
        port: TCP/IP port number
        user: username

    Returns:
        dictionary of MySQL variables/values

    """
    cmdargs = [
        mysql,
        "-h", host,
        "-P", str(port),
        "-e", "SHOW VARIABLES; SHOW STATUS",
        "-u", user,
        "-p"  # prompt for password
    ]
    log.info(f"Connecting to MySQL with user: {user}")
    log.debug(cmdargs)
    process = subprocess.Popen(cmdargs, stdout=subprocess.PIPE)
    out, err = process.communicate()
    lines = out.decode("utf8").splitlines()
    mysqlvars = {}
    for line in lines:
        var, val = line.split("\t")
        mysqlvars[var] = val
    return mysqlvars

def val_mb(valstr: Union[int, str]) -> str:
    """
    Converts a value in bytes (in string format) to megabytes.
    """
    try:
        return "{:.3f}".format(int(valstr) / (1024 * 1024))
    except (TypeError, ValueError):
        return '?'

def val_int(val: int) -> str:
    """
    Formats an integer value.
    """
    return str(val) + " " * 4

def add_var_mb(table: PrettyTable,
               vardict: Dict[str, str],
               varname: str) -> None:
    """
    Adds a row to ``table`` for ``varname``, in megabytes.
    """
    valstr = vardict.get(varname, None)
    table.add_row([varname, val_mb(valstr), UNITS_MB])

def add_blank_row(table: PrettyTable) -> None:
    """
    Adds a blank row to ``table``.
    """
    table.add_row([''] * 3)

def main():
    """
    Command-line processor. See ``--help`` for details.
    """
    main_only_quicksetup_rootlogger(level=logging.DEBUG)
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--mysql", default="mysql",
        help="MySQL program (default=mysql)")
    parser.add_argument(
        "--host", default="127.0.0.1",
        help="MySQL server/host (prefer '127.0.0.1' to 'localhost')")
    parser.add_argument(
        "--port", type=int, default=MYSQL_DEFAULT_PORT,
        help=f"MySQL port (default={MYSQL_DEFAULT_PORT})")
    parser.add_argument(
        "--user", default=MYSQL_DEFAULT_USER,
        help=f"MySQL user (default={MYSQL_DEFAULT_USER})")
    args = parser.parse_args()

    vardict = get_mysql_vars(
        mysql=args.mysql,
        host=args.host,
        port=args.port,
        user=args.user,
    )
    max_conn = int(vardict["max_connections"])
    max_used_conn = int(vardict["Max_used_connections"])
    base_mem = (
        int(vardict["key_buffer_size"]) +
        int(vardict["query_cache_size"]) +
        int(vardict["innodb_buffer_pool_size"]) +
        # int(vardict["innodb_additional_mem_pool_size"]) +
        int(vardict["innodb_log_buffer_size"])
    )
    mem_per_conn = (
        int(vardict["read_buffer_size"]) +
        int(vardict["read_rnd_buffer_size"]) +
        int(vardict["sort_buffer_size"]) +
        int(vardict["join_buffer_size"]) +
        int(vardict["binlog_cache_size"]) +
        int(vardict["thread_stack"]) +
        int(vardict["tmp_table_size"])
    )
    mem_total_min = base_mem + mem_per_conn * max_used_conn
    mem_total_max = base_mem + mem_per_conn * max_conn

    table = PrettyTable(["Variable", "Value", "Units"])
    table.align["Variable"] = "l"
    table.align["Value"] = "r"
    table.align["Units"] = "l"

    add_var_mb(table, vardict, "key_buffer_size")
    add_var_mb(table, vardict, "query_cache_size")
    add_var_mb(table, vardict, "innodb_buffer_pool_size")
    # print_var_mb(table, vardict, "innodb_additional_mem_pool_size")
    add_var_mb(table, vardict, "innodb_log_buffer_size")
    add_blank_row(table)
    table.add_row(["BASE MEMORY", val_mb(base_mem), UNITS_MB])
    add_blank_row(table)
    add_var_mb(table, vardict, "sort_buffer_size")
    add_var_mb(table, vardict, "read_buffer_size")
    add_var_mb(table, vardict, "read_rnd_buffer_size")
    add_var_mb(table, vardict, "join_buffer_size")
    add_var_mb(table, vardict, "thread_stack")
    add_var_mb(table, vardict, "binlog_cache_size")
    add_var_mb(table, vardict, "tmp_table_size")
    add_blank_row(table)
    table.add_row(["MEMORY PER CONNECTION", val_mb(mem_per_conn), UNITS_MB])
    add_blank_row(table)
    table.add_row(["Max_used_connections", val_int(max_used_conn), ''])
    table.add_row(["max_connections", val_int(max_conn), ''])
    add_blank_row(table)
    table.add_row(["TOTAL (MIN)", val_mb(mem_total_min), UNITS_MB])
    table.add_row(["TOTAL (MAX)", val_mb(mem_total_max), UNITS_MB])

    print(table.get_string())

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from copy import deepcopy

import dictdiffer

import virtool.db.otus
import virtool.errors
import virtool.otus
import virtool.utils
import virtool.history
from virtool.api.utils import paginate

MOST_RECENT_PROJECTION = [
    "_id",
    "description",
    "method_name",
    "user",
    "otu",
    "created_at"
]

LIST_PROJECTION = [
    "_id",
    "description",
    "method_name",
    "created_at",
    "index",
    "otu",
    "reference",
    "user"
]

PROJECTION = LIST_PROJECTION + [
    "diff"
]

async def add(db, method_name, old, new, description, user_id):
    """
    Add a change document to the history collection.

    :param db: the application database client
    :type db: :class:`~motor.motor_asyncio.AsyncIOMotorClient`

    :param method_name: the name of the handler method that executed the change
    :type method_name: str

    :param old: the otu document prior to the change
    :type new: Union[dict, None]

    :param new: the otu document after the change
    :type new: Union[dict, None]

    :param description: a human readable description of the change
    :type description: str

    :param user_id: the id of the requesting user
    :type user_id: str

    :return: the change document
    :rtype: Coroutine[dict]

    """
    try:
        otu_id = old["_id"]
    except TypeError:
        otu_id = new["_id"]

    try:
        otu_name = old["name"]
    except TypeError:
        otu_name = new["name"]

    try:
        otu_version = int(new["version"])
    except (TypeError, KeyError):
        otu_version = "removed"

    try:
        ref_id = old["reference"]["id"]
    except (TypeError, KeyError):
        ref_id = new["reference"]["id"]

    document = {
        "_id": ".".join([str(otu_id), str(otu_version)]),
        "method_name": method_name,
        "description": description,
        "created_at": virtool.utils.timestamp(),
        "otu": {
            "id": otu_id,
            "name": otu_name,
            "version": otu_version
        },
        "reference": {
            "id": ref_id
        },
        "index": {
            "id": "unbuilt",
            "version": "unbuilt"
        },
        "user": {
            "id": user_id
        }
    }

    if method_name == "create":
        document["diff"] = new

    elif method_name == "remove":
        document["diff"] = old

    else:
        document["diff"] = virtool.history.calculate_diff(old, new)

    await db.history.insert_one(document)

    return document

async def find(db, req_query, base_query=None):
    data = await paginate(
        db.history,
        {},
        req_query,
        base_query=base_query,
        sort="otu.version",
        projection=LIST_PROJECTION,
        reverse=True
    )

    return data

async def get_contributors(db, query):
    """
    Return an list of contributors and their contribution count for a specific set of history.

    :param db: the application database client
    :type db: :class:`~motor.motor_asyncio.AsyncIOMotorClient`

    :param query: a query to filter scanned history by
    :type query: dict

    :return: a list of contributors to the scanned history changes
    :rtype: List[dict]

    """
    cursor = db.history.aggregate([
        {"$match": query},
        {"$group": {
            "_id": "$user.id",
            "count": {"$sum": 1}
        }}
    ])

    return [{"id": c["_id"], "count": c["count"]} async for c in cursor]

async def get_most_recent_change(db, otu_id):
    """
    Get the most recent change for the otu identified by the passed ``otu_id``.

    :param db: the application database client
    :type db: :class:`~motor.motor_asyncio.AsyncIOMotorClient`

    :param otu_id: the target otu_id
    :type otu_id: str

    :return: the most recent change document
    :rtype: Coroutine[dict]

    """
    return await db.history.find_one({
        "otu.id": otu_id,
        "index.id": "unbuilt"
    }, MOST_RECENT_PROJECTION, sort=[("otu.version", -1)])

async def patch_to_verified(db, otu_id):
    current = await virtool.db.otus.join(db, otu_id) or dict()

    if current and current["verified"]:
        return current

    patched = deepcopy(current)

    async for change in db.history.find({"otu.id": otu_id}, sort=[("otu.version", -1)]):
        if change["method_name"] == "remove":
            patched = change["diff"]

        elif change["method_name"] == "create":
            return None

        else:
            diff = dictdiffer.swap(change["diff"])
            patched = dictdiffer.patch(diff, patched)

        if patched["verified"]:
            return patched

async def patch_to_version(db, otu_id, version):
    """
    Take a joined otu back in time to the passed ``version``. Uses the diffs in the change documents associated with
    the otu.

    :param db: the application database client
    :type db: :class:`~motor.motor_asyncio.AsyncIOMotorClient`

    :param otu_id: the id of the otu to patch
    :type otu_id: str

    :param version: the version to patch to
    :type version: str or int

    :return: the current joined otu, patched otu, and the ids of changes reverted in the process
    :rtype: Coroutine[tuple]

    """
    # A list of history_ids reverted to produce the patched entry.
    reverted_history_ids = list()

    current = await virtool.db.otus.join(db, otu_id) or dict()

    if "version" in current and current["version"] == version:
        return current, deepcopy(current), reverted_history_ids

    patched = deepcopy(current)

    # Sort the changes by descending timestamp.
    async for change in db.history.find({"otu.id": otu_id}, sort=[("otu.version", -1)]):
        if change["otu"]["version"] == "removed" or change["otu"]["version"] > version:
            reverted_history_ids.append(change["_id"])

            if change["method_name"] == "remove":
                patched = change["diff"]

            elif change["method_name"] == "create":
                patched = None

            else:
                diff = dictdiffer.swap(change["diff"])
                patched = dictdiffer.patch(diff, patched)
        else:
            break

    if current == {}:
        current = None

    return current, patched, reverted_history_ids

async def revert(db, change_id):
    """
    Revert a history change given by the passed ``change_id``.

    :param db: the application database client
    :type db: :class:`~motor.motor_asyncio.AsyncIOMotorClient`

    :param change_id: a unique id for the cah
    :return:
    """
    change = await db.history.find_one({"_id": change_id}, ["index"])

    if change["index"]["id"] != "unbuilt" or change["index"]["version"] != "unbuilt":
        raise virtool.errors.DatabaseError("Change is included in a build an not revertible")

    otu_id, otu_version = change_id.split(".")

    if otu_version != "removed":
        otu_version = int(otu_version)

    _, patched, history_to_delete = await patch_to_version(
        db,
        otu_id,
        otu_version - 1
    )

    # Remove the old sequences from the collection.
    await db.sequences.delete_many({"otu_id": otu_id})

    if patched is not None:
        patched_otu, sequences = virtool.otus.split(patched)

        # Add the reverted sequences to the collection.
        for sequence in sequences:
            await db.sequences.insert_one(sequence)

        # Replace the existing otu with the patched one. If it doesn't exist, insert it.
        await db.otus.replace_one({"_id": otu_id}, patched_otu, upsert=True)

    else:
        await db.otus.delete_one({"_id": otu_id})

    await db.history.delete_many({"_id": {"$in": history_to_delete}})

    return patched
<commit_msg>Add silent kwarg for db.history.add()<commit_after>from copy import deepcopy

import dictdiffer

import virtool.db.otus
import virtool.errors
import virtool.otus
import virtool.utils
import virtool.history
from virtool.api.utils import paginate

MOST_RECENT_PROJECTION = [
    "_id",
    "description",
    "method_name",
    "user",
    "otu",
    "created_at"
]

LIST_PROJECTION = [
    "_id",
    "description",
    "method_name",
    "created_at",
    "index",
    "otu",
    "reference",
    "user"
]

PROJECTION = LIST_PROJECTION + [
    "diff"
]

async def add(db, method_name, old, new, description, user_id, silent=False):
    """
    Add a change document to the history collection.

    :param db: the application database client
    :type db: :class:`~motor.motor_asyncio.AsyncIOMotorClient`

    :param method_name: the name of the handler method that executed the change
    :type method_name: str

    :param old: the otu document prior to the change
    :type new: Union[dict, None]

    :param new: the otu document after the change
    :type new: Union[dict, None]

    :param description: a human readable description of the change
    :type description: str

    :param user_id: the id of the requesting user
    :type user_id: str

    :param silent: don't dispatch a message
    :type: silent: bool

    :return: the change document
    :rtype: Coroutine[dict]

    """
    try:
        otu_id = old["_id"]
    except TypeError:
        otu_id = new["_id"]

    try:
        otu_name = old["name"]
    except TypeError:
        otu_name = new["name"]

    try:
        otu_version = int(new["version"])
    except (TypeError, KeyError):
        otu_version = "removed"

    try:
        ref_id = old["reference"]["id"]
    except (TypeError, KeyError):
        ref_id = new["reference"]["id"]

    document = {
        "_id": ".".join([str(otu_id), str(otu_version)]),
        "method_name": method_name,
        "description": description,
        "created_at": virtool.utils.timestamp(),
        "otu": {
            "id": otu_id,
            "name": otu_name,
            "version": otu_version
        },
        "reference": {
            "id": ref_id
        },
        "index": {
            "id": "unbuilt",
            "version": "unbuilt"
        },
        "user": {
            "id": user_id
        }
    }

    if method_name == "create":
        document["diff"] = new

    elif method_name == "remove":
        document["diff"] = old

    else:
        document["diff"] = virtool.history.calculate_diff(old, new)

    await db.history.insert_one(document, silent=silent)

    return document

async def find(db, req_query, base_query=None):
    data = await paginate(
        db.history,
        {},
        req_query,
        base_query=base_query,
        sort="otu.version",
        projection=LIST_PROJECTION,
        reverse=True
    )

    return data

async def get_contributors(db, query):
    """
    Return an list of contributors and their contribution count for a specific set of history.

    :param db: the application database client
    :type db: :class:`~motor.motor_asyncio.AsyncIOMotorClient`

    :param query: a query to filter scanned history by
    :type query: dict

    :return: a list of contributors to the scanned history changes
    :rtype: List[dict]

    """
    cursor = db.history.aggregate([
        {"$match": query},
        {"$group": {
            "_id": "$user.id",
            "count": {"$sum": 1}
        }}
    ])

    return [{"id": c["_id"], "count": c["count"]} async for c in cursor]

async def get_most_recent_change(db, otu_id):
    """
    Get the most recent change for the otu identified by the passed ``otu_id``.

    :param db: the application database client
    :type db: :class:`~motor.motor_asyncio.AsyncIOMotorClient`

    :param otu_id: the target otu_id
    :type otu_id: str

    :return: the most recent change document
    :rtype: Coroutine[dict]

    """
    return await db.history.find_one({
        "otu.id": otu_id,
        "index.id": "unbuilt"
    }, MOST_RECENT_PROJECTION, sort=[("otu.version", -1)])

async def patch_to_verified(db, otu_id):
    current = await virtool.db.otus.join(db, otu_id) or dict()

    if current and current["verified"]:
        return current

    patched = deepcopy(current)

    async for change in db.history.find({"otu.id": otu_id}, sort=[("otu.version", -1)]):
        if change["method_name"] == "remove":
            patched = change["diff"]

        elif change["method_name"] == "create":
            return None

        else:
            diff = dictdiffer.swap(change["diff"])
            patched = dictdiffer.patch(diff, patched)

        if patched["verified"]:
            return patched

async def patch_to_version(db, otu_id, version):
    """
    Take a joined otu back in time to the passed ``version``. Uses the diffs in the change documents associated with
    the otu.

    :param db: the application database client
    :type db: :class:`~motor.motor_asyncio.AsyncIOMotorClient`

    :param otu_id: the id of the otu to patch
    :type otu_id: str

    :param version: the version to patch to
    :type version: str or int

    :return: the current joined otu, patched otu, and the ids of changes reverted in the process
    :rtype: Coroutine[tuple]

    """
    # A list of history_ids reverted to produce the patched entry.
    reverted_history_ids = list()

    current = await virtool.db.otus.join(db, otu_id) or dict()

    if "version" in current and current["version"] == version:
        return current, deepcopy(current), reverted_history_ids

    patched = deepcopy(current)

    # Sort the changes by descending timestamp.
    async for change in db.history.find({"otu.id": otu_id}, sort=[("otu.version", -1)]):
        if change["otu"]["version"] == "removed" or change["otu"]["version"] > version:
            reverted_history_ids.append(change["_id"])

            if change["method_name"] == "remove":
                patched = change["diff"]

            elif change["method_name"] == "create":
                patched = None

            else:
                diff = dictdiffer.swap(change["diff"])
                patched = dictdiffer.patch(diff, patched)
        else:
            break

    if current == {}:
        current = None

    return current, patched, reverted_history_ids

async def revert(db, change_id):
    """
    Revert a history change given by the passed ``change_id``.

    :param db: the application database client
    :type db: :class:`~motor.motor_asyncio.AsyncIOMotorClient`

    :param change_id: a unique id for the cah
    :return:
    """
    change = await db.history.find_one({"_id": change_id}, ["index"])

    if change["index"]["id"] != "unbuilt" or change["index"]["version"] != "unbuilt":
        raise virtool.errors.DatabaseError("Change is included in a build an not revertible")

    otu_id, otu_version = change_id.split(".")

    if otu_version != "removed":
        otu_version = int(otu_version)

    _, patched, history_to_delete = await patch_to_version(
        db,
        otu_id,
        otu_version - 1
    )

    # Remove the old sequences from the collection.
    await db.sequences.delete_many({"otu_id": otu_id})

    if patched is not None:
        patched_otu, sequences = virtool.otus.split(patched)

        # Add the reverted sequences to the collection.
        for sequence in sequences:
            await db.sequences.insert_one(sequence)

        # Replace the existing otu with the patched one. If it doesn't exist, insert it.
        await db.otus.replace_one({"_id": otu_id}, patched_otu, upsert=True)

    else:
        await db.otus.delete_one({"_id": otu_id})

    await db.history.delete_many({"_id": {"$in": history_to_delete}})

    return patched
<|endoftext|>


<commit_before>from os import listdir, getcwd
from os.path import isfile, join
from random import choice

import datetime

from speaker_db import SpeakerDB
from speakerbot_plugins import *

try:
    from uwsgidecorators import lock

except ImportError:
    def lock(f):
        return f

def create_deferred(function, *args, **kwargs):

    def _exec():
        function(*args, **kwargs)

    return _exec

def parse_and_route_speech(speakerbot, text):

    actions = {
        'random':random_utterance,
        'dada':dada,
        'slinging':slinging_burgers,
        'weather':weather,
        'lunch': lunch,
        'datefact':datefact,
        'horoscope':horoscope,
        'yoda':yoda,
        'ross':ross,
        'jon':jon,
        'josh':josh,
        'spin':price_is_right,
        'suspense':random_drumroll,
        'urban': urban
    }

    token = None
    argument = None

    if text[0] == "!":
        space_pos = text.find(" ")
        if space_pos > 0:
            token = text[1:space_pos]
            argument = text[space_pos:]
        else:
            token = text[1:]

        method = actions.get(token, None)

        if method:
            try:
                if argument:
                    text_output = method(speakerbot, argument.strip())
                else:
                    text_output = method(speakerbot)

                text = text_output

            except TypeError:
                text = "I need an argument for that function, dummy."

    if text:
        play_speech(speakerbot.say_classy, run_filters(text))

def niceify_number(i):
    #swiped from http://codegolf.stackexchange.com/questions/4707/outputting-ordinal-numbers-1st-2nd-3rd
    k=i%10
    return "%d%s" % (i,"tsnrhtdd"[(i/10%10!=1)*(k<4)*k::4])

def queue_speech_for_tweet(*args, **kwargs):

    text = kwargs["speech_text"]

    if text[0] == "!" or text[0:2] == "..":
        return

    if not text:
        return

    db = SpeakerDB()
    text = text[:139]
    db.execute("INSERT INTO publish_queue (tweet_text) VALUES (?)", [text])

def queue_sound_for_tweet(name, event_result):

    db = SpeakerDB()
    matched_sound = db.execute("SELECT votes FROM sounds where name=?", [name]).fetchone()

    if matched_sound:
        text = "I just played %s for the %s time" % (name, niceify_number(matched_sound["votes"]))
        db.execute("INSERT INTO publish_queue (tweet_text) VALUES (?)", [text])

def play_speech(speech_func, text):

    run_with_lock(speech_func, speech_text=text)

@lock
def run_with_lock(func, *args, **kwargs):
    func(*args, **kwargs)

def get_image(checker_func=lambda x: True, depth=5):

    path = getcwd() + "/static/r_gifs/"

    files = [ f for f in listdir(path) if isfile(join(path,f)) ]

    file_path = choice(files)

    if "?" in file_path:
        file_path = get_image()

    if not checker_func(file_path) and depth != 0:
        print "trying to get passable image"
        depth -= 1
        file_path = get_image(checker_func, depth)

    return file_path<commit_msg>Lock the speech parser/router<commit_after>from os import listdir, getcwd
from os.path import isfile, join
from random import choice

import datetime

from speaker_db import SpeakerDB
from speakerbot_plugins import *

try:
    from uwsgidecorators import lock

except ImportError:
    def lock(f):
        return f

def create_deferred(function, *args, **kwargs):

    def _exec():
        function(*args, **kwargs)

    return _exec

@lock
def parse_and_route_speech(speakerbot, text):

    actions = {
        'random':random_utterance,
        'dada':dada,
        'slinging':slinging_burgers,
        'weather':weather,
        'lunch': lunch,
        'datefact':datefact,
        'horoscope':horoscope,
        'yoda':yoda,
        'ross':ross,
        'jon':jon,
        'josh':josh,
        'spin':price_is_right,
        'suspense':random_drumroll,
        'urban': urban
    }

    token = None
    argument = None

    if text[0] == "!":
        space_pos = text.find(" ")
        if space_pos > 0:
            token = text[1:space_pos]
            argument = text[space_pos:]
        else:
            token = text[1:]

        method = actions.get(token, None)

        if method:
            try:
                if argument:
                    text_output = method(speakerbot, argument.strip())
                else:
                    text_output = method(speakerbot)

                text = text_output

            except TypeError:
                text = "I need an argument for that function, dummy."

    if text:
        play_speech(speakerbot.say_classy, run_filters(text))

def niceify_number(i):
    #swiped from http://codegolf.stackexchange.com/questions/4707/outputting-ordinal-numbers-1st-2nd-3rd
    k=i%10
    return "%d%s" % (i,"tsnrhtdd"[(i/10%10!=1)*(k<4)*k::4])

def queue_speech_for_tweet(*args, **kwargs):

    text = kwargs["speech_text"]

    if text[0] == "!" or text[0:2] == "..":
        return

    if not text:
        return

    db = SpeakerDB()
    text = text[:139]
    db.execute("INSERT INTO publish_queue (tweet_text) VALUES (?)", [text])

def queue_sound_for_tweet(name, event_result):

    db = SpeakerDB()
    matched_sound = db.execute("SELECT votes FROM sounds where name=?", [name]).fetchone()

    if matched_sound:
        text = "I just played %s for the %s time" % (name, niceify_number(matched_sound["votes"]))
        db.execute("INSERT INTO publish_queue (tweet_text) VALUES (?)", [text])

def play_speech(speech_func, text):

    run_with_lock(speech_func, speech_text=text)

@lock
def run_with_lock(func, *args, **kwargs):
    func(*args, **kwargs)

def get_image(checker_func=lambda x: True, depth=5):

    path = getcwd() + "/static/r_gifs/"

    files = [ f for f in listdir(path) if isfile(join(path,f)) ]

    file_path = choice(files)

    if "?" in file_path:
        file_path = get_image()

    if not checker_func(file_path) and depth != 0:
        print "trying to get passable image"
        depth -= 1
        file_path = get_image(checker_func, depth)

    return file_path<|endoftext|>


<commit_before>#  Copyright (c) 2015 SONATA-NFV, UBIWHERE
# ALL RIGHTS RESERVED.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Neither the name of the SONATA-NFV, UBIWHERE
# nor the names of its contributors may be used to endorse or promote
# products derived from this software without specific prior written
# permission.
#
# This work has been performed in the framework of the SONATA project,
# funded by the European Commission under Grant number 671517 through
# the Horizon 2020 and 5G-PPP programmes. The authors would like to
# acknowledge the contributions of their colleagues of the SONATA
# partner consortium (www.sonata-nfv.eu).

import os
import sys
import yaml
import logging
import coloredlogs
import networkx as nx
from son.schema.validator import SchemaValidator

from son.workspace.workspace import Workspace, Project

log = logging.getLogger(__name__)

class Validator(object):

    def __init__(self, workspace, project):
        self._workspace = workspace
        self._project = project
        coloredlogs.install(level=workspace.log_level)

        self._schema_validator = SchemaValidator(self._workspace)
        self._nsd = None
        self._vnfds = {}

    def validate(self):
        """
        Validate a project.
        It performs the syntax and network topology validation of a service.
        :return:
        """
        success = True

        # load and correlate project descriptors
        log.info("Loading project descriptors")
        if not self._load_project_descriptors():
            return False

        # validate syntax
        log.info("Validating syntax of descriptors")
        if not self._validate_service_syntax():
            return False

        # validate topology
        log.info("Validating service network topology")
        if not self._validate_service_graph():
            return False

        log.info("SUCCESS")
        return True

    def _load_project_descriptors(self):

        # load project service descriptor (NSD)
        nsd_files = self._project.get_ns_descriptor()
        if not nsd_files:
            return

        if len(nsd_files) > 1:
            log.error("Found multiple service descriptors in project '{0}': "
                      "{1}"
                      .format(self._project.project_root,
                              nsd_files))
            return

        entry_nsd_file = nsd_files[0]
        with open(entry_nsd_file, 'r') as _file:
            self._nsd = yaml.load(_file)
            if not self._nsd:
                log.error("Couldn't read service descriptor file: '{0}'"
                          .format(entry_nsd_file))
                return

        # read VNFD files in project source
        prj_vnfds = {}
        vnfd_files = self._project.get_vnf_descriptors()
        if vnfd_files:
            for vnfd_file in vnfd_files:
                with open(vnfd_file, 'r') as _file:
                    vnfd = yaml.load(_file)
                    if not vnfd:
                        log.error("Couldn't read VNF descriptor file: '{0}'"
                                  .format(vnfd_file))
                        return

                    vnf_combo_id = vnfd['vendor'] + '.' + vnfd['name'] + '.' \
                        + vnfd['version']

                    if vnf_combo_id in prj_vnfds:
                        log.error("Duplicate VNF descriptor in file: '{0}'"
                                  .format(vnfd_file))
                        return

                    prj_vnfds[vnf_combo_id] = vnfd

        if not prj_vnfds:
            log.warning("Project source does not contain VNF descriptors")

        # read NSD file and get its referenced function descriptors (VNFDS)
        self._vnfds = {}

        if len(self._nsd['network_functions']) > 0 and len(prj_vnfds) == 0:
            log.error("Service descriptor '{0}' references function "
                      "descriptors (VNFs) but none were found in project "
                      "sources."
                      .format(entry_nsd_file))
            return

        for vnf in self._nsd['network_functions']:
            vnf_combo_id = vnf['vnf_vendor'] + '.' + vnf['vnf_name'] + \
                           '.' + vnf['vnf_version']
            if vnf_combo_id not in prj_vnfds.keys():
                log.error("Referenced VNF descriptor '{0}' could not be found"
                          .format(vnf_combo_id))
                return
            self._vnfds[vnf['vnf_id']] = prj_vnfds.pop(vnf_combo_id)

        if len(prj_vnfds) > 0:
            log.warning("The following VNFs are present in project sources "
                        "but are not referenced in the service descriptor: {0}"
                        .format(prj_vnfds.keys()))
        return True

    def _validate_service_syntax(self):
        """
        Validate a the syntax of a service and all of its descriptors.
        :return:
        """

        return True

    def _validate_service_graph(self):
        """
        Validate the network topology of a service.
        :return:
        """
        # build service network graph
        self._build_service_graph()

        # check for forwarding cycles
        self._find_service_graph_cycles()

        return True

    def _build_service_graph(self):
        """
        Build the network graph of a service.
        This graph will be later utilized for checking invalid or cyclic
        paths.
        :return:
        """

        # init service network graph
        sg = nx.Graph()

        # add connection points as nodes to the service graph
        for cp in self._nsd['connection_points']:
            if cp['type'] == 'interface':
                sg.add_node(cp['id'], attr_dict={'used': False})
        for vnf_id in self._vnfds.keys():
            for cp in self._vnfds[vnf_id]['connection_points']:
                if cp['type'] == 'interface':
                    sg.add_node(vnf_id + ':' + cp['id'], attr_dict={'used':
                                                                    False})

        # add edges to the graph
        for vlink in self._nsd['virtual_links']:
            ctype = vlink['connectivity_type']
            if ctype != 'E-Line':  # TODO: add support for 'E-Tree'
                continue

            cp_ref = vlink['connection_points_reference']
            if len(cp_ref) != 2:
                log.error("[virtual_link id: '{}'] E-Line must only comprise 2"
                          " connection points"
                          .format(vlink['id']))
                return

            for cp in cp_ref:
                if not sg.has_node(cp):
                    log.error("[virtual_link id: '{}'] The following "
                              "connection point is not defined: {}"
                              .format(vlink['id'], cp))
                    return

            sg.add_edge(cp_ref[0], cp_ref[1])

    def _find_service_graph_cycles(self):
        """
        Check if network graph has simple cycles.
        :return:
        """
        pass

def main():
    import argparse

    parser = argparse.ArgumentParser(
        description="Validate a SONATA project service")

    parser.add_argument(
        "--workspace",
        help="Specify workspace. If not specified "
             "will assume '{}'".format(Workspace.DEFAULT_WORKSPACE_DIR),
        required=False)

    parser.add_argument(
        "--project",
        help="Validate the project at the specified location. "
             "If not specified will assume current directory '{}'"
             .format(os.getcwd()),
        required=False)

    args = parser.parse_args()

    if args.workspace:
        ws_root = args.workspace
    else:
        ws_root = Workspace.DEFAULT_WORKSPACE_DIR

    prj_root = args.project if args.project else os.getcwd()

    # Obtain Workspace object
    workspace = Workspace.__create_from_descriptor__(ws_root)
    if not workspace:
        sys.stderr.write("Invalid workspace path: '%s'\n" % ws_root)
        exit(1)

    project = Project.__create_from_descriptor__(workspace, prj_root)
    if not project:
        sys.stderr.write("Invalid project path: '%s'\n  " % prj_root)
        exit(1)

    val = Validator(workspace, project)
    val.validate()
<commit_msg>- Syntax validation against schemas for service and function descriptors<commit_after>#  Copyright (c) 2015 SONATA-NFV, UBIWHERE
# ALL RIGHTS RESERVED.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Neither the name of the SONATA-NFV, UBIWHERE
# nor the names of its contributors may be used to endorse or promote
# products derived from this software without specific prior written
# permission.
#
# This work has been performed in the framework of the SONATA project,
# funded by the European Commission under Grant number 671517 through
# the Horizon 2020 and 5G-PPP programmes. The authors would like to
# acknowledge the contributions of their colleagues of the SONATA
# partner consortium (www.sonata-nfv.eu).

import os
import sys
import yaml
import logging
import coloredlogs
import networkx as nx
from son.schema.validator import SchemaValidator

from son.workspace.workspace import Workspace, Project

log = logging.getLogger(__name__)

class Validator(object):

    def __init__(self, workspace, project):
        self._workspace = workspace
        self._project = project
        coloredlogs.install(level=workspace.log_level)

        self._schema_validator = SchemaValidator(self._workspace)
        self._nsd_file = None
        self._nsd = None
        self._vnfd_files = {}
        self._vnfds = {}

    def validate(self):
        """
        Validate a project.
        It performs the syntax and network topology validation of a service.
        :return:
        """
        success = True

        # load and correlate project descriptors
        log.info("Loading project descriptors")
        if not self._load_project_descriptors():
            return False

        # validate syntax
        log.info("Validating syntax of descriptors")
        if not self._validate_service_syntax():
            return False

        # validate topology
        log.info("Validating service network topology")
        if not self._validate_service_graph():
            return False

        log.info("SUCCESS")
        return True

    def _load_project_descriptors(self):

        # load project service descriptor (NSD)
        nsd_files = self._project.get_ns_descriptor()
        if not nsd_files:
            return

        if len(nsd_files) > 1:
            log.error("Found multiple service descriptors in project '{0}': "
                      "{1}".format(self._project.project_root, nsd_files))
            return

        entry_nsd_file = nsd_files[0]
        with open(entry_nsd_file, 'r') as _file:
            self._nsd = yaml.load(_file)
            if not self._nsd:
                log.error("Couldn't read service descriptor file: '{0}'"
                          .format(entry_nsd_file))
                return
            self._nsd_file = entry_nsd_file

        # read VNFD files in project source
        prj_vnfds = {}
        prj_vnfd_files = {}
        vnfd_files = self._project.get_vnf_descriptors()
        if vnfd_files:
            for vnfd_file in vnfd_files:
                with open(vnfd_file, 'r') as _file:
                    vnfd = yaml.load(_file)
                    if not vnfd:
                        log.error("Couldn't read VNF descriptor file: '{0}'"
                                  .format(vnfd_file))
                        return

                    vnf_combo_id = vnfd['vendor'] + '.' + vnfd['name'] + '.' \
                        + vnfd['version']

                    if vnf_combo_id in prj_vnfds:
                        log.error("Duplicate VNF descriptor in file: '{0}'"
                                  .format(vnfd_file))
                        return

                    prj_vnfds[vnf_combo_id] = vnfd
                    prj_vnfd_files[vnf_combo_id] = vnfd_file

        if not prj_vnfds:
            log.warning("Project source does not contain VNF descriptors")

        # read NSD file and get its referenced function descriptors (VNFDS)
        self._vnfds = {}

        if len(self._nsd['network_functions']) > 0 and len(prj_vnfds) == 0:
            log.error("Service descriptor '{0}' references function "
                      "descriptors (VNFs) but none were found in project "
                      "sources."
                      .format(entry_nsd_file))
            return

        for vnf in self._nsd['network_functions']:
            vnf_combo_id = vnf['vnf_vendor'] + '.' + vnf['vnf_name'] + \
                           '.' + vnf['vnf_version']
            if vnf_combo_id not in prj_vnfds.keys():
                log.error("Referenced VNF descriptor '{0}' could not be found"
                          .format(vnf_combo_id))
                return
            self._vnfds[vnf['vnf_id']] = prj_vnfds.pop(vnf_combo_id)
            self._vnfd_files[vnf['vnf_id']] = prj_vnfd_files.pop(vnf_combo_id)

        if len(prj_vnfds) > 0:
            log.warning("The following VNFs are present in project sources "
                        "but are not referenced in the service descriptor: {0}"
                        .format(prj_vnfds.keys()))
        return True

    def _validate_service_syntax(self):
        """
        Validate a the syntax of a service and all of its descriptors.
        :return:
        """

        log.debug("Validate syntax of Service Descriptor file '{0}'"
                  .format(self._nsd_file))
        if not self._schema_validator.validate(
              self._nsd, SchemaValidator.SCHEMA_SERVICE_DESCRIPTOR):
            log.error("Bad Service Descriptor file: '{0}'"
                      .format(self._nsd_file))
            return

        log.debug("Validate syntax of Function Descriptor files:")
        for vnfd in self._vnfds.keys():
            log.debug("... '{0}'".format(self._vnfd_files[vnfd]))
            if not self._schema_validator.validate(
                  self._vnfds[vnfd],
                    SchemaValidator.SCHEMA_FUNCTION_DESCRIPTOR):
                log.error("Bad Function Descriptor file: {0}"
                          .format(self._vnfd_files[vnfd]))
                return

        return True

    def _validate_service_graph(self):
        """
        Validate the network topology of a service.
        :return:
        """
        # build service network graph
        self._build_service_graph()

        # check for forwarding cycles
        self._find_service_graph_cycles()

        return True

    def _build_service_graph(self):
        """
        Build the network graph of a service.
        This graph will be later utilized for checking invalid or cyclic
        paths.
        :return:
        """

        # init service network graph
        sg = nx.Graph()

        # add connection points as nodes to the service graph
        for cp in self._nsd['connection_points']:
            if cp['type'] == 'interface':
                sg.add_node(cp['id'], attr_dict={'used': False})
        for vnf_id in self._vnfds.keys():
            for cp in self._vnfds[vnf_id]['connection_points']:
                if cp['type'] == 'interface':
                    sg.add_node(vnf_id + ':' + cp['id'], attr_dict={'used':
                                                                    False})

        # add edges to the graph
        for vlink in self._nsd['virtual_links']:
            ctype = vlink['connectivity_type']
            if ctype != 'E-Line':  # TODO: add support for 'E-Tree'
                continue

            cp_ref = vlink['connection_points_reference']
            if len(cp_ref) != 2:
                log.error("[virtual_link id: '{}'] E-Line must only comprise 2"
                          " connection points"
                          .format(vlink['id']))
                return

            for cp in cp_ref:
                if not sg.has_node(cp):
                    log.error("[virtual_link id: '{}'] The following "
                              "connection point is not defined: {}"
                              .format(vlink['id'], cp))
                    return

            sg.add_edge(cp_ref[0], cp_ref[1])

    def _find_service_graph_cycles(self):
        """
        Check if network graph has simple cycles.
        :return:
        """
        pass

def main():
    import argparse

    parser = argparse.ArgumentParser(
        description="Validate a SONATA project service")

    parser.add_argument(
        "--workspace",
        help="Specify workspace. If not specified "
             "will assume '{}'".format(Workspace.DEFAULT_WORKSPACE_DIR),
        required=False)

    parser.add_argument(
        "--project",
        help="Validate the project at the specified location. "
             "If not specified will assume current directory '{}'"
             .format(os.getcwd()),
        required=False)

    args = parser.parse_args()

    if args.workspace:
        ws_root = args.workspace
    else:
        ws_root = Workspace.DEFAULT_WORKSPACE_DIR

    prj_root = args.project if args.project else os.getcwd()

    # Obtain Workspace object
    workspace = Workspace.__create_from_descriptor__(ws_root)
    if not workspace:
        sys.stderr.write("Invalid workspace path: '%s'\n" % ws_root)
        exit(1)

    project = Project.__create_from_descriptor__(workspace, prj_root)
    if not project:
        sys.stderr.write("Invalid project path: '%s'\n  " % prj_root)
        exit(1)

    val = Validator(workspace, project)
    val.validate()
<|endoftext|>


<commit_before>import re

from functools import reduce

from livestreamer.compat import urlparse, range
from livestreamer.plugin import Plugin
from livestreamer.plugin.api import http, validate
from livestreamer.stream import HDSStream, HTTPStream, RTMPStream
from livestreamer.stream.playlist import FLVPlaylist

COOKIES = {
    "family_filter": "off",
    "ff": "off"
}
QUALITY_MAP = {
    "ld": "240p",
    "sd": "360p",
    "hq": "480p",
    "hd720": "720p",
    "hd1080": "1080p",
    "custom": "live",
    "auto": "hds",
    "source": "hds"
}
STREAM_INFO_URL = "http://www.dailymotion.com/sequence/full/{0}"

_rtmp_re = re.compile("""
    (?P<host>rtmp://[^/]+)
    /(?P<app>[^/]+)
    /(?P<playpath>.+)
""", re.VERBOSE)
_url_re = re.compile("""
    http(s)?://(\w+\.)?
    dailymotion.com
    (/embed)?/(video|live)
    /(?P<media_id>[^_?/]+)
""", re.VERBOSE)

_media_inner_schema = validate.Schema([{
    "layerList": [{
        "name": validate.text,
        validate.optional("sequenceList"): [{
            "layerList": validate.all(
                [{
                    "name": validate.text,
                    validate.optional("param"): dict
                }],
                validate.filter(lambda l: l["name"] in ("video", "reporting"))
            )
        }]
    }]
}])
_media_schema = validate.Schema(
    validate.any(
        _media_inner_schema,
        validate.all(
            {"sequence": _media_inner_schema},
            validate.get("sequence")
        )
    )
)
_vod_playlist_schema = validate.Schema({
    "duration": float,
    "fragments": [[int, float]],
    "template": validate.text
})
_vod_manifest_schema = validate.Schema({
    "alternates": [{
        "height": int,
        "template": validate.text,
        validate.optional("failover"): [validate.text]
    }]
})

class DailyMotion(Plugin):
    @classmethod
    def can_handle_url(self, url):
        return _url_re.match(url)

    def _get_streams_from_media(self, media_id):
        res = http.get(STREAM_INFO_URL.format(media_id), cookies=COOKIES)
        media = http.json(res, schema=_media_schema)

        params = extra_params = swf_url = None
        for __ in media:
            for __ in __["layerList"]:
                for __ in __.get("sequenceList", []):
                    for layer in __["layerList"]:
                        name = layer["name"]
                        if name == "video":
                            params = layer.get("param")
                        elif name == "reporting":
                            extra_params = layer.get("param", {})
                            extra_params = extra_params.get("extraParams", {})

        if not params:
            return

        if extra_params:
            swf_url = extra_params.get("videoSwfURL")

        mode = params.get("mode")
        if mode == "live":
            return self._get_live_streams(params, swf_url)
        elif mode == "vod":
            return self._get_vod_streams(params)

    def _get_live_streams(self, params, swf_url):
        for key, quality in QUALITY_MAP.items():
            key_url = "{0}URL".format(key)
            url = params.get(key_url)

            if not url:
                continue

            try:
                res = http.get(url, exception=IOError)
            except IOError:
                continue

            if quality == "hds":
                streams = HDSStream.parse_manifest(self.session, res.url)
                for name, stream in streams.items():
                    if key == "source":
                        name += "+"

                    yield name, stream
            elif res.text.startswith("rtmp"):
                match = _rtmp_re.match(res.text)
                if not match:
                    continue

                stream = RTMPStream(self.session, {
                    "rtmp": match.group("host"),
                    "app": match.group("app"),
                    "playpath": match.group("playpath"),
                    "swfVfy": swf_url,
                    "live": True
                })

                yield quality, stream

    def _create_flv_playlist(self, template):
        res = http.get(template)
        playlist = http.json(res, schema=_vod_playlist_schema)

        parsed = urlparse(template)
        url_template = "{0}://{1}{2}".format(
            parsed.scheme, parsed.netloc, playlist["template"]
        )
        segment_max = reduce(lambda i,j: i + j[0], playlist["fragments"], 0)

        substreams = [HTTPStream(self.session,
                                 url_template.replace("$fragment$", str(i)))
                      for i in range(1, segment_max + 1)]

        return FLVPlaylist(self.session,
                           duration=playlist["duration"],
                           flatten_timestamps=True,
                           skip_header=True,
                           streams=substreams)

    def _get_vod_streams(self, params):
        manifest_url = params.get("autoURL")
        if not manifest_url:
            return

        res = http.get(manifest_url)
        manifest = http.json(res, schema=_vod_manifest_schema)
        for params in manifest["alternates"]:
            name = "{0}p".format(params["height"])
            stream = self._create_flv_playlist(params["template"])
            yield name, stream

            failovers = params.get("failover", [])
            for failover in failovers:
                stream = self._create_flv_playlist(failover)
                yield name, stream

    def _get_streams(self):
        match = _url_re.match(self.url)
        media_id = match.group("media_id")

        return self._get_streams_from_media(media_id)

__plugin__ = DailyMotion
<commit_msg>plugins.dailymotion: Add support for HDS VODs.<commit_after>import re

from functools import reduce

from livestreamer.compat import urlparse, range
from livestreamer.plugin import Plugin
from livestreamer.plugin.api import http, validate
from livestreamer.stream import HDSStream, HTTPStream, RTMPStream
from livestreamer.stream.playlist import FLVPlaylist

COOKIES = {
    "family_filter": "off",
    "ff": "off"
}
QUALITY_MAP = {
    "ld": "240p",
    "sd": "360p",
    "hq": "480p",
    "hd720": "720p",
    "hd1080": "1080p",
    "custom": "live",
    "auto": "hds",
    "source": "hds"
}
STREAM_INFO_URL = "http://www.dailymotion.com/sequence/full/{0}"

_rtmp_re = re.compile("""
    (?P<host>rtmp://[^/]+)
    /(?P<app>[^/]+)
    /(?P<playpath>.+)
""", re.VERBOSE)
_url_re = re.compile("""
    http(s)?://(\w+\.)?
    dailymotion.com
    (/embed)?/(video|live)
    /(?P<media_id>[^_?/]+)
""", re.VERBOSE)

_media_inner_schema = validate.Schema([{
    "layerList": [{
        "name": validate.text,
        validate.optional("sequenceList"): [{
            "layerList": validate.all(
                [{
                    "name": validate.text,
                    validate.optional("param"): dict
                }],
                validate.filter(lambda l: l["name"] in ("video", "reporting"))
            )
        }]
    }]
}])
_media_schema = validate.Schema(
    validate.any(
        _media_inner_schema,
        validate.all(
            {"sequence": _media_inner_schema},
            validate.get("sequence")
        )
    )
)
_vod_playlist_schema = validate.Schema({
    "duration": float,
    "fragments": [[int, float]],
    "template": validate.text
})
_vod_manifest_schema = validate.Schema({
    "alternates": [{
        "height": int,
        "template": validate.text,
        validate.optional("failover"): [validate.text]
    }]
})

class DailyMotion(Plugin):
    @classmethod
    def can_handle_url(self, url):
        return _url_re.match(url)

    def _get_streams_from_media(self, media_id):
        res = http.get(STREAM_INFO_URL.format(media_id), cookies=COOKIES)
        media = http.json(res, schema=_media_schema)

        params = extra_params = swf_url = None
        for __ in media:
            for __ in __["layerList"]:
                for __ in __.get("sequenceList", []):
                    for layer in __["layerList"]:
                        name = layer["name"]
                        if name == "video":
                            params = layer.get("param")
                        elif name == "reporting":
                            extra_params = layer.get("param", {})
                            extra_params = extra_params.get("extraParams", {})

        if not params:
            return

        if extra_params:
            swf_url = extra_params.get("videoSwfURL")

        mode = params.get("mode")
        if mode == "live":
            return self._get_live_streams(params, swf_url)
        elif mode == "vod":
            return self._get_vod_streams(params)

    def _get_live_streams(self, params, swf_url):
        for key, quality in QUALITY_MAP.items():
            key_url = "{0}URL".format(key)
            url = params.get(key_url)

            if not url:
                continue

            try:
                res = http.get(url, exception=IOError)
            except IOError:
                continue

            if quality == "hds":
                streams = HDSStream.parse_manifest(self.session, res.url)
                for name, stream in streams.items():
                    if key == "source":
                        name += "+"

                    yield name, stream
            elif res.text.startswith("rtmp"):
                match = _rtmp_re.match(res.text)
                if not match:
                    continue

                stream = RTMPStream(self.session, {
                    "rtmp": match.group("host"),
                    "app": match.group("app"),
                    "playpath": match.group("playpath"),
                    "swfVfy": swf_url,
                    "live": True
                })

                yield quality, stream

    def _create_flv_playlist(self, template):
        res = http.get(template)
        playlist = http.json(res, schema=_vod_playlist_schema)

        parsed = urlparse(template)
        url_template = "{0}://{1}{2}".format(
            parsed.scheme, parsed.netloc, playlist["template"]
        )
        segment_max = reduce(lambda i,j: i + j[0], playlist["fragments"], 0)

        substreams = [HTTPStream(self.session,
                                 url_template.replace("$fragment$", str(i)))
                      for i in range(1, segment_max + 1)]

        return FLVPlaylist(self.session,
                           duration=playlist["duration"],
                           flatten_timestamps=True,
                           skip_header=True,
                           streams=substreams)

    def _get_vod_streams(self, params):
        manifest_url = params.get("autoURL")
        if not manifest_url:
            return

        res = http.get(manifest_url)
        if res.headers.get("Content-Type") == "application/f4m+xml":
            streams = HDSStream.parse_manifest(self.session, res.url)

            # TODO: Replace with "yield from" when dropping Python 2.
            for __ in streams.items():
                yield __
        else:
            manifest = http.json(res, schema=_vod_manifest_schema)
            for params in manifest["alternates"]:
                name = "{0}p".format(params["height"])
                stream = self._create_flv_playlist(params["template"])
                yield name, stream

                failovers = params.get("failover", [])
                for failover in failovers:
                    stream = self._create_flv_playlist(failover)
                    yield name, stream

    def _get_streams(self):
        match = _url_re.match(self.url)
        media_id = match.group("media_id")

        return self._get_streams_from_media(media_id)

__plugin__ = DailyMotion
<|endoftext|>


<commit_before># coding=utf-8
# Filename: h5concat.py
"""
Convert ROOT and EVT files to HDF5.

Usage:
    h5concat [--verbose] OUTFILE INFILES...
    h5concat (-h | --help)
    h5concat --version

Options:
    -h --help               Show this screen.
    --verbose               Print out more progress. [default: False].
"""

from __future__ import division, absolute_import, print_function

from km3modules import StatusBar
from km3pipe import version

__author__ = "Tamas Gal"
__copyright__ = "Copyright 2016, Tamas Gal and the KM3NeT collaboration."
__credits__ = []
__license__ = "MIT"
__maintainer__ = "Tamas Gal and Moritz Lotze"
__email__ = "tgal@km3net.de"
__status__ = "Development"

def h5concat(output_file, input_files, verbose=False):
    """Convert Any file to HDF5 file"""
    from km3pipe import Pipeline  # noqa
    from km3pipe.io import HDF5Pump, HDF5Sink  # noqa

    pipe = Pipeline()
    pipe.attach(HDF5Pump, filenames=input_files, verbose=verbose)
    if verbose:
        pipe.attach(StatusBar, every=100)
    else:
        pipe.attach(StatusBar, every=1000)
    pipe.attach(HDF5Sink, filename=output_file)
    pipe.drain()

def main():
    from docopt import docopt
    args = docopt(__doc__, version=version)

    infiles = args['INFILES']
    outfile = args['OUTFILE']
    verb = bool(args['--verbose'])

    h5concat(outfile, infiles, verbose=verb)
<commit_msg>H5concat: use H5mono pump, optionally<commit_after># coding=utf-8
# Filename: h5concat.py
"""
Convert ROOT and EVT files to HDF5.

Usage:
    h5concat [--verbose] [--ignore-id] OUTFILE INFILES...
    h5concat (-h | --help)
    h5concat --version

Options:
    -h --help               Show this screen.
    --verbose               Print out more progress. [default: False].
    --ignore-id             use "Events are rows", not "read_where('X == event_id')"
"""

from __future__ import division, absolute_import, print_function

from km3modules import StatusBar
from km3pipe import version

__author__ = "Tamas Gal"
__copyright__ = "Copyright 2016, Tamas Gal and the KM3NeT collaboration."
__credits__ = []
__license__ = "MIT"
__maintainer__ = "Tamas Gal and Moritz Lotze"
__email__ = "tgal@km3net.de"
__status__ = "Development"

def h5concat(output_file, input_files, verbose=False, ignore_id=False):
    """Convert Any file to HDF5 file"""
    from km3pipe import Pipeline  # noqa
    from km3pipe.io import HDF5Pump, HDF5Sink  # noqa

    pipe = Pipeline()
    pipe.attach(HDF5Pump, filenames=input_files, verbose=verbose)
    if verbose:
        pipe.attach(StatusBar, every=100)
    else:
        pipe.attach(StatusBar, every=1000)
    if ignore_id:
        pipe.attach(H5Mono, filename=output_file)
    else:
        pipe.attach(HDF5Sink, filename=output_file)
    pipe.drain()

def main():
    from docopt import docopt
    args = docopt(__doc__, version=version)

    infiles = args['INFILES']
    outfile = args['OUTFILE']
    ignore_id=bool(args['--ignore-id'])
    verb = bool(args['--verbose'])

    h5concat(outfile, infiles, verbose=verb, ignore_id=ignore_id)
<|endoftext|>