<commit_before># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class LlvmFlang(CMakePackage, CudaPackage):
    """LLVM-Flang is the Flang fork of LLVM needed by the Flang package."""

    homepage = "https://github.com/flang-compiler"

    git      = "https://github.com/flang-compiler/llvm.git"

    maintainer = ['naromero77']

    version('master', branch='master')
    version('release_70', branch='release_70')
    version('release_60', branch='release_60')
    version('20190329', tag='flang_20190329')
    version('20181226_70', tag='20181226_70')
    version('20181226_60', tag='20181226_60')
    version('20180921', tag='20180921')
    version('20180319', tag='20180319')
    version('20180328', tag='20180308')

    # Variants
    variant('all_targets', default=False,
            description='Build all supported targets')

    variant('build_type', default='Release',
            description='The CMake build type to build',
            values=('Debug', 'Release', 'RelWithDebInfo', 'MinSizeRel'))

    # Universal dependency
    depends_on('cmake@3.8:', type='build')
    depends_on('python@2.7:', type='build')

    # openmp dependencies
    depends_on('perl-data-dumper', type=('build'))
    depends_on('hwloc')

    # libomptarget dependencies
    depends_on('libelf', when='+cuda')
    depends_on('libffi', when='+cuda')
    depends_on('cuda@:9', when='+cuda')  # llvm 7 not compatible with newer version of cuda

    # LLVM-Flang Componentes: Driver, OpenMP
    resource(name='flang-driver',
             git='https://github.com/flang-compiler/flang-driver.git',
             branch='master',
             destination='tools',
             placement='clang',
             when='@master')

    resource(name='flang-driver',
             git='https://github.com/flang-compiler/flang-driver.git',
             branch='release_70',
             destination='tools',
             placement='clang',
             when='@release_70')

    resource(name='flang-driver',
             git='https://github.com/flang-compiler/flang-driver.git',
             branch='release_60',
             destination='tools',
             placement='clang',
             when='@release_60')

    resource(name='flang-driver',
             git='https://github.com/flang-compiler/flang-driver.git',
             tag='flang_20190329',
             destination='tools',
             placement='clang',
             when='@20190329')

    resource(name='flang-driver',
             git='https://github.com/flang-compiler/flang-driver.git',
             tag='flang_20181226_70',
             destination='tools',
             placement='clang',
             when='@20181226_70')

    resource(name='flang-driver',
             git='https://github.com/flang-compiler/flang-driver.git',
             tag='flang_20181226_60',
             destination='tools',
             placement='clang',
             when='@20181226_60')

    resource(name='flang-driver',
             git='https://github.com/flang-compiler/flang-driver.git',
             tag='flang_20180921',
             destination='tools',
             placement='clang',
             when='@20180921')

    resource(name='flang-driver',
             git='https://github.com/flang-compiler/flang-driver.git',
             tag='flang_20180921',
             destination='tools',
             placement='clang',
             when='@20180308')

    resource(name='openmp',
             git='https://github.com/flang-compiler/openmp.git',
             branch='master',
             destination='projects',
             placement='openmp',
             when='@master')

    resource(name='openmp',
             git='https://github.com/flang-compiler/openmp.git',
             branch='release_70',
             destination='projects',
             placement='openmp',
             when='@release_70')

    resource(name='openmp',
             git='https://github.com/flang-compiler/openmp.git',
             branch='release_60',
             destination='projects',
             placement='openmp',
             when='@release_60')

    resource(name='openmp',
             git='https://github.com/flang-compiler/openmp.git',
             tag='flang_20190329',
             destination='projects',
             placement='openmp',
             when='@20190329')

    resource(name='openmp',
             git='https://github.com/flang-compiler/openmp.git',
             tag='flang_20181226_70',
             destination='projects',
             placement='openmp',
             when='@20181226_70')

    resource(name='openmp',
             git='https://github.com/flang-compiler/openmp.git',
             tag='flang_20181226_60',
             destination='projects',
             placement='openmp',
             when='@20181226_60')

    resource(name='openmp',
             git='https://github.com/flang-compiler/openmp.git',
             tag='flang_20180921',
             destination='projects',
             placement='openmp',
             when='@20180921')

    def cmake_args(self):
        spec = self.spec
        # universal
        args = [
            '-DLLVM_ENABLE_RTTI:BOOL=ON',
            '-DLLVM_ENABLE_EH:BOOL=ON',
            '-DCLANG_DEFAULT_OPENMP_RUNTIME:STRING=libomp',
        ]
        args.append('-DPYTHON_EXECUTABLE={0}'.format(
            spec['python'].command.path))

        # needed by flang-driver
        args.append('-DFLANG_LLVM_EXTENSIONS=ON')

        if '+all_targets' not in spec:  # all is default in cmake
            if spec.target.family == 'x86' or spec.target.family == 'x86_64':
                target = 'X86'
            elif spec.target.family == 'arm':
                target = 'ARM'
            elif spec.target.family == 'aarch64':
                target = 'AArch64'
            elif (spec.target.family == 'ppc64' or
                  spec.target.family == 'ppc64le' or
                  spec.target.family == 'ppc' or
                  spec.target.family == 'ppcle'):
                target = 'PowerPC'
            else:
                raise InstallError(
                    'Unsupported architecture: ' + spec.target.family)

            if '+cuda' in spec:
                args.append(
                    '-DLLVM_TARGETS_TO_BUILD:STRING=NVPTX;' + target)
            else:
                args.append(
                    '-DLLVM_TARGETS_TO_BUILD:STRING=' + target)

        # used by openmp
        args.append('-DLIBOMP_USE_HWLOC=On')
        args.append('-DLIBOMP_FORTRAN_MODULES=ON')
        args.append('-DLIBOMP_ENABLE_SHARED=TRUE')

        # used by libomptarget for NVidia gpu
        if '+cuda' in spec:
            args.append('-DOPENMP_ENABLE_LIBOMPTARGET=ON')
            cuda_arch_list = spec.variants['cuda_arch'].value
            args.append('-DCUDA_TOOLKIT_ROOT_DIR=%s' % spec['cuda'].prefix)
            args.append('-DLIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES={0}'.format(
                ','.join(cuda_arch_list)))
            args.append('-DCLANG_OPENMP_NVPTX_DEFAULT_ARCH=sm_{0}'.format(
                cuda_arch_list[-1]))
        else:
            args.append('-DOPENMP_ENABLE_LIBOMPTARGET=OFF')

        return args

    @run_after("install")
    def post_install(self):
        spec = self.spec

        # Manual bootstrap needed to get NVidia BC compiled with the
        # clang that was just built
        if '+cuda' in spec:
            ompdir = 'build-bootstrapped-omp'
            # rebuild libomptarget to get bytecode runtime library files
            with working_dir(ompdir, create=True):
                args = [
                    self.stage.source_path + '/projects/openmp',
                    '-DCMAKE_C_COMPILER:PATH={0}'.format(
                        spec.prefix.bin + '/clang'),
                    '-DCMAKE_CXX_COMPILER:PATH={0}'.format(
                        spec.prefix.bin + '/clang++'),
                    '-DCMAKE_INSTALL_PREFIX:PATH={0}'.format(
                        spec.prefix)
                ]
                args = args + self.cmake_args()
                # args = self.cmake_args()
                # enable CUDA bitcode
                args.append('-DLIBOMPTARGET_NVPTX_ENABLE_BCLIB=true')
                # work around bad libelf detection in libomptarget
                args.append(
                    '-DCMAKE_CXX_FLAGS:String=-I{0} -I{1}'.format(
                        spec['libelf'].prefix.include,
                        spec['hwloc'].prefix.include))

        cmake(*args)
        make()
        make('install')
<commit_msg>llvm-flang: Only build offload code if cuda enabled (#17466)<commit_after># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class LlvmFlang(CMakePackage, CudaPackage):
    """LLVM-Flang is the Flang fork of LLVM needed by the Flang package."""

    homepage = "https://github.com/flang-compiler"

    git      = "https://github.com/flang-compiler/llvm.git"

    maintainer = ['naromero77']

    version('master', branch='master')
    version('release_70', branch='release_70')
    version('release_60', branch='release_60')
    version('20190329', tag='flang_20190329')
    version('20181226_70', tag='20181226_70')
    version('20181226_60', tag='20181226_60')
    version('20180921', tag='20180921')
    version('20180319', tag='20180319')
    version('20180328', tag='20180308')

    # Variants
    variant('all_targets', default=False,
            description='Build all supported targets')

    variant('build_type', default='Release',
            description='The CMake build type to build',
            values=('Debug', 'Release', 'RelWithDebInfo', 'MinSizeRel'))

    # Universal dependency
    depends_on('cmake@3.8:', type='build')
    depends_on('python@2.7:', type='build')

    # openmp dependencies
    depends_on('perl-data-dumper', type=('build'))
    depends_on('hwloc')

    # libomptarget dependencies
    depends_on('libelf', when='+cuda')
    depends_on('libffi', when='+cuda')
    depends_on('cuda@:9', when='+cuda')  # llvm 7 not compatible with newer version of cuda

    # LLVM-Flang Componentes: Driver, OpenMP
    resource(name='flang-driver',
             git='https://github.com/flang-compiler/flang-driver.git',
             branch='master',
             destination='tools',
             placement='clang',
             when='@master')

    resource(name='flang-driver',
             git='https://github.com/flang-compiler/flang-driver.git',
             branch='release_70',
             destination='tools',
             placement='clang',
             when='@release_70')

    resource(name='flang-driver',
             git='https://github.com/flang-compiler/flang-driver.git',
             branch='release_60',
             destination='tools',
             placement='clang',
             when='@release_60')

    resource(name='flang-driver',
             git='https://github.com/flang-compiler/flang-driver.git',
             tag='flang_20190329',
             destination='tools',
             placement='clang',
             when='@20190329')

    resource(name='flang-driver',
             git='https://github.com/flang-compiler/flang-driver.git',
             tag='flang_20181226_70',
             destination='tools',
             placement='clang',
             when='@20181226_70')

    resource(name='flang-driver',
             git='https://github.com/flang-compiler/flang-driver.git',
             tag='flang_20181226_60',
             destination='tools',
             placement='clang',
             when='@20181226_60')

    resource(name='flang-driver',
             git='https://github.com/flang-compiler/flang-driver.git',
             tag='flang_20180921',
             destination='tools',
             placement='clang',
             when='@20180921')

    resource(name='flang-driver',
             git='https://github.com/flang-compiler/flang-driver.git',
             tag='flang_20180921',
             destination='tools',
             placement='clang',
             when='@20180308')

    resource(name='openmp',
             git='https://github.com/flang-compiler/openmp.git',
             branch='master',
             destination='projects',
             placement='openmp',
             when='@master')

    resource(name='openmp',
             git='https://github.com/flang-compiler/openmp.git',
             branch='release_70',
             destination='projects',
             placement='openmp',
             when='@release_70')

    resource(name='openmp',
             git='https://github.com/flang-compiler/openmp.git',
             branch='release_60',
             destination='projects',
             placement='openmp',
             when='@release_60')

    resource(name='openmp',
             git='https://github.com/flang-compiler/openmp.git',
             tag='flang_20190329',
             destination='projects',
             placement='openmp',
             when='@20190329')

    resource(name='openmp',
             git='https://github.com/flang-compiler/openmp.git',
             tag='flang_20181226_70',
             destination='projects',
             placement='openmp',
             when='@20181226_70')

    resource(name='openmp',
             git='https://github.com/flang-compiler/openmp.git',
             tag='flang_20181226_60',
             destination='projects',
             placement='openmp',
             when='@20181226_60')

    resource(name='openmp',
             git='https://github.com/flang-compiler/openmp.git',
             tag='flang_20180921',
             destination='projects',
             placement='openmp',
             when='@20180921')

    def cmake_args(self):
        spec = self.spec
        # universal
        args = [
            '-DLLVM_ENABLE_RTTI:BOOL=ON',
            '-DLLVM_ENABLE_EH:BOOL=ON',
            '-DCLANG_DEFAULT_OPENMP_RUNTIME:STRING=libomp',
        ]
        args.append('-DPYTHON_EXECUTABLE={0}'.format(
            spec['python'].command.path))

        # needed by flang-driver
        args.append('-DFLANG_LLVM_EXTENSIONS=ON')

        if '+all_targets' not in spec:  # all is default in cmake
            if spec.target.family == 'x86' or spec.target.family == 'x86_64':
                target = 'X86'
            elif spec.target.family == 'arm':
                target = 'ARM'
            elif spec.target.family == 'aarch64':
                target = 'AArch64'
            elif (spec.target.family == 'ppc64' or
                  spec.target.family == 'ppc64le' or
                  spec.target.family == 'ppc' or
                  spec.target.family == 'ppcle'):
                target = 'PowerPC'
            else:
                raise InstallError(
                    'Unsupported architecture: ' + spec.target.family)

            if '+cuda' in spec:
                args.append(
                    '-DLLVM_TARGETS_TO_BUILD:STRING=NVPTX;' + target)
            else:
                args.append(
                    '-DLLVM_TARGETS_TO_BUILD:STRING=' + target)

        # used by openmp
        args.append('-DLIBOMP_USE_HWLOC=On')
        args.append('-DLIBOMP_FORTRAN_MODULES=ON')
        args.append('-DLIBOMP_ENABLE_SHARED=TRUE')

        # used by libomptarget for NVidia gpu
        if '+cuda' in spec:
            args.append('-DOPENMP_ENABLE_LIBOMPTARGET=ON')
            cuda_arch_list = spec.variants['cuda_arch'].value
            args.append('-DCUDA_TOOLKIT_ROOT_DIR=%s' % spec['cuda'].prefix)
            args.append('-DLIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES={0}'.format(
                ','.join(cuda_arch_list)))
            args.append('-DCLANG_OPENMP_NVPTX_DEFAULT_ARCH=sm_{0}'.format(
                cuda_arch_list[-1]))
        else:
            args.append('-DOPENMP_ENABLE_LIBOMPTARGET=OFF')

        return args

    @run_after("install")
    def post_install(self):
        spec = self.spec

        # Manual bootstrap needed to get NVidia BC compiled with the
        # clang that was just built
        if '+cuda' in spec:
            ompdir = 'build-bootstrapped-omp'
            # rebuild libomptarget to get bytecode runtime library files
            with working_dir(ompdir, create=True):
                args = [
                    self.stage.source_path + '/projects/openmp',
                    '-DCMAKE_C_COMPILER:PATH={0}'.format(
                        spec.prefix.bin + '/clang'),
                    '-DCMAKE_CXX_COMPILER:PATH={0}'.format(
                        spec.prefix.bin + '/clang++'),
                    '-DCMAKE_INSTALL_PREFIX:PATH={0}'.format(
                        spec.prefix)
                ]
                args = args + self.cmake_args()
                # args = self.cmake_args()
                # enable CUDA bitcode
                args.append('-DLIBOMPTARGET_NVPTX_ENABLE_BCLIB=true')
                # work around bad libelf detection in libomptarget
                args.append(
                    '-DCMAKE_CXX_FLAGS:String=-I{0} -I{1}'.format(
                        spec['libelf'].prefix.include,
                        spec['hwloc'].prefix.include))

            # Only build if offload target.
            cmake(*args)
            make()
            make('install')
<|endoftext|>


<commit_before>##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *

class Postgresql(AutotoolsPackage):
    """PostgreSQL is a powerful, open source object-relational database system.
    It has more than 15 years of active development and a proven architecture
    that has earned it a strong reputation for reliability, data integrity, and
    correctness."""

    homepage = "http://www.postgresql.org/"
    url      = "http://ftp.postgresql.org/pub/source/v9.3.4/postgresql-9.3.4.tar.bz2"

    version('10.2', 'e97c3cc72bdf661441f29069299b260a')
    version('9.3.4', 'd0a41f54c377b2d2fab4a003b0dac762')
    version('9.5.3', '3f0c388566c688c82b01a0edf1e6b7a0')

    depends_on('openssl')
    depends_on('readline')

    variant('threadsafe', default=False, description='Build with thread safe.')

    def configure_arg(self):
        config_args = ["--with-openssl"]
        if '+threadsafe' in self.spec:
            config_args.append("--enable-thread-safety")
        else:
            config_args.append("--disable-thread-safety")

        return config_args
<commit_msg>postgresql: add newest version 10.3 (#7455)<commit_after>##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *

class Postgresql(AutotoolsPackage):
    """PostgreSQL is a powerful, open source object-relational database system.
    It has more than 15 years of active development and a proven architecture
    that has earned it a strong reputation for reliability, data integrity, and
    correctness."""

    homepage = "http://www.postgresql.org/"
    url      = "http://ftp.postgresql.org/pub/source/v9.3.4/postgresql-9.3.4.tar.bz2"

    version('10.3', '506498796a314c549388cafb3d5c717a')
    version('10.2', 'e97c3cc72bdf661441f29069299b260a')
    version('9.3.4', 'd0a41f54c377b2d2fab4a003b0dac762')
    version('9.5.3', '3f0c388566c688c82b01a0edf1e6b7a0')

    depends_on('openssl')
    depends_on('readline')

    variant('threadsafe', default=False, description='Build with thread safe.')

    def configure_arg(self):
        config_args = ["--with-openssl"]
        if '+threadsafe' in self.spec:
            config_args.append("--enable-thread-safety")
        else:
            config_args.append("--disable-thread-safety")

        return config_args
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys
import os

import argparse
import h5py as h5
from keras import backend as K
from keras import layers as kl
import numpy as np
import logging

from deepcpg import data as dat
from deepcpg import models as mod
from deepcpg.data import hdf, dna
from deepcpg.utils import ProgressBar, to_list, linear_weights

def get_layer_by_depth(layers, depth=1, layer_class=kl.Activation):
    idx = 1
    for layer in layers:
        if isinstance(layer, layer_class):
            if idx == depth:
                return layer
            else:
                idx += 1
    return None

class App(object):

    def run(self, args):
        name = os.path.basename(args[0])
        parser = self.create_parser(name)
        opts = parser.parse_args(args[1:])
        self.opts = opts
        return self.main(name, opts)

    def create_parser(self, name):
        p = argparse.ArgumentParser(
            prog=name,
            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
            description='Compute filter activations')
        p.add_argument(
            'data_files',
            help='Data files',
            nargs='+')
        p.add_argument(
            '--model_files',
            help='Model files',
            nargs='+')
        p.add_argument(
            '-o', '--out_file',
            help='Output file')
        p.add_argument(
            '--act_layer',
            help='Name or depth of activation layer',
            default='1')
        p.add_argument(
            '--weight_layer',
            help='Name or depth of weight layer',
            default='1')
        p.add_argument(
            '--act_fun',
            help='Function applied to reduce sequence window activations',
            choices=['mean', 'wmean', 'max'])
        p.add_argument(
            '--act_wlen',
            help='Slice wlen at center',
            type=int)
        p.add_argument(
            '--store_preds',
            help='Store model predictions',
            action='store_true')
        p.add_argument(
            '--store_inputs',
            help='Store model inputs',
            action='store_true')
        p.add_argument(
            '--nb_sample',
            help='Number of samples',
            type=int)
        p.add_argument(
            '--batch_size',
            help='Batch size',
            type=int,
            default=128)
        p.add_argument(
            '--seed',
            help='Seed of rng',
            type=int,
            default=0)
        p.add_argument(
            '--verbose',
            help='More detailed log messages',
            action='store_true')
        p.add_argument(
            '--log_file',
            help='Write log messages to file')
        return p

    def main(self, name, opts):
        logging.basicConfig(filename=opts.log_file,
                            format='%(levelname)s (%(asctime)s): %(message)s')
        log = logging.getLogger(name)
        if opts.verbose:
            log.setLevel(logging.DEBUG)
        else:
            log.setLevel(logging.INFO)
            log.debug(opts)

        if opts.seed is not None:
            np.random.seed(opts.seed)

        if not opts.model_files:
            raise ValueError('No model files provided!')

        if opts.act_wlen is not None and opts.act_wlen % 2 == 0:
            raise ValueError('Activation window length must be divisible by ' +
                             'two!')

        log.info('Loading model ...')
        K.set_learning_phase(0)
        model = mod.load_model(opts.model_files)

        if opts.act_layer.isdigit():
            act_layer = get_layer_by_depth(model.layers, int(opts.act_layer),
                                           layer_class=kl.Activation)
        else:
            act_layer = model.get_layer(opts.act_layer)
        if not act_layer:
            raise ValueError('Activation layer not found!')
        log.info('Using activation layer "%s"' % act_layer.name)
        fun_outputs = to_list(act_layer.output)
        if opts.store_preds:
            fun_outputs += to_list(model.output)
        fun = K.function(to_list(model.input), fun_outputs)

        log.info('Reading data ...')
        nb_sample = dat.get_nb_sample(opts.data_files, opts.nb_sample)
        data_reader = mod.data_reader_from_model(model)
        data_reader = data_reader(opts.data_files,
                                  nb_sample=nb_sample,
                                  batch_size=opts.batch_size,
                                  loop=False,
                                  shuffle=False)

        meta_reader = hdf.reader(opts.data_files, ['chromo', 'pos'],
                                 nb_sample=nb_sample,
                                 batch_size=opts.batch_size,
                                 loop=False,
                                 shuffle=False)

        out_file = h5.File(opts.out_file, 'w')
        out_group = out_file

        if opts.weight_layer:
            if opts.weight_layer.isdigit():
                weight_layer = get_layer_by_depth(model.layers,
                                                  int(opts.weight_layer),
                                                  layer_class=kl.Conv1D)
            else:
                weight_layer = model.get_layer(opts.weight_layer)
            if not weight_layer:
                raise ValueError('Weight layer not found!')
            log.info('Using weight layer "%s"' % weight_layer.name)
            weights = weight_layer.get_weights()
            out_group['weights/weights'] = weights[0]
            out_group['weights/bias'] = weights[1]

        def h5_dump(path, data, idx, dtype=None, compression='gzip'):
            if path not in out_group:
                if dtype is None:
                    dtype = data.dtype
                out_group.create_dataset(
                    name=path,
                    shape=[nb_sample] + list(data.shape[1:]),
                    dtype=dtype,
                    compression=compression
                )
            out_group[path][idx:idx+len(data)] = data

        progbar = ProgressBar(nb_sample, log.info)
        idx = 0
        for inputs, outputs, weights in data_reader:
            if isinstance(inputs, dict):
                inputs = list(inputs.values())
            batch_size = len(inputs[0])
            progbar.update(batch_size)

            if opts.store_inputs:
                for i, name in enumerate(model.input_names):
                    h5_dump('inputs/%s' % name,
                            dna.onehot_to_int(inputs[i]), idx)

            fun_eval = fun(inputs)
            act = fun_eval[0]

            if opts.act_wlen is not None:
                delta = opts.act_wlen // 2
                ctr = act.shape[1] // 2
                act = act[:, (ctr-delta):(ctr+delta+1)]

            if opts.act_fun is not None:
                if opts.act_fun == 'mean':
                    act = act.mean(axis=1)
                elif opts.act_fun == 'wmean':
                    weights = linear_weights(act.shape[1])
                    act = np.average(act, axis=1, weights=weights)
                elif opts.act_fun == 'max':
                    act = act.max(axis=1)
                else:
                    raise ValueError('Invalid function "%s"!' % (opts.act_fun))

            h5_dump('act', act, idx)

            if opts.store_preds:
                preds = fun_eval[1:]
                for i, name in enumerate(model.output_names):
                    h5_dump('preds/%s' % name, preds[i].squeeze(), idx)

            for name, value in next(meta_reader).items():
                h5_dump(name, value, idx)

            idx += batch_size
        progbar.close()

        out_file.close()
        log.info('Done!')

        return 0

if __name__ == '__main__':
    app = App()
    app.run(sys.argv)
<commit_msg>Update dcpg_filter_act.py<commit_after>#!/usr/bin/env python

import sys
import os

import argparse
import h5py as h5
from keras import backend as K
from keras import layers as kl
import numpy as np
import logging

from deepcpg import data as dat
from deepcpg import models as mod
from deepcpg.data import hdf, dna
from deepcpg.utils import ProgressBar, to_list, linear_weights

def get_layer_by_depth(layers, depth=1, layer_class=kl.Activation):
    idx = 1
    for layer in layers:
        if isinstance(layer, layer_class):
            if idx == depth:
                return layer
            else:
                idx += 1
    return None

class App(object):

    def run(self, args):
        name = os.path.basename(args[0])
        parser = self.create_parser(name)
        opts = parser.parse_args(args[1:])
        self.opts = opts
        return self.main(name, opts)

    def create_parser(self, name):
        p = argparse.ArgumentParser(
            prog=name,
            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
            description='Compute filter activations')
        p.add_argument(
            'data_files',
            help='Data files',
            nargs='+')
        p.add_argument(
            '--model_files',
            help='Model files',
            nargs='+')
        p.add_argument(
            '-o', '--out_file',
            help='Output file')
        p.add_argument(
            '--act_layer',
            help='Name or depth of activation layer',
            default='1')
        p.add_argument(
            '--weight_layer',
            help='Name or depth of weight layer',
            default='1')
        p.add_argument(
            '--act_fun',
            help='Function applied to reduce sequence window activations',
            choices=['mean', 'wmean', 'max'])
        p.add_argument(
            '--act_wlen',
            help='Slice wlen at center',
            type=int)
        p.add_argument(
            '--store_outputs',
            help='Store output labels',
            action='store_true')
        p.add_argument(
            '--store_preds',
            help='Store model predictions',
            action='store_true')
        p.add_argument(
            '--store_inputs',
            help='Store model inputs',
            action='store_true')
        p.add_argument(
            '--nb_sample',
            help='Number of samples',
            type=int)
        p.add_argument(
            '--batch_size',
            help='Batch size',
            type=int,
            default=128)
        p.add_argument(
            '--seed',
            help='Seed of rng',
            type=int,
            default=0)
        p.add_argument(
            '--verbose',
            help='More detailed log messages',
            action='store_true')
        p.add_argument(
            '--log_file',
            help='Write log messages to file')
        return p

    def main(self, name, opts):
        logging.basicConfig(filename=opts.log_file,
                            format='%(levelname)s (%(asctime)s): %(message)s')
        log = logging.getLogger(name)
        if opts.verbose:
            log.setLevel(logging.DEBUG)
        else:
            log.setLevel(logging.INFO)
            log.debug(opts)

        if opts.seed is not None:
            np.random.seed(opts.seed)

        if not opts.model_files:
            raise ValueError('No model files provided!')

        if opts.act_wlen is not None and opts.act_wlen % 2 == 0:
            raise ValueError('Activation window length must be divisible by ' +
                             'two!')

        log.info('Loading model ...')
        K.set_learning_phase(0)
        model = mod.load_model(opts.model_files)

        if opts.act_layer.isdigit():
            act_layer = get_layer_by_depth(model.layers, int(opts.act_layer),
                                           layer_class=kl.Activation)
        else:
            act_layer = model.get_layer(opts.act_layer)
        if not act_layer:
            raise ValueError('Activation layer not found!')
        log.info('Using activation layer "%s"' % act_layer.name)
        fun_outputs = to_list(act_layer.output)
        if opts.store_preds:
            fun_outputs += to_list(model.output)
        fun = K.function(to_list(model.input), fun_outputs)

        log.info('Reading data ...')
        nb_sample = dat.get_nb_sample(opts.data_files, opts.nb_sample)
        data_reader = mod.data_reader_from_model(model)
        data_reader = data_reader(opts.data_files,
                                  nb_sample=nb_sample,
                                  batch_size=opts.batch_size,
                                  loop=False,
                                  shuffle=False)

        meta_reader = hdf.reader(opts.data_files, ['chromo', 'pos'],
                                 nb_sample=nb_sample,
                                 batch_size=opts.batch_size,
                                 loop=False,
                                 shuffle=False)

        out_file = h5.File(opts.out_file, 'w')
        out_group = out_file

        if opts.weight_layer:
            if opts.weight_layer.isdigit():
                weight_layer = get_layer_by_depth(model.layers,
                                                  int(opts.weight_layer),
                                                  layer_class=kl.Conv1D)
            else:
                weight_layer = model.get_layer(opts.weight_layer)
            if not weight_layer:
                raise ValueError('Weight layer not found!')
            log.info('Using weight layer "%s"' % weight_layer.name)
            weights = weight_layer.get_weights()
            out_group['weights/weights'] = weights[0]
            out_group['weights/bias'] = weights[1]

        def h5_dump(path, data, idx, dtype=None, compression='gzip'):
            if path not in out_group:
                if dtype is None:
                    dtype = data.dtype
                out_group.create_dataset(
                    name=path,
                    shape=[nb_sample] + list(data.shape[1:]),
                    dtype=dtype,
                    compression=compression
                )
            out_group[path][idx:idx+len(data)] = data

        progbar = ProgressBar(nb_sample, log.info)
        idx = 0
        for inputs, outputs, weights in data_reader:
            if isinstance(inputs, dict):
                inputs = list(inputs.values())
            batch_size = len(inputs[0])
            progbar.update(batch_size)

            if opts.store_inputs:
                for i, name in enumerate(model.input_names):
                    h5_dump('inputs/%s' % name,
                            dna.onehot_to_int(inputs[i]), idx)

            if opts.store_outputs:
                for name, output in outputs.items():
                    h5_dump('outputs/%s' % name, output, idx)

            fun_eval = fun(inputs)
            act = fun_eval[0]

            if opts.act_wlen is not None:
                delta = opts.act_wlen // 2
                ctr = act.shape[1] // 2
                act = act[:, (ctr-delta):(ctr+delta+1)]

            if opts.act_fun is not None:
                if opts.act_fun == 'mean':
                    act = act.mean(axis=1)
                elif opts.act_fun == 'wmean':
                    weights = linear_weights(act.shape[1])
                    act = np.average(act, axis=1, weights=weights)
                elif opts.act_fun == 'max':
                    act = act.max(axis=1)
                else:
                    raise ValueError('Invalid function "%s"!' % (opts.act_fun))

            h5_dump('act', act, idx)

            if opts.store_preds:
                preds = fun_eval[1:]
                for i, name in enumerate(model.output_names):
                    h5_dump('preds/%s' % name, preds[i].squeeze(), idx)

            for name, value in next(meta_reader).items():
                h5_dump(name, value, idx)

            idx += batch_size
        progbar.close()

        out_file.close()
        log.info('Done!')

        return 0

if __name__ == '__main__':
    app = App()
    app.run(sys.argv)
<|endoftext|>


<commit_before>import re
import string

class Interface(object):
    """
    Represents a physical interface on a switch.
    """
    def __init__(self, switch, identifier):
        """
        Construct a new Interface with the given `identifier` located on the given `switch`.
        """
        self.switch = switch
        self.identifier = identifier

    def _get_name(self):
        """
        Get the friendly name configured for this interface.
        """
        run_output = self.switch.execute_command("show running-config interface " + str(self.identifier))
        # Try to extract the interface name, which may also contain spaces. This is achieved by greedily matching
        # whitespace at the end of the line and matching the `   name ` prefix a the beginning and using whatever
        # remains of the string as the interface name. The `name` group is matched in a non-greedy fashion as to not
        # "eat up" all the following whitespace which is not part of the name.
        name_match = re.search(r"^   name \"(?P<name>.*?)\"\s*$", run_output, re.MULTILINE)
        # Only return the name of the interface if a name was found.
        if name_match:
            return name_match.group('name')
        # If no name was configured, return None.
        return None

    def _set_name(self, value):
        """
        Configure the name `value` as the friendly name for this interface.
        """
        # Make sure that the name is legal according to the allowed interface names detailed in section 2-23 of the HP
        # Management and Configuration Guide
        assert(all(map(lambda letter: letter in (string.ascii_letters + string.digits), value)))
        # Issue the commands on the switch to set the new name.
        self.switch.execute_command("config")
        self.switch.execute_command("interface {0} name {1}".format(self.identifier, value))
        self.switch.execute_command("exit")

    def _del_name(self):
        """
        Deconfigure the friendly name on this interface.
        """
        self.switch.execute_command("config")
        self.switch.execute_command("no interface {0} name".format(self.identifier))
        self.switch.execute_command("exit")

    name = property(_get_name, _set_name, _del_name)
<commit_msg>Add ability to enable or disable an interface<commit_after>import re
import string

class Interface(object):
    """
    Represents a physical interface on a switch.
    """
    def __init__(self, switch, identifier):
        """
        Construct a new Interface with the given `identifier` located on the given `switch`.
        """
        self.switch = switch
        self.identifier = identifier

    def _get_name(self):
        """
        Get the friendly name configured for this interface.
        """
        run_output = self.switch.execute_command("show running-config interface " + str(self.identifier))
        # Try to extract the interface name, which may also contain spaces. This is achieved by greedily matching
        # whitespace at the end of the line and matching the `   name ` prefix a the beginning and using whatever
        # remains of the string as the interface name. The `name` group is matched in a non-greedy fashion as to not
        # "eat up" all the following whitespace which is not part of the name.
        name_match = re.search(r"^   name \"(?P<name>.*?)\"\s*$", run_output, re.MULTILINE)
        # Only return the name of the interface if a name was found.
        if name_match:
            return name_match.group('name')
        # If no name was configured, return None.
        return None

    def _set_name(self, value):
        """
        Configure the name `value` as the friendly name for this interface.
        """
        # Make sure that the name is legal according to the allowed interface names detailed in section 2-23 of the HP
        # Management and Configuration Guide
        assert(all(map(lambda letter: letter in (string.ascii_letters + string.digits), value)))
        # Issue the commands on the switch to set the new name.
        self.switch.execute_command("config")
        self.switch.execute_command("interface {0} name {1}".format(self.identifier, value))
        self.switch.execute_command("exit")

    def _del_name(self):
        """
        Deconfigure the friendly name on this interface.
        """
        self.switch.execute_command("config")
        self.switch.execute_command("no interface {0} name".format(self.identifier))
        self.switch.execute_command("exit")

    name = property(_get_name, _set_name, _del_name)

    def _get_enabled(self):
        """
        Get the admin status of this interface.
        """
        run_output = self.switch.execute_command("show running-config interface " + str(self.identifier))
        disable_match = re.search(r"^   disable\s*$", run_output, re.MULTILINE)
        # If nothing hinting at a disabled port has been matched, return True, else False.
        return disable_match is None

    def _set_enabled(self, value):
        """
        Set the admin status of this interface.
        """
        self.switch.execute_command("config")
        if value:
            self.switch.execute_command("interface {0} enable".format(self.identifier))
        else:
            self.switch.execute_command("interface {0} disable".format(self.identifier))
        self.switch.execute_command("exit")

    enabled = property(_get_enabled, _set_enabled)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
ENCODER TO ODOM

ROS Node for converting encoder ticks to a wheel odometry measurement of
position.
"""

import rospy
from tf import transformations as tft
import math
from nav_msgs.msg import Odometry
from geometry_msgs.msg import Quaternion
from snowmower_msgs.msg import EncMsg

def heading_to_quaternion(heading):
    """
    Converts a Euler yaw/heading angle to equivalent quaternion
    input: euler heading in radians
    output: nav_msgs.msg.Quaternion
    """

    quat = tft.quaternion_from_euler(0, 0, heading)

    quaternion = Quaternion()
    quaternion.x = quat[0]
    quaternion.y = quat[1]
    quaternion.z = quat[2]
    quaternion.w = quat[3]
    return quaternion

class WheelOdometryGenerator(object):
    """
    WheelOdometryGenerator: ROS Node that implements encoder ticks to position

    wog = WheelOdometryGenerator()
    wog.start_node()
    """

    # pylint: disable=too-many-instance-attributes
    # These attributes represent state, last data, and pub/sub

    # pylint: disable=invalid-name
    # The variables for state are valid as x, y, v (linear velocity)

    track_width = 1
    wheel_diam = .333
    encoders_per_rev = 100

    def __init__(self):
        self.x = 0
        self.y = 0
        self.heading = 0
        self.v = 0
        self.omega = 0

        self.last_enc = None

        self.seq = 1

        self.sub = None
        self.pub = None

    def process_encoder_msg(self, msg):
        """
        process_encoder_msg: callback to take in new data, and update state

        input: snowmower_msgs.msg.EncMsg
        output: None
        """

        if self.last_enc is None:
            self.last_enc = msg
        dleft = msg.left - self.last_enc.left
        dright = msg.right - self.last_enc.right
        # time is in secs
        dt = (msg.header.stamp.secs - self.last_enc.header.stamp.secs)*1.0
        if dt < 0:
            dt = 0.0

        if dleft == dright:
            v = self.distance(dleft) / dt
            omega = 0
        else:
            arc_left = self.distance(dleft)
            arc_right = self.distance(dright)

            if arc_left < arc_right:
                r = ((self.track_width/2)*
                    ((arc_right+arc_left)/(arc_right-arc_left)))
                theta = arc_left/(r-(self.track_width/2))
            else: # arc_left > arc_right:
                r = ((self.track_width/2)*
                    ((arc_right+arc_left)/(arc_left-arc_right)))
                theta = arc_left/(r+(self.track_width/2))
            v = ((dleft+dright)/2)/dt
            omega = theta/dt

        # based on v, omega, update state
        self.update_state(v, omega, dt)

        self.send_current_odom()
        self.last_enc = msg

    def send_current_odom(self):
        """
        send_current_odom: publish the current state as an Odometry message

        input: None
        output: (ROS publish) nav_msgs.msg.Odometry
        """

        o = Odometry()
        o.header.seq = self.seq
        o.header.stamp = rospy.Time.now()
        o.header.frame_id = '/odom'

        o.pose.pose.position.x = self.x
        o.pose.pose.position.y = self.y

        o.pose.pose.orientation = heading_to_quaternion(self.heading)

        o.twist.twist.linear.x = self.v
        o.twist.twist.angular.z = self.omega

        self.seq += 1

        self.pub.publish(o)

    def start_node(self):
        """
        start_node: Run the ROS node by intializing the publisher and subscriber
        """
        rospy.init_node('encoder2odom')
        self.pub = rospy.Publisher('/odom', Odometry, queue_size=1)
        self.sub = rospy.Subscriber('/enc', EncMsg, self.process_encoder_msg)
        rospy.loginfo('enc2odom: wheel odometry ready')
        rate = rospy.Rate(10)
        while not rospy.is_shutdown():
            self.send_current_odom()
            rate.sleep()

    # GEOMETRY

    def distance(self, ticks):
        """
        distance: calculate the distance traveled per given ticks based on given
            robot geometry
        """
        return ticks*1.0/self.encoders_per_rev*math.pi*self.wheel_diam

    def update_state(self, avg_v, avg_omega, dt):
        """
        update_state: move the state forward one step

        state has:
        initial x, y, heading, linear velocity, angular velocity

        input:
        average linear velocity, average angular velocity and change in time

        output:
        None
        (modifies instance state based on calculated final x, y,
            linear velocity, angular velocity)
        """
        # input is observed average velocity (distance / time), angular velocity
        # calculates the linear value from start v (current state) to end v (new
        # state) based on this average.

        dr = avg_v*dt # distance traveled forward
        dtheta = avg_omega*dt # angle swept

        average_heading = self.heading + avg_omega/2.0

        dy = dr * math.sin(average_heading) # distance traveled in x, y
        dx = dr * math.cos(average_heading)

        end_v = avg_v + (avg_v - self.v) # final linear, angular velocity
        end_omega = avg_omega + (avg_omega - self.omega)

        # assign new values to update state
        self.x += dx
        self.y += dy
        self.heading += dtheta
        self.v = end_v
        self.omega = end_omega

if __name__ == '__main__':
    # pylint: disable=invalid-name
    # wog is a placeholder to just run the node
    wog = WheelOdometryGenerator()
    wog.start_node()
<commit_msg>Enc->odom shows promising results<commit_after>#!/usr/bin/env python

"""
ENCODER TO ODOM

ROS Node for converting encoder ticks to a wheel odometry measurement of
position.
"""

import rospy
from tf import transformations as tft
import math
from nav_msgs.msg import Odometry
from std_msgs.msg import Float64
from geometry_msgs.msg import Quaternion
from snowmower_msgs.msg import EncMsg

def heading_to_quaternion(heading):
    """
    Converts a Euler yaw/heading angle to equivalent quaternion
    input: euler heading in radians
    output: nav_msgs.msg.Quaternion
    """
    # rospy.loginfo('heading out: '+str(heading))
    quat = tft.quaternion_from_euler(0, 0, heading)

    quaternion = Quaternion()
    quaternion.x = quat[0]
    quaternion.y = quat[1]
    quaternion.z = quat[2]
    quaternion.w = quat[3]
    return quaternion

class WheelOdometryGenerator(object):
    """
    WheelOdometryGenerator: ROS Node that implements encoder ticks to position

    wog = WheelOdometryGenerator()
    wog.start_node()
    """

    # pylint: disable=too-many-instance-attributes
    # These attributes represent state, last data, and pub/sub

    # pylint: disable=invalid-name
    # The variables for state are valid as x, y, v (linear velocity)

    track_width = 1.05
    wheel_diam = .3302
    encoders_per_rev = 10000

    tpm_right = 26500
    tpm_left = 27150

    debug = False

    def __init__(self):
        self.x = 0
        self.y = 0
        self.heading = 0
        self.v = 0
        self.omega = 0

        self.last_enc = None

        self.seq = 1

        self.sub = None
        self.pub = None

    def process_encoder_msg(self, msg):
        """
        process_encoder_msg: callback to take in new data, and update state

        input: snowmower_msgs.msg.EncMsg
        output: None
        """
        rospy.loginfo('msg recieved!')

        if self.last_enc is None:
            self.last_enc = msg
        dleft = msg.left - self.last_enc.left
        dright = msg.right - self.last_enc.right
        # time is in secs
        dt = (msg.header.stamp.nsecs - self.last_enc.header.stamp.nsecs)*pow(10,-9)
        if dt < 0:
            dt = 0.0

        if dleft == dright:
            if dt<= 0.0:
                v = 0.0
            else:
                v = self.distance(dleft, "left") / dt
            omega = 0.0
            arc_left = 0.0
            arc_right = 0.0
        else:
            arc_left = self.distance(dleft, "left")
            arc_right = self.distance(dright, "right")

            if arc_left < arc_right:
                r = ((self.track_width/2.0)*
                    ((arc_right+arc_left)/(arc_right-arc_left)))
                try:
                    # if r = track_width/2.0, calculate with right arc
                    theta = arc_left/(r-(self.track_width/2.0))
                except:
                    theta = arc_right/(r+(self.track_width/2.0))
            else: # arc_left > arc_right:
                r = ((self.track_width/2.0)*
                    ((arc_right+arc_left)/(arc_left-arc_right)))
                try:
                    # if r = - track_width/2.0, calculate with right arc
                    theta = arc_left/(r+(self.track_width/2.0))
                except:
                    theta = arc_right/(r-(self.track_width/2.0))
            if dt <= 0.0:
                if self.debug:
                    rospy.loginfo('negative time v = 0')
                v = 0
                omega = 0
            else:
                if self.debug:
                    rospy.loginfo('positive time v = ((dl+dr)/2) / dt')
                v = ((arc_left+arc_right)/2.0)/dt
                omega = theta/dt

        # based on v, omega, update state
        if dleft and dright and arc_left and arc_right:
            if self.debug:
                rospy.loginfo('dleft: '+str(dleft)+' dright: '+str(dright)+' dt: '+str(dt))
                rospy.loginfo('aleft: '+str(arc_left)+' aright: '+str(arc_right)+' dt: '+str(dt))
        self.update_state(v, omega, dt)
        if self.debug:
            rospy.loginfo('x: '+str(self.x)+' y: '+str(self.y))

        self.send_current_odom()
        self.last_enc = msg

    def send_current_odom(self):
        """
        send_current_odom: publish the current state as an Odometry message

        input: None
        output: (ROS publish) nav_msgs.msg.Odometry
        """

        o = Odometry()
        o.header.seq = self.seq
        o.header.stamp = rospy.Time.now()
        o.header.frame_id = '/odom'

        o.pose.pose.position.x = self.x
        o.pose.pose.position.y = self.y

        o.pose.pose.orientation = heading_to_quaternion(self.heading)

        o.twist.twist.linear.x = self.v
        o.twist.twist.angular.z = self.omega

        self.seq += 1

        self.pub.publish(o)

    def start_node(self):
        """
        start_node: Run the ROS node by intializing the publisher and subscriber
        """
        rospy.init_node('encoder2odom')
        self.pub = rospy.Publisher('/odom', Odometry, queue_size=1)
        self.pub_head = rospy.Publisher('/heading', Float64)
        self.sub = rospy.Subscriber('/enc', EncMsg, self.process_encoder_msg)
        rospy.loginfo('enc2odom: wheel odometry ready')
        rospy.loginfo('track_width  | wheel_diam   | encoders/rev')
        rospy.loginfo(str(self.track_width)+'         | '+str(self.wheel_diam)+'       | '+str(self.encoders_per_rev))
        rate = rospy.Rate(1)
        while not rospy.is_shutdown():
            self.send_current_odom()
            floa = Float64(data=self.heading)
            self.pub_head.publish(floa)
            rate.sleep()

    # GEOMETRY

    def distance(self, ticks, side):
        """
        distance: calculate the distance traveled per given ticks based on given
            robot geometry
        """
        if side == "right":
            return ticks*1.0/self.tpm_right
        else:
            return ticks*1.0/self.tpm_left
        # return (ticks*1.0/self.encoders_per_rev)*math.pi*self.wheel_diam

    def update_state(self, avg_v, avg_omega, dt):
        """
        update_state: move the state forward one step

        state has:
        initial x, y, heading, linear velocity, angular velocity

        input:
        average linear velocity, average angular velocity and change in time

        output:
        None
        (modifies instance state based on calculated final x, y,
            linear velocity, angular velocity)
        """
        # input is observed average velocity (distance / time), angular velocity
        # calculates the linear value from start v (current state) to end v (new
        # state) based on this average.

        dr = avg_v*dt # distance traveled forward
        dtheta = avg_omega*dt # angle swept
        if self.debug:
            rospy.loginfo('v: '+str(avg_v)+' w: '+str(avg_omega))
            rospy.loginfo('dr: '+str(dr)+' dtheta: '+str(dtheta))

        average_heading = self.heading + dtheta/2.0

        dy = dr * math.sin(average_heading) # distance traveled in x, y
        dx = dr * math.cos(average_heading)

        end_v = avg_v + (avg_v - self.v) # final linear, angular velocity
        end_omega = avg_omega + (avg_omega - self.omega)

        # assign new values to update state
        self.x += dx
        self.y += dy
        rospy.loginfo('dtheta: '+str(dtheta))
        self.heading += dtheta
        self.v = end_v
        self.omega = end_omega

if __name__ == '__main__':
    # pylint: disable=invalid-name
    # wog is a placeholder to just run the node
    wog = WheelOdometryGenerator()
    wog.start_node()
<|endoftext|>


<commit_before><commit_msg>scripts/gdb: fix typo in exception name<commit_after><|endoftext|>


<commit_before>import gen.build.target as target
import gen.build.nix as nix
from gen.env import Environment
import gen.shell as shell
import gen.path as path

Path = path.Path

class ExtractDebug(target.Commands):
    """Extract debug symbols from an executable."""
    __slots__ = ['_dest', '_src', '_env']
    def __init__(self, dest, src, env):
        target.Commands.__init__(self, env)
        if not isinstance(dest, Path):
            raise TypeError('dest must be Path')
        if not isinstance(src, Path):
            raise TypeError('src must be Path')
        self._dest = dest
        self._src = src

    def input(self):
        yield self._src

    def output(self):
        yield self._dest

    def name(self):
        return 'OBJCOPY'

    def commands(self):
        return [['objcopy', '--only-keep-debug', self._src, self._dest],
                ['chmod', '-x', self._dest]]

class Strip(target.Commands):
    """Copy an executable, stripping the debug symbols from it.

    Add a link to the external debug symbols.
    """
    __slots__ = ['_dest', '_src', '_debugsyms', '_env']
    def __init__(self, dest, src, debugsyms, env):
        target.Commands.__init__(self, env)
        if not isinstance(dest, Path):
            raise TypeError('dest must be Path')
        if not isinstance(src, Path):
            raise TypeError('src must be Path')
        if not isinstance(debugsyms, Path):
            raise TypeError('debugsymse must be Path')
        self._dest = dest
        self._src = src
        self._debugsyms = debugsyms

    def input(self):
        yield self._src
        yield self._debugsyms

    def output(self):
        yield self._dest

    def name(self):
        yield 'OBJCOPY'

    def commands(self):
        return [['objcopy', '--strip-unneeded',
                 ('--add-gnu-debuglink=', self._debugsyms),
                 self._src, self._dest]]

########################################################################

def customconfig(cmd):
    """Get the environment for a package from a config program."""
    cflags = shell.getoutput(cmd + ['--cflags'])
    libs = shell.getoutput(cmd + ['--libs'])
    return Environment(CFLAGS=cflags, LIBS=libs)

def pkgconfig(pkg):
    """Get the environment for a package from pkg-config."""
    return customconfig(['pkg-config', pkg])

def getmachine(env):
    """Get the name of the target machine."""
    m = shell.getoutput([env.CC, '-dumpmachine'] +
                        env.CPPFLAGS + env.CFLAGS)
    i = m.find('-')
    if i < 0:
        raise Exception('unable to parse machine name: %r' % (m,))
    return m[:i]

def add_targets(graph, proj, userenv):
    import platform
    if platform.system() == 'Linux':
        build_linux(graph, proj, userenv)

def build_linux(graph, proj, userenv):
    """Generate targets for a normal Linux build.

    This produces the actual executable as a target.
    """
    penv = Environment(
        proj.env,
        CFLAGS='-g',
        CXXFLAGS='-g',
        LDFLAGS='-Wl,--as-needed -Wl,--gc-sections',
        LIBS='-lGL -lGLU',
    )
    uenv = nix.get_user_env(userenv)

    machine = getmachine(uenv)
    if machine == 'x86_64':
        machine = 'linux64'
    elif re.match(r'i\d86', machine):
        machine = 'linux32'
    else:
        print >>sys.stderr, 'warning: unknown machine %s' % machine

    libs = {
        'LIBJPEG':  lambda: Environment(LIBS='-ljpeg'),
        'GTK':      lambda: pkgconfig('gtk+-2.0 gtkglext-1.0'),
        'LIBPNG':   lambda: pkgconfig('libpng'),
        'PANGO':    lambda: pkgconfig('pangocairo'),
    }

    def libenvf(atom):
        try:
            thunk = libs[atom]
        except KeyError:
            return None
        else:
            env = thunk()
            # This flag is added by the gmodule indirect dependency.
            # We don't need this flag and it's a waste.
            env.remove('LIBS', '-Wl,--export-dynamic')
            return env

    products = genbuild_linux(graph, proj, penv, uenv, libenvf, machine)
    graph.add(target.DepTarget('build', products, uenv))

def genbuild_linux(graph, proj, pkgenv, userenv, libenvf, machine):
    """Generate all targets for any Linux build.

    The package environment is pkgenv, the user environment is
    userenv.

    The libenvf parameter is a function which looks up the environment
    for the library identified by the given atom.  If it returns None
    for any atom, then sources with that atom will be excluded from
    the build.

    The machine parameter is the machine name to add to the name of
    the executable.  It may be empty.
    """

    envs1 = {}
    def getenv1(atom):
        """Memoized version of libenvf."""
        try:
            return envs1[atom]
        except KeyError:
            e = libenvf(atom)
            envs1[atom] = e
            return e

    envs = {}
    allatoms = set()
    def getenv(atoms):
        """Multiple-atom version of libenvf."""
        x = list(sorted(set(atoms)))
        xn = ' '.join(x)
        try:
            return envs[xn]
        except KeyError:
            es = [pkgenv]
            for a in x:
                e = getenv1(a)
                if e is None:
                    envs[xn] = None
                    return None
                es.append(e)
            es.append(userenv)
            e = Environment(*es)
            envs[xn] = e
            allatoms.update(x)
            return e

    objs = []
    for source in proj.sourcelist.sources():
        if not source.match_platform('LINUX'):
            continue
        env = getenv(source.other_atoms())
        if env is None:
            continue
        ext = source.grouppath.ext
        try:
            stype = path.EXTS[ext]
        except KeyError:
            raise Exception(
                'unknown extension %s for path %s' %
                (ext, source.relpath.posixpath))
        if stype in ('c', 'cxx'):
            opath = Path('build/obj', source.group.name,
                         source.grouppath.withext('.o'))
            objs.append(opath)
            graph.add(nix.CC(opath, source.relpath, env, stype))
        elif stype in ('h', 'hxx'):
            pass
        else:
            raise Exception(
                'cannot handle file type %s for path %s' %
                (stype, source.relpath.posixpath))

    env = getenv(allatoms)
    if machine:
        exename = '%s-%s' % (env.EXE_LINUX, machine)
    else:
        exename = env.EXE_LINUX
    rawpath = Path('build/exe', exename)
    exepath = Path('build/product', exename)
    dbgpath = Path('build/product', exename + '.dbg')

    graph.add(nix.LD(rawpath, objs, env))
    graph.add(ExtractDebug(dbgpath, rawpath, env))
    graph.add(Strip(exepath, rawpath, dbgpath, env))

    return [exepath, rawpath]
<commit_msg>build: fix incorrect name method<commit_after>import gen.build.target as target
import gen.build.nix as nix
from gen.env import Environment
import gen.shell as shell
import gen.path as path

Path = path.Path

class ExtractDebug(target.Commands):
    """Extract debug symbols from an executable."""
    __slots__ = ['_dest', '_src', '_env']
    def __init__(self, dest, src, env):
        target.Commands.__init__(self, env)
        if not isinstance(dest, Path):
            raise TypeError('dest must be Path')
        if not isinstance(src, Path):
            raise TypeError('src must be Path')
        self._dest = dest
        self._src = src

    def input(self):
        yield self._src

    def output(self):
        yield self._dest

    def name(self):
        return 'OBJCOPY'

    def commands(self):
        return [['objcopy', '--only-keep-debug', self._src, self._dest],
                ['chmod', '-x', self._dest]]

class Strip(target.Commands):
    """Copy an executable, stripping the debug symbols from it.

    Add a link to the external debug symbols.
    """
    __slots__ = ['_dest', '_src', '_debugsyms', '_env']
    def __init__(self, dest, src, debugsyms, env):
        target.Commands.__init__(self, env)
        if not isinstance(dest, Path):
            raise TypeError('dest must be Path')
        if not isinstance(src, Path):
            raise TypeError('src must be Path')
        if not isinstance(debugsyms, Path):
            raise TypeError('debugsymse must be Path')
        self._dest = dest
        self._src = src
        self._debugsyms = debugsyms

    def input(self):
        yield self._src
        yield self._debugsyms

    def output(self):
        yield self._dest

    def name(self):
        return 'OBJCOPY'

    def commands(self):
        return [['objcopy', '--strip-unneeded',
                 ('--add-gnu-debuglink=', self._debugsyms),
                 self._src, self._dest]]

########################################################################

def customconfig(cmd):
    """Get the environment for a package from a config program."""
    cflags = shell.getoutput(cmd + ['--cflags'])
    libs = shell.getoutput(cmd + ['--libs'])
    return Environment(CFLAGS=cflags, LIBS=libs)

def pkgconfig(pkg):
    """Get the environment for a package from pkg-config."""
    return customconfig(['pkg-config', pkg])

def getmachine(env):
    """Get the name of the target machine."""
    m = shell.getoutput([env.CC, '-dumpmachine'] +
                        env.CPPFLAGS + env.CFLAGS)
    i = m.find('-')
    if i < 0:
        raise Exception('unable to parse machine name: %r' % (m,))
    return m[:i]

def add_targets(graph, proj, userenv):
    import platform
    if platform.system() == 'Linux':
        build_linux(graph, proj, userenv)

def build_linux(graph, proj, userenv):
    """Generate targets for a normal Linux build.

    This produces the actual executable as a target.
    """
    penv = Environment(
        proj.env,
        CFLAGS='-g',
        CXXFLAGS='-g',
        LDFLAGS='-Wl,--as-needed -Wl,--gc-sections',
        LIBS='-lGL -lGLU',
    )
    uenv = nix.get_user_env(userenv)

    machine = getmachine(uenv)
    if machine == 'x86_64':
        machine = 'linux64'
    elif re.match(r'i\d86', machine):
        machine = 'linux32'
    else:
        print >>sys.stderr, 'warning: unknown machine %s' % machine

    libs = {
        'LIBJPEG':  lambda: Environment(LIBS='-ljpeg'),
        'GTK':      lambda: pkgconfig('gtk+-2.0 gtkglext-1.0'),
        'LIBPNG':   lambda: pkgconfig('libpng'),
        'PANGO':    lambda: pkgconfig('pangocairo'),
    }

    def libenvf(atom):
        try:
            thunk = libs[atom]
        except KeyError:
            return None
        else:
            env = thunk()
            # This flag is added by the gmodule indirect dependency.
            # We don't need this flag and it's a waste.
            env.remove('LIBS', '-Wl,--export-dynamic')
            return env

    products = genbuild_linux(graph, proj, penv, uenv, libenvf, machine)
    graph.add(target.DepTarget('build', products, uenv))

def genbuild_linux(graph, proj, pkgenv, userenv, libenvf, machine):
    """Generate all targets for any Linux build.

    The package environment is pkgenv, the user environment is
    userenv.

    The libenvf parameter is a function which looks up the environment
    for the library identified by the given atom.  If it returns None
    for any atom, then sources with that atom will be excluded from
    the build.

    The machine parameter is the machine name to add to the name of
    the executable.  It may be empty.
    """

    envs1 = {}
    def getenv1(atom):
        """Memoized version of libenvf."""
        try:
            return envs1[atom]
        except KeyError:
            e = libenvf(atom)
            envs1[atom] = e
            return e

    envs = {}
    allatoms = set()
    def getenv(atoms):
        """Multiple-atom version of libenvf."""
        x = list(sorted(set(atoms)))
        xn = ' '.join(x)
        try:
            return envs[xn]
        except KeyError:
            es = [pkgenv]
            for a in x:
                e = getenv1(a)
                if e is None:
                    envs[xn] = None
                    return None
                es.append(e)
            es.append(userenv)
            e = Environment(*es)
            envs[xn] = e
            allatoms.update(x)
            return e

    objs = []
    for source in proj.sourcelist.sources():
        if not source.match_platform('LINUX'):
            continue
        env = getenv(source.other_atoms())
        if env is None:
            continue
        ext = source.grouppath.ext
        try:
            stype = path.EXTS[ext]
        except KeyError:
            raise Exception(
                'unknown extension %s for path %s' %
                (ext, source.relpath.posixpath))
        if stype in ('c', 'cxx'):
            opath = Path('build/obj', source.group.name,
                         source.grouppath.withext('.o'))
            objs.append(opath)
            graph.add(nix.CC(opath, source.relpath, env, stype))
        elif stype in ('h', 'hxx'):
            pass
        else:
            raise Exception(
                'cannot handle file type %s for path %s' %
                (stype, source.relpath.posixpath))

    env = getenv(allatoms)
    if machine:
        exename = '%s-%s' % (env.EXE_LINUX, machine)
    else:
        exename = env.EXE_LINUX
    rawpath = Path('build/exe', exename)
    exepath = Path('build/product', exename)
    dbgpath = Path('build/product', exename + '.dbg')

    graph.add(nix.LD(rawpath, objs, env))
    graph.add(ExtractDebug(dbgpath, rawpath, env))
    graph.add(Strip(exepath, rawpath, dbgpath, env))

    return [exepath, rawpath]
<|endoftext|>


<commit_before>import pandas as pd
import numpy as np
import itertools
from collections import namedtuple

def aggregate(indexes, df):
    df = df.drop(['gender', 'ethnicity', 'school_type'], axis=1)

    agg_rules = {
        "age": np.mean,
        "enrolled": np.sum,
        "entrants": np.sum,
        "graduates": np.sum,
        "student_id": pd.Series.nunique,
        "morning": np.sum,
        "afternoon": np.sum,
        "night": np.sum,
        "full_time": np.sum,
        "entrants": np.sum,
    }

    pk_types = set([type(t) for t in indexes])
    if pk_types == set([str]) and indexes == ["year", "bra_id"]:
        agg_rules["university_id"] = pd.Series.nunique

    aggregated_dfs = []
    aggregated_dfs.append(df.groupby(indexes).agg(agg_rules))

    if 'course_hedu_id' in indexes:
        df_fields = df.reset_index()
        df_fields["course_hedu_id"] = df_fields["course_hedu_id"].str.slice(0, 2)
        aggregated_dfs.append(df_fields.groupby(indexes).agg(agg_rules))

    if 'bra_id' in indexes:
        df_region = df.reset_index()
        df_region["bra_id"] = df_region["bra_id"].str.slice(0, 1)
        aggregated_dfs.append(df_region.groupby(indexes).agg(agg_rules))

        df_state = df.reset_index()
        df_state["bra_id"] = df_state["bra_id"].str.slice(0, 3)
        aggregated_dfs.append(df_state.groupby(indexes).agg(agg_rules))

        df_meso = df.reset_index()
        df_meso["bra_id"] = df_meso["bra_id"].str.slice(0, 5)
        aggregated_dfs.append(df_meso.groupby(indexes).agg(agg_rules))

        df_micro = df.reset_index()
        df_micro["bra_id"] = df_micro["bra_id"].str.slice(0, 7)
        aggregated_dfs.append(df_micro.groupby(indexes).agg(agg_rules))

    return pd.concat(aggregated_dfs)
<commit_msg>Fixes aggregation logic to aggregate product of depth levels<commit_after>import pandas as pd
import numpy as np
import itertools
from collections import namedtuple

def aggregate(indexes, df):
    df = df.drop(['gender', 'ethnicity', 'school_type'], axis=1)

    Level = namedtuple('Level', ['column', 'depth'])

    depths = {
        'bra_id': [
            Level('bra_id', 1),
            Level('bra_id', 3),
            Level('bra_id', 5),
            Level('bra_id', 7),
            Level('bra_id', 9),
        ],

        'course_hedu_id': [
            Level('course_hedu_id', 2),
            Level('course_hedu_id', 6),
        ],
    }

    agg_rules = {
        "age": np.mean,
        "enrolled": np.sum,
        "entrants": np.sum,
        "graduates": np.sum,
        "student_id": pd.Series.nunique,
        "morning": np.sum,
        "afternoon": np.sum,
        "night": np.sum,
        "full_time": np.sum,
        "entrants": np.sum,
    }

    if indexes == ["year", "bra_id"]:
        agg_rules["university_id"] = pd.Series.nunique

    index_depths = list(set(indexes) & set(depths.keys()))

    aggregation_levels = list(itertools.product(
        *[[level for level in depths[index]] for index in index_depths]
    ))

    aggregated_dfs = []

    for levels in aggregation_levels:
        aggregated_df = df.reset_index()

        for level in levels:
            print level.depth, level.column
            aggregated_df[level.column] = aggregated_df[level.column].str.slice(0, level.depth)

        aggregated_dfs.append(aggregated_df.groupby(indexes).agg(agg_rules))

    return pd.concat(aggregated_dfs) if aggregated_dfs else df.groupby(indexes).agg(agg_rules)
<|endoftext|>


<commit_before>"""
Download 1000 ft reflectivity from the NCEP HRRR
"""
import sys
import datetime
import urllib2

def run(valid):
    """ run for this valid time! """
    output = open("/tmp/ncep_hrrr_%s.grib2" % (valid.strftime("%Y%m%d%H"),),
                  'wb')
    for hr in range(0,16):
        shr = "%02i" % (hr,)
        uri = valid.strftime(("http://www.ftp.ncep.noaa.gov/data/nccf/"
            +"nonoperational/com/hrrr/prod/hrrr.%Y%m%d/hrrr.t%Hz.wrfsubhf"+shr
            +".grib2.idx"))
        attempt = 0
        data = None
        while data is None and attempt < 10:
            try:
                data = urllib2.urlopen(uri, timeout=30)
            except Exception,exp:
                print("dl_hrrrref FAIL %s %s %s" % (valid, hr, exp))
            attempt += 1
        if data is None:
            print("ABORT")
            sys.exit()

        offsets = []
        neednext = False
        for line in data:
            tokens = line.split(":")
            if neednext:
                offsets[-1].append( int(tokens[1]))
                neednext = False
            if tokens[3] == 'REFD' and tokens[4] == '1000 m above ground':
                offsets.append( [int(tokens[1])] )
                neednext = True

        req = urllib2.Request(uri[:-4])

        for pr in offsets:
            req.headers['Range'] = 'bytes=%s-%s' % (pr[0], pr[1])
            f = urllib2.urlopen(req)
            output.write( f.read() )

    output.close()

def main():
    """ Go Main Go """
    valid = datetime.datetime( int(sys.argv[1]), int(sys.argv[2]),
                               int(sys.argv[3]), int(sys.argv[4]))
    run(valid)

if __name__ == '__main__':
    # do main
    main()<commit_msg>Make dl a bit more robust<commit_after>"""
Download 1000 ft reflectivity from the NCEP HRRR
"""
import sys
import datetime
import urllib2

def run(valid):
    """ run for this valid time! """
    output = open("/tmp/ncep_hrrr_%s.grib2" % (valid.strftime("%Y%m%d%H"),),
                  'wb')
    for hr in range(0,16):
        shr = "%02i" % (hr,)
        uri = valid.strftime(("http://www.ftp.ncep.noaa.gov/data/nccf/"
            +"nonoperational/com/hrrr/prod/hrrr.%Y%m%d/hrrr.t%Hz.wrfsubhf"+shr
            +".grib2.idx"))
        attempt = 0
        data = None
        while data is None and attempt < 10:
            try:
                data = urllib2.urlopen(uri, timeout=30)
            except Exception,exp:
                print("dl_hrrrref FAIL %s %s %s" % (valid, hr, exp))
            attempt += 1
        if data is None:
            print("ABORT")
            sys.exit()

        offsets = []
        neednext = False
        for line in data:
            tokens = line.split(":")
            if neednext:
                offsets[-1].append( int(tokens[1]))
                neednext = False
            if tokens[3] == 'REFD' and tokens[4] == '1000 m above ground':
                offsets.append( [int(tokens[1])] )
                neednext = True

        req = urllib2.Request(uri[:-4])

        for pr in offsets:
            req.headers['Range'] = 'bytes=%s-%s' % (pr[0], pr[1])
            f = None
            attempt = 0
            while f is None and attempt < 10:
                try:
                    f = urllib2.urlopen(req)
                except Exception, exp:
                    print("dl_hrrrref FAIL %s %s %s" % (valid, hr, exp))
                attempt += 1
            output.write( f.read() )

    output.close()

def main():
    """ Go Main Go """
    valid = datetime.datetime( int(sys.argv[1]), int(sys.argv[2]),
                               int(sys.argv[3]), int(sys.argv[4]))
    run(valid)

if __name__ == '__main__':
    # do main
    main()<|endoftext|>


<commit_before>"""
@brief      test log(time=10s)

"""
import os
import sys
import unittest
import warnings

try:
    import src
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..")))
    if path not in sys.path:
        sys.path.append(path)
    import src

try:
    import pyquickhelper as skip_
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..",
                "..",
                "pyquickhelper",
                "src")))
    if path not in sys.path:
        sys.path.append(path)
    import pyquickhelper as skip_

from pyquickhelper.loghelper import fLOG
from pyquickhelper.pycode import get_temp_folder, is_travis_or_appveyor, is_virtual_environment, run_base_script
from src.ensae_teaching_cs.helpers.video_helper import make_video

context = {}

class TestVideoHelper(unittest.TestCase):

    def test_make_video(self):
        """
        This test does not work under a virtual environment.
        """
        fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__")

        if is_travis_or_appveyor() == "travis":
            warnings.warn("cv2 is not available")
            return

        if "--force" in context or not is_virtual_environment():
            fLOG("running in base environment", sys.prefix)
            temp = get_temp_folder(__file__, "temp_make_video")
            img = os.path.join(temp, "..", "data")
            imgs = os.listdir(img)
            png = [os.path.join(img, _)
                   for _ in imgs if os.path.splitext(_)[-1] == ".png" and "00" in _]
            assert len(png) > 0
            out = os.path.join(temp, "out_video.avi")

            v = make_video(png, out, size=(1000, 300))
            assert os.path.exists(out)
            assert os.stat(out).st_size > 90000
            assert v is not None
            if "--force" in context:
                fLOG("success")
        else:
            # we run it with the original interpreter
            import pyquickhelper
            fLOG("switch from virtual environment", sys.base_prefix)
            fold = os.path.abspath(os.path.join(
                os.path.dirname(pyquickhelper.__file__), ".."))
            os.environ["PYTHONPATH"] = fold
            this = os.path.abspath(__file__)
            if sys.version_info[0] == 2:
                this = this.replace(".pyc", ".py")
            out = run_base_script(this, file=True, fLOG=fLOG, argv=["--force"])
            os.environ["PYTHONPATH"] = ""
            if "success" not in out:
                raise Exception(
                    "CMD:\n{0}\nOUT:\n{1}\nPYTHONPATH={2}".format(os.path.split(__file__)[-1], out, fold))

if __name__ == "__main__":

    if "--force" in sys.argv:
        context["--force"] = True
        del sys.argv[sys.argv.index('--force')]

    unittest.main()
<commit_msg>improve unittest<commit_after>"""
@brief      test log(time=10s)

"""
import os
import sys
import warnings

try:
    import src
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..")))
    if path not in sys.path:
        sys.path.append(path)
    import src

try:
    import pyquickhelper as skip_
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..",
                "..",
                "pyquickhelper",
                "src")))
    if path not in sys.path:
        sys.path.append(path)
    import pyquickhelper as skip_

from pyquickhelper.loghelper import fLOG
from pyquickhelper.pycode import get_temp_folder, is_travis_or_appveyor, is_virtual_environment, run_base_script, ExtTestCase
from src.ensae_teaching_cs.helpers.video_helper import make_video

context = {}

class TestVideoHelper(ExtTestCase):

    def test_make_video(self):
        """
        This test does not work under a virtual environment.
        """
        fLOG(__file__, self._testMethodName, OutputPrint=__name__ == "__main__")

        if is_travis_or_appveyor() == "travis":
            warnings.warn("cv2 is not available")
            return

        if "--force" in context or not is_virtual_environment():
            fLOG("running in base environment", sys.prefix)
            temp = get_temp_folder(__file__, "temp_make_video")
            img = os.path.join(temp, "..", "data")
            imgs = os.listdir(img)
            png = [os.path.join(img, _)
                   for _ in imgs if os.path.splitext(_)[-1] == ".png" and "00" in _]
            assert len(png) > 0
            out = os.path.join(temp, "out_video.avi")

            v = make_video(png, out, size=(1000, 300))
            assert os.path.exists(out)
            assert os.stat(out).st_size > 90000
            assert v is not None
            if "--force" in context:
                fLOG("success")
        else:
            # we run it with the original interpreter
            import pyquickhelper
            fLOG("switch from virtual environment", sys.base_prefix)
            fold = os.path.abspath(os.path.join(
                os.path.dirname(pyquickhelper.__file__), ".."))
            os.environ["PYTHONPATH"] = fold
            this = os.path.abspath(__file__)
            if sys.version_info[0] == 2:
                this = this.replace(".pyc", ".py")
            out = run_base_script(this, file=True, fLOG=fLOG, argv=["--force"])
            os.environ["PYTHONPATH"] = ""
            if "success" not in out:
                raise Exception(
                    "CMD:\n{0}\nOUT:\n{1}\nPYTHONPATH={2}".format(os.path.split(__file__)[-1], out, fold))
<|endoftext|>