<commit_before>## Copyright (c) 2012 Aldebaran Robotics. All rights reserved.
## Use of this source code is governed by a BSD-style license that can be
## found in the COPYING file.

"""This module contains function to
manipulate archives.

We will always manipulate .tar.gz archives on UNIX,
and .zip on windows. (keeping it close to the more common
used format of the given platform)

All archives should have a unique top dir.
"""

import os
import sys
import copy
import posixpath
import logging
import operator
import tarfile
import shutil
import zipfile

import qibuild

LOGGER = logging.getLogger("buildtool.archive")

class InvalidArchive(Exception):
    """Just a custom exception """
    def __init__(self, message):
        self._message = message
        Exception.__init__(self)

    def __str__(self):
        return self._message

def extract_tar(archive_path, dest_dir):
    """ Extract a .tar.gz archive

    :return: path to the extracted archive
             (dest_dir/topdir)

    """
    # Algorithm taken from tarfile.extractall():
    # First, extract everything directory in 700 mode,
    # then reverse-sort the order of the directory and
    # re-fix the permissions as they were in the original archive
    # We have to do this in case the archive contains something like:
    # src
    #   ro
    #     a
    # where ro is a read-only directory
    #
    # Obviously, creating ro first with the same permissions as
    # in the archive will prevent 'a' to be created.
    # See test_archive.py for relevant test.

    LOGGER.debug("Extracting %s to %s", archive_path, dest_dir)
    archive = tarfile.open(archive_path)
    members = archive.getmembers()
    size = len(members)
    orig_topdir = members[0].name.split(posixpath.sep)[0]
    if orig_topdir == ".":
        archive_name = os.path.basename(archive_path)
        archive_name = archive_name.split(".")[0]
        dest_dir = os.path.join(dest_dir, archive_name)
    done = 0
    # Extract directories with a safe mode.
    directories = list()
    for tarinfo in members:
        member_top_dir = tarinfo.name.split(posixpath.sep)[0]
        if done != 0 and member_top_dir != orig_topdir:
            # something wrong: members do not have the
            # same basename
            mess  = "Invalid member %s in archive:\n" % tarinfo.name
            mess += "Every files sould be in the same top dir (%s != %s)" % \
                 (orig_topdir, member_top_dir)
            raise InvalidArchive(mess)
        if tarinfo.isdir():
            directories.append(tarinfo)
            tarinfo = copy.copy(tarinfo)
            tarinfo.mode = 0700
        archive.extract(tarinfo, dest_dir)
        done += 1
        if sys.stdout.isatty():
            percent = float(done) /size * 100
            sys.stdout.write("Done: %.0f%%\r" % percent)
            sys.stdout.flush()

    # Reverse sort directories.
    directories.sort(key=operator.attrgetter('name'))
    directories.reverse()

    # Set correct owner, mtime and filemode on directories.
    for tarinfo in directories:
        dirpath = os.path.join(dest_dir, tarinfo.name)
        archive.chown(tarinfo, dirpath)
        archive.utime(tarinfo, dirpath)
        archive.chmod(tarinfo, dirpath)

    archive.close()
    LOGGER.debug("%s extracted to %s", archive_path, dest_dir)
    if orig_topdir == ".":
        res = dest_dir
    else:
        res = os.path.join(dest_dir, orig_topdir)
    return res

def extract_zip(archive_path, dest_dir):
    """ Extract a zip archive
    :return: path to the extracted archive
             (dest_dir/topdir)

    """
    dest_dir = qibuild.sh.to_native_path(dest_dir)
    LOGGER.debug("Extracting %s to %s", archive_path, dest_dir)
    archive = zipfile.ZipFile(archive_path)
    members = archive.infolist()
    # There is always the top dir as the first element of the archive
    # (or so we hope)
    orig_topdir = members[0].filename.split(posixpath.sep)[0]
    size = len(members)
    directories = list()
    for (i, member) in enumerate(members):
        member_top_dir = member.filename.split(posixpath.sep)[0]
        if i != 0 and member_top_dir != orig_topdir:
            # something wrong: members do not have the
            # same basename
            mess  = "Invalid member %s in archive:\n" % member.filename
            mess += "Every files sould be in the same top dir (%s != %s)" % \
                 (orig_topdir, member_top_dir)
            raise InvalidArchive(mess)
        # By-pass buggy zipfile for python 2.6:
        if sys.version_info < (2, 7):
            if member.filename.endswith("/"):
                # upstream buggy code would create an empty filename
                # instead of a directory, thus preventing next members
                # to be extracted
                to_create = member.filename[:-1]
                posix_dest_dir = qibuild.sh.to_posix_path(dest_dir)
                to_create = posixpath.join(posix_dest_dir, to_create)
                qibuild.sh.mkdir(to_create, recursive=True)
                continue
        archive.extract(member, path=dest_dir)
        # Fix permision on extracted file unless it is a directory
        # or if we are on windows
        if member.filename.endswith("/"):
            directories.append(member)
            new_path = os.path.join(dest_dir, member.filename)
            qibuild.sh.mkdir(new_path, recursive=True)
            new_st = 0777
        else:
            new_path = os.path.join(dest_dir, member.filename)
            new_st = member.external_attr >> 16L
        # permissions are meaningless on windows, here only the exension counts
        if not sys.platform.startswith("win"):
            os.chmod(new_path, new_st)

        percent = float(i) / size * 100
        if sys.stdout.isatty():
            sys.stdout.write("Done: %.0f%%\r" % percent)
            sys.stdout.flush()

    # Reverse sort directories, and then fix perm on these
    directories.sort(key=operator.attrgetter('filename'))
    directories.reverse()

    for zipinfo in directories:
        dirpath = os.path.join(dest_dir, zipinfo.filename)
        new_st = zipinfo.external_attr >> 16L
        os.chmod(dirpath, new_st)

    archive.close()
    LOGGER.debug("%s extracted to %s", archive_path, dest_dir)
    res = os.path.join(dest_dir, orig_topdir)
    return res

def extract(archive_path, directory, topdir=None):
    """Extract an archive, calling extract_zip or extract_tar
    when necessary

    The top directory of the archive will be replaced by topdir
    if it is given

    :return: path to the extracted archive
    """
    if archive_path.endswith(".zip"):
        extract_fun = extract_zip
    else:
        extract_fun = extract_tar

    # Errors returned by tarfile of zipfile are not very good,
    # so let's just catch everything
    try:
        if topdir:
            extracted = extract_fun(archive_path, directory)
            if os.path.basename(extracted) == topdir:
                return
            res = os.path.join(directory, topdir)
            qibuild.sh.rm(res)
            shutil.move(extracted, res)
        else:
            res = extract_fun(archive_path, directory)
        return res
    except Exception, err:
        mess = "Error occured when extracting %s\n" % archive_path
        mess += "Original error was: %s" % err
        raise InvalidArchive(mess)

def zip_win(directory):
    """Compress the directory in a .zip file

    """
    archive_name = directory + ".zip"
    # Convert to DOS path just to be sure:
    directory    = qibuild.sh.to_native_path(directory)
    archive_name = qibuild.sh.to_native_path(archive_name)
    archive = zipfile.ZipFile(archive_name, "w", zipfile.ZIP_DEFLATED)
    for (root, _directories, filenames) in os.walk(directory):
        for filename in filenames:
            full_path = os.path.join(root, filename)
            rel_path = os.path.relpath(full_path, directory)
            arcname  = os.path.join(os.path.basename(directory), rel_path)
            archive.write(full_path, arcname)
    archive.close()
    return archive_name

def zip_unix(directory):
    """
    Call tar cvfz on a directory

    """
    # Do not use tarfile python module when tar:
    # - is faster
    # - is not buggy
    base_dir = os.path.basename(directory)
    work_dir = os.path.abspath(os.path.join(directory, ".."))
    base_archive_name = base_dir + ".tar.gz"
    cmd = ["tar", "cfz", base_archive_name, base_dir]
    qibuild.command.call(cmd, cwd=work_dir)
    full_archive_name = os.path.join(work_dir, base_archive_name)
    return full_archive_name

def zip( directory):
    """Zip a directory, using .tar.gz or .zip according to the current
    platform

    """
    if sys.platform.startswith("win"):
        return zip_win(directory)
    else:
        return zip_unix(directory)

def extracted_name(archive_name):
    """Return the extracted name from the archive name.

    Warning: this assumes the archive is well-formed
    (ie the root dir of foo.tar.gz is foo)

    >>> extracted_name("foo.zip")
    'foo'
    >>> extracted_name("foo.tar.gz")
    'foo'
    """
    known_exts = [".zip", ".tar.gz", ".tar.bz2"]
    for ext in known_exts:
        if archive_name.endswith(ext):
            return archive_name[:-len(ext)]

def archive_name(directory):
    """ Return the name an archive made from the
    directory would have.
    (platform-dependant)

    On windows::

      >>> archive_name('foo')
      foo.zip

    Elsewhere::

      >>> archive_name('foo')
      foo.tar.gz
    """
    if sys.platform.startswith("win"):
        return directory + ".zip"
    else:
        return directory + ".tar.gz"

if __name__ == "__main__":
    extract(sys.argv[1], sys.argv[2], topdir=sys.argv[3])
<commit_msg>qibuild.archive: do not try to change utime<commit_after>## Copyright (c) 2012 Aldebaran Robotics. All rights reserved.
## Use of this source code is governed by a BSD-style license that can be
## found in the COPYING file.

"""This module contains function to
manipulate archives.

We will always manipulate .tar.gz archives on UNIX,
and .zip on windows. (keeping it close to the more common
used format of the given platform)

All archives should have a unique top dir.
"""

import os
import sys
import copy
import posixpath
import logging
import operator
import tarfile
import shutil
import zipfile

import qibuild

LOGGER = logging.getLogger("buildtool.archive")

class InvalidArchive(Exception):
    """Just a custom exception """
    def __init__(self, message):
        self._message = message
        Exception.__init__(self)

    def __str__(self):
        return self._message

def extract_tar(archive_path, dest_dir):
    """ Extract a .tar.gz archive

    :return: path to the extracted archive
             (dest_dir/topdir)

    """
    # Algorithm taken from tarfile.extractall():
    # First, extract everything directory in 700 mode,
    # then reverse-sort the order of the directory and
    # re-fix the permissions as they were in the original archive
    # We have to do this in case the archive contains something like:
    # src
    #   ro
    #     a
    # where ro is a read-only directory
    #
    # Obviously, creating ro first with the same permissions as
    # in the archive will prevent 'a' to be created.
    # See test_archive.py for relevant test.

    LOGGER.debug("Extracting %s to %s", archive_path, dest_dir)
    archive = tarfile.open(archive_path)
    members = archive.getmembers()
    size = len(members)
    orig_topdir = members[0].name.split(posixpath.sep)[0]
    if orig_topdir == ".":
        archive_name = os.path.basename(archive_path)
        archive_name = archive_name.split(".")[0]
        dest_dir = os.path.join(dest_dir, archive_name)
    done = 0
    # Extract directories with a safe mode.
    directories = list()
    for tarinfo in members:
        member_top_dir = tarinfo.name.split(posixpath.sep)[0]
        if done != 0 and member_top_dir != orig_topdir:
            # something wrong: members do not have the
            # same basename
            mess  = "Invalid member %s in archive:\n" % tarinfo.name
            mess += "Every files sould be in the same top dir (%s != %s)" % \
                 (orig_topdir, member_top_dir)
            raise InvalidArchive(mess)
        if tarinfo.isdir():
            directories.append(tarinfo)
            tarinfo = copy.copy(tarinfo)
            tarinfo.mode = 0700
        archive.extract(tarinfo, dest_dir)
        done += 1
        if sys.stdout.isatty():
            percent = float(done) /size * 100
            sys.stdout.write("Done: %.0f%%\r" % percent)
            sys.stdout.flush()

    # Reverse sort directories.
    directories.sort(key=operator.attrgetter('name'))
    directories.reverse()

    # Set correct owner, mtime and filemode on directories.
    for tarinfo in directories:
        dirpath = os.path.join(dest_dir, tarinfo.name)
        archive.chown(tarinfo, dirpath)
        archive.chmod(tarinfo, dirpath)

    archive.close()
    LOGGER.debug("%s extracted to %s", archive_path, dest_dir)
    if orig_topdir == ".":
        res = dest_dir
    else:
        res = os.path.join(dest_dir, orig_topdir)
    return res

def extract_zip(archive_path, dest_dir):
    """ Extract a zip archive
    :return: path to the extracted archive
             (dest_dir/topdir)

    """
    dest_dir = qibuild.sh.to_native_path(dest_dir)
    LOGGER.debug("Extracting %s to %s", archive_path, dest_dir)
    archive = zipfile.ZipFile(archive_path)
    members = archive.infolist()
    # There is always the top dir as the first element of the archive
    # (or so we hope)
    orig_topdir = members[0].filename.split(posixpath.sep)[0]
    size = len(members)
    directories = list()
    for (i, member) in enumerate(members):
        member_top_dir = member.filename.split(posixpath.sep)[0]
        if i != 0 and member_top_dir != orig_topdir:
            # something wrong: members do not have the
            # same basename
            mess  = "Invalid member %s in archive:\n" % member.filename
            mess += "Every files sould be in the same top dir (%s != %s)" % \
                 (orig_topdir, member_top_dir)
            raise InvalidArchive(mess)
        # By-pass buggy zipfile for python 2.6:
        if sys.version_info < (2, 7):
            if member.filename.endswith("/"):
                # upstream buggy code would create an empty filename
                # instead of a directory, thus preventing next members
                # to be extracted
                to_create = member.filename[:-1]
                posix_dest_dir = qibuild.sh.to_posix_path(dest_dir)
                to_create = posixpath.join(posix_dest_dir, to_create)
                qibuild.sh.mkdir(to_create, recursive=True)
                continue
        archive.extract(member, path=dest_dir)
        # Fix permision on extracted file unless it is a directory
        # or if we are on windows
        if member.filename.endswith("/"):
            directories.append(member)
            new_path = os.path.join(dest_dir, member.filename)
            qibuild.sh.mkdir(new_path, recursive=True)
            new_st = 0777
        else:
            new_path = os.path.join(dest_dir, member.filename)
            new_st = member.external_attr >> 16L
        # permissions are meaningless on windows, here only the exension counts
        if not sys.platform.startswith("win"):
            os.chmod(new_path, new_st)

        percent = float(i) / size * 100
        if sys.stdout.isatty():
            sys.stdout.write("Done: %.0f%%\r" % percent)
            sys.stdout.flush()

    # Reverse sort directories, and then fix perm on these
    directories.sort(key=operator.attrgetter('filename'))
    directories.reverse()

    for zipinfo in directories:
        dirpath = os.path.join(dest_dir, zipinfo.filename)
        new_st = zipinfo.external_attr >> 16L
        os.chmod(dirpath, new_st)

    archive.close()
    LOGGER.debug("%s extracted to %s", archive_path, dest_dir)
    res = os.path.join(dest_dir, orig_topdir)
    return res

def extract(archive_path, directory, topdir=None):
    """Extract an archive, calling extract_zip or extract_tar
    when necessary

    The top directory of the archive will be replaced by topdir
    if it is given

    :return: path to the extracted archive
    """
    if archive_path.endswith(".zip"):
        extract_fun = extract_zip
    else:
        extract_fun = extract_tar

    # Errors returned by tarfile of zipfile are not very good,
    # so let's just catch everything
    try:
        if topdir:
            extracted = extract_fun(archive_path, directory)
            if os.path.basename(extracted) == topdir:
                return
            res = os.path.join(directory, topdir)
            qibuild.sh.rm(res)
            shutil.move(extracted, res)
        else:
            res = extract_fun(archive_path, directory)
        return res
    except Exception, err:
        mess = "Error occured when extracting %s\n" % archive_path
        mess += "Original error was: %s" % err
        raise InvalidArchive(mess)

def zip_win(directory):
    """Compress the directory in a .zip file

    """
    archive_name = directory + ".zip"
    # Convert to DOS path just to be sure:
    directory    = qibuild.sh.to_native_path(directory)
    archive_name = qibuild.sh.to_native_path(archive_name)
    archive = zipfile.ZipFile(archive_name, "w", zipfile.ZIP_DEFLATED)
    for (root, _directories, filenames) in os.walk(directory):
        for filename in filenames:
            full_path = os.path.join(root, filename)
            rel_path = os.path.relpath(full_path, directory)
            arcname  = os.path.join(os.path.basename(directory), rel_path)
            archive.write(full_path, arcname)
    archive.close()
    return archive_name

def zip_unix(directory):
    """
    Call tar cvfz on a directory

    """
    # Do not use tarfile python module when tar:
    # - is faster
    # - is not buggy
    base_dir = os.path.basename(directory)
    work_dir = os.path.abspath(os.path.join(directory, ".."))
    base_archive_name = base_dir + ".tar.gz"
    cmd = ["tar", "cfz", base_archive_name, base_dir]
    qibuild.command.call(cmd, cwd=work_dir)
    full_archive_name = os.path.join(work_dir, base_archive_name)
    return full_archive_name

def zip( directory):
    """Zip a directory, using .tar.gz or .zip according to the current
    platform

    """
    if sys.platform.startswith("win"):
        return zip_win(directory)
    else:
        return zip_unix(directory)

def extracted_name(archive_name):
    """Return the extracted name from the archive name.

    Warning: this assumes the archive is well-formed
    (ie the root dir of foo.tar.gz is foo)

    >>> extracted_name("foo.zip")
    'foo'
    >>> extracted_name("foo.tar.gz")
    'foo'
    """
    known_exts = [".zip", ".tar.gz", ".tar.bz2"]
    for ext in known_exts:
        if archive_name.endswith(ext):
            return archive_name[:-len(ext)]

def archive_name(directory):
    """ Return the name an archive made from the
    directory would have.
    (platform-dependant)

    On windows::

      >>> archive_name('foo')
      foo.zip

    Elsewhere::

      >>> archive_name('foo')
      foo.tar.gz
    """
    if sys.platform.startswith("win"):
        return directory + ".zip"
    else:
        return directory + ".tar.gz"

if __name__ == "__main__":
    extract(sys.argv[1], sys.argv[2], topdir=sys.argv[3])
<|endoftext|>


<commit_before># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.

from __future__ import absolute_import, print_function

from distutils.spawn import find_executable
from distutils.version import LooseVersion
import json
import os
import distro
import shutil
import subprocess
import six
import six.moves.urllib as urllib
from six.moves import input
from subprocess import PIPE
from zipfile import BadZipfile

import servo.packages as packages
from servo.util import extract, download_file, host_triple

def check_gstreamer_lib():
    return subprocess.call(["pkg-config", "--atleast-version=1.16", "gstreamer-1.0"],
                           stdout=PIPE, stderr=PIPE) == 0

def run_as_root(command, force=False):
    if os.geteuid() != 0:
        command.insert(0, 'sudo')
    if force:
        command.append('-y')
    return subprocess.call(command)

def install_linux_deps(context, pkgs_ubuntu, pkgs_fedora, pkgs_void, force):
    install = False
    pkgs = []
    if context.distro in ['Ubuntu', 'Debian GNU/Linux']:
        command = ['apt-get', 'install']
        pkgs = pkgs_ubuntu
        if subprocess.call(['dpkg', '-s'] + pkgs, stdout=PIPE, stderr=PIPE) != 0:
            install = True
    elif context.distro in ['CentOS', 'CentOS Linux', 'Fedora']:
        installed_pkgs = str(subprocess.check_output(['rpm', '-qa'])).replace('\n', '|')
        pkgs = pkgs_fedora
        for p in pkgs:
            command = ['dnf', 'install']
            if "|{}".format(p) not in installed_pkgs:
                install = True
                break
    elif context.distro == 'void':
        installed_pkgs = str(subprocess.check_output(['xbps-query', '-l']))
        pkgs = pkgs_void
        for p in pkgs:
            command = ['xbps-install', '-A']
            if "ii {}-".format(p) not in installed_pkgs:
                install = force = True
                break

    if install:
        print("Installing missing dependencies...")
        run_as_root(command + pkgs, force)

    return install

def install_salt_dependencies(context, force):
    pkgs_apt = ['build-essential', 'libssl-dev', 'libffi-dev', 'python-dev']
    pkgs_dnf = ['gcc', 'libffi-devel', 'python-devel', 'openssl-devel']
    pkgs_xbps = ['gcc', 'libffi-devel', 'python-devel']
    if not install_linux_deps(context, pkgs_apt, pkgs_dnf, pkgs_xbps, force):
        print("Dependencies are already installed")

def gstreamer(context, force=False):
    cur = os.curdir
    gstdir = os.path.join(cur, "support", "linux", "gstreamer")
    if not os.path.isdir(os.path.join(gstdir, "gst", "lib")):
        subprocess.check_call(["bash", "gstreamer.sh"], cwd=gstdir)
        return True
    return False

def bootstrap_gstreamer(context, force=False):
    if not gstreamer(context, force):
        print("gstreamer is already set up")
    return 0

def linux(context, force=False):
    # Please keep these in sync with the packages in README.md
    pkgs_apt = ['git', 'curl', 'autoconf', 'libx11-dev', 'libfreetype6-dev',
                'libgl1-mesa-dri', 'libglib2.0-dev', 'xorg-dev', 'gperf', 'g++',
                'build-essential', 'cmake', 'libssl-dev',
                'liblzma-dev', 'libxmu6', 'libxmu-dev',
                'libgles2-mesa-dev', 'libegl1-mesa-dev', 'libdbus-1-dev',
                'libharfbuzz-dev', 'ccache', 'clang', 'libunwind-dev',
                'libgstreamer1.0-dev', 'libgstreamer-plugins-base1.0-dev',
                'libgstreamer-plugins-bad1.0-dev', 'autoconf2.13',
                'libunwind-dev', 'llvm-dev']
    pkgs_dnf = ['libtool', 'gcc-c++', 'libXi-devel', 'freetype-devel',
                'libunwind-devel', 'mesa-libGL-devel', 'mesa-libEGL-devel',
                'glib2-devel', 'libX11-devel', 'libXrandr-devel', 'gperf',
                'fontconfig-devel', 'cabextract', 'ttmkfdir', 'expat-devel',
                'rpm-build', 'openssl-devel', 'cmake',
                'libXcursor-devel', 'libXmu-devel',
                'dbus-devel', 'ncurses-devel', 'harfbuzz-devel', 'ccache',
                'clang', 'clang-libs', 'autoconf213', 'python3-devel',
                'gstreamer1-devel', 'gstreamer1-plugins-base-devel',
                'gstreamer1-plugins-bad-free-devel']
    pkgs_xbps = ['libtool', 'gcc', 'libXi-devel', 'freetype-devel',
                 'libunwind-devel', 'MesaLib-devel', 'glib-devel', 'pkg-config',
                 'libX11-devel', 'libXrandr-devel', 'gperf', 'bzip2-devel',
                 'fontconfig-devel', 'cabextract', 'expat-devel', 'cmake',
                 'cmake', 'libXcursor-devel', 'libXmu-devel', 'dbus-devel',
                 'ncurses-devel', 'harfbuzz-devel', 'ccache', 'glu-devel',
                 'clang', 'gstreamer1-devel', 'autoconf213',
                 'gst-plugins-base1-devel', 'gst-plugins-bad1-devel']

    installed_something = install_linux_deps(context, pkgs_apt, pkgs_dnf,
                                             pkgs_xbps, force)

    if not check_gstreamer_lib():
        installed_something |= gstreamer(context, force)

    if not installed_something:
        print("Dependencies were already installed!")

    return 0

def salt(context, force=False):
    # Ensure Salt dependencies are installed
    install_salt_dependencies(context, force)
    # Ensure Salt is installed in the virtualenv
    # It's not installed globally because it's a large, non-required dependency,
    # and the installation fails on Windows
    print("Checking Salt installation...", end='')
    reqs_path = os.path.join(context.topdir, 'python', 'requirements-salt.txt')
    process = subprocess.Popen(
        ["pip", "install", "-q", "-I", "-r", reqs_path],
        stdout=PIPE,
        stderr=PIPE
    )
    process.wait()
    if process.returncode:
        out, err = process.communicate()
        print('failed to install Salt via pip:')
        print('Output: {}\nError: {}'.format(out, err))
        return 1
    print("done")

    salt_root = os.path.join(context.sharedir, 'salt')
    config_dir = os.path.join(salt_root, 'etc', 'salt')
    pillar_dir = os.path.join(config_dir, 'pillars')

    # In order to allow `mach bootstrap` to work from any CWD,
    # the `root_dir` must be an absolute path.
    # We place it under `context.sharedir` because
    # Salt caches data (e.g. gitfs files) in its `var` subdirectory.
    # Hence, dynamically generate the config with an appropriate `root_dir`
    # and serialize it as JSON (which is valid YAML).
    config = {
        'hash_type': 'sha384',
        'master': 'localhost',
        'root_dir': salt_root,
        'state_output': 'changes',
        'state_tabular': True,
    }
    if 'SERVO_SALTFS_ROOT' in os.environ:
        config.update({
            'fileserver_backend': ['roots'],
            'file_roots': {
                'base': [os.path.abspath(os.environ['SERVO_SALTFS_ROOT'])],
            },
        })
    else:
        config.update({
            'fileserver_backend': ['git'],
            'gitfs_env_whitelist': 'base',
            'gitfs_provider': 'gitpython',
            'gitfs_remotes': [
                'https://github.com/servo/saltfs.git',
            ],
        })

    if not os.path.exists(config_dir):
        os.makedirs(config_dir, mode=0o700)
    with open(os.path.join(config_dir, 'minion'), 'w') as config_file:
        config_file.write(json.dumps(config) + '\n')

    # Similarly, the pillar data is created dynamically
    # and temporarily serialized to disk.
    # This dynamism is not yet used, but will be in the future
    # to enable Android bootstrapping by using
    # context.sharedir as a location for Android packages.
    pillar = {
        'top.sls': {
            'base': {
                '*': ['bootstrap'],
            },
        },
        'bootstrap.sls': {
            'fully_managed': False,
        },
    }
    if os.path.exists(pillar_dir):
        shutil.rmtree(pillar_dir)
    os.makedirs(pillar_dir, mode=0o700)
    for filename in pillar:
        with open(os.path.join(pillar_dir, filename), 'w') as pillar_file:
            pillar_file.write(json.dumps(pillar[filename]) + '\n')

    cmd = [
        # sudo escapes from the venv, need to use full path
        find_executable('salt-call'),
        '--local',
        '--config-dir={}'.format(config_dir),
        '--pillar-root={}'.format(pillar_dir),
        'state.apply',
        'servo-build-dependencies',
    ]

    if not force:
        print('Running bootstrap in dry-run mode to show changes')
        # Because `test=True` mode runs each state individually without
        # considering how required/previous states affect the system,
        # it will often report states with requisites as failing due
        # to the requisites not actually being run,
        # even though these are spurious and will succeed during
        # the actual highstate.
        # Hence `--retcode-passthrough` is not helpful in dry-run mode,
        # so only detect failures of the actual salt-call binary itself.
        retcode = run_as_root(cmd + ['test=True'])
        if retcode != 0:
            print('Something went wrong while bootstrapping')
            return retcode

        proceed = input(
            'Proposed changes are above, proceed with bootstrap? [y/N]: '
        )
        if proceed.lower() not in ['y', 'yes']:
            return 0

        print('')

    print('Running Salt bootstrap')
    retcode = run_as_root(cmd + ['--retcode-passthrough'])
    if retcode == 0:
        print('Salt bootstrapping complete')
    else:
        print('Salt bootstrapping encountered errors')
    return retcode

def windows_msvc(context, force=False):
    '''Bootstrapper for MSVC building on Windows.'''

    deps_dir = os.path.join(context.sharedir, "msvc-dependencies")
    deps_url = "https://servo-deps.s3.amazonaws.com/msvc-deps/"

    def version(package):
        return packages.WINDOWS_MSVC[package]

    def package_dir(package):
        return os.path.join(deps_dir, package, version(package))

    def check_cmake(version):
        cmake_path = find_executable("cmake")
        if cmake_path:
            cmake = subprocess.Popen([cmake_path, "--version"], stdout=PIPE)
            cmake_version = cmake.stdout.read().splitlines()[0].replace("cmake version ", "")
            if LooseVersion(cmake_version) >= LooseVersion(version):
                return True
        return False

    def prepare_file(zip_path, full_spec):
        if not os.path.isfile(zip_path):
            zip_url = "{}{}.zip".format(deps_url, urllib.parse.quote(full_spec))
            download_file(full_spec, zip_url, zip_path)

        print("Extracting {}...".format(full_spec), end='')
        try:
            extract(zip_path, deps_dir)
        except BadZipfile:
            print("\nError: %s.zip is not a valid zip file, redownload..." % full_spec)
            os.remove(zip_path)
            prepare_file(zip_path, full_spec)
        else:
            print("done")

    to_install = {}
    for package in packages.WINDOWS_MSVC:
        # Don't install CMake if it already exists in PATH
        if package == "cmake" and check_cmake(version("cmake")):
            continue

        if not os.path.isdir(package_dir(package)):
            to_install[package] = version(package)

    if not to_install:
        return 0

    print("Installing missing MSVC dependencies...")
    for package in to_install:
        full_spec = '{}-{}'.format(package, version(package))

        parent_dir = os.path.dirname(package_dir(package))
        if not os.path.isdir(parent_dir):
            os.makedirs(parent_dir)

        zip_path = package_dir(package) + ".zip"
        prepare_file(zip_path, full_spec)

        extracted_path = os.path.join(deps_dir, full_spec)
        os.rename(extracted_path, package_dir(package))

    return 0

LINUX_SPECIFIC_BOOTSTRAPPERS = {
    "salt": salt,
    "gstreamer": bootstrap_gstreamer,
}

def get_linux_distribution():
    distrib, version, _ = distro.linux_distribution()
    distrib = six.ensure_str(distrib)
    version = six.ensure_str(version)

    if distrib == 'LinuxMint' or distrib == 'Linux Mint':
        if '.' in version:
            major, _ = version.split('.', 1)
        else:
            major = version

        if major == '19':
            base_version = '18.04'
        elif major == '18':
            base_version = '16.04'
        else:
            raise Exception('unsupported version of %s: %s' % (distrib, version))

        distrib, version = 'Ubuntu', base_version
    elif distrib == 'Pop!_OS':
        if '.' in version:
            major, _ = version.split('.', 1)
        else:
            major = version

        if major == '19':
            base_version = '18.04'
        elif major == '18':
            base_version = '16.04'
        else:
            raise Exception('unsupported version of %s: %s' % (distrib, version))

        distrib, version = 'Ubuntu', base_version
    elif distrib.lower() == 'elementary':
        if version == '5.0':
            base_version = '18.04'
        elif version[0:3] == '0.4':
            base_version = '16.04'
        else:
            raise Exception('unsupported version of %s: %s' % (distrib, version))
        distrib, version = 'Ubuntu', base_version
    elif distrib.lower() == 'ubuntu':
        if version > '20.04':
            raise Exception('unsupported version of %s: %s' % (distrib, version))
    # Fixme: we should allow checked/supported versions only
    elif distrib.lower() not in [
        'centos',
        'centos linux',
        'debian gnu/linux',
        'fedora',
        'void',
    ]:
        raise Exception('mach bootstrap does not support %s, please file a bug' % distrib)

    return distrib, version

def bootstrap(context, force=False, specific=None):
    '''Dispatches to the right bootstrapping function for the OS.'''

    bootstrapper = None
    if "windows-msvc" in host_triple():
        bootstrapper = windows_msvc
    elif "linux-gnu" in host_triple():
        distrib, version = get_linux_distribution()

        context.distro = distrib
        context.distro_version = version
        bootstrapper = LINUX_SPECIFIC_BOOTSTRAPPERS.get(specific, linux)

    if bootstrapper is None:
        print('Bootstrap support is not yet available for your OS.')
        return 1

    return bootstrapper(context, force=force)
<commit_msg>support Pop!_OS 20.04<commit_after># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.

from __future__ import absolute_import, print_function

from distutils.spawn import find_executable
from distutils.version import LooseVersion
import json
import os
import distro
import shutil
import subprocess
import six
import six.moves.urllib as urllib
from six.moves import input
from subprocess import PIPE
from zipfile import BadZipfile

import servo.packages as packages
from servo.util import extract, download_file, host_triple

def check_gstreamer_lib():
    return subprocess.call(["pkg-config", "--atleast-version=1.16", "gstreamer-1.0"],
                           stdout=PIPE, stderr=PIPE) == 0

def run_as_root(command, force=False):
    if os.geteuid() != 0:
        command.insert(0, 'sudo')
    if force:
        command.append('-y')
    return subprocess.call(command)

def install_linux_deps(context, pkgs_ubuntu, pkgs_fedora, pkgs_void, force):
    install = False
    pkgs = []
    if context.distro in ['Ubuntu', 'Debian GNU/Linux']:
        command = ['apt-get', 'install']
        pkgs = pkgs_ubuntu
        if subprocess.call(['dpkg', '-s'] + pkgs, stdout=PIPE, stderr=PIPE) != 0:
            install = True
    elif context.distro in ['CentOS', 'CentOS Linux', 'Fedora']:
        installed_pkgs = str(subprocess.check_output(['rpm', '-qa'])).replace('\n', '|')
        pkgs = pkgs_fedora
        for p in pkgs:
            command = ['dnf', 'install']
            if "|{}".format(p) not in installed_pkgs:
                install = True
                break
    elif context.distro == 'void':
        installed_pkgs = str(subprocess.check_output(['xbps-query', '-l']))
        pkgs = pkgs_void
        for p in pkgs:
            command = ['xbps-install', '-A']
            if "ii {}-".format(p) not in installed_pkgs:
                install = force = True
                break

    if install:
        print("Installing missing dependencies...")
        run_as_root(command + pkgs, force)

    return install

def install_salt_dependencies(context, force):
    pkgs_apt = ['build-essential', 'libssl-dev', 'libffi-dev', 'python-dev']
    pkgs_dnf = ['gcc', 'libffi-devel', 'python-devel', 'openssl-devel']
    pkgs_xbps = ['gcc', 'libffi-devel', 'python-devel']
    if not install_linux_deps(context, pkgs_apt, pkgs_dnf, pkgs_xbps, force):
        print("Dependencies are already installed")

def gstreamer(context, force=False):
    cur = os.curdir
    gstdir = os.path.join(cur, "support", "linux", "gstreamer")
    if not os.path.isdir(os.path.join(gstdir, "gst", "lib")):
        subprocess.check_call(["bash", "gstreamer.sh"], cwd=gstdir)
        return True
    return False

def bootstrap_gstreamer(context, force=False):
    if not gstreamer(context, force):
        print("gstreamer is already set up")
    return 0

def linux(context, force=False):
    # Please keep these in sync with the packages in README.md
    pkgs_apt = ['git', 'curl', 'autoconf', 'libx11-dev', 'libfreetype6-dev',
                'libgl1-mesa-dri', 'libglib2.0-dev', 'xorg-dev', 'gperf', 'g++',
                'build-essential', 'cmake', 'libssl-dev',
                'liblzma-dev', 'libxmu6', 'libxmu-dev',
                'libgles2-mesa-dev', 'libegl1-mesa-dev', 'libdbus-1-dev',
                'libharfbuzz-dev', 'ccache', 'clang', 'libunwind-dev',
                'libgstreamer1.0-dev', 'libgstreamer-plugins-base1.0-dev',
                'libgstreamer-plugins-bad1.0-dev', 'autoconf2.13',
                'libunwind-dev', 'llvm-dev']
    pkgs_dnf = ['libtool', 'gcc-c++', 'libXi-devel', 'freetype-devel',
                'libunwind-devel', 'mesa-libGL-devel', 'mesa-libEGL-devel',
                'glib2-devel', 'libX11-devel', 'libXrandr-devel', 'gperf',
                'fontconfig-devel', 'cabextract', 'ttmkfdir', 'expat-devel',
                'rpm-build', 'openssl-devel', 'cmake',
                'libXcursor-devel', 'libXmu-devel',
                'dbus-devel', 'ncurses-devel', 'harfbuzz-devel', 'ccache',
                'clang', 'clang-libs', 'autoconf213', 'python3-devel',
                'gstreamer1-devel', 'gstreamer1-plugins-base-devel',
                'gstreamer1-plugins-bad-free-devel']
    pkgs_xbps = ['libtool', 'gcc', 'libXi-devel', 'freetype-devel',
                 'libunwind-devel', 'MesaLib-devel', 'glib-devel', 'pkg-config',
                 'libX11-devel', 'libXrandr-devel', 'gperf', 'bzip2-devel',
                 'fontconfig-devel', 'cabextract', 'expat-devel', 'cmake',
                 'cmake', 'libXcursor-devel', 'libXmu-devel', 'dbus-devel',
                 'ncurses-devel', 'harfbuzz-devel', 'ccache', 'glu-devel',
                 'clang', 'gstreamer1-devel', 'autoconf213',
                 'gst-plugins-base1-devel', 'gst-plugins-bad1-devel']

    installed_something = install_linux_deps(context, pkgs_apt, pkgs_dnf,
                                             pkgs_xbps, force)

    if not check_gstreamer_lib():
        installed_something |= gstreamer(context, force)

    if not installed_something:
        print("Dependencies were already installed!")

    return 0

def salt(context, force=False):
    # Ensure Salt dependencies are installed
    install_salt_dependencies(context, force)
    # Ensure Salt is installed in the virtualenv
    # It's not installed globally because it's a large, non-required dependency,
    # and the installation fails on Windows
    print("Checking Salt installation...", end='')
    reqs_path = os.path.join(context.topdir, 'python', 'requirements-salt.txt')
    process = subprocess.Popen(
        ["pip", "install", "-q", "-I", "-r", reqs_path],
        stdout=PIPE,
        stderr=PIPE
    )
    process.wait()
    if process.returncode:
        out, err = process.communicate()
        print('failed to install Salt via pip:')
        print('Output: {}\nError: {}'.format(out, err))
        return 1
    print("done")

    salt_root = os.path.join(context.sharedir, 'salt')
    config_dir = os.path.join(salt_root, 'etc', 'salt')
    pillar_dir = os.path.join(config_dir, 'pillars')

    # In order to allow `mach bootstrap` to work from any CWD,
    # the `root_dir` must be an absolute path.
    # We place it under `context.sharedir` because
    # Salt caches data (e.g. gitfs files) in its `var` subdirectory.
    # Hence, dynamically generate the config with an appropriate `root_dir`
    # and serialize it as JSON (which is valid YAML).
    config = {
        'hash_type': 'sha384',
        'master': 'localhost',
        'root_dir': salt_root,
        'state_output': 'changes',
        'state_tabular': True,
    }
    if 'SERVO_SALTFS_ROOT' in os.environ:
        config.update({
            'fileserver_backend': ['roots'],
            'file_roots': {
                'base': [os.path.abspath(os.environ['SERVO_SALTFS_ROOT'])],
            },
        })
    else:
        config.update({
            'fileserver_backend': ['git'],
            'gitfs_env_whitelist': 'base',
            'gitfs_provider': 'gitpython',
            'gitfs_remotes': [
                'https://github.com/servo/saltfs.git',
            ],
        })

    if not os.path.exists(config_dir):
        os.makedirs(config_dir, mode=0o700)
    with open(os.path.join(config_dir, 'minion'), 'w') as config_file:
        config_file.write(json.dumps(config) + '\n')

    # Similarly, the pillar data is created dynamically
    # and temporarily serialized to disk.
    # This dynamism is not yet used, but will be in the future
    # to enable Android bootstrapping by using
    # context.sharedir as a location for Android packages.
    pillar = {
        'top.sls': {
            'base': {
                '*': ['bootstrap'],
            },
        },
        'bootstrap.sls': {
            'fully_managed': False,
        },
    }
    if os.path.exists(pillar_dir):
        shutil.rmtree(pillar_dir)
    os.makedirs(pillar_dir, mode=0o700)
    for filename in pillar:
        with open(os.path.join(pillar_dir, filename), 'w') as pillar_file:
            pillar_file.write(json.dumps(pillar[filename]) + '\n')

    cmd = [
        # sudo escapes from the venv, need to use full path
        find_executable('salt-call'),
        '--local',
        '--config-dir={}'.format(config_dir),
        '--pillar-root={}'.format(pillar_dir),
        'state.apply',
        'servo-build-dependencies',
    ]

    if not force:
        print('Running bootstrap in dry-run mode to show changes')
        # Because `test=True` mode runs each state individually without
        # considering how required/previous states affect the system,
        # it will often report states with requisites as failing due
        # to the requisites not actually being run,
        # even though these are spurious and will succeed during
        # the actual highstate.
        # Hence `--retcode-passthrough` is not helpful in dry-run mode,
        # so only detect failures of the actual salt-call binary itself.
        retcode = run_as_root(cmd + ['test=True'])
        if retcode != 0:
            print('Something went wrong while bootstrapping')
            return retcode

        proceed = input(
            'Proposed changes are above, proceed with bootstrap? [y/N]: '
        )
        if proceed.lower() not in ['y', 'yes']:
            return 0

        print('')

    print('Running Salt bootstrap')
    retcode = run_as_root(cmd + ['--retcode-passthrough'])
    if retcode == 0:
        print('Salt bootstrapping complete')
    else:
        print('Salt bootstrapping encountered errors')
    return retcode

def windows_msvc(context, force=False):
    '''Bootstrapper for MSVC building on Windows.'''

    deps_dir = os.path.join(context.sharedir, "msvc-dependencies")
    deps_url = "https://servo-deps.s3.amazonaws.com/msvc-deps/"

    def version(package):
        return packages.WINDOWS_MSVC[package]

    def package_dir(package):
        return os.path.join(deps_dir, package, version(package))

    def check_cmake(version):
        cmake_path = find_executable("cmake")
        if cmake_path:
            cmake = subprocess.Popen([cmake_path, "--version"], stdout=PIPE)
            cmake_version = cmake.stdout.read().splitlines()[0].replace("cmake version ", "")
            if LooseVersion(cmake_version) >= LooseVersion(version):
                return True
        return False

    def prepare_file(zip_path, full_spec):
        if not os.path.isfile(zip_path):
            zip_url = "{}{}.zip".format(deps_url, urllib.parse.quote(full_spec))
            download_file(full_spec, zip_url, zip_path)

        print("Extracting {}...".format(full_spec), end='')
        try:
            extract(zip_path, deps_dir)
        except BadZipfile:
            print("\nError: %s.zip is not a valid zip file, redownload..." % full_spec)
            os.remove(zip_path)
            prepare_file(zip_path, full_spec)
        else:
            print("done")

    to_install = {}
    for package in packages.WINDOWS_MSVC:
        # Don't install CMake if it already exists in PATH
        if package == "cmake" and check_cmake(version("cmake")):
            continue

        if not os.path.isdir(package_dir(package)):
            to_install[package] = version(package)

    if not to_install:
        return 0

    print("Installing missing MSVC dependencies...")
    for package in to_install:
        full_spec = '{}-{}'.format(package, version(package))

        parent_dir = os.path.dirname(package_dir(package))
        if not os.path.isdir(parent_dir):
            os.makedirs(parent_dir)

        zip_path = package_dir(package) + ".zip"
        prepare_file(zip_path, full_spec)

        extracted_path = os.path.join(deps_dir, full_spec)
        os.rename(extracted_path, package_dir(package))

    return 0

LINUX_SPECIFIC_BOOTSTRAPPERS = {
    "salt": salt,
    "gstreamer": bootstrap_gstreamer,
}

def get_linux_distribution():
    distrib, version, _ = distro.linux_distribution()
    distrib = six.ensure_str(distrib)
    version = six.ensure_str(version)

    if distrib == 'LinuxMint' or distrib == 'Linux Mint':
        if '.' in version:
            major, _ = version.split('.', 1)
        else:
            major = version

        if major == '19':
            base_version = '18.04'
        elif major == '18':
            base_version = '16.04'
        else:
            raise Exception('unsupported version of %s: %s' % (distrib, version))

        distrib, version = 'Ubuntu', base_version
    elif distrib == 'Pop!_OS':
        if '.' in version:
            major, _ = version.split('.', 1)
        else:
            major = version

        if major == '20':
            base_version = '20.04'
        elif major == '19':
            base_version = '18.04'
        elif major == '18':
            base_version = '16.04'
        else:
            raise Exception('unsupported version of %s: %s' % (distrib, version))

        distrib, version = 'Ubuntu', base_version
    elif distrib.lower() == 'elementary':
        if version == '5.0':
            base_version = '18.04'
        elif version[0:3] == '0.4':
            base_version = '16.04'
        else:
            raise Exception('unsupported version of %s: %s' % (distrib, version))
        distrib, version = 'Ubuntu', base_version
    elif distrib.lower() == 'ubuntu':
        if version > '20.04':
            raise Exception('unsupported version of %s: %s' % (distrib, version))
    # Fixme: we should allow checked/supported versions only
    elif distrib.lower() not in [
        'centos',
        'centos linux',
        'debian gnu/linux',
        'fedora',
        'void',
    ]:
        raise Exception('mach bootstrap does not support %s, please file a bug' % distrib)

    return distrib, version

def bootstrap(context, force=False, specific=None):
    '''Dispatches to the right bootstrapping function for the OS.'''

    bootstrapper = None
    if "windows-msvc" in host_triple():
        bootstrapper = windows_msvc
    elif "linux-gnu" in host_triple():
        distrib, version = get_linux_distribution()

        context.distro = distrib
        context.distro_version = version
        bootstrapper = LINUX_SPECIFIC_BOOTSTRAPPERS.get(specific, linux)

    if bootstrapper is None:
        print('Bootstrap support is not yet available for your OS.')
        return 1

    return bootstrapper(context, force=force)
<|endoftext|>


<commit_before>from numpy import fft, random,frombuffer, mean, zeros

class ThunderDetector:
    def __init__(self, chunk):
        self.intensity=0
        self.ledindex=0
        self.time_since_thunder=0
        self.minfbin=0    #thunder is expected between these two frequencies
        self.maxfbin = int(2500/(44100 / chunk))
        self.prev_avg=zeros((1, chunk))
        self.prev_max=zeros((1,chunk))
        self.initflag=1;

    def detect(self, chunk, onthreshold, offthreshold):
        data = frombuffer(chunk, dtype='<i2')
        fcontent = abs(fft.fft(data))
        levels = fcontent[self.minfbin:self.maxfbin]
        if self.initflag==1:
            self.initflag=0
            self.prev_avg = fcontent[self.minfbin:self.maxfbin]
        else:
            if (self.time_since_thunder==0 and any(levels >offthreshold*self.prev_avg)):
                #continuing old thunder
                self.intensity = min(255, 255 * max(levels) / max(self.prev_max)) # normalize max brightness to 255
                if self.intensity==255:
                    self.prev_max=levels
                self.time_since_thunder = 0
            elif any(levels > onthreshold*self.prev_avg): #new thunder
                self.intensity= 255 #normalize max brightness to 255
                self.prev_max=levels
                self.time_since_thunder=0
            else: #no thunder
                self.intensity=0
                self.prev_avg = (fcontent[self.minfbin:self.maxfbin]+self.prev_avg)/2
                self.time_since_thunder=self.time_since_thunder+1
            return self.intensity

    def setindex(self):
        self.ledindex = random.uniform(0,256)
<commit_msg>fixed how max happens<commit_after>from numpy import fft, random,frombuffer, mean, zeros

class ThunderDetector:
    def __init__(self, chunk):
        self.intensity=0
        self.ledindex=0
        self.time_since_thunder=0
        self.minfbin=0    #thunder is expected between these two frequencies
        self.maxfbin = int(2500/(44100 / chunk))
        self.prev_avg=zeros((1, chunk))
        self.prev_max=1
        self.initflag=1;

    def detect(self, chunk, onthreshold, offthreshold):
        data = frombuffer(chunk, dtype='<i2')
        fcontent = abs(fft.fft(data))
        levels = fcontent[self.minfbin:self.maxfbin]
        if self.initflag==1:
            self.initflag=0
            self.prev_avg = fcontent[self.minfbin:self.maxfbin]
            self.prev_max=max(self.prev_avg)
        else:
            if (self.time_since_thunder==0 and any(levels >offthreshold*self.prev_avg)):
                #continuing old thunder
                self.intensity = min(255, 255 * max(levels) / self.prev_max) # normalize max brightness to 255
                if self.intensity==255:
                    self.prev_max=max(levels)
                self.time_since_thunder = 0
            elif any(levels > onthreshold*self.prev_avg): #new thunder
                self.intensity= 255 #normalize max brightness to 255
                self.prev_max=max(levels)
                self.time_since_thunder=0
            else: #no thunder
                self.intensity=0
                self.prev_avg = (fcontent[self.minfbin:self.maxfbin]+self.prev_avg)/2
                self.time_since_thunder=self.time_since_thunder+1
            return self.intensity

    def setindex(self):
        self.ledindex = random.uniform(0,256)
<|endoftext|>


<commit_before><commit_msg>Added __version__<commit_after>#!/usr/bin/env python
__version__ = "0.1.0"
<|endoftext|>


<commit_before># ----------------------------------------------------------------------------
# Copyright (c) 2016-2017, QIIME 2 development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file LICENSE, distributed with this software.
# ----------------------------------------------------------------------------
import qiime2.plugin
import importlib
from q2_types.feature_table import FeatureTable, Frequency
from q2_types.feature_data import FeatureData, Sequence
from q2_types.sample_data import SampleData
from q2_types.per_sample_sequences import (SequencesWithQuality,
                                           PairedEndSequencesWithQuality)

import q2_deblur

from q2_deblur._type import DeblurStats
from q2_deblur._format import DeblurStatsFmt, DeblurStatsDirFmt

plugin = qiime2.plugin.Plugin(
    name='deblur',
    version=q2_deblur.__version__,
    website='https://github.com/biocore/deblur',
    package='q2_deblur',
    user_support_text='https://github.com/biocore/deblur/issues',
    citation_text=("Deblur rapidly resolves single-nucleotide community "
                   "sequence patterns. Amnon Amir, Daniel McDonald, Jose "
                   "A. Navas-Molina, Evguenia Kopylova, Jamie Morton, "
                   "Zhenjiang Zech Xu, Eric P. Kightley, Luke R. Thompson, "
                   "Embriette R. Hyde, Antonio Gonzalez, Rob Knight. mSystems "
                   "Mar 2017, 2 (2) doi:10.1128/mSystems.00191-16.")
)

plugin.register_formats(DeblurStatsFmt, DeblurStatsDirFmt)
plugin.register_semantic_types(DeblurStats)
plugin.register_semantic_type_to_format(DeblurStats,
                                        artifact_format=DeblurStatsDirFmt)

_parameter_descriptions = {
    'mean_error': ("The mean per nucleotide error, used for original "
                   "sequence estimate."),
    'indel_prob': ('Insertion/deletion (indel) probability (same for N '
                   'indels).'),
    'indel_max': "Maximum number of insertion/deletions.",
    'trim_length': "Sequence trim length, specify -1 to disable trimming.",
    'min_reads': ("Retain only features appearing at least min_reads "
                  "times across all samples in the resulting feature "
                  "table."),
    'min_size': ("In each sample, discard all features with an abundance "
                 "less than min_size."),
    'jobs_to_start': "Number of jobs to start (if to run in parallel).",
    'hashed_feature_ids': "If true, hash the feature IDs.",
    'sample_stats': "If true, gather stats per sample."
}

_output_descriptions = {
    'table': 'The resulting denoised feature table.',
    'representative_sequences': 'The resulting feature sequences.',
    'stats': 'Per-sample stats if requested.'
}

_parameters = {
    'mean_error': qiime2.plugin.Float,
    'indel_prob': qiime2.plugin.Float,
    'indel_max': qiime2.plugin.Int,
    'trim_length': qiime2.plugin.Int,
    'min_reads': qiime2.plugin.Int,
    'min_size': qiime2.plugin.Int,
    'jobs_to_start': qiime2.plugin.Int,
    'hashed_feature_ids': qiime2.plugin.Bool,
    'sample_stats': qiime2.plugin.Bool
}

_outputs = [('table', FeatureTable[Frequency]),
            ('representative_sequences', FeatureData[Sequence]),
            ('stats', DeblurStats)]

plugin.methods.register_function(
    function=q2_deblur.denoise_16S,
    inputs={
        'demultiplexed_seqs': SampleData[SequencesWithQuality |
                                         PairedEndSequencesWithQuality],
    },
    parameters=_parameters,
    outputs=_outputs,
    input_descriptions={
        'demultiplexed_seqs': 'The demultiplexed sequences to be denoised.',
    },
    parameter_descriptions=_parameter_descriptions,
    output_descriptions=_output_descriptions,
    name='Deblur sequences using a 16S positive filter.',
    description=('Perform sequence quality control for Illumina data using '
                 'the Deblur workflow with a 16S reference as a positive '
                 'filter. Only forward reads are supported at this time. The '
                 'specific reference used is the 88% OTUs from Greengenes '
                 '13_8. This mode of operation should only be used when data '
                 'were generated from a 16S amplicon protocol on an Illumina '
                 'platform. The reference is only used to assess whether each '
                 'sequence is likely to be 16S by a local alignment using '
                 'SortMeRNA with a permissive e-value; the reference is not '
                 'used to characterize the sequences.')
)

plugin.methods.register_function(
    function=q2_deblur.denoise_other,
    inputs={
        'demultiplexed_seqs': SampleData[SequencesWithQuality |
                                         PairedEndSequencesWithQuality],
        'reference_seqs': FeatureData[Sequence],
    },
    parameters=_parameters,
    outputs=_outputs,
    input_descriptions={
        'demultiplexed_seqs': 'The demultiplexed sequences to be denoised.',
        'reference_seqs': ("Positive filtering database. Keep all "
                           "sequences aligning to these sequences."),
    },
    parameter_descriptions=_parameter_descriptions,
    output_descriptions=_output_descriptions,
    name='Deblur sequences using a user-specified positive filter.',
    description=('Perform sequence quality control for Illumina data using '
                 'the Deblur workflow, including positive alignment-based '
                 'filtering. Only forward reads are supported at this time. '
                 'This mode of execution is particularly useful when '
                 'operating on non-16S data. For example, to apply Deblur to '
                 '18S data, you would want to specify a reference composed of '
                 '18S sequences in order to filter out sequences which do not '
                 'appear to be 18S. The assessment is performed by local '
                 'alignment using SortMeRNA with a permissive e-value '
                 'threshold.')
)

plugin.visualizers.register_function(
    function=q2_deblur.visualize_stats,
    inputs={'deblur_stats': DeblurStats},
    parameters={},
    input_descriptions={
        'deblur_stats': 'Summary statistics of the Deblur process.'
    },
    parameter_descriptions={},
    name='Visualize Deblur stats per sample.',
    description='Display Deblur statistics per sample'
)

importlib.import_module('q2_deblur._transformer')
<commit_msg>DOC: add plugin description (#38)<commit_after># ----------------------------------------------------------------------------
# Copyright (c) 2016-2017, QIIME 2 development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file LICENSE, distributed with this software.
# ----------------------------------------------------------------------------
import qiime2.plugin
import importlib
from q2_types.feature_table import FeatureTable, Frequency
from q2_types.feature_data import FeatureData, Sequence
from q2_types.sample_data import SampleData
from q2_types.per_sample_sequences import (SequencesWithQuality,
                                           PairedEndSequencesWithQuality)

import q2_deblur

from q2_deblur._type import DeblurStats
from q2_deblur._format import DeblurStatsFmt, DeblurStatsDirFmt

plugin = qiime2.plugin.Plugin(
    name='deblur',
    version=q2_deblur.__version__,
    website='https://github.com/biocore/deblur',
    package='q2_deblur',
    citation_text=("Deblur rapidly resolves single-nucleotide community "
                   "sequence patterns. Amnon Amir, Daniel McDonald, Jose "
                   "A. Navas-Molina, Evguenia Kopylova, Jamie Morton, "
                   "Zhenjiang Zech Xu, Eric P. Kightley, Luke R. Thompson, "
                   "Embriette R. Hyde, Antonio Gonzalez, Rob Knight. mSystems "
                   "Mar 2017, 2 (2) doi:10.1128/mSystems.00191-16."),
    description=('This QIIME 2 plugin wraps the Deblur software for '
                 'performing sequence quality control.'),
    short_description='Plugin for sequence quality control with Deblur.'
)

plugin.register_formats(DeblurStatsFmt, DeblurStatsDirFmt)
plugin.register_semantic_types(DeblurStats)
plugin.register_semantic_type_to_format(DeblurStats,
                                        artifact_format=DeblurStatsDirFmt)

_parameter_descriptions = {
    'mean_error': ("The mean per nucleotide error, used for original "
                   "sequence estimate."),
    'indel_prob': ('Insertion/deletion (indel) probability (same for N '
                   'indels).'),
    'indel_max': "Maximum number of insertion/deletions.",
    'trim_length': "Sequence trim length, specify -1 to disable trimming.",
    'min_reads': ("Retain only features appearing at least min_reads "
                  "times across all samples in the resulting feature "
                  "table."),
    'min_size': ("In each sample, discard all features with an abundance "
                 "less than min_size."),
    'jobs_to_start': "Number of jobs to start (if to run in parallel).",
    'hashed_feature_ids': "If true, hash the feature IDs.",
    'sample_stats': "If true, gather stats per sample."
}

_output_descriptions = {
    'table': 'The resulting denoised feature table.',
    'representative_sequences': 'The resulting feature sequences.',
    'stats': 'Per-sample stats if requested.'
}

_parameters = {
    'mean_error': qiime2.plugin.Float,
    'indel_prob': qiime2.plugin.Float,
    'indel_max': qiime2.plugin.Int,
    'trim_length': qiime2.plugin.Int,
    'min_reads': qiime2.plugin.Int,
    'min_size': qiime2.plugin.Int,
    'jobs_to_start': qiime2.plugin.Int,
    'hashed_feature_ids': qiime2.plugin.Bool,
    'sample_stats': qiime2.plugin.Bool
}

_outputs = [('table', FeatureTable[Frequency]),
            ('representative_sequences', FeatureData[Sequence]),
            ('stats', DeblurStats)]

plugin.methods.register_function(
    function=q2_deblur.denoise_16S,
    inputs={
        'demultiplexed_seqs': SampleData[SequencesWithQuality |
                                         PairedEndSequencesWithQuality],
    },
    parameters=_parameters,
    outputs=_outputs,
    input_descriptions={
        'demultiplexed_seqs': 'The demultiplexed sequences to be denoised.',
    },
    parameter_descriptions=_parameter_descriptions,
    output_descriptions=_output_descriptions,
    name='Deblur sequences using a 16S positive filter.',
    description=('Perform sequence quality control for Illumina data using '
                 'the Deblur workflow with a 16S reference as a positive '
                 'filter. Only forward reads are supported at this time. The '
                 'specific reference used is the 88% OTUs from Greengenes '
                 '13_8. This mode of operation should only be used when data '
                 'were generated from a 16S amplicon protocol on an Illumina '
                 'platform. The reference is only used to assess whether each '
                 'sequence is likely to be 16S by a local alignment using '
                 'SortMeRNA with a permissive e-value; the reference is not '
                 'used to characterize the sequences.')
)

plugin.methods.register_function(
    function=q2_deblur.denoise_other,
    inputs={
        'demultiplexed_seqs': SampleData[SequencesWithQuality |
                                         PairedEndSequencesWithQuality],
        'reference_seqs': FeatureData[Sequence],
    },
    parameters=_parameters,
    outputs=_outputs,
    input_descriptions={
        'demultiplexed_seqs': 'The demultiplexed sequences to be denoised.',
        'reference_seqs': ("Positive filtering database. Keep all "
                           "sequences aligning to these sequences."),
    },
    parameter_descriptions=_parameter_descriptions,
    output_descriptions=_output_descriptions,
    name='Deblur sequences using a user-specified positive filter.',
    description=('Perform sequence quality control for Illumina data using '
                 'the Deblur workflow, including positive alignment-based '
                 'filtering. Only forward reads are supported at this time. '
                 'This mode of execution is particularly useful when '
                 'operating on non-16S data. For example, to apply Deblur to '
                 '18S data, you would want to specify a reference composed of '
                 '18S sequences in order to filter out sequences which do not '
                 'appear to be 18S. The assessment is performed by local '
                 'alignment using SortMeRNA with a permissive e-value '
                 'threshold.')
)

plugin.visualizers.register_function(
    function=q2_deblur.visualize_stats,
    inputs={'deblur_stats': DeblurStats},
    parameters={},
    input_descriptions={
        'deblur_stats': 'Summary statistics of the Deblur process.'
    },
    parameter_descriptions={},
    name='Visualize Deblur stats per sample.',
    description='Display Deblur statistics per sample'
)

importlib.import_module('q2_deblur._transformer')
<|endoftext|>


<commit_before># Copyright (c) 2010-2012, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import tempfile
import numpy
import StringIO
import shutil

from qa_tests import _utils as qa_utils
from tests.utils import helpers

from openquake.export import risk as risk_export
from openquake.db import models

class BaseRiskQATestCase(qa_utils.BaseQATestCase):
    """
    Base abstract class for risk QA tests. Derived class must define

    0) a test method (properly annotated) that executes run_test
    1) a cfg property holding the path to job.ini to execute
    2) a method hazard_id() that creates a proper hazard input
    3) a method actual_data that gets a list of
       individual risk output (e.g. a `openquake.db.models.LossCurveData`)
    4) a method expected_data to assert against the actual_data
    5) a method actual_outputs that gets a list of the created risk
       `openquake.db.models.Output`
    6) a method expected_outputs to assert against the actual_outputs
    """

    def run_risk(self, cfg, hazard_id):
        """
        Given the path to job config file, run the job and assert that it was
        successful. If this assertion passes, return the completed job.

        :param str cfg:
            Path to a job config file.
        :param int hazard_id:
            ID of the hazard output used by the risk calculation
        :returns:
            The completed :class:`~openquake.db.models.OqJob`.
        :raises:
            :exc:`AssertionError` if the job was not successfully run.
        """
        job_status = helpers.run_risk_job_sp(cfg, hazard_id=hazard_id,
                                             silence=True)
        self.assertEqual(0, job_status)

        completed_job = models.OqJob.objects.latest('last_update')
        self.assertEqual('complete', completed_job.status)
        return completed_job

    def run_test(self):
        result_dir = tempfile.mkdtemp()

        try:
            expected_data = self.expected_data()

            job = self.run_risk(self.cfg, self.hazard_id())

            actual_data = self.actual_data(job)

            for i, actual in enumerate(actual_data):
                numpy.allclose(actual, expected_data[i], rtol=0.01, atol=0.0)

            expected_outputs = self.expected_outputs()

            for i, output in enumerate(models.Output.objects.filter(
                    oq_job=job).order_by('id')):
                [exported_file] = risk_export.export(output.id, result_dir)
                self.assert_xml_equal(
                    StringIO.StringIO(expected_outputs[i]), exported_file)
        finally:
            shutil.rmtree(result_dir)
<commit_msg>Fixed values check<commit_after># Copyright (c) 2010-2012, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import tempfile
import numpy
import StringIO
import shutil

from qa_tests import _utils as qa_utils
from tests.utils import helpers

from openquake.export import risk as risk_export
from openquake.db import models

class BaseRiskQATestCase(qa_utils.BaseQATestCase):
    """
    Base abstract class for risk QA tests. Derived class must define

    0) a test method (properly annotated) that executes run_test
    1) a cfg property holding the path to job.ini to execute
    2) a method hazard_id() that creates a proper hazard input
    3) a method actual_data that gets a list of
       individual risk output (e.g. a `openquake.db.models.LossCurveData`)
    4) a method expected_data to assert against the actual_data
    5) a method actual_outputs that gets a list of the created risk
       `openquake.db.models.Output`
    6) a method expected_outputs to assert against the actual_outputs
    """

    def run_risk(self, cfg, hazard_id):
        """
        Given the path to job config file, run the job and assert that it was
        successful. If this assertion passes, return the completed job.

        :param str cfg:
            Path to a job config file.
        :param int hazard_id:
            ID of the hazard output used by the risk calculation
        :returns:
            The completed :class:`~openquake.db.models.OqJob`.
        :raises:
            :exc:`AssertionError` if the job was not successfully run.
        """
        job_status = helpers.run_risk_job_sp(cfg, hazard_id=hazard_id,
                                             silence=True)
        self.assertEqual(0, job_status)

        completed_job = models.OqJob.objects.latest('last_update')
        self.assertEqual('complete', completed_job.status)
        return completed_job

    def run_test(self):
        result_dir = tempfile.mkdtemp()

        try:
            expected_data = self.expected_data()

            job = self.run_risk(self.cfg, self.hazard_id())

            actual_data = self.actual_data(job)

            for i, actual in enumerate(actual_data):
                self.assertTrue(numpy.allclose(actual, expected_data[i],
                    rtol=0.01, atol=0.0))

            expected_outputs = self.expected_outputs()

            for i, output in enumerate(models.Output.objects.filter(
                    oq_job=job).order_by('id')):
                [exported_file] = risk_export.export(output.id, result_dir)
                self.assert_xml_equal(
                    StringIO.StringIO(expected_outputs[i]), exported_file)
        finally:
            shutil.rmtree(result_dir)
<|endoftext|>


<commit_before>import os, glob
import nipype.pipeline.engine as pe
from nipype.interfaces.utility import IdentityInterface, Function
from nipype.interfaces.io import DataFinder, DataSink
from qipipe.staging.group_dicom import group_dicom_files
from qipipe.interfaces import FixDicom
from .pipeline_helper import pvs
from .pipeline_helper import compress as cmpfunc

__all__ = ['run', 'stage']

def run(collection, dest, *patient_dirs):
    """
    Stages the given AIRC patient directories for import into CTP.
    The destination directory is populated with two subdirectories as follows:
        - airc: the patient/visit/series hierarchy linked to the AIRC source
        - ctp: the CTP import staging area

    The C{airc} area contains links to the AIRC study DICOM files.
    The staging area contains compressed DICOM files suitable for import into
    CTP. The DICOM headers are modified to correct the C{Patient ID} and
    C{Body Part Examined} tags.

    @param collection: the CTP image collection (C{Breast} or C{Sarcoma})
    @param dest: the destination directory
    @param patient_dirs: the AIRC source patient directories to stage
    """
    wf.inputs.infosource.collection = collection
    wf.inputs.infosource.ctp = os.path.join(dest, 'ctp')
    airc = os.path.join(dest, 'airc')
    # group_dicom_files is not included as a workflow node because nipype
    # structural constraints are too unwieldly, specifically:
    # 1) The group output might be empty, and nipype unconditionally executes
    #    all nodes, regardless of whether there are inputs.
    # 2) The group input is a patient directory and the output is the list of
    #    new series, which is input to the successor nodes. nipype cannot handle
    #    this structural mismatch without resorting to obscure kludges.
    dirs = group_dicom_files(*patient_dirs, dest=airc, include='*concat*/*')
    if dirs:
        wf.get_node('infosource').iterables = ('series_dir', dirs)
        wf.run()

def _run(collection, dest, patient_dir):
    from qipipe.pipelines.stage import run

    run(collection, dest, patient_dir)

stage = Function(input_names=['collection', 'dest', 'patient_dir'], output_names=[], function=_run)
"""The staging pipeline Function."""

# The staging pipeline prepares the AIRC image files for submission to CTP.
wf = pe.Workflow(name='stage')

# The stage workflow facade node.
infosource = pe.Node(interface=IdentityInterface(fields=['collection', 'ctp', 'series_dir']), name='infosource')

# Fix the AIRC DICOM tags. The fix input is a series directory. The fix output is
# the modified DICOM files.
fix = pe.Node(interface=FixDicom(dest='data'), name='fix')

# The DICOM file compressor. The compress input is a DICOM file. The compress output
# is the compressed file in the local run context data subdirectory. Since the fix
# output is a list of files, the fix output is multiplexed into a compress MapNode.
# The MapNode output is a list of files.
compress = pe.MapNode(cmpfunc, name='compress', iterfield=['in_file'])
compress.inputs.dest = 'data'

# The patient/visit/series substitution.
# The inputs are the patient, visit and series singleton lists matched on a series directory.
# The output is a substitutions assignment.
def pvs_substitution(patient, visit, series):
    return dict(_patient=patient, _visit=visit, _series=series).items()
pvsfunc = Function(input_names=['patient', 'visit', 'series'], output_names=['substitutions'],
    function=pvs_substitution)
pvs_substitution = pe.Node(pvsfunc, name='pvs_substitution')

# The result copier. The input field specifies the patient/visit/series hierarchy.
# Unsetting the parameterization flag prevents the DataSink from injecting an extraneous directory,
# e.g. _compress0, into the target location.
ASSEMBLY_FLD = '_patient._visit._series.@file'
assemble = pe.Node(interface=DataSink(infields=[ASSEMBLY_FLD], parameterization=False), name='assemble')

# Build the pipeline.
wf.connect([
    (infosource, pvs, [('series_dir', 'root_paths')]),
    (infosource, fix, [('collection', 'collection'), ('series_dir', 'source')]),
    (infosource, assemble, [('ctp', 'base_directory')]),
    (fix, compress, [('out_files', 'in_file')]),
    (pvs, pvs_substitution, [('patient', 'patient'), ('visit', 'visit'), ('series', 'series')]),
    (pvs_substitution, assemble, [('substitutions', 'substitutions')]),
    (compress, assemble, [('out_file', ASSEMBLY_FLD)])])
<commit_msg>Use absolute destination directory.<commit_after>import os, glob
import nipype.pipeline.engine as pe
from nipype.interfaces.utility import IdentityInterface, Function
from nipype.interfaces.io import DataFinder, DataSink
from qipipe.staging.group_dicom import group_dicom_files
from qipipe.interfaces import FixDicom
from .pipeline_helper import pvs
from .pipeline_helper import compress as cmpfunc

__all__ = ['run', 'stage']

def run(collection, dest, *patient_dirs):
    """
    Stages the given AIRC patient directories for import into CTP.
    The destination directory is populated with two subdirectories as follows:
        - airc: the patient/visit/series hierarchy linked to the AIRC source
        - ctp: the CTP import staging area

    The C{airc} area contains links to the AIRC study DICOM files.
    The staging area contains compressed DICOM files suitable for import into
    CTP. The DICOM headers are modified to correct the C{Patient ID} and
    C{Body Part Examined} tags.

    @param collection: the CTP image collection (C{Breast} or C{Sarcoma})
    @param dest: the destination directory
    @param patient_dirs: the AIRC source patient directories to stage
    """
    wf.inputs.infosource.collection = collection
    dest = os.path.abspath(dest)
    wf.inputs.infosource.ctp = os.path.join(dest, 'ctp')
    airc = os.path.join(dest, 'airc')
    # group_dicom_files is not included as a workflow node because nipype
    # structural constraints are too unwieldly, specifically:
    # 1) The group output might be empty, and nipype unconditionally executes
    #    all nodes, regardless of whether there are inputs.
    # 2) The group input is a patient directory and the output is the list of
    #    new series, which is input to the successor nodes. nipype cannot handle
    #    this structural mismatch without resorting to obscure kludges.
    dirs = group_dicom_files(*patient_dirs, dest=airc, include='*concat*/*')
    if dirs:
        wf.get_node('infosource').iterables = ('series_dir', dirs)
        wf.run()

def _run(collection, dest, patient_dir):
    from qipipe.pipelines.stage import run

    run(collection, dest, patient_dir)

stage = Function(input_names=['collection', 'dest', 'patient_dir'], output_names=[], function=_run)
"""The staging pipeline Function."""

# The staging pipeline prepares the AIRC image files for submission to CTP.
wf = pe.Workflow(name='stage')

# The stage workflow facade node.
infosource = pe.Node(interface=IdentityInterface(fields=['collection', 'ctp', 'series_dir']), name='infosource')

# Fix the AIRC DICOM tags. The fix input is a series directory. The fix output is
# the modified DICOM files.
fix = pe.Node(interface=FixDicom(dest='data'), name='fix')

# The DICOM file compressor. The compress input is a DICOM file. The compress output
# is the compressed file in the local run context data subdirectory. Since the fix
# output is a list of files, the fix output is multiplexed into a compress MapNode.
# The MapNode output is a list of files.
compress = pe.MapNode(cmpfunc, name='compress', iterfield=['in_file'])
compress.inputs.dest = 'data'

# The patient/visit/series substitution.
# The inputs are the patient, visit and series singleton lists matched on a series directory.
# The output is a substitutions assignment.
def pvs_substitution(patient, visit, series):
    return dict(_patient=patient, _visit=visit, _series=series).items()
pvsfunc = Function(input_names=['patient', 'visit', 'series'], output_names=['substitutions'],
    function=pvs_substitution)
pvs_substitution = pe.Node(pvsfunc, name='pvs_substitution')

# The result copier. The input field specifies the patient/visit/series hierarchy.
# Unsetting the parameterization flag prevents the DataSink from injecting an extraneous directory,
# e.g. _compress0, into the target location.
ASSEMBLY_FLD = '_patient._visit._series.@file'
assemble = pe.Node(interface=DataSink(infields=[ASSEMBLY_FLD], parameterization=False), name='assemble')

# Build the pipeline.
wf.connect([
    (infosource, pvs, [('series_dir', 'root_paths')]),
    (infosource, fix, [('collection', 'collection'), ('series_dir', 'source')]),
    (infosource, assemble, [('ctp', 'base_directory')]),
    (fix, compress, [('out_files', 'in_file')]),
    (pvs, pvs_substitution, [('patient', 'patient'), ('visit', 'visit'), ('series', 'series')]),
    (pvs_substitution, assemble, [('substitutions', 'substitutions')]),
    (compress, assemble, [('out_file', ASSEMBLY_FLD)])])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Main files, contains the version, the url to the documention.
"""

__version__ = "1.5"
__author__ = "Xavier Dupré"
__github__ = "https://github.com/sdpython/tkinterquickhelper"
__url__ = "http://www.xavierdupre.fr/app/tkinterquickhelper/helpsphinx/index.html"
__license__ = "MIT License"
__blog__ = """
<?xml version="1.0" encoding="UTF-8"?>
<opml version="1.0">
    <head>
        <title>blog</title>
    </head>
    <body>
        <outline text="tkinterquickhelper"
            title="tkinterquickhelper"
            type="rss"
            xmlUrl="http://www.xavierdupre.fr/app/tkinterquickhelper/helpsphinx/_downloads/rss.xml"
            htmlUrl="http://www.xavierdupre.fr/app/tkinterquickhelper/helpsphinx/blog/main_0000.html" />
    </body>
</opml>
"""

def check():
    """
    Checks the library is working.
    It raises an exception if it does not.

    @return         boolean
    """
    from .funcwin import check_icon
    from pyquickhelper.loghelper import check_log
    check_icon()
    check_log()
    return True

def _setup_hook(add_print=False, unit_test=False):
    """
    if this function is added to the module,
    the help automation and unit tests call it first before
    anything goes on as an initialization step.
    It should be run in a separate process.

    @param      add_print       print *Success: _setup_hook*
    @param      unit_test       used only for unit testing purpose
    """
    # we can check many things, needed module
    # any others things before unit tests are started
    if add_print:
        print("Success: _setup_hook")
<commit_msg>documentation<commit_after># -*- coding: utf-8 -*-
"""
@file
@brief Module *tkinterquickhelper*.
Helpers for :epkg:`*py:tkinter`.
"""

__version__ = "1.5"
__author__ = "Xavier Dupré"
__github__ = "https://github.com/sdpython/tkinterquickhelper"
__url__ = "http://www.xavierdupre.fr/app/tkinterquickhelper/helpsphinx/index.html"
__license__ = "MIT License"
__blog__ = """
<?xml version="1.0" encoding="UTF-8"?>
<opml version="1.0">
    <head>
        <title>blog</title>
    </head>
    <body>
        <outline text="tkinterquickhelper"
            title="tkinterquickhelper"
            type="rss"
            xmlUrl="http://www.xavierdupre.fr/app/tkinterquickhelper/helpsphinx/_downloads/rss.xml"
            htmlUrl="http://www.xavierdupre.fr/app/tkinterquickhelper/helpsphinx/blog/main_0000.html" />
    </body>
</opml>
"""

def check():
    """
    Checks the library is working.
    It raises an exception if it does not.

    @return         boolean
    """
    from .funcwin import check_icon
    from pyquickhelper.loghelper import check_log
    check_icon()
    check_log()
    return True

def _setup_hook(add_print=False, unit_test=False):
    """
    if this function is added to the module,
    the help automation and unit tests call it first before
    anything goes on as an initialization step.
    It should be run in a separate process.

    @param      add_print       print *Success: _setup_hook*
    @param      unit_test       used only for unit testing purpose
    """
    # we can check many things, needed module
    # any others things before unit tests are started
    if add_print:
        print("Success: _setup_hook")
<|endoftext|>


<commit_before># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class RReadbitmap(RPackage):
    """Identifies and reads Windows BMP, JPEG, PNG, and TIFF format bitmap
    images. Identification defaults to the use of the magic number embedded in
    the file rather than the file extension. Reading of JPEG and PNG image
    depends on libjpg and libpng libraries. See file INSTALL for details if
    necessary."""

    homepage = "https://github.com/jefferis/readbitmap"
    url      = "https://cloud.r-project.org/src/contrib/readbitmap_0.1.5.tar.gz"
    list_url = "https://cloud.r-project.org/src/contrib/Archive/readbitmap"

    version('0.1.5', sha256='737d7d585eb33de2c200da64d16781e3c9522400fe2af352e1460c6a402a0291')

    depends_on('r-bmp', type=('build', 'run'))
    depends_on('r-jpeg', type=('build', 'run'))
    depends_on('r-png', type=('build', 'run'))
    depends_on('r-tiff', type=('build', 'run'))

    depends_on('libjpeg')
    depends_on('libpng')
<commit_msg>package r-readbitmap Change jpeg dependency (#15004)<commit_after># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class RReadbitmap(RPackage):
    """Identifies and reads Windows BMP, JPEG, PNG, and TIFF format bitmap
    images. Identification defaults to the use of the magic number embedded in
    the file rather than the file extension. Reading of JPEG and PNG image
    depends on libjpg and libpng libraries. See file INSTALL for details if
    necessary."""

    homepage = "https://github.com/jefferis/readbitmap"
    url      = "https://cloud.r-project.org/src/contrib/readbitmap_0.1.5.tar.gz"
    list_url = "https://cloud.r-project.org/src/contrib/Archive/readbitmap"

    version('0.1.5', sha256='737d7d585eb33de2c200da64d16781e3c9522400fe2af352e1460c6a402a0291')

    depends_on('r-bmp', type=('build', 'run'))
    depends_on('r-jpeg', type=('build', 'run'))
    depends_on('r-png', type=('build', 'run'))
    depends_on('r-tiff', type=('build', 'run'))

    depends_on('jpeg')
    depends_on('libpng')
<|endoftext|>


<commit_before>from django import template

from wagtail.wagtailimages.models import Filter, SourceImageIOError

register = template.Library()

# Local cache of filters, avoid hitting the DB
filters = {}

@register.tag(name="image")
def image(parser, token):
    bits = token.split_contents()[1:]
    image_var = bits[0]
    filter_spec = bits[1]
    bits = bits[2:]

    if len(bits) == 2 and bits[0] == 'as':
        # token is of the form {% image self.photo max-320x200 as img %}
        return ImageNode(image_var, filter_spec, output_var_name=bits[1])
    else:
        # token is of the form {% image self.photo max-320x200 %} - all additional tokens
        # should be kwargs, which become attributes
        attrs = {}
        for bit in bits:
            try:
                name, value = bit.split('=')
            except ValueError:
                raise template.TemplateSyntaxError("'image' tag should be of the form {% image self.photo max-320x200 [ custom-attr=\"value\" ... ] %} or {% image self.photo max-320x200 as img %}")
            attrs[name] = parser.compile_filter(value) # setup to resolve context variables as value

        return ImageNode(image_var, filter_spec, attrs=attrs)

class ImageNode(template.Node):
    def __init__(self, image_var_name, filter_spec, output_var_name=None, attrs={}):
        self.image_var = template.Variable(image_var_name)
        self.output_var_name = output_var_name
        self.attrs = attrs

        if filter_spec not in filters:
            filters[filter_spec], _ = Filter.objects.get_or_create(spec=filter_spec)
        self.filter = filters[filter_spec]

    def render(self, context):
        try:
            image = self.image_var.resolve(context)
        except template.VariableDoesNotExist:
            return ''

        if not image:
            return ''

        try:
            rendition = image.get_rendition(self.filter)
        except SourceImageIOError:
            # It's fairly routine for people to pull down remote databases to their
            # local dev versions without retrieving the corresponding image files.
            # In such a case, we would get a SourceImageIOError at the point where we try to
            # create the resized version of a non-existent image. Since this is a
            # bit catastrophic for a missing image, we'll substitute a dummy
            # Rendition object so that we just output a broken link instead.
            Rendition = image.renditions.model  # pick up any custom Image / Rendition classes that may be in use
            rendition = Rendition(image=image, width=0, height=0)
            rendition.file.name = 'not-found'

        if self.output_var_name:
            # return the rendition object in the given variable
            context[self.output_var_name] = rendition
            return ''
        else:
            # render the rendition's image tag now
            resolved_attrs = {}
            for key in self.attrs:
                resolved_attrs[key] = self.attrs[key].resolve(context)
            return rendition.img_tag(resolved_attrs)
<commit_msg>images tags lazy filter loading<commit_after>from django import template

from wagtail.wagtailimages.models import Filter, SourceImageIOError

register = template.Library()

# Local cache of filters, avoid hitting the DB
filters = {}

@register.tag(name="image")
def image(parser, token):
    bits = token.split_contents()[1:]
    image_var = bits[0]
    filter_spec = bits[1]
    bits = bits[2:]

    if len(bits) == 2 and bits[0] == 'as':
        # token is of the form {% image self.photo max-320x200 as img %}
        return ImageNode(image_var, filter_spec, output_var_name=bits[1])
    else:
        # token is of the form {% image self.photo max-320x200 %} - all additional tokens
        # should be kwargs, which become attributes
        attrs = {}
        for bit in bits:
            try:
                name, value = bit.split('=')
            except ValueError:
                raise template.TemplateSyntaxError("'image' tag should be of the form {% image self.photo max-320x200 [ custom-attr=\"value\" ... ] %} or {% image self.photo max-320x200 as img %}")
            attrs[name] = parser.compile_filter(value) # setup to resolve context variables as value

        return ImageNode(image_var, filter_spec, attrs=attrs)

class ImageNode(template.Node):
    def __init__(self, image_var_name, filter_spec, output_var_name=None, attrs={}):
        self.image_var = template.Variable(image_var_name)
        self.output_var_name = output_var_name
        self.attrs = attrs

        self._filter_spec = filter_spec
        self._filter = None

    @property
    def filter(self):
        if self._filter is None:
            if self._filter_spec not in filters:
                filters[self._filter_spec], _ = Filter.objects.get_or_create(spec=self._filter_spec)
            self._filter = filters[self._filter_spec]

        return self._filter

    def render(self, context):
        try:
            image = self.image_var.resolve(context)
        except template.VariableDoesNotExist:
            return ''

        if not image:
            return ''

        try:
            rendition = image.get_rendition(self.filter)
        except SourceImageIOError:
            # It's fairly routine for people to pull down remote databases to their
            # local dev versions without retrieving the corresponding image files.
            # In such a case, we would get a SourceImageIOError at the point where we try to
            # create the resized version of a non-existent image. Since this is a
            # bit catastrophic for a missing image, we'll substitute a dummy
            # Rendition object so that we just output a broken link instead.
            Rendition = image.renditions.model  # pick up any custom Image / Rendition classes that may be in use
            rendition = Rendition(image=image, width=0, height=0)
            rendition.file.name = 'not-found'

        if self.output_var_name:
            # return the rendition object in the given variable
            context[self.output_var_name] = rendition
            return ''
        else:
            # render the rendition's image tag now
            resolved_attrs = {}
            for key in self.attrs:
                resolved_attrs[key] = self.attrs[key].resolve(context)
            return rendition.img_tag(resolved_attrs)
<|endoftext|>


<commit_before># Copyright (C) 2010-2011 Mathijs de Bruin <mathijs@mathijsfietst.nl>
#
# This file is part of django-webshop.
#
# django-webshop is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.

from webshop.extensions.discounts.models import *

from webshop.extensions.discounts.advanced.models.discount_models import *
from webshop.extensions.discounts.advanced.models.order_models import *
<commit_msg>Buggy bug<commit_after># Copyright (C) 2010-2011 Mathijs de Bruin <mathijs@mathijsfietst.nl>
#
# This file is part of django-webshop.
#
# django-webshop is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.

from webshop.extensions.discounts.advanced.models.discount_models import *
from webshop.extensions.discounts.advanced.models.order_models import *
<|endoftext|>


<commit_before>from django.test import TestCase
from urllib.parse import quote
from gwells.documents import MinioClient

class MockObject():
    def __init__(self, bucket_name, object_name):
        self.bucket_name = bucket_name
        self.object_name = object_name

class DocumentsTestCase(TestCase):

    def test_document_url_with_space(self):
        """ test creating a URL from an object with a space in the object key"""
        minio_client = MinioClient(disable_private=True)

        test_document = MockObject("test_bucket", "test key")
        test_url = minio_client.create_url(test_document, "example.com", test_document.bucket_name)

        self.assertEqual(test_url, quote("https://example.com/test_bucket/test key"))

    def test_document_url_with_plus(self):
        """ test creating a URL from and object key containing a plus sign """
        minio_client = MinioClient(disable_private=True)

        # use a key that contains a plus.
        # if this was a real plus in the filename it should be %2B in the listing.
        # spaces get encoded into + (so in this test case, this object_name originally had a space).
        test_document = MockObject("test_bucket", "test+key")

        test_url = minio_client.create_url(test_document, "example.com", test_document.bucket_name)

        self.assertEqual(test_url, quote("https://example.com/test_bucket/test key"))
<commit_msg>add another test case<commit_after>from django.test import TestCase
from urllib.parse import quote
from gwells.documents import MinioClient

class MockObject():
    def __init__(self, bucket_name, object_name):
        self.bucket_name = bucket_name
        self.object_name = object_name

class DocumentsTestCase(TestCase):

    def test_document_url_with_space(self):
        """ test creating a URL from an object with a space in the object key"""
        minio_client = MinioClient(disable_private=True)

        test_document = MockObject("test_bucket", "test key")
        test_url = minio_client.create_url(test_document, "example.com", test_document.bucket_name)

        self.assertEqual(test_url, quote("https://example.com/test_bucket/test key"))

    def test_document_url_with_plus(self):
        """ test creating a URL from an object key containing a plus sign """
        minio_client = MinioClient(disable_private=True)

        # use a key that contains a plus.
        # if this was a real plus in the filename it should be %2B in the listing.
        # spaces get encoded into + (so in this test case, this object_name originally had a space).
        test_document = MockObject("test_bucket", "test+key")

        test_url = minio_client.create_url(test_document, "example.com", test_document.bucket_name)

        self.assertEqual(test_url, quote("https://example.com/test_bucket/test key"))

    def test_document_url_with_encoded_plus(self):
        """ test creating a URL from and object key containing an encoded plus sign
            e.g., the original filename contained a plus.
        """
        minio_client = MinioClient(disable_private=True)

        # use a key that contains an encoded plus (e.g. ).
        test_document = MockObject("test_bucket", "test%2Bkey")

        test_url = minio_client.create_url(test_document, "example.com", test_document.bucket_name)

        self.assertEqual(test_url, quote("https://example.com/test_bucket/test+key"))
<|endoftext|>