<commit_before>import os
import tempfile
import subprocess
import shutil
import threading

from datetime import datetime

from .config import _cfg, _cfgi
from .database import r, _k
from .files import processing_needed, extension

converters = {
    'mp4': lambda path, outputpath: TimeLimitedCommand(["ffmpeg", "-i", path, "-pix_fmt", "yuv420p", "-vf", "scale=trunc(in_w/2)*2:trunc(in_h/2)*2", "%s.mp4" % outputpath]),
    'ogv': lambda path, outputpath: TimeLimitedCommand(["ffmpeg", "-i", path, "-q", "5", "-pix_fmt", "yuv420p", "%s.ogv" % outputpath]),
}

processors = {
    'jpg': lambda path: TimeLimitedCommand(["jhead", "-purejpg", path, path])
}

class TimeLimitedCommand(object):

    def __init__(self, *args):
        self.args = args
        self.process = None

    def _target(self):
        with open(os.devnull, "w") as devnull:
            self.process = subprocess.Popen(
                *self.args, stdout=devnull, stderr=devnull)
            self.process.communicate()

    def run(self, timeout=_cfgi("max_processing_time")):
        exited = False

        thread = threading.Thread(target=self._target)
        thread.start()
        thread.join(timeout)

        if thread.is_alive():
            print "Terminating process"
            self.process.terminate()
            thread.join()
            exited = True

        return self.process.returncode, exited

def process_gif(filename):
    f = r.get(_k("%s.file" % filename))
    ext = extension(f)
    path = os.path.join(_cfg("storage_folder"), f)

    statuscode = 0
    exited = False
    start = datetime.now()

    # Check if we know how to treat this file
    if ext not in processing_needed:
        r.delete(_k("%s.lock" % filename))
        return

    config = processing_needed[ext]
    # Do processing
    if ext in processors:
        code, exit = processors[ext](path).run(timeout=config['time'])

    # Do conversions
    outputpath = os.path.join(_cfg("storage_folder"), filename)
    for conversion in config['formats']:
        code, exit = converters[conversion](path, outputpath).run(timeout=config['time'])
        statuscode += code
        exited |= exit

    # Remove "processing lock"
    r.delete(_k("%s.lock" % filename))
    failed = False
    if statuscode != 0:
        r.set(_k("%s.error") % filename, "status")
        failed = True
    if exited:
        r.set(_k("%s.error") % filename, "timeout")
        failed = True

    # Remove artifacts if the conversion fails
    if failed:
        for artifact_ext in config['formats']:
            path = outputpath + "." + artifact_ext
            if os.path.exists(path):
                os.unlink(path)

    end = datetime.now()
    print "Processed", filename, end - start
<commit_msg>Make processing count towards failure<commit_after>import os
import tempfile
import subprocess
import shutil
import threading

from datetime import datetime

from .config import _cfg, _cfgi
from .database import r, _k
from .files import processing_needed, extension

converters = {
    'mp4': lambda path, outputpath: TimeLimitedCommand(["ffmpeg", "-i", path, "-pix_fmt", "yuv420p", "-vf", "scale=trunc(in_w/2)*2:trunc(in_h/2)*2", "%s.mp4" % outputpath]),
    'ogv': lambda path, outputpath: TimeLimitedCommand(["ffmpeg", "-i", path, "-q", "5", "-pix_fmt", "yuv420p", "%s.ogv" % outputpath]),
}

processors = {
    'jpg': lambda path: TimeLimitedCommand(["jhead", "-purejpg", path, path])
}

class TimeLimitedCommand(object):

    def __init__(self, *args):
        self.args = args
        self.process = None

    def _target(self):
        with open(os.devnull, "w") as devnull:
            self.process = subprocess.Popen(
                *self.args, stdout=devnull, stderr=devnull)
            self.process.communicate()

    def run(self, timeout=_cfgi("max_processing_time")):
        exited = False

        thread = threading.Thread(target=self._target)
        thread.start()
        thread.join(timeout)

        if thread.is_alive():
            print "Terminating process"
            self.process.terminate()
            thread.join()
            exited = True

        return self.process.returncode, exited

def process_gif(filename):
    f = r.get(_k("%s.file" % filename))
    ext = extension(f)
    path = os.path.join(_cfg("storage_folder"), f)

    statuscode = 0
    exited = False
    start = datetime.now()

    # Check if we know how to treat this file
    if ext not in processing_needed:
        r.delete(_k("%s.lock" % filename))
        return

    config = processing_needed[ext]
    # Do processing
    if ext in processors:
        code, exit = processors[ext](path).run(timeout=config['time'])
    statuscode += code
    exited |= exit

    # Do conversions
    outputpath = os.path.join(_cfg("storage_folder"), filename)
    for conversion in config['formats']:
        code, exit = converters[conversion](path, outputpath).run(timeout=config['time'])
        statuscode += code
        exited |= exit

    # Remove "processing lock"
    r.delete(_k("%s.lock" % filename))
    failed = False
    if statuscode != 0:
        r.set(_k("%s.error") % filename, "status")
        failed = True
    if exited:
        r.set(_k("%s.error") % filename, "timeout")
        failed = True

    # Remove artifacts if the conversion fails
    if failed:
        for artifact_ext in config['formats']:
            path = outputpath + "." + artifact_ext
            if os.path.exists(path):
                os.unlink(path)

    end = datetime.now()
    print "Processed", filename, end - start
<|endoftext|>


<commit_before># coding: utf-8
"""Find the path to xgboost dynamic library files."""

import os
import platform
import sys

class XGBoostLibraryNotFound(Exception):
    """Error throwed by when xgboost is not found"""
    pass

def find_lib_path():
    """Load find the path to xgboost dynamic library files.

    Returns
    -------
    lib_path: list(string)
       List of all found library path to xgboost
    """
    curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
    # make pythonpack hack: copy this directory one level upper for setup.py
    dll_path = [curr_path, os.path.join(curr_path, '../../lib/'),
                os.path.join(curr_path, './lib/'),
                os.path.join(sys.prefix, 'xgboost')]
    if os.name == 'nt':
        if platform.architecture()[0] == '64bit':
            dll_path.append(os.path.join(curr_path, '../../windows/x64/Release/'))
            # hack for pip installation when copy all parent source directory here
            dll_path.append(os.path.join(curr_path, './windows/x64/Release/'))
        else:
            dll_path.append(os.path.join(curr_path, '../../windows/Release/'))
            # hack for pip installation when copy all parent source directory here
            dll_path.append(os.path.join(curr_path, './windows/Release/'))
    if os.name == 'nt':
        dll_path = [os.path.join(p, 'libxgboost.dll') for p in dll_path]
    else:
        dll_path = [os.path.join(p, 'libxgboost.so') for p in dll_path]
    lib_path = [p for p in dll_path if os.path.exists(p) and os.path.isfile(p)]

    # From github issues, most of installation errors come from machines w/o compilers
    if len(lib_path) == 0 and not os.environ.get('XGBOOST_BUILD_DOC', False):
        raise XGBoostLibraryNotFound(
            'Cannot find XGBoost Libarary in the candicate path, ' +
            'did you install compilers and run build.sh in root path?\n'
            'List of candidates:\n' + ('\n'.join(dll_path)))
    return lib_path
<commit_msg>Fixes typo "candicate" (#1508)<commit_after># coding: utf-8
"""Find the path to xgboost dynamic library files."""

import os
import platform
import sys

class XGBoostLibraryNotFound(Exception):
    """Error throwed by when xgboost is not found"""
    pass

def find_lib_path():
    """Load find the path to xgboost dynamic library files.

    Returns
    -------
    lib_path: list(string)
       List of all found library path to xgboost
    """
    curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
    # make pythonpack hack: copy this directory one level upper for setup.py
    dll_path = [curr_path, os.path.join(curr_path, '../../lib/'),
                os.path.join(curr_path, './lib/'),
                os.path.join(sys.prefix, 'xgboost')]
    if os.name == 'nt':
        if platform.architecture()[0] == '64bit':
            dll_path.append(os.path.join(curr_path, '../../windows/x64/Release/'))
            # hack for pip installation when copy all parent source directory here
            dll_path.append(os.path.join(curr_path, './windows/x64/Release/'))
        else:
            dll_path.append(os.path.join(curr_path, '../../windows/Release/'))
            # hack for pip installation when copy all parent source directory here
            dll_path.append(os.path.join(curr_path, './windows/Release/'))
    if os.name == 'nt':
        dll_path = [os.path.join(p, 'libxgboost.dll') for p in dll_path]
    else:
        dll_path = [os.path.join(p, 'libxgboost.so') for p in dll_path]
    lib_path = [p for p in dll_path if os.path.exists(p) and os.path.isfile(p)]

    # From github issues, most of installation errors come from machines w/o compilers
    if len(lib_path) == 0 and not os.environ.get('XGBOOST_BUILD_DOC', False):
        raise XGBoostLibraryNotFound(
            'Cannot find XGBoost Libarary in the candidate path, ' +
            'did you install compilers and run build.sh in root path?\n'
            'List of candidates:\n' + ('\n'.join(dll_path)))
    return lib_path
<|endoftext|>


<commit_before>#!/usr/bin/env python

#
# Copyright 2016 The BigDL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import os
import re
from os.path import isfile, join

scala_nn_root = "./spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/"
python_nn_root = "./pyspark/bigdl/nn/"

scala_to_python = {"Graph": "Model"}

def extract_scala_class(class_path):
    exclude_key_words = set(["*", "abstract", "Const", "Fill", "Shape",
                             "SplitAndSelect", "StrideSlice", "Scheduler"])
    include_key_words = set(["Module", "Criterion", "Container", "Cell", "TensorNumeric"])  # noqa
    content = "\n".join([line for line in open(class_path).readlines() if all([key not in line for key in exclude_key_words])])  # noqa
    match = re.search(r"class ([\w]+)[^{]+", content)
    if match and any([key in match.group() for key in include_key_words]):
        return match.group(1)
    else:
        return None

def get_scala_classes(scala_nn_root):
    raw_result = [extract_scala_class(join(scala_nn_root, name)) for name in os.listdir(scala_nn_root) if isfile(join(scala_nn_root, name))]  # noqa
    return set(
        class_name for class_name in raw_result if class_name is not None)

def get_python_classes(nn_root):
    exclude_classes = set(["Criterion"])
    raw_classes = []
    python_nn_layers = [join(nn_root, name) for name in os.listdir(nn_root) if isfile(join(nn_root, name)) and name.endswith('py') and "__" not in name]  # noqa
    for p in python_nn_layers:
        with open(p) as f:
            raw_classes.extend([line for line in f.readlines() if line.startswith("class")])  # noqa
    classes = [name.split()[1].split("(")[0]for name in raw_classes]
    return set([name for name in classes if name not in exclude_classes])

scala_layers = get_scala_classes(scala_nn_root)
python_layers = get_python_classes(python_nn_root)

print("scala_layers: {0}, {1}".format(len(scala_layers), scala_layers))
print("\n")
print("python_layers: {0}, {1}".format(len(python_layers), python_layers))

print("In scala, not in python: "),
diff_count = 0
for name in scala_layers:
    if name not in python_layers and scala_to_python[name] not in python_layers:
        print(name),
        diff_count += 1

if diff_count > 0:
    raise Exception("There are difference between python and scala")
<commit_msg>Fix 1923: Use static graph execution when there's no control operation in graph container (#2008)<commit_after>#!/usr/bin/env python

#
# Copyright 2016 The BigDL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import os
import re
from os.path import isfile, join

scala_nn_root = "./spark/dl/src/main/scala/com/intel/analytics/bigdl/nn/"
python_nn_root = "./pyspark/bigdl/nn/"

scala_to_python = {"Graph": "Model"}

def extract_scala_class(class_path):
    exclude_key_words = set(["*", "abstract", "Const", "Fill", "Shape",
                             "SplitAndSelect", "StrideSlice", "Scheduler", "StaticGraph", "DynamicGraph"])   # noqa
    include_key_words = set(["Module", "Criterion", "Container", "Cell", "TensorNumeric"])  # noqa
    content = "\n".join([line for line in open(class_path).readlines() if all([key not in line for key in exclude_key_words])])  # noqa
    match = re.search(r"class ([\w]+)[^{]+", content)
    if match and any([key in match.group() for key in include_key_words]):
        return match.group(1)
    else:
        return None

def get_scala_classes(scala_nn_root):
    raw_result = [extract_scala_class(join(scala_nn_root, name)) for name in os.listdir(scala_nn_root) if isfile(join(scala_nn_root, name))]  # noqa
    return set(
        class_name for class_name in raw_result if class_name is not None)

def get_python_classes(nn_root):
    exclude_classes = set(["Criterion"])
    raw_classes = []
    python_nn_layers = [join(nn_root, name) for name in os.listdir(nn_root) if isfile(join(nn_root, name)) and name.endswith('py') and "__" not in name]  # noqa
    for p in python_nn_layers:
        with open(p) as f:
            raw_classes.extend([line for line in f.readlines() if line.startswith("class")])  # noqa
    classes = [name.split()[1].split("(")[0]for name in raw_classes]
    return set([name for name in classes if name not in exclude_classes])

scala_layers = get_scala_classes(scala_nn_root)
python_layers = get_python_classes(python_nn_root)

print("scala_layers: {0}, {1}".format(len(scala_layers), scala_layers))
print("\n")
print("python_layers: {0}, {1}".format(len(python_layers), python_layers))

print("In scala, not in python: "),
diff_count = 0
for name in scala_layers:
    if name not in python_layers and scala_to_python[name] not in python_layers:
        print(name),
        diff_count += 1

if diff_count > 0:
    raise Exception("There are difference between python and scala")
<|endoftext|>


<commit_before>__linelist__ = {
                'OII_1' : 3727.09,
                'OII_2' : 3729.88,
                'H_theta' : 3798.98,
                'H_eta' : 3836.47,
                'H_xi' : 3890.16,
                'H_epsilon' : 3971.20,
                'SII_1' : 4072.30,
                'H_delta' : 4102.89,
                'H_gamma' : 4341.68,
                'OIII_1' : 4364.44,
                'H_beta' : 4862.68,
                'OIII_2' : 4932.60,
                'OIII_3' : 4960.30,
                'OIII_4' : 5008.24,
                'HeI' : 5877.65,
                'OI_1' : 6302.05,
                'OI_2' : 6365.54,
                'NI' : 6529.03,
                'NII' : 6549.86,
                'H_alpha' : 6564.61,
                'NII' : 6585.27,
                'SII_2' : 6718.29,
                'SII_3' : 6732.67
                }
<commit_msg>added mask creation function<commit_after>import numpy as np

__linelist__ = {
                'OII_1' : 3727.09,
                'OII_2' : 3729.88,
                'H_theta' : 3798.98,
                'H_eta' : 3836.47,
                'H_xi' : 3890.16,
                'H_epsilon' : 3971.20,
                'SII_1' : 4072.30,
                'H_delta' : 4102.89,
                'H_gamma' : 4341.68,
                'OIII_1' : 4364.44,
                'H_beta' : 4862.68,
                'OIII_2' : 4932.60,
                'OIII_3' : 4960.30,
                'OIII_4' : 5008.24,
                'HeI' : 5877.65,
                'OI_1' : 6302.05,
                'OI_2' : 6365.54,
                'NI' : 6529.03,
                'NII' : 6549.86,
                'H_alpha' : 6564.61,
                'NII' : 6585.27,
                'SII_2' : 6718.29,
                'SII_3' : 6732.67
                }

def create_mask(lines, wave):
    wave = np.array(wave)
    mask = np.ones(wave.shape[0])
    for line in lines.values():
        loc = np.abs(wave-line).argmin()
        mask[loc-5:loc+5] = 0
    return mask
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
# Imports =====================================================================
from os.path import join

from bottle import post
from bottle_rest import form_to_params

from ..connectors import aleph
from ..settings import API_PATH

# Variables ===================================================================
API_PATH = join(API_PATH, "aleph")

# Functions & classes =========================================================
@post(join(API_PATH, "records_by_issn"))
@form_to_params
def records_by_issn(issn):
    return [
        result.get_mapping()
        for result in aleph.by_issn(issn)
    ]

@post(join(API_PATH, "authors_by_name"))
@form_to_params
def authors_by_name(name):
    return [
        author._asdict()
        for author in aleph.Author.search_by_name(name)
    ]
<commit_msg>#25: Added docstrings to aleph_api.py.<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
"""
REST API provider for the Aleph system.
"""
#
# Imports =====================================================================
from os.path import join

from bottle import post
from bottle_rest import form_to_params

from ..connectors import aleph
from ..settings import API_PATH

# Variables ===================================================================
API_PATH = join(API_PATH, "aleph")

# Functions & classes =========================================================
@post(join(API_PATH, "records_by_issn"))
@form_to_params
def records_by_issn(issn):
    """
    Search NTK Aleph ISSN base for given `issn`.

    Returns:
        list: List of ISSN records.
    """
    return [
        result.get_mapping()
        for result in aleph.by_issn(issn)
    ]

@post(join(API_PATH, "authors_by_name"))
@form_to_params
def authors_by_name(name):
    """
    Search NK Aleph authority base for authors matching the `name`.

    Returns:
        list: List of matching authors.
    """
    return [
        author._asdict()
        for author in aleph.Author.search_by_name(name)
    ]
<|endoftext|>


<commit_before>try:
    from configparser import ConfigParser
except ImportError:
    from ConfigParser import ConfigParser

try:
    from urllib.parse import urlsplit
except ImportError:
    from urlparse import urlsplit

try:
    FileExistsException = FileExistsError
except NameError:
    FileExistsException = OSError

__all__ = [ConfigParser, urlsplit, FileExistsError]
<commit_msg>name error<commit_after>try:
    from configparser import ConfigParser
except ImportError:
    from ConfigParser import ConfigParser

try:
    from urllib.parse import urlsplit
except ImportError:
    from urlparse import urlsplit

try:
    FileExistsException = FileExistsError
except NameError:
    FileExistsException = OSError

__all__ = [ConfigParser, urlsplit, FileExistsException]
<|endoftext|>


<commit_before>import argparse
import logging.config
import sys
import textwrap
import traceback

import cli_ui

from gitlabform import EXIT_INVALID_INPUT, EXIT_PROCESSING_ERROR
from gitlabform.configuration.core import ConfigFileNotFoundException
from gitlabform.filter import NonEmptyConfigsProvider
from gitlabform.gitlab import GitLab
from gitlabform.gitlab.core import TestRequestFailedException
from gitlabform.input import GroupsAndProjectsProvider
from gitlabform.output import EffectiveConfiguration
from gitlabform.processors.group import GroupProcessors
from gitlabform.processors.project import ProjectProcessors
from gitlabform.ui import (
    info_group_count,
    info_project_count,
    show_version,
    show_summary,
    show_header,
)

class Formatter(
    argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter
):
    pass

class GitLabForm(object):
    def __init__(self, project_or_group=None, config_string=None):

        if project_or_group and config_string:
            # this mode is basically only for testing

            self.project_or_group = project_or_group
            self.config_string = config_string
            self.verbose = True
            self.debug = True
            self.strict = True
            self.start_from = 1
            self.start_from_group = 1
            self.noop = False
            self.output_file = None
            self.skip_version_check = True
            self.include_archived_projects = True  # for unarchive tests
            self.just_show_version = False
            self.terminate_after_error = True

            self.configure_output(tests=True)
        else:
            # normal mode

            (
                self.project_or_group,
                self.config,
                self.verbose,
                self.debug,
                self.strict,
                self.start_from,
                self.start_from_group,
                self.noop,
                self.output_file,
                self.skip_version_check,
                self.include_archived_projects,
                self.just_show_version,
                self.terminate_after_error,
            ) = self.parse_args()

            self.configure_output()

            show_version(self.skip_version_check)
            if self.just_show_version:
                sys.exit(0)

            if not self.project_or_group:
                cli_ui.fatal(
                    "project_or_group parameter is required.",
                    exit_code=EXIT_INVALID_INPUT,
                )

        self.gitlab, self.configuration = self.initialize_configuration_and_gitlab()

        self.group_processors = GroupProcessors(self.gitlab)
        self.project_processors = ProjectProcessors(
            self.gitlab, self.configuration, self.strict
        )
        self.groups_and_projects_provider = GroupsAndProjectsProvider(
            self.gitlab, self.configuration, self.include_archived_projects
        )

        self.non_empty_configs_provider = NonEmptyConfigsProvider(
            self.configuration, self.group_processors, self.project_processors
        )

    def parse_args(self):

        parser = argparse.ArgumentParser(
            description=textwrap.dedent(
                f"""
            Specialized "configuration as a code" tool for GitLab projects, groups and more
            using hierarchical configuration written in YAML.

            Exits with code {EXIT_INVALID_INPUT} on invalid input errors (f.e. config file not found),
            and with code {EXIT_PROCESSING_ERROR} if the are processing errors (f.e. if GitLab returns 400).
            """
            ),
            formatter_class=Formatter,
        )

        parser.add_argument(
            "project_or_group",
            nargs="?",
            help='Project name in "group/project" format '
            "OR a single group name "
            'OR "ALL_DEFINED" to run for all groups and projects defined the config '
            'OR "ALL" to run for all projects that you have access to',
        )

        parser.add_argument(
            "-V",
            "--version",
            dest="just_show_version",
            action="store_true",
            help="show version and exit",
        )

        parser.add_argument(
            "-c", "--config", default="config.yml", help="config file path and filename"
        )

        verbosity_args = parser.add_mutually_exclusive_group()

        verbosity_args.add_argument(
            "-v", "--verbose", action="store_true", help="verbose mode"
        )

        verbosity_args.add_argument(
            "-d", "--debug", action="store_true", help="debug mode (most verbose)"
        )

        parser.add_argument(
            "-s",
            "--strict",
            action="store_true",
            help="stop on missing branches and tags",
        )

        parser.add_argument(
            "-n",
            "--noop",
            dest="noop",
            action="store_true",
            help="run in no-op (dry run) mode",
        )

        parser.add_argument(
            "-o",
            "--output-file",
            dest="output_file",
            default=None,
            help="name/path of a file to write the effective configs to",
        )

        parser.add_argument(
            "-k",
            "--skip-version-check",
            dest="skip_version_check",
            action="store_true",
            help="Skips checking if the latest version is used",
        )

        parser.add_argument(
            "-a",
            "--include-archived-projects",
            dest="include_archived_projects",
            action="store_true",
            help="Includes processing projects that are archived",
        )

        parser.add_argument(
            "-t",
            "--terminate",
            dest="terminate_after_error",
            action="store_true",
            help=f"exit with {EXIT_PROCESSING_ERROR} after the first group/project processing error."
            f" (default: process all the requested groups/projects and skip the failing ones."
            f" At the end, if there were any groups/projects, exit with {EXIT_PROCESSING_ERROR}.)",
        )

        parser.add_argument(
            "-sf",
            "--start-from",
            dest="start_from",
            default=1,
            type=int,
            help="start processing projects from the given one"
            ' (as numbered by "x/y Processing group/project" messages)',
        )

        parser.add_argument(
            "-sfg",
            "--start-from-group",
            dest="start_from_group",
            default=1,
            type=int,
            help="start processing groups from the given one "
            '(as numbered by "x/y Processing group/project" messages)',
        )

        args = parser.parse_args()

        return (
            args.project_or_group,
            args.config,
            args.verbose,
            args.debug,
            args.strict,
            args.start_from,
            args.start_from_group,
            args.noop,
            args.output_file,
            args.skip_version_check,
            args.include_archived_projects,
            args.just_show_version,
            args.terminate_after_error,
        )

    def configure_output(self, tests=False):

        # normal mode - print cli_ui.info+
        # verbose mode - print cli_ui.*
        # debug / tests mode - like above + logging.debug

        logging.basicConfig()

        if not self.verbose and not self.debug:  # normal
            cli_ui.setup()
            level = (
                logging.FATAL
            )  # de facto disabled as we don't use logging different than debug in this project
        else:
            if (
                self.debug or tests
            ):  # debug (BUT verbose may also be set, that's why we check this first)
                cli_ui.setup(verbose=True)
                level = logging.DEBUG
            elif self.verbose:  # verbose
                cli_ui.setup(verbose=True)
                level = (
                    logging.FATAL
                )  # de facto disabled as we don't use logging different than debug in this project

        logging.getLogger().setLevel(level)
        fmt = logging.Formatter("%(message)s")
        logging.getLogger().handlers[0].setFormatter(fmt)

    def initialize_configuration_and_gitlab(self):

        try:
            if hasattr(self, "config_string"):
                gitlab = GitLab(config_string=self.config_string)
            else:
                gitlab = GitLab(config_path=self.config)
            configuration = gitlab.get_configuration()
            return gitlab, configuration
        except ConfigFileNotFoundException as e:
            cli_ui.fatal(
                f"Config file not found at: {e}",
                exit_code=EXIT_INVALID_INPUT,
            )
        except TestRequestFailedException as e:
            cli_ui.fatal(
                f"GitLab test request failed. Exception: '{e}'",
                EXIT_PROCESSING_ERROR,
            )

    def run(self):

        projects_with_non_empty_configs, groups_with_non_empty_configs = show_header(
            self.project_or_group,
            self.groups_and_projects_provider,
            self.non_empty_configs_provider,
        )

        group_number = 0
        successful_groups = 0
        failed_groups = {}

        effective_configuration = EffectiveConfiguration(self.output_file)

        for group in groups_with_non_empty_configs:

            group_number += 1

            if group_number < self.start_from_group:
                info_group_count(
                    "@",
                    group_number,
                    len(groups_with_non_empty_configs),
                    cli_ui.yellow,
                    f"Skipping group {group} as requested to start from {self.start_from_group}...",
                    cli_ui.reset,
                )
                continue

            configuration = self.configuration.get_effective_config_for_group(group)

            effective_configuration.add_placeholder(group)

            info_group_count(
                "@",
                group_number,
                len(groups_with_non_empty_configs),
                f"Processing group: {group}",
            )

            try:
                self.group_processors.process_group(
                    group,
                    configuration,
                    dry_run=self.noop,
                    effective_configuration=effective_configuration,
                )

                successful_groups += 1

            except Exception as e:

                failed_groups[group_number] = group

                trace = traceback.format_exc()
                message = f"Error occurred while processing group {group}, exception:\n\n{e}\n\n{trace}"

                if self.terminate_after_error:
                    effective_configuration.write_to_file()
                    cli_ui.fatal(
                        message,
                        exit_code=EXIT_PROCESSING_ERROR,
                    )
                else:
                    cli_ui.warning(message)
            finally:
                logging.debug(
                    f"@ ({group_number}/{len(groups_with_non_empty_configs)}) FINISHED Processing group: {group}"
                )

        project_number = 0
        successful_projects = 0
        failed_projects = {}

        for project_and_group in projects_with_non_empty_configs:

            project_number += 1

            if project_number < self.start_from:
                info_project_count(
                    "*",
                    project_number,
                    len(projects_with_non_empty_configs),
                    cli_ui.yellow,
                    f"Skipping project {project_and_group} as requested to start from {self.start_from}...",
                    cli_ui.reset,
                )
                continue

            configuration = self.configuration.get_effective_config_for_project(
                project_and_group
            )

            effective_configuration.add_placeholder(project_and_group)

            info_project_count(
                "*",
                project_number,
                len(projects_with_non_empty_configs),
                f"Processing project: {project_and_group}",
            )

            try:
                self.project_processors.process_project(
                    project_and_group,
                    configuration,
                    dry_run=self.noop,
                    effective_configuration=effective_configuration,
                )

                successful_projects += 1

            except Exception as e:

                failed_projects[project_number] = project_and_group

                trace = traceback.format_exc()
                message = f"Error occurred while processing project {project_and_group}, exception:\n\n{e}\n\n{trace}"

                if self.terminate_after_error:
                    effective_configuration.write_to_file()
                    cli_ui.fatal(
                        message,
                        exit_code=EXIT_PROCESSING_ERROR,
                    )
                else:
                    cli_ui.warning(message)

            finally:

                logging.debug(
                    f"* ({project_number}/{len(projects_with_non_empty_configs)}) FINISHED Processing project: {project_and_group}",
                )

        effective_configuration.write_to_file()

        show_summary(
            groups_with_non_empty_configs,
            projects_with_non_empty_configs,
            successful_groups,
            successful_projects,
            failed_groups,
            failed_projects,
        )
<commit_msg>Warn about debug and output file use<commit_after>import argparse
import logging.config
import sys
import textwrap
import traceback

import cli_ui

from gitlabform import EXIT_INVALID_INPUT, EXIT_PROCESSING_ERROR
from gitlabform.configuration.core import ConfigFileNotFoundException
from gitlabform.filter import NonEmptyConfigsProvider
from gitlabform.gitlab import GitLab
from gitlabform.gitlab.core import TestRequestFailedException
from gitlabform.input import GroupsAndProjectsProvider
from gitlabform.output import EffectiveConfiguration
from gitlabform.processors.group import GroupProcessors
from gitlabform.processors.project import ProjectProcessors
from gitlabform.ui import (
    info_group_count,
    info_project_count,
    show_version,
    show_summary,
    show_header,
)

class Formatter(
    argparse.ArgumentDefaultsHelpFormatter,
    argparse.RawTextHelpFormatter,
):
    pass

class GitLabForm(object):
    def __init__(self, project_or_group=None, config_string=None):

        if project_or_group and config_string:
            # this mode is basically only for testing

            self.project_or_group = project_or_group
            self.config_string = config_string
            self.verbose = True
            self.debug = True
            self.strict = True
            self.start_from = 1
            self.start_from_group = 1
            self.noop = False
            self.output_file = None
            self.skip_version_check = True
            self.include_archived_projects = True  # for unarchive tests
            self.just_show_version = False
            self.terminate_after_error = True

            self.configure_output(tests=True)
        else:
            # normal mode

            (
                self.project_or_group,
                self.config,
                self.verbose,
                self.debug,
                self.strict,
                self.start_from,
                self.start_from_group,
                self.noop,
                self.output_file,
                self.skip_version_check,
                self.include_archived_projects,
                self.just_show_version,
                self.terminate_after_error,
            ) = self.parse_args()

            self.configure_output()

            show_version(self.skip_version_check)
            if self.just_show_version:
                sys.exit(0)

            if not self.project_or_group:
                cli_ui.fatal(
                    "project_or_group parameter is required.",
                    exit_code=EXIT_INVALID_INPUT,
                )

        self.gitlab, self.configuration = self.initialize_configuration_and_gitlab()

        self.group_processors = GroupProcessors(self.gitlab)
        self.project_processors = ProjectProcessors(
            self.gitlab, self.configuration, self.strict
        )
        self.groups_and_projects_provider = GroupsAndProjectsProvider(
            self.gitlab, self.configuration, self.include_archived_projects
        )

        self.non_empty_configs_provider = NonEmptyConfigsProvider(
            self.configuration, self.group_processors, self.project_processors
        )

    def parse_args(self):

        parser = argparse.ArgumentParser(
            description=textwrap.dedent(
                f"""
            Specialized "configuration as a code" tool for GitLab projects, groups and more
            using hierarchical configuration written in YAML.

            Exits with code {EXIT_INVALID_INPUT} on invalid input errors (f.e. config file not found),
            and with code {EXIT_PROCESSING_ERROR} if the are processing errors (f.e. if GitLab returns 400).
            """
            ),
            formatter_class=Formatter,
        )

        parser.add_argument(
            "project_or_group",
            nargs="?",
            help='Project name in "group/project" format \n'
            "OR a single group name \n"
            'OR "ALL_DEFINED" to run for all groups and projects defined the config \n'
            'OR "ALL" to run for all projects that you have access to',
        )

        parser.add_argument(
            "-V",
            "--version",
            dest="just_show_version",
            action="store_true",
            help="show version and exit",
        )

        parser.add_argument(
            "-k",
            "--skip-version-check",
            dest="skip_version_check",
            action="store_true",
            help="Skips checking if the latest version is used",
        )

        parser.add_argument(
            "-c", "--config", default="config.yml", help="config file path and filename"
        )

        verbosity_args = parser.add_mutually_exclusive_group()

        verbosity_args.add_argument(
            "-v", "--verbose", action="store_true", help="verbose output"
        )

        verbosity_args.add_argument(
            "-d",
            "--debug",
            action="store_true",
            help="debug output (!!! WARNING !!!: sensitive data and secrets may"
            " be printed in this mode - all the data sent to GitLab API"
            " will be printed in plain-text.)",
        )

        parser.add_argument(
            "-s",
            "--strict",
            action="store_true",
            help="stop on missing branches and tags",
        )

        parser.add_argument(
            "-n",
            "--noop",
            dest="noop",
            action="store_true",
            help="run in no-op (dry run) mode",
        )

        parser.add_argument(
            "-o",
            "--output-file",
            dest="output_file",
            default=None,
            help="name/path of a file to write the effective configs to"
            " (!!! WARNING !!!: if your config contains sensitive data or secrets, then this file will also"
            " contain them.)",
        )

        parser.add_argument(
            "-a",
            "--include-archived-projects",
            dest="include_archived_projects",
            action="store_true",
            help="Includes processing projects that are archived",
        )

        parser.add_argument(
            "-t",
            "--terminate",
            dest="terminate_after_error",
            action="store_true",
            help=f"exit with {EXIT_PROCESSING_ERROR} after the first group/project processing error."
            f" (default: process all the requested groups/projects and skip the failing ones."
            f" At the end, if there were any groups/projects, exit with {EXIT_PROCESSING_ERROR}.)",
        )

        parser.add_argument(
            "-sf",
            "--start-from",
            dest="start_from",
            default=1,
            type=int,
            help="start processing projects from the given one"
            ' (as numbered by "x/y Processing group/project" messages)',
        )

        parser.add_argument(
            "-sfg",
            "--start-from-group",
            dest="start_from_group",
            default=1,
            type=int,
            help="start processing groups from the given one "
            '(as numbered by "x/y Processing group/project" messages)',
        )

        args = parser.parse_args()

        return (
            args.project_or_group,
            args.config,
            args.verbose,
            args.debug,
            args.strict,
            args.start_from,
            args.start_from_group,
            args.noop,
            args.output_file,
            args.skip_version_check,
            args.include_archived_projects,
            args.just_show_version,
            args.terminate_after_error,
        )

    def configure_output(self, tests=False):

        # normal mode - print cli_ui.info+
        # verbose mode - print cli_ui.*
        # debug / tests mode - like above + logging.debug

        logging.basicConfig()

        if not self.verbose and not self.debug:  # normal
            cli_ui.setup()
            level = (
                logging.FATAL
            )  # de facto disabled as we don't use logging different than debug in this project
        else:
            if (
                self.debug or tests
            ):  # debug (BUT verbose may also be set, that's why we check this first)
                cli_ui.setup(verbose=True)
                level = logging.DEBUG
            elif self.verbose:  # verbose
                cli_ui.setup(verbose=True)
                level = (
                    logging.FATAL
                )  # de facto disabled as we don't use logging different than debug in this project

        logging.getLogger().setLevel(level)
        fmt = logging.Formatter("%(message)s")
        logging.getLogger().handlers[0].setFormatter(fmt)

    def initialize_configuration_and_gitlab(self):

        try:
            if hasattr(self, "config_string"):
                gitlab = GitLab(config_string=self.config_string)
            else:
                gitlab = GitLab(config_path=self.config)
            configuration = gitlab.get_configuration()
            return gitlab, configuration
        except ConfigFileNotFoundException as e:
            cli_ui.fatal(
                f"Config file not found at: {e}",
                exit_code=EXIT_INVALID_INPUT,
            )
        except TestRequestFailedException as e:
            cli_ui.fatal(
                f"GitLab test request failed. Exception: '{e}'",
                EXIT_PROCESSING_ERROR,
            )

    def run(self):

        projects_with_non_empty_configs, groups_with_non_empty_configs = show_header(
            self.project_or_group,
            self.groups_and_projects_provider,
            self.non_empty_configs_provider,
        )

        group_number = 0
        successful_groups = 0
        failed_groups = {}

        effective_configuration = EffectiveConfiguration(self.output_file)

        for group in groups_with_non_empty_configs:

            group_number += 1

            if group_number < self.start_from_group:
                info_group_count(
                    "@",
                    group_number,
                    len(groups_with_non_empty_configs),
                    cli_ui.yellow,
                    f"Skipping group {group} as requested to start from {self.start_from_group}...",
                    cli_ui.reset,
                )
                continue

            configuration = self.configuration.get_effective_config_for_group(group)

            effective_configuration.add_placeholder(group)

            info_group_count(
                "@",
                group_number,
                len(groups_with_non_empty_configs),
                f"Processing group: {group}",
            )

            try:
                self.group_processors.process_group(
                    group,
                    configuration,
                    dry_run=self.noop,
                    effective_configuration=effective_configuration,
                )

                successful_groups += 1

            except Exception as e:

                failed_groups[group_number] = group

                trace = traceback.format_exc()
                message = f"Error occurred while processing group {group}, exception:\n\n{e}\n\n{trace}"

                if self.terminate_after_error:
                    effective_configuration.write_to_file()
                    cli_ui.fatal(
                        message,
                        exit_code=EXIT_PROCESSING_ERROR,
                    )
                else:
                    cli_ui.warning(message)
            finally:
                logging.debug(
                    f"@ ({group_number}/{len(groups_with_non_empty_configs)}) FINISHED Processing group: {group}"
                )

        project_number = 0
        successful_projects = 0
        failed_projects = {}

        for project_and_group in projects_with_non_empty_configs:

            project_number += 1

            if project_number < self.start_from:
                info_project_count(
                    "*",
                    project_number,
                    len(projects_with_non_empty_configs),
                    cli_ui.yellow,
                    f"Skipping project {project_and_group} as requested to start from {self.start_from}...",
                    cli_ui.reset,
                )
                continue

            configuration = self.configuration.get_effective_config_for_project(
                project_and_group
            )

            effective_configuration.add_placeholder(project_and_group)

            info_project_count(
                "*",
                project_number,
                len(projects_with_non_empty_configs),
                f"Processing project: {project_and_group}",
            )

            try:
                self.project_processors.process_project(
                    project_and_group,
                    configuration,
                    dry_run=self.noop,
                    effective_configuration=effective_configuration,
                )

                successful_projects += 1

            except Exception as e:

                failed_projects[project_number] = project_and_group

                trace = traceback.format_exc()
                message = f"Error occurred while processing project {project_and_group}, exception:\n\n{e}\n\n{trace}"

                if self.terminate_after_error:
                    effective_configuration.write_to_file()
                    cli_ui.fatal(
                        message,
                        exit_code=EXIT_PROCESSING_ERROR,
                    )
                else:
                    cli_ui.warning(message)

            finally:

                logging.debug(
                    f"* ({project_number}/{len(projects_with_non_empty_configs)}) FINISHED Processing project: {project_and_group}",
                )

        effective_configuration.write_to_file()

        show_summary(
            groups_with_non_empty_configs,
            projects_with_non_empty_configs,
            successful_groups,
            successful_projects,
            failed_groups,
            failed_projects,
        )
<|endoftext|>


<commit_before>import datetime
from decimal import Decimal

BIRTHDAY = datetime.date(2012, 6, 5)
def age():
    age = datetime.date.today() - BIRTHDAY
    return "%d days" % age.days

db = None # This global is wired in wireup. It's an instance of
          # gittip.postgres.PostgresManager.
AMOUNTS= [Decimal(a) for a in ('0.00', '0.08', '0.16', '0.32', '0.64', '1.28')]

__version__ = "1.2.7"

def get_tip(tipper, tippee):
    """Given two user ids, return a Decimal.
    """
    TIP = """\

        SELECT amount
          FROM tips
         WHERE tipper=%s
           AND tippee=%s
      ORDER BY mtime DESC
         LIMIT 1

    """
    rec = db.fetchone(TIP, (tipper, tippee))
    if rec is None:
        tip = Decimal(0.00)
    else:
        tip = rec['amount']
    return tip

def get_tipjar(participant_id, pronoun="their"):
    """Given a participant id, return a unicode.
    """

    # Compute the amount.
    # ===================

    TIPJAR = """\

        SELECT sum(amount) AS tipjar
          FROM ( SELECT DISTINCT ON (tipper)
                        amount
                      , tipper
                   FROM tips
                   JOIN participants p ON p.id = tipper
                  WHERE tippee=%s
                    AND last_bill_result = ''
               ORDER BY tipper
                      , mtime DESC
                ) AS foo

    """
    rec = db.fetchone(TIPJAR, (participant_id,))
    if rec is None:
        amount = None
    else:
        amount = rec['tipjar']  # might be None

    if amount is None:
        amount = Decimal(0.00)

    # Compute a unicode describing the amount.
    # ========================================
    # This is down here because we use it in multiple places in the app,
    # basically in the claimed participant page and the non-claimed GitHub page
    # (hopefully soon to be Facebook and Twitter as well).

    if pronoun not in ('your', 'their'):
        raise Exception("Unknown, um, pronoun: %s" % pronoun)

    if amount == 0:
        if pronoun == "your":
            tipjar = u"have no backed tips."
        elif pronoun == "their":
            tipjar = u"has no backed tips."
    else:
        if pronoun == "your":
            tipjar = u"have $%s in backed tips."
        elif pronoun == "their":
            tipjar = u"has $%s in backed tips."
        tipjar %= amount

    return tipjar

def get_tips_and_total(tipper, for_payday=False):
    """Given a participant id, return a list and a Decimal.

    This function is used to populate a participant's page for their own
    viewing pleasure, and also by the payday function.

    """
    if for_payday:

        # For payday we want the oldest relationship to be paid first.
        order_by = "ctime ASC"

        # This is where it gets crash-proof.
        # ==================================
        # We need to account for the fact that we may have crashed during
        # Payday and we're re-running that function. We only want to select
        # tips that existed before Payday started, but haven't been processed
        # as part of this Payday yet.
        #
        # It's a bug if the paydays subselect returns > 1 rows.
        #
        # XXX If we crash during Payday and we rerun it after a timezone
        # change, will we get burned? How?

        ts_filter = """\

               AND mtime < %s
               AND ( SELECT id
                       FROM transfers
                      WHERE tipper=tips.tipper
                        AND tippee=tips.tippee
                        AND timestamp >= %s
                    ) IS NULL

        """
        args = (tipper, for_payday, for_payday)
    else:
        order_by = "amount DESC"
        ts_filter = ""
        args = (tipper,)

    TIPS = """\

        SELECT * FROM (
            SELECT DISTINCT ON (tippee)
                   amount
                 , tippee
                 , ctime
              FROM tips
             WHERE tipper = %%s
               %s
          ORDER BY tippee
                 , mtime DESC
        ) AS foo
        ORDER BY %s
               , tippee

    """ % (ts_filter, order_by)
    tips = list(db.fetchall(TIPS, args))
    total = sum([tip['amount'] for tip in tips])
    if not total:
        total = Decimal('0.00')
    return tips, total

# canonizer
# =========
# This is an Aspen hook to ensure that requests are served on a certain root
# URL, even if multiple domains point to the application.

class X: pass
canonical_scheme = None
canonical_host = None

def canonize(request):
    """Enforce a certain scheme and hostname. Store these on request as well.
    """
    scheme = request.headers.get('X-Forwarded-Proto', 'http') # per Heroku
    host = request.headers['Host']
    bad_scheme = scheme != canonical_scheme
    bad_host = bool(canonical_host) and (host != canonical_host)
                # '' and False => ''
    if bad_scheme or bad_host:
        url = '%s://%s/' % (canonical_scheme, canonical_host)
        request.redirect(url, permanent=True)
<commit_msg>Resetting version in gittip/__init__.py.<commit_after>import datetime
from decimal import Decimal

BIRTHDAY = datetime.date(2012, 6, 5)
def age():
    age = datetime.date.today() - BIRTHDAY
    return "%d days" % age.days

db = None # This global is wired in wireup. It's an instance of
          # gittip.postgres.PostgresManager.
AMOUNTS= [Decimal(a) for a in ('0.00', '0.08', '0.16', '0.32', '0.64', '1.28')]

__version__ = "~~VERSION~~"

def get_tip(tipper, tippee):
    """Given two user ids, return a Decimal.
    """
    TIP = """\

        SELECT amount
          FROM tips
         WHERE tipper=%s
           AND tippee=%s
      ORDER BY mtime DESC
         LIMIT 1

    """
    rec = db.fetchone(TIP, (tipper, tippee))
    if rec is None:
        tip = Decimal(0.00)
    else:
        tip = rec['amount']
    return tip

def get_tipjar(participant_id, pronoun="their"):
    """Given a participant id, return a unicode.
    """

    # Compute the amount.
    # ===================

    TIPJAR = """\

        SELECT sum(amount) AS tipjar
          FROM ( SELECT DISTINCT ON (tipper)
                        amount
                      , tipper
                   FROM tips
                   JOIN participants p ON p.id = tipper
                  WHERE tippee=%s
                    AND last_bill_result = ''
               ORDER BY tipper
                      , mtime DESC
                ) AS foo

    """
    rec = db.fetchone(TIPJAR, (participant_id,))
    if rec is None:
        amount = None
    else:
        amount = rec['tipjar']  # might be None

    if amount is None:
        amount = Decimal(0.00)

    # Compute a unicode describing the amount.
    # ========================================
    # This is down here because we use it in multiple places in the app,
    # basically in the claimed participant page and the non-claimed GitHub page
    # (hopefully soon to be Facebook and Twitter as well).

    if pronoun not in ('your', 'their'):
        raise Exception("Unknown, um, pronoun: %s" % pronoun)

    if amount == 0:
        if pronoun == "your":
            tipjar = u"have no backed tips."
        elif pronoun == "their":
            tipjar = u"has no backed tips."
    else:
        if pronoun == "your":
            tipjar = u"have $%s in backed tips."
        elif pronoun == "their":
            tipjar = u"has $%s in backed tips."
        tipjar %= amount

    return tipjar

def get_tips_and_total(tipper, for_payday=False):
    """Given a participant id, return a list and a Decimal.

    This function is used to populate a participant's page for their own
    viewing pleasure, and also by the payday function.

    """
    if for_payday:

        # For payday we want the oldest relationship to be paid first.
        order_by = "ctime ASC"

        # This is where it gets crash-proof.
        # ==================================
        # We need to account for the fact that we may have crashed during
        # Payday and we're re-running that function. We only want to select
        # tips that existed before Payday started, but haven't been processed
        # as part of this Payday yet.
        #
        # It's a bug if the paydays subselect returns > 1 rows.
        #
        # XXX If we crash during Payday and we rerun it after a timezone
        # change, will we get burned? How?

        ts_filter = """\

               AND mtime < %s
               AND ( SELECT id
                       FROM transfers
                      WHERE tipper=tips.tipper
                        AND tippee=tips.tippee
                        AND timestamp >= %s
                    ) IS NULL

        """
        args = (tipper, for_payday, for_payday)
    else:
        order_by = "amount DESC"
        ts_filter = ""
        args = (tipper,)

    TIPS = """\

        SELECT * FROM (
            SELECT DISTINCT ON (tippee)
                   amount
                 , tippee
                 , ctime
              FROM tips
             WHERE tipper = %%s
               %s
          ORDER BY tippee
                 , mtime DESC
        ) AS foo
        ORDER BY %s
               , tippee

    """ % (ts_filter, order_by)
    tips = list(db.fetchall(TIPS, args))
    total = sum([tip['amount'] for tip in tips])
    if not total:
        total = Decimal('0.00')
    return tips, total

# canonizer
# =========
# This is an Aspen hook to ensure that requests are served on a certain root
# URL, even if multiple domains point to the application.

class X: pass
canonical_scheme = None
canonical_host = None

def canonize(request):
    """Enforce a certain scheme and hostname. Store these on request as well.
    """
    scheme = request.headers.get('X-Forwarded-Proto', 'http') # per Heroku
    host = request.headers['Host']
    bad_scheme = scheme != canonical_scheme
    bad_host = bool(canonical_host) and (host != canonical_host)
                # '' and False => ''
    if bad_scheme or bad_host:
        url = '%s://%s/' % (canonical_scheme, canonical_host)
        request.redirect(url, permanent=True)
<|endoftext|>


<commit_before>import Adafruit_BBIO.ADC as ADC
ADC.setup()

from .mq5 import mq5
<commit_msg>cleaned ip __init__<commit_after>from .mq5 import mq5
<|endoftext|>


<commit_before>from aiohttp import web
import amen.audio
import pydub
import time
import asyncio
import logging
import subprocess
from . import database

# To determine the "effective length" of the last beat, we
# average the last N beats prior to it. Higher numbers give
# smoother results but may have issues with a close-out rall.
LAST_BEAT_AVG = 10

app = web.Application()

songs = []
position = 0
track_list = []

def route(url):
	def deco(f):
		app.router.add_get(url, f)
		return f
	return deco

# ------ Helper functions for infinitely_glitch() -------

# The rate-limiting sleep will wait until the clock catches up to this point.
# We start it "ten seconds ago" so we get a bit of buffer to start off.
rendered_until = time.time() - 10
ffmpeg = None # aio subprocess where we're compressing to MP3
async def _render_output_audio(seg, fn):
	logging.info("Sending %d bytes of data for %s secs of %s", len(seg.raw_data), seg.duration_seconds, fn)
	ffmpeg.stdin.write(seg.raw_data)
	await ffmpeg.stdin.drain()
	global rendered_until; rendered_until += seg.duration_seconds
	delay = rendered_until - time.time()
	if delay > 0:
		logging.debug("And sleeping for %ds until %s", delay, rendered_until)
		await asyncio.sleep(delay)

def _get_track():
	"""Get a track and load everything we need."""
	# TODO: Have proper async database calls (if we can do it without
	# massively breaking encapsulation); psycopg2 has an async mode, and
	# aiopg links that in with asyncio.
	nexttrack = database.get_track_to_play()
	dub2 = pydub.AudioSegment.from_mp3("audio/" + nexttrack.filename).set_channels(2)
	# NOTE: Calling amen with a filename invokes a second load from disk,
	# duplicating work done above. However, it will come straight from the
	# disk cache, so the only real duplication is the decode-from-MP3; and
	# timing tests show that this is a measurable but not overly costly
	# addition on top of the time to do the actual analysis. KISS.
	t2 = amen.audio.Audio("audio/" + nexttrack.filename)
	return nexttrack, t2, dub2

async def infinitely_glitch():
	try:
		nexttrack, t2, dub2 = _get_track()
		skip = 0.0
		while True:
			track = nexttrack; t1 = t2; dub1 = dub2
			nexttrack, t2, dub2 = _get_track()
			# Combine this into the next track.
			# 1) Analyze using amen
			#    t1 = amen.audio.Audio(track.filename)
			# 2) Locate the end of the effective last beat
			#    t1.timings['beats'][-10:-1][*].duration -> avg
			#    t1_end = t1.timings['beats'][-1].time + avg_duration
			# 3) Locate the first beat of the next track
			#    t2 = amen.audio.Audio(nexttrack.filename)
			#    t2_start = t2.timings['beats'][0].time
			# 4) Count back from the end of the last beat
			#    t1_end - t2_start
			# 5) Cross-fade from that point to t1_end to t2_start
			# Possibly do the cross-fade in two sections, as the times
			# won't be the same. They're the fade-in duration of t1 and
			# the fade-out duration of t2. Note that they depend on each
			# other, so they can't just be stored as-is (although the
			# beat positions and durations can).

			# Note on units:
			# The Timedelta that we find in timings['beats'] can provide us
			# with float total_seconds(), or with a .value in nanoseconds.
			# We later on will prefer milliseconds, though, so we rescale.
			# In this code, all variables store ms unless otherwise stated.
			t1b = t1.timings['beats']
			beat_ns = sum(b.duration.value for b in t1b[-1-LAST_BEAT_AVG : -1]) // LAST_BEAT_AVG
			t1_end = (t1b[-1].time.value + beat_ns) // 1000000
			t1_length = int(t1.duration * 1000)
			t2_start = t2.timings['beats'][1].time.value // 1000000
			# 1) Render t1 from skip up to (t1_end-t2_start) - the bulk of the track
			bulk = dub1[skip : t1_end - t2_start]
			track_list.append({
				"id": track.id,
				"start_time": rendered_until,
				"details": track.track_details,
			})
			await _render_output_audio(bulk, track.filename)
			# 2) Fade across t2_start ms - this will get us to the downbeat
			# 3) Fade across (t1_length-t1_end) ms - this nicely rounds out the last track
			# 4) Go get the next track, but skip the first (t2_start+t1_length-t1_end) ms
			skip = t2_start + t1_length - t1_end
			# Dumb fade mode. Doesn't actually fade, just overlays.
			fadeout1 = dub1[t1_end - t2_start : t1_end]
			fadein1 = dub2[:t2_start]
			fade1 = fadeout1.overlay(fadein1)
			fadeout2 = dub1[t1_end:]
			fadein2 = dub2[t2_start:skip]
			fade2 = fadeout2.overlay(fadein2)
			await _render_output_audio(fade1, "xfade 1")
			await _render_output_audio(fade2, "xfade 2")
	finally:
		ffmpeg.stdin.close()

# ------ Main renderer coroutine -------

async def ffmpeg():
	logging.debug("renderer started")
	global ffmpeg
	ffmpeg = await asyncio.create_subprocess_exec("ffmpeg", "-ac", "2", "-f", "s16le", "-i", "-", "-f", "mp3", "-",
		stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
	asyncio.ensure_future(infinitely_glitch())
	totdata = 0
	logging.debug("Waiting for data from ffmpeg...")
	chunk = b""
	while ffmpeg.returncode is None:
		data = await ffmpeg.stdout.read(4096)
		if not data: break
		totdata += len(data)
		chunk += data
		# logging.debug("Received %d bytes [%d]", totdata, len(data))
		if data.startswith(b"\xFF\xFB") and len(chunk) > 1024*1024:
			global position
			songs.append(chunk)
			logging.debug("Adding another song section [%d, %d bytes]", len(songs) + position, len(chunk))
			chunk = b""
			if len(songs) > 32:
				songs.pop(0)
				position += 1
	if ffmpeg.returncode is None:
		ffmpeg.terminate()

# ------ End of main renderer. Simpler stuff follows. :) -------

@route("/all.mp3")
async def moosic(req):
	logging.debug("/all.mp3 requested")
	resp = web.StreamResponse()
	resp.content_type = "audio/mpeg"
	await resp.prepare(req)
	pos = position
	while True:
		if pos - position >= len(songs):
			# Not enough content. This should only happen
			# on startup, so we just queue ourselves a bit.
			await asyncio.sleep(1)
			continue
		if pos < position:
			# The client is so far behind that we've dropped
			# the chunk that would have been next. There's
			# really not much we can do; disconnect.
			break
		logging.debug("songs %d, pos %d, position %d", len(songs), pos, position)
		resp.write(songs[pos - position])
		await resp.drain()
		pos += 1
	return resp

@route("/status.json")
async def info(req):
	logging.debug("/status.json requested")
	# TODO: Clean up the track list, ditching entries way in the past.
	# It doesn't need to be an ever-growing history.
	return web.json_response({"ts": time.time(), "tracks": track_list}, headers={"Access-Control-Allow-Origin": "*"})

def run(port=8889):
	asyncio.ensure_future(ffmpeg())
	web.run_app(app, port=port)

if __name__ == '__main__':
	run()
<commit_msg>renderer: Try/finally the termination signal for ffmpeg<commit_after>from aiohttp import web
import amen.audio
import pydub
import time
import asyncio
import logging
import subprocess
from . import database

# To determine the "effective length" of the last beat, we
# average the last N beats prior to it. Higher numbers give
# smoother results but may have issues with a close-out rall.
LAST_BEAT_AVG = 10

app = web.Application()

songs = []
position = 0
track_list = []

def route(url):
	def deco(f):
		app.router.add_get(url, f)
		return f
	return deco

# ------ Helper functions for infinitely_glitch() -------

# The rate-limiting sleep will wait until the clock catches up to this point.
# We start it "ten seconds ago" so we get a bit of buffer to start off.
rendered_until = time.time() - 10
ffmpeg = None # aio subprocess where we're compressing to MP3
async def _render_output_audio(seg, fn):
	logging.info("Sending %d bytes of data for %s secs of %s", len(seg.raw_data), seg.duration_seconds, fn)
	ffmpeg.stdin.write(seg.raw_data)
	await ffmpeg.stdin.drain()
	global rendered_until; rendered_until += seg.duration_seconds
	delay = rendered_until - time.time()
	if delay > 0:
		logging.debug("And sleeping for %ds until %s", delay, rendered_until)
		await asyncio.sleep(delay)

def _get_track():
	"""Get a track and load everything we need."""
	# TODO: Have proper async database calls (if we can do it without
	# massively breaking encapsulation); psycopg2 has an async mode, and
	# aiopg links that in with asyncio.
	nexttrack = database.get_track_to_play()
	dub2 = pydub.AudioSegment.from_mp3("audio/" + nexttrack.filename).set_channels(2)
	# NOTE: Calling amen with a filename invokes a second load from disk,
	# duplicating work done above. However, it will come straight from the
	# disk cache, so the only real duplication is the decode-from-MP3; and
	# timing tests show that this is a measurable but not overly costly
	# addition on top of the time to do the actual analysis. KISS.
	t2 = amen.audio.Audio("audio/" + nexttrack.filename)
	return nexttrack, t2, dub2

async def infinitely_glitch():
	try:
		nexttrack, t2, dub2 = _get_track()
		skip = 0.0
		while True:
			track = nexttrack; t1 = t2; dub1 = dub2
			nexttrack, t2, dub2 = _get_track()
			# Combine this into the next track.
			# 1) Analyze using amen
			#    t1 = amen.audio.Audio(track.filename)
			# 2) Locate the end of the effective last beat
			#    t1.timings['beats'][-10:-1][*].duration -> avg
			#    t1_end = t1.timings['beats'][-1].time + avg_duration
			# 3) Locate the first beat of the next track
			#    t2 = amen.audio.Audio(nexttrack.filename)
			#    t2_start = t2.timings['beats'][0].time
			# 4) Count back from the end of the last beat
			#    t1_end - t2_start
			# 5) Cross-fade from that point to t1_end to t2_start
			# Possibly do the cross-fade in two sections, as the times
			# won't be the same. They're the fade-in duration of t1 and
			# the fade-out duration of t2. Note that they depend on each
			# other, so they can't just be stored as-is (although the
			# beat positions and durations can).

			# Note on units:
			# The Timedelta that we find in timings['beats'] can provide us
			# with float total_seconds(), or with a .value in nanoseconds.
			# We later on will prefer milliseconds, though, so we rescale.
			# In this code, all variables store ms unless otherwise stated.
			t1b = t1.timings['beats']
			beat_ns = sum(b.duration.value for b in t1b[-1-LAST_BEAT_AVG : -1]) // LAST_BEAT_AVG
			t1_end = (t1b[-1].time.value + beat_ns) // 1000000
			t1_length = int(t1.duration * 1000)
			t2_start = t2.timings['beats'][1].time.value // 1000000
			# 1) Render t1 from skip up to (t1_end-t2_start) - the bulk of the track
			bulk = dub1[skip : t1_end - t2_start]
			track_list.append({
				"id": track.id,
				"start_time": rendered_until,
				"details": track.track_details,
			})
			await _render_output_audio(bulk, track.filename)
			# 2) Fade across t2_start ms - this will get us to the downbeat
			# 3) Fade across (t1_length-t1_end) ms - this nicely rounds out the last track
			# 4) Go get the next track, but skip the first (t2_start+t1_length-t1_end) ms
			skip = t2_start + t1_length - t1_end
			# Dumb fade mode. Doesn't actually fade, just overlays.
			fadeout1 = dub1[t1_end - t2_start : t1_end]
			fadein1 = dub2[:t2_start]
			fade1 = fadeout1.overlay(fadein1)
			fadeout2 = dub1[t1_end:]
			fadein2 = dub2[t2_start:skip]
			fade2 = fadeout2.overlay(fadein2)
			await _render_output_audio(fade1, "xfade 1")
			await _render_output_audio(fade2, "xfade 2")
	finally:
		ffmpeg.stdin.close()

# ------ Main renderer coroutine -------

async def ffmpeg():
	logging.debug("renderer started")
	global ffmpeg
	ffmpeg = await asyncio.create_subprocess_exec("ffmpeg", "-ac", "2", "-f", "s16le", "-i", "-", "-f", "mp3", "-",
		stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
	asyncio.ensure_future(infinitely_glitch())
	totdata = 0
	logging.debug("Waiting for data from ffmpeg...")
	chunk = b""
	try:
		while ffmpeg.returncode is None:
			data = await ffmpeg.stdout.read(4096)
			if not data: break
			totdata += len(data)
			chunk += data
			# logging.debug("Received %d bytes [%d]", totdata, len(data))
			if data.startswith(b"\xFF\xFB") and len(chunk) > 1024*1024:
				global position
				songs.append(chunk)
				logging.debug("Adding another song section [%d, %d bytes]", len(songs) + position, len(chunk))
				chunk = b""
				if len(songs) > 32:
					songs.pop(0)
					position += 1
	finally:
		if ffmpeg.returncode is None:
			ffmpeg.terminate()

# ------ End of main renderer. Simpler stuff follows. :) -------

@route("/all.mp3")
async def moosic(req):
	logging.debug("/all.mp3 requested")
	resp = web.StreamResponse()
	resp.content_type = "audio/mpeg"
	await resp.prepare(req)
	pos = position
	while True:
		if pos - position >= len(songs):
			# Not enough content. This should only happen
			# on startup, so we just queue ourselves a bit.
			await asyncio.sleep(1)
			continue
		if pos < position:
			# The client is so far behind that we've dropped
			# the chunk that would have been next. There's
			# really not much we can do; disconnect.
			break
		logging.debug("songs %d, pos %d, position %d", len(songs), pos, position)
		resp.write(songs[pos - position])
		await resp.drain()
		pos += 1
	return resp

@route("/status.json")
async def info(req):
	logging.debug("/status.json requested")
	# TODO: Clean up the track list, ditching entries way in the past.
	# It doesn't need to be an ever-growing history.
	return web.json_response({"ts": time.time(), "tracks": track_list}, headers={"Access-Control-Allow-Origin": "*"})

def run(port=8889):
	asyncio.ensure_future(ffmpeg())
	web.run_app(app, port=port)

if __name__ == '__main__':
	run()
<|endoftext|>


<commit_before>"""glymur - read, write, and interrogate JPEG 2000 files
"""

from .jp2k import Jp2k
from .jp2dump import jp2dump

from . import data
from . import test

def runtests():
    """Discover and run all tests for the glymur package.
    """
    import unittest
    suite = unittest.defaultTestLoader.discover(__path__[0])
    unittest.TextTestRunner(verbosity=2).run(suite)
<commit_msg>pep8 work<commit_after>"""glymur - read, write, and interrogate JPEG 2000 files
"""

from .jp2k import Jp2k
from .jp2dump import jp2dump

from . import data
from . import test

def runtests():
    """Discover and run all tests for the glymur package.
    """
    import unittest
    suite = unittest.defaultTestLoader.discover(__path__[0])
    unittest.TextTestRunner(verbosity=2).run(suite)
<|endoftext|>


<commit_before>from elasticsearch_dsl.connections import connections
from elasticsearch_dsl import Index
from qanta.guesser.abstract import AbstractGuesser
from qanta.guesser.experimental.elasticsearch_instance_of import ElasticSearchWikidataGuesser
from qanta.buzzer.rnn_0 import dense_vector0 as dense_vector

gspec = AbstractGuesser.list_enabled_guessers()[0]
guesser_dir = AbstractGuesser.output_path(gspec.guesser_module,
        gspec.guesser_class, '')
guesser = ElasticSearchWikidataGuesser.load(guesser_dir)

guesses = guesser.guess_single('Sviatoslav Teofilovich Richter was a Soviet pianist \
    known for the depth of his interpretations, virtuoso technique, and vast \
    repertoire.')

guesses = [[guesses]]
print(guesses)
vecs = dense_vector(guesses)
print(vecs)

<commit_msg>update<commit_after>from elasticsearch_dsl.connections import connections
from elasticsearch_dsl import Index
from qanta.guesser.abstract import AbstractGuesser
from qanta.guesser.experimental.elasticsearch_instance_of import ElasticSearchWikidataGuesser
from qanta.buzzer.rnn_0 import dense_vector0 as dense_vector

gspec = AbstractGuesser.list_enabled_guessers()[0]
guesser_dir = AbstractGuesser.output_path(gspec.guesser_module,
        gspec.guesser_class, '')
guesser = ElasticSearchWikidataGuesser.load(guesser_dir)

guesses = guesser.guess_single("wrote a novel about a boy who encounters the\
        Duke and the Dauphin while traveling down the Mississippi with the slave\
        Jim")

guesses = [[guesses]]
print(guesses)
vecs = dense_vector(guesses)
print(vecs)

<|endoftext|>


<commit_before># Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging

from oslo.config import cfg

from gnocchi.openstack.common import log

LOG = log.getLogger(__name__)

def prepare_service():
    cfg.CONF()
    log.setup('gnocchi')
    cfg.CONF.log_opt_values(LOG, logging.DEBUG)
<commit_msg>Set project name when parsing configuration<commit_after># Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging

from oslo.config import cfg

from gnocchi.openstack.common import log

LOG = log.getLogger(__name__)

def prepare_service():
    cfg.CONF(project='gnocchi')
    log.setup('gnocchi')
    cfg.CONF.log_opt_values(LOG, logging.DEBUG)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# This code is part of Qiskit.
#
# (C) Copyright IBM 2019.
#
# This code is licensed under the Apache License, Version 2.0. You may
# obtain a copy of this license in the LICENSE.txt file in the root directory
# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
#
# Any modifications or derivative works of this code must retain this
# copyright notice, and modified files need to carry a notice indicating
# that they have been altered from the originals.

"""
The most straightforward scheduling methods: scheduling **as early** or **as late** as possible.
"""

from collections import defaultdict, namedtuple
from typing import List

from qiskit.circuit.measure import Measure
from qiskit.circuit.quantumcircuit import QuantumCircuit
from qiskit.exceptions import QiskitError
from qiskit.extensions.standard.barrier import Barrier
from qiskit.pulse.exceptions import PulseError
from qiskit.pulse.schedule import Schedule
from qiskit.pulse.channels import AcquireChannel
from qiskit.scheduler.utils import measure

from qiskit.scheduler.config import ScheduleConfig

CircuitPulseDef = namedtuple('CircuitPulseDef', [
    'schedule',  # The schedule which implements the quantum circuit command
    'qubits'])   # The labels of the qubits involved in the command according to the circuit

def as_soon_as_possible(circuit: QuantumCircuit,
                        schedule_config: ScheduleConfig) -> Schedule:
    """
    Return the pulse Schedule which implements the input circuit using an "as soon as possible"
    (asap) scheduling policy.

    Circuit instructions are first each mapped to equivalent pulse
    Schedules according to the command definition given by the schedule_config. Then, this circuit
    instruction-equivalent Schedule is appended at the earliest time at which all qubits involved
    in the instruction are available.

    Args:
        circuit: The quantum circuit to translate.
        schedule_config: Backend specific parameters used for building the Schedule.

    Returns:
        A schedule corresponding to the input ``circuit`` with pulses occurring as early as
        possible.
    """
    sched = Schedule(name=circuit.name)

    qubit_time_available = defaultdict(int)

    def update_times(inst_qubits: List[int], time: int = 0) -> None:
        """Update the time tracker for all inst_qubits to the given time."""
        for q in inst_qubits:
            qubit_time_available[q] = time

    circ_pulse_defs = translate_gates_to_pulse_defs(circuit, schedule_config)
    for circ_pulse_def in circ_pulse_defs:
        time = max(qubit_time_available[q] for q in circ_pulse_def.qubits)
        if isinstance(circ_pulse_def.schedule, Barrier):
            update_times(circ_pulse_def.qubits, time)
        else:
            sched = sched.insert(time, circ_pulse_def.schedule)
            update_times(circ_pulse_def.qubits, time + circ_pulse_def.schedule.duration)
    return sched

def as_late_as_possible(circuit: QuantumCircuit,
                        schedule_config: ScheduleConfig) -> Schedule:
    """
    Return the pulse Schedule which implements the input circuit using an "as late as possible"
    (alap) scheduling policy.

    Circuit instructions are first each mapped to equivalent pulse
    Schedules according to the command definition given by the schedule_config. Then, this circuit
    instruction-equivalent Schedule is appended at the latest time that it can be without allowing
    unnecessary time between instructions or allowing instructions with common qubits to overlap.

    This method should improves the outcome fidelity over ASAP scheduling, because we may
    maximize the time that the qubit remains in the ground state.

    Args:
        circuit: The quantum circuit to translate.
        schedule_config: Backend specific parameters used for building the Schedule.

    Returns:
        A schedule corresponding to the input ``circuit`` with pulses occurring as late as
        possible.
    """
    sched = Schedule(name=circuit.name)
    # Align channel end times.
    circuit.barrier()
    # We schedule in reverse order to get ALAP behaviour. We need to know how far out from t=0 any
    # qubit will become occupied. We add positive shifts to these times as we go along.
    # The time is initialized to 0 because all qubits are involved in the final barrier.
    qubit_available_until = defaultdict(lambda: 0)

    def update_times(inst_qubits: List[int], shift: int = 0, inst_start_time: int = 0) -> None:
        """Update the time tracker for all inst_qubits to the given time."""
        for q in inst_qubits:
            qubit_available_until[q] = inst_start_time
        for q in qubit_available_until.keys():
            if q not in inst_qubits:
                # Uninvolved qubits might be free for the duration of the new instruction
                qubit_available_until[q] += shift

    circ_pulse_defs = translate_gates_to_pulse_defs(circuit, schedule_config)
    for circ_pulse_def in reversed(circ_pulse_defs):
        inst_sched = circ_pulse_def.schedule
        # The new instruction should end when one of its qubits becomes occupied
        inst_start_time = (min([qubit_available_until[q] for q in circ_pulse_def.qubits])
                           - getattr(inst_sched, 'duration', 0))  # Barrier has no duration
        # We have to translate qubit times forward when the inst_start_time is negative
        shift_amount = max(0, -inst_start_time)
        inst_start_time = max(inst_start_time, 0)
        if not isinstance(circ_pulse_def.schedule, Barrier):
            sched = inst_sched.shift(inst_start_time).insert(shift_amount, sched, name=sched.name)
        update_times(circ_pulse_def.qubits, shift_amount, inst_start_time)
    return sched

def translate_gates_to_pulse_defs(circuit: QuantumCircuit,
                                  schedule_config: ScheduleConfig) -> List[CircuitPulseDef]:
    """
    Return a list of Schedules and the qubits they operate on, for each element encountered in th
    input circuit.

    Without concern for the final schedule, extract and return a list of Schedules and the qubits
    they operate on, for each element encountered in the input circuit. Measures are grouped when
    possible, so ``qc.measure(q0, c0)`` or ``qc.measure(q1, c1)`` will generate a synchronous
    measurement pulse.

    Args:
        circuit: The quantum circuit to translate.
        schedule_config: Backend specific parameters used for building the Schedule.

    Returns:
        A list of CircuitPulseDefs: the pulse definition for each circuit element.

    Raises:
        QiskitError: If circuit uses a command that isn't defined in config.inst_map.
    """
    circ_pulse_defs = []

    inst_map = schedule_config.inst_map
    qubit_mem_slots = {}  # Map measured qubit index to classical bit index

    def get_measure_schedule() -> CircuitPulseDef:
        """Create a schedule to measure the qubits queued for measuring."""
        sched = Schedule()
        sched += measure(qubits=list(qubit_mem_slots.keys()),
                         inst_map=inst_map,
                         meas_map=schedule_config.meas_map,
                         qubit_mem_slots=qubit_mem_slots)
        qubit_mem_slots.clear()
        return CircuitPulseDef(schedule=sched,
                               qubits=[chan.index for chan in sched.channels
                                       if isinstance(chan, AcquireChannel)])

    for inst, qubits, clbits in circuit.data:
        inst_qubits = [qubit.index for qubit in qubits]  # We want only the indices of the qubits
        if any(q in qubit_mem_slots for q in inst_qubits):
            # If we are operating on a qubit that was scheduled to be measured, process that first
            circ_pulse_defs.append(get_measure_schedule())
        if isinstance(inst, Barrier):
            circ_pulse_defs.append(CircuitPulseDef(schedule=inst, qubits=inst_qubits))
        elif isinstance(inst, Measure):
            if (len(inst_qubits) != 1 and len(clbits) != 1):
                raise QiskitError("Qubit '{0}' or classical bit '{1}' errored because the "
                                  "circuit Measure instruction only takes one of "
                                  "each.".format(inst_qubits, clbits))
            qubit_mem_slots[inst_qubits[0]] = clbits[0].index
        else:
            try:
                circ_pulse_defs.append(
                    CircuitPulseDef(schedule=inst_map.get(inst.name, inst_qubits, *inst.params),
                                    qubits=inst_qubits))
            except PulseError:
                raise QiskitError("Operation '{0}' on qubit(s) {1} not supported by the backend "
                                  "command definition. Did you remember to transpile your input "
                                  "circuit for the same backend?".format(inst.name, inst_qubits))
    if qubit_mem_slots:
        circ_pulse_defs.append(get_measure_schedule())

    return circ_pulse_defs
<commit_msg>speed up schedulers (#3795)<commit_after># -*- coding: utf-8 -*-

# This code is part of Qiskit.
#
# (C) Copyright IBM 2019.
#
# This code is licensed under the Apache License, Version 2.0. You may
# obtain a copy of this license in the LICENSE.txt file in the root directory
# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
#
# Any modifications or derivative works of this code must retain this
# copyright notice, and modified files need to carry a notice indicating
# that they have been altered from the originals.

"""
The most straightforward scheduling methods: scheduling **as early** or **as late** as possible.
"""

from collections import defaultdict, namedtuple
from typing import List

from qiskit.circuit.measure import Measure
from qiskit.circuit.quantumcircuit import QuantumCircuit
from qiskit.exceptions import QiskitError
from qiskit.extensions.standard.barrier import Barrier
from qiskit.pulse.exceptions import PulseError
from qiskit.pulse.schedule import Schedule
from qiskit.pulse.channels import AcquireChannel
from qiskit.scheduler.utils import measure

from qiskit.scheduler.config import ScheduleConfig

CircuitPulseDef = namedtuple('CircuitPulseDef', [
    'schedule',  # The schedule which implements the quantum circuit command
    'qubits'])   # The labels of the qubits involved in the command according to the circuit

def as_soon_as_possible(circuit: QuantumCircuit,
                        schedule_config: ScheduleConfig) -> Schedule:
    """
    Return the pulse Schedule which implements the input circuit using an "as soon as possible"
    (asap) scheduling policy.

    Circuit instructions are first each mapped to equivalent pulse
    Schedules according to the command definition given by the schedule_config. Then, this circuit
    instruction-equivalent Schedule is appended at the earliest time at which all qubits involved
    in the instruction are available.

    Args:
        circuit: The quantum circuit to translate.
        schedule_config: Backend specific parameters used for building the Schedule.

    Returns:
        A schedule corresponding to the input ``circuit`` with pulses occurring as early as
        possible.
    """
    qubit_time_available = defaultdict(int)

    def update_times(inst_qubits: List[int], time: int = 0) -> None:
        """Update the time tracker for all inst_qubits to the given time."""
        for q in inst_qubits:
            qubit_time_available[q] = time

    start_times = []
    circ_pulse_defs = translate_gates_to_pulse_defs(circuit, schedule_config)
    for circ_pulse_def in circ_pulse_defs:
        start_time = max(qubit_time_available[q] for q in circ_pulse_def.qubits)
        stop_time = start_time
        if not isinstance(circ_pulse_def.schedule, Barrier):
            stop_time += circ_pulse_def.schedule.duration

        start_times.append(start_time)
        update_times(circ_pulse_def.qubits, stop_time)

    timed_schedules = [(time, cpd.schedule) for time, cpd in zip(start_times, circ_pulse_defs)
                       if not isinstance(cpd.schedule, Barrier)]
    return Schedule(*timed_schedules, name=circuit.name)

def as_late_as_possible(circuit: QuantumCircuit,
                        schedule_config: ScheduleConfig) -> Schedule:
    """
    Return the pulse Schedule which implements the input circuit using an "as late as possible"
    (alap) scheduling policy.

    Circuit instructions are first each mapped to equivalent pulse
    Schedules according to the command definition given by the schedule_config. Then, this circuit
    instruction-equivalent Schedule is appended at the latest time that it can be without allowing
    unnecessary time between instructions or allowing instructions with common qubits to overlap.

    This method should improves the outcome fidelity over ASAP scheduling, because we may
    maximize the time that the qubit remains in the ground state.

    Args:
        circuit: The quantum circuit to translate.
        schedule_config: Backend specific parameters used for building the Schedule.

    Returns:
        A schedule corresponding to the input ``circuit`` with pulses occurring as late as
        possible.
    """
    qubit_time_available = defaultdict(int)

    def update_times(inst_qubits: List[int], time: int = 0) -> None:
        """Update the time tracker for all inst_qubits to the given time."""
        for q in inst_qubits:
            qubit_time_available[q] = time

    rev_stop_times = []
    circ_pulse_defs = translate_gates_to_pulse_defs(circuit, schedule_config)
    for circ_pulse_def in reversed(circ_pulse_defs):
        start_time = max(qubit_time_available[q] for q in circ_pulse_def.qubits)
        stop_time = start_time
        if not isinstance(circ_pulse_def.schedule, Barrier):
            stop_time += circ_pulse_def.schedule.duration

        rev_stop_times.append(stop_time)
        update_times(circ_pulse_def.qubits, stop_time)

    last_stop = max(t for t in qubit_time_available.values())
    start_times = [last_stop - t for t in reversed(rev_stop_times)]
    timed_schedules = [(time, cpd.schedule) for time, cpd in zip(start_times, circ_pulse_defs)
                       if not isinstance(cpd.schedule, Barrier)]
    return Schedule(*timed_schedules, name=circuit.name)

def translate_gates_to_pulse_defs(circuit: QuantumCircuit,
                                  schedule_config: ScheduleConfig) -> List[CircuitPulseDef]:
    """
    Return a list of Schedules and the qubits they operate on, for each element encountered in th
    input circuit.

    Without concern for the final schedule, extract and return a list of Schedules and the qubits
    they operate on, for each element encountered in the input circuit. Measures are grouped when
    possible, so ``qc.measure(q0, c0)`` or ``qc.measure(q1, c1)`` will generate a synchronous
    measurement pulse.

    Args:
        circuit: The quantum circuit to translate.
        schedule_config: Backend specific parameters used for building the Schedule.

    Returns:
        A list of CircuitPulseDefs: the pulse definition for each circuit element.

    Raises:
        QiskitError: If circuit uses a command that isn't defined in config.inst_map.
    """
    circ_pulse_defs = []

    inst_map = schedule_config.inst_map
    qubit_mem_slots = {}  # Map measured qubit index to classical bit index

    def get_measure_schedule() -> CircuitPulseDef:
        """Create a schedule to measure the qubits queued for measuring."""
        sched = Schedule()
        sched += measure(qubits=list(qubit_mem_slots.keys()),
                         inst_map=inst_map,
                         meas_map=schedule_config.meas_map,
                         qubit_mem_slots=qubit_mem_slots)
        qubit_mem_slots.clear()
        return CircuitPulseDef(schedule=sched,
                               qubits=[chan.index for chan in sched.channels
                                       if isinstance(chan, AcquireChannel)])

    for inst, qubits, clbits in circuit.data:
        inst_qubits = [qubit.index for qubit in qubits]  # We want only the indices of the qubits
        if any(q in qubit_mem_slots for q in inst_qubits):
            # If we are operating on a qubit that was scheduled to be measured, process that first
            circ_pulse_defs.append(get_measure_schedule())
        if isinstance(inst, Barrier):
            circ_pulse_defs.append(CircuitPulseDef(schedule=inst, qubits=inst_qubits))
        elif isinstance(inst, Measure):
            if (len(inst_qubits) != 1 and len(clbits) != 1):
                raise QiskitError("Qubit '{0}' or classical bit '{1}' errored because the "
                                  "circuit Measure instruction only takes one of "
                                  "each.".format(inst_qubits, clbits))
            qubit_mem_slots[inst_qubits[0]] = clbits[0].index
        else:
            try:
                circ_pulse_defs.append(
                    CircuitPulseDef(schedule=inst_map.get(inst.name, inst_qubits, *inst.params),
                                    qubits=inst_qubits))
            except PulseError:
                raise QiskitError("Operation '{0}' on qubit(s) {1} not supported by the backend "
                                  "command definition. Did you remember to transpile your input "
                                  "circuit for the same backend?".format(inst.name, inst_qubits))
    if qubit_mem_slots:
        circ_pulse_defs.append(get_measure_schedule())

    return circ_pulse_defs
<|endoftext|>


<commit_before># future imports
from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals

# stdlib imports
import logging

# third-party imports
import soundcloud

# local imports
from radiobabel.errors import TrackNotFound

logger = logging.getLogger('radiobabel.backends.soundcloud')

def _transform_track(track):
    """Transform result into a format that more closely matches our unified API.
    """
    transformed_track = dict([
        ('source_type', 'soundcloud'),
        ('source_id', track['id']),
        ('name', track['title']),
        ('duration_ms', track['duration']),
        ('preview_url', track.get('stream_url')),
        ('track_number', 0),
    ])
    transformed_track['artists'] = [
        dict([
            ('source_type', 'soundcloud'),
            ('source_id', track['user']['id']),
            ('name', track['user']['username']),
        ]),
    ]
    transformed_track['album'] = None
    if track['artwork_url']:
        transformed_track['image_url'] = track['artwork_url']
    else:
        transformed_track['image_url'] = 'http://judejohnstone.com/wp-content/themes/soundcheck/images/default-artwork.png'
    return transformed_track

class SoundcloudClient(object):

    def __init__(self, client_id):
        """Initialise soundcloud API client.
        """
        self.client = soundcloud.Client(client_id=client_id)

    def track(self, track_id):
        """Lookup a single track using the soundcloud API
        """
        logger.info('Track lookup: {0}'.format(track_id))

        try:
            track = self.client.get('/tracks', ids=track_id)[0].obj
        except IndexError:
            raise TrackNotFound('Soundcloud: {0}'.format(track_id))
        return _transform_track(track)

    def search(self, query, limit=200, offset=0):
        """Search for tracks using the soundcloud API
        """
        logger.info('Searching: Limit {0}, Offset {1}'.format(limit, offset))

        tracks = self.client.get('/tracks', q=query, limit=limit, offset=offset)
        return [_transform_track(x.obj) for x in tracks]
<commit_msg>radiobabel > backends > soundcloud - if track has image, returns 3 image sizes<commit_after># future imports
from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals

# stdlib imports
import logging

# third-party imports
import soundcloud

# local imports
from radiobabel.errors import TrackNotFound

logger = logging.getLogger('radiobabel.backends.soundcloud')

def _transform_track(track):
    """Transform result into a format that
    more closely matches our unified API.
    """
    large_artwork = None
    medium_artwork = None
    small_artwork = None

    if track['artwork_url']:
        large_artwork = (track['artwork_url']).replace('large', 't500x500')
        medium_artwork = (track['artwork_url']).replace('large', 't300x300')
        small_artwork = (track['artwork_url']).replace('large', 't67x67')

    transformed_track = dict([
        ('source_type', 'soundcloud'),
        ('source_id', track['id']),
        ('name', track['title']),
        ('duration_ms', track['duration']),
        ('preview_url', track.get('stream_url')),
        ('track_number', 0),
        ('image_small', small_artwork),
        ('image_medium', medium_artwork),
        ('image_large', large_artwork),
    ])
    transformed_track['artists'] = [
        dict([
            ('source_type', 'soundcloud'),
            ('source_id', track['user']['id']),
            ('name', track['user']['username']),
        ]),
    ]
    transformed_track['album'] = None

    return transformed_track

class SoundcloudClient(object):

    def __init__(self, client_id):
        """Initialise soundcloud API client.
        """
        self.client = soundcloud.Client(client_id=client_id)

    def track(self, track_id):
        """Lookup a single track using the soundcloud API
        """
        logger.info('Track lookup: {0}'.format(track_id))

        try:
            track = self.client.get('/tracks', ids=track_id)[0].obj
        except IndexError:
            raise TrackNotFound('Soundcloud: {0}'.format(track_id))
        return _transform_track(track)

    def search(self, query, limit=200, offset=0):
        """Search for tracks using the soundcloud API
        """
        logger.info('Searching: Limit {0}, Offset {1}'.format(limit, offset))

        tracks = self.client.get('/tracks', q=query, limit=limit, offset=offset)
        return [_transform_track(x.obj) for x in tracks]
<|endoftext|>