<commit_before># -*- coding: utf-8 -*-
#
# Copyright 2017 Ricequant, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from datetime import datetime
import traceback
import logbook
import better_exceptions
from logbook import Logger
from logbook.more import ColorizedStderrHandler

from rqalpha.utils.py2 import to_utf8, from_utf8

logbook.set_datetime_format("local")

# patch warn
logbook.base._level_names[logbook.base.WARNING] = 'WARN'

# better_exceptions hot patch
def format_exception(exc, value, tb):
    formatted, colored_source = better_exceptions.format_traceback(tb)

    if not str(value) and exc is AssertionError:
        value.args = (colored_source,)
    title = traceback.format_exception_only(exc, value)
    title = from_utf8(title[0].strip())
    full_trace = u'Traceback (most recent call last):\n{}{}\n'.format(formatted, title)

    return full_trace

better_exceptions.format_exception = format_exception

__all__ = [
    "user_log",
    "system_log",
]

DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f"

def user_std_handler_log_formatter(record, handler):
    from rqalpha.environment import Environment
    try:
        dt = Environment.get_instance().calendar_dt.strftime(DATETIME_FORMAT)[:-3]
    except Exception:
        dt = datetime.now().strftime(DATETIME_FORMAT)[:-3]

    log = "{dt} {level} {msg}".format(
        dt=dt,
        level=record.level_name,
        msg=to_utf8(record.message),
    )
    return log

user_std_handler = ColorizedStderrHandler(bubble=True)
user_std_handler.formatter = user_std_handler_log_formatter

def formatter_builder(tag):
    def formatter(record, handler):

        log = "[{formatter_tag}] [{time}] {level}: {msg}".format(
            formatter_tag=tag,
            level=record.level_name,
            msg=to_utf8(record.message),
            time=record.time,
        )

        if record.formatted_exception:
            log += "\n" + record.formatted_exception
        return log
    return formatter

# loggers
# 用户代码logger日志
user_log = Logger("user_log")
# 给用户看的系统日志
user_system_log = Logger("user_system_log")

# 用于用户异常的详细日志打印
user_detail_log = Logger("user_detail_log")
# user_detail_log.handlers.append(ColorizedStderrHandler(bubble=True))

# 系统日志
system_log = Logger("system_log")
basic_system_log = Logger("basic_system_log")

# 标准输出日志
std_log = Logger("std_log")

def init_logger():
    system_log.handlers = [ColorizedStderrHandler(bubble=True)]
    basic_system_log.handlers = [ColorizedStderrHandler(bubble=True)]
    std_log.handlers = [ColorizedStderrHandler(bubble=True)]
    user_log.handlers = []
    user_system_log.handlers = []

def user_print(*args, **kwargs):
    sep = kwargs.get("sep", " ")
    end = kwargs.get("end", "")

    message = sep.join(map(str, args)) + end

    user_log.info(message)

init_logger()<commit_msg>fix log date<commit_after># -*- coding: utf-8 -*-
#
# Copyright 2017 Ricequant, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from datetime import datetime
import traceback
import logbook
import better_exceptions
from logbook import Logger
from logbook.more import ColorizedStderrHandler

from rqalpha.utils.py2 import to_utf8, from_utf8

logbook.set_datetime_format("local")

# patch warn
logbook.base._level_names[logbook.base.WARNING] = 'WARN'

# better_exceptions hot patch
def format_exception(exc, value, tb):
    formatted, colored_source = better_exceptions.format_traceback(tb)

    if not str(value) and exc is AssertionError:
        value.args = (colored_source,)
    title = traceback.format_exception_only(exc, value)
    title = from_utf8(title[0].strip())
    full_trace = u'Traceback (most recent call last):\n{}{}\n'.format(formatted, title)

    return full_trace

better_exceptions.format_exception = format_exception

__all__ = [
    "user_log",
    "system_log",
]

DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S.%f"

def user_std_handler_log_formatter(record, handler):
    from rqalpha.environment import Environment
    try:
        dt = Environment.get_instance().calendar_dt.strftime(DATETIME_FORMAT)
    except Exception:
        dt = datetime.now().strftime(DATETIME_FORMAT)

    log = "{dt} {level} {msg}".format(
        dt=dt,
        level=record.level_name,
        msg=to_utf8(record.message),
    )
    return log

user_std_handler = ColorizedStderrHandler(bubble=True)
user_std_handler.formatter = user_std_handler_log_formatter

def formatter_builder(tag):
    def formatter(record, handler):

        log = "[{formatter_tag}] [{time}] {level}: {msg}".format(
            formatter_tag=tag,
            level=record.level_name,
            msg=to_utf8(record.message),
            time=record.time,
        )

        if record.formatted_exception:
            log += "\n" + record.formatted_exception
        return log
    return formatter

# loggers
# 用户代码logger日志
user_log = Logger("user_log")
# 给用户看的系统日志
user_system_log = Logger("user_system_log")

# 用于用户异常的详细日志打印
user_detail_log = Logger("user_detail_log")
# user_detail_log.handlers.append(ColorizedStderrHandler(bubble=True))

# 系统日志
system_log = Logger("system_log")
basic_system_log = Logger("basic_system_log")

# 标准输出日志
std_log = Logger("std_log")

def init_logger():
    system_log.handlers = [ColorizedStderrHandler(bubble=True)]
    basic_system_log.handlers = [ColorizedStderrHandler(bubble=True)]
    std_log.handlers = [ColorizedStderrHandler(bubble=True)]
    user_log.handlers = []
    user_system_log.handlers = []

def user_print(*args, **kwargs):
    sep = kwargs.get("sep", " ")
    end = kwargs.get("end", "")

    message = sep.join(map(str, args)) + end

    user_log.info(message)

init_logger()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# cerbero - a multi-platform build system for Open Source software
# Copyright (C) 2012 Thiago Santos <thiago.sousa.santos@collabora.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

import os
import subprocess
import shutil

file_types = [
    ('Mach-O', 'merge'),
    ('ar archive', 'merge'),
    ('libtool archive', 'skip'),
    ('libtool library', 'skip'),
    ('symbolic link', 'link'),
    ('data', 'copy'),
    ('text', 'copy'),
    ('document', 'copy'),
    ('catalog', 'copy'),
    ('python', 'copy'),
    ('image', 'copy'),
    ('icon', 'copy'),
    ('FORTRAN', 'copy'),
    ('LaTeX', 'copy'),
    ('Zip', 'copy'),
    ('empty', 'copy'),
    ('data', 'copy'),
]

class OSXUniversalGenerator(object):
    '''
    Wrapper for OS X's lipo command to help generating universal binaries
    from single arch binaries.

    It takes multiple input directories and parses through them. For every
    file it finds, it looks at file type and compare with the file types
    of the other input directories, in case the file is a single arch executable
    or dynamic library, it will be merged into a universal binary and copied to
    the output directory. All the other files (text/headers/scripts) are just
    copied directly.

    This tool assumes that the input roots have the same structures and files
    as they should be results from building the same project to different
    architectures

    '''

    LIPO_CMD = 'lipo'
    FILE_CMD = 'file'

    def __init__(self, output_root):
        '''
        @output_root: the output directory where the result will be generated

        '''
        self.output_root = output_root
        self.missing = []

    def merge_files(self, filelist, dirs):
        if len(filelist) == 0:
            return
        for f in filelist:
            self.do_merge(f, dirs)

    def merge_dirs(self, input_roots):
        if not os.path.exists(self.output_root):
            os.mkdir(self.output_root)
        self.parse_dirs(input_roots)

    def create_universal_file(self, output, inputlist):
        cmd = '%s -create %s -output %s' % (self.LIPO_CMD,
               ' '.join(inputlist), output)
        self._call(cmd)

    def get_file_type(self, filepath):
        cmd = '%s -bh "%s"' % (self.FILE_CMD, filepath)
        return self._call(cmd)[0:-1] #remove trailing \n

    def _detect_merge_action(self, files_list):
        actions = []
        for f in files_list:
            if not os.path.exists(f):
                continue #TODO what can we do here? fontconfig has
                         #some random generated filenames it seems
            ftype = self.get_file_type(f)
            action = ''
            for ft in file_types:
                if ft[0] in ftype:
                    action = ft[1]
                    break
            if not action:
                raise Exception, 'Unexpected file type %s %s' % (str(ftype), f)
            actions.append(action)
        if len(actions) == 0:
            return 'skip' #we should skip this one, the file doesn't exist
        all_same = all(x == actions[0] for x in actions)
        if not all_same:
            raise Exception, 'Different file types found: %s : %s' \
                             % (str(ftype), str(files_list))
        return actions[0]

    def do_merge(self, filepath, dirs):
        full_filepaths = [os.path.join(d, filepath) for d in dirs]
        action = self._detect_merge_action(full_filepaths)

        #pick the first file as the base one in case of copying/linking
        current_file = full_filepaths[0]
        output_file = os.path.join(self.output_root, filepath)
        output_dir = os.path.dirname(output_file)

        if action == 'copy':
            self._copy(current_file, output_file)
        elif action == 'link':
            self._link(current_file, output_file, filepath)
        elif action == 'merge':
            if not os.path.exists(output_dir):
                os.makedirs(output_dir)
            self.create_universal_file(output_file, full_filepaths)
        elif action == 'skip':
            pass #just pass
        else:
            raise Exception, 'unexpected action %s' % action

    def parse_dirs(self, dirs, filters=None):
        self.missing = []

        dir_path = dirs[0]
        if dir_path.endswith('/'):
            dir_path = dir_path[:-1]
        for dirpath, dirnames, filenames in os.walk(dir_path):
            current_dir = ''
            token = ' '
            remaining_dirpath = dirpath
            while remaining_dirpath != dir_path or token == '':
                remaining_dirpath, token = os.path.split(remaining_dirpath)
                current_dir = os.path.join(token, current_dir)

            for f in filenames:
                if filters is not None and os.path.splitext(f)[1] not in filters:
                    continue
                current_file = os.path.join(current_dir, f)
                self.do_merge(current_file, dirs)

    def _copy(self, src, dest):
        if not os.path.exists(os.path.dirname(dest)):
            os.makedirs(os.path.dirname(dest))
        shutil.copy(src, dest)

    def _link(self, src, dest, filepath):
        if not os.path.exists(os.path.dirname(dest)):
            os.makedirs(os.path.dirname(dest))
        if os.path.lexists(dest):
            return #link exists, skip it

        # read the link, and extract the relative filepath
        target = os.path.join(os.path.dirname(src), os.readlink(src))
        src_prefix = src.split(filepath)[0]
        dest_prefix = dest.split(filepath)[0]
        rel_target = os.path.relpath(target, src_prefix)
        dest_target = os.path.join(dest_prefix, rel_target)
        if not os.path.exists(dest_target):
            shutil.copy(target, dest_target)
        os.symlink(dest_target, dest)

    def _call(self, cmd, cwd=None):
        cmd = cmd or self.root
        process = subprocess.Popen(cmd, cwd=cwd,
            stdout=subprocess.PIPE, shell=True)
        output, unused_err = process.communicate()
        return output

class Main(object):

    def run(self):
        # We use OptionParser instead of ArgumentsParse because this script might
        # be run in OS X 10.6 or older, which do not provide the argparse module
        import optparse
        usage = "usage: %prog [options] outputdir inputdir1 inputdir2 ..."
        description='Merges multiple architecture build trees into a single '\
                    'universal binary build tree'
        parser = optparse.OptionParser(usage=usage, description=description)
        options, args = parser.parse_args()
        if len(args) < 3:
            parser.print_usage()
            exit(1)
        generator = OSXUniversalGenerator(args[0])
        generator.merge_dirs(args[1:])
        exit(0)

if __name__ == "__main__":
    main = Main()
    main.run()
<commit_msg>osxuniversalgenerator: copy .la files too<commit_after>#!/usr/bin/env python
# cerbero - a multi-platform build system for Open Source software
# Copyright (C) 2012 Thiago Santos <thiago.sousa.santos@collabora.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

import os
import subprocess
import shutil

file_types = [
    ('Mach-O', 'merge'),
    ('ar archive', 'merge'),
    ('libtool archive', 'skip'),
    ('libtool library', 'copy-la'),
    ('symbolic link', 'link'),
    ('data', 'copy'),
    ('text', 'copy'),
    ('document', 'copy'),
    ('catalog', 'copy'),
    ('python', 'copy'),
    ('image', 'copy'),
    ('icon', 'copy'),
    ('FORTRAN', 'copy'),
    ('LaTeX', 'copy'),
    ('Zip', 'copy'),
    ('empty', 'copy'),
    ('data', 'copy'),
]

class OSXUniversalGenerator(object):
    '''
    Wrapper for OS X's lipo command to help generating universal binaries
    from single arch binaries.

    It takes multiple input directories and parses through them. For every
    file it finds, it looks at file type and compare with the file types
    of the other input directories, in case the file is a single arch executable
    or dynamic library, it will be merged into a universal binary and copied to
    the output directory. All the other files (text/headers/scripts) are just
    copied directly.

    This tool assumes that the input roots have the same structures and files
    as they should be results from building the same project to different
    architectures

    '''

    LIPO_CMD = 'lipo'
    FILE_CMD = 'file'

    def __init__(self, output_root):
        '''
        @output_root: the output directory where the result will be generated

        '''
        self.output_root = output_root
        self.missing = []

    def merge_files(self, filelist, dirs):
        if len(filelist) == 0:
            return
        for f in filelist:
            self.do_merge(f, dirs)

    def merge_dirs(self, input_roots):
        if not os.path.exists(self.output_root):
            os.mkdir(self.output_root)
        self.parse_dirs(input_roots)

    def create_universal_file(self, output, inputlist):
        cmd = '%s -create %s -output %s' % (self.LIPO_CMD,
               ' '.join(inputlist), output)
        self._call(cmd)

    def get_file_type(self, filepath):
        cmd = '%s -bh "%s"' % (self.FILE_CMD, filepath)
        return self._call(cmd)[0:-1] #remove trailing \n

    def _detect_merge_action(self, files_list):
        actions = []
        for f in files_list:
            if not os.path.exists(f):
                continue #TODO what can we do here? fontconfig has
                         #some random generated filenames it seems
            ftype = self.get_file_type(f)
            action = ''
            for ft in file_types:
                if ft[0] in ftype:
                    action = ft[1]
                    break
            if not action:
                raise Exception, 'Unexpected file type %s %s' % (str(ftype), f)
            actions.append(action)
        if len(actions) == 0:
            return 'skip' #we should skip this one, the file doesn't exist
        all_same = all(x == actions[0] for x in actions)
        if not all_same:
            raise Exception, 'Different file types found: %s : %s' \
                             % (str(ftype), str(files_list))
        return actions[0]

    def do_merge(self, filepath, dirs):
        full_filepaths = [os.path.join(d, filepath) for d in dirs]
        action = self._detect_merge_action(full_filepaths)

        #pick the first file as the base one in case of copying/linking
        current_file = full_filepaths[0]
        output_file = os.path.join(self.output_root, filepath)
        output_dir = os.path.dirname(output_file)

        if action == 'copy':
            self._copy(current_file, output_file)
        elif action == 'copy-la':
            self._copy_la(current_file, output_file, dirs)
        elif action == 'link':
            self._link(current_file, output_file, filepath)
        elif action == 'merge':
            if not os.path.exists(output_dir):
                os.makedirs(output_dir)
            self.create_universal_file(output_file, full_filepaths)
        elif action == 'skip':
            pass #just pass
        else:
            raise Exception, 'unexpected action %s' % action

    def parse_dirs(self, dirs, filters=None):
        self.missing = []

        dir_path = dirs[0]
        if dir_path.endswith('/'):
            dir_path = dir_path[:-1]
        for dirpath, dirnames, filenames in os.walk(dir_path):
            current_dir = ''
            token = ' '
            remaining_dirpath = dirpath
            while remaining_dirpath != dir_path or token == '':
                remaining_dirpath, token = os.path.split(remaining_dirpath)
                current_dir = os.path.join(token, current_dir)

            for f in filenames:
                if filters is not None and os.path.splitext(f)[1] not in filters:
                    continue
                current_file = os.path.join(current_dir, f)
                self.do_merge(current_file, dirs)

    def _copy(self, src, dest):
        if not os.path.exists(os.path.dirname(dest)):
            os.makedirs(os.path.dirname(dest))
        shutil.copy(src, dest)

    def _copy_la(self, src, dest, dirs):
        self._copy(src, dest)
        self._replace(dest, {d:self.output_root for d in dirs})

    def _link(self, src, dest, filepath):
        if not os.path.exists(os.path.dirname(dest)):
            os.makedirs(os.path.dirname(dest))
        if os.path.lexists(dest):
            return #link exists, skip it

        # read the link, and extract the relative filepath
        target = os.path.join(os.path.dirname(src), os.readlink(src))
        src_prefix = src.split(filepath)[0]
        dest_prefix = dest.split(filepath)[0]
        rel_target = os.path.relpath(target, src_prefix)
        dest_target = os.path.join(dest_prefix, rel_target)
        if not os.path.exists(dest_target):
            shutil.copy(target, dest_target)
        os.symlink(dest_target, dest)

    def _call(self, cmd, cwd=None):
        cmd = cmd or self.root
        process = subprocess.Popen(cmd, cwd=cwd,
            stdout=subprocess.PIPE, shell=True)
        output, unused_err = process.communicate()
        return output

    def _replace(self, filepath, replacements):
        with open(filepath, 'r') as f:
            content = f.read()
        for k, v in replacements.iteritems():
            content = content.replace(k, v)
        with open(filepath, 'w+') as f:
            f.write(content)

class Main(object):

    def run(self):
        # We use OptionParser instead of ArgumentsParse because this script might
        # be run in OS X 10.6 or older, which do not provide the argparse module
        import optparse
        usage = "usage: %prog [options] outputdir inputdir1 inputdir2 ..."
        description='Merges multiple architecture build trees into a single '\
                    'universal binary build tree'
        parser = optparse.OptionParser(usage=usage, description=description)
        options, args = parser.parse_args()
        if len(args) < 3:
            parser.print_usage()
            exit(1)
        generator = OSXUniversalGenerator(args[0])
        generator.merge_dirs(args[1:])
        exit(0)

if __name__ == "__main__":
    main = Main()
    main.run()
<|endoftext|>


<commit_before>import numpy as np
import tensorflow as tf

from tensor_train import TensorTrain

def tt_rand_matrix(shape, rank=2):
    """Generate a random TT-matrix of given shape.

    Args:
        shape: 2d array, shape[0] is the shape of the matrix row-index,
            shape[1] is the shape of the column index.
            shape[0] and shape[1] should have the same number of elements (d)
        rank: a number or a (d-1)-element array with ranks.

    Returns:
        TensorTrain containing a TT-matrix of size
            np.prod(shape[0]) x np.prod(shape[1])
    """
    # TODO: good distribution to init training.
    shape = np.array(shape)
    rank = np.array(rank)
    if len(shape.shape) != 2:
        raise ValueError('shape should be 2d array')
    if shape[0].size != shape[1].size:
        raise ValueError('shape[0] should have the same length as shape[1]')
    if np.any(shape.flatten() < 1):
        raise ValueError('all elements in `shape` should be positive')
    if np.any(rank < 1):
        raise ValueError('`rank` should be positive')
    if rank.size != 1 and rank.size != (shape[0].size-1):
        raise ValueError('`rank` array has inappropriate size')

    num_dims = shape[0].size
    if rank.size == 1:
        rank = rank * np.ones(num_dims)
    # Add 1 to the beggining and end to simplify working with ranks.
    rank_ext = np.insert(rank, 0, 1)
    rank_ext = np.append(rank_ext, 1)
    # TODO: check that ints?
    shape = shape.astype(int)
    rank_ext = rank_ext.astype(int)

    tt_cores = [None] * num_dims
    for i in range(num_dims):
        curr_core_shape = (rank_ext[i], shape[0][i], shape[1][i], rank_ext[i+1])
        tt_cores[i] = tf.random_normal(curr_core_shape)

    return TensorTrain(tt_cores)
<commit_msg>Implement tt_rand_matrix<commit_after>import numpy as np
import tensorflow as tf

from tensor_train import TensorTrain

def tt_rand_matrix(shape, rank=2):
    """Generate a random TT-matrix of given shape.

    Args:
        shape: 2d array, shape[0] is the shape of the matrix row-index,
            shape[1] is the shape of the column index.
            shape[0] and shape[1] should have the same number of elements (d)
        rank: a number or a (d-1)-element array with ranks.

    Returns:
        TensorTrain containing a TT-matrix of size
            np.prod(shape[0]) x np.prod(shape[1])
    """
    # TODO: good distribution to init training.
    shape = np.array(shape)
    rank = np.array(rank)
    if len(shape.shape) != 2:
        raise ValueError('shape should be 2d array')
    if shape[0].size != shape[1].size:
        raise ValueError('shape[0] should have the same length as shape[1]')
    if np.any(shape.flatten() < 1):
        raise ValueError('all elements in `shape` should be positive')
    if np.any(rank < 1):
        raise ValueError('`rank` should be positive')
    if rank.size != 1 and rank.size != (shape[0].size-1):
        raise ValueError('`rank` array has inappropriate size')

    num_dims = shape[0].size
    if rank.size == 1:
        rank = rank * np.ones(num_dims - 1)
    # Add 1 to the beggining and end to simplify working with ranks.
    rank_ext = np.insert(rank, 0, 1)
    rank_ext = np.append(rank_ext, 1)
    # TODO: check that ints?
    shape = shape.astype(int)
    rank_ext = rank_ext.astype(int)

    tt_cores = [None] * num_dims
    for i in range(num_dims):
        curr_core_shape = (rank_ext[i], shape[0][i], shape[1][i], rank_ext[i+1])
        tt_cores[i] = tf.random_normal(curr_core_shape)

    return TensorTrain(tt_cores)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# cerbero - a multi-platform build system for Open Source software
# Copyright (C) 2012 Thiago Santos <thiago.sousa.santos@collabora.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

import os
import subprocess
import shutil
import tempfile

from cerbero.utils import shell
from cerbero.tools.osxrelocator import OSXRelocator

file_types = [
    ('Mach-O', 'merge'),
    ('ar archive', 'merge'),
    ('libtool archive', 'skip'),
    ('libtool library', 'copy-la'),
    ('symbolic link', 'link'),
    ('data', 'copy'),
    ('text', 'copy'),
    ('document', 'copy'),
    ('catalog', 'copy'),
    ('python', 'copy'),
    ('image', 'copy'),
    ('icon', 'copy'),
    ('FORTRAN', 'copy'),
    ('LaTeX', 'copy'),
    ('Zip', 'copy'),
    ('empty', 'copy'),
    ('data', 'copy'),
]

class OSXUniversalGenerator(object):
    '''
    Wrapper for OS X's lipo command to help generating universal binaries
    from single arch binaries.

    It takes multiple input directories and parses through them. For every
    file it finds, it looks at file type and compare with the file types
    of the other input directories, in case the file is a single arch executable
    or dynamic library, it will be merged into a universal binary and copied to
    the output directory. All the other files (text/headers/scripts) are just
    copied directly.

    This tool assumes that the input roots have the same structures and files
    as they should be results from building the same project to different
    architectures

    '''

    LIPO_CMD = 'lipo'
    FILE_CMD = 'file'

    def __init__(self, output_root):
        '''
        @output_root: the output directory where the result will be generated

        '''
        self.output_root = output_root
        if self.output_root.endswith('/'):
            self.output_root = self.output_root[:-1]
        self.missing = []

    def merge_files(self, filelist, dirs):
        if len(filelist) == 0:
            return
        for f in filelist:
            self.do_merge(f, dirs)

    def merge_dirs(self, input_roots):
        if not os.path.exists(self.output_root):
            os.mkdir(self.output_root)
        self.parse_dirs(input_roots)

    def create_universal_file(self, output, inputlist, dirs):
        tmp_inputs = []
        # relocate all files with the prefix of the merged file.
        # which must be done before merging them.
        for f in inputlist:
            # keep the filename in the suffix to preserve the filename extension
            tmp = tempfile.NamedTemporaryFile(suffix=os.path.basename(f))
            tmp_inputs.append(tmp)
            shutil.copy(f, tmp.name)
            prefix_to_replace = [d for d in dirs if d in f][0]
            relocator = OSXRelocator (self.output_root, prefix_to_replace, self.output_root,
                                      False)
            # since we are using a temporary file, we must force the library id
            # name to real one and not based on the filename
            relocator.relocate_file(tmp.name,
                    id=f.replace(prefix_to_replace, self.output_root))
        cmd = '%s -create %s -output %s' % (self.LIPO_CMD,
            ' '.join([f.name for f in tmp_inputs]), output)
        self._call(cmd)

    def get_file_type(self, filepath):
        cmd = '%s -bh "%s"' % (self.FILE_CMD, filepath)
        return self._call(cmd)[0:-1] #remove trailing \n

    def _detect_merge_action(self, files_list):
        actions = []
        for f in files_list:
            if not os.path.exists(f):
                continue #TODO what can we do here? fontconfig has
                         #some random generated filenames it seems
            ftype = self.get_file_type(f)
            action = ''
            for ft in file_types:
                if ft[0] in ftype:
                    if ft[0] == 'text' and f.endswith('.pc'):
                        action = 'copy-pc'
                    else:
                        action = ft[1]
                    break
            if not action:
                raise Exception, 'Unexpected file type %s %s' % (str(ftype), f)
            actions.append(action)
        if len(actions) == 0:
            return 'skip' #we should skip this one, the file doesn't exist
        all_same = all(x == actions[0] for x in actions)
        if not all_same:
            raise Exception, 'Different file types found: %s : %s' \
                             % (str(ftype), str(files_list))
        return actions[0]

    def do_merge(self, filepath, dirs):
        full_filepaths = [os.path.join(d, filepath) for d in dirs]
        action = self._detect_merge_action(full_filepaths)

        #pick the first file as the base one in case of copying/linking
        current_file = full_filepaths[0]
        output_file = os.path.join(self.output_root, filepath)
        output_dir = os.path.dirname(output_file)

        if action == 'copy':
            self._copy(current_file, output_file)
        elif action == 'copy-la' or action == 'copy-pc':
            self._copy_and_replace_paths(current_file, output_file, dirs)
        elif action == 'link':
            self._link(current_file, output_file, filepath)
        elif action == 'merge':
            if not os.path.exists(output_dir):
                os.makedirs(output_dir)
            self.create_universal_file(output_file, full_filepaths, dirs)
        elif action == 'skip':
            pass #just pass
        else:
            raise Exception, 'unexpected action %s' % action

    def parse_dirs(self, dirs, filters=None):
        self.missing = []

        dir_path = dirs[0]
        if dir_path.endswith('/'):
            dir_path = dir_path[:-1]
        for dirpath, dirnames, filenames in os.walk(dir_path):
            current_dir = ''
            token = ' '
            remaining_dirpath = dirpath
            while remaining_dirpath != dir_path or token == '':
                remaining_dirpath, token = os.path.split(remaining_dirpath)
                current_dir = os.path.join(token, current_dir)

            for f in filenames:
                if filters is not None and os.path.splitext(f)[1] not in filters:
                    continue
                current_file = os.path.join(current_dir, f)
                self.do_merge(current_file, dirs)

    def _copy(self, src, dest):
        if not os.path.exists(os.path.dirname(dest)):
            os.makedirs(os.path.dirname(dest))
        shutil.copy(src, dest)

    def _copy_and_replace_paths(self, src, dest, dirs):
        self._copy(src, dest)
        replacements = {}
        for d in dirs:
            replacements[d]=self.output_root
        shell.replace(dest, replacements)

    def _link(self, src, dest, filepath):
        if not os.path.exists(os.path.dirname(dest)):
            os.makedirs(os.path.dirname(dest))
        if os.path.lexists(dest):
            return #link exists, skip it

        # read the link, and extract the relative filepath
        target = os.path.join(os.path.dirname(src), os.readlink(src))
        src_prefix = src.split(filepath)[0]
        dest_prefix = dest.split(filepath)[0]
        rel_target = os.path.relpath(target, src_prefix)
        dest_target = os.path.join(dest_prefix, rel_target)
        if not os.path.exists(dest_target):
            if not os.path.exists(os.path.dirname(dest_target)):
                os.makedirs(os.path.dirname(dest_target))
            shutil.copy(target, dest_target)
        os.symlink(dest_target, dest)

    def _call(self, cmd, cwd=None):
        cmd = cmd or self.root
        process = subprocess.Popen(cmd, cwd=cwd,
            stdout=subprocess.PIPE, shell=True)
        output, unused_err = process.communicate()
        return output

class Main(object):

    def run(self):
        # We use OptionParser instead of ArgumentsParse because this script might
        # be run in OS X 10.6 or older, which do not provide the argparse module
        import optparse
        usage = "usage: %prog [options] outputdir inputdir1 inputdir2 ..."
        description='Merges multiple architecture build trees into a single '\
                    'universal binary build tree'
        parser = optparse.OptionParser(usage=usage, description=description)
        options, args = parser.parse_args()
        if len(args) < 3:
            parser.print_usage()
            exit(1)
        generator = OSXUniversalGenerator(args[0])
        generator.merge_dirs(args[1:])
        exit(0)

if __name__ == "__main__":
    main = Main()
    main.run()
<commit_msg>osxuniversalgenerator: handle links with realtive paths correctly<commit_after>#!/usr/bin/env python
# cerbero - a multi-platform build system for Open Source software
# Copyright (C) 2012 Thiago Santos <thiago.sousa.santos@collabora.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

import os
import subprocess
import shutil
import tempfile

from cerbero.utils import shell
from cerbero.tools.osxrelocator import OSXRelocator

file_types = [
    ('Mach-O', 'merge'),
    ('ar archive', 'merge'),
    ('libtool archive', 'skip'),
    ('libtool library', 'copy-la'),
    ('symbolic link', 'link'),
    ('data', 'copy'),
    ('text', 'copy'),
    ('document', 'copy'),
    ('catalog', 'copy'),
    ('python', 'copy'),
    ('image', 'copy'),
    ('icon', 'copy'),
    ('FORTRAN', 'copy'),
    ('LaTeX', 'copy'),
    ('Zip', 'copy'),
    ('empty', 'copy'),
    ('data', 'copy'),
]

class OSXUniversalGenerator(object):
    '''
    Wrapper for OS X's lipo command to help generating universal binaries
    from single arch binaries.

    It takes multiple input directories and parses through them. For every
    file it finds, it looks at file type and compare with the file types
    of the other input directories, in case the file is a single arch executable
    or dynamic library, it will be merged into a universal binary and copied to
    the output directory. All the other files (text/headers/scripts) are just
    copied directly.

    This tool assumes that the input roots have the same structures and files
    as they should be results from building the same project to different
    architectures

    '''

    LIPO_CMD = 'lipo'
    FILE_CMD = 'file'

    def __init__(self, output_root):
        '''
        @output_root: the output directory where the result will be generated

        '''
        self.output_root = output_root
        if self.output_root.endswith('/'):
            self.output_root = self.output_root[:-1]
        self.missing = []

    def merge_files(self, filelist, dirs):
        if len(filelist) == 0:
            return
        for f in filelist:
            self.do_merge(f, dirs)

    def merge_dirs(self, input_roots):
        if not os.path.exists(self.output_root):
            os.mkdir(self.output_root)
        self.parse_dirs(input_roots)

    def create_universal_file(self, output, inputlist, dirs):
        tmp_inputs = []
        # relocate all files with the prefix of the merged file.
        # which must be done before merging them.
        for f in inputlist:
            # keep the filename in the suffix to preserve the filename extension
            tmp = tempfile.NamedTemporaryFile(suffix=os.path.basename(f))
            tmp_inputs.append(tmp)
            shutil.copy(f, tmp.name)
            prefix_to_replace = [d for d in dirs if d in f][0]
            relocator = OSXRelocator (self.output_root, prefix_to_replace, self.output_root,
                                      False)
            # since we are using a temporary file, we must force the library id
            # name to real one and not based on the filename
            relocator.relocate_file(tmp.name,
                    id=f.replace(prefix_to_replace, self.output_root))
        cmd = '%s -create %s -output %s' % (self.LIPO_CMD,
            ' '.join([f.name for f in tmp_inputs]), output)
        self._call(cmd)

    def get_file_type(self, filepath):
        cmd = '%s -bh "%s"' % (self.FILE_CMD, filepath)
        return self._call(cmd)[0:-1] #remove trailing \n

    def _detect_merge_action(self, files_list):
        actions = []
        for f in files_list:
            if not os.path.exists(f):
                continue #TODO what can we do here? fontconfig has
                         #some random generated filenames it seems
            ftype = self.get_file_type(f)
            action = ''
            for ft in file_types:
                if ft[0] in ftype:
                    if ft[0] == 'text' and f.endswith('.pc'):
                        action = 'copy-pc'
                    else:
                        action = ft[1]
                    break
            if not action:
                raise Exception, 'Unexpected file type %s %s' % (str(ftype), f)
            actions.append(action)
        if len(actions) == 0:
            return 'skip' #we should skip this one, the file doesn't exist
        all_same = all(x == actions[0] for x in actions)
        if not all_same:
            raise Exception, 'Different file types found: %s : %s' \
                             % (str(ftype), str(files_list))
        return actions[0]

    def do_merge(self, filepath, dirs):
        full_filepaths = [os.path.join(d, filepath) for d in dirs]
        action = self._detect_merge_action(full_filepaths)

        #pick the first file as the base one in case of copying/linking
        current_file = full_filepaths[0]
        output_file = os.path.join(self.output_root, filepath)
        output_dir = os.path.dirname(output_file)

        if action == 'copy':
            self._copy(current_file, output_file)
        elif action == 'copy-la' or action == 'copy-pc':
            self._copy_and_replace_paths(current_file, output_file, dirs)
        elif action == 'link':
            self._link(current_file, output_file, filepath)
        elif action == 'merge':
            if not os.path.exists(output_dir):
                os.makedirs(output_dir)
            self.create_universal_file(output_file, full_filepaths, dirs)
        elif action == 'skip':
            pass #just pass
        else:
            raise Exception, 'unexpected action %s' % action

    def parse_dirs(self, dirs, filters=None):
        self.missing = []

        dir_path = dirs[0]
        if dir_path.endswith('/'):
            dir_path = dir_path[:-1]
        for dirpath, dirnames, filenames in os.walk(dir_path):
            current_dir = ''
            token = ' '
            remaining_dirpath = dirpath
            while remaining_dirpath != dir_path or token == '':
                remaining_dirpath, token = os.path.split(remaining_dirpath)
                current_dir = os.path.join(token, current_dir)

            for f in filenames:
                if filters is not None and os.path.splitext(f)[1] not in filters:
                    continue
                current_file = os.path.join(current_dir, f)
                self.do_merge(current_file, dirs)

    def _copy(self, src, dest):
        if not os.path.exists(os.path.dirname(dest)):
            os.makedirs(os.path.dirname(dest))
        shutil.copy(src, dest)

    def _copy_and_replace_paths(self, src, dest, dirs):
        self._copy(src, dest)
        replacements = {}
        for d in dirs:
            replacements[d]=self.output_root
        shell.replace(dest, replacements)

    def _link(self, src, dest, filepath):
        if not os.path.exists(os.path.dirname(dest)):
            os.makedirs(os.path.dirname(dest))
        if os.path.lexists(dest):
            return #link exists, skip it

        # read the link, and extract the relative filepath
        target = os.readlink(src)

        # if it's a relative path use it directly
        if not os.path.isabs(target):
            os.symlink(target, dest)
            return

        # if it's an absolute path replace the prefix
        target = os.path.join(os.path.dirname(src), target)
        src_prefix = src.split(filepath)[0]
        dest_prefix = dest.split(filepath)[0]
        rel_target = os.path.relpath(target, src_prefix)
        dest_target = os.path.join(dest_prefix, rel_target)
        os.symlink(dest_target, dest)

    def _call(self, cmd, cwd=None):
        cmd = cmd or self.root
        process = subprocess.Popen(cmd, cwd=cwd,
            stdout=subprocess.PIPE, shell=True)
        output, unused_err = process.communicate()
        return output

class Main(object):

    def run(self):
        # We use OptionParser instead of ArgumentsParse because this script might
        # be run in OS X 10.6 or older, which do not provide the argparse module
        import optparse
        usage = "usage: %prog [options] outputdir inputdir1 inputdir2 ..."
        description='Merges multiple architecture build trees into a single '\
                    'universal binary build tree'
        parser = optparse.OptionParser(usage=usage, description=description)
        options, args = parser.parse_args()
        if len(args) < 3:
            parser.print_usage()
            exit(1)
        generator = OSXUniversalGenerator(args[0])
        generator.merge_dirs(args[1:])
        exit(0)

if __name__ == "__main__":
    main = Main()
    main.run()
<|endoftext|>


<commit_before>import numpy

import chainer
from chainer import cuda
from chainer import function
from chainer.utils import type_check

def _as_mat(x):
    if x.ndim == 2:
        return x
    return x.reshape(len(x), -1)

class Dropconnect(function.Function):

    """Linear unit regularized by dropconnect."""

    def __init__(self, ratio, mask=None):
        self.ratio = ratio
        self.mask = mask

    def check_type_forward(self, in_types):
        n_in = in_types.size()
        type_check.expect(2 <= n_in, n_in <= 3)
        x_type, w_type = in_types[:2]

        type_check.expect(
            x_type.dtype.kind == 'f',
            w_type.dtype.kind == 'f',
            x_type.ndim >= 2,
            w_type.ndim == 2,
            type_check.prod(x_type.shape[1:]) == w_type.shape[1],
        )
        if n_in.eval() == 3:
            b_type = in_types[2]
            type_check.expect(
                b_type.dtype == x_type.dtype,
                b_type.ndim == 1,
                b_type.shape[0] == w_type.shape[0],
            )

    def forward(self, inputs):
        scale = inputs[1].dtype.type(1. / (1 - self.ratio))
        xp = cuda.get_array_module(*inputs)
        if self.mask is None:
            if xp == numpy:
                flag = xp.random.rand(*inputs[1].shape) >= self.ratio
            else:
                flag = xp.random.rand(*inputs[1].shape,
                                      dtype=numpy.float32) >= self.ratio
            self.mask = scale * flag
        elif isinstance(self.mask, chainer.Variable):
            self.mask = self.mask.data

        x = _as_mat(inputs[0])
        W = inputs[1] * self.mask
        y = x.dot(W.T).astype(x.dtype, copy=False)
        if len(inputs) == 3:
            b = inputs[2]
            y += b
        return y,

    def backward(self, inputs, grad_outputs):
        x = _as_mat(inputs[0])
        W = inputs[1] * self.mask
        gy = grad_outputs[0]

        gx = gy.dot(W).astype(x.dtype, copy=False).reshape(inputs[0].shape)
        gW = gy.T.dot(x).astype(W.dtype, copy=False) * self.mask
        if len(inputs) == 3:
            gb = gy.sum(0)
            return gx, gW, gb
        else:
            return gx, gW

def dropconnect(x, W, b=None, ratio=.5, train=True, mask=None):
    """Linear unit regularized by dropconnect.

    Dropconnect drops weight matrix elements randomly with probability
    ``ratio`` and scales the remaining elements by factor ``1 / (1 - ratio)``.
    It accepts two or three arguments: an input minibatch ``x``, a weight
    matrix ``W``, and optionally a bias vector ``b``. It computes
    :math:`Y = xW^\\top + b`.

    In testing mode, zero will be used as dropconnect ratio instead of
    ``ratio``.

    Args:
        x (chainer.Variable or :class:`numpy.ndarray` or cupy.ndarray):
            Input variable. Its first dimension is assumed
            to be the *minibatch dimension*. The other dimensions are treated
            as concatenated one dimension whose size must be ``N``.
        W (~chainer.Variable): Weight variable of shape ``(M, N)``.
        b (~chainer.Variable): Bias variable (optional) of shape ``(M,)``.
        ratio (float):
            Dropconnect ratio.
            If ``mask`` is not ``None``, this value is ignored.
        train (bool):
            If ``True``, executes dropconnect.
            Otherwise, dropconnect function works as a linear function.
        mask (chainer.Variable or :class:`numpy.ndarray` or cupy.ndarray):
            If ``None``, randomized dropconnect mask is generated.
            If not ``None``, this value is used as a dropconnect mask.
            Scaling will not be executed.
            The mask shape must be ``(M, N)``.
            Main purpose of the latter option is debugging.

    Returns:
        ~chainer.Variable: Output variable.

    .. seealso:: :class:`~chainer.links.Dropconnect`

    """
    if not train:
        ratio = 0
    if b is None:
        return Dropconnect(ratio, mask)(x, W)
    else:
        return Dropconnect(ratio, mask)(x, W, b)
<commit_msg>Add doc about defference with original dropconnect.<commit_after>import numpy

import chainer
from chainer import cuda
from chainer import function
from chainer.utils import type_check

def _as_mat(x):
    if x.ndim == 2:
        return x
    return x.reshape(len(x), -1)

class Dropconnect(function.Function):

    """Linear unit regularized by dropconnect."""

    def __init__(self, ratio, mask=None):
        self.ratio = ratio
        self.mask = mask

    def check_type_forward(self, in_types):
        n_in = in_types.size()
        type_check.expect(2 <= n_in, n_in <= 3)
        x_type, w_type = in_types[:2]

        type_check.expect(
            x_type.dtype.kind == 'f',
            w_type.dtype.kind == 'f',
            x_type.ndim >= 2,
            w_type.ndim == 2,
            type_check.prod(x_type.shape[1:]) == w_type.shape[1],
        )
        if n_in.eval() == 3:
            b_type = in_types[2]
            type_check.expect(
                b_type.dtype == x_type.dtype,
                b_type.ndim == 1,
                b_type.shape[0] == w_type.shape[0],
            )

    def forward(self, inputs):
        scale = inputs[1].dtype.type(1. / (1 - self.ratio))
        xp = cuda.get_array_module(*inputs)
        if self.mask is None:
            if xp == numpy:
                flag = xp.random.rand(*inputs[1].shape) >= self.ratio
            else:
                flag = xp.random.rand(*inputs[1].shape,
                                      dtype=numpy.float32) >= self.ratio
            self.mask = scale * flag
        elif isinstance(self.mask, chainer.Variable):
            self.mask = self.mask.data

        x = _as_mat(inputs[0])
        W = inputs[1] * self.mask
        y = x.dot(W.T).astype(x.dtype, copy=False)
        if len(inputs) == 3:
            b = inputs[2]
            y += b
        return y,

    def backward(self, inputs, grad_outputs):
        x = _as_mat(inputs[0])
        W = inputs[1] * self.mask
        gy = grad_outputs[0]

        gx = gy.dot(W).astype(x.dtype, copy=False).reshape(inputs[0].shape)
        gW = gy.T.dot(x).astype(W.dtype, copy=False) * self.mask
        if len(inputs) == 3:
            gb = gy.sum(0)
            return gx, gW, gb
        else:
            return gx, gW

def dropconnect(x, W, b=None, ratio=.5, train=True, mask=None):
    """Linear unit regularized by dropconnect.

    Dropconnect drops weight matrix elements randomly with probability
    ``ratio`` and scales the remaining elements by factor ``1 / (1 - ratio)``.
    It accepts two or three arguments: an input minibatch ``x``, a weight
    matrix ``W``, and optionally a bias vector ``b``. It computes
    :math:`Y = xW^\\top + b`.

    In testing mode, zero will be used as dropconnect ratio instead of
    ``ratio``.

    There are two differences between the current implementation and the
    original version dropconnect.
    The original version uses a different mask for each sample in the mini
    batch, but the current implementation uses a single same mask.
    In addition, not approximation by Gaussian distribution as original
    version, current implementation uses approximation by averaging before
    activation at inference time.

    Args:
        x (chainer.Variable or :class:`numpy.ndarray` or cupy.ndarray):
            Input variable. Its first dimension is assumed
            to be the *minibatch dimension*. The other dimensions are treated
            as concatenated one dimension whose size must be ``N``.
        W (~chainer.Variable): Weight variable of shape ``(M, N)``.
        b (~chainer.Variable): Bias variable (optional) of shape ``(M,)``.
        ratio (float):
            Dropconnect ratio.
            If ``mask`` is not ``None``, this value is ignored.
        train (bool):
            If ``True``, executes dropconnect.
            Otherwise, dropconnect function works as a linear function.
        mask (chainer.Variable or :class:`numpy.ndarray` or cupy.ndarray):
            If ``None``, randomized dropconnect mask is generated.
            If not ``None``, this value is used as a dropconnect mask.
            Scaling will not be executed.
            The mask shape must be ``(M, N)``.
            Main purpose of the latter option is debugging.

    Returns:
        ~chainer.Variable: Output variable.

    .. seealso:: :class:`~chainer.links.Dropconnect`

    """
    if not train:
        ratio = 0
    if b is None:
        return Dropconnect(ratio, mask)(x, W)
    else:
        return Dropconnect(ratio, mask)(x, W, b)
<|endoftext|>


<commit_before>'''
 _ __ ___  ___ _ __   ___  _ __  ___  ___   _ __  _   _
| '__/ _ \/ __| '_ \ / _ \| '_ \/ __|/ _ \ | '_ \| | | |
| | |  __/\__ \ |_) | (_) | | | \__ \  __/_| |_) | |_| |
|_|  \___||___/ .__/ \___/|_| |_|___/\___(_) .__/ \__, |
              |_|                          |_|    |___/
'''

from contextlib import contextmanager
from threading import local

# We _really_ want _everything_ from `requests`.
from requests import *

__title__     = 'response'
__author__    = 'Genadi Samokovarov'
__copyright__ = 'Copyrignt 2012 Genadi Samokovarov'

class ResponseStack(local):
    '''
    Thread-local stack for responses.
    '''

    def __init__(self):
        self.content = list()

    def top(self):
        return self.content[-1] if self.content else None

    def push(self, response):
        self.content.append(response)

    def pop(self):
        try:
            return self.top()
        finally:
            del self.content[-1]

    __iter__    = lambda self: iter(self.content)
    __len__     = lambda self: len(self.content)
    __nonzero__ = lambda self: bool(self.content)
    __bool__    = __nonzero__

class ResponseProxy(local):
    '''
    Thread-local response objects proxy.
    '''

    def __init__(self, response_factory):
        self.response_factory = response_factory

    @property
    def response(self):
        return self.response_factory()

    @property
    def __class__(self):
        return self.response.__class__

    @property
    def __dict__(self):
        return self.response.__dict__

    __getattr__ = lambda self, name: getattr(self.response, name)
    __repr__    = lambda self: repr(self.response)
    __nonzero__ = lambda self: bool(self.response)
    __bool__    = __nonzero__

def convert_to_context_manager(func):
    '''
    Converts a `requests` API function to a context manager.
    '''

    def wrapper(*args, **kw):
        response = func(*args, **kw)

        try:
            yield responses.push(response)
        finally:
            responses.pop()

    return contextmanager(wrapper)

get     = convert_to_context_manager(get)
options = convert_to_context_manager(options)
head    = convert_to_context_manager(head)
post    = convert_to_context_manager(post)
put     = convert_to_context_manager(put)
patch   = convert_to_context_manager(patch)
delete  = convert_to_context_manager(delete)

request = convert_to_context_manager(request)

responses = ResponseStack()
response  = ResponseProxy(responses.top)
last      = response
<commit_msg>Minor refactor - style and whitespace changes<commit_after>'''
 _ __ ___  ___ _ __   ___  _ __  ___  ___   _ __  _   _
| '__/ _ \/ __| '_ \ / _ \| '_ \/ __|/ _ \ | '_ \| | | |
| | |  __/\__ \ |_) | (_) | | | \__ \  __/_| |_) | |_| |
|_|  \___||___/ .__/ \___/|_| |_|___/\___(_) .__/ \__, |
              |_|                          |_|    |___/
'''

from contextlib import contextmanager
from threading import local

# We _really_ want _everything_ from `requests`.
from requests import *

__title__     = 'response'
__author__    = 'Genadi Samokovarov'
__copyright__ = 'Copyrignt 2012 Genadi Samokovarov'

class ResponseStack(local):
    '''
    Thread-local stack for responses.
    '''

    def __init__(self):
        self.content = []

    def top(self):
        if self:
            return self.content[-1]

    def push(self, response):
        self.content.append(response)

    def pop(self):
        if self:
            try:
                return self.top()
            finally:
                del self.content[-1]

    __iter__    = lambda self: iter(self.content)
    __len__     = lambda self: len(self.content)
    __nonzero__ = lambda self: bool(self.content)
    __bool__    = __nonzero__

class ResponseProxy(local):
    '''
    Thread-local response objects proxy.
    '''

    def __init__(self, response_factory):
        self.response_factory = response_factory

    @property
    def response(self):
        return self.response_factory()

    @property
    def __class__(self):
        return self.response.__class__

    @property
    def __dict__(self):
        return self.response.__dict__

    __getattr__ = lambda self, name: getattr(self.response, name)
    __repr__    = lambda self: repr(self.response)
    __nonzero__ = lambda self: bool(self.response)
    __bool__    = __nonzero__

def convert_to_context_manager(func):
    '''
    Converts a `requests` API function to a context manager.
    '''

    def wrapper(*args, **kw):
        response = func(*args, **kw)

        try:
            yield responses.push(response)
        finally:
            responses.pop()

    return contextmanager(wrapper)

get     = convert_to_context_manager(get)
options = convert_to_context_manager(options)
head    = convert_to_context_manager(head)
post    = convert_to_context_manager(post)
put     = convert_to_context_manager(put)
patch   = convert_to_context_manager(patch)
delete  = convert_to_context_manager(delete)

request = convert_to_context_manager(request)

responses = ResponseStack()
response  = ResponseProxy(responses.top)
last      = response
<|endoftext|>


<commit_before>from django.db import DatabaseError
from django.forms import ModelForm, Field, ValidationError, BooleanField, CharField
from django.forms.widgets import CheckboxInput, Select

from explorer import app_settings
from explorer.models import Query, MSG_FAILED_BLACKLIST

class SqlField(Field):

    def validate(self, value):
        """
        Ensure that the SQL passes the blacklist.

        :param value: The SQL for this Query model.
        """

        query = Query(sql=value)

        passes_blacklist, failing_words = query.passes_blacklist()

        error = MSG_FAILED_BLACKLIST % ', '.join(failing_words) if not passes_blacklist else None

        if error:
            raise ValidationError(
                error,
                code="InvalidSql"
            )

class QueryForm(ModelForm):

    sql = SqlField()
    snapshot = BooleanField(widget=CheckboxInput, required=False)
    connection = CharField(widget=Select, required=False)

    def __init__(self, *args, **kwargs):
        super(QueryForm, self).__init__(*args, **kwargs)
        self.fields['connection'].widget.choices = self.connections
        if not self.instance.connection:
            self.initial['connection'] = app_settings.EXPLORER_DEFAULT_CONNECTION
        self.fields['connection'].widget.attrs['class'] = 'form-control'

    def clean(self):
        if self.instance and self.data.get('created_by_user', None):
            self.cleaned_data['created_by_user'] = self.instance.created_by_user
        return super(QueryForm, self).clean()

    @property
    def created_by_user_email(self):
        return self.instance.created_by_user.email if self.instance.created_by_user else '--'

    @property
    def created_at_time(self):
        return self.instance.created_at.strftime('%Y-%m-%d')

    @property
    def connections(self):
        return zip(app_settings.EXPLORER_CONNECTIONS.values(), app_settings.EXPLORER_CONNECTIONS.keys())

    class Meta:
        model = Query
        fields = ['title', 'sql', 'description', 'snapshot', 'connection']
<commit_msg>change QueryForm.connections property to return a list so it can be iterated over multiple times<commit_after>from django.db import DatabaseError
from django.forms import ModelForm, Field, ValidationError, BooleanField, CharField
from django.forms.widgets import CheckboxInput, Select

from explorer import app_settings
from explorer.models import Query, MSG_FAILED_BLACKLIST

class SqlField(Field):

    def validate(self, value):
        """
        Ensure that the SQL passes the blacklist.

        :param value: The SQL for this Query model.
        """

        query = Query(sql=value)

        passes_blacklist, failing_words = query.passes_blacklist()

        error = MSG_FAILED_BLACKLIST % ', '.join(failing_words) if not passes_blacklist else None

        if error:
            raise ValidationError(
                error,
                code="InvalidSql"
            )

class QueryForm(ModelForm):

    sql = SqlField()
    snapshot = BooleanField(widget=CheckboxInput, required=False)
    connection = CharField(widget=Select, required=False)

    def __init__(self, *args, **kwargs):
        super(QueryForm, self).__init__(*args, **kwargs)
        self.fields['connection'].widget.choices = self.connections
        if not self.instance.connection:
            self.initial['connection'] = app_settings.EXPLORER_DEFAULT_CONNECTION
        self.fields['connection'].widget.attrs['class'] = 'form-control'

    def clean(self):
        if self.instance and self.data.get('created_by_user', None):
            self.cleaned_data['created_by_user'] = self.instance.created_by_user
        return super(QueryForm, self).clean()

    @property
    def created_by_user_email(self):
        return self.instance.created_by_user.email if self.instance.created_by_user else '--'

    @property
    def created_at_time(self):
        return self.instance.created_at.strftime('%Y-%m-%d')

    @property
    def connections(self):
        return [(v, k) for k, v in app_settings.EXPLORER_CONNECTIONS.items()]

    class Meta:
        model = Query
        fields = ['title', 'sql', 'description', 'snapshot', 'connection']
<|endoftext|>


<commit_before>__description__ = "general utilities"
__author__ = "Reed Essick (reed.essick@ligo.org)"

#-------------------------------------------------

import os
import gzip

import numpy as np

#-------------------------------------------------

DEFAULT_ACCOUNTING_GROUP = 'ligo.dev.o1.burst.explore.test'
DEFAULT_RETRY = 0

#-------------------------------------------------

### define template strings here for easy reference between multiple scripts!
compute_horizon_sub = '''\
universe = %(universe)s
executable = %(exe)s
arguments = "$(path) --output-dir $(outdir) $(tag) --flow %(flow)d --fhigh %(fhigh)d --distance %(distance)f --snr-thr %(snr_thr)f -v"
getenv = true
accounting_group = %(accounting_group)s
accounting_group_user = %(accounting_group_user)s
log    = $(outdir)/condor-compute_horizon%(filetag)s_$(Cluster)-$(Process).log
error  = $(outdir)/condor-compute_horizon%(filetag)s_$(Cluster)-$(Process).err
output = $(outdir)/condor-compute_horizon%(filetag)s_$(Cluster)-$(Process).out
notification = never
queue 1'''

compute_horizon_jobid = "compute_horizon_%s"
compute_horizon_dag = '''\
JOB    compute_horizon_%(jobid)s %(sub)s
VARS   compute_horizon_%(jobid)s path="%(path)s" tag="%(tag)s" outdir="%(outdir)s"
RETRY  compute_horizon_%(jobid)s %(retry)d
'''

compute_network_sensitivity_sub = '''\
universe = %(universe)s
executable = %(exe)s
arguments = "$(gps) $(ifo_horizons) --nside %(nside)d --output-dir $(outdir) --tag $(tag) -V"
getenv = true
accounting_group = %(accounting_group)s
accounting_group_user = %(accounting_group_user)s
log    = $(outdir)/condor-compute_network_sensitivity%(filetag)s_$(Cluster)-$(Process).log
error  = $(outdir)/condor-compute_network_sensitivity%(filetag)s_$(Cluster)-$(Process).err
output = $(outdir)/condor-compute_network_sensitivity%(filetag)s_$(Cluster)-$(Process).out
notification = never
queue 1'''

compute_network_sensitivity_jobid = "compute_network_sensitivity_%s"
compute_network_sensitivity_dag = '''\
JOB    compute_network_sensitivity_%(jobid)s %(sub)s
VARS   compute_network_sensitivity_%(jobid)s gps="%(gps)d" tag="%(tag)s" outdir="%(outdir)s" ifo_horizons="%(ifo_horizons)s"
RETRY  compute_network_sensitivity_%(jobid)s %(retry)d
'''

compute_psd_sub = '''\
universe = %(universe)s
executable = %(exe)s
arguments = "%(channel)s %(frametype)s $(gpsstart) $(gpsstop) --seglen %(seglen)d --overlap %(overlap)d --tukey-alpha %(tukey_alpha)f --output-dir $(outdir) %(tag)s -V"
getenv = true
accounting_group = %(accounting_group)s
accounting_group_user = %(accounting_group_user)s
log    = $(outdir)/condor-compute_psd%(filetag)s_$(Cluster)-$(Process).log
error  = $(outdir)/condor-compute_psd%(filetag)s_$(Cluster)-$(Process).err
output = $(outdir)/condor-compute_psd%(filetag)s_$(Cluster)-$(Process).out
notification = never
queue 1'''

compute_psd_jobid = "compute_psd_%s"
compute_psd_dag = '''\
JOB    compute_psd_%(jobid)s %(sub)s
VARS   compute_psd_%(jobid)s gpsstart="%(gpsstart)d" gpsstop="%(gpsstop)d" outdir="%(outdir)s"
RETRY  compute_psd_%(jobid)s %(retry)d
'''

plot_psd_sub = '''\
universe = %(universe)s
executable = %(exe)s
arguments = "$(psd) --f-min %(fmin)d --f-max %(fmax)d --y-min %(ymin)e --y-max %(ymax)e --output-dir %(outdir)s %(tag)s -v"
getenv = true
accounting_group = %(accounting_group)s
accounting_group_user = %(accounting_group_user)s
log    = %(outdir)s/condor-compute_psd%(filetag)s_$(Cluster)-$(Process).log
error  = %(outdir)s/condor-compute_psd%(filetag)s_$(Cluster)-$(Process).err
output = %(outdir)s/condor-compute_psd%(filetag)s_$(Cluster)-$(Process).out
notification = never
queue 1'''

plot_psd_jobid = "plot_psd_%s"
plot_psd_dag = '''\
JOB    plot_psd_%(jobid)s %(sub)s
VARS   plot_psd_%(jobid)s psd="%(path)s"
RETRY  plot_psd_%(jobid)s %(retry)d
'''

compute_exposure_sub = '''\
universe = %(universe)s
executable = %(exe)s
arguments = "$(FITS) $(normalize) --nside %(nside)d --index %(index)d --output-dir %(outdir)s %(tag)s"
getenv = true
accounting_group = %(accounting_group)s
accounting_group_user = %(accounting_group_user)s
log    = %(outdir)s/condor-compute_exposure%(filetag)s_$(Cluster)-$(Process).log
error  = %(outdir)s/condor-compute_exposure%(filetag)s_$(Cluster)-$(Process).err
output = %(outdir)s/condor-compute_exposure%(filetag)s_$(Cluster)-$(Process).out
notification = never
queue 1'''

compute_exposure_jobid = " compute_exposure_%s"
compute_exposure_dag = '''\
JOB    compute_exposure_%(jobid)s %(sub)s
VARS   compute_exposure_%(jobid)s FITS="%(FITS)s" normalize="%(normalize)s"
RETRY  compute_exposure_%(jobid)s %(retry)d
'''

plot_maps_sub = '''\
universe = %(universe)s
executable = %(exe)s
arguments = "$(base),$(path) --coord %(coord)s --grid --projection %(projection)s $(tag) -v"
getenv = true
accounting_group = %(accounting_group)s
accounting_group_user = %(accounting_group_user)s
log    = $(outdir)/condor-plot_maps%(filetag)s_$(Cluster)-$(Process).log
error  = $(outdir)/condor-plot_maps%(filetag)s_$(Cluster)-$(Process).err
output = $(outdir)/condor-plot_maps%(filetag)s_$(Cluster)-$(Process).out
notification = never
queue 1'''

plot_maps_jobid = "compute_horizon_%s"
plot_maps_dag = '''\
JOB    compute_horizon_%(jobid)s %(sub)s
VARS   compute_horizon_%(jobid)s path="%(path)s" tag="%(tag)s"
RETRY  compute_horizon_%(jobid)s %(retry)d
'''

monte_carlo_vt_sub = '''\
universe = %(universe)s
executable = %(exe)s
arguments = "--verbose %(population)s $(gpsstart) $(gpsstop) --snr-threshold %(snr_threshold).6f --min-num-samples %(min_num_samples)d --fractional-systematic-error %(error).6f $(sources) --output-dir $(outdir) %(tag)s"
getenv = true
accounting_group = %(accounting_group)s
accounting_group_user = %(accounting_group_user)s
log    = $(outdir)/condor-monte-carlo-vt%(filetag)s_$(Cluster)-$(Process).log
error  = $(outdir)/condor-monte-carlo-vt%(filetag)s_$(Cluster)-$(Process).err
output = $(outdir)/condor-monte-carlo-vt%(filetag)s_$(Cluster)-$(Process).out
notification = never
queue 1'''

monte_carlo_vt_jobid = "monte_carlo_vt_%s"
monte_carlo_vt_dag = '''\
JOB    monte_carlo_vt_%(jobid)s %(sub)s
VARS   monte_carlo_vt_%(jobid)s gpsstart="%(gpsstart)d" gpsstop="%(gpsstop)d" outdir="%(outdir)s" sources="%(sources)s"
RETRY  monte_carlo_vt_%(jobid)s %(retry)d
'''

#-------------------------------------------------

MOD_STRIDE = 100000

def segs_path(output_dir, tag, gpsstart, gpsdur):
    return "%s/seg%s-%d-%d.txt.gz"%(gps2moddir(output_dir, gpsstart), tag, (int(gpsstart)/MOD_STRIDE)*MOD_STRIDE, MOD_STRIDE)

def psd_path(output_dir, tag, gpsstart, gpsdur):
    return "%s/psd%s-%d-%d.csv.gz"%(gps2dir(output_dir, gpsstart, gpsdur), tag, gpsstart, gpsdur)

def samples_path(output_dir, tag, gpsstart, gpsdur):
    return "%s/samples%s-%d-%d.csv.gz"%(gps2dir(output_dir, gpsstart, gpsdur), tag, gpsstart, gpsdur)

def horizon_path(output_dir, tag, gpstart, gpsdur):
    return "%s/horizon%s-%d-%d.txt.gz"%(output_dir, tag, gpstart, gpsdur)

def sensitivity_path(output_dir, tag, gps, gzip=False):
    ans = "%s/network_sensitivity%s-%d.fits"%(output_dir, tag, gps)
    if gzip:
        ans = ans+".gz"
    return ans

def gps2moddir(directory, start):
    return "%s/%d/"%(directory, int(start)/MOD_STRIDE)

def gps2dir(directory, start, dur):
    return "%s/%d-%d/"%(gps2moddir(directory, start), start, dur)

#-------------------------------------------------

def extract_start_dur(path, suffix='.gwf'):
    return [int(_) for _ in path[:-len(suffix)].split('-')[-2:]]

def report_psd(path, freqs, psd):
    """writes the PSD to disk
    """
    ### save to a temporary file
    tmp = os.path.join(os.path.dirname(path), '.'+os.path.basename(path))
    np.savetxt(tmp, np.array(zip(freqs, psd)), comments='', delimiter=',', header='frequency,psd')
    ### move to the final location
    os.system('mv %s %s'%(tmp, path))

def retrieve_psd(path):
    """read the PSD from disk
    return freqs, psd
    """
    ans = np.genfromtxt(path, delimiter=',', names=True)
    return ans['frequency'], ans['psd']

def report_segs(path, new_segs):
    """reads in the segments contained in path and appends the current seg of segs to them
    assumes you are only adding segments in time order, so it will either start a new segment or merge the last segment in the file
    """
    if len(new_segs): ### there must be something to report...
        new_segs = [(a,b) for a, b in new_segs] ### make a copy so we don't mess up a shared reference

        ### read in and merge segs if necessary
        if os.path.exists(path):
            segs = retrieve_segs(path)
            if len(segs):
                if np.ndim(segs)==1: ### a single segment
                    segs = [segs]

                while len(segs) and (segs[-1][0] > new_segs[0][1]): ### remove existing segs that are redundant with where new_segs starts
                    segs.pop(-1)

                if len(segs):
                    if (segs[-1][1] >= new_segs[0][0]): ### these things overlap
                        segs[-1][1] = new_segs.pop(0)[1] ### remove the first element from segs

                segs = segs+list(new_segs)

            else: ### either empty or a single se
                segs = new_segs

        else:
            segs = new_segs

        ### write the result back out to a temporary location
        tmp = os.path.join(os.path.dirname(path), '.'+os.path.basename(path))
        np.savetxt(tmp, segs, fmt='%d')
        ### move to final location
        os.system('mv %s %s'%(tmp, path))

def retrieve_segs(path):
    """retrieves segments from disk
    """
    return list(np.loadtxt(path))

def report_horizon(path, horizon, flow, fhigh, psd_path):
    """report the horizon to disk
    """
    with open(path, 'w') as f:
        f.write('%.9e\n%d\n%d\n%s'%(horizon, flow, fhigh, psd_path))

def retrieve_horizon(path):
    """retrieve the horizon from disk
    """
    with open(path, 'r') as f:
        horizon = float(f.readline().strip())
        flow = float(f.readline().strip())
        fhigh = float(f.readlin().strip())
        psd_path = f.readline().strip()
    return horizon, flow, fhigh, psd_path

def report_sensitivity(path, sensitivity, *extra_header):
    basepath = path.strip('.gz')
    hp.write_map(
        basepath,
        network,
        coord='C',
        extra_header=extra_header,
    )

    if path.endswith('.gz'):
        if opts.verbose:
            print( "gzipping : %s -> %s"%(basepath, path) )
        with open(basepath, 'r') as f:
            with gzip.open(path, 'w') as gzf:
                gzf.write(f.read())

#-------------------------------------------------

def livetime(segs):
    return np.sum(e-s for s, e in segs)

def andsegments(list1, list2):
    """
    computes the intersection of 2 lists of segments, returning a new list
    taken from laldetchar.idq.event.andtwosegmentlists
    """
    newsegments = []
    index = 0
    for seg in list1:
        while index > 0 and list2[index][0] > seg[0]:
            index -= 1
        while index < len(list2) and list2[index][1] <= seg[0]:
            index += 1
        while index < len(list2) and list2[index][0] < seg[1]:
            newsegments.append([max(seg[0], list2[index][0]), min(seg[1], list2[index][1])])
            index += 1
        if index > 0:
            index -= 1
    return newsegments

def invsegments(start, stop, segs):
    """
    takes the inverse of a list of segments between start and stop
    assumes that all segments are already between start and stop
    """
    if not segs:
        return [[start, stop]]

    newsegments = []
    for s, e in segs:
        if start < s <= stop:
            newsegments.append( [start, s] )
        start = e
    if e < stop:
        newsegments.append( [e, stop] )

    return newsegments

def mergesegments(segments):
    """
    assumes segments are in the correct order and non-overlapping.
    simply merges any touching segments into one bigger segemnt
    """
    if len(segments)<2:
        return segments

    segs = []
    s, e = segments[0]
    for S, E in segments[1:]:
        if e==S:
            e=E
        else:
            segs.append( [s,e] )
            s = S
            e = E
    segs.append( [s, e] )
    return segs

def checksegments(seg, window=None):
    """
    ensures segment makes sense
    if provided, segment is sliced so that we only keep the part that interesects with window=[start, end]
    """
    s, e = seg
    if window:
        start, end = window
        if s < start:
            s = start # truncate the start of seg
        elif not (s < end):
            return False # no overlap between current segment and window
        if e > end:
            e = end # truncate the end of seg
        elif not (e > start):
            return False # no overlap between currnet segment and window

    if s < e:
        return [s,e]
    elif s > e:
        raise ValueError("something is very wrong with segment generation... seg[1]=%.3f < seg[0]=%.3f"%tuple(seg))
    else:
        return False
<commit_msg>changed default accounting group to something more sensible<commit_after>__description__ = "general utilities"
__author__ = "Reed Essick (reed.essick@ligo.org)"

#-------------------------------------------------

import os
import gzip

import numpy as np

#-------------------------------------------------

DEFAULT_ACCOUNTING_GROUP = 'ligo.sim.o3.cbc.bayesianpopulations.parametric'
DEFAULT_RETRY = 0

#-------------------------------------------------

### define template strings here for easy reference between multiple scripts!
compute_horizon_sub = '''\
universe = %(universe)s
executable = %(exe)s
arguments = "$(path) --output-dir $(outdir) $(tag) --flow %(flow)d --fhigh %(fhigh)d --distance %(distance)f --snr-thr %(snr_thr)f -v"
getenv = true
accounting_group = %(accounting_group)s
accounting_group_user = %(accounting_group_user)s
log    = $(outdir)/condor-compute_horizon%(filetag)s_$(Cluster)-$(Process).log
error  = $(outdir)/condor-compute_horizon%(filetag)s_$(Cluster)-$(Process).err
output = $(outdir)/condor-compute_horizon%(filetag)s_$(Cluster)-$(Process).out
notification = never
queue 1'''

compute_horizon_jobid = "compute_horizon_%s"
compute_horizon_dag = '''\
JOB    compute_horizon_%(jobid)s %(sub)s
VARS   compute_horizon_%(jobid)s path="%(path)s" tag="%(tag)s" outdir="%(outdir)s"
RETRY  compute_horizon_%(jobid)s %(retry)d
'''

compute_network_sensitivity_sub = '''\
universe = %(universe)s
executable = %(exe)s
arguments = "$(gps) $(ifo_horizons) --nside %(nside)d --output-dir $(outdir) --tag $(tag) -V"
getenv = true
accounting_group = %(accounting_group)s
accounting_group_user = %(accounting_group_user)s
log    = $(outdir)/condor-compute_network_sensitivity%(filetag)s_$(Cluster)-$(Process).log
error  = $(outdir)/condor-compute_network_sensitivity%(filetag)s_$(Cluster)-$(Process).err
output = $(outdir)/condor-compute_network_sensitivity%(filetag)s_$(Cluster)-$(Process).out
notification = never
queue 1'''

compute_network_sensitivity_jobid = "compute_network_sensitivity_%s"
compute_network_sensitivity_dag = '''\
JOB    compute_network_sensitivity_%(jobid)s %(sub)s
VARS   compute_network_sensitivity_%(jobid)s gps="%(gps)d" tag="%(tag)s" outdir="%(outdir)s" ifo_horizons="%(ifo_horizons)s"
RETRY  compute_network_sensitivity_%(jobid)s %(retry)d
'''

compute_psd_sub = '''\
universe = %(universe)s
executable = %(exe)s
arguments = "%(channel)s %(frametype)s $(gpsstart) $(gpsstop) --seglen %(seglen)d --overlap %(overlap)d --tukey-alpha %(tukey_alpha)f --output-dir $(outdir) %(tag)s -V"
getenv = true
accounting_group = %(accounting_group)s
accounting_group_user = %(accounting_group_user)s
log    = $(outdir)/condor-compute_psd%(filetag)s_$(Cluster)-$(Process).log
error  = $(outdir)/condor-compute_psd%(filetag)s_$(Cluster)-$(Process).err
output = $(outdir)/condor-compute_psd%(filetag)s_$(Cluster)-$(Process).out
notification = never
queue 1'''

compute_psd_jobid = "compute_psd_%s"
compute_psd_dag = '''\
JOB    compute_psd_%(jobid)s %(sub)s
VARS   compute_psd_%(jobid)s gpsstart="%(gpsstart)d" gpsstop="%(gpsstop)d" outdir="%(outdir)s"
RETRY  compute_psd_%(jobid)s %(retry)d
'''

plot_psd_sub = '''\
universe = %(universe)s
executable = %(exe)s
arguments = "$(psd) --f-min %(fmin)d --f-max %(fmax)d --y-min %(ymin)e --y-max %(ymax)e --output-dir %(outdir)s %(tag)s -v"
getenv = true
accounting_group = %(accounting_group)s
accounting_group_user = %(accounting_group_user)s
log    = %(outdir)s/condor-compute_psd%(filetag)s_$(Cluster)-$(Process).log
error  = %(outdir)s/condor-compute_psd%(filetag)s_$(Cluster)-$(Process).err
output = %(outdir)s/condor-compute_psd%(filetag)s_$(Cluster)-$(Process).out
notification = never
queue 1'''

plot_psd_jobid = "plot_psd_%s"
plot_psd_dag = '''\
JOB    plot_psd_%(jobid)s %(sub)s
VARS   plot_psd_%(jobid)s psd="%(path)s"
RETRY  plot_psd_%(jobid)s %(retry)d
'''

compute_exposure_sub = '''\
universe = %(universe)s
executable = %(exe)s
arguments = "$(FITS) $(normalize) --nside %(nside)d --index %(index)d --output-dir %(outdir)s %(tag)s"
getenv = true
accounting_group = %(accounting_group)s
accounting_group_user = %(accounting_group_user)s
log    = %(outdir)s/condor-compute_exposure%(filetag)s_$(Cluster)-$(Process).log
error  = %(outdir)s/condor-compute_exposure%(filetag)s_$(Cluster)-$(Process).err
output = %(outdir)s/condor-compute_exposure%(filetag)s_$(Cluster)-$(Process).out
notification = never
queue 1'''

compute_exposure_jobid = " compute_exposure_%s"
compute_exposure_dag = '''\
JOB    compute_exposure_%(jobid)s %(sub)s
VARS   compute_exposure_%(jobid)s FITS="%(FITS)s" normalize="%(normalize)s"
RETRY  compute_exposure_%(jobid)s %(retry)d
'''

plot_maps_sub = '''\
universe = %(universe)s
executable = %(exe)s
arguments = "$(base),$(path) --coord %(coord)s --grid --projection %(projection)s $(tag) -v"
getenv = true
accounting_group = %(accounting_group)s
accounting_group_user = %(accounting_group_user)s
log    = $(outdir)/condor-plot_maps%(filetag)s_$(Cluster)-$(Process).log
error  = $(outdir)/condor-plot_maps%(filetag)s_$(Cluster)-$(Process).err
output = $(outdir)/condor-plot_maps%(filetag)s_$(Cluster)-$(Process).out
notification = never
queue 1'''

plot_maps_jobid = "compute_horizon_%s"
plot_maps_dag = '''\
JOB    compute_horizon_%(jobid)s %(sub)s
VARS   compute_horizon_%(jobid)s path="%(path)s" tag="%(tag)s"
RETRY  compute_horizon_%(jobid)s %(retry)d
'''

monte_carlo_vt_sub = '''\
universe = %(universe)s
executable = %(exe)s
arguments = "--verbose %(population)s $(gpsstart) $(gpsstop) --snr-threshold %(snr_threshold).6f --min-num-samples %(min_num_samples)d --fractional-systematic-error %(error).6f $(sources) --output-dir $(outdir) %(tag)s"
getenv = true
accounting_group = %(accounting_group)s
accounting_group_user = %(accounting_group_user)s
log    = $(outdir)/condor-monte-carlo-vt%(filetag)s_$(Cluster)-$(Process).log
error  = $(outdir)/condor-monte-carlo-vt%(filetag)s_$(Cluster)-$(Process).err
output = $(outdir)/condor-monte-carlo-vt%(filetag)s_$(Cluster)-$(Process).out
notification = never
queue 1'''

monte_carlo_vt_jobid = "monte_carlo_vt_%s"
monte_carlo_vt_dag = '''\
JOB    monte_carlo_vt_%(jobid)s %(sub)s
VARS   monte_carlo_vt_%(jobid)s gpsstart="%(gpsstart)d" gpsstop="%(gpsstop)d" outdir="%(outdir)s" sources="%(sources)s"
RETRY  monte_carlo_vt_%(jobid)s %(retry)d
'''

#-------------------------------------------------

MOD_STRIDE = 100000

def segs_path(output_dir, tag, gpsstart, gpsdur):
    return "%s/seg%s-%d-%d.txt.gz"%(gps2moddir(output_dir, gpsstart), tag, (int(gpsstart)/MOD_STRIDE)*MOD_STRIDE, MOD_STRIDE)

def psd_path(output_dir, tag, gpsstart, gpsdur):
    return "%s/psd%s-%d-%d.csv.gz"%(gps2dir(output_dir, gpsstart, gpsdur), tag, gpsstart, gpsdur)

def samples_path(output_dir, tag, gpsstart, gpsdur):
    return "%s/samples%s-%d-%d.csv.gz"%(gps2dir(output_dir, gpsstart, gpsdur), tag, gpsstart, gpsdur)

def horizon_path(output_dir, tag, gpstart, gpsdur):
    return "%s/horizon%s-%d-%d.txt.gz"%(output_dir, tag, gpstart, gpsdur)

def sensitivity_path(output_dir, tag, gps, gzip=False):
    ans = "%s/network_sensitivity%s-%d.fits"%(output_dir, tag, gps)
    if gzip:
        ans = ans+".gz"
    return ans

def gps2moddir(directory, start):
    return "%s/%d/"%(directory, int(start)/MOD_STRIDE)

def gps2dir(directory, start, dur):
    return "%s/%d-%d/"%(gps2moddir(directory, start), start, dur)

#-------------------------------------------------

def extract_start_dur(path, suffix='.gwf'):
    return [int(_) for _ in path[:-len(suffix)].split('-')[-2:]]

def report_psd(path, freqs, psd):
    """writes the PSD to disk
    """
    ### save to a temporary file
    tmp = os.path.join(os.path.dirname(path), '.'+os.path.basename(path))
    np.savetxt(tmp, np.array(zip(freqs, psd)), comments='', delimiter=',', header='frequency,psd')
    ### move to the final location
    os.system('mv %s %s'%(tmp, path))

def retrieve_psd(path):
    """read the PSD from disk
    return freqs, psd
    """
    ans = np.genfromtxt(path, delimiter=',', names=True)
    return ans['frequency'], ans['psd']

def report_segs(path, new_segs):
    """reads in the segments contained in path and appends the current seg of segs to them
    assumes you are only adding segments in time order, so it will either start a new segment or merge the last segment in the file
    """
    if len(new_segs): ### there must be something to report...
        new_segs = [(a,b) for a, b in new_segs] ### make a copy so we don't mess up a shared reference

        ### read in and merge segs if necessary
        if os.path.exists(path):
            segs = retrieve_segs(path)
            if len(segs):
                if np.ndim(segs)==1: ### a single segment
                    segs = [segs]

                while len(segs) and (segs[-1][0] > new_segs[0][1]): ### remove existing segs that are redundant with where new_segs starts
                    segs.pop(-1)

                if len(segs):
                    if (segs[-1][1] >= new_segs[0][0]): ### these things overlap
                        segs[-1][1] = new_segs.pop(0)[1] ### remove the first element from segs

                segs = segs+list(new_segs)

            else: ### either empty or a single se
                segs = new_segs

        else:
            segs = new_segs

        ### write the result back out to a temporary location
        tmp = os.path.join(os.path.dirname(path), '.'+os.path.basename(path))
        np.savetxt(tmp, segs, fmt='%d')
        ### move to final location
        os.system('mv %s %s'%(tmp, path))

def retrieve_segs(path):
    """retrieves segments from disk
    """
    return list(np.loadtxt(path))

def report_horizon(path, horizon, flow, fhigh, psd_path):
    """report the horizon to disk
    """
    with open(path, 'w') as f:
        f.write('%.9e\n%d\n%d\n%s'%(horizon, flow, fhigh, psd_path))

def retrieve_horizon(path):
    """retrieve the horizon from disk
    """
    with open(path, 'r') as f:
        horizon = float(f.readline().strip())
        flow = float(f.readline().strip())
        fhigh = float(f.readlin().strip())
        psd_path = f.readline().strip()
    return horizon, flow, fhigh, psd_path

def report_sensitivity(path, sensitivity, *extra_header):
    basepath = path.strip('.gz')
    hp.write_map(
        basepath,
        network,
        coord='C',
        extra_header=extra_header,
    )

    if path.endswith('.gz'):
        if opts.verbose:
            print( "gzipping : %s -> %s"%(basepath, path) )
        with open(basepath, 'r') as f:
            with gzip.open(path, 'w') as gzf:
                gzf.write(f.read())

#-------------------------------------------------

def livetime(segs):
    return np.sum(e-s for s, e in segs)

def andsegments(list1, list2):
    """
    computes the intersection of 2 lists of segments, returning a new list
    taken from laldetchar.idq.event.andtwosegmentlists
    """
    newsegments = []
    index = 0
    for seg in list1:
        while index > 0 and list2[index][0] > seg[0]:
            index -= 1
        while index < len(list2) and list2[index][1] <= seg[0]:
            index += 1
        while index < len(list2) and list2[index][0] < seg[1]:
            newsegments.append([max(seg[0], list2[index][0]), min(seg[1], list2[index][1])])
            index += 1
        if index > 0:
            index -= 1
    return newsegments

def invsegments(start, stop, segs):
    """
    takes the inverse of a list of segments between start and stop
    assumes that all segments are already between start and stop
    """
    if not segs:
        return [[start, stop]]

    newsegments = []
    for s, e in segs:
        if start < s <= stop:
            newsegments.append( [start, s] )
        start = e
    if e < stop:
        newsegments.append( [e, stop] )

    return newsegments

def mergesegments(segments):
    """
    assumes segments are in the correct order and non-overlapping.
    simply merges any touching segments into one bigger segemnt
    """
    if len(segments)<2:
        return segments

    segs = []
    s, e = segments[0]
    for S, E in segments[1:]:
        if e==S:
            e=E
        else:
            segs.append( [s,e] )
            s = S
            e = E
    segs.append( [s, e] )
    return segs

def checksegments(seg, window=None):
    """
    ensures segment makes sense
    if provided, segment is sliced so that we only keep the part that interesects with window=[start, end]
    """
    s, e = seg
    if window:
        start, end = window
        if s < start:
            s = start # truncate the start of seg
        elif not (s < end):
            return False # no overlap between current segment and window
        if e > end:
            e = end # truncate the end of seg
        elif not (e > start):
            return False # no overlap between currnet segment and window

    if s < e:
        return [s,e]
    elif s > e:
        raise ValueError("something is very wrong with segment generation... seg[1]=%.3f < seg[0]=%.3f"%tuple(seg))
    else:
        return False
<|endoftext|>