<commit_before>#!/usr/bin/python
# @lint-avoid-python-3-compatibility-imports
#
# ustat  Activity stats from high-level languages, including exceptions,
#        method calls, class loads, garbage collections, and more.
#        For Linux, uses BCC, eBPF.
#
# USAGE: ustat [-l {java,python,ruby,node}] [-C]
#        [-S {cload,excp,gc,method,objnew,thread}] [-r MAXROWS] [-d]
#        [interval [count]]
#
# This uses in-kernel eBPF maps to store per process summaries for efficiency.
# Newly-created processes might only be traced at the next interval, if the
# relevant USDT probe requires enabling through a semaphore.
#
# Copyright 2016 Sasha Goldshtein
# Licensed under the Apache License, Version 2.0 (the "License")
#
# 26-Oct-2016   Sasha Goldshtein    Created this.

from __future__ import print_function
import argparse
from bcc import BPF, USDT
import os
from subprocess import call
from time import sleep, strftime

class Category(object):
    THREAD = "THREAD"
    METHOD = "METHOD"
    OBJNEW = "OBJNEW"
    CLOAD = "CLOAD"
    EXCP = "EXCP"
    GC = "GC"

class Probe(object):
    def __init__(self, language, procnames, events):
        """
        Initialize a new probe object with a specific language, set of process
        names to monitor for that language, and a dictionary of events and
        categories. The dictionary is a mapping of USDT probe names (such as
        'gc__start') to event categories supported by this tool -- from the
        Category class.
        """
        self.language = language
        self.procnames = procnames
        self.events = events

    def _find_targets(self):
        """Find pids where the comm is one of the specified list"""
        self.targets = {}
        all_pids = [int(pid) for pid in os.listdir('/proc') if pid.isdigit()]
        for pid in all_pids:
            try:
                comm = open('/proc/%d/comm' % pid).read().strip()
                if comm in self.procnames:
                    cmdline = open('/proc/%d/cmdline' % pid).read()
                    self.targets[pid] = cmdline.replace('\0', ' ')
            except IOError:
                continue    # process may already have terminated

    def _enable_probes(self):
        self.usdts = []
        for pid in self.targets:
            usdt = USDT(pid=pid)
            for event in self.events:
                try:
                    usdt.enable_probe(event, "%s_%s" % (self.language, event))
                except Exception:
                    # This process might not have a recent version of the USDT
                    # probes enabled, or might have been compiled without USDT
                    # probes at all. The process could even have been shut down
                    # and the pid been recycled. We have to gracefully handle
                    # the possibility that we can't attach probes to it at all.
                    pass
            self.usdts.append(usdt)

    def _generate_tables(self):
        text = """
BPF_HASH(%s_%s_counts, u32, u64);   // pid to event count
        """
        return str.join('', [text % (self.language, event)
                             for event in self.events])

    def _generate_functions(self):
        text = """
int %s_%s(void *ctx) {
    u64 *valp, zero = 0;
    u32 tgid = bpf_get_current_pid_tgid() >> 32;
    valp = %s_%s_counts.lookup_or_init(&tgid, &zero);
    ++(*valp);
    return 0;
}
        """
        lang = self.language
        return str.join('', [text % (lang, event, lang, event)
                             for event in self.events])

    def get_program(self):
        self._find_targets()
        self._enable_probes()
        return self._generate_tables() + self._generate_functions()

    def get_usdts(self):
        return self.usdts

    def get_counts(self, bpf):
        """Return a map of event counts per process"""
        event_dict = dict([(category, 0) for category in self.events.values()])
        result = dict([(pid, event_dict.copy()) for pid in self.targets])
        for event, category in self.events.items():
            counts = bpf["%s_%s_counts" % (self.language, event)]
            for pid, count in counts.items():
                result[pid.value][category] = count.value
            counts.clear()
        return result

    def cleanup(self):
        self.usdts = None

class Tool(object):
    def _parse_args(self):
        examples = """examples:
  ./ustat              # stats for all languages, 1 second refresh
  ./ustat -C           # don't clear the screen
  ./ustat -l java      # Java processes only
  ./ustat 5            # 5 second summaries
  ./ustat 5 10         # 5 second summaries, 10 times only
        """
        parser = argparse.ArgumentParser(
            description="Activity stats from high-level languages.",
            formatter_class=argparse.RawDescriptionHelpFormatter,
            epilog=examples)
        parser.add_argument("-l", "--language",
            choices=["java", "python", "ruby", "node"],
            help="language to trace (default: all languages)")
        parser.add_argument("-C", "--noclear", action="store_true",
            help="don't clear the screen")
        parser.add_argument("-S", "--sort",
            choices=[cat.lower() for cat in dir(Category) if cat.isupper()],
            help="sort by this field (descending order)")
        parser.add_argument("-r", "--maxrows", default=20, type=int,
            help="maximum rows to print, default 20")
        parser.add_argument("-d", "--debug", action="store_true",
            help="Print the resulting BPF program (for debugging purposes)")
        parser.add_argument("interval", nargs="?", default=1, type=int,
            help="output interval, in seconds")
        parser.add_argument("count", nargs="?", default=99999999, type=int,
            help="number of outputs")
        self.args = parser.parse_args()

    def _create_probes(self):
        probes_by_lang = {
                "node": Probe("node", ["node"], {
                    "gc__start": Category.GC
                    }),
                "python": Probe("python", ["python"], {
                    "function__entry": Category.METHOD,
                    "gc__start": Category.GC
                    }),
                "ruby": Probe("ruby", ["ruby", "irb"], {
                    "method__entry": Category.METHOD,
                    "cmethod__entry": Category.METHOD,
                    "gc__mark__begin": Category.GC,
                    "gc__sweep__begin": Category.GC,
                    "object__create": Category.OBJNEW,
                    "hash__create": Category.OBJNEW,
                    "string__create": Category.OBJNEW,
                    "array__create": Category.OBJNEW,
                    "require__entry": Category.CLOAD,
                    "load__entry": Category.CLOAD,
                    "raise": Category.EXCP
                    }),
                "java": Probe("java", ["java"], {
                    "gc__begin": Category.GC,
                    "mem__pool__gc__begin": Category.GC,
                    "thread__start": Category.THREAD,
                    "class__loaded": Category.CLOAD,
                    "object__alloc": Category.OBJNEW,
                    "method__entry": Category.METHOD,
                    "ExceptionOccurred__entry": Category.EXCP
                    })
                }

        if self.args.language:
            self.probes = [probes_by_lang[self.args.language]]
        else:
            self.probes = probes_by_lang.values()

    def _attach_probes(self):
        program = str.join('\n', [p.get_program() for p in self.probes])
        if self.args.debug:
            print(program)
            for probe in self.probes:
                print("Attached to %s processes:" % probe.language,
                        str.join(', ', map(str, probe.targets)))
        self.bpf = BPF(text=program)
        usdts = [usdt for probe in self.probes for usdt in probe.get_usdts()]
        # Filter out duplicates when we have multiple processes with the same
        # uprobe. We are attaching to these probes manually instead of using
        # the USDT support from the bcc module, because the USDT class attaches
        # to each uprobe with a specific pid. When there is more than one
        # process from some language, we end up attaching more than once to the
        # same uprobe (albeit with different pids), which is not allowed.
        # Instead, we use a global attach (with pid=-1).
        uprobes = set([(path, func, addr) for usdt in usdts
                       for (path, func, addr, _)
                       in usdt.enumerate_active_probes()])
        for (path, func, addr) in uprobes:
            self.bpf.attach_uprobe(name=path, fn_name=func, addr=addr, pid=-1)

    def _detach_probes(self):
        for probe in self.probes:
            probe.cleanup()     # Cleans up USDT contexts
        self.bpf.cleanup()      # Cleans up all attached probes
        self.bpf = None

    def _loop_iter(self):
        self._attach_probes()
        try:
            sleep(self.args.interval)
        except KeyboardInterrupt:
            self.exiting = True

        if not self.args.noclear:
            call("clear")
        else:
            print()
        with open("/proc/loadavg") as stats:
            print("%-8s loadavg: %s" % (strftime("%H:%M:%S"), stats.read()))
        print("%-6s %-20s %-10s %-6s %-10s %-8s %-6s %-6s" % (
            "PID", "CMDLINE", "METHOD/s", "GC/s", "OBJNEW/s",
            "CLOAD/s", "EXC/s", "THR/s"))

        line = 0
        counts = {}
        targets = {}
        for probe in self.probes:
            counts.update(probe.get_counts(self.bpf))
            targets.update(probe.targets)
        if self.args.sort:
            counts = sorted(counts.items(), key=lambda (_, v):
                            -v.get(self.args.sort.upper(), 0))
        else:
            counts = sorted(counts.items(), key=lambda (k, _): k)
        for pid, stats in counts:
            print("%-6d %-20s %-10d %-6d %-10d %-8d %-6d %-6d" % (
                  pid, targets[pid][:20],
                  stats.get(Category.METHOD, 0) / self.args.interval,
                  stats.get(Category.GC, 0) / self.args.interval,
                  stats.get(Category.OBJNEW, 0) / self.args.interval,
                  stats.get(Category.CLOAD, 0) / self.args.interval,
                  stats.get(Category.EXCP, 0) / self.args.interval,
                  stats.get(Category.THREAD, 0) / self.args.interval
                  ))
            line += 1
            if line >= self.args.maxrows:
                break
        self._detach_probes()

    def run(self):
        self._parse_args()
        self._create_probes()
        print('Tracing... Output every %d secs. Hit Ctrl-C to end' %
              self.args.interval)
        countdown = self.args.count
        self.exiting = False
        while True:
            self._loop_iter()
            countdown -= 1
            if self.exiting or countdown == 0:
                print("Detaching...")
                exit()

if __name__ == "__main__":
    try:
        Tool().run()
    except KeyboardInterrupt:
        pass
<commit_msg>ustat: fix lambda syntax on python3.<commit_after>#!/usr/bin/python
# @lint-avoid-python-3-compatibility-imports
#
# ustat  Activity stats from high-level languages, including exceptions,
#        method calls, class loads, garbage collections, and more.
#        For Linux, uses BCC, eBPF.
#
# USAGE: ustat [-l {java,python,ruby,node}] [-C]
#        [-S {cload,excp,gc,method,objnew,thread}] [-r MAXROWS] [-d]
#        [interval [count]]
#
# This uses in-kernel eBPF maps to store per process summaries for efficiency.
# Newly-created processes might only be traced at the next interval, if the
# relevant USDT probe requires enabling through a semaphore.
#
# Copyright 2016 Sasha Goldshtein
# Licensed under the Apache License, Version 2.0 (the "License")
#
# 26-Oct-2016   Sasha Goldshtein    Created this.

from __future__ import print_function
import argparse
from bcc import BPF, USDT
import os
from subprocess import call
from time import sleep, strftime

class Category(object):
    THREAD = "THREAD"
    METHOD = "METHOD"
    OBJNEW = "OBJNEW"
    CLOAD = "CLOAD"
    EXCP = "EXCP"
    GC = "GC"

class Probe(object):
    def __init__(self, language, procnames, events):
        """
        Initialize a new probe object with a specific language, set of process
        names to monitor for that language, and a dictionary of events and
        categories. The dictionary is a mapping of USDT probe names (such as
        'gc__start') to event categories supported by this tool -- from the
        Category class.
        """
        self.language = language
        self.procnames = procnames
        self.events = events

    def _find_targets(self):
        """Find pids where the comm is one of the specified list"""
        self.targets = {}
        all_pids = [int(pid) for pid in os.listdir('/proc') if pid.isdigit()]
        for pid in all_pids:
            try:
                comm = open('/proc/%d/comm' % pid).read().strip()
                if comm in self.procnames:
                    cmdline = open('/proc/%d/cmdline' % pid).read()
                    self.targets[pid] = cmdline.replace('\0', ' ')
            except IOError:
                continue    # process may already have terminated

    def _enable_probes(self):
        self.usdts = []
        for pid in self.targets:
            usdt = USDT(pid=pid)
            for event in self.events:
                try:
                    usdt.enable_probe(event, "%s_%s" % (self.language, event))
                except Exception:
                    # This process might not have a recent version of the USDT
                    # probes enabled, or might have been compiled without USDT
                    # probes at all. The process could even have been shut down
                    # and the pid been recycled. We have to gracefully handle
                    # the possibility that we can't attach probes to it at all.
                    pass
            self.usdts.append(usdt)

    def _generate_tables(self):
        text = """
BPF_HASH(%s_%s_counts, u32, u64);   // pid to event count
        """
        return str.join('', [text % (self.language, event)
                             for event in self.events])

    def _generate_functions(self):
        text = """
int %s_%s(void *ctx) {
    u64 *valp, zero = 0;
    u32 tgid = bpf_get_current_pid_tgid() >> 32;
    valp = %s_%s_counts.lookup_or_init(&tgid, &zero);
    ++(*valp);
    return 0;
}
        """
        lang = self.language
        return str.join('', [text % (lang, event, lang, event)
                             for event in self.events])

    def get_program(self):
        self._find_targets()
        self._enable_probes()
        return self._generate_tables() + self._generate_functions()

    def get_usdts(self):
        return self.usdts

    def get_counts(self, bpf):
        """Return a map of event counts per process"""
        event_dict = dict([(category, 0) for category in self.events.values()])
        result = dict([(pid, event_dict.copy()) for pid in self.targets])
        for event, category in self.events.items():
            counts = bpf["%s_%s_counts" % (self.language, event)]
            for pid, count in counts.items():
                result[pid.value][category] = count.value
            counts.clear()
        return result

    def cleanup(self):
        self.usdts = None

class Tool(object):
    def _parse_args(self):
        examples = """examples:
  ./ustat              # stats for all languages, 1 second refresh
  ./ustat -C           # don't clear the screen
  ./ustat -l java      # Java processes only
  ./ustat 5            # 5 second summaries
  ./ustat 5 10         # 5 second summaries, 10 times only
        """
        parser = argparse.ArgumentParser(
            description="Activity stats from high-level languages.",
            formatter_class=argparse.RawDescriptionHelpFormatter,
            epilog=examples)
        parser.add_argument("-l", "--language",
            choices=["java", "python", "ruby", "node"],
            help="language to trace (default: all languages)")
        parser.add_argument("-C", "--noclear", action="store_true",
            help="don't clear the screen")
        parser.add_argument("-S", "--sort",
            choices=[cat.lower() for cat in dir(Category) if cat.isupper()],
            help="sort by this field (descending order)")
        parser.add_argument("-r", "--maxrows", default=20, type=int,
            help="maximum rows to print, default 20")
        parser.add_argument("-d", "--debug", action="store_true",
            help="Print the resulting BPF program (for debugging purposes)")
        parser.add_argument("interval", nargs="?", default=1, type=int,
            help="output interval, in seconds")
        parser.add_argument("count", nargs="?", default=99999999, type=int,
            help="number of outputs")
        self.args = parser.parse_args()

    def _create_probes(self):
        probes_by_lang = {
                "node": Probe("node", ["node"], {
                    "gc__start": Category.GC
                    }),
                "python": Probe("python", ["python"], {
                    "function__entry": Category.METHOD,
                    "gc__start": Category.GC
                    }),
                "ruby": Probe("ruby", ["ruby", "irb"], {
                    "method__entry": Category.METHOD,
                    "cmethod__entry": Category.METHOD,
                    "gc__mark__begin": Category.GC,
                    "gc__sweep__begin": Category.GC,
                    "object__create": Category.OBJNEW,
                    "hash__create": Category.OBJNEW,
                    "string__create": Category.OBJNEW,
                    "array__create": Category.OBJNEW,
                    "require__entry": Category.CLOAD,
                    "load__entry": Category.CLOAD,
                    "raise": Category.EXCP
                    }),
                "java": Probe("java", ["java"], {
                    "gc__begin": Category.GC,
                    "mem__pool__gc__begin": Category.GC,
                    "thread__start": Category.THREAD,
                    "class__loaded": Category.CLOAD,
                    "object__alloc": Category.OBJNEW,
                    "method__entry": Category.METHOD,
                    "ExceptionOccurred__entry": Category.EXCP
                    })
                }

        if self.args.language:
            self.probes = [probes_by_lang[self.args.language]]
        else:
            self.probes = probes_by_lang.values()

    def _attach_probes(self):
        program = str.join('\n', [p.get_program() for p in self.probes])
        if self.args.debug:
            print(program)
            for probe in self.probes:
                print("Attached to %s processes:" % probe.language,
                        str.join(', ', map(str, probe.targets)))
        self.bpf = BPF(text=program)
        usdts = [usdt for probe in self.probes for usdt in probe.get_usdts()]
        # Filter out duplicates when we have multiple processes with the same
        # uprobe. We are attaching to these probes manually instead of using
        # the USDT support from the bcc module, because the USDT class attaches
        # to each uprobe with a specific pid. When there is more than one
        # process from some language, we end up attaching more than once to the
        # same uprobe (albeit with different pids), which is not allowed.
        # Instead, we use a global attach (with pid=-1).
        uprobes = set([(path, func, addr) for usdt in usdts
                       for (path, func, addr, _)
                       in usdt.enumerate_active_probes()])
        for (path, func, addr) in uprobes:
            self.bpf.attach_uprobe(name=path, fn_name=func, addr=addr, pid=-1)

    def _detach_probes(self):
        for probe in self.probes:
            probe.cleanup()     # Cleans up USDT contexts
        self.bpf.cleanup()      # Cleans up all attached probes
        self.bpf = None

    def _loop_iter(self):
        self._attach_probes()
        try:
            sleep(self.args.interval)
        except KeyboardInterrupt:
            self.exiting = True

        if not self.args.noclear:
            call("clear")
        else:
            print()
        with open("/proc/loadavg") as stats:
            print("%-8s loadavg: %s" % (strftime("%H:%M:%S"), stats.read()))
        print("%-6s %-20s %-10s %-6s %-10s %-8s %-6s %-6s" % (
            "PID", "CMDLINE", "METHOD/s", "GC/s", "OBJNEW/s",
            "CLOAD/s", "EXC/s", "THR/s"))

        line = 0
        counts = {}
        targets = {}
        for probe in self.probes:
            counts.update(probe.get_counts(self.bpf))
            targets.update(probe.targets)
        if self.args.sort:
            counts = sorted(counts.items(), key=lambda _, v:
                            -v.get(self.args.sort.upper(), 0))
        else:
            counts = sorted(counts.items(), key=lambda k, _: k)
        for pid, stats in counts:
            print("%-6d %-20s %-10d %-6d %-10d %-8d %-6d %-6d" % (
                  pid, targets[pid][:20],
                  stats.get(Category.METHOD, 0) / self.args.interval,
                  stats.get(Category.GC, 0) / self.args.interval,
                  stats.get(Category.OBJNEW, 0) / self.args.interval,
                  stats.get(Category.CLOAD, 0) / self.args.interval,
                  stats.get(Category.EXCP, 0) / self.args.interval,
                  stats.get(Category.THREAD, 0) / self.args.interval
                  ))
            line += 1
            if line >= self.args.maxrows:
                break
        self._detach_probes()

    def run(self):
        self._parse_args()
        self._create_probes()
        print('Tracing... Output every %d secs. Hit Ctrl-C to end' %
              self.args.interval)
        countdown = self.args.count
        self.exiting = False
        while True:
            self._loop_iter()
            countdown -= 1
            if self.exiting or countdown == 0:
                print("Detaching...")
                exit()

if __name__ == "__main__":
    try:
        Tool().run()
    except KeyboardInterrupt:
        pass
<|endoftext|>


<commit_before>"""
mbed SDK
Copyright (c) 2011-2013 ARM Limited

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import sys
import inspect
import os
import argparse
import math
from os import listdir, remove, makedirs
from shutil import copyfile
from os.path import isdir, join, exists, split, relpath, splitext
from subprocess import Popen, PIPE, STDOUT, call
import json
from collections import OrderedDict

def cmd(l, check=True, verbose=False, shell=False, cwd=None):
    text = l if shell else ' '.join(l)
    if verbose:
        print text
    rc = call(l, shell=shell, cwd=cwd)
    if check and rc != 0:
        raise Exception('ERROR %d: "%s"' % (rc, text))

def run_cmd(command, wd=None, redirect=False):
    assert is_cmd_valid(command[0])
    try:
        p = Popen(command, stdout=PIPE, stderr=STDOUT if redirect else PIPE, cwd=wd)
        _stdout, _stderr = p.communicate()
    except OSError as e:
        print "[OS ERROR] Command: "+(' '.join(command))
        raise

    return _stdout, _stderr, p.returncode

def run_cmd_ext(command):
    assert is_cmd_valid(command[0])
    p = Popen(command, stdout=PIPE, stderr=PIPE)
    _stdout, _stderr = p.communicate()
    return _stdout, _stderr, p.returncode

def is_cmd_valid(cmd):
    caller = get_caller_name()
    abspath = find_cmd_abspath(cmd)
    if not abspath:
        error("%s: Command '%s' can't be found" % (caller, cmd))
    if not is_exec(abspath):
        error("%s: Command '%s' resolves to file '%s' which is not executable" % (caller, cmd, abspath))
    return True

def is_exec(path):
    return os.access(path, os.X_OK) or os.access(path+'.exe', os.X_OK)

def find_cmd_abspath(cmd):
    """ Returns the absolute path to a command.
        None is returned if no absolute path was found.
    """
    if exists(cmd) or exists(cmd + '.exe'):
        return os.path.abspath(cmd)
    if not 'PATH' in os.environ:
        raise Exception("Can't find command path for current platform ('%s')" % sys.platform)
    PATH=os.environ['PATH']
    for path in PATH.split(os.pathsep):
        abspath = '%s/%s' % (path, cmd)
        if exists(abspath) or exists(abspath + '.exe'):
            return abspath

def mkdir(path):
    if not exists(path):
        makedirs(path)

def copy_file(src, dst):
    """ Implement the behaviour of "shutil.copy(src, dst)" without copying the
        permissions (this was causing errors with directories mounted with samba)
    """
    if isdir(dst):
        _, file = split(src)
        dst = join(dst, file)
    copyfile(src, dst)

def delete_dir_files(dir):
    if not exists(dir):
        return

    for f in listdir(dir):
        file = join(dir, f)
        if not isdir(file):
            remove(file)

def get_caller_name(steps=2):
    """
    When called inside a function, it returns the name
    of the caller of that function.
    """
    return inspect.stack()[steps][3]

def error(msg):
    print("ERROR: %s" % msg)
    sys.exit(1)

def rel_path(path, base, dot=False):
    p = relpath(path, base)
    if dot and not p.startswith('.'):
        p = './' + p
    return p

class ToolException(Exception):
    pass

class NotSupportedException(Exception):
    pass

def split_path(path):
    base, file = split(path)
    name, ext = splitext(file)
    return base, name, ext

def args_error(parser, message):
    print "\n\n%s\n\n" % message
    parser.print_help()
    sys.exit()

def construct_enum(**enums):
    """ Create your own pseudo-enums """
    return type('Enum', (), enums)

def check_required_modules(required_modules, verbose=True):
    """ Function checks for Python modules which should be "importable" (installed)
        before test suite can be used.
        @return returns True if all modules are installed already
    """
    import imp
    not_installed_modules = []
    for module_name in required_modules:
        try:
            imp.find_module(module_name)
        except ImportError as e:
            # We also test against a rare case: module is an egg file
            try:
                __import__(module_name)
            except ImportError as e:
                not_installed_modules.append(module_name)
                if verbose:
                    print "Error: %s" % e

    if verbose:
        if not_installed_modules:
            print "Warning: Module(s) %s not installed. Please install required module(s) before using this script."% (', '.join(not_installed_modules))

    if not_installed_modules:
        return False
    else:
        return True

# Utility function: traverse a dictionary and change all the strings in the dictionary to
# ASCII from Unicode. Useful when reading ASCII JSON data, because the JSON decoder always
# returns Unicode string.
# Based on http://stackoverflow.com/a/13105359
def dict_to_ascii(input):
    if isinstance(input, dict):
        return OrderedDict([(dict_to_ascii(key), dict_to_ascii(value)) for key, value in input.iteritems()])
    elif isinstance(input, list):
        return [dict_to_ascii(element) for element in input]
    elif isinstance(input, unicode):
        return input.encode('ascii')
    else:
        return input

# Read a JSON file and return its Python representation, transforming all the strings from Unicode
# to ASCII. The order of keys in the JSON file is preserved.
def json_file_to_dict(fname):
    with open(fname, "rt") as f:
        return dict_to_ascii(json.load(f, object_pairs_hook=OrderedDict))

# Wowza, double closure
def argparse_type(casedness, prefer_hyphen=False) :
    def middle(list, type_name):
        def parse_type(string):
            if prefer_hyphen: newstring = casedness(string).replace("_","-")
            else:             newstring = casedness(string).replace("-","_")
            if string in list:
                return string
            elif string not in list and newstring in list:
                raise argparse.ArgumentTypeError("{0} is not a supported {1}. Did you mean {2}?".format(string, type_name, newstring))
            else:
                raise argparse.ArgumentTypeError("{0} is not a supported {1}. Supported {1}s are:\n{2}".format(string, type_name, columnate(list)))
        return parse_type
    return middle

argparse_uppercase_type = argparse_type(str.upper, False)
argparse_lowercase_type = argparse_type(str.lower, False)
argparse_uppercase_hyphen_type = argparse_type(str.upper, True)
argparse_lowercase_hyphen_type = argparse_type(str.lower, True)

def argparse_many(fn):
    def wrap(string):
        return [fn(s) for s in string.split(",")]
    return wrap

def argparse_filestring_type(string) :
    if exists(string) :
        return string
    else :
        raise argparse.ArgumentTypeError("{0}"" does not exist in the filesystem.".format(string))

def columnate(strings, seperator=", ", chars=80):
    col_width = max(len(s) for s in strings)
    total_width = col_width + len(seperator)
    columns = math.floor(chars / total_width)
    output = ""
    for i, s in zip(range(len(strings)), strings):
        append = s
        if i != len(strings) - 1:
            append += seperator
        if i % columns == columns - 1:
            append += "\n"
        else:
            append = append.ljust(total_width)
        output += append
    return output
<commit_msg>Add documentation for the argparser utilities<commit_after>"""
mbed SDK
Copyright (c) 2011-2013 ARM Limited

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import sys
import inspect
import os
import argparse
import math
from os import listdir, remove, makedirs
from shutil import copyfile
from os.path import isdir, join, exists, split, relpath, splitext
from subprocess import Popen, PIPE, STDOUT, call
import json
from collections import OrderedDict

def cmd(l, check=True, verbose=False, shell=False, cwd=None):
    text = l if shell else ' '.join(l)
    if verbose:
        print text
    rc = call(l, shell=shell, cwd=cwd)
    if check and rc != 0:
        raise Exception('ERROR %d: "%s"' % (rc, text))

def run_cmd(command, wd=None, redirect=False):
    assert is_cmd_valid(command[0])
    try:
        p = Popen(command, stdout=PIPE, stderr=STDOUT if redirect else PIPE, cwd=wd)
        _stdout, _stderr = p.communicate()
    except OSError as e:
        print "[OS ERROR] Command: "+(' '.join(command))
        raise

    return _stdout, _stderr, p.returncode

def run_cmd_ext(command):
    assert is_cmd_valid(command[0])
    p = Popen(command, stdout=PIPE, stderr=PIPE)
    _stdout, _stderr = p.communicate()
    return _stdout, _stderr, p.returncode

def is_cmd_valid(cmd):
    caller = get_caller_name()
    abspath = find_cmd_abspath(cmd)
    if not abspath:
        error("%s: Command '%s' can't be found" % (caller, cmd))
    if not is_exec(abspath):
        error("%s: Command '%s' resolves to file '%s' which is not executable" % (caller, cmd, abspath))
    return True

def is_exec(path):
    return os.access(path, os.X_OK) or os.access(path+'.exe', os.X_OK)

def find_cmd_abspath(cmd):
    """ Returns the absolute path to a command.
        None is returned if no absolute path was found.
    """
    if exists(cmd) or exists(cmd + '.exe'):
        return os.path.abspath(cmd)
    if not 'PATH' in os.environ:
        raise Exception("Can't find command path for current platform ('%s')" % sys.platform)
    PATH=os.environ['PATH']
    for path in PATH.split(os.pathsep):
        abspath = '%s/%s' % (path, cmd)
        if exists(abspath) or exists(abspath + '.exe'):
            return abspath

def mkdir(path):
    if not exists(path):
        makedirs(path)

def copy_file(src, dst):
    """ Implement the behaviour of "shutil.copy(src, dst)" without copying the
        permissions (this was causing errors with directories mounted with samba)
    """
    if isdir(dst):
        _, file = split(src)
        dst = join(dst, file)
    copyfile(src, dst)

def delete_dir_files(dir):
    if not exists(dir):
        return

    for f in listdir(dir):
        file = join(dir, f)
        if not isdir(file):
            remove(file)

def get_caller_name(steps=2):
    """
    When called inside a function, it returns the name
    of the caller of that function.
    """
    return inspect.stack()[steps][3]

def error(msg):
    print("ERROR: %s" % msg)
    sys.exit(1)

def rel_path(path, base, dot=False):
    p = relpath(path, base)
    if dot and not p.startswith('.'):
        p = './' + p
    return p

class ToolException(Exception):
    pass

class NotSupportedException(Exception):
    pass

def split_path(path):
    base, file = split(path)
    name, ext = splitext(file)
    return base, name, ext

def args_error(parser, message):
    print "\n\n%s\n\n" % message
    parser.print_help()
    sys.exit()

def construct_enum(**enums):
    """ Create your own pseudo-enums """
    return type('Enum', (), enums)

def check_required_modules(required_modules, verbose=True):
    """ Function checks for Python modules which should be "importable" (installed)
        before test suite can be used.
        @return returns True if all modules are installed already
    """
    import imp
    not_installed_modules = []
    for module_name in required_modules:
        try:
            imp.find_module(module_name)
        except ImportError as e:
            # We also test against a rare case: module is an egg file
            try:
                __import__(module_name)
            except ImportError as e:
                not_installed_modules.append(module_name)
                if verbose:
                    print "Error: %s" % e

    if verbose:
        if not_installed_modules:
            print "Warning: Module(s) %s not installed. Please install required module(s) before using this script."% (', '.join(not_installed_modules))

    if not_installed_modules:
        return False
    else:
        return True

# Utility function: traverse a dictionary and change all the strings in the dictionary to
# ASCII from Unicode. Useful when reading ASCII JSON data, because the JSON decoder always
# returns Unicode string.
# Based on http://stackoverflow.com/a/13105359
def dict_to_ascii(input):
    if isinstance(input, dict):
        return OrderedDict([(dict_to_ascii(key), dict_to_ascii(value)) for key, value in input.iteritems()])
    elif isinstance(input, list):
        return [dict_to_ascii(element) for element in input]
    elif isinstance(input, unicode):
        return input.encode('ascii')
    else:
        return input

# Read a JSON file and return its Python representation, transforming all the strings from Unicode
# to ASCII. The order of keys in the JSON file is preserved.
def json_file_to_dict(fname):
    with open(fname, "rt") as f:
        return dict_to_ascii(json.load(f, object_pairs_hook=OrderedDict))

# Wowza, double closure
def argparse_type(casedness, prefer_hyphen=False) :
    def middle(list, type_name):
        # validate that an argument passed in (as string) is a member of the list of possible
        # arguments. Offer a suggestion if the case of the string, or the hyphens/underscores
        # do not match the expected style of the argument.
        def parse_type(string):
            if prefer_hyphen: newstring = casedness(string).replace("_","-")
            else:             newstring = casedness(string).replace("-","_")
            if string in list:
                return string
            elif string not in list and newstring in list:
                raise argparse.ArgumentTypeError("{0} is not a supported {1}. Did you mean {2}?".format(string, type_name, newstring))
            else:
                raise argparse.ArgumentTypeError("{0} is not a supported {1}. Supported {1}s are:\n{2}".format(string, type_name, columnate(list)))
        return parse_type
    return middle

# short cuts for the argparse_type versions
argparse_uppercase_type = argparse_type(str.upper, False)
argparse_lowercase_type = argparse_type(str.lower, False)
argparse_uppercase_hyphen_type = argparse_type(str.upper, True)
argparse_lowercase_hyphen_type = argparse_type(str.lower, True)

# An argument parser combinator that takes in an argument parser and creates a new parser that
# accepts a comma separated list of the same thing.
def argparse_many(fn):
    def wrap(string):
        return [fn(s) for s in string.split(",")]
    return wrap

# An argument parser that verifies that a string passed in corresponds to a file
def argparse_filestring_type(string) :
    if exists(string) :
        return string
    else :
        raise argparse.ArgumentTypeError("{0}"" does not exist in the filesystem.".format(string))

# render a list of strings as a in a bunch of columns
def columnate(strings, seperator=", ", chars=80):
    col_width = max(len(s) for s in strings)
    total_width = col_width + len(seperator)
    columns = math.floor(chars / total_width)
    output = ""
    for i, s in zip(range(len(strings)), strings):
        append = s
        if i != len(strings) - 1:
            append += seperator
        if i % columns == columns - 1:
            append += "\n"
        else:
            append = append.ljust(total_width)
        output += append
    return output
<|endoftext|>


<commit_before>#SPARQL_BAMS_Store_Persist_Example.py
#This program is used to open up BAMS data, persist it to a store for speed up of querying

#Needed after IPython installation:
#import sys
#print(sys.path)
#shows that the path was changed after downloading anaconda and IPython Notebook
#sys.path += ['', 'C:\Python27\Lib\idlelib', 'C:\Python27\lib\site-packages\pip-1.1-py2.7.egg', 'C:\Python27\lib\site-packages\gensim-0.8.6-py2.7.egg', 'C:\WINDOWS\system32\python27.zip', 'C:\Python27\DLLs', 'C:\Python27\lib', 'C:\Python27\lib\plat-win', 'C:\Python27\lib\lib-tk', 'C:\Python27', 'C:\Python27\lib\site-packages', 'C:\Python27\lib\site-packages\PIL']

#For Parsing
import rdflib
from rdflib import plugin

#for getting the length of the files
import os

#for working with tempfiles
import os.path as op
import tempfile

#For Unzipping
import zipfile
from StringIO import StringIO

plugin.register(
    'sparql', rdflib.query.Processor,
    'rdfextras.sparql.processor', 'Processor')
plugin.register(
    'sparql', rdflib.query.Result,
    'rdfextras.sparql.query', 'SPARQLQueryResult')

zipdata = StringIO()

# open the file using a relative path
#r = open("../Data/BAMS1.zip")

# adding the BAMS Thesaurus instead of the more limited set of data:
#r = open("../Data/bams_thesaurus_2013-09-24_17-12-40.xml.zip")
# Fixed RDF
r = open("../Data/bams_thesaurus_2013-10-06_14-58-56.xml.zip")

# zipdata is a buffer holding the contents of the zip file in memory
zipdata.write(r.read())

print("~40 seconds for zip to open...")

#myzipfile opens the contents of the zip file as an object that knows how to unzip
myzipfile = zipfile.ZipFile(zipdata)

#grab the contents out of myzipfile by name
#foofile = myzipfile.open('bams_ontology_2013-07-10_03-20-00.xml')

#changing the foofile to be the file we upen above^^^^^ in r = open()....etc.
#foofile = myzipfile.open('bams_thesaurus_2013-09-24_17-12-40.xml')
# Fixed RDF
foofile = myzipfile.open('bams_thesaurus_2013-10-06_14-58-56.xml')

print("loading up the BAMS file in memory...")

#Get a Graph object using a Sleepycat persistent store
g = rdflib.Graph('Sleepycat',identifier='BAMS')

# first time create the store
#  put the store in a temp directory so it doesn't get confused with stuff we should commit
tempStore = op.join( tempfile.gettempdir(), 'myRDF_BAMS_Store')
g.open(tempStore, create = True)

#pull in the BAMS RDF document, parse, and store.
#result = g.parse(file=myzipfile.open('bams_ontology_2013-07-10_03-20-00.xml'), format="application/rdf+xml")

#do the same thing but with the BAMS thesaurus file
#result = g.parse(file=myzipfile.open('bams_thesaurus_2013-09-24_17-12-40.xml'), format="application/rdf+xml")
# Fixed RDF
result = g.parse(file=myzipfile.open('bams_thesaurus_2013-10-06_14-58-56.xml'), format="application/rdf+xml")

foofile.close()

# when done!
g.close()

print("Graph stored to disk")<commit_msg>Storing BAMS Thesaurus to Disk<commit_after>#SPARQL_BAMS_Store_Persist_Example.py
#This program is used to open up BAMS data, persist it to a store for speed up of querying
#This document is currently working with the Fixed RDF BAMS Thesaurus Data

#Needed after IPython installation:
#import sys
#print(sys.path)
#shows that the path was changed after downloading anaconda and IPython Notebook
#sys.path += ['', 'C:\Python27\Lib\idlelib', 'C:\Python27\lib\site-packages\pip-1.1-py2.7.egg', 'C:\Python27\lib\site-packages\gensim-0.8.6-py2.7.egg', 'C:\WINDOWS\system32\python27.zip', 'C:\Python27\DLLs', 'C:\Python27\lib', 'C:\Python27\lib\plat-win', 'C:\Python27\lib\lib-tk', 'C:\Python27', 'C:\Python27\lib\site-packages', 'C:\Python27\lib\site-packages\PIL']

#For Parsing
import rdflib
from rdflib import plugin

#for getting the length of the files
import os

#for working with tempfiles
import os.path as op
import tempfile

#For Unzipping
import zipfile
from StringIO import StringIO

plugin.register(
    'sparql', rdflib.query.Processor,
    'rdfextras.sparql.processor', 'Processor')
plugin.register(
    'sparql', rdflib.query.Result,
    'rdfextras.sparql.query', 'SPARQLQueryResult')

zipdata = StringIO()

# open the file using a relative path
#r = open("../Data/BAMS1.zip")

# adding the BAMS Thesaurus instead of the more limited set of data:
#r = open("../Data/bams_thesaurus_2013-09-24_17-12-40.xml.zip")
# Fixed RDF
r = open("../Data/bams_thesaurus_2013-10-06_14-58-56.xml.zip")

# zipdata is a buffer holding the contents of the zip file in memory
zipdata.write(r.read())

print("~40 seconds for zip to open...")

#myzipfile opens the contents of the zip file as an object that knows how to unzip
myzipfile = zipfile.ZipFile(zipdata)

#grab the contents out of myzipfile by name
#foofile = myzipfile.open('bams_ontology_2013-07-10_03-20-00.xml')

#changing the foofile to be the file we upen above^^^^^ in r = open()....etc.
#foofile = myzipfile.open('bams_thesaurus_2013-09-24_17-12-40.xml')
# Fixed RDF
foofile = myzipfile.open('bams_thesaurus_2013-10-06_14-58-56.xml')

print("loading up the BAMS file in memory...")

#Get a Graph object using a Sleepycat persistent store
g = rdflib.Graph('Sleepycat',identifier='BAMS')

# first time create the store
#  put the store in a temp directory so it doesn't get confused with stuff we should commit
tempStore = op.join( tempfile.gettempdir(), 'myRDF_BAMS_Store')
g.open(tempStore, create = True)

#pull in the BAMS RDF document, parse, and store.
#result = g.parse(file=myzipfile.open('bams_ontology_2013-07-10_03-20-00.xml'), format="application/rdf+xml")

#do the same thing but with the BAMS thesaurus file
#result = g.parse(file=myzipfile.open('bams_thesaurus_2013-09-24_17-12-40.xml'), format="application/rdf+xml")
# Fixed RDF
result = g.parse(file=myzipfile.open('bams_thesaurus_2013-10-06_14-58-56.xml'), format="application/rdf+xml")

foofile.close()

# when done!
g.close()

print("Graph stored to disk")<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

## This script reads from TextGrid files produced in Praat the timestamps and labels of any annotations.
## It then extracts appropriately-named JPGs from those time points in the video file.

import sys
import re
import datetime
import subprocess

## CONSTANTS:
## (change these with each run)
C = 'LB' # C for consultant; change depending on whose files you're looking at.
SECTIONS = [3, 4, 5, 8] # change according to which numbered sections are available for each consultant; the video files should be labeled "(consultant's initials)(section number).mov", e.g. "LB3.mov"

def counting(vowels_so_far, tuple):
    """Return a three-digit number determined by how many tokens of a particular vowel have been previously identified: 004, 023, etc. (as EdgeTrak prefers).
    """
    if len(str(vowels_so_far.count(tuple[1])+1)) == 1:
        return '00'+str(vowels_so_far.count(tuple[1])+1)
    elif len(str(vowels_so_far.count(tuple[1])+1)) == 2:
        return '0'+str(vowels_so_far.count(tuple[1])+1)
    elif len(str(vowels_so_far.count(tuple[1])+1)) == 3:
        return str(vowels_so_far.count(tuple[1])+1)

def extract_section_frames(s, vowels_so_far):
    tg = open(C+str(s)+'.TextGrid', 'r').read()
    raw_times = re.findall("number = \d+\.\d\d\d", tg)
    times = []
    for item in raw_times:
        times.append(item[9:])
    raw_vowels = re.findall('mark = "\w+"', tg)
    vowels = []
    for item in raw_vowels:
        vowels.append(item[8:-1])
    tuples = zip(times, vowels)

    for t in tuples:
        timestamp = str(datetime.timedelta(seconds=float(t[0])))[:-3]
        subprocess.call(['ffmpeg', '-i', str(C)+str(s)+'.mov', '-ss', '0'+timestamp, '-vframes', '1', t[1]+'-'+counting(vowels_so_far, t)+'.jpg'])
        vowels_so_far.append(t[1])
    return vowels_so_far

if __name__ == '__main__':
    vowels_so_far = [] # set of all vowels encountered so far (for numbering)
    for s in SECTIONS:
        vowels_so_far = extract_section_frames(s, vowels_so_far)

<commit_msg>Enable more diverse point tier labels<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

## This script reads from TextGrid files produced in Praat the timestamps and labels of any annotations.
## It then extracts appropriately-named JPGs from those time points in the video file.

import sys
import re
import datetime
import subprocess

## CONSTANTS:
## (change these with each run)
C = 'LB' # C for consultant; change depending on whose files you're looking at.
SECTIONS = [3, 4, 5, 8] # change according to which numbered sections are available for each consultant; the video files should be labeled "(consultant's initials)(section number).mov", e.g. "LB3.mov"

def counting(vowels_so_far, tuple):
    """Return a three-digit number determined by how many tokens of a particular vowel have been previously identified: 004, 023, etc. (as EdgeTrak prefers).
    """
    if len(str(vowels_so_far.count(tuple[1])+1)) == 1:
        return '00'+str(vowels_so_far.count(tuple[1])+1)
    elif len(str(vowels_so_far.count(tuple[1])+1)) == 2:
        return '0'+str(vowels_so_far.count(tuple[1])+1)
    elif len(str(vowels_so_far.count(tuple[1])+1)) == 3:
        return str(vowels_so_far.count(tuple[1])+1)

def extract_section_frames(s, vowels_so_far):
    tg = open(C+str(s)+'.TextGrid', 'r').read()
    raw_times = re.findall("number = \d+\.\d\d\d", tg)
    times = []
    for item in raw_times:
        times.append(item[9:])
    raw_vowels = re.findall('mark = ".+"', tg)
    vowels = []
    for item in raw_vowels:
        vowels.append(item[8:-1])
    tuples = zip(times, vowels)

    for t in tuples:
        timestamp = str(datetime.timedelta(seconds=float(t[0])))[:-3]
        subprocess.call(['ffmpeg', '-i', str(C)+str(s)+'.mov', '-ss', '0'+timestamp, '-vframes', '1', t[1]+'-'+counting(vowels_so_far, t)+'.jpg'])
        vowels_so_far.append(t[1])
    return vowels_so_far

if __name__ == '__main__':
    vowels_so_far = [] # set of all vowels encountered so far (for numbering)
    for s in SECTIONS:
        vowels_so_far = extract_section_frames(s, vowels_so_far)

<|endoftext|>


<commit_before>load("//infra/bazel:build.bzl", "foreign_go_binary")
load("//infra/bazel:gpg.bzl", "gpg_sign")

def gen_targets(matrix):
  pkg = "v2ray.com/core/main"
  output = "v2ray"

  for (os, arch) in matrix:
    bin_name = "v2ray_" + os + "_" + arch
    foreign_go_binary(
      name = bin_name,
      pkg = pkg,
      output = output,
      os = os,
      arch = arch,
    )

    gpg_sign(
      name = bin_name + "_sig",
      base = ":" + bin_name,
    )

    if os in ["windows"]:
      bin_name = "v2ray_" + os + "_" + arch + "_nowindow"
      foreign_go_binary(
        name = bin_name,
        pkg = pkg,
        output = "w" + output,
        os = os,
        arch = arch,
        ld = "-H windowsgui",
      )

      gpg_sign(
        name = bin_name + "_sig",
        base = ":" + bin_name,
      )

      bin_name = "v2ray_" + os + "_" + arch + "_armv7_nowindow"
      foreign_go_binary(
        name = bin_name,
        pkg = pkg,
        output = "w" + output + "_armv7",
        os = os,
        arch = arch,
        arm = "7",
        ld = "-H windowsgui",
      )

      gpg_sign(
        name = bin_name + "_sig",
        base = ":" + bin_name,
      )

      bin_name = "v2ray_" + os + "_" + arch + "_armv6_nowindow"
      foreign_go_binary(
        name = bin_name,
        pkg = pkg,
        output = "w" + output + "_armv6",
        os = os,
        arch = arch,
        arm = "6",
        ld = "-H windowsgui",
      )

      gpg_sign(
        name = bin_name + "_sig",
        base = ":" + bin_name,
      )

      bin_name = "v2ray_" + os + "_" + arch + "_armv5_nowindow"
      foreign_go_binary(
        name = bin_name,
        pkg = pkg,
        output = "w" + output + "_armv5",
        os = os,
        arch = arch,
        arm = "5",
        ld = "-H windowsgui",
      )

      gpg_sign(
        name = bin_name + "_sig",
        base = ":" + bin_name,
      )

    if arch in ["mips", "mipsle"]:
      bin_name = "v2ray_" + os + "_" + arch + "_softfloat"
      foreign_go_binary(
        name = bin_name,
        pkg = pkg,
        output = output+"_softfloat",
        os = os,
        arch = arch,
        mips = "softfloat",
      )

      gpg_sign(
        name = bin_name + "_sig",
        base = ":" + bin_name,
      )

    if arch in ["arm"]:
      bin_name = "v2ray_" + os + "_" + arch + "_armv7"
      foreign_go_binary(
        name = bin_name,
        pkg = pkg,
        output = output+"_armv7",
        os = os,
        arch = arch,
        arm = "7",
      )

      gpg_sign(
        name = bin_name + "_sig",
        base = ":" + bin_name,
      )

      bin_name = "v2ray_" + os + "_" + arch + "_armv6"
      foreign_go_binary(
        name = bin_name,
        pkg = pkg,
        output = output+"_armv6",
        os = os,
        arch = arch,
        arm = "6",
      )

      gpg_sign(
        name = bin_name + "_sig",
        base = ":" + bin_name,
      )

      bin_name = "v2ray_" + os + "_" + arch + "_armv5"
      foreign_go_binary(
        name = bin_name,
        pkg = pkg,
        output = output+"_armv5",
        os = os,
        arch = arch,
        arm = "5",
      )

      gpg_sign(
        name = bin_name + "_sig",
        base = ":" + bin_name,
      )
<commit_msg>Adjust arm build<commit_after>load("//infra/bazel:build.bzl", "foreign_go_binary")
load("//infra/bazel:gpg.bzl", "gpg_sign")

def gen_targets(matrix):
  pkg = "v2ray.com/core/main"
  output = "v2ray"

  for (os, arch) in matrix:
    bin_name = "v2ray_" + os + "_" + arch
    foreign_go_binary(
      name = bin_name,
      pkg = pkg,
      output = output,
      os = os,
      arch = arch,
    )

    gpg_sign(
      name = bin_name + "_sig",
      base = ":" + bin_name,
    )

    if os in ["windows"]:
      bin_name = "v2ray_" + os + "_" + arch + "_nowindow"
      foreign_go_binary(
        name = bin_name,
        pkg = pkg,
        output = "w" + output,
        os = os,
        arch = arch,
        ld = "-H windowsgui",
      )

      gpg_sign(
        name = bin_name + "_sig",
        base = ":" + bin_name,
      )

      bin_name = "v2ray_" + os + "_" + arch + "_armv7_nowindow"
      foreign_go_binary(
        name = bin_name,
        pkg = pkg,
        output = "w" + output + "_armv7",
        os = os,
        arch = arch,
        arm = "7",
        ld = "-H windowsgui",
      )

      gpg_sign(
        name = bin_name + "_sig",
        base = ":" + bin_name,
      )

      bin_name = "v2ray_" + os + "_" + arch + "_armv6_nowindow"
      foreign_go_binary(
        name = bin_name,
        pkg = pkg,
        output = "w" + output + "_armv6",
        os = os,
        arch = arch,
        arm = "6",
        ld = "-H windowsgui",
      )

      gpg_sign(
        name = bin_name + "_sig",
        base = ":" + bin_name,
      )

      bin_name = "v2ray_" + os + "_" + arch + "_armv5_nowindow"
      foreign_go_binary(
        name = bin_name,
        pkg = pkg,
        output = "w" + output,
        os = os,
        arch = arch,
        arm = "5",
        ld = "-H windowsgui",
      )

      gpg_sign(
        name = bin_name + "_sig",
        base = ":" + bin_name,
      )

    if arch in ["mips", "mipsle"]:
      bin_name = "v2ray_" + os + "_" + arch + "_softfloat"
      foreign_go_binary(
        name = bin_name,
        pkg = pkg,
        output = output+"_softfloat",
        os = os,
        arch = arch,
        mips = "softfloat",
      )

      gpg_sign(
        name = bin_name + "_sig",
        base = ":" + bin_name,
      )

    if arch in ["arm"]:
      bin_name = "v2ray_" + os + "_" + arch + "_armv7"
      foreign_go_binary(
        name = bin_name,
        pkg = pkg,
        output = output + "_armv7",
        os = os,
        arch = arch,
        arm = "7",
      )

      gpg_sign(
        name = bin_name + "_sig",
        base = ":" + bin_name,
      )

      bin_name = "v2ray_" + os + "_" + arch + "_armv6"
      foreign_go_binary(
        name = bin_name,
        pkg = pkg,
        output = output + "_armv6",
        os = os,
        arch = arch,
        arm = "6",
      )

      gpg_sign(
        name = bin_name + "_sig",
        base = ":" + bin_name,
      )

      bin_name = "v2ray_" + os + "_" + arch + "_armv5"
      foreign_go_binary(
        name = bin_name,
        pkg = pkg,
        output = output,
        os = os,
        arch = arch,
        arm = "5",
      )

      gpg_sign(
        name = bin_name + "_sig",
        base = ":" + bin_name,
      )
<|endoftext|>


<commit_before>import numpy, sys, time, pylab
from astropy.io import fits as pyfits
from pylab import *
from matplotlib import *
from math import *
import kepio, kepmsg, kepkey, kepfunc, kepstat

def kepfilter(infile,outfile,datacol,function,cutoff,passband,plot,plotlab,
              clobber,verbose,logfile,status,cmdLine=False):

## startup parameters

    status = 0
    numpy.seterr(all="ignore")
    labelsize = 24
    ticksize = 16
    xsize = 16
    ysize = 6
    lcolor = '#0000ff'
    lwidth = 1.0
    fcolor = '#ffff00'
    falpha = 0.2

## log the call

    hashline = '----------------------------------------------------------------------------'
    kepmsg.log(logfile,hashline,verbose)
    call = 'KEPFILTER -- '
    call += 'infile='+infile+' '
    call += 'outfile='+outfile+' '
    call += 'datacol='+str(datacol)+' '
    call += 'function='+str(function)+' '
    call += 'cutoff='+str(cutoff)+' '
    call += 'passband='+str(passband)+' '
    plotit = 'n'
    if (plot): plotit = 'y'
    call += 'plot='+plotit+ ' '
    call += 'plotlab='+str(plotlab)+' '
    overwrite = 'n'
    if (clobber): overwrite = 'y'
    call += 'clobber='+overwrite+ ' '
    chatter = 'n'
    if (verbose): chatter = 'y'
    call += 'verbose='+chatter+' '
    call += 'logfile='+logfile
    kepmsg.log(logfile,call+'\n',verbose)

## start time

    kepmsg.clock('KEPFILTER started at',logfile,verbose)

## test log file

    logfile = kepmsg.test(logfile)

## clobber output file

    if clobber: status = kepio.clobber(outfile,logfile,verbose)
    if kepio.fileexists(outfile):
	    message = 'ERROR -- KEPFILTER: ' + outfile + ' exists. Use clobber=yes'
	    status = kepmsg.err(logfile,message,verbose)

## open input file

    if status == 0:
        instr, status = kepio.openfits(infile,'readonly',logfile,verbose)
        tstart, tstop, bjdref, cadence, status = kepio.timekeys(instr,infile,logfile,verbose,status)
    if status == 0:
        try:
            work = instr[0].header['FILEVER']
            cadenom = 1.0
        except:
            cadenom = cadence

## fudge non-compliant FITS keywords with no values

    if status == 0:
        instr = kepkey.emptykeys(instr,file,logfile,verbose)

## read table structure

    if status == 0:
	table, status = kepio.readfitstab(infile,instr[1],logfile,verbose)

# read time and flux columns

    if status == 0:
        barytime, status = kepio.readtimecol(infile,table,logfile,verbose)
        flux, status = kepio.readsapcol(infile,table,logfile,verbose)

# filter input data table

    if status == 0:
        try:
            nanclean = instr[1].header['NANCLEAN']
        except:
            naxis2 = 0
            for i in range(len(table.field(0))):
                if (numpy.isfinite(barytime[i]) and numpy.isfinite(flux[i]) and flux[i] != 0.0):
                    table[naxis2] = table[i]
                    naxis2 += 1
            instr[1].data = table[:naxis2]
            comment = 'NaN cadences removed from data'
            status = kepkey.new('NANCLEAN',True,comment,instr[1],outfile,logfile,verbose)

## read table columns

    if status == 0:
        intime, status = kepio.readtimecol(infile,instr[1].data,logfile,verbose)
    if status == 0:
	indata, status = kepio.readfitscol(infile,instr[1].data,datacol,logfile,verbose)
    if status == 0:
        intime = intime + bjdref
        indata = indata / cadenom

## define data sampling

    if status == 0:
        tr = 1.0 / (cadence / 86400)
        timescale = 1.0 / (cutoff / tr)

## define convolution function

    if status == 0:
        if function == 'boxcar':
            filtfunc = numpy.ones(numpy.ceil(timescale))
        elif function == 'gauss':
            timescale /= 2
            dx = numpy.ceil(timescale * 10 + 1)
            filtfunc = kepfunc.gauss()
            filtfunc = filtfunc([1.0,dx/2-1.0,timescale],linspace(0,dx-1,dx))
        elif function == 'sinc':
            dx = numpy.ceil(timescale * 12 + 1)
            fx = linspace(0,dx-1,dx)
            fx = fx - dx / 2 + 0.5
            fx /= timescale
            filtfunc = numpy.sinc(fx)
        filtfunc /= numpy.sum(filtfunc)

## pad time series at both ends with noise model

    if status == 0:
        ave, sigma  = kepstat.stdev(indata[:len(filtfunc)])
        padded = append(kepstat.randarray(np.ones(len(filtfunc)) * ave,
                                          np.ones(len(filtfunc)) * sigma), indata)
        ave, sigma  = kepstat.stdev(indata[-len(filtfunc):])
        padded = append(padded, kepstat.randarray(np.ones(len(filtfunc)) * ave,
                                                  np.ones(len(filtfunc)) * sigma))

## convolve data

    if status == 0:
        convolved = convolve(padded,filtfunc,'same')

## remove padding from the output array

    if status == 0:
        if function == 'boxcar':
            outdata = convolved[len(filtfunc):-len(filtfunc)]
        else:
            outdata = convolved[len(filtfunc):-len(filtfunc)]

## subtract low frequencies

    if status == 0 and passband == 'high':
        outmedian = median(outdata)
        outdata = indata - outdata + outmedian

## comment keyword in output file

    if status == 0:
        status = kepkey.history(call,instr[0],outfile,logfile,verbose)

## clean up x-axis unit

    if status == 0:
	intime0 = float(int(tstart / 100) * 100.0)
        if intime0 < 2.4e6: intime0 += 2.4e6
	ptime = intime - intime0
	xlab = 'BJD $-$ %d' % intime0

## clean up y-axis units

    if status == 0:
        pout = indata * 1.0
        pout2 = outdata * 1.0
	nrm = len(str(int(numpy.nanmax(pout))))-1
	pout = pout / 10**nrm
	pout2 = pout2 / 10**nrm
	ylab = '10$^%d$ %s' % (nrm, plotlab)

## data limits

	xmin = ptime.min()
	xmax = ptime.max()
	ymin = numpy.nanmin(pout)
	ymax = numpy.nanmax(pout)
	xr = xmax - xmin
	yr = ymax - ymin
        ptime = insert(ptime,[0],[ptime[0]])
        ptime = append(ptime,[ptime[-1]])
        pout = insert(pout,[0],[0.0])
        pout = append(pout,0.0)
        pout2 = insert(pout2,[0],[0.0])
        pout2 = append(pout2,0.0)

## plot light curve

    if status == 0 and plot:
        try:
            params = {'backend': 'png',
                      'axes.linewidth': 2.5,
                      'axes.labelsize': labelsize,
                      'axes.font': 'sans-serif',
                      'axes.fontweight' : 'bold',
                      'text.fontsize': 12,
                      'legend.fontsize': 12,
                      'xtick.labelsize': ticksize,
                      'ytick.labelsize': ticksize}
            rcParams.update(params)
        except:
            print 'ERROR -- KEPFILTER: install latex for scientific plotting'
            status = 1
    if status == 0 and plot:
        pylab.figure(figsize=[xsize,ysize])
        pylab.clf()

## plot filtered data

        ax = pylab.axes([0.06,0.1,0.93,0.87])
        pylab.gca().xaxis.set_major_formatter(pylab.ScalarFormatter(useOffset=False))
        pylab.gca().yaxis.set_major_formatter(pylab.ScalarFormatter(useOffset=False))
        labels = ax.get_yticklabels()
        setp(labels, 'rotation', 90, fontsize=12)
        pylab.plot(ptime,pout,color='#ff9900',linestyle='-',linewidth=lwidth)
        fill(ptime,pout,color=fcolor,linewidth=0.0,alpha=falpha)
        if passband == 'low':
            pylab.plot(ptime[1:-1],pout2[1:-1],color=lcolor,linestyle='-',linewidth=lwidth)
        else:
            pylab.plot(ptime,pout2,color=lcolor,linestyle='-',linewidth=lwidth)
            fill(ptime,pout2,color=lcolor,linewidth=0.0,alpha=falpha)
	xlabel(xlab, {'color' : 'k'})
	ylabel(ylab, {'color' : 'k'})
	xlim(xmin-xr*0.01,xmax+xr*0.01)
        if ymin >= 0.0:
            ylim(ymin-yr*0.01,ymax+yr*0.01)
        else:
            ylim(1.0e-10,ymax+yr*0.01)
        pylab.grid()

# render plot

        if cmdLine:
            pylab.show()
        else:
            pylab.ion()
            pylab.plot([])
            pylab.ioff()

## write output file

    if status == 0:
        for i in range(len(outdata)):
            instr[1].data.field(datacol)[i] = outdata[i]
        instr.writeto(outfile)

## close input file

    if status == 0:
        status = kepio.closefits(instr,logfile,verbose)

## end time

    if (status == 0):
	    message = 'KEPFILTER completed at'
    else:
	    message = '\nKEPFILTER aborted at'
    kepmsg.clock(message,logfile,verbose)

## main
if '--shell' in sys.argv:
    import argparse

    parser = argparse.ArgumentParser(description='Low bandpass or high bandpass signal filtering')
    parser.add_argument('--shell', action='store_true', help='Are we running from the shell?')

    parser.add_argument('infile', help='Name of input file', type=str)
    parser.add_argument('outfile', help='Name of FITS file to output', type=str)

    parser.add_argument('--datacol', default='SAP_FLUX', help='Name of data column', type=str)
    parser.add_argument('--function', default='boxcar', help='The bandpass convolution function', type=str,
        choices=['boxcar','gauss','sinc'])

    parser.add_argument('--cutoff', default=1.0, help='Characteristic frequency cutoff of filter [1/days]', type=float)
    parser.add_argument('--passband', help='low- or high-bandpass filter', type=str, choices=['low','high'])

    parser.add_argument('--plot', action='store_true', help='Plot result?')

    parser.add_argument('--clobber', action='store_true', help='Overwrite output file?')
    parser.add_argument('--verbose', action='store_true', help='Write to a log file?')
    parser.add_argument('--logfile', '-l', help='Name of ascii log file', default='kepcotrend.log', dest='logfile', type=str)
    parser.add_argument('--status', '-e', help='Exit status (0=good)', default=0, dest='status', type=int)

    args = parser.parse_args()

    cmdLine=True

    kepfilter(args.infile,args.outfile,args.datacol,args.function,args.cutoff,args.passband,args.plot,'e$^-$ s$^{-1}$',
              args.clobber,args.verbose,args.logfile,args.status,cmdLine)

else:
    from pyraf import iraf
    parfile = iraf.osfn("kepler$kepfilter.par")
    t = iraf.IrafTaskFactory(taskname="kepfilter", value=parfile, function=kepfilter)
<commit_msg>fix kepfilter<commit_after>import sys, time
from astropy.io import fits as pyfits
from matplotlib import pyplot as plt
import numpy as np
import kepio, kepmsg, kepkey, kepfunc, kepstat

def kepfilter(infile,outfile,datacol,function,cutoff,passband,plot,plotlab,
              clobber,verbose,logfile,status,cmdLine=False):

## startup parameters

    status = 0
    np.seterr(all="ignore")
    labelsize = 24
    ticksize = 16
    xsize = 16
    ysize = 6
    lcolor = '#0000ff'
    lwidth = 1.0
    fcolor = '#ffff00'
    falpha = 0.2

## log the call

    hashline = '----------------------------------------------------------------------------'
    kepmsg.log(logfile,hashline,verbose)
    call = 'KEPFILTER -- '
    call += 'infile='+infile+' '
    call += 'outfile='+outfile+' '
    call += 'datacol='+str(datacol)+' '
    call += 'function='+str(function)+' '
    call += 'cutoff='+str(cutoff)+' '
    call += 'passband='+str(passband)+' '
    plotit = 'n'
    if (plot): plotit = 'y'
    call += 'plot='+plotit+ ' '
    call += 'plotlab='+str(plotlab)+' '
    overwrite = 'n'
    if (clobber): overwrite = 'y'
    call += 'clobber='+overwrite+ ' '
    chatter = 'n'
    if (verbose): chatter = 'y'
    call += 'verbose='+chatter+' '
    call += 'logfile='+logfile
    kepmsg.log(logfile,call+'\n',verbose)

## start time

    kepmsg.clock('KEPFILTER started at',logfile,verbose)

## test log file

    logfile = kepmsg.test(logfile)

## clobber output file

    if clobber: status = kepio.clobber(outfile,logfile,verbose)
    if kepio.fileexists(outfile):
	    message = 'ERROR -- KEPFILTER: ' + outfile + ' exists. Use clobber=yes'
	    status = kepmsg.err(logfile,message,verbose)

## open input file

    if status == 0:
        instr, status = kepio.openfits(infile,'readonly',logfile,verbose)
        tstart, tstop, bjdref, cadence, status = kepio.timekeys(instr,infile,logfile,verbose,status)
    if status == 0:
        try:
            work = instr[0].header['FILEVER']
            cadenom = 1.0
        except:
            cadenom = cadence

## fudge non-compliant FITS keywords with no values

    if status == 0:
        instr = kepkey.emptykeys(instr,file,logfile,verbose)

## read table structure

    if status == 0:
	table, status = kepio.readfitstab(infile,instr[1],logfile,verbose)

# read time and flux columns

    if status == 0:
        barytime, status = kepio.readtimecol(infile,table,logfile,verbose)
        flux, status = kepio.readsapcol(infile,table,logfile,verbose)

# filter input data table

    if status == 0:
        try:
            nanclean = instr[1].header['NANCLEAN']
        except:
            naxis2 = 0
            for i in range(len(table.field(0))):
                if (np.isfinite(barytime[i]) and np.isfinite(flux[i]) and flux[i] != 0.0):
                    table[naxis2] = table[i]
                    naxis2 += 1
            instr[1].data = table[:naxis2]
            comment = 'NaN cadences removed from data'
            status = kepkey.new('NANCLEAN',True,comment,instr[1],outfile,logfile,verbose)

## read table columns

    if status == 0:
        intime, status = kepio.readtimecol(infile,instr[1].data,logfile,verbose)
    if status == 0:
	indata, status = kepio.readfitscol(infile,instr[1].data,datacol,logfile,verbose)
    if status == 0:
        intime = intime + bjdref
        indata = indata / cadenom

## define data sampling

    if status == 0:
        tr = 1.0 / (cadence / 86400)
        timescale = 1.0 / (cutoff / tr)

## define convolution function

    if status == 0:
        if function == 'boxcar':
            filtfunc = np.ones(int(np.ceil(timescale)))
        elif function == 'gauss':
            timescale /= 2
            dx = np.ceil(timescale * 10 + 1)
            filtfunc = kepfunc.gauss()
            filtfunc = filtfunc([1.0,dx/2-1.0,timescale],linspace(0,dx-1,dx))
        elif function == 'sinc':
            dx = np.ceil(timescale * 12 + 1)
            fx = np.linspace(0,dx-1,dx)
            fx = fx - dx / 2 + 0.5
            fx /= timescale
            filtfunc = np.sinc(fx)
        filtfunc /= np.sum(filtfunc)

## pad time series at both ends with noise model

    if status == 0:
        ave, sigma  = kepstat.stdev(indata[:len(filtfunc)])
        padded = np.append(kepstat.randarray(np.ones(len(filtfunc)) * ave,
                                          np.ones(len(filtfunc)) * sigma), indata)
        ave, sigma  = kepstat.stdev(indata[-len(filtfunc):])
        padded = np.append(padded, kepstat.randarray(np.ones(len(filtfunc)) * ave,
                                                  np.ones(len(filtfunc)) * sigma))

## convolve data

    if status == 0:
        convolved = np.convolve(padded,filtfunc,'same')

## remove padding from the output array

    if status == 0:
        if function == 'boxcar':
            outdata = convolved[len(filtfunc):-len(filtfunc)]
        else:
            outdata = convolved[len(filtfunc):-len(filtfunc)]

## subtract low frequencies

    if status == 0 and passband == 'high':
        outmedian = np.median(outdata)
        outdata = indata - outdata + outmedian

## comment keyword in output file

    if status == 0:
        status = kepkey.history(call,instr[0],outfile,logfile,verbose)

## clean up x-axis unit

    if status == 0:
	intime0 = float(int(tstart / 100) * 100.0)
        if intime0 < 2.4e6: intime0 += 2.4e6
	ptime = intime - intime0
	xlab = 'BJD $-$ %d' % intime0

## clean up y-axis units

    if status == 0:
        pout = indata * 1.0
        pout2 = outdata * 1.0
	nrm = len(str(int(np.nanmax(pout))))-1
	pout = pout / 10**nrm
	pout2 = pout2 / 10**nrm
	ylab = '10$^%d$ %s' % (nrm, plotlab)

## data limits

	xmin = ptime.min()
	xmax = ptime.max()
	ymin = np.nanmin(pout)
	ymax = np.nanmax(pout)
	xr = xmax - xmin
	yr = ymax - ymin
        ptime = np.insert(ptime,[0],[ptime[0]])
        ptime = np.append(ptime,[ptime[-1]])
        pout = np.insert(pout,[0],[0.0])
        pout = np.append(pout,0.0)
        pout2 = np.insert(pout2,[0],[0.0])
        pout2 = np.append(pout2,0.0)

## plot light curve

    if status == 0 and plot:
        plt.figure(figsize=[xsize,ysize])
        plt.clf()

## plot filtered data

        ax = plt.axes([0.06,0.1,0.93,0.87])
        plt.gca().xaxis.set_major_formatter(plt.ScalarFormatter(useOffset=False))
        plt.gca().yaxis.set_major_formatter(plt.ScalarFormatter(useOffset=False))
        labels = ax.get_yticklabels()
        plt.setp(labels, 'rotation', 90, fontsize=12)
        plt.plot(ptime,pout,color='#ff9900',linestyle='-',linewidth=lwidth)
        plt.fill(ptime,pout,color=fcolor,linewidth=0.0,alpha=falpha)
        if passband == 'low':
            plt.plot(ptime[1:-1],pout2[1:-1],color=lcolor,linestyle='-',linewidth=lwidth)
        else:
            plt.plot(ptime,pout2,color=lcolor,linestyle='-',linewidth=lwidth)
            plt.fill(ptime,pout2,color=lcolor,linewidth=0.0,alpha=falpha)
	plt.xlabel(xlab, {'color' : 'k'})
	plt.ylabel(ylab, {'color' : 'k'})
	plt.xlim(xmin-xr*0.01,xmax+xr*0.01)
        if ymin >= 0.0:
            plt.ylim(ymin-yr*0.01,ymax+yr*0.01)
        else:
            plt.ylim(1.0e-10,ymax+yr*0.01)
        plt.grid()
# render plot
        plt.show()
## write output file

    if status == 0:
        for i in range(len(outdata)):
            instr[1].data.field(datacol)[i] = outdata[i]
        instr.writeto(outfile)

## close input file

    if status == 0:
        status = kepio.closefits(instr,logfile,verbose)

## end time

    if (status == 0):
	    message = 'KEPFILTER completed at'
    else:
	    message = '\nKEPFILTER aborted at'
    kepmsg.clock(message,logfile,verbose)

## main
if '--shell' in sys.argv:
    import argparse

    parser = argparse.ArgumentParser(description='Low bandpass or high bandpass signal filtering')
    parser.add_argument('--shell', action='store_true', help='Are we running from the shell?')
    parser.add_argument('infile', help='Name of input file', type=str)
    parser.add_argument('outfile', help='Name of FITS file to output', type=str)
    parser.add_argument('--datacol', default='SAP_FLUX', help='Name of data column', type=str)
    parser.add_argument('--function', default='boxcar', help='The bandpass convolution function', type=str,
                        choices=['boxcar','gauss','sinc'])
    parser.add_argument('--cutoff', default=1.0, help='Characteristic frequency cutoff of filter [1/days]', type=float)
    parser.add_argument('--passband', help='low- or high-bandpass filter', type=str, choices=['low','high'])
    parser.add_argument('--plot', action='store_true', help='Plot result?')
    parser.add_argument('--clobber', action='store_true', help='Overwrite output file?')
    parser.add_argument('--verbose', action='store_true', help='Write to a log file?')
    parser.add_argument('--logfile', '-l', help='Name of ascii log file',
                        default='kepcotrend.log', dest='logfile', type=str)
    parser.add_argument('--status', '-e', help='Exit status (0=good)', default=0, dest='status', type=int)

    args = parser.parse_args()
    cmdLine=True
    kepfilter(args.infile,args.outfile,args.datacol,args.function,args.cutoff,args.passband,args.plot,'e$^-$ s$^{-1}$',
              args.clobber,args.verbose,args.logfile,args.status,cmdLine)
else:
    from pyraf import iraf
    parfile = iraf.osfn("kepler$kepfilter.par")
    t = iraf.IrafTaskFactory(taskname="kepfilter", value=parfile, function=kepfilter)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import asyncio
import functools
# import aiohttp_jinja2
from aiohttp import web
from aiohttp.abc import AbstractView
from aiohttp_auth.auth import get_auth
from aiohttp_jinja2 import APP_KEY, render_template

async def http_400_response(error_reason):
    return web.json_response({
        'status': 'error',
        'content': error_reason
    }, status=400)

async def http_401_response(error_reason):
    return web.json_response({
        'status': 'error',
        'content': error_reason
    }, status=401)

async def http_404_response(request, msg='Page Not Found!'):
    res = render_template('error/404.html', request, {'msg': msg})
    res.set_status(404)
    return res

def geass(template_name, *, app_key=APP_KEY, encoding='utf-8', status=200):

    def wrapper(func):
        @functools.wraps(func)
        async def wrapped(*args):
            if asyncio.iscoroutinefunction(func):
                coro = func
            else:
                coro = asyncio.coroutine(func)
            context = await coro(*args)

            # if isinstance(context, web.StreamResponse):
            #     return context

            if isinstance(args[0], AbstractView):
                request = args[0].request
            else:
                request = args[-1]

            if (await get_auth(request)) is None:
                # 未登录的
                if context is not None:
                    context['isauth'] = False
            else:
                if context is not None:
                    context['isauth'] = True

            response = render_template(template_name, request, context,
                                       app_key=app_key, encoding=encoding)
            response.set_status(status)
            return response
        return wrapped
    return wrapper
<commit_msg>Fix object does not support item assignment<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import asyncio
import functools
# import aiohttp_jinja2
from aiohttp import web
from aiohttp.abc import AbstractView
from aiohttp_auth.auth import get_auth
from aiohttp_jinja2 import APP_KEY, render_template

async def http_400_response(error_reason):
    return web.json_response({
        'status': 'error',
        'content': error_reason
    }, status=400)

async def http_401_response(error_reason):
    return web.json_response({
        'status': 'error',
        'content': error_reason
    }, status=401)

async def http_404_response(request, msg='Page Not Found!'):
    res = render_template('error/404.html', request, {'msg': msg})
    res.set_status(404)
    return res

def geass(template_name, *, app_key=APP_KEY, encoding='utf-8', status=200):

    def wrapper(func):
        @functools.wraps(func)
        async def wrapped(*args):
            if asyncio.iscoroutinefunction(func):
                coro = func
            else:
                coro = asyncio.coroutine(func)
            context = await coro(*args)

            # if isinstance(context, web.StreamResponse):
            #     return context

            if isinstance(args[0], AbstractView):
                request = args[0].request
            else:
                request = args[-1]

            if context is None:
                 context = {}
            if (await get_auth(request)) is None:
                # 未登录的
                context['isauth'] = False
            else:
                context['isauth'] = True

            response = render_template(template_name, request, context,
                                       app_key=app_key, encoding=encoding)
            response.set_status(status)
            return response
        return wrapped
    return wrapper
<|endoftext|>


<commit_before># Convert MACS output file to XLS spreadsheet
#
import sys
import os
import string
import xlwt

def write_data_to_sheet(worksheet,data):
    """Write a list of data to an xlwt sheet

    Given a list of rows with tab-separated data items,
    write the data to an xlwt worksheet.

    Arguments:
      worksheet: an xlwt.Workbook object (i.e. XLS worksheet)
      data: Python list representing rows of tab-separated
        data items
    """
    rindex = 0
    for line in data:
        cindex = 0
        for item in line.split('\t'):
            if str(item).startswith('='):
                # Formula item
                formula = ''
                for c in item[1:]:
                    formula += c
                    if c.isalpha() and c.isupper:
                        # Add the row number afterwards
                        # NB xlwt takes row numbers from zero,
                        # while XLS starts from 1
                        formula += str(rindex+1)
                ##print "%s" % formula
                worksheet.write(rindex,cindex,xlwt.Formula(formula))
            else:
                # Data item
                worksheet.write(rindex,cindex,item)
            cindex += 1
        rindex += 1

def get_column_id(data,name):
    """Lookup XLS column id from name of column

    If there is no data, or if the name isn't in the header
    row of the data, then an exception is raised.
    """
    i = data[0].split('\t').index(name)
    return string.uppercase[i]

def insert_column_into_data(data,position,title='',insert_item=''):
    """Insert a column into a list of data

    This inserts a new column into each row of data, at the
    specified positional index (starting from 0).

    If insert_item starts with '=' then it's interpreted as a row-wise
    formula. Formulas can be written in the form e.g. "=A+B-C", where
    the letters indicate columns in the final XLS spreadsheet. When the
    formulae are written they are expanded to include the row number
    e.g. "=A1+B1-C1", "A2+B2-C2" etc.

    Arguments:
      data: Python list representing rows of tab-separated data items
      position: positional index for the column to be inserted
        at (0=A, 1=B etc)
      title: value to be written to the first row (i.e. a column
        title)
      insert_item: value to be inserted; can be blank, a constant
        value, or a formula.
    """
    insert_title = True
    new_data = []
    # Loop over rows
    for line in data:
        items = line.split('\t')
        new_items = []
        for i in range(len(items)):
            # Insert appropriate item at this position
            if i == position:
                if insert_title:
                    # Title for new column
                    new_items.append(title)
                    insert_title = False
                else:
                    # Data item
                    new_items.append(insert_item)
            new_items.append(items[i])
        new_data.append('\t'.join(new_items))
    return new_data

if __name__ == "__main__":
    # Get input file name
    if len(sys.argv) != 2:
        print "Usage: %s <macs_file>" % sys.argv[0]
        sys.exit(1)
    macs_in = sys.argv[1]

    # Build output file name
    # XLS_<input_name>.xls
    # Note that MACS output file might already have an .xls extension
    # but we'll add an explicit .xls extension
    xls_out = "XLS_"+os.path.splitext(os.path.basename(macs_in))[0]+".xls"
    print "Input file: %s" % macs_in
    print "Output XLS: %s" % xls_out

    # Read in the MACS file and split into header and data
    header = []
    data = []
    fp = open(macs_in,'r')
    for line in fp:
        # Strip trailing newlines
        line0 = line.strip('\n')
        if line0.strip() == '':
            # Skip empty lines
            pass
        elif line0.startswith('#'):
            # Header line
            header.append(line0)
        else:
            # Data line
            if not data:
                # First line of data: prepend with '#'
                line0 = '#'+line0
            data.append(line0)
    fp.close()

    # Insert formulae columns
    #
    # Summit-100 = ("start"+"summit" columns) - 100
    data = insert_column_into_data(data,3,
                                   title="summit-100",
                                   insert_item="=(B+G)-100")
    # Summit+100 = ("start"+"summit" columns) + 100
    data = insert_column_into_data(data,4,
                                   title="summit+100",
                                   insert_item="=(B+G)+100")

    # Alternative (commented out)
    ##start = get_column_id(data,"start")
    ##end = get_column_id(data,"end")
    ##print "start = %s end = %s" % (start,end)
    ##formula = "=%s-%s+1" % (end,start)
    ##data = insert_column_into_data(data,3,insert_item=formula,title="Diff")

    # Notes text example
    notes = """Here is some example text for Ian

Edit this as you wish and it will be added to the "notes" page
of the spreadsheet, preserving the line breaks etc

(hopefully it will work okay :)
"""
    # Create a new spreadsheet
    wb = xlwt.Workbook()

    # Create sheets: notes, data and header
    ws_notes = wb.add_sheet("Notes")
    ws_data = wb.add_sheet("Data")
    ws_header = wb.add_sheet("Header")

    # Write data to the data sheet
    write_data_to_sheet(ws_data,data)

    # Write header to the header sheet
    write_data_to_sheet(ws_header,header)

    # Write notes to the note sheet
    write_data_to_sheet(ws_notes,notes.split('\n'))

    # Write the spreadsheet to file
    wb.save(xls_out)

<commit_msg>make_macs_xls.py: Added comment to the effect that putting tab characters into the "notes" text will make new columns in a single line.<commit_after># Convert MACS output file to XLS spreadsheet
#
import sys
import os
import string
import xlwt

def write_data_to_sheet(worksheet,data):
    """Write a list of data to an xlwt sheet

    Given a list of rows with tab-separated data items,
    write the data to an xlwt worksheet.

    Arguments:
      worksheet: an xlwt.Workbook object (i.e. XLS worksheet)
      data: Python list representing rows of tab-separated
        data items
    """
    rindex = 0
    for line in data:
        cindex = 0
        for item in line.split('\t'):
            if str(item).startswith('='):
                # Formula item
                formula = ''
                for c in item[1:]:
                    formula += c
                    if c.isalpha() and c.isupper:
                        # Add the row number afterwards
                        # NB xlwt takes row numbers from zero,
                        # while XLS starts from 1
                        formula += str(rindex+1)
                ##print "%s" % formula
                worksheet.write(rindex,cindex,xlwt.Formula(formula))
            else:
                # Data item
                worksheet.write(rindex,cindex,item)
            cindex += 1
        rindex += 1

def get_column_id(data,name):
    """Lookup XLS column id from name of column

    If there is no data, or if the name isn't in the header
    row of the data, then an exception is raised.
    """
    i = data[0].split('\t').index(name)
    return string.uppercase[i]

def insert_column_into_data(data,position,title='',insert_item=''):
    """Insert a column into a list of data

    This inserts a new column into each row of data, at the
    specified positional index (starting from 0).

    If insert_item starts with '=' then it's interpreted as a row-wise
    formula. Formulas can be written in the form e.g. "=A+B-C", where
    the letters indicate columns in the final XLS spreadsheet. When the
    formulae are written they are expanded to include the row number
    e.g. "=A1+B1-C1", "A2+B2-C2" etc.

    Arguments:
      data: Python list representing rows of tab-separated data items
      position: positional index for the column to be inserted
        at (0=A, 1=B etc)
      title: value to be written to the first row (i.e. a column
        title)
      insert_item: value to be inserted; can be blank, a constant
        value, or a formula.
    """
    insert_title = True
    new_data = []
    # Loop over rows
    for line in data:
        items = line.split('\t')
        new_items = []
        for i in range(len(items)):
            # Insert appropriate item at this position
            if i == position:
                if insert_title:
                    # Title for new column
                    new_items.append(title)
                    insert_title = False
                else:
                    # Data item
                    new_items.append(insert_item)
            new_items.append(items[i])
        new_data.append('\t'.join(new_items))
    return new_data

if __name__ == "__main__":
    # Get input file name
    if len(sys.argv) != 2:
        print "Usage: %s <macs_file>" % sys.argv[0]
        sys.exit(1)
    macs_in = sys.argv[1]

    # Build output file name
    # XLS_<input_name>.xls
    # Note that MACS output file might already have an .xls extension
    # but we'll add an explicit .xls extension
    xls_out = "XLS_"+os.path.splitext(os.path.basename(macs_in))[0]+".xls"
    print "Input file: %s" % macs_in
    print "Output XLS: %s" % xls_out

    # Read in the MACS file and split into header and data
    header = []
    data = []
    fp = open(macs_in,'r')
    for line in fp:
        # Strip trailing newlines
        line0 = line.strip('\n')
        if line0.strip() == '':
            # Skip empty lines
            pass
        elif line0.startswith('#'):
            # Header line
            header.append(line0)
        else:
            # Data line
            if not data:
                # First line of data: prepend with '#'
                line0 = '#'+line0
            data.append(line0)
    fp.close()

    # Insert formulae columns
    #
    # Summit-100 = ("start"+"summit" columns) - 100
    data = insert_column_into_data(data,3,
                                   title="summit-100",
                                   insert_item="=(B+G)-100")
    # Summit+100 = ("start"+"summit" columns) + 100
    data = insert_column_into_data(data,4,
                                   title="summit+100",
                                   insert_item="=(B+G)+100")

    # Alternative (commented out)
    ##start = get_column_id(data,"start")
    ##end = get_column_id(data,"end")
    ##print "start = %s end = %s" % (start,end)
    ##formula = "=%s-%s+1" % (end,start)
    ##data = insert_column_into_data(data,3,insert_item=formula,title="Diff")

    # Notes text example
    # NB put tabs into the text to create new columns within a single line
    notes = """Here is some example text for\tIan

Edit this as you wish and it will be added to the "notes" page
of the spreadsheet, preserving the line breaks etc

(hopefully it will work okay :)
"""
    # Create a new spreadsheet
    wb = xlwt.Workbook()

    # Create sheets: notes, data and header
    ws_notes = wb.add_sheet("Notes")
    ws_data = wb.add_sheet("Data")
    ws_header = wb.add_sheet("Header")

    # Write data to the data sheet
    write_data_to_sheet(ws_data,data)

    # Write header to the header sheet
    write_data_to_sheet(ws_header,header)

    # Write notes to the note sheet
    write_data_to_sheet(ws_notes,notes.split('\n'))

    # Write the spreadsheet to file
    wb.save(xls_out)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
JSON-LD extractor
"""

import json
import re

import lxml.etree
import lxml.html

HTML_OR_JS_COMMENTLINE = re.compile('^\s*(//.*|<!--.*-->)')

class JsonLdExtractor(object):
    _xp_jsonld = lxml.etree.XPath('descendant-or-self::script[@type="application/ld+json"]')

    def extract(self, htmlstring, base_url=None, encoding="UTF-8"):
        parser = lxml.html.HTMLParser(encoding=encoding)
        lxmldoc = lxml.html.fromstring(htmlstring, parser=parser)
        return self.extract_items(lxmldoc, base_url=base_url)

    def extract_items(self, document, base_url=None):
        return [item for items in map(self._extract_items,
                                      self._xp_jsonld(document))
                     for item in items
                         if item]

    def _extract_items(self, node):
        script = node.xpath('string()')
        try:
            data = json.loads(script)
        except ValueError:
            # sometimes JSON-decoding errors are due to leading HTML or JavaScript comments
            try:
                data = json.loads(HTML_OR_JS_COMMENTLINE.sub('', script))
            except json.JSONDecodeError:
                # some pages have JSON-LD data with control characters, json.loads should use strict=False
                data = json.loads(script, strict=False)
        if isinstance(data, list):
            return data
        elif isinstance(data, dict):
            return [data]
<commit_msg>Fix: Make tests pass, `json.JSONDecodeError` is only available for Python 3.5 and on.<commit_after># -*- coding: utf-8 -*-
"""
JSON-LD extractor
"""

import json
import re

import lxml.etree
import lxml.html

HTML_OR_JS_COMMENTLINE = re.compile('^\s*(//.*|<!--.*-->)')

class JsonLdExtractor(object):
    _xp_jsonld = lxml.etree.XPath('descendant-or-self::script[@type="application/ld+json"]')

    def extract(self, htmlstring, base_url=None, encoding="UTF-8"):
        parser = lxml.html.HTMLParser(encoding=encoding)
        lxmldoc = lxml.html.fromstring(htmlstring, parser=parser)
        return self.extract_items(lxmldoc, base_url=base_url)

    def extract_items(self, document, base_url=None):
        return [item for items in map(self._extract_items,
                                      self._xp_jsonld(document))
                     for item in items
                         if item]

    def _extract_items(self, node):
        script = node.xpath('string()')
        try:
            data = json.loads(script)
        except ValueError:
            # sometimes JSON-decoding errors are due to leading HTML or JavaScript comments
            try:
                data = json.loads(HTML_OR_JS_COMMENTLINE.sub('', script))
            except ValueError: # ValueError again because json.JSONDecodeError appears since Python 3.5
                # some pages have JSON-LD data with control characters, json.loads should use strict=False
                data = json.loads(script, strict=False)
            except json.JSONDecodeError: # From Python 3.5 and on, this is the proper error
                # some pages have JSON-LD data with control characters, json.loads should use strict=False
                data = json.loads(script, strict=False)
        if isinstance(data, list):
            return data
        elif isinstance(data, dict):
            return [data]
<|endoftext|>