<commit_before>#!/usr/bin/python
# @lint-avoid-python-3-compatibility-imports
#
# ucalls  Summarize method calls in high-level languages and/or system calls.
#         For Linux, uses BCC, eBPF.
#
# USAGE: ucalls [-l {java,python,ruby}] [-h] [-T TOP] [-L] [-S] [-v] [-m]
#        pid [interval]
#
# Copyright 2016 Sasha Goldshtein
# Licensed under the Apache License, Version 2.0 (the "License")
#
# 19-Oct-2016   Sasha Goldshtein   Created this.

from __future__ import print_function
import argparse
from bcc import BPF, USDT
from time import sleep

examples = """examples:
    ./ucalls -l java 185        # trace Java calls and print statistics on ^C
    ./ucalls -l python 2020 1   # trace Python calls and print every second
    ./ucalls -l java 185 -S     # trace Java calls and syscalls
    ./ucalls 6712 -S            # trace only syscall counts
    ./ucalls -l ruby 1344 -T 10 # trace top 10 Ruby method calls
    ./ucalls -l ruby 1344 -L    # trace Ruby calls including latency
    ./ucalls -l ruby 1344 -LS   # trace Ruby calls and syscalls with latency
    ./ucalls -l python 2020 -mL # trace Python calls including latency in ms
"""
parser = argparse.ArgumentParser(
    description="Summarize method calls in high-level languages.",
    formatter_class=argparse.RawDescriptionHelpFormatter,
    epilog=examples)
parser.add_argument("pid", type=int, help="process id to attach to")
parser.add_argument("interval", type=int, nargs='?',
    help="print every specified number of seconds")
parser.add_argument("-l", "--language", choices=["java", "python", "ruby"],
    help="language to trace (if none, trace syscalls only)")
parser.add_argument("-T", "--top", type=int,
    help="number of most frequent/slow calls to print")
parser.add_argument("-L", "--latency", action="store_true",
    help="record method latency from enter to exit (except recursive calls)")
parser.add_argument("-S", "--syscalls", action="store_true",
    help="record syscall latency (adds overhead)")
parser.add_argument("-v", "--verbose", action="store_true",
    help="verbose mode: print the BPF program (for debugging purposes)")
parser.add_argument("-m", "--milliseconds", action="store_true",
    help="report times in milliseconds (default is microseconds)")
args = parser.parse_args()

# We assume that the entry and return probes have the same arguments. This is
# the case for Java, Python, and Ruby. If there's a language where it's not the
# case, we will need to build a custom correlator from entry to exit.
if args.language == "java":
    # TODO for JVM entries, we actually have the real length of the class
    #      and method strings in arg3 and arg5 respectively, so we can insert
    #      the null terminator in its proper position.
    entry_probe = "method__entry"
    return_probe = "method__return"
    read_class = "bpf_usdt_readarg(2, ctx, &clazz);"
    read_method = "bpf_usdt_readarg(4, ctx, &method);"
elif args.language == "python":
    entry_probe = "function__entry"
    return_probe = "function__return"
    read_class = "bpf_usdt_readarg(1, ctx, &clazz);"    # filename really
    read_method = "bpf_usdt_readarg(2, ctx, &method);"
elif args.language == "ruby":
    entry_probe = "method__entry"
    return_probe = "method__return"
    read_class = "bpf_usdt_readarg(1, ctx, &clazz);"
    read_method = "bpf_usdt_readarg(2, ctx, &method);"
elif not args.language:
    if not args.syscalls:
        print("Nothing to do; use -S to trace syscalls.")
        exit(1)
    entry_probe, return_probe, read_class, read_method = ("", "", "", "")

program = """
#include <linux/ptrace.h>

#define MAX_STRING_LENGTH 80
DEFINE_NOLANG
DEFINE_LATENCY
DEFINE_SYSCALLS

struct method_t {
    char clazz[MAX_STRING_LENGTH];
    char method[MAX_STRING_LENGTH];
};
struct entry_t {
    u64 pid;
    struct method_t method;
};
struct info_t {
    u64 num_calls;
    u64 total_ns;
};
struct syscall_entry_t {
    u64 timestamp;
    u64 ip;
};

#ifndef LATENCY
  BPF_HASH(counts, struct method_t, u64);            // number of calls
  #ifdef SYSCALLS
    BPF_HASH(syscounts, u64, u64);                   // number of calls per IP
  #endif  // SYSCALLS
#else
  BPF_HASH(times, struct method_t, struct info_t);
  BPF_HASH(entry, struct entry_t, u64);              // timestamp at entry
  #ifdef SYSCALLS
    BPF_HASH(systimes, u64, struct info_t);          // latency per IP
    BPF_HASH(sysentry, u64, struct syscall_entry_t); // ts + IP at entry
  #endif  // SYSCALLS
#endif

#ifndef NOLANG
int trace_entry(struct pt_regs *ctx) {
    u64 clazz = 0, method = 0, val = 0;
    u64 *valp;
    struct entry_t data = {0};
#ifdef LATENCY
    u64 timestamp = bpf_ktime_get_ns();
    data.pid = bpf_get_current_pid_tgid();
#endif
    READ_CLASS
    READ_METHOD
    bpf_probe_read(&data.method.clazz, sizeof(data.method.clazz),
                   (void *)clazz);
    bpf_probe_read(&data.method.method, sizeof(data.method.method),
                   (void *)method);
#ifndef LATENCY
    valp = counts.lookup_or_init(&data.method, &val);
    ++(*valp);
#endif
#ifdef LATENCY
    entry.update(&data, &timestamp);
#endif
    return 0;
}

#ifdef LATENCY
int trace_return(struct pt_regs *ctx) {
    u64 *entry_timestamp, clazz = 0, method = 0;
    struct info_t *info, zero = {};
    struct entry_t data = {};
    data.pid = bpf_get_current_pid_tgid();
    READ_CLASS
    READ_METHOD
    bpf_probe_read(&data.method.clazz, sizeof(data.method.clazz),
                   (void *)clazz);
    bpf_probe_read(&data.method.method, sizeof(data.method.method),
                   (void *)method);
    entry_timestamp = entry.lookup(&data);
    if (!entry_timestamp) {
        return 0;   // missed the entry event
    }
    info = times.lookup_or_init(&data.method, &zero);
    info->num_calls += 1;
    info->total_ns += bpf_ktime_get_ns() - *entry_timestamp;
    entry.delete(&data);
    return 0;
}
#endif  // LATENCY
#endif  // NOLANG

#ifdef SYSCALLS
int syscall_entry(struct pt_regs *ctx) {
    u64 pid = bpf_get_current_pid_tgid();
    u64 *valp, ip = ctx->ip, val = 0;
    PID_FILTER
#ifdef LATENCY
    struct syscall_entry_t data = {};
    data.timestamp = bpf_ktime_get_ns();
    data.ip = ip;
#endif
#ifndef LATENCY
    valp = syscounts.lookup_or_init(&ip, &val);
    ++(*valp);
#endif
#ifdef LATENCY
    sysentry.update(&pid, &data);
#endif
    return 0;
}

#ifdef LATENCY
int syscall_return(struct pt_regs *ctx) {
    struct syscall_entry_t *e;
    struct info_t *info, zero = {};
    u64 pid = bpf_get_current_pid_tgid(), ip;
    PID_FILTER
    e = sysentry.lookup(&pid);
    if (!e) {
        return 0;   // missed the entry event
    }
    ip = e->ip;
    info = systimes.lookup_or_init(&ip, &zero);
    info->num_calls += 1;
    info->total_ns += bpf_ktime_get_ns() - e->timestamp;
    sysentry.delete(&pid);
    return 0;
}
#endif  // LATENCY
#endif  // SYSCALLS
""".replace("READ_CLASS", read_class) \
   .replace("READ_METHOD", read_method) \
   .replace("PID_FILTER", "if ((pid >> 32) != %d) { return 0; }" % args.pid) \
   .replace("DEFINE_NOLANG", "#define NOLANG" if not args.language else "") \
   .replace("DEFINE_LATENCY", "#define LATENCY" if args.latency else "") \
   .replace("DEFINE_SYSCALLS", "#define SYSCALLS" if args.syscalls else "")

if args.language:
    usdt = USDT(pid=args.pid)
    usdt.enable_probe(entry_probe, "trace_entry")
    if args.latency:
        usdt.enable_probe(return_probe, "trace_return")
else:
    usdt = None

if args.verbose:
    if usdt:
        print(usdt.get_text())
    print(program)

bpf = BPF(text=program, usdt_contexts=[usdt] if usdt else [])
if args.syscalls:
    syscall_regex = "^[Ss]y[Ss]_.*"
    bpf.attach_kprobe(event_re=syscall_regex, fn_name="syscall_entry")
    if args.latency:
        bpf.attach_kretprobe(event_re=syscall_regex, fn_name="syscall_return")
    print("Attached %d kernel probes for syscall tracing." %
          bpf.num_open_kprobes())

def get_data():
    # Will be empty when no language was specified for tracing
    if args.latency:
        data = map(lambda (k, v): (k.clazz + "." + k.method,
                                   (v.num_calls, v.total_ns)),
                   bpf["times"].items())
    else:
        data = map(lambda (k, v): (k.clazz + "." + k.method, (v.value, 0)),
                   bpf["counts"].items())

    if args.syscalls:
        if args.latency:
            syscalls = map(lambda (k, v): (bpf.ksym(k.value),
                                           (v.num_calls, v.total_ns)),
                           bpf["systimes"].items())
            data.extend(syscalls)
        else:
            syscalls = map(lambda (k, v): (bpf.ksym(k.value), (v.value, 0)),
                           bpf["syscounts"].items())
            data.extend(syscalls)

    return sorted(data, key=lambda (k, v): v[1 if args.latency else 0])

def clear_data():
    if args.latency:
        bpf["times"].clear()
    else:
        bpf["counts"].clear()

    if args.syscalls:
        if args.latency:
            bpf["systimes"].clear()
        else:
            bpf["syscounts"].clear()

exit_signaled = False
print("Tracing calls in process %d (language: %s)... Ctrl-C to quit." %
      (args.pid, args.language or "none"))
while True:
    try:
        sleep(args.interval or 99999999)
    except KeyboardInterrupt:
        exit_signaled = True
    print()
    data = get_data()   # [(function, (num calls, latency in ns))]
    if args.latency:
        time_col = "TIME (ms)" if args.milliseconds else "TIME (us)"
        print("%-50s %8s %8s" % ("METHOD", "# CALLS", time_col))
    else:
        print("%-50s %8s" % ("METHOD", "# CALLS"))
    if args.top:
        data = data[-args.top:]
    for key, value in data:
        if args.latency:
            time = value[1]/1000000.0 if args.milliseconds else \
                   value[1]/1000.0
            print("%-50s %8d %6.2f" % (key, value[0], time))
        else:
            print("%-50s %8d" % (key, value[0]))
    if args.interval and not exit_signaled:
        clear_data()
    else:
        if args.syscalls:
            print("Detaching kernel probes, please wait...")
        exit()
<commit_msg>ucalls: TODO reminder to probe C methods<commit_after>#!/usr/bin/python
# @lint-avoid-python-3-compatibility-imports
#
# ucalls  Summarize method calls in high-level languages and/or system calls.
#         For Linux, uses BCC, eBPF.
#
# USAGE: ucalls [-l {java,python,ruby}] [-h] [-T TOP] [-L] [-S] [-v] [-m]
#        pid [interval]
#
# Copyright 2016 Sasha Goldshtein
# Licensed under the Apache License, Version 2.0 (the "License")
#
# 19-Oct-2016   Sasha Goldshtein   Created this.

from __future__ import print_function
import argparse
from bcc import BPF, USDT
from time import sleep

examples = """examples:
    ./ucalls -l java 185        # trace Java calls and print statistics on ^C
    ./ucalls -l python 2020 1   # trace Python calls and print every second
    ./ucalls -l java 185 -S     # trace Java calls and syscalls
    ./ucalls 6712 -S            # trace only syscall counts
    ./ucalls -l ruby 1344 -T 10 # trace top 10 Ruby method calls
    ./ucalls -l ruby 1344 -L    # trace Ruby calls including latency
    ./ucalls -l ruby 1344 -LS   # trace Ruby calls and syscalls with latency
    ./ucalls -l python 2020 -mL # trace Python calls including latency in ms
"""
parser = argparse.ArgumentParser(
    description="Summarize method calls in high-level languages.",
    formatter_class=argparse.RawDescriptionHelpFormatter,
    epilog=examples)
parser.add_argument("pid", type=int, help="process id to attach to")
parser.add_argument("interval", type=int, nargs='?',
    help="print every specified number of seconds")
parser.add_argument("-l", "--language", choices=["java", "python", "ruby"],
    help="language to trace (if none, trace syscalls only)")
parser.add_argument("-T", "--top", type=int,
    help="number of most frequent/slow calls to print")
parser.add_argument("-L", "--latency", action="store_true",
    help="record method latency from enter to exit (except recursive calls)")
parser.add_argument("-S", "--syscalls", action="store_true",
    help="record syscall latency (adds overhead)")
parser.add_argument("-v", "--verbose", action="store_true",
    help="verbose mode: print the BPF program (for debugging purposes)")
parser.add_argument("-m", "--milliseconds", action="store_true",
    help="report times in milliseconds (default is microseconds)")
args = parser.parse_args()

# We assume that the entry and return probes have the same arguments. This is
# the case for Java, Python, and Ruby. If there's a language where it's not the
# case, we will need to build a custom correlator from entry to exit.
if args.language == "java":
    # TODO for JVM entries, we actually have the real length of the class
    #      and method strings in arg3 and arg5 respectively, so we can insert
    #      the null terminator in its proper position.
    entry_probe = "method__entry"
    return_probe = "method__return"
    read_class = "bpf_usdt_readarg(2, ctx, &clazz);"
    read_method = "bpf_usdt_readarg(4, ctx, &method);"
elif args.language == "python":
    entry_probe = "function__entry"
    return_probe = "function__return"
    read_class = "bpf_usdt_readarg(1, ctx, &clazz);"    # filename really
    read_method = "bpf_usdt_readarg(2, ctx, &method);"
elif args.language == "ruby":
    # TODO Also probe cmethod__entry and cmethod__return with same arguments
    entry_probe = "method__entry"
    return_probe = "method__return"
    read_class = "bpf_usdt_readarg(1, ctx, &clazz);"
    read_method = "bpf_usdt_readarg(2, ctx, &method);"
elif not args.language:
    if not args.syscalls:
        print("Nothing to do; use -S to trace syscalls.")
        exit(1)
    entry_probe, return_probe, read_class, read_method = ("", "", "", "")

program = """
#include <linux/ptrace.h>

#define MAX_STRING_LENGTH 80
DEFINE_NOLANG
DEFINE_LATENCY
DEFINE_SYSCALLS

struct method_t {
    char clazz[MAX_STRING_LENGTH];
    char method[MAX_STRING_LENGTH];
};
struct entry_t {
    u64 pid;
    struct method_t method;
};
struct info_t {
    u64 num_calls;
    u64 total_ns;
};
struct syscall_entry_t {
    u64 timestamp;
    u64 ip;
};

#ifndef LATENCY
  BPF_HASH(counts, struct method_t, u64);            // number of calls
  #ifdef SYSCALLS
    BPF_HASH(syscounts, u64, u64);                   // number of calls per IP
  #endif  // SYSCALLS
#else
  BPF_HASH(times, struct method_t, struct info_t);
  BPF_HASH(entry, struct entry_t, u64);              // timestamp at entry
  #ifdef SYSCALLS
    BPF_HASH(systimes, u64, struct info_t);          // latency per IP
    BPF_HASH(sysentry, u64, struct syscall_entry_t); // ts + IP at entry
  #endif  // SYSCALLS
#endif

#ifndef NOLANG
int trace_entry(struct pt_regs *ctx) {
    u64 clazz = 0, method = 0, val = 0;
    u64 *valp;
    struct entry_t data = {0};
#ifdef LATENCY
    u64 timestamp = bpf_ktime_get_ns();
    data.pid = bpf_get_current_pid_tgid();
#endif
    READ_CLASS
    READ_METHOD
    bpf_probe_read(&data.method.clazz, sizeof(data.method.clazz),
                   (void *)clazz);
    bpf_probe_read(&data.method.method, sizeof(data.method.method),
                   (void *)method);
#ifndef LATENCY
    valp = counts.lookup_or_init(&data.method, &val);
    ++(*valp);
#endif
#ifdef LATENCY
    entry.update(&data, &timestamp);
#endif
    return 0;
}

#ifdef LATENCY
int trace_return(struct pt_regs *ctx) {
    u64 *entry_timestamp, clazz = 0, method = 0;
    struct info_t *info, zero = {};
    struct entry_t data = {};
    data.pid = bpf_get_current_pid_tgid();
    READ_CLASS
    READ_METHOD
    bpf_probe_read(&data.method.clazz, sizeof(data.method.clazz),
                   (void *)clazz);
    bpf_probe_read(&data.method.method, sizeof(data.method.method),
                   (void *)method);
    entry_timestamp = entry.lookup(&data);
    if (!entry_timestamp) {
        return 0;   // missed the entry event
    }
    info = times.lookup_or_init(&data.method, &zero);
    info->num_calls += 1;
    info->total_ns += bpf_ktime_get_ns() - *entry_timestamp;
    entry.delete(&data);
    return 0;
}
#endif  // LATENCY
#endif  // NOLANG

#ifdef SYSCALLS
int syscall_entry(struct pt_regs *ctx) {
    u64 pid = bpf_get_current_pid_tgid();
    u64 *valp, ip = ctx->ip, val = 0;
    PID_FILTER
#ifdef LATENCY
    struct syscall_entry_t data = {};
    data.timestamp = bpf_ktime_get_ns();
    data.ip = ip;
#endif
#ifndef LATENCY
    valp = syscounts.lookup_or_init(&ip, &val);
    ++(*valp);
#endif
#ifdef LATENCY
    sysentry.update(&pid, &data);
#endif
    return 0;
}

#ifdef LATENCY
int syscall_return(struct pt_regs *ctx) {
    struct syscall_entry_t *e;
    struct info_t *info, zero = {};
    u64 pid = bpf_get_current_pid_tgid(), ip;
    PID_FILTER
    e = sysentry.lookup(&pid);
    if (!e) {
        return 0;   // missed the entry event
    }
    ip = e->ip;
    info = systimes.lookup_or_init(&ip, &zero);
    info->num_calls += 1;
    info->total_ns += bpf_ktime_get_ns() - e->timestamp;
    sysentry.delete(&pid);
    return 0;
}
#endif  // LATENCY
#endif  // SYSCALLS
""".replace("READ_CLASS", read_class) \
   .replace("READ_METHOD", read_method) \
   .replace("PID_FILTER", "if ((pid >> 32) != %d) { return 0; }" % args.pid) \
   .replace("DEFINE_NOLANG", "#define NOLANG" if not args.language else "") \
   .replace("DEFINE_LATENCY", "#define LATENCY" if args.latency else "") \
   .replace("DEFINE_SYSCALLS", "#define SYSCALLS" if args.syscalls else "")

if args.language:
    usdt = USDT(pid=args.pid)
    usdt.enable_probe(entry_probe, "trace_entry")
    if args.latency:
        usdt.enable_probe(return_probe, "trace_return")
else:
    usdt = None

if args.verbose:
    if usdt:
        print(usdt.get_text())
    print(program)

bpf = BPF(text=program, usdt_contexts=[usdt] if usdt else [])
if args.syscalls:
    syscall_regex = "^[Ss]y[Ss]_.*"
    bpf.attach_kprobe(event_re=syscall_regex, fn_name="syscall_entry")
    if args.latency:
        bpf.attach_kretprobe(event_re=syscall_regex, fn_name="syscall_return")
    print("Attached %d kernel probes for syscall tracing." %
          bpf.num_open_kprobes())

def get_data():
    # Will be empty when no language was specified for tracing
    if args.latency:
        data = map(lambda (k, v): (k.clazz + "." + k.method,
                                   (v.num_calls, v.total_ns)),
                   bpf["times"].items())
    else:
        data = map(lambda (k, v): (k.clazz + "." + k.method, (v.value, 0)),
                   bpf["counts"].items())

    if args.syscalls:
        if args.latency:
            syscalls = map(lambda (k, v): (bpf.ksym(k.value),
                                           (v.num_calls, v.total_ns)),
                           bpf["systimes"].items())
            data.extend(syscalls)
        else:
            syscalls = map(lambda (k, v): (bpf.ksym(k.value), (v.value, 0)),
                           bpf["syscounts"].items())
            data.extend(syscalls)

    return sorted(data, key=lambda (k, v): v[1 if args.latency else 0])

def clear_data():
    if args.latency:
        bpf["times"].clear()
    else:
        bpf["counts"].clear()

    if args.syscalls:
        if args.latency:
            bpf["systimes"].clear()
        else:
            bpf["syscounts"].clear()

exit_signaled = False
print("Tracing calls in process %d (language: %s)... Ctrl-C to quit." %
      (args.pid, args.language or "none"))
while True:
    try:
        sleep(args.interval or 99999999)
    except KeyboardInterrupt:
        exit_signaled = True
    print()
    data = get_data()   # [(function, (num calls, latency in ns))]
    if args.latency:
        time_col = "TIME (ms)" if args.milliseconds else "TIME (us)"
        print("%-50s %8s %8s" % ("METHOD", "# CALLS", time_col))
    else:
        print("%-50s %8s" % ("METHOD", "# CALLS"))
    if args.top:
        data = data[-args.top:]
    for key, value in data:
        if args.latency:
            time = value[1]/1000000.0 if args.milliseconds else \
                   value[1]/1000.0
            print("%-50s %8d %6.2f" % (key, value[0], time))
        else:
            print("%-50s %8d" % (key, value[0]))
    if args.interval and not exit_signaled:
        clear_data()
    else:
        if args.syscalls:
            print("Detaching kernel probes, please wait...")
        exit()
<|endoftext|>


<commit_before># Released under The MIT License (MIT)
# http://opensource.org/licenses/MIT
# Copyright (c) 2013-2015 SCoT Development Team

""" Use scikit-learn routines as backend.
"""

from __future__ import absolute_import

import scipy as sp
from . import backend_builtin as builtin
from . import config, datatools
from .varbase import VARBase
from .datatools import atleast_3d

def wrapper_fastica(data):
    """ Call FastICA implementation from scikit-learn.
    """
    from sklearn.decomposition import FastICA
    ica = FastICA()
    ica.fit(datatools.cat_trials(data).T)
    u = ica.components_.T
    m = ica.mixing_.T
    return m, u

def wrapper_pca(x, reducedim):
    """ Call PCA implementation from scikit-learn.
    """
    from sklearn.decomposition import PCA
    pca = PCA(n_components=reducedim)
    pca.fit(datatools.cat_trials(x).T)
    d = pca.components_
    c = pca.components_.T
    y = datatools.dot_special(c.T, x)
    return c, d, y

class VAR(VARBase):
    """ Scikit-learn based implementation of VARBase.

    This class fits VAR models using various implementations of generalized linear model fitting available in scikit-learn.

    Parameters
    ----------
    model_order : int
        Autoregressive model order
    fitobj : class, optional
        Instance of a linear model implementation.
    """
    def __init__(self, model_order, fitobj=None):
        VARBase.__init__(self, model_order)
        if fitobj is None:
            from sklearn.linear_model import LinearRegression
            fitobj = LinearRegression()
        self.fitting_model = fitobj

    def fit(self, data):
        """ Fit VAR model to data.

        Parameters
        ----------
        data : array, shape (n_trials, n_channels, n_samples) or (n_channels, n_samples)
            Continuous or segmented data set.

        Returns
        -------
        self : :class:`VAR`
            The :class:`VAR` object.
        """
        data = atleast_3d(data)
        (x, y) = self._construct_eqns(data)
        self.fitting_model.fit(x, y)

        self.coef = self.fitting_model.coef_

        self.residuals = data - self.predict(data)
        self.rescov = sp.cov(datatools.cat_trials(self.residuals[:, :, self.p:]))

        return self

backend = builtin.backend.copy()
backend.update({
    'ica': wrapper_fastica,
    'pca': wrapper_pca,
    'var': VAR
})

def activate():
    """ Set backend attribute in the config module.
    """
    config.backend = backend

activate()
<commit_msg>Do not fit intercept - this gives weird numerical differences compared to the original data orientation<commit_after># Released under The MIT License (MIT)
# http://opensource.org/licenses/MIT
# Copyright (c) 2013-2015 SCoT Development Team

""" Use scikit-learn routines as backend.
"""

from __future__ import absolute_import

import scipy as sp
from . import backend_builtin as builtin
from . import config, datatools
from .varbase import VARBase
from .datatools import atleast_3d

def wrapper_fastica(data):
    """ Call FastICA implementation from scikit-learn.
    """
    from sklearn.decomposition import FastICA
    ica = FastICA()
    ica.fit(datatools.cat_trials(data).T)
    u = ica.components_.T
    m = ica.mixing_.T
    return m, u

def wrapper_pca(x, reducedim):
    """ Call PCA implementation from scikit-learn.
    """
    from sklearn.decomposition import PCA
    pca = PCA(n_components=reducedim)
    pca.fit(datatools.cat_trials(x).T)
    d = pca.components_
    c = pca.components_.T
    y = datatools.dot_special(c.T, x)
    return c, d, y

class VAR(VARBase):
    """ Scikit-learn based implementation of VARBase.

    This class fits VAR models using various implementations of generalized linear model fitting available in scikit-learn.

    Parameters
    ----------
    model_order : int
        Autoregressive model order
    fitobj : class, optional
        Instance of a linear model implementation.
    """
    def __init__(self, model_order, fitobj=None):
        VARBase.__init__(self, model_order)
        if fitobj is None:
            from sklearn.linear_model import LinearRegression
            fitobj = LinearRegression(fit_intercept=False)
        self.fitting_model = fitobj

    def fit(self, data):
        """ Fit VAR model to data.

        Parameters
        ----------
        data : array, shape (n_trials, n_channels, n_samples) or (n_channels, n_samples)
            Continuous or segmented data set.

        Returns
        -------
        self : :class:`VAR`
            The :class:`VAR` object.
        """
        data = atleast_3d(data)
        (x, y) = self._construct_eqns(data)
        self.fitting_model.fit(x, y)

        self.coef = self.fitting_model.coef_

        self.residuals = data - self.predict(data)
        self.rescov = sp.cov(datatools.cat_trials(self.residuals[:, :, self.p:]))

        return self

backend = builtin.backend.copy()
backend.update({
    'ica': wrapper_fastica,
    'pca': wrapper_pca,
    'var': VAR
})

def activate():
    """ Set backend attribute in the config module.
    """
    config.backend = backend

activate()
<|endoftext|>


<commit_before>"""Terminal Wrapper which renders 2d arrays of characters to terminal"""

import sys
import os
import logging

from fmtstr.fmtstr import fmtstr
from fmtstr.fsarray import FSArray

import events

logging.basicConfig(filename='terminal.log',level=logging.DEBUG, format='%(asctime)s %(message)s')

class Terminal(object):
    """ Renders 2D arrays of characters and cursor position """
    #TODO: when less than whole screen owned, deal with that:
    #    -render the top of the screen at the first clear row
    #    -scroll down before rendering as necessary
    def __init__(self, tc):
        """
        tc expected to have must have methods:
            get_cursor_position()
            get_screen_size() -> (row, col)
            set_cursor_position((row, column))
            write(msg)
            scroll_down()
            erase_line()
            down, up, left, back()
            get_event() -> 'c' | events.WindowChangeEvent(rows, columns)
        """
        logging.debug('-------initializing Terminal object %r------' % self)
        self.tc = tc

    def __enter__(self):
        self.top_usable_row, _ = self.tc.get_cursor_position()
        logging.debug('initial top_usable_row: %d' % self.top_usable_row)
        return self

    def __exit__(self, type, value, traceback):
        logging.debug("running Terminal.__exit__")
        #self.tc.scroll_down()
        #row, _ = self.tc.get_cursor_position()
        #for i in range(1000):
        #    self.tc.erase_line()
        #    self.tc.down()
        #self.tc.set_cursor_position((row, 1))
        #self.tc.erase_rest_of_line()

    def render_to_terminal(self, array, cursor_pos=(0,0)):
        """Renders array to terminal, returns the number of lines
            scrolled offscreen
        outputs:
         -number of times scrolled

        If array received is of width too small, render it anyway
        if array received is of width too large, render it anyway
        if array received is of height too small, render it anyway
        if array received is of height too large, render it, scroll down,
            and render the rest of it, then return how much we scrolled down
        """
        #TODO add cool render-on-change caching
        #TODO take a formatting array with same dimensions as array

        height, width = self.tc.get_screen_size()
        rows_for_use = range(self.top_usable_row, height + 1)
        shared = min(len(array), len(rows_for_use))
        for row, line in zip(rows_for_use[:shared], array[:shared]):
            self.tc.set_cursor_position((row, 1))
            self.tc.write(str(line))
            if len(line) < width:
                self.tc.erase_rest_of_line()
        rest_of_lines = array[shared:]
        rest_of_rows = rows_for_use[shared:]
        for row in rest_of_rows: # if array too small
            self.tc.set_cursor_position((row, 1))
            self.tc.erase_line()
        offscreen_scrolls = 0
        for line in rest_of_lines: # if array too big
            logging.debug('sending scroll down message')
            self.tc.scroll_down()
            if self.top_usable_row > 1:
                self.top_usable_row -= 1
            else:
                offscreen_scrolls += 1
            logging.debug('new top_usable_row: %d' % self.top_usable_row)
            self.tc.set_cursor_position((height, 1)) # since scrolling moves the cursor
            self.tc.write(str(line))

        self.tc.set_cursor_position((cursor_pos[0]-offscreen_scrolls+self.top_usable_row, cursor_pos[1]+1))
        return offscreen_scrolls

    def array_from_text(self, msg):
        rows, columns = self.tc.get_screen_size()
        arr = FSArray(0, columns)
        i = 0
        for c in msg:
            if i >= rows * columns:
                return arr
            elif c in '\r\n':
                i = ((i / columns) + 1) * columns
            else:
                arr[i / arr.columns, i % arr.columns] = [fmtstr(c)]
            i += 1
        return arr

def test():
    import terminalcontrol
    with terminalcontrol.TCPartialler(sys.stdin, sys.stdout) as tc:
        with Terminal(tc) as t:
            rows, columns = t.tc.get_screen_size()
            while True:
                c = t.tc.get_event()
                if c == "":
                    sys.exit() # same as raise SystemExit()
                elif c == "h":
                    a = t.array_from_text("a for small array")
                elif c == "a":
                    a = [fmtstr(c*columns) for _ in range(rows)]
                elif c == "s":
                    a = [fmtstr(c*columns) for _ in range(rows-1)]
                elif c == "d":
                    a = [fmtstr(c*columns) for _ in range(rows+1)]
                elif c == "f":
                    a = [fmtstr(c*columns) for _ in range(rows-2)]
                elif c == "q":
                    a = [fmtstr(c*columns) for _ in range(1)]
                elif c == "w":
                    a = [fmtstr(c*columns) for _ in range(1)]
                elif c == "e":
                    a = [fmtstr(c*columns) for _ in range(1)]
                elif isinstance(c, events.WindowChangeEvent):
                    a = t.array_from_text("window just changed to %d rows and %d columns" % (c.rows, c.columns))
                elif c == "":
                    [t.tc.write('\n') for _ in range(rows)]
                    continue
                else:
                    a = t.array_from_text("unknown command")
                t.render_to_terminal(a)

def main():
    t = Terminal(sys.stdin, sys.stdout)
    rows, columns = t.tc.get_screen_size()
    import random
    goop = lambda l: [random.choice('aaabcddeeeefghiiijklmnooprssttuv        ') for _ in range(l)]
    a = [fmtstr(goop) for _ in range(rows)]
    t.render_to_terminal(a)
    while True:
        c = t.tc.get_event()
        if c == "":
            t.cleanup()
            sys.exit()
        t.render_to_terminal([fmtstr(c*columns) for _ in range(rows)])

def test_array_from_text():
    class FakeTC(object): get_screen_size = lambda self: (30, 50)
    t = Terminal(FakeTC())
    a = t.array_from_text('\n\nhey there\nyo')
    os.system('reset')
    for line in a:
        print str(line)
    raw_input()

if __name__ == '__main__':
    test_array_from_text()
    #test()
<commit_msg>fix terminal.py test<commit_after>"""Terminal Wrapper which renders 2d arrays of characters to terminal"""

import sys
import os
import logging

from fmtstr.fmtstr import fmtstr
from fmtstr.fsarray import FSArray

import events

logging.basicConfig(filename='terminal.log',level=logging.DEBUG, format='%(asctime)s %(message)s')

class Terminal(object):
    """ Renders 2D arrays of characters and cursor position """
    #TODO: when less than whole screen owned, deal with that:
    #    -render the top of the screen at the first clear row
    #    -scroll down before rendering as necessary
    def __init__(self, tc):
        """
        tc expected to have must have methods:
            get_cursor_position()
            get_screen_size() -> (row, col)
            set_cursor_position((row, column))
            write(msg)
            scroll_down()
            erase_line()
            down, up, left, back()
            get_event() -> 'c' | events.WindowChangeEvent(rows, columns)
        """
        logging.debug('-------initializing Terminal object %r------' % self)
        self.tc = tc

    def __enter__(self):
        self.top_usable_row, _ = self.tc.get_cursor_position()
        logging.debug('initial top_usable_row: %d' % self.top_usable_row)
        return self

    def __exit__(self, type, value, traceback):
        logging.debug("running Terminal.__exit__")
        self.tc.scroll_down()
        row, _ = self.tc.get_cursor_position()
        for i in range(1000):
            self.tc.erase_line()
            self.tc.down()
        self.tc.set_cursor_position((row, 1))
        self.tc.erase_rest_of_line()

    def render_to_terminal(self, array, cursor_pos=(0,0)):
        """Renders array to terminal, returns the number of lines
            scrolled offscreen
        outputs:
         -number of times scrolled

        If array received is of width too small, render it anyway
        if array received is of width too large, render it anyway
        if array received is of height too small, render it anyway
        if array received is of height too large, render it, scroll down,
            and render the rest of it, then return how much we scrolled down
        """
        #TODO add cool render-on-change caching
        #TODO take a formatting array with same dimensions as array

        height, width = self.tc.get_screen_size()
        rows_for_use = range(self.top_usable_row, height + 1)
        shared = min(len(array), len(rows_for_use))
        for row, line in zip(rows_for_use[:shared], array[:shared]):
            self.tc.set_cursor_position((row, 1))
            self.tc.write(str(line))
            if len(line) < width:
                self.tc.erase_rest_of_line()
        rest_of_lines = array[shared:]
        rest_of_rows = rows_for_use[shared:]
        for row in rest_of_rows: # if array too small
            self.tc.set_cursor_position((row, 1))
            self.tc.erase_line()
        offscreen_scrolls = 0
        for line in rest_of_lines: # if array too big
            logging.debug('sending scroll down message')
            self.tc.scroll_down()
            if self.top_usable_row > 1:
                self.top_usable_row -= 1
            else:
                offscreen_scrolls += 1
            logging.debug('new top_usable_row: %d' % self.top_usable_row)
            self.tc.set_cursor_position((height, 1)) # since scrolling moves the cursor
            self.tc.write(str(line))

        self.tc.set_cursor_position((cursor_pos[0]-offscreen_scrolls+self.top_usable_row, cursor_pos[1]+1))
        return offscreen_scrolls

    def array_from_text(self, msg):
        rows, columns = self.tc.get_screen_size()
        arr = FSArray(0, columns)
        i = 0
        for c in msg:
            if i >= rows * columns:
                return arr
            elif c in '\r\n':
                i = ((i / columns) + 1) * columns
            else:
                arr[i / arr.columns, i % arr.columns] = [fmtstr(c)]
            i += 1
        return arr

def test():
    import terminalcontrol
    with terminalcontrol.TerminalController(sys.stdin, sys.stdout) as tc:
        with Terminal(tc) as t:
            rows, columns = t.tc.get_screen_size()
            while True:
                c = t.tc.get_event()
                if c == "":
                    sys.exit() # same as raise SystemExit()
                elif c == "h":
                    a = t.array_from_text("a for small array")
                elif c == "a":
                    a = [fmtstr(c*columns) for _ in range(rows)]
                elif c == "s":
                    a = [fmtstr(c*columns) for _ in range(rows-1)]
                elif c == "d":
                    a = [fmtstr(c*columns) for _ in range(rows+1)]
                elif c == "f":
                    a = [fmtstr(c*columns) for _ in range(rows-2)]
                elif c == "q":
                    a = [fmtstr(c*columns) for _ in range(1)]
                elif c == "w":
                    a = [fmtstr(c*columns) for _ in range(1)]
                elif c == "e":
                    a = [fmtstr(c*columns) for _ in range(1)]
                elif isinstance(c, events.WindowChangeEvent):
                    a = t.array_from_text("window just changed to %d rows and %d columns" % (c.rows, c.columns))
                elif c == "":
                    [t.tc.write('\n') for _ in range(rows)]
                    continue
                else:
                    a = t.array_from_text("unknown command")
                t.render_to_terminal(a)

def main():
    t = Terminal(sys.stdin, sys.stdout)
    rows, columns = t.tc.get_screen_size()
    import random
    goop = lambda l: [random.choice('aaabcddeeeefghiiijklmnooprssttuv        ') for _ in range(l)]
    a = [fmtstr(goop) for _ in range(rows)]
    t.render_to_terminal(a)
    while True:
        c = t.tc.get_event()
        if c == "":
            t.cleanup()
            sys.exit()
        t.render_to_terminal([fmtstr(c*columns) for _ in range(rows)])

def test_array_from_text():
    class FakeTC(object): get_screen_size = lambda self: (30, 50)
    t = Terminal(FakeTC())
    a = t.array_from_text('\n\nhey there\nyo')
    os.system('reset')
    for line in a:
        print str(line)
    raw_input()

if __name__ == '__main__':
    test_array_from_text()
    test()
<|endoftext|>


<commit_before># linkFollower.py
# follows links at a specific position a specific number of times
# part of the assignments for University of Michigan course:
# Using Python to Access Web Data - offered on Coursera.org
# Author: Joe Ryan
# 19 November, 2015

import urllib
from BeautifulSoup import *

url = raw_input('Enter URL: ')
count = raw_input('Enter count: ')
position = raw_input('Enter position: ')

for i in range(0, int(count)):
    print 'Retrieving ' + url + '.....'
    html = urllib.urlopen(url).read()
    soup = BeautifulSoup(html)
    tags = soup('a')
    url = tags[int(position)].get('href', None)

print 'Last URL: ', url
<commit_msg>fix index of desired link polition<commit_after># linkFollower.py
# follows links at a specific position a specific number of times
# part of the assignments for University of Michigan course:
# Using Python to Access Web Data - offered on Coursera.org
# Author: Joe Ryan
# 19 November, 2015

import urllib
from BeautifulSoup import *

url = raw_input('Enter URL: ')
count = raw_input('Enter count: ')
position = raw_input('Enter position: ')

for i in range(0, int(count)):
    print 'Retrieving ' + url + '.....'
    html = urllib.urlopen(url).read()
    soup = BeautifulSoup(html)
    tags = soup('a')
    url = tags[int(position)-1].get('href', None)

print 'Last URL: ', url
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Allow for flexible configuration.

There are two config dict models here:: one is to recursively lock the
dictionary.  This is to aid in debugging; one can assume the config hasn't
changed from the moment of locking.  This is the original mozharness model.

The other model is to log any changes to the dict or its children.  When
debugging, config changes will be marked in the log.

Attributes:
  SUPPORTED_LOGGING_TYPES (dict): a non-logging to logging class map, e.g.
    dict: LoggingDict.  Not yet supporting collections / OrderedDicts.
"""

from __future__ import absolute_import, division, print_function
from copy import deepcopy
from scriptharness import ScriptHarnessException
import logging
import six

# TODO use memo like deepcopy to prevent loop recursion

# LoggingDict and helpers {{{1
# logging classes {{{2
class LoggingClass(object):
    """General logging methods for the Logging* classes to subclass.

    Attributes:
      level (int): the logging level for changes
      logger_name (str): the logger name to use
      name (str): the name of the class for logs
      parent (str): the name of the parent, if applicable, for logs
    """
    level = None
    logger_name = None
    name = None
    parent = None
    def __new__(cls, super_class, items, level=logging.INFO,
                logger_name='scriptharness.config'):
        self = super_class.__new__(
            cls,
            (enable_logging(x, logger_name, level) for x in items)
        )
        self.level = level
        self.logger_name = logger_name
        return self
    def items(self):
        """Shut up pylint"""
        return super(LoggingClass, self).items()
    def recusively_set_parent(self, name, parent=None):
        """Recursively set name + parent.

        If our LoggingDict is a multi-level nested Logging* class, then
        seeing a log message that something in one of the Logging* classes
        has changed can be confusing.  If we know that it's
        grandparent[parent][self][child] that has changed, then the log
        message is helpful.

        name (str): set self.name, for later logging purposes.
        parent (Logging* object, optional): set self.parent, for later logging
          purposes.
        """
        if name is not None:
            self.name = name
        if parent is not None:
            self.parent = parent
        if issubclass(self, dict):
            for child_name, child in self.items():
                if is_logging_class(child):
                    child.recursively_set_parent(child_name, self)
        else:
            for count, elem in enumerate(self):
                if is_logging_class(elem):
                    elem.recursively_set_parent(six.text_type(count), self)
    def log_change(self, message, child_list=None):
        """Log a change to self.

        Args:
          message (str): The message to log.
          child_list (list, automatically generated): in a multi-level nested
            Logging* class, generate the list of children's names so we can log
            which Logging* class has changed.  This list will be built by
            prepending our name and calling log_change() on self.parent.
        """
        if self.parent:
            if child_list is None:
                child_list = []
            child_list.insert(0, self.name)
            # TODO what happens on deletion?
            return self.parent.log_change(message, child_list)
        logger = logging.getLogger(self.logger_name)
        if child_list:
            name = six.text_type(child_list.pop(0))
            for item in child_list:
                name += "[{}]".format(six.text_type(item))
            message = "{}: {}".format(name, message)
        return logger.log(self.level, message)

class LoggingList(LoggingClass, list):
    """A list that logs any changes, as do its children.
    """
    def __new__(cls, *args, **kwargs):
        return LoggingClass.__new__(cls, list, *args, **kwargs)
    def __deepcopy__(self, memo):
        """Return a list on deepcopy.
        """
        return [deepcopy(elem, memo) for elem in self]  # pragma: no branch
    # TODO add logging

class LoggingTuple(LoggingClass, tuple):
    """A tuple whose children log any changes.
    """
    def __new__(cls, *args, **kwargs):
        return LoggingClass.__new__(cls, tuple, *args, **kwargs)
    def __deepcopy__(self, memo):
        """Return a tuple on deepcopy.
        """
        return tuple(  # pragma: no branch
            [deepcopy(elem, memo) for elem in self]
        )

class LoggingSet(LoggingClass, set):
    """A set that logs any changes, as do its children.
    """
    def __new__(cls, *args, **kwargs):
        return LoggingClass.__new__(cls, set, *args, **kwargs)
    def __deepcopy__(self, memo):
        """Return a set on deepcopy.
        """
        return set(  # pragma: no branch
            [deepcopy(elem, memo) for elem in self]
        )
    # TODO add logging

class LoggingFrozenSet(LoggingClass, frozenset):
    """A frozenset whose children log any changes.
    """
    def __new__(cls, *args, **kwargs):
        return LoggingClass.__new__(cls, frozenset, *args, **kwargs)
    def __deepcopy__(self, memo):
        """Return a set on deepcopy.
        """
        return frozenset(  # pragma: no branch
            [deepcopy(elem, memo) for elem in self]
        )

class LoggingDict(LoggingClass, dict):
    """A dict that logs any changes, as do its children.
    """
    def __new__(cls, *args, **kwargs):
        return LoggingClass.__new__(cls, dict, *args, **kwargs)
    def __setitem__(self, *args):
        return super(LoggingDict, self).__setitem__(*args)
    def __delitem__(self, *args):
        return super(LoggingDict, self).__delitem__(*args)
    def clear(self, *args):
        return super(LoggingDict, self).clear(*args)
    def pop(self, *args):
        return super(LoggingDict, self).pop(*args)
    def popitem(self, *args):
        return super(LoggingDict, self).popitem(*args)
    def setdefault(self, *args):
        return super(LoggingDict, self).setdefault(*args)
    def update(self, *args):
        return super(LoggingDict, self).update(*args)
    def __deepcopy__(self, memo):
        """Return a dict on deepcopy()
        """
        # TODO needed?
        result = {}
        memo[id(self)] = result
        for key, value in self.items():
            result[key] = deepcopy(value, memo)
        return result
    # TODO add logging
    # TODO secret key, e.g. {'credentials': {}} that notes changes but
    # doesn't log them?

# end logging classes 2}}}

SUPPORTED_LOGGING_TYPES = {
    dict: LoggingDict,
    frozenset: LoggingFrozenSet,
    list: LoggingList,
    set: LoggingSet,
    tuple: LoggingTuple,
}

def is_logging_class(item):
    """Determine if a class is one of the Logging* classes.
    """
    return item in SUPPORTED_LOGGING_TYPES.values()

def enable_logging(item, logger_name=None, level=logging.INFO):
    """Recursively add logging to all contents of a LoggingDict.

    Any children of supported types will also have logging enabled.
    Currently supported:: list, tuple, dict, set, frozenset.

    Note:: a tuple or frozenset will become a LoggingList or LoggingSet,
    respectively; this means they will become read/write.  We can
    add non-recursive locking capability to these if this becomes a problem.

    Args:
      item (object): a child of a LoggingDict.

    Returns:
      A logging version of item, when applicable, or item.
    """
    result = item
    for key, value in SUPPORTED_LOGGING_TYPES.items():
        if isinstance(item, key):
            result = value(item, logger_name=logger_name, level=level)
    return result

# ReadOnlyDict {{{1
def make_immutable(item):
    """Recursively lock all contents of a ReadOnlyDict.

    Any children of supported types will also be locked.
    Currently supported:: list, tuple, dict, set, frozenset.

    and we locked r on a shallow level, we could still r['b'].append() or
    r['c']['key2'] = 'value2'.  So to avoid that, we need to recursively
    lock r via make_immutable.

    Taken from mozharness, but added LockedFrozenSet.

    Args:
      item (object): a child of a ReadOnlyDict.

    Returns:
      A locked version of item, when applicable, or item.
    """
    if isinstance(item, list) or isinstance(item, tuple):
        result = LockedTuple(item)
    elif isinstance(item, dict):
        result = ReadOnlyDict(item)
        result.lock()
    elif isinstance(item, set) or isinstance(item, frozenset):
        result = LockedFrozenSet(item)
    else:
        result = item
    return result

class LockedTuple(tuple):
    """A tuple with its children recursively locked.

    Tuples are read-only by nature, but we need to be able to recursively lock
    the contents of the tuple, since the tuple can contain dicts or lists.

    Taken straight from mozharness.
    """
    def __new__(cls, items):
        return tuple.__new__(cls, (make_immutable(x) for x in items))
    def __deepcopy__(self, memo):
        """Return a list on deepcopy.
        """
        return [deepcopy(elem, memo) for elem in self]  # pragma: no branch

class LockedFrozenSet(frozenset):
    """A frozenset with its children recursively locked.

    Frozensets are read-only by nature, but we need to be able to recursively
    lock the contents of the frozenset, since the frozenset can contain dicts
    or lists.
    """
    def __new__(cls, items):
        return frozenset.__new__(cls, (make_immutable(x) for x in items))
    def __deepcopy__(self, memo):
        """Return a set on deepcopy.
        """
        return set(  # pragma: no branch
            [deepcopy(elem, memo) for elem in self]
        )

class ReadOnlyDict(dict):
    '''A dict that is lockable.  When locked, any changes raise exceptions.

    Slightly modified version of mozharness.base.config.ReadOnlyDict,
    largely for pylint.
    '''
    def __init__(self, *args, **kwargs):
        self._lock = False
        super(ReadOnlyDict, self).__init__(*args, **kwargs)

    def _check_lock(self):
        """Throw an exception if we try to change anything while locked.
        """
        if self._lock:
            raise ScriptHarnessException("ReadOnlyDict is locked!")

    def lock(self):
        """Recursively lock the dictionary.
        """
        for (key, value) in self.items():
            self[key] = make_immutable(value)
        self._lock = True

    def __setitem__(self, *args):
        self._check_lock()
        return super(ReadOnlyDict, self).__setitem__(*args)

    def __delitem__(self, *args):
        self._check_lock()
        return super(ReadOnlyDict, self).__delitem__(*args)

    def clear(self, *args):
        self._check_lock()
        return super(ReadOnlyDict, self).clear(*args)

    def pop(self, *args):
        self._check_lock()
        return super(ReadOnlyDict, self).pop(*args)

    def popitem(self, *args):
        self._check_lock()
        return super(ReadOnlyDict, self).popitem(*args)

    def setdefault(self, *args):
        self._check_lock()
        return super(ReadOnlyDict, self).setdefault(*args)

    def update(self, *args):
        self._check_lock()
        return super(ReadOnlyDict, self).update(*args)

    def __deepcopy__(self, memo):
        """Create an unlocked ReadOnlyDict on deepcopy()
        """
        result = self.__class__()
        memo[id(self)] = result
        for key, value in self.items():
            result[key] = deepcopy(value, memo)
        return result
<commit_msg>add logging to LoggingList.<commit_after>#!/usr/bin/env python
"""Allow for flexible configuration.

There are two config dict models here:: one is to recursively lock the
dictionary.  This is to aid in debugging; one can assume the config hasn't
changed from the moment of locking.  This is the original mozharness model.

The other model is to log any changes to the dict or its children.  When
debugging, config changes will be marked in the log.

Attributes:
  SUPPORTED_LOGGING_TYPES (dict): a non-logging to logging class map, e.g.
    dict: LoggingDict.  Not yet supporting collections / OrderedDicts.
"""

from __future__ import absolute_import, division, print_function
from copy import deepcopy
from scriptharness import ScriptHarnessException
import logging
import six

# TODO use memo like deepcopy to prevent loop recursion

# LoggingDict and helpers {{{1
# logging classes {{{2
class LoggingClass(object):
    """General logging methods for the Logging* classes to subclass.

    Attributes:
      level (int): the logging level for changes
      logger_name (str): the logger name to use
      name (str): the name of the class for logs
      parent (str): the name of the parent, if applicable, for logs
    """
    level = None
    logger_name = None
    name = None
    parent = None
    def __new__(cls, super_class, items, level=logging.INFO,
                logger_name='scriptharness.config'):
        self = super_class.__new__(
            cls,
            (enable_logging(x, logger_name, level) for x in items)
        )
        self.level = level
        self.logger_name = logger_name
        return self
    def items(self):
        """Shut up pylint"""
        return super(LoggingClass, self).items()
    def recusively_set_parent(self, name, parent=None):
        """Recursively set name + parent.

        If our LoggingDict is a multi-level nested Logging* instance, then
        seeing a log message that something in one of the Logging* instances
        has changed can be confusing.  If we know that it's
        grandparent[parent][self][child] that has changed, then the log
        message is helpful.

        name (str): set self.name, for later logging purposes.
        parent (Logging* object, optional): set self.parent, for later logging
          purposes.
        """
        if name is not None:
            self.name = name
        if parent is not None:
            self.parent = parent
        if issubclass(self, dict):
            for child_name, child in self.items():
                if is_logging_class(child):
                    child.recursively_set_parent(child_name, self)
        else:
            for count, elem in enumerate(self):
                if is_logging_class(elem):
                    elem.recursively_set_parent(six.text_type(count - 1), self)
    def child_set_parent(self, child, child_name):
        """If child is a Logging* instance, set its parent and name.

        Args:
          child: an object, which might be a Logging* instance
          child_name: the name to set in the child
        """
        if is_logging_class(child):
            child.recursively_set_parent(child_name, parent=self)
    def log_change(self, message, child_list=None):
        """Log a change to self.

        Args:
          message (str): The message to log.
          child_list (list, automatically generated): in a multi-level nested
            Logging* class, generate the list of children's names so we can log
            which Logging* class has changed.  This list will be built by
            prepending our name and calling log_change() on self.parent.
        """
        if self.parent:
            if child_list is None:
                child_list = []
            child_list.insert(0, self.name)
            # TODO what happens on deletion?
            return self.parent.log_change(message, child_list)
        logger = logging.getLogger(self.logger_name)
        if child_list:
            name = six.text_type(child_list.pop(0))
            for item in child_list:
                name += "[{0}]".format(six.text_type(item))
            message = "{0}: {1}".format(name, message)
        return logger.log(self.level, message)

class LoggingList(LoggingClass, list):
    """A list that logs any changes, as do its children.
    """
    def __new__(cls, *args, **kwargs):
        return LoggingClass.__new__(cls, list, *args, **kwargs)

    def __deepcopy__(self, memo):
        """Return a list on deepcopy.
        """
        return [deepcopy(elem, memo) for elem in self]  # pragma: no branch

    def child_set_parent(self, position=0):
        """When the list changes, we either want to change all of the
        children's names (which correspond to indeces) or a subset of
        [position:]

        Override child_set_parent to make this simpler.
        """
        for count, elem in enumerate(self, start=position):
            super(LoggingList, self).child_set_parent(
                elem,
                six_text_type(count)
            )

    def append(self, item):
        self.log_change("appending {0}".format(six.text_type(item)))
        super(LoggingList, self).append(item)
        self.log_change("now looks like {0}".format(self))
        self.child_set_parent(item, six.text_type(len(self) - 1))

    def extend(self, items):
        self.log_change("extending with {0}".format(items))
        super(LoggingList, self).extend(items)
        self.log_change("now looks like {0}".format(self))
        for count, elem in enumerate(self):
            self.child_set_parent(elem, six.text_type(count))

    def insert(self, position, item):
        self.log_change("inserting {0} at position {1}".format(item, position))
        super(LoggingList, self).insert(position, item)
        self.log_change("now looks like {0}".format(self))
        for count, elem in enumerate(self, start=position):
            self.child_set_parent(elem, six.text_type(count))

    def remove(self, item):
        self.log_change("removing {0}".format(item))
        position = self.index(item)
        super(LoggingList, self).remove(item)
        self.log_change("now looks like {0}".format(self))
        for count, elem in enumerate(self, start=position):
            self.child_set_parent(elem, six.text_type(count))

    def pop(self, position=None):
        message = "popping"
        if position:
            message += " position {0}".format(position)
        self.log_change(message)
        value = super(LoggingList, self).pop(position)
        self.log_change("now looks like {0}".format(self))
        if position:
            for count, elem in enumerate(self, start=position):
                self.child_set_parent(elem, six.text_type(count))
        return value

    def sort(self, *args, **kwargs):
        self.log_change("sorting")
        super(LoggingList, self).sort(*args, **kwargs)
        self.log_change("now looks like {0}".format(self))
        for count, elem in enumerate(self):
            self.child_set_parent(elem, six.text_type(count))

    def reverse(self):
        self.log_change("reversing")
        super(LoggingList, self).reverse()
        self.log_change("now looks like {0}".format(self))
        for count, elem in enumerate(self):
            self.child_set_parent(elem, six.text_type(count))

class LoggingTuple(LoggingClass, tuple):
    """A tuple whose children log any changes.
    """
    def __new__(cls, *args, **kwargs):
        return LoggingClass.__new__(cls, tuple, *args, **kwargs)
    def __deepcopy__(self, memo):
        """Return a tuple on deepcopy.
        """
        return tuple(  # pragma: no branch
            [deepcopy(elem, memo) for elem in self]
        )

class LoggingSet(LoggingClass, set):
    """A set that logs any changes, as do its children.
    """
    def __new__(cls, *args, **kwargs):
        return LoggingClass.__new__(cls, set, *args, **kwargs)
    def __deepcopy__(self, memo):
        """Return a set on deepcopy.
        """
        return set(  # pragma: no branch
            [deepcopy(elem, memo) for elem in self]
        )
    # TODO add logging

class LoggingFrozenSet(LoggingClass, frozenset):
    """A frozenset whose children log any changes.
    """
    def __new__(cls, *args, **kwargs):
        return LoggingClass.__new__(cls, frozenset, *args, **kwargs)
    def __deepcopy__(self, memo):
        """Return a set on deepcopy.
        """
        return frozenset(  # pragma: no branch
            [deepcopy(elem, memo) for elem in self]
        )

class LoggingDict(LoggingClass, dict):
    """A dict that logs any changes, as do its children.
    """
    def __new__(cls, *args, **kwargs):
        return LoggingClass.__new__(cls, dict, *args, **kwargs)
    def __setitem__(self, *args):
        return super(LoggingDict, self).__setitem__(*args)
    def __delitem__(self, *args):
        return super(LoggingDict, self).__delitem__(*args)
    def clear(self, *args):
        return super(LoggingDict, self).clear(*args)
    def pop(self, *args):
        return super(LoggingDict, self).pop(*args)
    def popitem(self, *args):
        return super(LoggingDict, self).popitem(*args)
    def setdefault(self, *args):
        return super(LoggingDict, self).setdefault(*args)
    def update(self, *args):
        return super(LoggingDict, self).update(*args)
    def __deepcopy__(self, memo):
        """Return a dict on deepcopy()
        """
        # TODO needed?
        result = {}
        memo[id(self)] = result
        for key, value in self.items():
            result[key] = deepcopy(value, memo)
        return result
    # TODO add logging
    # TODO secret key, e.g. {'credentials': {}} that notes changes but
    # doesn't log them?

# end logging classes 2}}}

SUPPORTED_LOGGING_TYPES = {
    dict: LoggingDict,
    frozenset: LoggingFrozenSet,
    list: LoggingList,
    set: LoggingSet,
    tuple: LoggingTuple,
}

def is_logging_class(item):
    """Determine if a class is one of the Logging* classes.
    """
    return item in SUPPORTED_LOGGING_TYPES.values()

def enable_logging(item, logger_name=None, level=logging.INFO):
    """Recursively add logging to all contents of a LoggingDict.

    Any children of supported types will also have logging enabled.
    Currently supported:: list, tuple, dict, set, frozenset.

    Note:: a tuple or frozenset will become a LoggingList or LoggingSet,
    respectively; this means they will become read/write.  We can
    add non-recursive locking capability to these if this becomes a problem.

    Args:
      item (object): a child of a LoggingDict.

    Returns:
      A logging version of item, when applicable, or item.
    """
    result = item
    for key, value in SUPPORTED_LOGGING_TYPES.items():
        if isinstance(item, key):
            result = value(item, logger_name=logger_name, level=level)
    return result

# ReadOnlyDict {{{1
def make_immutable(item):
    """Recursively lock all contents of a ReadOnlyDict.

    Any children of supported types will also be locked.
    Currently supported:: list, tuple, dict, set, frozenset.

    and we locked r on a shallow level, we could still r['b'].append() or
    r['c']['key2'] = 'value2'.  So to avoid that, we need to recursively
    lock r via make_immutable.

    Taken from mozharness, but added LockedFrozenSet.

    Args:
      item (object): a child of a ReadOnlyDict.

    Returns:
      A locked version of item, when applicable, or item.
    """
    if isinstance(item, list) or isinstance(item, tuple):
        result = LockedTuple(item)
    elif isinstance(item, dict):
        result = ReadOnlyDict(item)
        result.lock()
    elif isinstance(item, set) or isinstance(item, frozenset):
        result = LockedFrozenSet(item)
    else:
        result = item
    return result

class LockedTuple(tuple):
    """A tuple with its children recursively locked.

    Tuples are read-only by nature, but we need to be able to recursively lock
    the contents of the tuple, since the tuple can contain dicts or lists.

    Taken straight from mozharness.
    """
    def __new__(cls, items):
        return tuple.__new__(cls, (make_immutable(x) for x in items))
    def __deepcopy__(self, memo):
        """Return a list on deepcopy.
        """
        return [deepcopy(elem, memo) for elem in self]  # pragma: no branch

class LockedFrozenSet(frozenset):
    """A frozenset with its children recursively locked.

    Frozensets are read-only by nature, but we need to be able to recursively
    lock the contents of the frozenset, since the frozenset can contain dicts
    or lists.
    """
    def __new__(cls, items):
        return frozenset.__new__(cls, (make_immutable(x) for x in items))
    def __deepcopy__(self, memo):
        """Return a set on deepcopy.
        """
        return set(  # pragma: no branch
            [deepcopy(elem, memo) for elem in self]
        )

class ReadOnlyDict(dict):
    '''A dict that is lockable.  When locked, any changes raise exceptions.

    Slightly modified version of mozharness.base.config.ReadOnlyDict,
    largely for pylint.
    '''
    def __init__(self, *args, **kwargs):
        self._lock = False
        super(ReadOnlyDict, self).__init__(*args, **kwargs)

    def _check_lock(self):
        """Throw an exception if we try to change anything while locked.
        """
        if self._lock:
            raise ScriptHarnessException("ReadOnlyDict is locked!")

    def lock(self):
        """Recursively lock the dictionary.
        """
        for (key, value) in self.items():
            self[key] = make_immutable(value)
        self._lock = True

    def __setitem__(self, *args):
        self._check_lock()
        return super(ReadOnlyDict, self).__setitem__(*args)

    def __delitem__(self, *args):
        self._check_lock()
        return super(ReadOnlyDict, self).__delitem__(*args)

    def clear(self, *args):
        self._check_lock()
        return super(ReadOnlyDict, self).clear(*args)

    def pop(self, *args):
        self._check_lock()
        return super(ReadOnlyDict, self).pop(*args)

    def popitem(self, *args):
        self._check_lock()
        return super(ReadOnlyDict, self).popitem(*args)

    def setdefault(self, *args):
        self._check_lock()
        return super(ReadOnlyDict, self).setdefault(*args)

    def update(self, *args):
        self._check_lock()
        return super(ReadOnlyDict, self).update(*args)

    def __deepcopy__(self, memo):
        """Create an unlocked ReadOnlyDict on deepcopy()
        """
        result = self.__class__()
        memo[id(self)] = result
        for key, value in self.items():
            result[key] = deepcopy(value, memo)
        return result
<|endoftext|>


<commit_before>from csacompendium.csa_practice.models import CsaPractice
from csacompendium.research_type.models import (
    ControlResearch,
    ExperimentRep,
    NitrogenApplied,
    ExperimentDetails,
)
from csacompendium.utils.hyperlinkedidentity import hyperlinked_identity
from csacompendium.utils.serializersutils import (
    get_related_content_url,
    valid_integer,
    CreateSerializerUtil,
    FieldMethodSerializer,
)
from rest_framework.serializers import (
    ModelSerializer,
    SerializerMethodField,
    ValidationError,
)

def control_research_serializers():
    """
    Control research serializers
    :return: All control research serializers
    :rtype: Object
    """

    def create_control_research_serializer(model_type=None, request_key=None, user=None):
        """
        Creates a model serializer
        :param model_type: Model
        :param request_key: A slug or primary key depending on what is passed
        :param user: Record owner
        :return: Serializer class
        :rtype: Object
        """

        class ControlResearchCreateSerializer(ModelSerializer, CreateSerializerUtil):
            """
            Create a record
            """
            class Meta:
                model = ControlResearch
                fields = ['id', 'csapractice', 'experimentrep', 'experimentdetails',
                          'nitrogenapplied', 'last_update', 'time_created', ]

            def __init__(self, *args, **kwargs):
                super(ControlResearchCreateSerializer, self).__init__(*args, **kwargs)
                self.model_type = model_type
                self.key = request_key
                self.user = user
                if valid_integer(request_key):
                    self.slugify = False
                else:
                    self.slugify = True
                self.auth_user = self.get_authenticated_user(self.user)

            def create(self, validated_data):
                """
                Created record from validated data
                :param validated_data: Validated data
                :return: Control research object
                :rtype: Object
                """
                csapractice = validated_data.get('csapractice')
                experimentrep = validated_data.get('experimentrep')
                experimentdetails = validated_data.get('experimentdetails')
                nitrogenapplied = validated_data.get('nitrogenapplied')
                control_research = ControlResearch.objects.create_by_model_type(
                    self.model_type,
                    self.key,
                    csapractice=csapractice,
                    experimentrep=experimentrep,
                    experimentdetails=experimentdetails,
                    nitrogenapplied=nitrogenapplied,
                    user=self.auth_user,
                    modified_by=self.auth_user
                )
                if control_research:
                    return control_research
                else:
                    raise ValidationError({"non_field_errors": ["This is not a valid content type"]})

        return ControlResearchCreateSerializer

    class ControlResearchListSerializer(ModelSerializer):
        """
        Serialize all records in given fields into an API
        """
        url = hyperlinked_identity('research_type_api:control_research_detail', 'pk')

        class Meta:
            model = ControlResearch
            fields = ['id', 'url', ]

    class ControlResearchDetailSerializer(ModelSerializer, FieldMethodSerializer):
        """
        Serialize single record into an API. This is dependent on fields given.
        """
        csa_practice_url = SerializerMethodField()
        experiment_replications_url = SerializerMethodField()
        experiment_details_url = SerializerMethodField()
        nitrogen_applied_url = SerializerMethodField()
        user = SerializerMethodField()
        modified_by = SerializerMethodField()
        content_type_url = SerializerMethodField()

        class Meta:
            common_fields = [
                'modified_by',
                'last_update',
                'time_created',
                'content_type_url',
            ]
            model = ControlResearch
            fields = ['id', 'csa_practice_url', 'experiment_replications_url',
                      'experiment_details_url', 'nitrogen_applied_url', 'user'] + common_fields
            read_only_fields = ['id', ] + common_fields

        def get_csa_practice_url(self, obj):
            """
            Get related content type/object url
            :param obj: Current record object
            :return: URL to related object
            :rtype: String
            """
            return get_related_content_url(CsaPractice, obj.csapractice.id)

        def get_experiment_details_url(self, obj):
            """
            Get related content type/object url
            :param obj: Current record object
            :return: URL to related object
            :rtype: String
            """
            return get_related_content_url(ExperimentRep, obj.experimentrep.id)

        def get_experiment_replications_url(self, obj):
            """
            Get related content type/object url
            :param obj: Current record object
            :return: URL to related object
            :rtype: String
            """
            return get_related_content_url(ExperimentDetails, obj.experimentdetails.id)

        def get_nitrogen_applied_url(self, obj):
            """
            Get related content type/object url
            :param obj: Current record object
            :return: URL to related object
            :rtype: String
            """
            return get_related_content_url(NitrogenApplied, obj.nitrogenapplied.id)

    return {
        'create_control_research_serializer': create_control_research_serializer,
        'ControlResearchListSerializer': ControlResearchListSerializer,
        'ControlResearchDetailSerializer': ControlResearchDetailSerializer
    }
<commit_msg>added "Experiment duration" url to "Control research" serializer<commit_after>from csacompendium.csa_practice.models import CsaPractice
from csacompendium.research_type.models import (
    ControlResearch,
    ExperimentRep,
    NitrogenApplied,
    ExperimentDetails,
    ExperimentDuration,
)
from csacompendium.utils.hyperlinkedidentity import hyperlinked_identity
from csacompendium.utils.serializersutils import (
    get_related_content_url,
    valid_integer,
    CreateSerializerUtil,
    FieldMethodSerializer,
)
from rest_framework.serializers import (
    ModelSerializer,
    SerializerMethodField,
    ValidationError,
)

def control_research_serializers():
    """
    Control research serializers
    :return: All control research serializers
    :rtype: Object
    """

    def create_control_research_serializer(model_type=None, request_key=None, user=None):
        """
        Creates a model serializer
        :param model_type: Model
        :param request_key: A slug or primary key depending on what is passed
        :param user: Record owner
        :return: Serializer class
        :rtype: Object
        """

        class ControlResearchCreateSerializer(ModelSerializer, CreateSerializerUtil):
            """
            Create a record
            """
            class Meta:
                model = ControlResearch
                fields = ['id', 'csapractice', 'experimentrep', 'experimentdetails',
                          'nitrogenapplied', 'experimentduration', 'last_update', 'time_created', ]

            def __init__(self, *args, **kwargs):
                super(ControlResearchCreateSerializer, self).__init__(*args, **kwargs)
                self.model_type = model_type
                self.key = request_key
                self.user = user
                if valid_integer(request_key):
                    self.slugify = False
                else:
                    self.slugify = True
                self.auth_user = self.get_authenticated_user(self.user)

            def create(self, validated_data):
                """
                Created record from validated data
                :param validated_data: Validated data
                :return: Control research object
                :rtype: Object
                """
                csapractice = validated_data.get('csapractice')
                experimentrep = validated_data.get('experimentrep')
                experimentdetails = validated_data.get('experimentdetails')
                nitrogenapplied = validated_data.get('nitrogenapplied')
                experimentduration = validated_data.get('experimentduration')
                control_research = ControlResearch.objects.create_by_model_type(
                    self.model_type,
                    self.key,
                    csapractice=csapractice,
                    experimentrep=experimentrep,
                    experimentdetails=experimentdetails,
                    nitrogenapplied=nitrogenapplied,
                    experimentduration=experimentduration,
                    user=self.auth_user,
                    modified_by=self.auth_user
                )
                if control_research:
                    return control_research
                else:
                    raise ValidationError({"non_field_errors": ["This is not a valid content type"]})

        return ControlResearchCreateSerializer

    class ControlResearchListSerializer(ModelSerializer):
        """
        Serialize all records in given fields into an API
        """
        url = hyperlinked_identity('research_type_api:control_research_detail', 'pk')

        class Meta:
            model = ControlResearch
            fields = ['id', 'url', ]

    class ControlResearchDetailSerializer(ModelSerializer, FieldMethodSerializer):
        """
        Serialize single record into an API. This is dependent on fields given.
        """
        csa_practice_url = SerializerMethodField()
        experiment_replications_url = SerializerMethodField()
        experiment_details_url = SerializerMethodField()
        nitrogen_applied_url = SerializerMethodField()
        experiment_duration_url = SerializerMethodField()
        user = SerializerMethodField()
        modified_by = SerializerMethodField()
        content_type_url = SerializerMethodField()

        class Meta:
            common_fields = [
                'modified_by',
                'last_update',
                'time_created',
                'content_type_url',
            ]
            model = ControlResearch
            fields = ['id', 'csa_practice_url', 'experiment_replications_url',
                      'experiment_details_url', 'nitrogen_applied_url', 'experiment_duration_url',
                      'user'] + common_fields
            read_only_fields = ['id', ] + common_fields

        def get_csa_practice_url(self, obj):
            """
            Get related content type/object url
            :param obj: Current record object
            :return: URL to related object
            :rtype: String
            """
            return get_related_content_url(CsaPractice, obj.csapractice.id)

        def get_experiment_details_url(self, obj):
            """
            Get related content type/object url
            :param obj: Current record object
            :return: URL to related object
            :rtype: String
            """
            return get_related_content_url(ExperimentRep, obj.experimentrep.id)

        def get_experiment_replications_url(self, obj):
            """
            Get related content type/object url
            :param obj: Current record object
            :return: URL to related object
            :rtype: String
            """
            return get_related_content_url(ExperimentDetails, obj.experimentdetails.id)

        def get_nitrogen_applied_url(self, obj):
            """
            Get related content type/object url
            :param obj: Current record object
            :return: URL to related object
            :rtype: String
            """
            return get_related_content_url(NitrogenApplied, obj.nitrogenapplied.id)

        def get_experiment_duration_url(self, obj):
            """
            Get related content type/object url
            :param obj: Current record object
            :return: URL to related object
            :rtype: String
            """
            return get_related_content_url(ExperimentDuration, obj.experimentduration.id)

    return {
        'create_control_research_serializer': create_control_research_serializer,
        'ControlResearchListSerializer': ControlResearchListSerializer,
        'ControlResearchDetailSerializer': ControlResearchDetailSerializer
    }
<|endoftext|>


<commit_before>#!/usr/bin/env python
#Script to build a scriptorium paper in a cross-platform friendly fashion

import argparse
import argcomplete
import shutil
import sys
import os
import os.path
import yaml

import scriptorium

def build_cmd(args):
    """Creates PDF from paper in the requested location."""
    pdf = scriptorium.to_pdf(args.paper, use_shell_escape=args.shell_escape, flatten=args.flatten)

    if args.output and pdf != args.output:
        shutil.move(pdf, args.output)

def info(args):
    """Function to attempt to extract useful information from a specified paper."""
    fname = scriptorium.paper_root(args.paper)

    if not fname:
        print('{0} does not contain a valid root document.'.format(args.paper))
        sys.exit(1)

    if args.template:
        template = scriptorium.get_template(os.path.join(args.paper, fname))
        if not template:
            print('Could not find footer indicating template name.')
            sys.exit(2)
        print(template)

def template_cmd(args):
    """Prints out all installed templates."""
    if args.update:
        rev = args.update[1] if len(args.update) > 1 else None
        scriptorium.update_template(args.update[0], args.template_dir, rev)

    if args.list:
        templates = scriptorium.all_templates(args.template_dir)
        print('\n'.join(templates))

    if args.readme:
        template = scriptorium.find_template(args.readme, args.template_dir)
        template_readme = os.path.join(template, 'README.md')
        if template and os.path.exists(template_readme):
            with open(template_readme, 'r') as readme:
                print(readme.read())

    if args.install:
        scriptorium.install_template(args.install, args.template_dir)

    if args.variables:
        variables = scriptorium.list_variables(args.variables, args.template_dir)
        print('\n'.join(variables))

    if args.manifest:
        manifest = scriptorium.get_manifest(args.manifest, args.template_dir)

        for kk, vv in manifest.items():
            print("{0} -> {1}".format(vv, kk))

def create_cmd(args):
    """Creates a new paper given flags."""
    args.config = {k.upper():v for k, v in args.config}
    if not scriptorium.create(args.output, args.template, force=args.force, config=args.config):
        sys.exit(3)

def doctor_cmd(_):
    """Command for checking the health of scriptorium."""
    missing_packages = scriptorium.find_missing_packages()
    if missing_packages:
        for package in missing_packages:
            print('Missing package {0}\n'.format(package))

def config_cmd(args):
    """Command to access configuration values."""
    if args.list:
        print(yaml.dump(scriptorium.CONFIG, default_flow_style=False))
    elif len(args.value) == 1:
        print(yaml.dump({args.value[0] : scriptorium.CONFIG[args.value[0]]}))
    elif len(args.value) == 2:
        setattr(scriptorium, args.value[0], args.value[1])
        scriptorium.save_config()

def main():
    """Main function for executing scriptorium as a standalone script."""
    parser = argparse.ArgumentParser()

    subparsers = parser.add_subparsers()

    # Build Command
    build_parser = subparsers.add_parser('build')
    build_parser.add_argument('paper', default='.', nargs='?',
                              help='Directory containing paper to build')
    build_parser.add_argument('-o', '--output', help='Destination of PDF')
    build_parser.add_argument('-s', '--shell-escape', action='store_true', default=False,
                              help='Flag indicating shell-escape should be used')
    build_parser.add_argument('-f', '--flatten', action='store_true', default=False,
                              help='Flatten root LaTeX file output')
    build_parser.set_defaults(func=build_cmd)

    # Info Command
    info_parser = subparsers.add_parser('info')
    info_parser.add_argument('paper', default='.', help='Directory containing paper to make')
    info_parser.add_argument('-t', '--template', action='store_true',
                             help='Flag to extract template')
    info_parser.set_defaults(func=info)

    # New Command
    new_parser = subparsers.add_parser("new")
    new_parser.add_argument("output", help="Directory to create paper in.")
    new_parser.add_argument("-f", "--force", action="store_true",
                            help="Overwrite files in paper creation.")
    new_parser.add_argument("-t", "--template", help="Template to use in paper.")
    new_parser.add_argument("-c", "--config", nargs=2, action='append', default=[],
                            help='Provide "key" "value" to replace in default paper.')
    new_parser.set_defaults(func=create_cmd)

    # Template Command
    template_parser = subparsers.add_parser("template")
    template_parser.add_argument('-l', '--list', action='store_true', default=False,
                                 help='List available templates')
    template_parser.add_argument('-u', '--update', nargs='+',
                                 help='Update the given template to the latest version')
    template_parser.add_argument('-r', '--readme', help='Print README for the specified template')
    template_parser.add_argument('-d', '--template_dir', default=None,
                                 help='Overrides template directory used for listing templates')
    template_parser.add_argument('-i', '--install',
                                 help='Install repository at given URL in template directory')
    template_parser.add_argument('-v', '--variables',
                                 help='List variables available when using the new command')
    template_parser.add_argument('-m', '--manifest',
                                help='List the manifest of files this template provides for new documents')
    template_parser.set_defaults(func=template_cmd)

    # Doctor Command
    doctor_parser = subparsers.add_parser('doctor')
    doctor_parser.set_defaults(func=doctor_cmd)

    # Config Command
    config_parser = subparsers.add_parser('config')
    config_parser.add_argument('-l', '--list', action='store_true',
                               help='List available configuration options and current vaules')
    config_parser.add_argument('value', nargs='*', help='Access configuration value')
    config_parser.set_defaults(func=config_cmd)

    argcomplete.autocomplete(parser)
    args = parser.parse_args()

    if 'func' in args:
        args.func(args)
    else:
        parser.print_help()

if __name__ == '__main__':
    # Only save configuration with main
    main()
<commit_msg>Fix configuration storing at command line<commit_after>#!/usr/bin/env python
#Script to build a scriptorium paper in a cross-platform friendly fashion

import argparse
import argcomplete
import shutil
import sys
import os
import os.path
import yaml

import scriptorium

def build_cmd(args):
    """Creates PDF from paper in the requested location."""
    pdf = scriptorium.to_pdf(args.paper, use_shell_escape=args.shell_escape, flatten=args.flatten)

    if args.output and pdf != args.output:
        shutil.move(pdf, args.output)

def info(args):
    """Function to attempt to extract useful information from a specified paper."""
    fname = scriptorium.paper_root(args.paper)

    if not fname:
        print('{0} does not contain a valid root document.'.format(args.paper))
        sys.exit(1)

    if args.template:
        template = scriptorium.get_template(os.path.join(args.paper, fname))
        if not template:
            print('Could not find footer indicating template name.')
            sys.exit(2)
        print(template)

def template_cmd(args):
    """Prints out all installed templates."""
    if args.update:
        rev = args.update[1] if len(args.update) > 1 else None
        scriptorium.update_template(args.update[0], args.template_dir, rev)

    if args.list:
        templates = scriptorium.all_templates(args.template_dir)
        print('\n'.join(templates))

    if args.readme:
        template = scriptorium.find_template(args.readme, args.template_dir)
        template_readme = os.path.join(template, 'README.md')
        if template and os.path.exists(template_readme):
            with open(template_readme, 'r') as readme:
                print(readme.read())

    if args.install:
        scriptorium.install_template(args.install, args.template_dir)

    if args.variables:
        variables = scriptorium.list_variables(args.variables, args.template_dir)
        print('\n'.join(variables))

    if args.manifest:
        manifest = scriptorium.get_manifest(args.manifest, args.template_dir)

        for kk, vv in manifest.items():
            print("{0} -> {1}".format(vv, kk))

def create_cmd(args):
    """Creates a new paper given flags."""
    args.config = {k.upper():v for k, v in args.config}
    if not scriptorium.create(args.output, args.template, force=args.force, config=args.config):
        sys.exit(3)

def doctor_cmd(_):
    """Command for checking the health of scriptorium."""
    missing_packages = scriptorium.find_missing_packages()
    if missing_packages:
        for package in missing_packages:
            print('Missing package {0}\n'.format(package))

def config_cmd(args):
    """Command to access configuration values."""
    if args.list:
        print(yaml.dump(scriptorium.CONFIG, default_flow_style=False))
    elif len(args.value) == 1:
        print(yaml.dump({args.value[0] : scriptorium.CONFIG[args.value[0]]}))
    elif len(args.value) == 2:
        scriptorium.CONFIG[args.value[0].upper()] = args.value[1]
        scriptorium.save_config()

def main():
    """Main function for executing scriptorium as a standalone script."""
    parser = argparse.ArgumentParser()

    subparsers = parser.add_subparsers()

    # Build Command
    build_parser = subparsers.add_parser('build')
    build_parser.add_argument('paper', default='.', nargs='?',
                              help='Directory containing paper to build')
    build_parser.add_argument('-o', '--output', help='Destination of PDF')
    build_parser.add_argument('-s', '--shell-escape', action='store_true', default=False,
                              help='Flag indicating shell-escape should be used')
    build_parser.add_argument('-f', '--flatten', action='store_true', default=False,
                              help='Flatten root LaTeX file output')
    build_parser.set_defaults(func=build_cmd)

    # Info Command
    info_parser = subparsers.add_parser('info')
    info_parser.add_argument('paper', default='.', help='Directory containing paper to make')
    info_parser.add_argument('-t', '--template', action='store_true',
                             help='Flag to extract template')
    info_parser.set_defaults(func=info)

    # New Command
    new_parser = subparsers.add_parser("new")
    new_parser.add_argument("output", help="Directory to create paper in.")
    new_parser.add_argument("-f", "--force", action="store_true",
                            help="Overwrite files in paper creation.")
    new_parser.add_argument("-t", "--template", help="Template to use in paper.")
    new_parser.add_argument("-c", "--config", nargs=2, action='append', default=[],
                            help='Provide "key" "value" to replace in default paper.')
    new_parser.set_defaults(func=create_cmd)

    # Template Command
    template_parser = subparsers.add_parser("template")
    template_parser.add_argument('-l', '--list', action='store_true', default=False,
                                 help='List available templates')
    template_parser.add_argument('-u', '--update', nargs='+',
                                 help='Update the given template to the latest version')
    template_parser.add_argument('-r', '--readme', help='Print README for the specified template')
    template_parser.add_argument('-d', '--template_dir', default=None,
                                 help='Overrides template directory used for listing templates')
    template_parser.add_argument('-i', '--install',
                                 help='Install repository at given URL in template directory')
    template_parser.add_argument('-v', '--variables',
                                 help='List variables available when using the new command')
    template_parser.add_argument('-m', '--manifest',
                                help='List the manifest of files this template provides for new documents')
    template_parser.set_defaults(func=template_cmd)

    # Doctor Command
    doctor_parser = subparsers.add_parser('doctor')
    doctor_parser.set_defaults(func=doctor_cmd)

    # Config Command
    config_parser = subparsers.add_parser('config')
    config_parser.add_argument('-l', '--list', action='store_true',
                               help='List available configuration options and current vaules')
    config_parser.add_argument('value', nargs='*', help='Access configuration value')
    config_parser.set_defaults(func=config_cmd)

    argcomplete.autocomplete(parser)
    args = parser.parse_args()

    if 'func' in args:
        args.func(args)
    else:
        parser.print_help()

if __name__ == '__main__':
    # Only save configuration with main
    main()
<|endoftext|>