<commit_before>r"""Functions for exclusive $B\to P\ell\nu$ decays."""

from math import sqrt
import flavio
from flavio.physics.bdecays.common import meson_quark, meson_ff
from flavio.physics import ckm
from flavio.classes import AuxiliaryQuantity
from flavio.config import config
from flavio.physics.running import running
from flavio.physics.bdecays import angular
from flavio.physics.bdecays.wilsoncoefficients import get_wceff_fccc
from flavio.classes import Observable, Prediction

def prefactor(q2, par, B, P, lep):
    GF = par['GF']
    ml = par['m_'+lep]
    scale = config['renormalization scale']['bpll']
    alphaem = running.get_alpha(par, scale)['alpha_e']
    di_dj = meson_quark[(B,P)]
    qi_qj = meson_quark[(B, P)]
    if qi_qj == 'bu':
        Vij = ckm.get_ckm(par)[0,2] # V_{ub} for b->u transitions
    if qi_qj == 'bc':
        Vij = ckm.get_ckm(par)[1,2] # V_{cb} for b->c transitions
    if q2 <= ml**2:
        return 0
    return 4*GF/sqrt(2)*Vij

def get_ff(q2, par, B, P):
    ff_name = meson_ff[(B,P)] + ' form factor'
    return AuxiliaryQuantity.get_instance(ff_name).prediction(par_dict=par, wc_obj=None, q2=q2)

def get_angularcoeff(q2, wc_obj, par, B, P, lep):
    ml = par['m_'+lep]
    mB = par['m_'+B]
    mP = par['m_'+P]
    scale = config['renormalization scale']['bpll']
    mb = running.get_mb(par, scale)
    wc = get_wceff_fccc(wc_obj, par, meson_quark[(B,P)], lep, mb, scale, nf=5)
    N = prefactor(q2, par, B, P, lep)
    ff = get_ff(q2, par, B, P)
    qi_qj = meson_quark[(B, P)]
    if qi_qj == 'bu':
        mlight = 0. # neglecting the up quark mass
    if qi_qj == 'bc':
        mlight = running.get_mc(par, scale) # this is needed for scalar contributions
    h = angular.helicity_amps_p(q2, mB, mP, mb, mlight, ml, 0, ff, wc, N)
    J = angular.angularcoeffs_general_p(h, q2, mB, mP, mb, mlight, ml, 0)
    return J

def dGdq2(J):
    return 2 * (J['a'] + J['c']/3.)

def dBRdq2_lep(q2, wc_obj, par, B, P, lep):
    ml = par['m_'+lep]
    mB = par['m_'+B]
    mP = par['m_'+P]
    if q2 < ml**2 or q2 > (mB-mP)**2:
        return 0
    tauB = par['tau_'+B]
    J = get_angularcoeff(q2, wc_obj, par, B, P, lep)
    if P == 'pi0':
        # factor of 1/2 for neutral pi due to pi = (uubar-ddbar)/sqrt(2)
        return tauB * dGdq2(J) / 2.
    return tauB * dGdq2(J)

def dBRdq2(q2, wc_obj, par, B, P, lep):
    if lep == 'l':
        # average of e and mu!
        return (dBRdq2_lep(q2, wc_obj, par, B, P, 'e') + dBRdq2_lep(q2, wc_obj, par, B, P, 'mu'))/2
    else:
        return dBRdq2_lep(q2, wc_obj, par, B, P, lep)

def dBRdq2_function(B, P, lep):
    return lambda wc_obj, par, q2: dBRdq2(q2, wc_obj, par, B, P, lep)

def BR_binned(q2min, q2max, wc_obj, par, B, P, lep):
    def integrand(q2):
        return dBRdq2(q2, wc_obj, par, B, P, lep)
    return flavio.math.integrate.nintegrate(integrand, q2min, q2max)

def BR_binned_function(B, P, lep):
    return lambda wc_obj, par, q2min, q2max: BR_binned(q2min, q2max, wc_obj, par, B, P, lep)

def BR_binned_leptonflavour(q2min, q2max, wc_obj, par, B, P, lnum, lden):
    num = BR_binned(q2min, q2max, wc_obj, par, B, P, lnum)
    if num == 0:
        return 0
    den = BR_binned(q2min, q2max, wc_obj, par, B, P, lden)
    return num/den

def BR_binned_leptonflavour_function(B, P, lnum, lden):
    return lambda wc_obj, par, q2min, q2max: BR_binned_leptonflavour(q2min, q2max, wc_obj, par, B, P, lnum, lden)

def _BR_tot(wc_obj, par, B, P, lep):
    mB = par['m_'+B]
    mP = par['m_'+P]
    ml = par['m_'+lep]
    q2max = (mB-mP)**2
    q2min = ml**2
    return BR_binned(q2min, q2max, wc_obj, par, B, P, lep)

def BR_tot(wc_obj, par, B, P, lep):
    if lep == 'l':
        # average of e and mu!
        return (_BR_tot(wc_obj, par, B, P, 'e')+_BR_tot(wc_obj, par, B, P, 'mu'))/2.
    else:
        return _BR_tot(wc_obj, par, B, P, lep)

def BR_tot_function(B, P, lep):
    return lambda wc_obj, par: BR_tot(wc_obj, par, B, P, lep)

def BR_tot_leptonflavour(wc_obj, par, B, P, lnum, lden):
    num = _BR_tot(wc_obj, par, B, P, lnum)
    if num == 0:
        return 0
    den = _BR_tot(wc_obj, par, B, P, lden)
    return num/den

def BR_tot_leptonflavour_function(B, P, lnum, lden):
    return lambda wc_obj, par: BR_tot_leptonflavour(wc_obj, par, B, P, lnum, lden)

# Observable and Prediction instances

_tex = {'e': 'e', 'mu': '\mu', 'tau': r'\tau', 'l': r'\ell'}
_func = {'dBR/dq2': dBRdq2_function, 'BR': BR_tot_function, '
': BR_binned_function}
_desc = {'dBR/dq2': 'Differential', 'BR': 'Total', '
': 'Binned'}
_tex_br = {'dBR/dq2': r'\frac{d\text{BR}}{dq^2}', 'BR': r'\text{BR}', '
': r'\langle\text{BR}\rangle'}
_args = {'dBR/dq2': ['q2'], 'BR': None, '
': ['q2min', 'q2max']}

_hadr = {
'B+->D': {'tex': r"B^+\to D^0", 'B': 'B+', 'P': 'D0', },
'B0->D': {'tex': r"B^0\to D^- ", 'B': 'B0', 'P': 'D+', },
'B+->pi': {'tex': r"B^+\to \pi^0", 'B': 'B+', 'P': 'pi0', },
'B0->pi': {'tex': r"B^0\to \pi^- ", 'B': 'B0', 'P': 'pi+', },
}

# for LF ratios we don't distinguish B+ and B0 (but take B0 because we have to choose sth)
_hadr_l = {
'B->D': {'tex': r"B\to D", 'B': 'B0', 'P': 'D+', 'decays': ['B+->D', 'B0->D'],},
'B->pi': {'tex': r"B\to \pi ", 'B': 'B0', 'P': 'pi+', 'decays': ['B+->pi', 'B0->pi'],},
}

_process_taxonomy = r'Process :: $b$ hadron decays :: Semi-leptonic tree-level decays :: $B\to P\ell\nu$ :: $'

for l in ['e', 'mu', 'tau', 'l']:
    for M in _hadr.keys():
        for br in ['dBR/dq2', 'BR', '
']:
            _obs_name = br + "("+M+l+"nu)"
            _process_tex = _hadr[M]['tex']+_tex[l]+r"^+\nu_"+_tex[l]
            _obs = Observable(_obs_name)
            _obs.set_description(_desc[br] + r" branching ratio of $" + _process_tex + r"$")
            _obs.tex = r'$' + _tex_br[br] + r"(" + _process_tex + r")$"
            _obs.arguments = _args[br]
            _obs.add_taxonomy(_process_taxonomy + _process_tex + r"$")
            Prediction(_obs_name, _func[br](_hadr[M]['B'], _hadr[M]['P'], l))

# Lepton flavour ratios
for l in [('mu','e'), ('tau','mu'),]:
    for M in _hadr_l.keys():

        # binned ratio of BRs
        _obs_name = "<R"+l[0]+l[1]+">("+M+"lnu)"
        _obs = Observable(name=_obs_name, arguments=['q2min', 'q2max'])
        _obs.set_description(r"Ratio of partial branching ratios of $" + _hadr_l[M]['tex'] +_tex[l[0]]+r"^+ \nu_"+_tex[l[0]]+r"$" + " and " + r"$" + _hadr_l[M]['tex'] +_tex[l[1]]+r"^+ \nu_"+_tex[l[1]]+r"$")
        _obs.tex = r"$\langle R_{" + _tex[l[0]] + ' ' + _tex[l[1]] + r"} \rangle(" + _hadr_l[M]['tex'] + r"\ell^+\nu)$"
        for li in l:
            for N in _hadr_l[M]['decays']:
                # add taxonomy for both processes (e.g. B->Penu and B->Pmunu) and for charged and neutral
                _obs.add_taxonomy(_process_taxonomy + _hadr[N]['tex'] + _tex[li]+r"^+\nu_"+_tex[li]+r"$")
        Prediction(_obs_name, BR_binned_leptonflavour_function(_hadr_l[M]['B'], _hadr_l[M]['P'], l[0], l[1]))

        # ratio of total BRs
        _obs_name = "R"+l[0]+l[1]+"("+M+"lnu)"
        _obs = Observable(name=_obs_name, arguments=['q2min', 'q2max'])
        _obs.set_description(r"Ratio of total branching ratios of $" + _hadr_l[M]['tex'] +_tex[l[0]]+r"^+ \nu_"+_tex[l[0]]+r"$" + " and " + r"$" + _hadr_l[M]['tex'] +_tex[l[1]]+r"^+ \nu_"+_tex[l[1]]+r"$")
        _obs.tex = r"$\langle R_{" + _tex[l[0]] + ' ' + _tex[l[1]] + r"} \rangle(" + _hadr_l[M]['tex'] + r"\ell^+\nu)$"
        for li in l:
            for N in _hadr_l[M]['decays']:
                # add taxonomy for both processes (e.g. B->Penu and B->Pmunu) and for charged and neutral
                _obs.add_taxonomy(_process_taxonomy + _hadr[N]['tex'] +_tex[li]+r"^+\nu_"+_tex[li]+r"$")
        Prediction(_obs_name, BR_tot_leptonflavour_function(_hadr_l[M]['B'], _hadr_l[M]['P'], l[0], l[1]))
<commit_msg>[bplnu] Added ratio of tau vs e/mu average for B->Plnu decays<commit_after>r"""Functions for exclusive $B\to P\ell\nu$ decays."""

from math import sqrt
import flavio
from flavio.physics.bdecays.common import meson_quark, meson_ff
from flavio.physics import ckm
from flavio.classes import AuxiliaryQuantity
from flavio.config import config
from flavio.physics.running import running
from flavio.physics.bdecays import angular
from flavio.physics.bdecays.wilsoncoefficients import get_wceff_fccc
from flavio.classes import Observable, Prediction

def prefactor(q2, par, B, P, lep):
    GF = par['GF']
    ml = par['m_'+lep]
    scale = config['renormalization scale']['bpll']
    alphaem = running.get_alpha(par, scale)['alpha_e']
    di_dj = meson_quark[(B,P)]
    qi_qj = meson_quark[(B, P)]
    if qi_qj == 'bu':
        Vij = ckm.get_ckm(par)[0,2] # V_{ub} for b->u transitions
    if qi_qj == 'bc':
        Vij = ckm.get_ckm(par)[1,2] # V_{cb} for b->c transitions
    if q2 <= ml**2:
        return 0
    return 4*GF/sqrt(2)*Vij

def get_ff(q2, par, B, P):
    ff_name = meson_ff[(B,P)] + ' form factor'
    return AuxiliaryQuantity.get_instance(ff_name).prediction(par_dict=par, wc_obj=None, q2=q2)

def get_angularcoeff(q2, wc_obj, par, B, P, lep):
    ml = par['m_'+lep]
    mB = par['m_'+B]
    mP = par['m_'+P]
    scale = config['renormalization scale']['bpll']
    mb = running.get_mb(par, scale)
    wc = get_wceff_fccc(wc_obj, par, meson_quark[(B,P)], lep, mb, scale, nf=5)
    N = prefactor(q2, par, B, P, lep)
    ff = get_ff(q2, par, B, P)
    qi_qj = meson_quark[(B, P)]
    if qi_qj == 'bu':
        mlight = 0. # neglecting the up quark mass
    if qi_qj == 'bc':
        mlight = running.get_mc(par, scale) # this is needed for scalar contributions
    h = angular.helicity_amps_p(q2, mB, mP, mb, mlight, ml, 0, ff, wc, N)
    J = angular.angularcoeffs_general_p(h, q2, mB, mP, mb, mlight, ml, 0)
    return J

def dGdq2(J):
    return 2 * (J['a'] + J['c']/3.)

def dBRdq2_lep(q2, wc_obj, par, B, P, lep):
    ml = par['m_'+lep]
    mB = par['m_'+B]
    mP = par['m_'+P]
    if q2 < ml**2 or q2 > (mB-mP)**2:
        return 0
    tauB = par['tau_'+B]
    J = get_angularcoeff(q2, wc_obj, par, B, P, lep)
    if P == 'pi0':
        # factor of 1/2 for neutral pi due to pi = (uubar-ddbar)/sqrt(2)
        return tauB * dGdq2(J) / 2.
    return tauB * dGdq2(J)

def dBRdq2(q2, wc_obj, par, B, P, lep):
    if lep == 'l':
        # average of e and mu!
        return (dBRdq2_lep(q2, wc_obj, par, B, P, 'e') + dBRdq2_lep(q2, wc_obj, par, B, P, 'mu'))/2
    else:
        return dBRdq2_lep(q2, wc_obj, par, B, P, lep)

def dBRdq2_function(B, P, lep):
    return lambda wc_obj, par, q2: dBRdq2(q2, wc_obj, par, B, P, lep)

def BR_binned(q2min, q2max, wc_obj, par, B, P, lep):
    def integrand(q2):
        return dBRdq2(q2, wc_obj, par, B, P, lep)
    return flavio.math.integrate.nintegrate(integrand, q2min, q2max)

def BR_binned_function(B, P, lep):
    return lambda wc_obj, par, q2min, q2max: BR_binned(q2min, q2max, wc_obj, par, B, P, lep)

def BR_binned_leptonflavour(q2min, q2max, wc_obj, par, B, P, lnum, lden):
    num = BR_binned(q2min, q2max, wc_obj, par, B, P, lnum)
    if num == 0:
        return 0
    den = BR_binned(q2min, q2max, wc_obj, par, B, P, lden)
    return num/den

def BR_binned_leptonflavour_function(B, P, lnum, lden):
    return lambda wc_obj, par, q2min, q2max: BR_binned_leptonflavour(q2min, q2max, wc_obj, par, B, P, lnum, lden)

def _BR_tot(wc_obj, par, B, P, lep):
    mB = par['m_'+B]
    mP = par['m_'+P]
    ml = par['m_'+lep]
    q2max = (mB-mP)**2
    q2min = ml**2
    return BR_binned(q2min, q2max, wc_obj, par, B, P, lep)

def BR_tot(wc_obj, par, B, P, lep):
    if lep == 'l':
        # average of e and mu!
        return (_BR_tot(wc_obj, par, B, P, 'e')+_BR_tot(wc_obj, par, B, P, 'mu'))/2.
    else:
        return _BR_tot(wc_obj, par, B, P, lep)

def BR_tot_function(B, P, lep):
    return lambda wc_obj, par: BR_tot(wc_obj, par, B, P, lep)

def BR_tot_leptonflavour(wc_obj, par, B, P, lnum, lden):
    num = BR_tot(wc_obj, par, B, P, lnum)
    if num == 0:
        return 0
    den = BR_tot(wc_obj, par, B, P, lden)
    return num/den

def BR_tot_leptonflavour_function(B, P, lnum, lden):
    return lambda wc_obj, par: BR_tot_leptonflavour(wc_obj, par, B, P, lnum, lden)

# Observable and Prediction instances

_tex = {'e': 'e', 'mu': '\mu', 'tau': r'\tau', 'l': r'\ell'}
_func = {'dBR/dq2': dBRdq2_function, 'BR': BR_tot_function, '
': BR_binned_function}
_desc = {'dBR/dq2': 'Differential', 'BR': 'Total', '
': 'Binned'}
_tex_br = {'dBR/dq2': r'\frac{d\text{BR}}{dq^2}', 'BR': r'\text{BR}', '
': r'\langle\text{BR}\rangle'}
_args = {'dBR/dq2': ['q2'], 'BR': None, '
': ['q2min', 'q2max']}

_hadr = {
'B+->D': {'tex': r"B^+\to D^0", 'B': 'B+', 'P': 'D0', },
'B0->D': {'tex': r"B^0\to D^- ", 'B': 'B0', 'P': 'D+', },
'B+->pi': {'tex': r"B^+\to \pi^0", 'B': 'B+', 'P': 'pi0', },
'B0->pi': {'tex': r"B^0\to \pi^- ", 'B': 'B0', 'P': 'pi+', },
}

# for LF ratios we don't distinguish B+ and B0 (but take B0 because we have to choose sth)
_hadr_l = {
'B->D': {'tex': r"B\to D", 'B': 'B0', 'P': 'D+', 'decays': ['B+->D', 'B0->D'],},
'B->pi': {'tex': r"B\to \pi ", 'B': 'B0', 'P': 'pi+', 'decays': ['B+->pi', 'B0->pi'],},
}

_process_taxonomy = r'Process :: $b$ hadron decays :: Semi-leptonic tree-level decays :: $B\to P\ell\nu$ :: $'

for l in ['e', 'mu', 'tau', 'l']:
    for M in _hadr.keys():
        for br in ['dBR/dq2', 'BR', '
']:
            _obs_name = br + "("+M+l+"nu)"
            _process_tex = _hadr[M]['tex']+_tex[l]+r"^+\nu_"+_tex[l]
            _obs = Observable(_obs_name)
            _obs.set_description(_desc[br] + r" branching ratio of $" + _process_tex + r"$")
            _obs.tex = r'$' + _tex_br[br] + r"(" + _process_tex + r")$"
            _obs.arguments = _args[br]
            _obs.add_taxonomy(_process_taxonomy + _process_tex + r"$")
            Prediction(_obs_name, _func[br](_hadr[M]['B'], _hadr[M]['P'], l))

# Lepton flavour ratios
for l in [('mu','e'), ('tau','mu'), ('tau', 'l')]:
    for M in _hadr_l.keys():

        # binned ratio of BRs
        _obs_name = "<R"+l[0]+l[1]+">("+M+"lnu)"
        _obs = Observable(name=_obs_name, arguments=['q2min', 'q2max'])
        _obs.set_description(r"Ratio of partial branching ratios of $" + _hadr_l[M]['tex'] +_tex[l[0]]+r"^+ \nu_"+_tex[l[0]]+r"$" + " and " + r"$" + _hadr_l[M]['tex'] +_tex[l[1]]+r"^+ \nu_"+_tex[l[1]]+r"$")
        _obs.tex = r"$\langle R_{" + _tex[l[0]] + ' ' + _tex[l[1]] + r"} \rangle(" + _hadr_l[M]['tex'] + r"\ell^+\nu)$"
        for li in l:
            for N in _hadr_l[M]['decays']:
                # add taxonomy for both processes (e.g. B->Penu and B->Pmunu) and for charged and neutral
                _obs.add_taxonomy(_process_taxonomy + _hadr[N]['tex'] + _tex[li]+r"^+\nu_"+_tex[li]+r"$")
        Prediction(_obs_name, BR_binned_leptonflavour_function(_hadr_l[M]['B'], _hadr_l[M]['P'], l[0], l[1]))

        # ratio of total BRs
        _obs_name = "R"+l[0]+l[1]+"("+M+"lnu)"
        _obs = Observable(name=_obs_name, arguments=['q2min', 'q2max'])
        _obs.set_description(r"Ratio of total branching ratios of $" + _hadr_l[M]['tex'] +_tex[l[0]]+r"^+ \nu_"+_tex[l[0]]+r"$" + " and " + r"$" + _hadr_l[M]['tex'] +_tex[l[1]]+r"^+ \nu_"+_tex[l[1]]+r"$")
        _obs.tex = r"$R_{" + _tex[l[0]] + ' ' + _tex[l[1]] + r"}(" + _hadr_l[M]['tex'] + r"\ell^+\nu)$"
        for li in l:
            for N in _hadr_l[M]['decays']:
                # add taxonomy for both processes (e.g. B->Penu and B->Pmunu) and for charged and neutral
                _obs.add_taxonomy(_process_taxonomy + _hadr[N]['tex'] +_tex[li]+r"^+\nu_"+_tex[li]+r"$")
        Prediction(_obs_name, BR_tot_leptonflavour_function(_hadr_l[M]['B'], _hadr_l[M]['P'], l[0], l[1]))
<|endoftext|>


<commit_before>import os
import time
import numpy as np
import argparse
import functools
import shutil

import paddle
import paddle.fluid as fluid
import reader
from mobilenet_ssd import mobile_net
from utility import add_arguments, print_arguments

parser = argparse.ArgumentParser(description=__doc__)
add_arg = functools.partial(add_arguments, argparser=parser)
# yapf: disable
add_arg('learning_rate',    float, 0.001,     "Learning rate.")
add_arg('batch_size',       int,   64,        "Minibatch size.")
add_arg('num_passes',       int,   120,       "Epoch number.")
add_arg('use_gpu',          bool,  True,      "Whether use GPU.")
add_arg('parallel',         bool,  True,      "Parallel.")
add_arg('dataset',          str,   'pascalvoc', "coco2014, coco2017, and pascalvoc.")
add_arg('model_save_dir',   str,   'model',     "The path to save model.")
add_arg('pretrained_model', str,   'pretrained/ssd_mobilenet_v1_coco/', "The init model path.")
add_arg('apply_distort',    bool,  True,   "Whether apply distort.")
add_arg('apply_expand',     bool,  True,   "Whether apply expand.")
add_arg('nms_threshold',    float, 0.45,   "NMS threshold.")
add_arg('ap_version',       str,   '11point',   "integral, 11point.")
add_arg('resize_h',         int,   300,    "The resized image height.")
add_arg('resize_w',         int,   300,    "The resized image height.")
add_arg('mean_value_B',     float, 127.5,  "Mean value for B channel which will be subtracted.")  #123.68
add_arg('mean_value_G',     float, 127.5,  "Mean value for G channel which will be subtracted.")  #116.78
add_arg('mean_value_R',     float, 127.5,  "Mean value for R channel which will be subtracted.")  #103.94
add_arg('is_toy',           int,   0, "Toy for quick debug, 0 means using all data, while n means using only n sample.")
add_arg('data_dir',         str,   'data/pascalvoc', "data directory")
add_arg('for_model_ce',     bool,  False, "Use CE to evaluate the model")
#yapf: enable

def train(args,
          train_file_list,
          val_file_list,
          data_args,
          learning_rate,
          batch_size,
          num_passes,
          model_save_dir,
          pretrained_model=None):
    image_shape = [3, data_args.resize_h, data_args.resize_w]
    if 'coco' in data_args.dataset:
        num_classes = 91
    elif 'pascalvoc' in data_args.dataset:
        num_classes = 21

    devices = os.getenv("CUDA_VISIBLE_DEVICES") or ""
    devices_num = len(devices.split(","))

    image = fluid.layers.data(name='image', shape=image_shape, dtype='float32')
    gt_box = fluid.layers.data(
        name='gt_box', shape=[4], dtype='float32', lod_level=1)
    gt_label = fluid.layers.data(
        name='gt_label', shape=[1], dtype='int32', lod_level=1)
    difficult = fluid.layers.data(
        name='gt_difficult', shape=[1], dtype='int32', lod_level=1)

    locs, confs, box, box_var = mobile_net(num_classes, image, image_shape)
    nmsed_out = fluid.layers.detection_output(
        locs, confs, box, box_var, nms_threshold=args.nms_threshold)
    loss = fluid.layers.ssd_loss(locs, confs, gt_box, gt_label, box,
                                 box_var)
    loss = fluid.layers.reduce_sum(loss)

    test_program = fluid.default_main_program().clone(for_test=True)
    with fluid.program_guard(test_program):
        map_eval = fluid.evaluator.DetectionMAP(
            nmsed_out,
            gt_label,
            gt_box,
            difficult,
            num_classes,
            overlap_threshold=0.5,
            evaluate_difficult=False,
            ap_version=args.ap_version)

    if 'coco' in data_args.dataset:
        # learning rate decay in 12, 19 pass, respectively
        if '2014' in train_file_list:
            epocs = 82783 / batch_size
            boundaries = [epocs * 12, epocs * 19]
        elif '2017' in train_file_list:
            epocs = 118287 / batch_size
            boundaries = [epocs * 12, epocs * 19]
        values = [
            learning_rate, learning_rate * 0.5, learning_rate * 0.25
        ]
    elif 'pascalvoc' in data_args.dataset:
        epocs = 19200 / batch_size
        boundaries = [epocs * 40, epocs * 60, epocs * 80, epocs * 100]
        values = [
            learning_rate, learning_rate * 0.5, learning_rate * 0.25,
            learning_rate * 0.1, learning_rate * 0.01
        ]
    optimizer = fluid.optimizer.RMSProp(
        learning_rate=fluid.layers.piecewise_decay(boundaries, values),
        regularization=fluid.regularizer.L2Decay(0.00005), )

    optimizer.minimize(loss)

    place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace()
    exe = fluid.Executor(place)
    exe.run(fluid.default_startup_program())

    if pretrained_model:
        def if_exist(var):
            return os.path.exists(os.path.join(pretrained_model, var.name))
        fluid.io.load_vars(exe, pretrained_model, predicate=if_exist)

    if args.parallel:
        train_exe = fluid.ParallelExecutor(
            use_cuda=args.use_gpu, loss_name=loss.name)

    train_reader = paddle.batch(
        reader.train(data_args, train_file_list), batch_size=batch_size)
    test_reader = paddle.batch(
        reader.test(data_args, val_file_list), batch_size=batch_size)
    feeder = fluid.DataFeeder(
        place=place, feed_list=[image, gt_box, gt_label, difficult])

    def save_model(postfix):
        model_path = os.path.join(model_save_dir, postfix)
        if os.path.isdir(model_path):
            shutil.rmtree(model_path)
        print 'save models to %s' % (model_path)
        fluid.io.save_persistables(exe, model_path)

    best_map = 0.

    def test(pass_id, best_map):
        _, accum_map = map_eval.get_map_var()
        map_eval.reset(exe)
        for batch_id, data in enumerate(test_reader()):
            test_map, = exe.run(test_program,
                               feed=feeder.feed(data),
                               fetch_list=[accum_map])
            if batch_id % 20 == 0:
                print("Batch {0}, map {1}".format(batch_id, test_map))
        if test_map[0] > best_map:
            best_map = test_map[0]
            save_model('best_model')
        print("Pass {0}, test map {1}".format(pass_id, test_map))
        return best_map

    def ce_map(pass_id, best_map):
        _, accum_map = map_eval.get_map_var()
        map_eval.reset(exe)
        every_train_map = []
        for batch_id, data in enumerate(train_reader()):
            out, = exe.run(test_program,
                               feed=feeder.feed(data),
                               fetch_list=[accum_map])
            if batch_id % 20 == 0:
                every_train_map.append(out)
        train_map = np.mean(every_train_map)

        _, accum_map = map_eval.get_map_var()
        map_eval.reset(exe)
        every_test_map = []
        for batch_id, data in enumerate(test_reader()):
            out, = exe.run(test_program,
                               feed=feeder.feed(data),
                               fetch_list=[accum_map])
            if batch_id % 20 == 0:
                every_test_map.append(out)
        test_map = np.mean(every_test_map)
        return (train_map, test_map)

    for pass_id in range(num_passes):
        start_time = time.time()
        prev_start_time = start_time
        every_pass_loss = []
        iter = 0
        pass_duration = 0.0
        for batch_id, data in enumerate(train_reader()):
            prev_start_time = start_time
            start_time = time.time()
            if len(data) < (devices_num * 2):
                print("There are too few data to train on all devices.")
                continue
            if args.parallel:
                loss_v, = train_exe.run(fetch_list=[loss.name],
                                        feed=feeder.feed(data))
            else:
                loss_v, = exe.run(fluid.default_main_program(),
                                  feed=feeder.feed(data),
                                  fetch_list=[loss])
            loss_v = np.mean(np.array(loss_v))
            every_pass_loss.append(loss_v)
            if batch_id % 20 == 0:
                print("Pass {0}, batch {1}, loss {2}, time {3}".format(
                    pass_id, batch_id, loss_v, start_time - prev_start_time))

        end_time = time.time()

        best_map = test(pass_id, best_map)
        if args.for_model_ce:
            train_avg_loss = np.mean(every_pass_loss)
            train_avg_acc, test_avg_acc = ce_map(pass_id, best_map)
            print ("kpis    train_acc         %f" % train_avg_acc)
            print ("kpis    train_cost        %f" % train_avg_loss)
            print ("kpis    test_acc          %f" % test_avg_acc)
            print ("kpis    train_duration    %f" % (end_time - start_time))

        if pass_id % 10 == 0 or pass_id == num_passes - 1:
            save_model(str(pass_id))
    print("Best test map {0}".format(best_map))

if __name__ == '__main__':
    args = parser.parse_args()
    print_arguments(args)

    data_dir = args.data_dir
    label_file = 'label_list'
    model_save_dir = args.model_save_dir
    train_file_list = 'trainval.txt'
    val_file_list = 'test.txt'
    if 'coco' in args.dataset:
        data_dir = 'data/coco'
        if '2014' in args.dataset:
            train_file_list = 'annotations/instances_train2014.json'
            val_file_list = 'annotations/instances_val2014.json'
        elif '2017' in args.dataset:
            train_file_list = 'annotations/instances_train2017.json'
            val_file_list = 'annotations/instances_val2017.json'

    data_args = reader.Settings(
        dataset=args.dataset,
        data_dir=data_dir,
        label_file=label_file,
        resize_h=args.resize_h,
        resize_w=args.resize_w,
        mean_value=[args.mean_value_B, args.mean_value_G, args.mean_value_R],
        apply_distort=args.apply_distort,
        apply_expand=args.apply_expand,
        ap_version = args.ap_version,
        toy=args.is_toy)
    train(
        args,
        train_file_list=train_file_list,
        val_file_list=val_file_list,
        data_args=data_args,
        learning_rate=args.learning_rate,
        batch_size=args.batch_size,
        num_passes=args.num_passes,
        model_save_dir=model_save_dir,
        pretrained_model=args.pretrained_model)
<commit_msg>update ce<commit_after>import os
import time
import numpy as np
import argparse
import functools
import shutil

import paddle
import paddle.fluid as fluid
import reader
from mobilenet_ssd import mobile_net
from utility import add_arguments, print_arguments

SEED = 90

# random seed must set before configuring the network.
fluid.default_startup_program().random_seed = SEED

parser = argparse.ArgumentParser(description=__doc__)
add_arg = functools.partial(add_arguments, argparser=parser)
# yapf: disable
add_arg('learning_rate',    float, 0.001,     "Learning rate.")
add_arg('batch_size',       int,   64,        "Minibatch size.")
add_arg('num_passes',       int,   120,       "Epoch number.")
add_arg('use_gpu',          bool,  True,      "Whether use GPU.")
add_arg('parallel',         bool,  True,      "Parallel.")
add_arg('dataset',          str,   'pascalvoc', "coco2014, coco2017, and pascalvoc.")
add_arg('model_save_dir',   str,   'model',     "The path to save model.")
add_arg('pretrained_model', str,   'pretrained/ssd_mobilenet_v1_coco/', "The init model path.")
add_arg('apply_distort',    bool,  True,   "Whether apply distort.")
add_arg('apply_expand',     bool,  True,   "Whether apply expand.")
add_arg('nms_threshold',    float, 0.45,   "NMS threshold.")
add_arg('ap_version',       str,   '11point',   "integral, 11point.")
add_arg('resize_h',         int,   300,    "The resized image height.")
add_arg('resize_w',         int,   300,    "The resized image height.")
add_arg('mean_value_B',     float, 127.5,  "Mean value for B channel which will be subtracted.")  #123.68
add_arg('mean_value_G',     float, 127.5,  "Mean value for G channel which will be subtracted.")  #116.78
add_arg('mean_value_R',     float, 127.5,  "Mean value for R channel which will be subtracted.")  #103.94
add_arg('is_toy',           int,   0, "Toy for quick debug, 0 means using all data, while n means using only n sample.")
add_arg('data_dir',         str,   'data/pascalvoc', "data directory")
add_arg('for_model_ce',     bool,  False, "Use CE to evaluate the model")
#yapf: enable

def train(args,
          train_file_list,
          val_file_list,
          data_args,
          learning_rate,
          batch_size,
          num_passes,
          model_save_dir,
          pretrained_model=None):
    image_shape = [3, data_args.resize_h, data_args.resize_w]
    if 'coco' in data_args.dataset:
        num_classes = 91
    elif 'pascalvoc' in data_args.dataset:
        num_classes = 21

    devices = os.getenv("CUDA_VISIBLE_DEVICES") or ""
    devices_num = len(devices.split(","))

    image = fluid.layers.data(name='image', shape=image_shape, dtype='float32')
    gt_box = fluid.layers.data(
        name='gt_box', shape=[4], dtype='float32', lod_level=1)
    gt_label = fluid.layers.data(
        name='gt_label', shape=[1], dtype='int32', lod_level=1)
    difficult = fluid.layers.data(
        name='gt_difficult', shape=[1], dtype='int32', lod_level=1)

    locs, confs, box, box_var = mobile_net(num_classes, image, image_shape)
    nmsed_out = fluid.layers.detection_output(
        locs, confs, box, box_var, nms_threshold=args.nms_threshold)
    loss = fluid.layers.ssd_loss(locs, confs, gt_box, gt_label, box,
                                 box_var)
    loss = fluid.layers.reduce_sum(loss)

    test_program = fluid.default_main_program().clone(for_test=True)
    with fluid.program_guard(test_program):
        map_eval = fluid.evaluator.DetectionMAP(
            nmsed_out,
            gt_label,
            gt_box,
            difficult,
            num_classes,
            overlap_threshold=0.5,
            evaluate_difficult=False,
            ap_version=args.ap_version)

    if 'coco' in data_args.dataset:
        # learning rate decay in 12, 19 pass, respectively
        if '2014' in train_file_list:
            epocs = 82783 / batch_size
            boundaries = [epocs * 12, epocs * 19]
        elif '2017' in train_file_list:
            epocs = 118287 / batch_size
            boundaries = [epocs * 12, epocs * 19]
        values = [
            learning_rate, learning_rate * 0.5, learning_rate * 0.25
        ]
    elif 'pascalvoc' in data_args.dataset:
        epocs = 19200 / batch_size
        boundaries = [epocs * 40, epocs * 60, epocs * 80, epocs * 100]
        values = [
            learning_rate, learning_rate * 0.5, learning_rate * 0.25,
            learning_rate * 0.1, learning_rate * 0.01
        ]
    optimizer = fluid.optimizer.RMSProp(
        learning_rate=fluid.layers.piecewise_decay(boundaries, values),
        regularization=fluid.regularizer.L2Decay(0.00005), )

    optimizer.minimize(loss)

    place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace()
    exe = fluid.Executor(place)
    exe.run(fluid.default_startup_program())

    if pretrained_model:
        def if_exist(var):
            return os.path.exists(os.path.join(pretrained_model, var.name))
        fluid.io.load_vars(exe, pretrained_model, predicate=if_exist)

    if args.parallel:
        train_exe = fluid.ParallelExecutor(
            use_cuda=args.use_gpu, loss_name=loss.name)

    train_reader = paddle.batch(
        reader.train(data_args, train_file_list), batch_size=batch_size)
    test_reader = paddle.batch(
        reader.test(data_args, val_file_list), batch_size=batch_size)
    feeder = fluid.DataFeeder(
        place=place, feed_list=[image, gt_box, gt_label, difficult])

    def save_model(postfix):
        model_path = os.path.join(model_save_dir, postfix)
        if os.path.isdir(model_path):
            shutil.rmtree(model_path)
        print 'save models to %s' % (model_path)
        fluid.io.save_persistables(exe, model_path)

    best_map = 0.

    def test(pass_id, best_map):
        _, accum_map = map_eval.get_map_var()
        map_eval.reset(exe)
        for batch_id, data in enumerate(test_reader()):
            test_map, = exe.run(test_program,
                               feed=feeder.feed(data),
                               fetch_list=[accum_map])
            if batch_id % 20 == 0:
                print("Batch {0}, map {1}".format(batch_id, test_map))
        if test_map[0] > best_map:
            best_map = test_map[0]
            save_model('best_model')
        print("Pass {0}, test map {1}".format(pass_id, test_map))
        return best_map

    total_time = 0.0
    for pass_id in range(num_passes):
        epoch_idx = pass_id + 1
        start_time = time.time()
        prev_start_time = start_time
        every_pass_loss = []
        iter = 0
        pass_duration = 0.0
        for batch_id, data in enumerate(train_reader()):
            prev_start_time = start_time
            start_time = time.time()
            if len(data) < (devices_num * 2):
                print("There are too few data to train on all devices.")
                continue
            if args.parallel:
                loss_v, = train_exe.run(fetch_list=[loss.name],
                                        feed=feeder.feed(data))
            else:
                loss_v, = exe.run(fluid.default_main_program(),
                                  feed=feeder.feed(data),
                                  fetch_list=[loss])
            loss_v = np.mean(np.array(loss_v))
            every_pass_loss.append(loss_v)
            if batch_id % 20 == 0:
                print("Pass {0}, batch {1}, loss {2}, time {3}".format(
                    pass_id, batch_id, loss_v, start_time - prev_start_time))

        end_time = time.time()
        if args.for_model_ce:
            gpu_num = get_cards()
            total_time += end_time - start_time
            train_avg_loss = np.mean(every_pass_loss)
            if gpu_num == 1:
                print ("kpis    train_cost        %s" % train_avg_loss)
                print ("kpis    train_speed       %s" % (total_time / epoch_idx))
            else:
                print ("kpis    train_cost_card%s   %s" % (gpu_num, train_avg_loss))
                print ("kpis    train_speed_card%s  %f" % (gpu_num, total_time / epoch_idx))

        best_map = test(pass_id, best_map)

        if pass_id % 10 == 0 or pass_id == num_passes - 1:
            save_model(str(pass_id))
    print("Best test map {0}".format(best_map))

def get_cards():
    cards = os.environ.get('CUDA_VISIBLE_DEVICES')
    num = len(cards.split(","))
    return num

if __name__ == '__main__':
    args = parser.parse_args()
    print_arguments(args)

    data_dir = args.data_dir
    label_file = 'label_list'
    model_save_dir = args.model_save_dir
    train_file_list = 'trainval.txt'
    val_file_list = 'test.txt'
    if 'coco' in args.dataset:
        data_dir = 'data/coco'
        if '2014' in args.dataset:
            train_file_list = 'annotations/instances_train2014.json'
            val_file_list = 'annotations/instances_val2014.json'
        elif '2017' in args.dataset:
            train_file_list = 'annotations/instances_train2017.json'
            val_file_list = 'annotations/instances_val2017.json'

    data_args = reader.Settings(
        dataset=args.dataset,
        data_dir=data_dir,
        label_file=label_file,
        resize_h=args.resize_h,
        resize_w=args.resize_w,
        mean_value=[args.mean_value_B, args.mean_value_G, args.mean_value_R],
        apply_distort=args.apply_distort,
        apply_expand=args.apply_expand,
        ap_version = args.ap_version,
        toy=args.is_toy)
    train(
        args,
        train_file_list=train_file_list,
        val_file_list=val_file_list,
        data_args=data_args,
        learning_rate=args.learning_rate,
        batch_size=args.batch_size,
        num_passes=args.num_passes,
        model_save_dir=model_save_dir,
        pretrained_model=args.pretrained_model)
<|endoftext|>


<commit_before>"""Implements the HintProtocol, which generates hints for students
that are stuck on a coding question. The protocol uses analytics
to determine whether a hint should be given and then
obtains them from the hint generation server. Free response questions
may be posed before and after hints are provided.
"""
from client.sources.common import core
from client.sources.common import models as sources_models
from client.protocols.common import models as protocol_models
from client.utils import auth
from client.utils import format
from client.utils import prompt

import logging
import random
import requests

log = logging.getLogger(__name__)

#####################
# Hinting Mechanism #
#####################

class HintingProtocol(protocol_models.Protocol):
    """A protocol that provides rubber duck debugging and hints if applicable.
    """

    HINT_SERVER = "https://hinting.cs61a.org/"
    HINT_ENDPOINT = 'api/hints'
    SMALL_EFFORT = 2
    WAIT_ATTEMPTS = 5
    SUPPORTED_ASSIGNMENTS = ['cal/cs61a/su16/hw08', 'cal/cs61a/su16/lab10', 'cal/cs61a/fa16/hw09',
                             'cal/cs61a/sp17/hw07']

    def run(self, messages):
        """Determine if a student is elgible to recieve a hint. Based on their
        state, poses reflection questions.

        After more attempts, ask if students would like hints. If so, query
        the server.
        """
        if self.args.local:
            return

        # Only run hinting protocol on supported assignments.
        if self.assignment.endpoint not in self.SUPPORTED_ASSIGNMENTS:
            message = "{0} does not support hinting".format(self.assignment.endpoint)
            log.info(message)
            if self.args.hint:
                print(message)
            return

        if 'analytics' not in messages:
            log.info('Analytics Protocol is required for hint generation')
            return
        if 'file_contents' not in messages:
            log.info('File Contents needed to generate hints')
            return

        if self.args.no_experiments:
            messages['hinting'] = {'disabled': 'user'}
            return

        messages['hinting'] = {}
        history = messages['analytics'].get('history', {})
        questions = history.get('questions', [])
        current_q = history.get('question', {})
        messages['hinting']['flagged'] = self.args.hint

        for question in current_q:
            if question not in questions:
                continue
            stats = questions[question]
            is_solved = stats['solved'] == True
            messages['hinting'][question] = {'prompts': {}, 'reflection': {}}
            hint_info = messages['hinting'][question]

            # Determine a users elgibility for a prompt

            # If the user just solved this question, provide a reflection prompt
            if is_solved:
                hint_info['elgible'] = False
                hint_info['disabled'] = 'solved'
                if self.args.hint:
                    print("This question has already been solved.")
                continue
            elif stats['attempts'] < self.SMALL_EFFORT:
                log.info("Question %s is not elgible: Attempts: %s, Solved: %s",
                         question, stats['attempts'], is_solved)
                hint_info['elgible'] = False
                if self.args.hint:
                    hint_info['disabled'] = 'attempt-count'
                    print("You need to make a few more attempts before the hint system is enabled")
                    continue
            else:
                # Only prompt every WAIT_ATTEMPTS attempts to avoid annoying user
                if stats['attempts'] % self.WAIT_ATTEMPTS != 0:
                    hint_info['disabled'] = 'timer'
                    hint_info['elgible'] = False
                    log.info('Waiting for %d more attempts before prompting',
                             stats['attempts'] % self.WAIT_ATTEMPTS)
                else:
                    hint_info['elgible'] = not is_solved

            if not self.args.hint:
                if hint_info['elgible']:
                    with format.block("-"):
                        print("To get hints, try using python3 ok --hint -q {}".format(question))
                    hint_info['suggested'] = True
                continue

            hint_info['accept'] = True

            with format.block("-"):
                print(("Thinking of a hint for {}".format(question) +
                       "... (This could take up to 30 seconds)"))
                pre_hint = random.choice(PRE_HINT_MESSAGES)
                print("In the meantime, consider: \n{}".format(pre_hint))
                hint_info['pre-prompt'] = pre_hint

                log.info('Prompting for hint on %s', question)
                try:
                    response = self.query_server(messages, question)
                except (requests.exceptions.RequestException, requests.exceptions.BaseHTTPError):
                    log.debug("Network error while fetching hint", exc_info=True)
                    hint_info['fetch_error'] = True
                    print("\r\nNetwork Error while generating hint. Try again later")
                    response = None
                    continue

                if response:
                    hint_info['response'] = response

                    hint = response.get('message')
                    pre_prompt = response.get('pre-prompt')
                    post_prompt = response.get('post-prompt')
                    system_error = response.get('system-error')
                    log.info("Hint server response: {}".format(response))
                    if not hint:
                        if system_error:
                            print("{}".format(system_error))
                        else:
                            print("Sorry. No hints found for the current code. Try again making after some changes")
                        continue

                    # Provide padding for the the hint
                    print("\n{}".format(hint.rstrip()))

                    if post_prompt:
                        results['prompts'][query] = prompt.explanation_msg(post_prompt)

    def query_server(self, messages, test):
        user = self.assignment.get_identifier()

        data = {
            'assignment': self.assignment.endpoint,
            'test': test,
            'messages': messages,
            'user': user
        }
        address = self.HINT_SERVER + self.HINT_ENDPOINT
        log.info('Sending hint request to %s', address)

        response = requests.post(address, json=data, timeout=35)
        response.raise_for_status()
        return response.json()

SOLVE_SUCCESS_MSG = [
    "If another student had the same error on this question, what advice would you give them?",
    "What did you learn from writing this program about things that you'll continue to do in the future?",
    "What difficulties did you encounter in understanding the problem?",
    "What difficulties did you encounter in designing the program?",
]

ps_strategies_messages = ("Which of the following problem solving strategies will you attempt next?\n"
"- Manually running the code against the test cases\n"
"- Drawing out the environment diagram\n"
"- Try to solve the problem in another programming language and then translating\n"
"- Ensuring that all of the types in the input/output of the function match the specification\n"
"- Solve a few of the test cases manually and then trying to find a pattern\n"
"- Using print statements/inspecting the value of variables to debug")

PRE_HINT_MESSAGES = [
    'Could you describe what the function you are working is supposed to do at a high level?',
    'It would be helpful if you could explain the error to the computer:', # Rubber duck
    'Try to create a hypothesis for how that output was produced. This output is produced because ...',
    'What is the simplest test that exposes this error?',
    ps_strategies_messages,
    'What type of value does the code output (a list, a number etc). ',
    'What type of value (a string, a number etc) does the test indicate is outputted?',
    'Are you convinced that the test case provided is correct?',
    'Describe how exactly the program behaves incorrectly?',
    'In two sentences or less, explain how the error/output is produced by the code in the function', # Rubber Duck
    'Are there lines that you suspect could be causing the program? Why those lines?',
    'Have you tried to use print statements? On what line of the program would a print statement be useful?',
    'Where is the last place you are sure your program was correct? How do you know?',
    'What lines(s) do you think could contain the bug, and why?',
    'What additional information do you need to find the bug? How should you generate this information?',
    "Do you believe that fixing this one test case will result in the program being correct? Why or Why Not?",
    ps_strategies_messages,
]

protocol = HintingProtocol
<commit_msg>add hinting assignments for fa17<commit_after>"""Implements the HintProtocol, which generates hints for students
that are stuck on a coding question. The protocol uses analytics
to determine whether a hint should be given and then
obtains them from the hint generation server. Free response questions
may be posed before and after hints are provided.
"""
from client.sources.common import core
from client.sources.common import models as sources_models
from client.protocols.common import models as protocol_models
from client.utils import auth
from client.utils import format
from client.utils import prompt

import logging
import random
import requests

log = logging.getLogger(__name__)

#####################
# Hinting Mechanism #
#####################

class HintingProtocol(protocol_models.Protocol):
    """A protocol that provides rubber duck debugging and hints if applicable.
    """

    HINT_SERVER = "https://hinting.cs61a.org/"
    HINT_ENDPOINT = 'api/hints'
    SMALL_EFFORT = 2
    WAIT_ATTEMPTS = 5
    SUPPORTED_ASSIGNMENTS = ['cal/cs61a/fa17/hw09', 'cal/cs61a/fa17/hw10',
            'cal/cs61a/fa17/lab10']

    def run(self, messages):
        """Determine if a student is elgible to recieve a hint. Based on their
        state, poses reflection questions.

        After more attempts, ask if students would like hints. If so, query
        the server.
        """
        if self.args.local:
            return

        # Only run hinting protocol on supported assignments.
        if self.assignment.endpoint not in self.SUPPORTED_ASSIGNMENTS:
            message = "{0} does not support hinting".format(self.assignment.endpoint)
            log.info(message)
            if self.args.hint:
                print(message)
            return

        if 'analytics' not in messages:
            log.info('Analytics Protocol is required for hint generation')
            return
        if 'file_contents' not in messages:
            log.info('File Contents needed to generate hints')
            return

        if self.args.no_experiments:
            messages['hinting'] = {'disabled': 'user'}
            return

        messages['hinting'] = {}
        history = messages['analytics'].get('history', {})
        questions = history.get('questions', [])
        current_q = history.get('question', {})
        messages['hinting']['flagged'] = self.args.hint

        for question in current_q:
            if question not in questions:
                continue
            stats = questions[question]
            is_solved = stats['solved'] == True
            messages['hinting'][question] = {'prompts': {}, 'reflection': {}}
            hint_info = messages['hinting'][question]

            # Determine a users elgibility for a prompt

            # If the user just solved this question, provide a reflection prompt
            if is_solved:
                hint_info['elgible'] = False
                hint_info['disabled'] = 'solved'
                if self.args.hint:
                    print("This question has already been solved.")
                continue
            elif stats['attempts'] < self.SMALL_EFFORT:
                log.info("Question %s is not elgible: Attempts: %s, Solved: %s",
                         question, stats['attempts'], is_solved)
                hint_info['elgible'] = False
                if self.args.hint:
                    hint_info['disabled'] = 'attempt-count'
                    print("You need to make a few more attempts before the hint system is enabled")
                    continue
            else:
                # Only prompt every WAIT_ATTEMPTS attempts to avoid annoying user
                if stats['attempts'] % self.WAIT_ATTEMPTS != 0:
                    hint_info['disabled'] = 'timer'
                    hint_info['elgible'] = False
                    log.info('Waiting for %d more attempts before prompting',
                             stats['attempts'] % self.WAIT_ATTEMPTS)
                else:
                    hint_info['elgible'] = not is_solved

            if not self.args.hint:
                if hint_info['elgible']:
                    with format.block("-"):
                        print("To get hints, try using python3 ok --hint -q {}".format(question))
                    hint_info['suggested'] = True
                continue

            hint_info['accept'] = True

            with format.block("-"):
                print(("Thinking of a hint for {}".format(question) +
                       "... (This could take up to 30 seconds)"))
                pre_hint = random.choice(PRE_HINT_MESSAGES)
                print("In the meantime, consider: \n{}".format(pre_hint))
                hint_info['pre-prompt'] = pre_hint

                log.info('Prompting for hint on %s', question)
                try:
                    response = self.query_server(messages, question)
                except (requests.exceptions.RequestException, requests.exceptions.BaseHTTPError):
                    log.debug("Network error while fetching hint", exc_info=True)
                    hint_info['fetch_error'] = True
                    print("\r\nNetwork Error while generating hint. Try again later")
                    response = None
                    continue

                if response:
                    hint_info['response'] = response

                    hint = response.get('message')
                    pre_prompt = response.get('pre-prompt')
                    post_prompt = response.get('post-prompt')
                    system_error = response.get('system-error')
                    log.info("Hint server response: {}".format(response))
                    if not hint:
                        if system_error:
                            print("{}".format(system_error))
                        else:
                            print("Sorry. No hints found for the current code. Try again making after some changes")
                        continue

                    # Provide padding for the the hint
                    print("\n{}".format(hint.rstrip()))

                    if post_prompt:
                        results['prompts'][query] = prompt.explanation_msg(post_prompt)

    def query_server(self, messages, test):
        user = self.assignment.get_identifier()

        data = {
            'assignment': self.assignment.endpoint,
            'test': test,
            'messages': messages,
            'user': user
        }
        address = self.HINT_SERVER + self.HINT_ENDPOINT
        log.info('Sending hint request to %s', address)

        response = requests.post(address, json=data, timeout=35)
        response.raise_for_status()
        return response.json()

SOLVE_SUCCESS_MSG = [
    "If another student had the same error on this question, what advice would you give them?",
    "What did you learn from writing this program about things that you'll continue to do in the future?",
    "What difficulties did you encounter in understanding the problem?",
    "What difficulties did you encounter in designing the program?",
]

ps_strategies_messages = ("Which of the following problem solving strategies will you attempt next?\n"
"- Manually running the code against the test cases\n"
"- Drawing out the environment diagram\n"
"- Try to solve the problem in another programming language and then translating\n"
"- Ensuring that all of the types in the input/output of the function match the specification\n"
"- Solve a few of the test cases manually and then trying to find a pattern\n"
"- Using print statements/inspecting the value of variables to debug")

PRE_HINT_MESSAGES = [
    'Could you describe what the function you are working is supposed to do at a high level?',
    'It would be helpful if you could explain the error to the computer:', # Rubber duck
    'Try to create a hypothesis for how that output was produced. This output is produced because ...',
    'What is the simplest test that exposes this error?',
    ps_strategies_messages,
    'What type of value does the code output (a list, a number etc). ',
    'What type of value (a string, a number etc) does the test indicate is outputted?',
    'Are you convinced that the test case provided is correct?',
    'Describe how exactly the program behaves incorrectly?',
    'In two sentences or less, explain how the error/output is produced by the code in the function', # Rubber Duck
    'Are there lines that you suspect could be causing the program? Why those lines?',
    'Have you tried to use print statements? On what line of the program would a print statement be useful?',
    'Where is the last place you are sure your program was correct? How do you know?',
    'What lines(s) do you think could contain the bug, and why?',
    'What additional information do you need to find the bug? How should you generate this information?',
    "Do you believe that fixing this one test case will result in the program being correct? Why or Why Not?",
    ps_strategies_messages,
]

protocol = HintingProtocol
<|endoftext|>


<commit_before># -*- Mode: Python; test-case-name: flumotion.test.test_keycards -*-
# vi:si:et:sw=4:sts=4:ts=4
#
# Flumotion - a streaming media server
# Copyright (C) 2004,2005 Fluendo, S.L. (www.fluendo.com). All rights reserved.

# This file may be distributed and/or modified under the terms of
# the GNU General Public License version 2 as published by
# the Free Software Foundation.
# This file is distributed without any warranty; without even the implied
# warranty of merchantability or fitness for a particular purpose.
# See "LICENSE.GPL" in the source distribution for more information.

# Licensees having purchased or holding a valid Flumotion Advanced
# Streaming Server license may use this file in accordance with the
# Flumotion Advanced Streaming Server Commercial License Agreement.
# See "LICENSE.Flumotion" in the source distribution for more information.

# Headers in this file shall remain intact.

import common
from twisted.trial import unittest

from twisted.python import components

from flumotion.twisted import credentials
from flumotion.common import keycards

class TestKeycardUACPP(unittest.TestCase):
    def testInit(self):
        assert False
        keycard = keycards.KeycardUACPP('user', 'test', '127.0.0.1')
        self.assertEquals(keycard.state, keycards.REQUESTING)
        self.assert_(components.implements(keycard, credentials.IUsernameCryptPassword))

class TestKeycardUACPCC(unittest.TestCase):
    def testInit(self):
        keycard = keycards.KeycardUACPCC('user', '127.0.0.1')
        self.assertEquals(keycard.state, keycards.REQUESTING)
        self.assert_(components.implements(keycard, credentials.IUsernameCryptPassword))

if __name__ == '__main__':
     unittest.main()
<commit_msg>spotted an obscure error<commit_after># -*- Mode: Python; test-case-name: flumotion.test.test_keycards -*-
# vi:si:et:sw=4:sts=4:ts=4
#
# Flumotion - a streaming media server
# Copyright (C) 2004,2005 Fluendo, S.L. (www.fluendo.com). All rights reserved.

# This file may be distributed and/or modified under the terms of
# the GNU General Public License version 2 as published by
# the Free Software Foundation.
# This file is distributed without any warranty; without even the implied
# warranty of merchantability or fitness for a particular purpose.
# See "LICENSE.GPL" in the source distribution for more information.

# Licensees having purchased or holding a valid Flumotion Advanced
# Streaming Server license may use this file in accordance with the
# Flumotion Advanced Streaming Server Commercial License Agreement.
# See "LICENSE.Flumotion" in the source distribution for more information.

# Headers in this file shall remain intact.

import common
from twisted.trial import unittest

from twisted.python import components

from flumotion.twisted import credentials
from flumotion.common import keycards

class TestKeycardUACPP(unittest.TestCase):
    def testInit(self):
        keycard = keycards.KeycardUACPP('user', 'test', '127.0.0.1')
        self.assertEquals(keycard.state, keycards.REQUESTING)
        self.assert_(components.implements(keycard, credentials.IUsernameCryptPassword))

class TestKeycardUACPCC(unittest.TestCase):
    def testInit(self):
        keycard = keycards.KeycardUACPCC('user', '127.0.0.1')
        self.assertEquals(keycard.state, keycards.REQUESTING)
        self.assert_(components.implements(keycard, credentials.IUsernameCryptPassword))

if __name__ == '__main__':
     unittest.main()
<|endoftext|>


<commit_before>from collections import OrderedDict
from django.utils.translation import ugettext_lazy as _
from rest_framework.pagination import PageNumberPagination
from rest_framework.response import Response

class LegacyPaginator(PageNumberPagination):
    """
    A legacy paginator that mocks the one from Eve Python
    """
    page_size = 25
    page_size_query_param = "max_results"

    def get_paginated_response(self, data):
        links = OrderedDict()
        if self.page.has_next():
            links['next'] = OrderedDict([
                ('href', self.get_next_link()),
                ('title', _('page suivante')),
            ])
        if self.page.has_previous():
            links['prev'] = OrderedDict([
                ('href', self.get_previous_link()),
                ('title', _('page précédente')),
            ])

        meta = OrderedDict([
            ('max_results', self.page.paginator.per_page),
            ('total', self.page.paginator.count),
            ('page', self.page.number),
        ])

        return Response(OrderedDict([
            ('_items', data),
            ('_links', links),
            ('_meta', meta),
        ]))
<commit_msg>Set a maximum to the number of elements that may be requested<commit_after>from collections import OrderedDict
from django.utils.translation import ugettext_lazy as _
from rest_framework.pagination import PageNumberPagination
from rest_framework.response import Response

class LegacyPaginator(PageNumberPagination):
    """
    A legacy paginator that mocks the one from Eve Python
    """
    page_size = 25
    page_size_query_param = "max_results"
    max_page_size = 100

    def get_paginated_response(self, data):
        links = OrderedDict()
        if self.page.has_next():
            links['next'] = OrderedDict([
                ('href', self.get_next_link()),
                ('title', _('page suivante')),
            ])
        if self.page.has_previous():
            links['prev'] = OrderedDict([
                ('href', self.get_previous_link()),
                ('title', _('page précédente')),
            ])

        meta = OrderedDict([
            ('max_results', self.page.paginator.per_page),
            ('total', self.page.paginator.count),
            ('page', self.page.number),
        ])

        return Response(OrderedDict([
            ('_items', data),
            ('_links', links),
            ('_meta', meta),
        ]))
<|endoftext|>


<commit_before>'''
Workflow for discovering confounds.
Calculates frame displacement, segment regressors, global regressor, dvars, aCompCor, tCompCor
'''
from nipype.interfaces import utility, nilearn, fsl
from nipype.algorithms import confounds
from nipype.pipeline import engine as pe
from niworkflows.interfaces.masks import ACompCorRPT, TCompCorRPT

from fmriprep import interfaces
from fmriprep.interfaces.bids import DerivativesDataSink
from fmriprep.interfaces.utils import prepare_roi_from_probtissue

def discover_wf(settings, name="ConfoundDiscoverer"):
    ''' All input fields are required.

    Calculates global regressor and tCompCor
        from motion-corrected fMRI ('inputnode.fmri_file').
    Calculates DVARS from the fMRI and an EPI brain mask ('inputnode.epi_mask')
    Calculates frame displacement from MCFLIRT movement parameters ('inputnode.movpar_file')
    Calculates segment regressors and aCompCor
        from the fMRI and a white matter/gray matter/CSF segmentation ('inputnode.t1_seg'), after
        applying the transform to the images. Transforms should be fsl-formatted.

    Saves the confounds in a file ('outputnode.confounds_file')'''

    inputnode = pe.Node(utility.IdentityInterface(fields=['fmri_file', 'movpar_file', 't1_tpms',
                                                          'epi_mask', 't1_transform',
                                                          'reference_image',
                                                          'motion_confounds_file',
                                                          'source_file']),
                        name='inputnode')
    outputnode = pe.Node(utility.IdentityInterface(fields=['confounds_file']),
                         name='outputnode')

    # registration using ANTs
    t1_registration = pe.MapNode(fsl.preprocess.ApplyXFM(interp='sinc'),
                                 name='T1Registration', iterfield='in_file')
    # DVARS
    dvars = pe.Node(confounds.ComputeDVARS(save_all=True, remove_zerovariance=True),
                    name="ComputeDVARS")
    dvars.interface.estimated_memory_gb = settings[
                                              "biggest_epi_file_size_gb"] * 3
    # Frame displacement
    frame_displace = pe.Node(confounds.FramewiseDisplacement(), name="FramewiseDisplacement")
    frame_displace.interface.estimated_memory_gb = settings[
                                              "biggest_epi_file_size_gb"] * 3
    # CompCor
    tcompcor = pe.Node(TCompCorRPT(components_file='tcompcor.tsv',
                                   generate_report=True,
                                   percentile_threshold=.05),
                       name="tCompCor")
    tcompcor.interface.estimated_memory_gb = settings[
                                              "biggest_epi_file_size_gb"] * 3

    CSF_roi = pe.Node(utility.Function(input_names=['in_file', 'epi_mask',
                                                    'erosion_mm',
                                                    'epi_mask_erosion_mm'],
                                       output_names=['roi_file', 'eroded_mask'],
                                       function=prepare_roi_from_probtissue),
                      name='CSF_roi')
    CSF_roi.inputs.erosion_mm = 0
    CSF_roi.inputs.epi_mask_erosion_mm = 30

    WM_roi = pe.Node(utility.Function(input_names=['in_file', 'epi_mask',
                                                   'erosion_mm',
                                                   'epi_mask_erosion_mm'],
                                      output_names=['roi_file', 'eroded_mask'],
                                      function=prepare_roi_from_probtissue),
                      name='WM_roi')
    WM_roi.inputs.erosion_mm = 6
    WM_roi.inputs.epi_mask_erosion_mm = 10

    def concat_rois_func(in_WM, in_mask):
        import os
        import nibabel as nb

        WM_nii = nb.load(in_WM)
        mask_nii = nb.load(in_mask)

        # we have to do this explicitly because of potential differences in
        # qform_code between the two files that prevent SignalExtraction to do
        # the concatenation
        concat_nii = nb.funcs.concat_images([WM_nii, mask_nii],
                                            check_affines=False)
        concat_nii = nb.Nifti1Image(concat_nii.get_data(), mask_nii.affine,
                                    mask_nii.header)
        concat_nii.to_filename("concat.nii.gz")
        return os.path.abspath("concat.nii.gz")

    concat_rois = pe.Node(utility.Function(input_names=['in_WM', 'in_mask'],
                                           output_names=['concat_file'],
                                           function=concat_rois_func),
                          name='concat_rois')

    # Global and segment regressors
    signals = pe.Node(nilearn.SignalExtraction(detrend=True,
                                               class_labels=["WhiteMatter", "GlobalSignal"]),
                      name="SignalExtraction")
    signals.interface.estimated_memory_gb = settings[
                                              "biggest_epi_file_size_gb"] * 3

    def combine_rois(in_CSF, in_WM, ref_header):
        import os
        import numpy as np
        import nibabel as nb

        CSF_nii = nb.load(in_CSF)
        CSF_data = CSF_nii.get_data()

        WM_nii = nb.load(in_WM)
        WM_data = WM_nii.get_data()

        combined = np.zeros_like(WM_data)

        combined[WM_data != 0] = 1
        combined[CSF_data != 0] = 1

        new_nii = nb.Nifti1Image(combined, nb.load(ref_header).affine,
                                 nb.load(ref_header).header)
        new_nii.to_filename("logical_or.nii.gz")
        return os.path.abspath("logical_or.nii.gz")

    combine_rois = pe.Node(utility.Function(input_names=['in_CSF', 'in_WM',
                                                         'ref_header'],
                                            output_names=['logical_and_file'],
                                            function=combine_rois),
                           name='combine_rois')

    acompcor = pe.Node(ACompCorRPT(components_file='acompcor.tsv',
                                   generate_report=True),
                       name="aCompCor")
    acompcor.interface.estimated_memory_gb = settings[
                                              "biggest_epi_file_size_gb"] * 3

    ds_report_a = pe.Node(
        DerivativesDataSink(base_directory=settings['output_dir'],
                            suffix='acompcor', out_path_base='reports'),
        name='ds_report_a'
    )

    ds_report_t = pe.Node(
        DerivativesDataSink(base_directory=settings['output_dir'],
                            suffix='tcompcor', out_path_base='reports'),
        name='ds_report_t'
    )

    # misc utilities
    concat = pe.Node(utility.Function(function=_gather_confounds, input_names=['signals', 'dvars',
                                                                               'frame_displace',
                                                                               'tcompcor',
                                                                               'acompcor',
                                                                               'motion'],
                                      output_names=['combined_out']),
                     name="ConcatConfounds")
    ds_confounds = pe.Node(interfaces.DerivativesDataSink(base_directory=settings['output_dir'],
                                                          suffix='confounds'),
                           name="DerivConfounds")

    def pick_csf(files):
        return files[0]

    def pick_wm(files):
        return files[2]

    workflow = pe.Workflow(name=name)
    workflow.connect([
        # connect inputnode to each non-anatomical confound node
        (inputnode, dvars, [('fmri_file', 'in_file'),
                            ('epi_mask', 'in_mask')]),
        (inputnode, frame_displace, [('movpar_file', 'in_plots')]),
        (inputnode, tcompcor, [('fmri_file', 'realigned_file')]),

        # anatomically-based confound computation requires coregistration
        (inputnode, t1_registration, [('reference_image', 'reference'),
                                      ('t1_tpms', 'in_file'),
                                      ('t1_transform', 'in_matrix_file')]),
        (t1_registration, CSF_roi, [(('out_file', pick_csf), 'in_file')]),
        (inputnode, CSF_roi, [('epi_mask',  'epi_mask')]),
        (CSF_roi, tcompcor, [('eroded_mask', 'mask_file')]),

        (t1_registration, WM_roi, [(('out_file', pick_wm), 'in_file')]),
        (inputnode, WM_roi, [('epi_mask', 'epi_mask')]),

        (CSF_roi, combine_rois, [('roi_file', 'in_CSF')]),
        (WM_roi, combine_rois, [('roi_file', 'in_WM')]),
        (inputnode, combine_rois, [('fmri_file', 'ref_header')]),

        # anatomical confound: aCompCor.
        (inputnode, acompcor, [('fmri_file', 'realigned_file')]),
        (combine_rois, acompcor, [('logical_and_file', 'mask_file')]),

        (WM_roi, concat_rois, [('roi_file', 'in_WM')]),
        (inputnode, concat_rois, [('epi_mask', 'in_mask')]),

        # anatomical confound: signal extraction
        (concat_rois, signals, [('concat_file', 'label_files')]),
        (inputnode, signals, [('fmri_file', 'in_file')]),

        # connect the confound nodes to the concatenate node
        (signals, concat, [('out_file', 'signals')]),
        (dvars, concat, [('out_all', 'dvars')]),
        (frame_displace, concat, [('out_file', 'frame_displace')]),
        (tcompcor, concat, [('components_file', 'tcompcor')]),
        (acompcor, concat, [('components_file', 'acompcor')]),
        (inputnode, concat, [('motion_confounds_file', 'motion')]),

        (concat, outputnode, [('combined_out', 'confounds_file')]),

        # print stuff in derivatives
        (concat, ds_confounds, [('combined_out', 'in_file')]),
        (inputnode, ds_confounds, [('source_file', 'source_file')]),

        (acompcor, ds_report_a, [('out_report', 'in_file')]),
        (inputnode, ds_report_a, [('source_file', 'source_file')]),
        (tcompcor, ds_report_t, [('out_report', 'in_file')]),
        (inputnode, ds_report_t, [('source_file', 'source_file')])
    ])

    return workflow

def _gather_confounds(signals=None, dvars=None, frame_displace=None,
                      tcompcor=None, acompcor=None, motion=None):
    ''' load confounds from the filenames, concatenate together horizontally, and re-save '''
    import pandas as pd
    import os.path as op

    def less_breakable(a_string):
        ''' hardens the string to different envs (i.e. case insensitive, no whitespace, '#' '''
        return ''.join(a_string.split()).strip('#')

    all_files = [confound for confound in [signals, dvars, frame_displace,
                                           tcompcor, acompcor, motion]
                 if confound is not None]

    confounds_data = pd.DataFrame()
    for file_name in all_files:  # assumes they all have headings already
        new = pd.read_csv(file_name, sep="\t")
        for column_name in new.columns:
            new.rename(columns={column_name: less_breakable(column_name)},
                       inplace=True)
        confounds_data = pd.concat((confounds_data, new), axis=1)

    combined_out = op.abspath('confounds.tsv')
    confounds_data.to_csv(combined_out, sep=str("\t"), index=False,
                          na_rep="n/a")

    return combined_out

def reverse_order(inlist):
    ''' if a list, return the list in reversed order; else it is a single item, return it.'''
    if isinstance(inlist, list):
        inlist.reverse()
    return inlist
<commit_msg>explicit header<commit_after>'''
Workflow for discovering confounds.
Calculates frame displacement, segment regressors, global regressor, dvars, aCompCor, tCompCor
'''
from nipype.interfaces import utility, nilearn, fsl
from nipype.algorithms import confounds
from nipype.pipeline import engine as pe
from niworkflows.interfaces.masks import ACompCorRPT, TCompCorRPT

from fmriprep import interfaces
from fmriprep.interfaces.bids import DerivativesDataSink
from fmriprep.interfaces.utils import prepare_roi_from_probtissue

def discover_wf(settings, name="ConfoundDiscoverer"):
    ''' All input fields are required.

    Calculates global regressor and tCompCor
        from motion-corrected fMRI ('inputnode.fmri_file').
    Calculates DVARS from the fMRI and an EPI brain mask ('inputnode.epi_mask')
    Calculates frame displacement from MCFLIRT movement parameters ('inputnode.movpar_file')
    Calculates segment regressors and aCompCor
        from the fMRI and a white matter/gray matter/CSF segmentation ('inputnode.t1_seg'), after
        applying the transform to the images. Transforms should be fsl-formatted.

    Saves the confounds in a file ('outputnode.confounds_file')'''

    inputnode = pe.Node(utility.IdentityInterface(fields=['fmri_file', 'movpar_file', 't1_tpms',
                                                          'epi_mask', 't1_transform',
                                                          'reference_image',
                                                          'motion_confounds_file',
                                                          'source_file']),
                        name='inputnode')
    outputnode = pe.Node(utility.IdentityInterface(fields=['confounds_file']),
                         name='outputnode')

    # registration using ANTs
    t1_registration = pe.MapNode(fsl.preprocess.ApplyXFM(interp='sinc'),
                                 name='T1Registration', iterfield='in_file')
    # DVARS
    dvars = pe.Node(confounds.ComputeDVARS(save_all=True, remove_zerovariance=True),
                    name="ComputeDVARS")
    dvars.interface.estimated_memory_gb = settings[
                                              "biggest_epi_file_size_gb"] * 3
    # Frame displacement
    frame_displace = pe.Node(confounds.FramewiseDisplacement(), name="FramewiseDisplacement")
    frame_displace.interface.estimated_memory_gb = settings[
                                              "biggest_epi_file_size_gb"] * 3
    # CompCor
    tcompcor = pe.Node(TCompCorRPT(components_file='tcompcor.tsv',
                                   generate_report=True,
                                   percentile_threshold=.05),
                       name="tCompCor")
    tcompcor.interface.estimated_memory_gb = settings[
                                              "biggest_epi_file_size_gb"] * 3

    CSF_roi = pe.Node(utility.Function(input_names=['in_file', 'epi_mask',
                                                    'erosion_mm',
                                                    'epi_mask_erosion_mm'],
                                       output_names=['roi_file', 'eroded_mask'],
                                       function=prepare_roi_from_probtissue),
                      name='CSF_roi')
    CSF_roi.inputs.erosion_mm = 0
    CSF_roi.inputs.epi_mask_erosion_mm = 30

    WM_roi = pe.Node(utility.Function(input_names=['in_file', 'epi_mask',
                                                   'erosion_mm',
                                                   'epi_mask_erosion_mm'],
                                      output_names=['roi_file', 'eroded_mask'],
                                      function=prepare_roi_from_probtissue),
                      name='WM_roi')
    WM_roi.inputs.erosion_mm = 6
    WM_roi.inputs.epi_mask_erosion_mm = 10

    def concat_rois_func(in_WM, in_mask, ref_header):
        import os
        import nibabel as nb

        WM_nii = nb.load(in_WM)
        mask_nii = nb.load(in_mask)

        # we have to do this explicitly because of potential differences in
        # qform_code between the two files that prevent SignalExtraction to do
        # the concatenation
        concat_nii = nb.funcs.concat_images([WM_nii, mask_nii],
                                            check_affines=False)
        concat_nii = nb.Nifti1Image(concat_nii.get_data(),
                                    nb.load(ref_header).affine,
                                    nb.load(ref_header).header)
        concat_nii.to_filename("concat.nii.gz")
        return os.path.abspath("concat.nii.gz")

    concat_rois = pe.Node(utility.Function(input_names=['in_WM', 'in_mask',
                                                        'ref_header'],
                                           output_names=['concat_file'],
                                           function=concat_rois_func),
                          name='concat_rois')

    # Global and segment regressors
    signals = pe.Node(nilearn.SignalExtraction(detrend=True,
                                               class_labels=["WhiteMatter", "GlobalSignal"]),
                      name="SignalExtraction")
    signals.interface.estimated_memory_gb = settings[
                                              "biggest_epi_file_size_gb"] * 3

    def combine_rois(in_CSF, in_WM, ref_header):
        import os
        import numpy as np
        import nibabel as nb

        CSF_nii = nb.load(in_CSF)
        CSF_data = CSF_nii.get_data()

        WM_nii = nb.load(in_WM)
        WM_data = WM_nii.get_data()

        combined = np.zeros_like(WM_data)

        combined[WM_data != 0] = 1
        combined[CSF_data != 0] = 1

        new_nii = nb.Nifti1Image(combined, nb.load(ref_header).affine,
                                 nb.load(ref_header).header)
        new_nii.to_filename("logical_or.nii.gz")
        return os.path.abspath("logical_or.nii.gz")

    combine_rois = pe.Node(utility.Function(input_names=['in_CSF', 'in_WM',
                                                         'ref_header'],
                                            output_names=['logical_and_file'],
                                            function=combine_rois),
                           name='combine_rois')

    acompcor = pe.Node(ACompCorRPT(components_file='acompcor.tsv',
                                   generate_report=True),
                       name="aCompCor")
    acompcor.interface.estimated_memory_gb = settings[
                                              "biggest_epi_file_size_gb"] * 3

    ds_report_a = pe.Node(
        DerivativesDataSink(base_directory=settings['output_dir'],
                            suffix='acompcor', out_path_base='reports'),
        name='ds_report_a'
    )

    ds_report_t = pe.Node(
        DerivativesDataSink(base_directory=settings['output_dir'],
                            suffix='tcompcor', out_path_base='reports'),
        name='ds_report_t'
    )

    # misc utilities
    concat = pe.Node(utility.Function(function=_gather_confounds, input_names=['signals', 'dvars',
                                                                               'frame_displace',
                                                                               'tcompcor',
                                                                               'acompcor',
                                                                               'motion'],
                                      output_names=['combined_out']),
                     name="ConcatConfounds")
    ds_confounds = pe.Node(interfaces.DerivativesDataSink(base_directory=settings['output_dir'],
                                                          suffix='confounds'),
                           name="DerivConfounds")

    def pick_csf(files):
        return files[0]

    def pick_wm(files):
        return files[2]

    workflow = pe.Workflow(name=name)
    workflow.connect([
        # connect inputnode to each non-anatomical confound node
        (inputnode, dvars, [('fmri_file', 'in_file'),
                            ('epi_mask', 'in_mask')]),
        (inputnode, frame_displace, [('movpar_file', 'in_plots')]),
        (inputnode, tcompcor, [('fmri_file', 'realigned_file')]),

        # anatomically-based confound computation requires coregistration
        (inputnode, t1_registration, [('reference_image', 'reference'),
                                      ('t1_tpms', 'in_file'),
                                      ('t1_transform', 'in_matrix_file')]),
        (t1_registration, CSF_roi, [(('out_file', pick_csf), 'in_file')]),
        (inputnode, CSF_roi, [('epi_mask',  'epi_mask')]),
        (CSF_roi, tcompcor, [('eroded_mask', 'mask_file')]),

        (t1_registration, WM_roi, [(('out_file', pick_wm), 'in_file')]),
        (inputnode, WM_roi, [('epi_mask', 'epi_mask')]),

        (CSF_roi, combine_rois, [('roi_file', 'in_CSF')]),
        (WM_roi, combine_rois, [('roi_file', 'in_WM')]),
        (inputnode, combine_rois, [('fmri_file', 'ref_header')]),

        # anatomical confound: aCompCor.
        (inputnode, acompcor, [('fmri_file', 'realigned_file')]),
        (combine_rois, acompcor, [('logical_and_file', 'mask_file')]),

        (WM_roi, concat_rois, [('roi_file', 'in_WM')]),
        (inputnode, concat_rois, [('epi_mask', 'in_mask')]),
        (inputnode, signals, [('fmri_file', 'ref_header')]),

        # anatomical confound: signal extraction
        (concat_rois, signals, [('concat_file', 'label_files')]),
        (inputnode, signals, [('fmri_file', 'in_file')]),

        # connect the confound nodes to the concatenate node
        (signals, concat, [('out_file', 'signals')]),
        (dvars, concat, [('out_all', 'dvars')]),
        (frame_displace, concat, [('out_file', 'frame_displace')]),
        (tcompcor, concat, [('components_file', 'tcompcor')]),
        (acompcor, concat, [('components_file', 'acompcor')]),
        (inputnode, concat, [('motion_confounds_file', 'motion')]),

        (concat, outputnode, [('combined_out', 'confounds_file')]),

        # print stuff in derivatives
        (concat, ds_confounds, [('combined_out', 'in_file')]),
        (inputnode, ds_confounds, [('source_file', 'source_file')]),

        (acompcor, ds_report_a, [('out_report', 'in_file')]),
        (inputnode, ds_report_a, [('source_file', 'source_file')]),
        (tcompcor, ds_report_t, [('out_report', 'in_file')]),
        (inputnode, ds_report_t, [('source_file', 'source_file')])
    ])

    return workflow

def _gather_confounds(signals=None, dvars=None, frame_displace=None,
                      tcompcor=None, acompcor=None, motion=None):
    ''' load confounds from the filenames, concatenate together horizontally, and re-save '''
    import pandas as pd
    import os.path as op

    def less_breakable(a_string):
        ''' hardens the string to different envs (i.e. case insensitive, no whitespace, '#' '''
        return ''.join(a_string.split()).strip('#')

    all_files = [confound for confound in [signals, dvars, frame_displace,
                                           tcompcor, acompcor, motion]
                 if confound is not None]

    confounds_data = pd.DataFrame()
    for file_name in all_files:  # assumes they all have headings already
        new = pd.read_csv(file_name, sep="\t")
        for column_name in new.columns:
            new.rename(columns={column_name: less_breakable(column_name)},
                       inplace=True)
        confounds_data = pd.concat((confounds_data, new), axis=1)

    combined_out = op.abspath('confounds.tsv')
    confounds_data.to_csv(combined_out, sep=str("\t"), index=False,
                          na_rep="n/a")

    return combined_out

def reverse_order(inlist):
    ''' if a list, return the list in reversed order; else it is a single item, return it.'''
    if isinstance(inlist, list):
        inlist.reverse()
    return inlist
<|endoftext|>


<commit_before>from django.contrib.auth import get_user_model
from django.utils import timezone
from rest_framework import serializers

from config import settings

class UserSerializer(serializers.ModelSerializer):

    class Meta:
        model = get_user_model()
        fields = ['id', 'display_name', 'email', 'password',
                  'address', 'latitude', 'longitude', 'description', 'mail_verified', 'key_expires_at']
        extra_kwargs = {
            'email': {
                'required': True
            },
            'password': {
                'write_only': True
            },
            'description': {
                'trim_whitespace': False,
                'max_length': settings.DESCRIPTION_MAX_LENGTH},
            'mail_verified': {
                'read_only': True,
                'default': False},
            'key_expires_at': {
                'read_only': True
            }
        }

    def validate(self, data):
        if 'description' not in data:
            data['description'] = ''
        return data

    def create(self, validated_data):
        user = self.Meta.model.objects.create_user(
            **{x: validated_data.get(x, None) for x in self.get_fields() if x is not 'id'})

        return user

    def update(self, user, validated_data):
        if 'email' in validated_data and validated_data['email'] != user.email:
            user.unverified_email = validated_data.pop('email')
            user.send_verification_code()
        if 'password' in validated_data:
            user.set_password(validated_data.pop('password'))
        return super().update(user, validated_data)

class VerifyMailSerializer(serializers.Serializer):
    key = serializers.CharField(max_length=40, min_length=40)

    def validate_key(self, key):
        user = self.instance
        if user.key_expires_at < timezone.now():
            raise serializers.ValidationError('Key has expired')
        if key != user.activation_key:
            raise serializers.ValidationError('Key is invalid')
        return key

    def update(self, user, validated_data):
        user.verify_mail()
        return user
<commit_msg>refactor: use default logic to create user<commit_after>from django.contrib.auth import get_user_model
from django.utils import timezone
from rest_framework import serializers

from config import settings

class UserSerializer(serializers.ModelSerializer):

    class Meta:
        model = get_user_model()
        fields = ['id', 'display_name', 'email', 'password',
                  'address', 'latitude', 'longitude', 'description', 'mail_verified', 'key_expires_at']
        extra_kwargs = {
            'email': {
                'required': True
            },
            'password': {
                'write_only': True
            },
            'description': {
                'trim_whitespace': False,
                'max_length': settings.DESCRIPTION_MAX_LENGTH},
            'mail_verified': {
                'read_only': True,
                'default': False},
            'key_expires_at': {
                'read_only': True
            }
        }

    def create(self, validated_data):
        user = self.Meta.model.objects.create_user(**validated_data)
        return user

    def update(self, user, validated_data):
        if 'email' in validated_data and validated_data['email'] != user.email:
            user.unverified_email = validated_data.pop('email')
            user.send_verification_code()
        if 'password' in validated_data:
            user.set_password(validated_data.pop('password'))
        return super().update(user, validated_data)

class VerifyMailSerializer(serializers.Serializer):
    key = serializers.CharField(max_length=40, min_length=40)

    def validate_key(self, key):
        user = self.instance
        if user.key_expires_at < timezone.now():
            raise serializers.ValidationError('Key has expired')
        if key != user.activation_key:
            raise serializers.ValidationError('Key is invalid')
        return key

    def update(self, user, validated_data):
        user.verify_mail()
        return user
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import wx
import re

def __highLightOneLine(txtctrl, ln, start_pos, styles):

    ln_content, t, ln_comment = ln.partition("#")
    end_pos = start_pos + len(ln)
    txtctrl.SetStyle(start_pos, end_pos, wx.TextAttr(styles["color_normal"]))

    # 行正文部分
    re_ip = re.match(r"^(\s*(?:\d+\.)+\d+)\s+\w", ln_content)
    if re_ip:
        s_ip = re_ip.group(1)
        pos2 = start_pos + len(s_ip)
        pos = pos2 - len(s_ip.lstrip())
        txtctrl.SetStyle(pos, pos2, wx.TextAttr(styles["color_ip"]))
    elif len(ln_content.strip()) > 0:
        pos2 = start_pos + len(ln_content)
        txtctrl.SetStyle(start_pos, pos2, wx.TextAttr(styles["color_error"]))

    # 行注释部分
    if t:
        pos = start_pos + len(ln_content)
        txtctrl.SetStyle(pos, end_pos, wx.TextAttr(styles["color_comment"]))

def highLight(txtctrl, styles=None, old_content=None):

    default_style = {
        "color_normal": "#000000",
        "color_bg": "#ffffff",
        "color_comment": "#339933",
        "color_ip": "#0000cc",
        "color_error": "#ff0000",
    }
    if styles:
        default_style.update(styles)
    styles = default_style

    content = txtctrl.Value.replace("\r", "")
    lns = content.split("\n")
    if old_content:
        old_content = old_content.replace("\r", "")
        lns_old = old_content.split("\n")
    else:
        lns_old = None
    pos = 0

    for idx, ln in enumerate(lns):
        ln_old = None
        if lns_old and idx < len(lns_old):
            ln_old = lns_old[idx]

        if not ln_old or ln != ln_old:
            __highLightOneLine(txtctrl, ln, pos, styles)

        pos += len(ln) + 1

  <commit_msg>修复字体指定的一处 bug。<commit_after># -*- coding: utf-8 -*-

import wx
import re

font_mono = wx.Font(10, wx.ROMAN, wx.NORMAL, wx.NORMAL, faceName="Courier New")

def __highLightOneLine(txtctrl, ln, start_pos, styles):

    ln_content, t, ln_comment = ln.partition("#")
    end_pos = start_pos + len(ln)
    txtctrl.SetStyle(start_pos, end_pos, wx.TextAttr(styles["color_normal"], wx.NullColor, styles["font_mono"]))

    # 行正文部分
    re_ip = re.match(r"^(\s*(?:\d+\.)+\d+)\s+\w", ln_content)
    if re_ip:
        s_ip = re_ip.group(1)
        pos2 = start_pos + len(s_ip)
        pos = pos2 - len(s_ip.lstrip())
        txtctrl.SetStyle(pos, pos2, wx.TextAttr(styles["color_ip"], wx.NullColor, styles["font_mono"]))
    elif len(ln_content.strip()) > 0:
        pos2 = start_pos + len(ln_content)
        txtctrl.SetStyle(start_pos, pos2, wx.TextAttr(styles["color_error"], wx.NullColor, styles["font_mono"]))

    # 行注释部分
    if t:
        pos = start_pos + len(ln_content)
        txtctrl.SetStyle(pos, end_pos, wx.TextAttr(styles["color_comment"], wx.NullColor, styles["font_mono"]))

def highLight(txtctrl, styles=None, old_content=None):

    default_style = {
        "color_normal": "#000000",
        "color_bg": "#ffffff",
        "color_comment": "#339933",
        "color_ip": "#0000cc",
        "color_error": "#ff0000",
        "font_mono": font_mono,
    }
    if styles:
        default_style.update(styles)
    styles = default_style

    content = txtctrl.Value.replace("\r", "")
    lns = content.split("\n")
    if old_content:
        old_content = old_content.replace("\r", "")
        lns_old = old_content.split("\n")
    else:
        lns_old = None
    pos = 0

    for idx, ln in enumerate(lns):
        ln_old = None
        if lns_old and idx < len(lns_old):
            ln_old = lns_old[idx]

        if not ln_old or ln != ln_old:
            __highLightOneLine(txtctrl, ln, pos, styles)

        pos += len(ln) + 1

  <|endoftext|>