<commit_before>from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
from builtins import range
from future import standard_library
standard_library.install_aliases()
from logging import getLogger
logger = getLogger(__name__)
import copy
import os
import multiprocessing as mp

import numpy as np
import chainer
from chainer import functions as F
from chainer import serializers

import copy_param
import async
from chainerrl.misc.makedirs import makedirs

class NSQ(object):
    """N-step Q-Learning.

    See http://arxiv.org/abs/1602.01783
    """

    def __init__(self, process_idx, q_function, optimizer,
                 t_max, gamma, i_target, explorer, phi=lambda x: x,
                 clip_reward=True):

        self.process_idx = process_idx

        self.shared_q_function = q_function
        self.target_q_function = copy.deepcopy(q_function)
        self.q_function = copy.deepcopy(self.shared_q_function)

        async.assert_params_not_shared(self.shared_q_function, self.q_function)

        self.optimizer = optimizer

        self.t_max = t_max
        self.gamma = gamma
        self.explorer = explorer
        self.i_target = i_target
        self.clip_reward = clip_reward
        self.phi = phi
        self.t_global = mp.Value('l', 0)

        self.t = 0
        self.t_start = 0
        self.past_action_values = {}
        self.past_states = {}
        self.past_rewards = {}

    def sync_parameters(self):
        copy_param.copy_param(target_link=self.q_function,
                              source_link=self.shared_q_function)

    @property
    def shared_attributes(self):
        return ('shared_q_function', 'target_q_function', 'optimizer',
                't_global')

    def update(self, statevar):
        assert self.t_start < self.t

        # Update
        if statevar is None:
            R = 0
        else:
            R = float(self.target_q_function(statevar).max.data)

        loss = 0
        for i in reversed(range(self.t_start, self.t)):
            R *= self.gamma
            R += self.past_rewards[i]
            q = F.reshape(self.past_action_values[i], (1, 1))
            # Accumulate gradients of Q-function
            # loss += (R - q) ** 2
            # loss += F.mean_squared_error(q, chainer.Variable(np.asarray([R])))
            loss += F.sum(F.huber_loss(
                q, chainer.Variable(np.asarray([[R]], dtype=np.float32)),
                delta=1.0))

        # Do we need to normalize losses by (self.t - self.t_start)?
        # Otherwise, loss scales can be different in case of self.t_max
        # and in case of termination.

        # I'm not sure but if we need to normalize losses...
        # loss /= self.t - self.t_start

        # Compute gradients using thread-specific model
        self.q_function.zerograds()
        loss.backward()
        # Copy the gradients to the globally shared model
        self.shared_q_function.zerograds()
        copy_param.copy_grad(self.shared_q_function, self.q_function)
        # Update the globally shared model
        self.optimizer.update()

        self.sync_parameters()

        self.past_action_values = {}
        self.past_states = {}
        self.past_rewards = {}

        self.t_start = self.t

    def act_and_train(self, obs, reward):

        if self.clip_reward:
            reward = np.clip(reward, -1, 1)

        statevar = chainer.Variable(np.expand_dims(self.phi(obs), 0))

        self.past_rewards[self.t - 1] = reward

        if self.t - self.t_start == self.t_max:
            self.update(statevar)

        self.past_states[self.t] = statevar
        qout = self.q_function(statevar)
        if self.process_idx == 0:
            logger.debug('t_global: %s t_local: %s action_value: %s',
                         self.t_global.value, self.t, qout)
        action = self.explorer.select_action(
            self.t, lambda: qout.greedy_actions.data[0])
        q = qout.evaluate_actions(np.asarray([action]))
        self.past_action_values[self.t] = q
        self.t += 1
        with self.t_global.get_lock():
            self.t_global.value += 1
            t_global = self.t_global.value

        if t_global % self.i_target == 0:
            logger.debug('target synchronized t_global:%s t_local:%s',
                         t_global, self.t)
            copy_param.copy_param(self.target_q_function, self.q_function)

        return action

    def act(self, obs):
        statevar = chainer.Variable(np.expand_dims(self.phi(obs), 0))
        qout = self.q_function(statevar)
        return qout.greedy_actions.data[0]

    def stop_episode_and_train(self, state, reward, done=False):
        if self.clip_reward:
            reward = np.clip(reward, -1, 1)

        self.past_rewards[self.t - 1] = reward
        if done:
            self.update(None)
        else:
            statevar = chainer.Variable(np.expand_dims(self.phi(state), 0))
            self.update(statevar)

    def stop_episode(self):
        # TODO support recurrent Q-function
        pass

    def save(self, dirname):
        makedirs(dirname, exist_ok=True)
        serializers.save_npz(os.path.join(dirname, 'q_function.npz'),
                             self.q_function)
        serializers.save_npz(os.path.join(dirname, 'target_q_function.npz'),
                             self.target_q_function)
        serializers.save_npz(os.path.join(dirname, 'optimizer.npz'),
                             self.optimizer)

    def load(self, dirname):
        serializers.load_npz(os.path.join(
            dirname, 'q_function.npz'), self.q_function)

        target_filename = os.path.join(dirname, 'target_q_function.npz')
        if os.path.exists(target_filename):
            serializers.load_npz(target_filename, self.target_q_function)
        else:
            print('WARNING: {0} was not found'.format(target_filename))
            copy_param.copy_param(target_link=self.target_q_function,
                                  source_link=self.q_function)

        opt_filename = os.path.join(dirname, 'optimizer.npz')
        if os.path.exists(opt_filename):
            serializers.load_npz(opt_filename, self.optimizer)
        else:
            print('WARNING: {0} was not found, so loaded only a model'.format(
                opt_filename))

    def get_stats_keys(self):
        return ()

    def get_stats_values(self):
        return ()
<commit_msg>Log obs and reward<commit_after>from __future__ import unicode_literals
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
from builtins import range
from future import standard_library
standard_library.install_aliases()
from logging import getLogger
logger = getLogger(__name__)
import copy
import os
import multiprocessing as mp

import numpy as np
import chainer
from chainer import functions as F
from chainer import serializers

import copy_param
import async
from chainerrl.misc.makedirs import makedirs

class NSQ(object):
    """N-step Q-Learning.

    See http://arxiv.org/abs/1602.01783
    """

    def __init__(self, process_idx, q_function, optimizer,
                 t_max, gamma, i_target, explorer, phi=lambda x: x,
                 clip_reward=True):

        self.process_idx = process_idx

        self.shared_q_function = q_function
        self.target_q_function = copy.deepcopy(q_function)
        self.q_function = copy.deepcopy(self.shared_q_function)

        async.assert_params_not_shared(self.shared_q_function, self.q_function)

        self.optimizer = optimizer

        self.t_max = t_max
        self.gamma = gamma
        self.explorer = explorer
        self.i_target = i_target
        self.clip_reward = clip_reward
        self.phi = phi
        self.t_global = mp.Value('l', 0)

        self.t = 0
        self.t_start = 0
        self.past_action_values = {}
        self.past_states = {}
        self.past_rewards = {}

    def sync_parameters(self):
        copy_param.copy_param(target_link=self.q_function,
                              source_link=self.shared_q_function)

    @property
    def shared_attributes(self):
        return ('shared_q_function', 'target_q_function', 'optimizer',
                't_global')

    def update(self, statevar):
        assert self.t_start < self.t

        # Update
        if statevar is None:
            R = 0
        else:
            R = float(self.target_q_function(statevar).max.data)

        loss = 0
        for i in reversed(range(self.t_start, self.t)):
            R *= self.gamma
            R += self.past_rewards[i]
            q = F.reshape(self.past_action_values[i], (1, 1))
            # Accumulate gradients of Q-function
            # loss += (R - q) ** 2
            # loss += F.mean_squared_error(q, chainer.Variable(np.asarray([R])))
            loss += F.sum(F.huber_loss(
                q, chainer.Variable(np.asarray([[R]], dtype=np.float32)),
                delta=1.0))

        # Do we need to normalize losses by (self.t - self.t_start)?
        # Otherwise, loss scales can be different in case of self.t_max
        # and in case of termination.

        # I'm not sure but if we need to normalize losses...
        # loss /= self.t - self.t_start

        # Compute gradients using thread-specific model
        self.q_function.zerograds()
        loss.backward()
        # Copy the gradients to the globally shared model
        self.shared_q_function.zerograds()
        copy_param.copy_grad(self.shared_q_function, self.q_function)
        # Update the globally shared model
        self.optimizer.update()

        self.sync_parameters()

        self.past_action_values = {}
        self.past_states = {}
        self.past_rewards = {}

        self.t_start = self.t

    def act_and_train(self, obs, reward):

        if self.clip_reward:
            reward = np.clip(reward, -1, 1)

        statevar = chainer.Variable(np.expand_dims(self.phi(obs), 0))

        self.past_rewards[self.t - 1] = reward

        if self.t - self.t_start == self.t_max:
            self.update(statevar)

        self.past_states[self.t] = statevar
        qout = self.q_function(statevar)
        if self.process_idx == 0:
            logger.debug(
                't_global: %s t_local: %s obs: %s r: %s action_value: %s',
                self.t_global.value, self.t, statevar.data.sum(), reward, qout)
        action = self.explorer.select_action(
            self.t, lambda: qout.greedy_actions.data[0])
        q = qout.evaluate_actions(np.asarray([action]))
        self.past_action_values[self.t] = q
        self.t += 1
        with self.t_global.get_lock():
            self.t_global.value += 1
            t_global = self.t_global.value

        if t_global % self.i_target == 0:
            logger.debug('target synchronized t_global:%s t_local:%s',
                         t_global, self.t)
            copy_param.copy_param(self.target_q_function, self.q_function)

        return action

    def act(self, obs):
        statevar = chainer.Variable(np.expand_dims(self.phi(obs), 0))
        qout = self.q_function(statevar)
        return qout.greedy_actions.data[0]

    def stop_episode_and_train(self, state, reward, done=False):
        if self.clip_reward:
            reward = np.clip(reward, -1, 1)

        self.past_rewards[self.t - 1] = reward
        if done:
            self.update(None)
        else:
            statevar = chainer.Variable(np.expand_dims(self.phi(state), 0))
            self.update(statevar)

    def stop_episode(self):
        # TODO support recurrent Q-function
        pass

    def save(self, dirname):
        makedirs(dirname, exist_ok=True)
        serializers.save_npz(os.path.join(dirname, 'q_function.npz'),
                             self.q_function)
        serializers.save_npz(os.path.join(dirname, 'target_q_function.npz'),
                             self.target_q_function)
        serializers.save_npz(os.path.join(dirname, 'optimizer.npz'),
                             self.optimizer)

    def load(self, dirname):
        serializers.load_npz(os.path.join(
            dirname, 'q_function.npz'), self.q_function)

        target_filename = os.path.join(dirname, 'target_q_function.npz')
        if os.path.exists(target_filename):
            serializers.load_npz(target_filename, self.target_q_function)
        else:
            print('WARNING: {0} was not found'.format(target_filename))
            copy_param.copy_param(target_link=self.target_q_function,
                                  source_link=self.q_function)

        opt_filename = os.path.join(dirname, 'optimizer.npz')
        if os.path.exists(opt_filename):
            serializers.load_npz(opt_filename, self.optimizer)
        else:
            print('WARNING: {0} was not found, so loaded only a model'.format(
                opt_filename))

    def get_stats_keys(self):
        return ()

    def get_stats_values(self):
        return ()
<|endoftext|>


<commit_before>from changes.config import db

from changes.constants import Status

from changes.models import Build, FileCoverage, Job, Project, Source

def get_coverage_by_source_id(source_id):
    """
    Takes a source_id and returns a dictionary of coverage for that source_id.  The
    coverage is generated for the most recently finished builds for each project.

    The dictionary maps file names to a string of the form 'UNCCCNCU', where U means
    'uncovered', C means 'covered' and 'N' means 'no coverage info'.
    """
    source = Source.query.get(source_id)

    projects = Project.query.filter(
        Project.repository_id == source.repository_id
    )

    newest_build_ids = set()
    for project in projects:
        newest_build_ids.add(db.session.query(Build.id).filter(
                Build.project_id == project.id,
                Build.source_id == source_id,
                Build.status == Status.finished
        ).order_by(Build.date_created.desc()).first())

    return get_coverage_by_build_ids(newest_build_ids)

def get_coverage_by_build_id(build_id):
    return get_coverage_by_build_ids([build_id])

def get_coverage_by_build_ids(build_ids):
    """
    Returns the coverage associated with some builds.

    The dictionary maps file names to a string of the form 'UNCCCNCU', where U means
    'uncovered', C means 'covered' and 'N' means 'no coverage info'.
    """
    all_job_ids = db.session.query(Job.id).filter(
        Job.build_id.in_(build_ids)
    )

    return get_coverage_by_job_ids(all_job_ids)

def get_coverage_by_job_ids(job_ids):
    """
    Returns the coverage associated with some jobs.

    The dictionary maps file names to a string of the form 'UNCCCNCU', where U means
    'uncovered', C means 'covered' and 'N' means 'no coverage info'.
    """
    all_coverages = FileCoverage.query.filter(
        FileCoverage.job_id.in_(job_ids)
    )

    combined_file_coverage = dict()

    for coverage in all_coverages:
        combined_file_coverage[coverage.filename] = coverage.data

    return combined_file_coverage
<commit_msg>grab the first element from the list in get_coverage_by_source_id<commit_after>from changes.config import db

from changes.constants import Status

from changes.models import Build, FileCoverage, Job, Project, Source

def get_coverage_by_source_id(source_id):
    """
    Takes a source_id and returns a dictionary of coverage for that source_id.  The
    coverage is generated for the most recently finished builds for each project.

    The dictionary maps file names to a string of the form 'UNCCCNCU', where U means
    'uncovered', C means 'covered' and 'N' means 'no coverage info'.
    """
    source = Source.query.get(source_id)

    projects = Project.query.filter(
        Project.repository_id == source.repository_id
    )

    newest_build_ids = set()
    for project in projects:
        newest_build_ids.add(db.session.query(Build.id).filter(
                Build.project_id == project.id,
                Build.source_id == source_id,
                Build.status == Status.finished
        ).order_by(Build.date_created.desc()).first()[0])

    return get_coverage_by_build_ids(newest_build_ids)

def get_coverage_by_build_id(build_id):
    return get_coverage_by_build_ids([build_id])

def get_coverage_by_build_ids(build_ids):
    """
    Returns the coverage associated with some builds.

    The dictionary maps file names to a string of the form 'UNCCCNCU', where U means
    'uncovered', C means 'covered' and 'N' means 'no coverage info'.
    """
    all_job_ids = db.session.query(Job.id).filter(
        Job.build_id.in_(build_ids)
    )

    return get_coverage_by_job_ids(all_job_ids)

def get_coverage_by_job_ids(job_ids):
    """
    Returns the coverage associated with some jobs.

    The dictionary maps file names to a string of the form 'UNCCCNCU', where U means
    'uncovered', C means 'covered' and 'N' means 'no coverage info'.
    """
    all_coverages = FileCoverage.query.filter(
        FileCoverage.job_id.in_(job_ids)
    )

    combined_file_coverage = dict()

    for coverage in all_coverages:
        combined_file_coverage[coverage.filename] = coverage.data

    return combined_file_coverage
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Built-in packages
from __future__ import division
import logging
import argparse
from argparse import RawDescriptionHelpFormatter

# Add-on packages
import pandas as pd
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
import statsmodels.formula.api as smf
from statsmodels.sandbox.regression.predstd import wls_prediction_std

# Local Packages
from interface import wideToDesign
import logger as sl

def getOptions():
    """ Function to pull in arguments """
    description = """ """
    parser = argparse.ArgumentParser(description=description, formatter_class=RawDescriptionHelpFormatter)
    group1 = parser.add_argument_group(title='Standard input', description='Standard input for SECIM tools.')
    group1.add_argument('--input', dest='fname', action='store', required=True, help="Input dataset in wide format.")
    group1.add_argument('--design', dest='dname', action='store', required=True, help="Design file.")
    group1.add_argument('--ID', dest='uniqID', action='store', required=True,
                        help="Name of the column with unique identifiers.")
    group1.add_argument('--group', dest='group', action='store', required=True,
                        help="Group/treatment identifier in design file [Optional].")
    group1.add_argument('--order', dest='order', action='store', required=True, help='')

    group2 = parser.add_argument_group(title='Required Output')
    group2.add_argument('--fig', dest='oname', action='store', required=True, help="Name of PDF to save scatter plots.")
    group2.add_argument('--table', dest='tname', action='store', required=True,
                        help="Name of table for scatter plots")

    group4 = parser.add_argument_group(title='Development Settings')
    group4.add_argument("--debug", dest="debug", action='store_true', required=False, help="Add debugging log output.")

    args = parser.parse_args()

    return (args)

def runOrder(col):
    """ Run order regression
    :Arguments:
        col (pd.Series): A Pandas series with run order ['run'] as the index
            and ['val'] as the value to regressed on.

    :Returns:
        name (str): Name of the column.

        X's (int): Values of X ['run']

        Y's (int): Values of Y ['val']

        results (sm.RegressionResults): Regression results.

    """
    # Drop missing values and make sure everything is numeric
    clean = col.dropna().reset_index().convert_objects(convert_numeric=True)
    clean.columns = ['run', 'val']

    if clean.shape[0] >= 5:
        # Fit model
        model = smf.ols(formula='val ~ run', data=clean, missing='drop')
        results = model.fit()
        return col.name, clean['run'], clean['val'], results
    else:
        logger.warn('{} had fewer than 5 samples without NaNs, it will be ignored.'.format(col.name))

def flagPval(val, alpha):
    if val <= alpha:
        flag = True
    else:
        flag = False

    return flag

def makeScatter(row, pdf):
    """ Plot a scatter plot of x vs y. """
    # Split out row results
    name, x, y, res = row

    # Get fitted values
    fitted = res.fittedvalues

    # Get slope and p-value
    slope = res.params['run']
    pval = res.pvalues['run']
    rsq = res.rsquared

    # Make Flag based on p-value
    flag_runOrder_pval_05 = flagPval(pval, 0.05)
    flag_runOrder_pval_01 = flagPval(pval, 0.01)

    # Make scatter plot if p-pvalue is less than 0.05
    if flag_runOrder_pval_05 == 1:
        # Get 95% CI
        prstd, lower, upper = wls_prediction_std(res)

        # Sort CIs for Plotting
        toPlot = pd.DataFrame({'x': x, 'lower': lower, 'upper': upper})
        toPlot.sort(columns='x', inplace=True)

        # Plot
        fig, ax = plt.subplots(1, 1, figsize=(8, 5))
        ax.scatter(x, y)
        ax.plot(toPlot['x'], toPlot['lower'], 'r-')
        ax.plot(x, fitted, 'c-')
        ax.plot(toPlot['x'], toPlot['upper'], 'r-')
        ax.set_xlabel('Run Order')
        ax.set_ylabel('Value')
        ax.set_title(u'{}\nScatter plot'.format(name))
        ax.text(.7, .85, u'Slope: {0:.4f}\n(p-value = {1:.4f})\nR^2: {2:4f}'.format(slope, pval, rsq),
                transform=ax.transAxes, fontsize=12)

        # Save to PDF
        pdf.savefig(fig)
        plt.close(fig)

    # Make output table
    out = pd.Series({
        'slope': slope,
        'pval': pval,
        'flag_feature_runOrder_pval_05': flag_runOrder_pval_05,
        'flag_feature_runOrder_pval_01': flag_runOrder_pval_01,
        'rSquared': rsq
    })

    return out

def main(args):
    # Import data
    logger.info('Importing Data')
    dat = wideToDesign(args.fname, args.dname, args.uniqID, args.group, anno=[args.order, ])

    # Transpose Data so compounds are columns
    trans = dat.transpose()
    trans.set_index(args.order, inplace=True)

    # Drop group
    trans.drop(args.group, axis=1, inplace=True)

    # Run each column through regression
    logger.info('Running Regressions')
    res = trans.apply(runOrder, axis=0)
    res.dropna(inplace=True)        # Drop rows that are missing regression results.

    # Plot Results
    # Open a multiple page PDF for plots
    logger.info('Plotting Results')
    pp = PdfPages(args.oname)
    outTable = res.apply(makeScatter, args=(pp, ))
    pp.close()

    # Write regression results to table
    outTable.index.name = args.uniqID
    outTable['flag_feature_runOrder_pval_01'] = outTable['flag_feature_runOrder_pval_01'].astype('int32')
    outTable['flag_feature_runOrder_pval_05'] = outTable['flag_feature_runOrder_pval_05'].astype('int32')
    outTable[['slope', 'pval', 'rSquared', 'flag_feature_runOrder_pval_01', 'flag_feature_runOrder_pval_05']].to_csv(args.tname, sep='\t', float_format='%.4f')

if __name__ == '__main__':
    # Command line options
    args = getOptions()

    logger = logging.getLogger()
    if args.debug:
        sl.setLogger(logger, logLevel='debug')
        DEBUG = True
    else:
        sl.setLogger(logger)

    main(args)
<commit_msg>Fixes the index error from the runOrder function.<commit_after>#!/usr/bin/env python

# Built-in packages
from __future__ import division
import logging
import argparse
from argparse import RawDescriptionHelpFormatter

# Add-on packages
import pandas as pd
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
import statsmodels.formula.api as smf
from statsmodels.sandbox.regression.predstd import wls_prediction_std

# Local Packages
from interface import wideToDesign
import logger as sl

def getOptions():
    """ Function to pull in arguments """
    description = """ """
    parser = argparse.ArgumentParser(description=description, formatter_class=RawDescriptionHelpFormatter)
    group1 = parser.add_argument_group(title='Standard input', description='Standard input for SECIM tools.')
    group1.add_argument('--input', dest='fname', action='store', required=True, help="Input dataset in wide format.")
    group1.add_argument('--design', dest='dname', action='store', required=True, help="Design file.")
    group1.add_argument('--ID', dest='uniqID', action='store', required=True,
                        help="Name of the column with unique identifiers.")
    group1.add_argument('--group', dest='group', action='store', required=True,
                        help="Group/treatment identifier in design file [Optional].")
    group1.add_argument('--order', dest='order', action='store', required=True, help='')

    group2 = parser.add_argument_group(title='Required Output')
    group2.add_argument('--fig', dest='oname', action='store', required=True, help="Name of PDF to save scatter plots.")
    group2.add_argument('--table', dest='tname', action='store', required=True,
                        help="Name of table for scatter plots")

    group4 = parser.add_argument_group(title='Development Settings')
    group4.add_argument("--debug", dest="debug", action='store_true', required=False, help="Add debugging log output.")

    args = parser.parse_args()

    return (args)

def runOrder(col):
    """ Run order regression
    :Arguments:
        col (pd.Series): A Pandas series with run order ['run'] as the index
            and ['val'] as the value to regressed on.

    :Returns:
        name (str): Name of the column.

        X's (int): Values of X ['run']

        Y's (int): Values of Y ['val']

        results (sm.RegressionResults): Regression results.

    """
    # Drop missing values and make sure everything is numeric
    clean = col.dropna().reset_index().convert_objects(convert_numeric=True)
    clean.columns = ['run', 'val']

    if clean.shape[0] >= 5:
        # Fit model
        model = smf.ols(formula='val ~ run', data=clean, missing='drop')
        results = model.fit()
        return col.name, clean['run'], clean['val'], results
    else:
        logger.warn('{} had fewer than 5 samples without NaNs, it will be ignored.'.format(col.name))

def flagPval(val, alpha):
    if val <= alpha:
        flag = True
    else:
        flag = False

    return flag

def makeScatter(row, pdf):
    """ Plot a scatter plot of x vs y. """
    # Split out row results
    name, x, y, res = row

    # Get fitted values
    fitted = res.fittedvalues

    # Get slope and p-value
    slope = res.params['run']
    pval = res.pvalues['run']
    rsq = res.rsquared

    # Make Flag based on p-value
    flag_runOrder_pval_05 = flagPval(pval, 0.05)
    flag_runOrder_pval_01 = flagPval(pval, 0.01)

    # Make scatter plot if p-pvalue is less than 0.05
    if flag_runOrder_pval_05 == 1:
        # Get 95% CI
        prstd, lower, upper = wls_prediction_std(res)

        # Sort CIs for Plotting
        toPlot = pd.DataFrame({'x': x, 'lower': lower, 'upper': upper})
        toPlot.sort(columns='x', inplace=True)

        # Plot
        fig, ax = plt.subplots(1, 1, figsize=(8, 5))
        ax.scatter(x, y)
        ax.plot(toPlot['x'], toPlot['lower'], 'r-')
        ax.plot(x, fitted, 'c-')
        ax.plot(toPlot['x'], toPlot['upper'], 'r-')
        ax.set_xlabel('Run Order')
        ax.set_ylabel('Value')
        ax.set_title(u'{}\nScatter plot'.format(name))
        ax.text(.7, .85, u'Slope: {0:.4f}\n(p-value = {1:.4f})\nR^2: {2:4f}'.format(slope, pval, rsq),
                transform=ax.transAxes, fontsize=12)

        # Save to PDF
        pdf.savefig(fig)
        plt.close(fig)

    # Make output table
    out = pd.Series({
        'slope': slope,
        'pval': pval,
        'flag_feature_runOrder_pval_05': flag_runOrder_pval_05,
        'flag_feature_runOrder_pval_01': flag_runOrder_pval_01,
        'rSquared': rsq
    })

    return out

def main(args):
    # Import data
    logger.info('Importing Data')
    dat = wideToDesign(args.fname, args.dname, args.uniqID, args.group, anno=[args.order, ])

    # Transpose Data so compounds are columns
    trans = dat.transpose()
    trans.set_index(args.order, inplace=True)

    # Drop group
    trans.drop(args.group, axis=1, inplace=True)

    # Run each column through regression
    logger.info('Running Regressions')

    res = list()
    # Iterate over columns
    # Was using an apply statment instead, but it kept giving me index errors
    for col in trans.columns:
        res.append(runOrder(trans[col]))

    # convert result list to a Series
    res = pd.Series({x[0]: x for x in res})

    # Drop rows that are missing regression results
    res.dropna(inplace=True)

    # Plot Results
    # Open a multiple page PDF for plots
    logger.info('Plotting Results')
    pp = PdfPages(args.oname)
    outTable = res.apply(makeScatter, args=(pp, ))
    pp.close()

    # Write regression results to table
    outTable.index.name = args.uniqID
    outTable['flag_feature_runOrder_pval_01'] = outTable['flag_feature_runOrder_pval_01'].astype('int32')
    outTable['flag_feature_runOrder_pval_05'] = outTable['flag_feature_runOrder_pval_05'].astype('int32')
    outTable[['slope', 'pval', 'rSquared', 'flag_feature_runOrder_pval_01', 'flag_feature_runOrder_pval_05']].to_csv(args.tname, sep='\t', float_format='%.4f')

if __name__ == '__main__':
    # Command line options
    args = getOptions()

    logger = logging.getLogger()
    if args.debug:
        sl.setLogger(logger, logLevel='debug')
        DEBUG = True
    else:
        sl.setLogger(logger)

    main(args)
<|endoftext|>


<commit_before>#! /usr/bin/env python

import json
import re
import time

import requests

import config

from utils.with_logger import WithLogger

class Slacker(WithLogger):

    def __init__(self, slack_name, token, init=True):
        """
        slack name is the short name of the slack (preceding '.slack.com')
        token should be a Slack API Token.
        """
        self.slack_name = slack_name
        self.token = token
        assert self.token, "Token should not be blank"
        self.url = self.api_url()
        self.config = config.Config()
        self.session = requests.Session()
        if init:
            self.get_users()
            self.get_channels()

    def get_emojis(self):
        url = self.url + "emoji.list?token={}".format(self.token)
        payload = self.session.get(url).json()
        return payload

    def get_user(self, uid):
        url = self.url + "users.info?token={}&user={}".format(self.token, uid)
        payload = self.session.get(url).json()
        return payload

    def get_users(self):
        users = self.get_all_user_objects()
        self.users_by_id = {x['id']: x['name'] for x in users}
        self.users_by_name = {x['name']: x['id'] for x in users}
        self.restricted_users = [x['id'] for x in users if x.get('is_restricted')]
        self.ultra_restricted_users = [x['id'] for x in users if x.get('is_ultra_restricted')]
        self.all_restricted_users = set(self.restricted_users + self.ultra_restricted_users)
        self.logger.debug("All restricted user names: %s", ', '.join([self.users_by_id[x] for x in self.all_restricted_users]))
        return users

    def asciify(self, text):
        return ''.join([x for x in list(text) if ord(x) in range(128)])

    def add_channel_markup(self, channel_name, fail_silently=True):
        channel_id = self.get_channelid(channel_name)
        if channel_id:
            return "<#{}|{}>".format(channel_id, channel_name)
        else:
            if fail_silently:
                return "#{}".format(channel_name)

    def get_messages_in_time_range(self, oldest, cid, latest=None):
        assert cid in self.channels_by_id, "Unknown channel ID {}".format(cid)
        cname = self.channels_by_id[cid]
        messages = []
        done = False
        while not done:
            murl = self.url + "channels.history?oldest={}&token={}&channel={}".format(oldest, self.token, cid)
            if latest:
                murl += "&latest={}".format(latest)
            else:
                murl += "&latest={}".format(int(time.time()))
            response = self.session.get(murl)
            content = response.content
            try:
                payload = response.json()
            except ValueError as e:
                self.logger.error("Value error %s for content: %s", e, content)
                raise e
            if payload.get('error') == 'ratelimited':
                retry_after = int(response.headers['Retry-After'])
                self.logger.debug('Ratelimited. Sleeping %s', retry_after)
                time.sleep(retry_after)
                continue
            messages += payload['messages']
            if payload['has_more'] is False:
                done = True
                continue
            ts = [float(x['ts']) for x in messages]
            earliest = min(ts)
            latest = earliest
        messages.sort(key=lambda x: float(x['ts']))
        for message in messages:
            message['channel'] = cname
        return messages

    def replace_id(self, cid):
        """
        Assuming either a #channelid or @personid, replace them with #channelname or @username
        """
        stripped = cid[1:]
        first = cid[0]
        if first == "#":
            m = [x for x in self.channels if self.channels[x] == stripped]
            if m:
                return "#" + m[0]
        elif first == "@":
            # occasionally input will have the format "userid|name".
            #  in case the name changed at some point,
            #  lookup user by userid in users_by_id
            if "|" in stripped:
                uname_parts = stripped.split("|")
                uname = self.users_by_id[uname_parts[0]]
            else:
                uname = self.users_by_id[stripped]
            if uname:
                return "@" + uname
        return cid

    def detokenize(self, message):
        new = []
        tokens = re.split("(<.*?>)", message)
        for token in tokens:
            if len(token) > 3 and token[0] == "<" and token[-1] == ">":
                token = self.replace_id(token[1:-1])
            new.append(token)
        message = " ".join(new)
        return message

    def api_url(self):
        return "https://{}.slack.com/api/".format(self.slack_name)

    def get_channels(self, exclude_archived=True):
        """
        return a {channel_name: channel_id} dictionary
        if exclude_archived (default: True), only shows non-archived channels
        """
        channels = self.get_all_channel_objects(exclude_archived=exclude_archived)
        self.channels_by_id = {x['id']: x['name'] for x in channels}
        self.channels_by_name = {x['name']: x['id'] for x in channels}
        self.channels = self.channels_by_name

    def get_channelid(self, channel_name):
        return self.channels_by_name.get(channel_name)

    def channel_exists(self, channel_name):
        try:
            # strip leading "#" if it exists, as Slack returns all channels without them
            if channel_name[0] == "#":
                channel = channel_name[1:]
            else:
                channel = channel_name
            return self.channels_by_name[channel]
        except KeyError:  # channel not found
            return None

    def delete_message(self, cid, message_timestamp):
        url_template = self.url + "chat.delete?token={}&channel={}&ts={}"
        url = url_template.format(self.token, cid, message_timestamp)
        ret = self.session.get(url).json()
        if not ret['ok']:
            self.logger.error("Failed to delete message; error: %s", ret)
        return ret['ok']

    def get_channel_members_ids(self, channel_name):
        """
        returns an array of member IDs for channel_name
        """
        return self.get_channel_info(channel_name)['members']

    def channel_has_only_restricted_members(self, channel_name):
        """
        returns True if the channel only has restricted/ultra_restricted
        members, False otherwise
        """

        mids = set(self.get_channel_members_ids(channel_name))
        self.logger.debug("Current members in %s are %s", channel_name, mids)
        return mids.intersection(self.all_restricted_users)

    def get_channel_member_names(self, channel_name):
        """
        returns an array of ["@member"] for members of the channel
        """
        members = self.get_channel_members_ids(channel_name)
        return ["@" + self.users_by_id[x] for x in members]

    def get_channel_info(self, channel_name):
        """
        returns JSON with channel information.  Adds 'age' in seconds to JSON
        """
        url_template = self.url + "channels.info?token={}&channel={}"
        cid = self.get_channelid(channel_name)
        now = int(time.time())
        url = url_template.format(self.token, cid)
        ret = self.session.get(url).json()
        if ret['ok'] is not True:
            m = "Attempted to get channel info for {}, but return was {}"
            m = m.format(channel_name, ret)
            raise RuntimeError(m)
        created = ret['channel']['created']
        age = now - created
        ret['channel']['age'] = age
        return ret['channel']

    def get_all_channel_objects(self, exclude_archived=True):
        """
        return all channels
        if exclude_archived (default: True), only shows non-archived channels
        """

        url_template = self.url + "channels.list?exclude_archived={}&token={}"
        if exclude_archived:
            exclude_archived = 1
        else:
            exclude_archived = 0
        url = url_template.format(exclude_archived, self.token)
        request = self.session.get(url)
        payload = request.json()
        assert 'channels' in payload
        return payload['channels']

    def get_all_user_objects(self):
        url = self.url + "users.list?token=" + self.token
        return self.session.get(url).json()['members']

    def archive(self, channel_name):
        url_template = self.url + "channels.archive?token={}&channel={}"
        cid = self.get_channelid(channel_name)
        url = url_template.format(self.token, cid)
        request = self.session.get(url)
        payload = request.json()
        return payload

    def post_message(self, channel, message, message_type=None):
        """
        Posts a `message` into a `channel`.
        Optionally append an invisible attachment with 'fallback' set to `message_type`.

        Note: `channel` value should not be preceded with '#'.
        """
        assert channel  # not blank
        if channel[0] == '#':
            channel = channel[1:]

        post_data = {
            'token': self.token,
            'channel': channel,
            'text': message.encode('utf-8')
        }

        bot_name = self.config.bot_name
        bot_avatar_url = self.config.bot_avatar_url
        if bot_name or bot_avatar_url:
            post_data['as_user'] = False
            if bot_name:
                post_data['username'] = bot_name
            if bot_avatar_url:
                post_data['icon_url'] = bot_avatar_url

        if message_type:
            post_data['attachments'] = json.dumps([{'fallback': message_type}], encoding='utf-8')

        p = self.session.post(self.url + "chat.postMessage", data=post_data)
        return p.json()
<commit_msg>Improve retry logic for rate limiting<commit_after>#! /usr/bin/env python

import json
import re
import time

import requests

import config

from utils.with_logger import WithLogger

class Slacker(WithLogger):

    def __init__(self, slack_name, token, init=True):
        """
        slack name is the short name of the slack (preceding '.slack.com')
        token should be a Slack API Token.
        """
        self.slack_name = slack_name
        self.token = token
        assert self.token, "Token should not be blank"
        self.url = self.api_url()
        self.config = config.Config()
        self.session = requests.Session()
        if init:
            self.get_users()
            self.get_channels()

    def get_emojis(self):
        url = self.url + "emoji.list?token={}".format(self.token)
        payload = self.session.get(url).json()
        return payload

    def get_user(self, uid):
        url = self.url + "users.info?token={}&user={}".format(self.token, uid)
        payload = self.session.get(url).json()
        return payload

    def get_users(self):
        users = self.get_all_user_objects()
        self.users_by_id = {x['id']: x['name'] for x in users}
        self.users_by_name = {x['name']: x['id'] for x in users}
        self.restricted_users = [x['id'] for x in users if x.get('is_restricted')]
        self.ultra_restricted_users = [x['id'] for x in users if x.get('is_ultra_restricted')]
        self.all_restricted_users = set(self.restricted_users + self.ultra_restricted_users)
        self.logger.debug("All restricted user names: %s", ', '.join([self.users_by_id[x] for x in self.all_restricted_users]))
        return users

    def asciify(self, text):
        return ''.join([x for x in list(text) if ord(x) in range(128)])

    def add_channel_markup(self, channel_name, fail_silently=True):
        channel_id = self.get_channelid(channel_name)
        if channel_id:
            return "<#{}|{}>".format(channel_id, channel_name)
        else:
            if fail_silently:
                return "#{}".format(channel_name)

    def get_messages_in_time_range(self, oldest, cid, latest=None):
        assert cid in self.channels_by_id, "Unknown channel ID {}".format(cid)
        cname = self.channels_by_id[cid]
        messages = []
        done = False
        retry_attempts = 0
        max_retry_attempts = 10
        while not done:
            murl = self.url + "channels.history?oldest={}&token={}&channel={}".format(oldest, self.token, cid)
            if latest:
                murl += "&latest={}".format(latest)
            else:
                murl += "&latest={}".format(int(time.time()))
            response = self.session.get(murl)

            try:
                response.raise_for_status()
            except requests.exceptions.HTTPError as e:
                if retry_attempts >= max_retry_attempts:
                    raise e
                if 'Retry-After' in response.headers:
                    retry_after = int(response.headers['Retry-After']) * 2
                    self.logger.debug('Ratelimited. Sleeping %s', retry_after)
                else:
                    retry_attempts += 1
                    retry_after = retry_attempts * 5
                    self.logger.debug('Unknown requests error. Sleeping %s. %s/%s retry attempts.', retry_after, retry_attempts, max_retry_attempts)
                time.sleep(retry_after)
                continue

            payload = response.json()
            messages += payload['messages']
            if payload['has_more'] is False:
                done = True
                continue
            ts = [float(x['ts']) for x in messages]
            earliest = min(ts)
            latest = earliest
        messages.sort(key=lambda x: float(x['ts']))
        for message in messages:
            message['channel'] = cname
        return messages

    def replace_id(self, cid):
        """
        Assuming either a #channelid or @personid, replace them with #channelname or @username
        """
        stripped = cid[1:]
        first = cid[0]
        if first == "#":
            m = [x for x in self.channels if self.channels[x] == stripped]
            if m:
                return "#" + m[0]
        elif first == "@":
            # occasionally input will have the format "userid|name".
            #  in case the name changed at some point,
            #  lookup user by userid in users_by_id
            if "|" in stripped:
                uname_parts = stripped.split("|")
                uname = self.users_by_id[uname_parts[0]]
            else:
                uname = self.users_by_id[stripped]
            if uname:
                return "@" + uname
        return cid

    def detokenize(self, message):
        new = []
        tokens = re.split("(<.*?>)", message)
        for token in tokens:
            if len(token) > 3 and token[0] == "<" and token[-1] == ">":
                token = self.replace_id(token[1:-1])
            new.append(token)
        message = " ".join(new)
        return message

    def api_url(self):
        return "https://{}.slack.com/api/".format(self.slack_name)

    def get_channels(self, exclude_archived=True):
        """
        return a {channel_name: channel_id} dictionary
        if exclude_archived (default: True), only shows non-archived channels
        """
        channels = self.get_all_channel_objects(exclude_archived=exclude_archived)
        self.channels_by_id = {x['id']: x['name'] for x in channels}
        self.channels_by_name = {x['name']: x['id'] for x in channels}
        self.channels = self.channels_by_name

    def get_channelid(self, channel_name):
        return self.channels_by_name.get(channel_name)

    def channel_exists(self, channel_name):
        try:
            # strip leading "#" if it exists, as Slack returns all channels without them
            if channel_name[0] == "#":
                channel = channel_name[1:]
            else:
                channel = channel_name
            return self.channels_by_name[channel]
        except KeyError:  # channel not found
            return None

    def delete_message(self, cid, message_timestamp):
        url_template = self.url + "chat.delete?token={}&channel={}&ts={}"
        url = url_template.format(self.token, cid, message_timestamp)
        ret = self.session.get(url).json()
        if not ret['ok']:
            self.logger.error("Failed to delete message; error: %s", ret)
        return ret['ok']

    def get_channel_members_ids(self, channel_name):
        """
        returns an array of member IDs for channel_name
        """
        return self.get_channel_info(channel_name)['members']

    def channel_has_only_restricted_members(self, channel_name):
        """
        returns True if the channel only has restricted/ultra_restricted
        members, False otherwise
        """

        mids = set(self.get_channel_members_ids(channel_name))
        self.logger.debug("Current members in %s are %s", channel_name, mids)
        return mids.intersection(self.all_restricted_users)

    def get_channel_member_names(self, channel_name):
        """
        returns an array of ["@member"] for members of the channel
        """
        members = self.get_channel_members_ids(channel_name)
        return ["@" + self.users_by_id[x] for x in members]

    def get_channel_info(self, channel_name):
        """
        returns JSON with channel information.  Adds 'age' in seconds to JSON
        """
        url_template = self.url + "channels.info?token={}&channel={}"
        cid = self.get_channelid(channel_name)
        now = int(time.time())
        url = url_template.format(self.token, cid)
        ret = self.session.get(url).json()
        if ret['ok'] is not True:
            m = "Attempted to get channel info for {}, but return was {}"
            m = m.format(channel_name, ret)
            raise RuntimeError(m)
        created = ret['channel']['created']
        age = now - created
        ret['channel']['age'] = age
        return ret['channel']

    def get_all_channel_objects(self, exclude_archived=True):
        """
        return all channels
        if exclude_archived (default: True), only shows non-archived channels
        """

        url_template = self.url + "channels.list?exclude_archived={}&token={}"
        if exclude_archived:
            exclude_archived = 1
        else:
            exclude_archived = 0
        url = url_template.format(exclude_archived, self.token)
        request = self.session.get(url)
        payload = request.json()
        assert 'channels' in payload
        return payload['channels']

    def get_all_user_objects(self):
        url = self.url + "users.list?token=" + self.token
        return self.session.get(url).json()['members']

    def archive(self, channel_name):
        url_template = self.url + "channels.archive?token={}&channel={}"
        cid = self.get_channelid(channel_name)
        url = url_template.format(self.token, cid)
        request = self.session.get(url)
        payload = request.json()
        return payload

    def post_message(self, channel, message, message_type=None):
        """
        Posts a `message` into a `channel`.
        Optionally append an invisible attachment with 'fallback' set to `message_type`.

        Note: `channel` value should not be preceded with '#'.
        """
        assert channel  # not blank
        if channel[0] == '#':
            channel = channel[1:]

        post_data = {
            'token': self.token,
            'channel': channel,
            'text': message.encode('utf-8')
        }

        bot_name = self.config.bot_name
        bot_avatar_url = self.config.bot_avatar_url
        if bot_name or bot_avatar_url:
            post_data['as_user'] = False
            if bot_name:
                post_data['username'] = bot_name
            if bot_avatar_url:
                post_data['icon_url'] = bot_avatar_url

        if message_type:
            post_data['attachments'] = json.dumps([{'fallback': message_type}], encoding='utf-8')

        p = self.session.post(self.url + "chat.postMessage", data=post_data)
        return p.json()
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

import babel.dates

from odoo import api, fields, models

from odoo.fields import Datetime

class FleetVehicle(models.Model):
    _inherit = 'fleet.vehicle'

    co2_fee = fields.Float(compute='_compute_co2_fee', string="CO2 Fee")
    total_depreciated_cost = fields.Float(compute='_compute_total_depreciated_cost',
        string="Total Cost (Depreciated)", track_visibility="onchange",
        help="This includes all the depreciated costs and the CO2 fee")
    total_cost = fields.Float(compute='_compute_total_cost', string="Total Cost", help="This include all the costs and the CO2 fee")
    fuel_type = fields.Selection(required=True, default='diesel')
    atn = fields.Float(compute='_compute_car_atn', string="ATN")
    acquisition_date = fields.Date(required=True)

    @api.depends('co2_fee', 'log_contracts', 'log_contracts.state', 'log_contracts.recurring_cost_amount_depreciated')
    def _compute_total_depreciated_cost(self):
        for car in self:
            car.total_depreciated_cost = car.co2_fee + \
                sum(car.log_contracts.filtered(
                    lambda contract: contract.state == 'open'
                ).mapped('recurring_cost_amount_depreciated'))

    @api.depends('co2_fee', 'log_contracts', 'log_contracts.state', 'log_contracts.cost_generated')
    def _compute_total_cost(self):
        for car in self:
            car.total_cost = car.co2_fee
            contracts = car.log_contracts.filtered(
                lambda contract: contract.state == 'open' and contract.cost_frequency != 'no'
            )
            for contract in contracts:
                if contract.cost_frequency == "daily":
                    car.total_cost += contract.cost_generated * 30.0
                elif contract.cost_frequency == "weekly":
                    car.total_cost += contract.cost_generated * 4.0
                elif contract.cost_frequency == "monthly":
                    car.total_cost += contract.cost_generated
                elif contract.cost_frequency == "yearly":
                    car.total_cost += contract.cost_generated / 12.0

    def _get_co2_fee(self, co2):
        return max((((co2 * 9.0) - 600.0) * 1.2488) / 12.0, 0.0)

    @api.depends('co2')
    def _compute_co2_fee(self):
        for car in self:
            car.co2_fee = self._get_co2_fee(car.co2)

    @api.depends('fuel_type', 'car_value', 'acquisition_date')
    def _compute_car_atn(self):
        for car in self:
            car.atn = car._get_car_atn(car.acquisition_date, car.car_value, car.fuel_type, car.co2)

    @api.depends('model_id', 'license_plate', 'log_contracts', 'acquisition_date',
                 'co2_fee', 'log_contracts', 'log_contracts.state', 'log_contracts.recurring_cost_amount_depreciated')
    def _compute_vehicle_name(self):
        super(FleetVehicle, self)._compute_vehicle_name()
        for vehicle in self:
            acquisition_date = vehicle._get_acquisition_date()
            vehicle.name += u" \u2022 " + str(round(vehicle.total_depreciated_cost, 2)) + u" \u2022 " + acquisition_date

    def _get_acquisition_date(self):
        self.ensure_one()
        return babel.dates.format_date(
            date=Datetime.from_string(self.acquisition_date),
            format='MMMM y',
            locale=self._context.get('lang') or 'en_US'
        )

    def _get_car_atn(self, acquisition_date, car_value, fuel_type, co2):
        # Compute the correction coefficient from the age of the car
        now = Datetime.from_string(Datetime.now())
        start = Datetime.from_string(acquisition_date)
        if start:
            number_of_month = (now.year - start.year) * 12.0 + now.month - start.month + int(bool(now.day - start.day + 1))
            if number_of_month <= 12:
                age_coefficient = 1.00
            elif number_of_month <= 24:
                age_coefficient = 0.94
            elif number_of_month <= 36:
                age_coefficient = 0.88
            elif number_of_month <= 48:
                age_coefficient = 0.82
            elif number_of_month <= 60:
                age_coefficient = 0.76
            else:
                age_coefficient = 0.70
            car_value = car_value * age_coefficient
            # Compute atn value from corrected car_value
            magic_coeff = 6.0 / 7.0  # Don't ask me why
            if fuel_type == 'electric':
                atn = 0.0
            else:
                if fuel_type in ['diesel', 'hybrid']:
                    reference = 87.0
                else:
                    reference = 105.0

                if co2 <= reference:
                    atn = car_value * max(0.04, (0.055 - 0.001 * (reference - co2))) * magic_coeff
                else:
                    atn = car_value * min(0.18, (0.055 + 0.001 * (co2 - reference))) * magic_coeff
            return max(1280, atn) / 12.0

class FleetVehicleLogContract(models.Model):
    _inherit = 'fleet.vehicle.log.contract'

    recurring_cost_amount_depreciated = fields.Float("Recurring Cost Amount (depreciated)")

class FleetVehicleModel(models.Model):
    _inherit = 'fleet.vehicle.model'

    default_recurring_cost_amount_depreciated = fields.Float(string="Cost (Depreciated)",
        help="Default recurring cost amount that should be applied to a new car from this model")
    default_co2 = fields.Float(string="CO2 emissions")
    default_fuel_type = fields.Selection([('gasoline', 'Gasoline'), ('diesel', 'Diesel'), ('electric', 'Electric'), ('hybrid', 'Hybrid')], 'Fuel Type', help='Fuel Used by the vehicle')
    default_car_value = fields.Float(string="Catalog Value (VAT Incl.)")
    can_be_requested = fields.Boolean(string="Can be requested", help="Can be requested on a contract as a new car")
    default_atn = fields.Float(compute='_compute_atn', string="ATN")
    default_total_depreciated_cost = fields.Float(compute='_compute_default_total_depreciated_cost', string="Total Cost (Depreciated)")
    co2_fee = fields.Float(compute='_compute_co2_fee', string="CO2 fee")

    @api.depends('default_car_value', 'default_co2', 'default_fuel_type')
    def _compute_atn(self):
        now = Datetime.now()
        for model in self:
            model.default_atn = self.env['fleet.vehicle']._get_car_atn(now, model.default_car_value, model.default_fuel_type, model.default_co2)

    @api.depends('co2_fee', 'default_recurring_cost_amount_depreciated')
    def _compute_default_total_depreciated_cost(self):
        for model in self:
            model.default_total_depreciated_cost = model.co2_fee + model.default_recurring_cost_amount_depreciated

    @api.multi
    @api.depends('name', 'brand_id', 'default_total_depreciated_cost')
    def name_get(self):
        res = super(FleetVehicleModel, self).name_get()
        new_res = []
        for res_item in res:
            model = self.browse(res_item[0])
            if model.default_total_depreciated_cost != 0.0:
                new_res.append((res_item[0], res_item[1] + u" \u2022 " + str(round(model.default_total_depreciated_cost, 2))))
            else:
                new_res.append(res_item)
        return new_res

    @api.depends('default_co2')
    def _compute_co2_fee(self):
        for model in self:
            model.co2_fee = self.env['fleet.vehicle']._get_co2_fee(model.default_co2)
<commit_msg>[FIX] l10n_be_hr_payroll_fleet: Don't display costs in model name<commit_after># -*- coding:utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

import babel.dates

from odoo import api, fields, models

from odoo.fields import Datetime

class FleetVehicle(models.Model):
    _inherit = 'fleet.vehicle'

    co2_fee = fields.Float(compute='_compute_co2_fee', string="CO2 Fee")
    total_depreciated_cost = fields.Float(compute='_compute_total_depreciated_cost',
        string="Total Cost (Depreciated)", track_visibility="onchange",
        help="This includes all the depreciated costs and the CO2 fee")
    total_cost = fields.Float(compute='_compute_total_cost', string="Total Cost", help="This include all the costs and the CO2 fee")
    fuel_type = fields.Selection(required=True, default='diesel')
    atn = fields.Float(compute='_compute_car_atn', string="ATN")
    acquisition_date = fields.Date(required=True)

    @api.depends('co2_fee', 'log_contracts', 'log_contracts.state', 'log_contracts.recurring_cost_amount_depreciated')
    def _compute_total_depreciated_cost(self):
        for car in self:
            car.total_depreciated_cost = car.co2_fee + \
                sum(car.log_contracts.filtered(
                    lambda contract: contract.state == 'open'
                ).mapped('recurring_cost_amount_depreciated'))

    @api.depends('co2_fee', 'log_contracts', 'log_contracts.state', 'log_contracts.cost_generated')
    def _compute_total_cost(self):
        for car in self:
            car.total_cost = car.co2_fee
            contracts = car.log_contracts.filtered(
                lambda contract: contract.state == 'open' and contract.cost_frequency != 'no'
            )
            for contract in contracts:
                if contract.cost_frequency == "daily":
                    car.total_cost += contract.cost_generated * 30.0
                elif contract.cost_frequency == "weekly":
                    car.total_cost += contract.cost_generated * 4.0
                elif contract.cost_frequency == "monthly":
                    car.total_cost += contract.cost_generated
                elif contract.cost_frequency == "yearly":
                    car.total_cost += contract.cost_generated / 12.0

    def _get_co2_fee(self, co2):
        return max((((co2 * 9.0) - 600.0) * 1.2488) / 12.0, 0.0)

    @api.depends('co2')
    def _compute_co2_fee(self):
        for car in self:
            car.co2_fee = self._get_co2_fee(car.co2)

    @api.depends('fuel_type', 'car_value', 'acquisition_date')
    def _compute_car_atn(self):
        for car in self:
            car.atn = car._get_car_atn(car.acquisition_date, car.car_value, car.fuel_type, car.co2)

    @api.depends('model_id', 'license_plate', 'log_contracts', 'acquisition_date',
                 'co2_fee', 'log_contracts', 'log_contracts.state', 'log_contracts.recurring_cost_amount_depreciated')
    def _compute_vehicle_name(self):
        super(FleetVehicle, self)._compute_vehicle_name()
        for vehicle in self:
            acquisition_date = vehicle._get_acquisition_date()
            vehicle.name += u" \u2022 " + str(round(vehicle.total_depreciated_cost, 2)) + u" \u2022 " + acquisition_date

    def _get_acquisition_date(self):
        self.ensure_one()
        return babel.dates.format_date(
            date=Datetime.from_string(self.acquisition_date),
            format='MMMM y',
            locale=self._context.get('lang') or 'en_US'
        )

    def _get_car_atn(self, acquisition_date, car_value, fuel_type, co2):
        # Compute the correction coefficient from the age of the car
        now = Datetime.from_string(Datetime.now())
        start = Datetime.from_string(acquisition_date)
        if start:
            number_of_month = (now.year - start.year) * 12.0 + now.month - start.month + int(bool(now.day - start.day + 1))
            if number_of_month <= 12:
                age_coefficient = 1.00
            elif number_of_month <= 24:
                age_coefficient = 0.94
            elif number_of_month <= 36:
                age_coefficient = 0.88
            elif number_of_month <= 48:
                age_coefficient = 0.82
            elif number_of_month <= 60:
                age_coefficient = 0.76
            else:
                age_coefficient = 0.70
            car_value = car_value * age_coefficient
            # Compute atn value from corrected car_value
            magic_coeff = 6.0 / 7.0  # Don't ask me why
            if fuel_type == 'electric':
                atn = 0.0
            else:
                if fuel_type in ['diesel', 'hybrid']:
                    reference = 87.0
                else:
                    reference = 105.0

                if co2 <= reference:
                    atn = car_value * max(0.04, (0.055 - 0.001 * (reference - co2))) * magic_coeff
                else:
                    atn = car_value * min(0.18, (0.055 + 0.001 * (co2 - reference))) * magic_coeff
            return max(1280, atn) / 12.0

class FleetVehicleLogContract(models.Model):
    _inherit = 'fleet.vehicle.log.contract'

    recurring_cost_amount_depreciated = fields.Float("Recurring Cost Amount (depreciated)")

class FleetVehicleModel(models.Model):
    _inherit = 'fleet.vehicle.model'

    default_recurring_cost_amount_depreciated = fields.Float(string="Cost (Depreciated)",
        help="Default recurring cost amount that should be applied to a new car from this model")
    default_co2 = fields.Float(string="CO2 emissions")
    default_fuel_type = fields.Selection([('gasoline', 'Gasoline'), ('diesel', 'Diesel'), ('electric', 'Electric'), ('hybrid', 'Hybrid')], 'Fuel Type', help='Fuel Used by the vehicle')
    default_car_value = fields.Float(string="Catalog Value (VAT Incl.)")
    can_be_requested = fields.Boolean(string="Can be requested", help="Can be requested on a contract as a new car")
    default_atn = fields.Float(compute='_compute_atn', string="ATN")
    default_total_depreciated_cost = fields.Float(compute='_compute_default_total_depreciated_cost', string="Total Cost (Depreciated)")
    co2_fee = fields.Float(compute='_compute_co2_fee', string="CO2 fee")

    @api.depends('default_car_value', 'default_co2', 'default_fuel_type')
    def _compute_atn(self):
        now = Datetime.now()
        for model in self:
            model.default_atn = self.env['fleet.vehicle']._get_car_atn(now, model.default_car_value, model.default_fuel_type, model.default_co2)

    @api.depends('co2_fee', 'default_recurring_cost_amount_depreciated')
    def _compute_default_total_depreciated_cost(self):
        for model in self:
            model.default_total_depreciated_cost = model.co2_fee + model.default_recurring_cost_amount_depreciated

    @api.depends('default_co2')
    def _compute_co2_fee(self):
        for model in self:
            model.co2_fee = self.env['fleet.vehicle']._get_co2_fee(model.default_co2)
<|endoftext|>


<commit_before>import scipy.optimize as spo
import fixed_points as fx

def local_solver(
    sampler,
    f,
    qg,
    H,
    stop_time=None,
    max_repeats=None,
    max_updates=0,
    logfile=None
    ):
    """
    A fixed point solver using local optimization (Sussillo and Barak 2013)
    Locally optimizes an objective function with minima at fixed points.
    Repeatedly samples and optimizes points along random trajectories.

    User provides function handle sampler
        sampler() returns random v in state space, an (N,1) ndarray
    User provides function handle f, which accepts (N,1) ndarray v
        f(v) returns change in v after one update
    User provides function handles qg and H, which accept (N,) ndarray v
    qg and H conform to scipy.optimize.minimize first parameter and "hess" parameter
        qg(v) returns
            q: the objective at v (a scalar)
            g: the gradient of the objective (a (N,) ndarray)
        H(v) returns
            H: an approximate Hessian of q at v (a (N,N) ndarray)
    If provided, the solver terminates at the clock time stop_time.
    If provided, the solver terminates after max_repeats samples.
    The solver iterates each sample under the system dynamics at most max_updates times before optimizing.

    A dictionary with the following entries is returned:
    "Seeds": ndarray where the n^th column is the n^th seed
    "Updates": ndarray where the n^th element is the number of system updates for the n^th seed
    "Optima": ndarray where the n^th column is the optimum from the n^th seed

    """
    seeds, updates, optima = [], [], []
    start = time.clock()
    for repeat in it.count(1):

        # Check termination criteria
        if stop_time is not None and time.clock() >= stop_time: break
        if max_repeats is not None and repeat >= max_repeats: break

        # get random initial seed anywhere in range
        v = sampler()
        seeds.append(v)

        # iterate trajectory a random number of steps
        num_updates = np.random.randint(max_updates)
        for u in range(num_updates):
            v = v + f(v)
        updates.append(num_updates)

        # run minimization
        result = spo.minimize(qg, v.flatten(), method='trust-ncg', jac=True, hess=H)
        optimum = result.x.reshape(v.shape)
        optima.append(optimum)

    # Format output
    return {
        "Seeds": np.concatenate(seeds, axis=1),
        "Updates": np.concatenate(updates),
        "Optima": np.concatenate(optima, axis=1),
    }

def fiber_solver(
    f,
    ef,
    Df,
    compute_step_amount,
    v=None,
    c=None,
    z=None,
    N=None,
    terminate=None,
    logfile=None,
    stop_time=None,
    max_traverse_steps=None,
    max_step_size=None,
    max_solve_iterations=None,
    solve_tolerance=None,
    ):
    """
    Fixed point location using directional fibers.
    All parameters are as described in directional_fibers.traverse_fiber().
    Returns solution, a dictionary with keys
        "Fiber": the result of traverse_fiber()
        "Fixed points": an array with one fixed point per column
        "Refinements": a list of local traverse_fiber results, one per fixed point
        "Fixed index": an array of fixed point candidate indices in the fiber
    """

    # Traverse fiber
    fiber_result = df.traverse_fiber(
        f,
        Df,
        compute_step_amount,
        v=v,
        c=c,
        z=z,
        N=N,
        terminate=terminate,
        logfile=logfile,
        stop_time=stop_time,
        max_traverse_steps=max_traverse_steps,
        max_step_size=max_step_size,
        max_solve_iterations=max_solve_iterations,
        solve_tolerance=solve_tolerance,
    )

    # Keep final direction vector in case random default was used (in theory shouldn't matter)
    c = fiber_result["c"]

    # Extract candidate fixed points: endpoints, local |alpha| minimum, alpha sign change
    # As well as immediate neighbors for close pairs of fixed points and added redundancy
    X = fiber_result["X"]
    a = X[-1,:]
    fixed_index = np.zeros(len(a), dtype=bool)
    # endpoints
    fixed_index[[0, -1]] = True
    # local magnitude minima
    fixed_index[1:-1] |= (np.fabs(a[1:-1]) <= np.fabs(a[2:])) & (np.fabs(a[1:-1]) <= np.fabs(a[:-2]))
    # sign changes
    fixed_index[:-1] |= np.sign(a[:-1]) != np.sign(a[1:])
    # extra redundancy with neighbors
    fixed_index[:-1] = np.logical_or(fixed_index[:-1], fixed_index[1:])
    fixed_index[1:] = np.logical_or(fixed_index[1:], fixed_index[:-1])

    # Set up within-fiber Newton-Raphson step computation
    def compute_refine_step_amount(x, DF, z):
        refine_step_amount = -x[-1,0]/z[-1,0]
        fiber_step_amount, fiber_step_data = compute_step_amount(x, DF, z)
        step_amount = np.sign(refine_step_amount)*min(np.fabs(refine_step_amount), fiber_step_amount)
        step_data = (refine_step_amount, fiber_step_amount, fiber_step_data)
        return step_amount, step_data
    refine_terminate = lambda x: fx.is_fixed(x[:-1,:], f, ef)[0] or terminate(x)

    # Run within-fiber Newton-Raphson at each candidate
    X = X[:, fixed_index]
    refinement_results = []
    fixed_points = []
    for i in range(X.shape[1]):
        refinement_result = df.traverse_fiber(
            f,
            Df,
            compute_refine_step_amount,
            v=X[:-1,[i]].copy(),
            c=c,
            terminate=refine_terminate,
            logfile=logfile,
            stop_time=stop_time,
            max_traverse_steps=2**5, # few steps needed for Newton-Raphson
            max_step_size=max_step_size,
            max_solve_iterations=max_solve_iterations,
            solve_tolerance=solve_tolerance,
        )
        refinement_results.append(refinement_result)
        fixed_points.append(refinement_result["X"][:-1,[-1]].copy())

    # Return output
    solution = {
        "Fiber": fiber_result,
        "Fixed points": np.concatenate(fixed_points,axis=1),
        "Refinements": refinement_results,
        "Fixed index": np.flatnonzero(fixed_index),
    }
    return solution
<commit_msg>tweaked comment<commit_after>import scipy.optimize as spo
import fixed_points as fx

def local_solver(
    sampler,
    f,
    qg,
    H,
    stop_time=None,
    max_repeats=None,
    max_updates=0,
    logfile=None
    ):
    """
    A fixed point solver using local optimization (Sussillo and Barak 2013)
    Locally optimizes an objective function with minima at fixed points.
    Finds multiple fixed points with repeated local optimization of seeds along random trajectories.

    User provides function handle sampler
        sampler() returns random v in state space, an (N,1) ndarray
    User provides function handle f, which accepts (N,1) ndarray v
        f(v) returns change in v after one update
    User provides function handles qg and H, which accept (N,) ndarray v
    qg and H conform to scipy.optimize.minimize first parameter and "hess" parameter
        qg(v) returns
            q: the objective at v (a scalar)
            g: the gradient of the objective (a (N,) ndarray)
        H(v) returns
            H: an approximate Hessian of q at v (a (N,N) ndarray)
    If provided, the solver terminates at the clock time stop_time.
    If provided, the solver terminates after max_repeats samples.
    The solver iterates each sample under the system dynamics at most max_updates times before optimizing.

    A dictionary with the following entries is returned:
    "Seeds": ndarray where the n^th column is the n^th seed
    "Updates": ndarray where the n^th element is the number of system updates for the n^th seed
    "Optima": ndarray where the n^th column is the optimum from the n^th seed

    """
    seeds, updates, optima = [], [], []
    start = time.clock()
    for repeat in it.count(1):

        # Check termination criteria
        if stop_time is not None and time.clock() >= stop_time: break
        if max_repeats is not None and repeat >= max_repeats: break

        # get random initial seed anywhere in range
        v = sampler()
        seeds.append(v)

        # iterate trajectory a random number of steps
        num_updates = np.random.randint(max_updates)
        for u in range(num_updates):
            v = v + f(v)
        updates.append(num_updates)

        # run minimization
        result = spo.minimize(qg, v.flatten(), method='trust-ncg', jac=True, hess=H)
        optimum = result.x.reshape(v.shape)
        optima.append(optimum)

    # Format output
    return {
        "Seeds": np.concatenate(seeds, axis=1),
        "Updates": np.concatenate(updates),
        "Optima": np.concatenate(optima, axis=1),
    }

def fiber_solver(
    f,
    ef,
    Df,
    compute_step_amount,
    v=None,
    c=None,
    z=None,
    N=None,
    terminate=None,
    logfile=None,
    stop_time=None,
    max_traverse_steps=None,
    max_step_size=None,
    max_solve_iterations=None,
    solve_tolerance=None,
    ):
    """
    Fixed point location using directional fibers.
    All parameters are as described in directional_fibers.traverse_fiber().
    Returns solution, a dictionary with keys
        "Fiber": the result of traverse_fiber()
        "Fixed points": an array with one fixed point per column
        "Refinements": a list of local traverse_fiber results, one per fixed point
        "Fixed index": an array of fixed point candidate indices in the fiber
    """

    # Traverse fiber
    fiber_result = df.traverse_fiber(
        f,
        Df,
        compute_step_amount,
        v=v,
        c=c,
        z=z,
        N=N,
        terminate=terminate,
        logfile=logfile,
        stop_time=stop_time,
        max_traverse_steps=max_traverse_steps,
        max_step_size=max_step_size,
        max_solve_iterations=max_solve_iterations,
        solve_tolerance=solve_tolerance,
    )

    # Keep final direction vector in case random default was used (in theory shouldn't matter)
    c = fiber_result["c"]

    # Extract candidate fixed points: endpoints, local |alpha| minimum, alpha sign change
    # As well as immediate neighbors for close pairs of fixed points and added redundancy
    X = fiber_result["X"]
    a = X[-1,:]
    fixed_index = np.zeros(len(a), dtype=bool)
    # endpoints
    fixed_index[[0, -1]] = True
    # local magnitude minima
    fixed_index[1:-1] |= (np.fabs(a[1:-1]) <= np.fabs(a[2:])) & (np.fabs(a[1:-1]) <= np.fabs(a[:-2]))
    # sign changes
    fixed_index[:-1] |= np.sign(a[:-1]) != np.sign(a[1:])
    # extra redundancy with neighbors
    fixed_index[:-1] = np.logical_or(fixed_index[:-1], fixed_index[1:])
    fixed_index[1:] = np.logical_or(fixed_index[1:], fixed_index[:-1])

    # Set up within-fiber Newton-Raphson step computation
    def compute_refine_step_amount(x, DF, z):
        refine_step_amount = -x[-1,0]/z[-1,0]
        fiber_step_amount, fiber_step_data = compute_step_amount(x, DF, z)
        step_amount = np.sign(refine_step_amount)*min(np.fabs(refine_step_amount), fiber_step_amount)
        step_data = (refine_step_amount, fiber_step_amount, fiber_step_data)
        return step_amount, step_data
    refine_terminate = lambda x: fx.is_fixed(x[:-1,:], f, ef)[0] or terminate(x)

    # Run within-fiber Newton-Raphson at each candidate
    X = X[:, fixed_index]
    refinement_results = []
    fixed_points = []
    for i in range(X.shape[1]):
        refinement_result = df.traverse_fiber(
            f,
            Df,
            compute_refine_step_amount,
            v=X[:-1,[i]].copy(),
            c=c,
            terminate=refine_terminate,
            logfile=logfile,
            stop_time=stop_time,
            max_traverse_steps=2**5, # few steps needed for Newton-Raphson
            max_step_size=max_step_size,
            max_solve_iterations=max_solve_iterations,
            solve_tolerance=solve_tolerance,
        )
        refinement_results.append(refinement_result)
        fixed_points.append(refinement_result["X"][:-1,[-1]].copy())

    # Return output
    solution = {
        "Fiber": fiber_result,
        "Fixed points": np.concatenate(fixed_points,axis=1),
        "Refinements": refinement_results,
        "Fixed index": np.flatnonzero(fixed_index),
    }
    return solution
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# Parse the YAML file, start the testrunners in parallel,
# and wait for them.

import os
import sys
import traceback
import threading
import subprocess

from yaml.scanner import ScannerError
from pykwalify.errors import SchemaError

import utils.parser as parser
import utils.common as common
import utils.ghupdate as ghupdate

def main():
    "Main entry point."

    try:
        suites = parse_suites()
    except ScannerError:
        update_gh('error', "Red Hat CI", "YAML syntax error.")
    except SchemaError as e:
        # print the error to give feedback in the logs, but exit nicely
        traceback.print_exc()
        update_gh('error', "Red Hat CI", "YAML semantic error.")
    else:
        n = len(suites)
        if n > 0:
            spawn_testrunners(n)
            inspect_suite_failures(suites)
            update_required_context(suites)
        else:
            print("INFO: No testsuites to run.")

def parse_suites():

    yml_file = os.path.join('checkouts',
                            os.environ['github_repo'],
                            '.redhat-ci.yml')

    # this should have been checked already
    assert os.path.isfile(yml_file)

    suites = []
    branch = os.environ.get('github_branch')
    suite_parser = parser.SuiteParser(yml_file)
    for idx, suite in enumerate(suite_parser.parse()):
        if len(os.environ.get('RHCI_DEBUG_ALWAYS_RUN', '')) == 0:
            branches = suite.get('branches', ['master'])
            if branch is not None and branch not in branches:
                print("INFO: %s suite not defined to run for branch %s." %
                      (common.ordinal(idx + 1), branch))
                continue
        suite_dir = 'state/suite-%d/parsed' % len(suites)
        parser.flush_suite(suite, suite_dir)
        suites.append(suite)

    return suites

def spawn_testrunners(n):

    testrunner = os.path.join(sys.path[0], "testrunner")

    runners = []
    threads = []
    for i in range(n):
        p = subprocess.Popen([testrunner, str(i)],
                             stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT)
        t = threading.Thread(target=read_pipe,
                             args=(i, p.stdout))
        t.start()
        runners.append(p)
        threads.append(t)

    # We don't implement any fail fast here, so just do a
    # naive wait to collect them all.
    failed = False
    for runner in runners:
        if runner.wait() != 0:
            failed = True

    for thread in threads:
        thread.join()

    # NB: When we say 'failed' here, we're talking about
    # infrastructure failure. Bad PR code should never cause
    # rc != 0.
    if failed:
        raise Exception("at least one runner failed")

def read_pipe(idx, fd):
    s = fd.readline()
    while s != b'':
        if not s.endswith(b'\n'):
            s += b'\n'
            # pylint: disable=no-member
        sys.stdout.buffer.write((b'[%d] ' % idx) + s)
        s = fd.readline()

def inspect_suite_failures(suites):

    for i, suite in enumerate(suites):
        assert 'rc' not in suite

        # If the rc file doesn't exist but the runner exited
        # nicely, then it means there was a semantic error
        # in the YAML (e.g. bad Docker image, bad ostree
        # revision, etc...).
        if not os.path.isfile("state/suite-%d/rc" % i):
            suite['rc'] = 1
        else:
            with open("state/suite-%d/rc" % i) as f:
                suite['rc'] = int(f.read().strip())

    # It's helpful to have an easy global way to figure out
    # if any of the suites failed, e.g. for integration in
    # Jenkins. Let's write a 'failures' file counting the
    # number of failed suites.
    with open("state/failures", "w") as f:
        f.write("%d" % count_failures(suites))

def count_failures(suites):
    return sum([int(suite['rc'] != 0) for suite in suites])

def update_required_context(suites):

    # only send 'required' context for branches
    if 'github_pull_id' in os.environ:
        return

    required_suites = [suite for suite in suites if suite.get('required')]
    total = len(required_suites)

    if total == 0:
        return

    failed = count_failures(required_suites)
    update_gh('success' if failed == 0 else 'failure', 'required',
              "%d/%d required testsuites passed" % (total - failed, total))

def update_gh(state, context, description):

    try:
        args = {'repo': os.environ['github_repo'],
                'commit': os.environ['github_commit'],
                'token': os.environ['github_token'],
                'state': state,
                'context': context,
                'description': description}

        ghupdate.send(**args)

        if os.path.isfile('state/is_merge_sha'):
            with open('state/sha') as f:
                args['commit'] = f.read().strip()
            ghupdate.send(**args)

    # it can happen that the commit doesn't even exist
    # anymore, so let's be tolerant of such errors
    except ghupdate.CommitNotFoundException:
        pass

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>spawner.py: abbreviate required status description<commit_after>#!/usr/bin/env python3

# Parse the YAML file, start the testrunners in parallel,
# and wait for them.

import os
import sys
import traceback
import threading
import subprocess

from yaml.scanner import ScannerError
from pykwalify.errors import SchemaError

import utils.parser as parser
import utils.common as common
import utils.ghupdate as ghupdate

def main():
    "Main entry point."

    try:
        suites = parse_suites()
    except ScannerError:
        update_gh('error', "Red Hat CI", "YAML syntax error.")
    except SchemaError as e:
        # print the error to give feedback in the logs, but exit nicely
        traceback.print_exc()
        update_gh('error', "Red Hat CI", "YAML semantic error.")
    else:
        n = len(suites)
        if n > 0:
            spawn_testrunners(n)
            inspect_suite_failures(suites)
            update_required_context(suites)
        else:
            print("INFO: No testsuites to run.")

def parse_suites():

    yml_file = os.path.join('checkouts',
                            os.environ['github_repo'],
                            '.redhat-ci.yml')

    # this should have been checked already
    assert os.path.isfile(yml_file)

    suites = []
    branch = os.environ.get('github_branch')
    suite_parser = parser.SuiteParser(yml_file)
    for idx, suite in enumerate(suite_parser.parse()):
        if len(os.environ.get('RHCI_DEBUG_ALWAYS_RUN', '')) == 0:
            branches = suite.get('branches', ['master'])
            if branch is not None and branch not in branches:
                print("INFO: %s suite not defined to run for branch %s." %
                      (common.ordinal(idx + 1), branch))
                continue
        suite_dir = 'state/suite-%d/parsed' % len(suites)
        parser.flush_suite(suite, suite_dir)
        suites.append(suite)

    return suites

def spawn_testrunners(n):

    testrunner = os.path.join(sys.path[0], "testrunner")

    runners = []
    threads = []
    for i in range(n):
        p = subprocess.Popen([testrunner, str(i)],
                             stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT)
        t = threading.Thread(target=read_pipe,
                             args=(i, p.stdout))
        t.start()
        runners.append(p)
        threads.append(t)

    # We don't implement any fail fast here, so just do a
    # naive wait to collect them all.
    failed = False
    for runner in runners:
        if runner.wait() != 0:
            failed = True

    for thread in threads:
        thread.join()

    # NB: When we say 'failed' here, we're talking about
    # infrastructure failure. Bad PR code should never cause
    # rc != 0.
    if failed:
        raise Exception("at least one runner failed")

def read_pipe(idx, fd):
    s = fd.readline()
    while s != b'':
        if not s.endswith(b'\n'):
            s += b'\n'
            # pylint: disable=no-member
        sys.stdout.buffer.write((b'[%d] ' % idx) + s)
        s = fd.readline()

def inspect_suite_failures(suites):

    for i, suite in enumerate(suites):
        assert 'rc' not in suite

        # If the rc file doesn't exist but the runner exited
        # nicely, then it means there was a semantic error
        # in the YAML (e.g. bad Docker image, bad ostree
        # revision, etc...).
        if not os.path.isfile("state/suite-%d/rc" % i):
            suite['rc'] = 1
        else:
            with open("state/suite-%d/rc" % i) as f:
                suite['rc'] = int(f.read().strip())

    # It's helpful to have an easy global way to figure out
    # if any of the suites failed, e.g. for integration in
    # Jenkins. Let's write a 'failures' file counting the
    # number of failed suites.
    with open("state/failures", "w") as f:
        f.write("%d" % count_failures(suites))

def count_failures(suites):
    return sum([int(suite['rc'] != 0) for suite in suites])

def update_required_context(suites):

    # only send 'required' context for branches
    if 'github_pull_id' in os.environ:
        return

    required_suites = [suite for suite in suites if suite.get('required')]
    total = len(required_suites)

    if total == 0:
        return

    failed = count_failures(required_suites)
    update_gh('success' if failed == 0 else 'failure', 'required',
              "%d/%d PASSES" % (total - failed, total))

def update_gh(state, context, description):

    try:
        args = {'repo': os.environ['github_repo'],
                'commit': os.environ['github_commit'],
                'token': os.environ['github_token'],
                'state': state,
                'context': context,
                'description': description}

        ghupdate.send(**args)

        if os.path.isfile('state/is_merge_sha'):
            with open('state/sha') as f:
                args['commit'] = f.read().strip()
            ghupdate.send(**args)

    # it can happen that the commit doesn't even exist
    # anymore, so let's be tolerant of such errors
    except ghupdate.CommitNotFoundException:
        pass

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before># -*- coding: utf-8-*-
"""
A Speaker handles audio output from Jasper to the user

Speaker methods:
    say - output 'phrase' as speech
    play - play the audio in 'filename'
    isAvailable - returns True if the platform supports this implementation
"""
import os
import json

class eSpeakSpeaker:

    """
    Uses the eSpeak speech synthesizer included in the Jasper disk image
    """
    @classmethod
    def isAvailable(cls):
        return os.system("which espeak") == 0

    def say(self, phrase, OPTIONS=" -vdefault+m3 -p 40 -s 160 --stdout > say.wav"):
        os.system("espeak " + json.dumps(phrase) + OPTIONS)
        self.play("say.wav")

    def play(self, filename):
        os.system("aplay " + filename)

class saySpeaker:

    """
    Uses the OS X built-in 'say' command
    """

    @classmethod
    def isAvailable(cls):
        return os.system("which say") == 0

    def shellquote(self, s):
        return "'" + s.replace("'", "'\\''") + "'"

    def say(self, phrase):
        os.system("say " + self.shellquote(phrase))

    def play(self, filename):
        os.system("afplay " + filename)

def newSpeaker():
    """
    Returns:
        A speaker implementation available on the current platform

    Raises:
        ValueError if no speaker implementation is supported on this platform
    """

    for cls in [eSpeakSpeaker, saySpeaker]:
        if cls.isAvailable():
            return cls()
    raise ValueError("Platform is not supported")
<commit_msg>test start<commit_after># -*- coding: utf-8-*-
"""
A Speaker handles audio output from Jasper to the user

Speaker methods:
    say - output 'phrase' as speech
    play - play the audio in 'filename'
    isAvailable - returns True if the platform supports this implementation
"""
import os
import json

class eSpeakSpeaker:

    """
    Uses the eSpeak speech synthesizer included in the Jasper disk image
    """
    @classmethod
    def isAvailable(cls):
        return os.system("which espeak") == 0

    def say(self, phrase, OPTIONS=" -vdefault+m3 -p 40 -s 160 --stdout > say.wav"):
        os.system("sudo espeak " + json.dumps(phrase) + OPTIONS)
        self.play("sudo say.wav")

    def play(self, filename):
        os.system("sudo aplay " + filename)

class saySpeaker:

    """
    Uses the OS X built-in 'say' command
    """

    @classmethod
    def isAvailable(cls):
        return os.system("which say") == 0

    def shellquote(self, s):
        return "'" + s.replace("'", "'\\''") + "'"

    def say(self, phrase):
        os.system("say " + self.shellquote(phrase))

    def play(self, filename):
        os.system("afplay " + filename)

def newSpeaker():
    """
    Returns:
        A speaker implementation available on the current platform

    Raises:
        ValueError if no speaker implementation is supported on this platform
    """

    for cls in [eSpeakSpeaker, saySpeaker]:
        if cls.isAvailable():
            return cls()
    raise ValueError("Platform is not supported")
<|endoftext|>