<commit_before>import numpy as np
from tabulate import tabulate
from PyAnalysisTools.base import _logger
from PyAnalysisTools.base.YAMLHandle import YAMLLoader
from PyAnalysisTools.ROOTUtils.FileHandle import FileHandle as FH
from PyAnalysisTools.PlottingUtils.HistTools import scale
from PyAnalysisTools.AnalysisTools.XSHandle import XSHandle
from PyAnalysisTools.PlottingUtils.PlotConfig import parse_and_build_process_config
from numpy.lib.recfunctions import rec_append_fields
import ROOT

class CutflowAnalyser(object):
    """
    Cutflow analyser
    """
    def __init__(self, **kwargs):
        kwargs.setdefault("output_file_name", None)
        kwargs.setdefault("lumi", None)
        kwargs.setdefault("process_config", None)
        kwargs.setdefault("no_merge", False)
        self.file_list = kwargs["file_list"]
        self.cutflow_hists = dict()
        self.cutflow_hists = dict()
        self.cutflow_tables = dict()
        self.dataset_config_file = kwargs["dataset_config"]
        self.lumi = kwargs["lumi"]
        self.output_file_name = kwargs["output_file_name"]
        self.systematics = kwargs["systematics"]
        self.cutflow_hists = dict()
        self.cutflows = dict()
        self.xs_handle = XSHandle(kwargs["dataset_config"])
        self.event_numbers = dict()
        self.process_config = None
        self.merge = True if not kwargs["no_merge"] else False
        if kwargs["process_config"] is not None:
            self.process_config = parse_and_build_process_config(kwargs["process_config"])

    def apply_cross_section_weight(self):
        for process in self.cutflow_hists.keys():
            lumi_weight = self.get_cross_section_weight(process)
            for systematic in self.cutflow_hists[process].keys():
                for cutflow in self.cutflow_hists[process][systematic].values():
                    scale(cutflow, lumi_weight)

    def analyse_cutflow(self):
        self.apply_cross_section_weight()
        if self.process_config is not None and self.merge:
            self.merge_histograms(self.cutflow_hists)
        for systematic in self.systematics:
            self.cutflows[systematic] = dict()
            for process in self.cutflow_hists.keys():
                self.cutflows[systematic][process] = dict()
                for k, v in self.cutflow_hists[process][systematic].iteritems():
                    if k.endswith("_raw"):
                        continue
                    raw_cutflow = self.cutflow_hists[process][systematic][k+"_raw"]
                    self.cutflows[systematic][process][k] = self._analyse_cutflow(v, raw_cutflow, process)
        self.calculate_cut_efficiencies()

    def merge_histograms(self, hist_dict):
        def merge(histograms):
            for process, process_config in self.process_config.iteritems():
                if not hasattr(process_config, "subprocesses"):
                    continue
                for sub_process in process_config.subprocesses:
                    if sub_process not in histograms.keys():
                        continue
                    for systematic in histograms[sub_process].keys():
                        for selection in histograms[sub_process][systematic].keys():
                            if not process in histograms.keys():
                                histograms[process] = dict((syst,
                                                            dict((sel, None) for sel in histograms[sub_process][syst].keys()))
                                                           for syst in histograms[sub_process].keys())
                            if histograms[process][systematic][selection] is None:
                                new_hist_name = histograms[sub_process][systematic][selection].GetName().replace(sub_process, process)
                                histograms[process][systematic][selection] = histograms[sub_process][systematic][selection].Clone(new_hist_name)
                            else:
                                histograms[process][systematic][selection].Add(histograms[sub_process][systematic][selection].Clone(new_hist_name))
                    histograms.pop(sub_process)

        merge(hist_dict)

    def get_cross_section_weight(self, process):
        if self.lumi is None or "data" in process.lower():
            return 1.
        lumi_weight = self.xs_handle.get_lumi_scale_factor(process, self.lumi, self.event_numbers[process])
        _logger.debug("Retrieved %.2f as cross section weight for process %s and lumi %.2f" % (lumi_weight, process,
                                                                                self.lumi))
        return lumi_weight

    def _analyse_cutflow(self, cutflow_hist, raw_cutflow_hist, process=None):
        parsed_info = np.array([(cutflow_hist.GetXaxis().GetBinLabel(b),
                                 cutflow_hist.GetBinContent(b),
                                 #raw_cutflow_hist.GetBinContent(b),
                                 cutflow_hist.GetBinError(b),
                                 #raw_cutflow_hist.GetBinError(b),
                                 -1.,
                                 -1.) for b in range(0, cutflow_hist.GetNbinsX() + 1)],
                               dtype=[("cut", "S100"), ("yield", "f4"), #("yield_raw", "f4"),
                                       ("yield_unc", "f4"),
                                      ("eff", float),
                                      ("eff_total", float)])  # todo: array dtype for string not a good choice

        #("yield_raw_unc", float),
        return parsed_info

    def calculate_cut_efficiencies(self):
        for systematic in self.systematics:
            for process in self.cutflows[systematic].keys():
                for cutflow in self.cutflows[systematic][process].values():
                    CutflowAnalyser.calculate_cut_efficiency(cutflow)

    @staticmethod
    def calculate_cut_efficiency(cutflow):
        for i in range(len(cutflow["cut"])):
            if i == 0:
                cutflow[i]["eff"] = 100.
                cutflow[i]["eff_total"] = 100.
                continue
            if cutflow[i-1]["yield"] != 0.:
                cutflow[i]["eff"] = round(100.0*cutflow[i]["yield"]/cutflow[i-1]["yield"], 3)
            else:
                cutflow[i]["eff"] = -1.
            if cutflow[0]["yield"] != 0.:
                cutflow[i]["eff_total"] = round(100.0*cutflow[i]["yield"]/cutflow[0]["yield"], 3)
            else:
                cutflow[i]["eff_total"] = -1.

    def make_cutflow_tables(self):
        for systematic in self.systematics:
            self.make_cutflow_table(systematic)

    def make_cutflow_table(self, systematic):
        cutflow_tables = dict()
        for process in self.cutflows[systematic].keys():
            for selection, cutflow in self.cutflows[systematic][process].items():
                cutflow_tmp = self.stringify(cutflow)
                if selection not in cutflow_tables.keys():
                    cutflow_tables[selection] = cutflow_tmp
                    continue
                cutflow_tables[selection] = rec_append_fields(cutflow_tables[selection],
                                                              [i+process for i in cutflow_tmp.dtype.names[1:]],
                                                              [cutflow_tmp[n] for n in cutflow_tmp.dtype.names[1:]])
            self.cutflow_tables= {k: tabulate(v.transpose(), headers=self.cutflows[systematic].keys()) for k, v in cutflow_tables.iteritems()}

    @staticmethod
    def stringify(cutflow):
        def format_yield(value, uncertainty):
            if value > 10000.:
                return "{:.3e} +- {:.3e}".format(value, uncertainty)
            else:
                return "{:.2f} +- {:.2f}".format(value, uncertainty)

        cutflow = np.array([(cutflow[i]["cut"],
                             format_yield(cutflow[i]["yield"], cutflow[i]["yield_unc"]),
                             #"%.2f +- %.2f" % (cutflow[i]["yield_raw"], cutflow[i]["yield_raw_unc"]),
                             cutflow[i]["eff"],
                             cutflow[i]["eff_total"]) for i in range(len(cutflow))],
                           dtype=[("cut", "S100"), ("yield", "S100"), ("eff", float), ("eff_total", float)])
                           #dtype=[("cut", "S100"), ("yield", "S100"), ("yield_raw", "S100"), ("eff", float), ("eff_total", float)])
        return cutflow

    def print_cutflow_table(self):
        for selection, cutflow in self.cutflow_tables.iteritems():
            if not selection == "BaseSelection":
                continue
            print
            print "Cutflow for region %s" % selection
            print cutflow

    def store_cutflow_table(self):
        pass

    def load_cutflows(self, file_name):
        file_handle = FH(file_name=file_name, dataset_info=self.dataset_config_file)
        process = file_handle.process
        if process not in self.event_numbers:
            self.event_numbers[process] = file_handle.get_number_of_total_events()
        else:
            self.event_numbers[process] += file_handle.get_number_of_total_events()
        if process not in self.cutflow_hists.keys():
            self.cutflow_hists[process] = dict()
        for systematic in self.systematics:
            cutflow_hists = file_handle.get_objects_by_pattern("^(cutflow_)", systematic)
            if systematic not in self.cutflow_hists[process]:
                self.cutflow_hists[process][systematic] = dict()
            for cutflow_hist in cutflow_hists:
                cutflow_hist.SetDirectory(0)
                try:
                    self.cutflow_hists[process][systematic][cutflow_hist.GetName().replace("cutflow_", "")].Add(cutflow_hist)
                except KeyError:
                    self.cutflow_hists[process][systematic][cutflow_hist.GetName().replace("cutflow_", "")] = cutflow_hist

    def read_cutflows(self):
        for file_name in self.file_list:
            self.load_cutflows(file_name)

    def execute(self):
        self.read_cutflows()
        self.analyse_cutflow()
        self.make_cutflow_tables()
<commit_msg>implement raw cutflow support<commit_after>import numpy as np
from tabulate import tabulate
from PyAnalysisTools.base import _logger
from PyAnalysisTools.base.YAMLHandle import YAMLLoader
from PyAnalysisTools.ROOTUtils.FileHandle import FileHandle as FH
from PyAnalysisTools.PlottingUtils.HistTools import scale
from PyAnalysisTools.AnalysisTools.XSHandle import XSHandle
from PyAnalysisTools.PlottingUtils.PlotConfig import parse_and_build_process_config
from numpy.lib.recfunctions import rec_append_fields
import ROOT

class CutflowAnalyser(object):
    """
    Cutflow analyser
    """
    def __init__(self, **kwargs):
        kwargs.setdefault("output_file_name", None)
        kwargs.setdefault("lumi", None)
        kwargs.setdefault("process_config", None)
        kwargs.setdefault("no_merge", False)
        kwargs.setdefault("raw", False)
        self.file_list = kwargs["file_list"]
        self.cutflow_hists = dict()
        self.cutflow_hists = dict()
        self.cutflow_tables = dict()
        self.dataset_config_file = kwargs["dataset_config"]
        self.lumi = kwargs["lumi"]
        self.output_file_name = kwargs["output_file_name"]
        self.systematics = kwargs["systematics"]
        self.cutflow_hists = dict()
        self.cutflows = dict()
        self.xs_handle = XSHandle(kwargs["dataset_config"])
        self.event_numbers = dict()
        self.process_config = None
        self.raw = kwargs["raw"]
        self.merge = True if not kwargs["no_merge"] else False
        if kwargs["process_config"] is not None:
            self.process_config = parse_and_build_process_config(kwargs["process_config"])

    def apply_cross_section_weight(self):
        for process in self.cutflow_hists.keys():
            lumi_weight = self.get_cross_section_weight(process)
            for systematic in self.cutflow_hists[process].keys():
                for cutflow in self.cutflow_hists[process][systematic].values():
                    scale(cutflow, lumi_weight)

    def analyse_cutflow(self):
        self.apply_cross_section_weight()
        if self.process_config is not None and self.merge:
            self.merge_histograms(self.cutflow_hists)
        for systematic in self.systematics:
            self.cutflows[systematic] = dict()
            for process in self.cutflow_hists.keys():
                self.cutflows[systematic][process] = dict()
                for k, v in self.cutflow_hists[process][systematic].iteritems():
                    if k.endswith("_raw"):
                        continue
                    raw_cutflow = self.cutflow_hists[process][systematic][k+"_raw"]
                    self.cutflows[systematic][process][k] = self._analyse_cutflow(v, raw_cutflow, process)
        self.calculate_cut_efficiencies()

    def merge_histograms(self, hist_dict):
        def merge(histograms):
            for process, process_config in self.process_config.iteritems():
                if not hasattr(process_config, "subprocesses"):
                    continue
                for sub_process in process_config.subprocesses:
                    if sub_process not in histograms.keys():
                        continue
                    for systematic in histograms[sub_process].keys():
                        for selection in histograms[sub_process][systematic].keys():
                            if not process in histograms.keys():
                                histograms[process] = dict((syst,
                                                            dict((sel, None) for sel in histograms[sub_process][syst].keys()))
                                                           for syst in histograms[sub_process].keys())
                            if histograms[process][systematic][selection] is None:
                                new_hist_name = histograms[sub_process][systematic][selection].GetName().replace(sub_process, process)
                                histograms[process][systematic][selection] = histograms[sub_process][systematic][selection].Clone(new_hist_name)
                            else:
                                histograms[process][systematic][selection].Add(histograms[sub_process][systematic][selection].Clone(new_hist_name))
                    histograms.pop(sub_process)

        merge(hist_dict)

    def get_cross_section_weight(self, process):
        if self.lumi is None or "data" in process.lower():
            return 1.
        lumi_weight = self.xs_handle.get_lumi_scale_factor(process, self.lumi, self.event_numbers[process])
        _logger.debug("Retrieved %.2f as cross section weight for process %s and lumi %.2f" % (lumi_weight, process,
                                                                                self.lumi))
        return lumi_weight

    def _analyse_cutflow(self, cutflow_hist, raw_cutflow_hist, process=None):
        if not self.raw:
            parsed_info = np.array([(cutflow_hist.GetXaxis().GetBinLabel(b),
                                     cutflow_hist.GetBinContent(b),
                                 #raw_cutflow_hist.GetBinContent(b),
                                 cutflow_hist.GetBinError(b),
                                 #raw_cutflow_hist.GetBinError(b),
                                 -1.,
                                 -1.) for b in range(1, cutflow_hist.GetNbinsX() + 1)],
                               dtype=[("cut", "S100"), ("yield", "f4"), #("yield_raw", "f4"),
                                       ("yield_unc", "f4"),
                                      ("eff", float),
                                      ("eff_total", float)])  # todo: array dtype for string not a good choice
        else:
            parsed_info = np.array([(cutflow_hist.GetXaxis().GetBinLabel(b),
                                     raw_cutflow_hist.GetBinContent(b),
                                     raw_cutflow_hist.GetBinError(b),
                                     -1.,
                                     -1.) for b in range(1, cutflow_hist.GetNbinsX() + 1)],
                               dtype=[("cut", "S100"), ("yield_raw", "f4"),
                                       ("yield_unc_raw", "f4"),
                                      ("eff", float),
                                      ("eff_total", float)])  # todo: array dtype for string not a good choice

        #("yield_raw_unc", float),
        return parsed_info

    def calculate_cut_efficiencies(self):
        for systematic in self.systematics:
            for process in self.cutflows[systematic].keys():
                for cutflow in self.cutflows[systematic][process].values():
                    self.calculate_cut_efficiency(cutflow)

    def calculate_cut_efficiency(self, cutflow):
        yield_str = "yield"
        if self.raw:
            yield_str = "yield_raw"
        for i in range(len(cutflow["cut"])):
            if i == 0:
                cutflow[i]["eff"] = 100.
                cutflow[i]["eff_total"] = 100.
                continue
            if cutflow[i-1][yield_str] != 0.:
                cutflow[i]["eff"] = round(100.0*cutflow[i][yield_str]/cutflow[i-1][yield_str], 3)
            else:
                cutflow[i]["eff"] = -1.
            if cutflow[0][yield_str] != 0.:
                cutflow[i]["eff_total"] = round(100.0*cutflow[i][yield_str]/cutflow[0][yield_str], 3)
            else:
                cutflow[i]["eff_total"] = -1.

    def make_cutflow_tables(self):
        for systematic in self.systematics:
            self.make_cutflow_table(systematic)

    def make_cutflow_table(self, systematic):
        cutflow_tables = dict()
        for process in self.cutflows[systematic].keys():
            for selection, cutflow in self.cutflows[systematic][process].items():
                cutflow_tmp = self.stringify(cutflow)
                if selection not in cutflow_tables.keys():
                    cutflow_tables[selection] = cutflow_tmp
                    continue
                cutflow_tables[selection] = rec_append_fields(cutflow_tables[selection],
                                                              [i+process for i in cutflow_tmp.dtype.names[1:]],
                                                              [cutflow_tmp[n] for n in cutflow_tmp.dtype.names[1:]])
            self.cutflow_tables= {k: tabulate(v.transpose(), headers=self.cutflows[systematic].keys()) for k, v in cutflow_tables.iteritems()}

    def stringify(self, cutflow):
        def format_yield(value, uncertainty):
            if value > 10000.:
                return "{:.3e} +- {:.3e}".format(value, uncertainty)
            else:
                return "{:.2f} +- {:.2f}".format(value, uncertainty)

        if not self.raw:
            cutflow = np.array([(cutflow[i]["cut"],
                                 format_yield(cutflow[i]["yield"], cutflow[i]["yield_unc"]),
                                 cutflow[i]["eff"],
                                 cutflow[i]["eff_total"]) for i in range(len(cutflow))],
                               dtype=[("cut", "S100"), ("yield", "S100"), ("eff", float), ("eff_total", float)])
        else:
            cutflow = np.array([(cutflow[i]["cut"],
                                 format_yield(cutflow[i]["yield_raw"], cutflow[i]["yield_unc_raw"]),
                                 cutflow[i]["eff"],
                                 cutflow[i]["eff_total"]) for i in range(len(cutflow))],
                               dtype=[("cut", "S100"), ("yield_raw", "S100"), ("eff", float), ("eff_total", float)])

        return cutflow

    def print_cutflow_table(self):
        for selection, cutflow in self.cutflow_tables.iteritems():
            if not selection == "BaseSelection":
                continue
            print
            print "Cutflow for region %s" % selection
            print cutflow

    def store_cutflow_table(self):
        pass

    def load_cutflows(self, file_name):
        file_handle = FH(file_name=file_name, dataset_info=self.dataset_config_file)
        process = file_handle.process
        if process not in self.event_numbers:
            self.event_numbers[process] = file_handle.get_number_of_total_events()
        else:
            self.event_numbers[process] += file_handle.get_number_of_total_events()
        if process not in self.cutflow_hists.keys():
            self.cutflow_hists[process] = dict()
        for systematic in self.systematics:
            cutflow_hists = file_handle.get_objects_by_pattern("^(cutflow_)", systematic)
            if systematic not in self.cutflow_hists[process]:
                self.cutflow_hists[process][systematic] = dict()
            for cutflow_hist in cutflow_hists:
                cutflow_hist.SetDirectory(0)
                try:
                    self.cutflow_hists[process][systematic][cutflow_hist.GetName().replace("cutflow_", "")].Add(cutflow_hist)
                except KeyError:
                    self.cutflow_hists[process][systematic][cutflow_hist.GetName().replace("cutflow_", "")] = cutflow_hist

    def read_cutflows(self):
        for file_name in self.file_list:
            self.load_cutflows(file_name)

    def execute(self):
        self.read_cutflows()
        self.analyse_cutflow()
        self.make_cutflow_tables()
<|endoftext|>


<commit_before>'''
------------------------------------------------------------------------
Last updated 6/19/2015

This will run the steady state solver as well as time path iteration.

This py-file calls the following other file(s):
            income_polynomials.py
            demographics.py
            wealth_data.py
            labor_data.py
            SS.py
            TPI.py

This py-file creates the following other file(s):
    (make sure that an OUTPUT folder exists)
            OUTPUT/Saved_moments/params_given.pkl
            OUTPUT/Saved_moments/params_changed.pkl
------------------------------------------------------------------------
'''

'''
------------------------------------------------------------------------
Import Packages
------------------------------------------------------------------------
'''

import cPickle as pickle
import os
from glob import glob
from subprocess import call

import numpy as np

import income_polynomials as income
import demographics
import wealth_data
import labor_data

'''
------------------------------------------------------------------------
Setting up the Model
------------------------------------------------------------------------
S            = number of periods an individual lives
J            = number of different ability groups
T            = number of time periods until steady state is reached
lambdas  = desired percentiles of ability groups
scal         = scalar multiplier used in SS files to make the initial value work
starting_age = age of first members of cohort
ending age   = age of the last members of cohort
E            = number of cohorts before S=1
beta_annual  = discount factor for one year
beta         = discount factor for each age cohort
sigma        = coefficient of relative risk aversion
alpha        = capital share of income
Z            = total factor productivity parameter in firms' production
               function
delta_annual = depreciation rate of capital for one year
delta        = depreciation rate of capital for each cohort
ltilde       = measure of time each individual is endowed with each
               period
g_y_annual   = annual growth rate of technology
g_y          = growth rate of technology for one cohort
slow_work    = time at which chi_n starts increasing from 1
chi_n_multiplier = scalar which is increased to force the labor
               distribution to 0
maxiter   = Maximum number of iterations that TPI will undergo
mindist_SS   = Cut-off distance between iterations for SS
mindist_TPI   = Cut-off distance between iterations for TPI
nu           = contraction parameter in steady state iteration process
               representing the weight on the new distribution gamma_nu
b_ellipse    = value of b for elliptical fit of utility function
k_ellipse    = value of k for elliptical fit of utility function
upsilon      = value of omega for elliptical fit of utility function
mean_income_data  = mean income from IRS data file used to calibrate income tax
               (scalar)
a_tax_income = used to calibrate income tax (scalar)
b_tax_income = used to calibrate income tax (scalar)
c_tax_income = used to calibrate income tax (scalar)
d_tax_income = used to calibrate income tax (scalar)
retire       = age in which individuals retire(scalar)
h_wealth     = wealth tax parameter h
m_wealth     = wealth tax parameter m
p_wealth     = wealth tax parameter p
tau_bq       = bequest tax (scalar)
tau_payroll  = payroll tax (scalar)
theta    = payback value for payroll tax (scalar)
e            = S x J matrix of age dependent possible working abilities
               e_s
omega        = T x S x J array of demographics
g_n_ss          = steady state population growth rate
omega_SS     = steady state population distribution
surv_rate    = S x 1 array of survival rates
rho    = S x 1 array of mortality rates
------------------------------------------------------------------------
'''

# Parameters
S = 80
J = 7
T = int(2 * S)
lambdas = np.array([.25, .25, .2, .1, .1, .09, .01])
starting_age = 20
ending_age = 100
E = int(starting_age * (S / float(ending_age-starting_age)))
beta_annual = .96
beta = beta_annual ** (float(ending_age-starting_age) / S)
sigma = 3.0
alpha = .35
Z = 1.0
delta_annual = .05
delta = 1 - ((1-delta_annual) ** (float(ending_age-starting_age) / S))
ltilde = 1.0
g_y_annual = 0.03
g_y = (1 + g_y_annual)**(float(ending_age-starting_age)/S) - 1
# TPI parameters
maxiter = 250
mindist_SS = 1e-9
mindist_TPI = 1e-6
nu = .40
# Ellipse parameters
b_ellipse = 25.6594
k_ellipse = -26.4902
upsilon = 3.0542
# Tax parameters:
mean_income_data = 84377.0
a_tax_income = 3.03452713268985e-06
b_tax_income = .222
c_tax_income = 133261.0
d_tax_income = .219
retire = np.round(9.0 * S / 16.0) - 1
# Wealth tax params
# These won't be used for the wealth tax, h and m just need
# need to be nonzero to avoid errors
h_wealth = 0.1
m_wealth = 1.0
p_wealth = 0.0
# Tax parameters that are zeroed out for SS
# Initial taxes below
tau_bq = np.zeros(J)
tau_payroll = 0.15
# Flag to prevent graphing from occuring in demographic, income, wealth, and labor files
flag_graphs = False
# Generate Income and Demographic parameters
omega, g_n_ss, omega_SS, surv_rate, rho, g_n_vector = demographics.get_omega(
    S, T, starting_age, ending_age, E, flag_graphs)

e = income.get_e(S, J, starting_age, ending_age, lambdas, omega_SS, flag_graphs)

# Calibration parameters
calibrate_model = False
chi_b_guess = np.array([2, 10, 90, 350, 1700, 22000, 120000])
chi_n_guess = np.array([47.12000874 , 22.22762421 , 14.34842241 , 10.67954008 ,  8.41097278
                         ,  7.15059004 ,  6.46771332 ,  5.85495452 ,  5.46242013 ,  5.00364263
                         ,  4.57322063 ,  4.53371545 ,  4.29828515 ,  4.10144524 ,  3.8617942  ,  3.57282
                         ,  3.47473172 ,  3.31111347 ,  3.04137299 ,  2.92616951 ,  2.58517969
                         ,  2.48761429 ,  2.21744847 ,  1.9577682  ,  1.66931057 ,  1.6878927
                         ,  1.63107201 ,  1.63390543 ,  1.5901486  ,  1.58143606 ,  1.58005578
                         ,  1.59073213 ,  1.60190899 ,  1.60001831 ,  1.67763741 ,  1.70451784
                         ,  1.85430468 ,  1.97291208 ,  1.97017228 ,  2.25518398 ,  2.43969757
                         ,  3.21870602 ,  4.18334822 ,  4.97772026 ,  6.37663164 ,  8.65075992
                         ,  9.46944758 , 10.51634777 , 12.13353793 , 11.89186997 , 12.07083882
                         , 13.2992811  , 14.07987878 , 14.19951571 , 14.97943562 , 16.05601334
                         , 16.42979341 , 16.91576867 , 17.62775142 , 18.4885405  , 19.10609921
                         , 20.03988031 , 20.86564363 , 21.73645892 , 22.6208256  , 23.37786072
                         , 24.38166073 , 25.22395387 , 26.21419653 , 27.05246704 , 27.86896121
                         , 28.90029708 , 29.83586775 , 30.87563699 , 31.91207845 , 33.07449767
                         , 34.27919965 , 35.57195873 , 36.95045988 , 38.62308152])

# Generate Wealth data moments
wealth_data.get_wealth_data(lambdas, J, flag_graphs)

# Generate labor data moments

labor_data.labor_data_moments(flag_graphs)

get_baseline = True

# List of parameter names that will not be changing (unless we decide to
# change them for a tax experiment.
param_names = ['S', 'J', 'T', 'lambdas', 'starting_age', 'ending_age',
             'beta', 'sigma', 'alpha', 'nu', 'Z', 'delta', 'E',
             'ltilde', 'g_y', 'maxiter', 'mindist_SS', 'mindist_TPI',
             'b_ellipse', 'k_ellipse', 'upsilon',
             'a_tax_income', 'chi_b_guess', 'chi_n_guess',
             'b_tax_income', 'c_tax_income', 'd_tax_income',
             'tau_payroll', 'tau_bq', 'calibrate_model',
             'retire', 'mean_income_data', 'g_n_vector',
             'h_wealth', 'p_wealth', 'm_wealth', 'get_baseline',
             'omega', 'g_n_ss', 'omega_SS', 'surv_rate', 'e', 'rho']

'''
------------------------------------------------------------------------
    Run SS with minimization to fit chi_b and chi_n
------------------------------------------------------------------------
'''

# This is the simulation before getting the replacement rate values

dictionary = {}
for key in param_names:
    dictionary[key] = globals()[key]
pickle.dump(dictionary, open("OUTPUT/Saved_moments/params_given.pkl", "w"))

import SS

'''
------------------------------------------------------------------------
    Run the baseline TPI simulation
------------------------------------------------------------------------
'''

import TPI

'''
------------------------------------------------------------------------
    Alter desired tax parameters
------------------------------------------------------------------------
'''

# New Tax Parameters

# get_baseline = False
# d_tax_income = .42

# var_names = ['get_baseline', 'd_tax_income']
# dictionary = {}
# for key in var_names:
#     dictionary[key] = globals()[key]
# pickle.dump(dictionary, open("OUTPUT/Saved_moments/params_changed.pkl", "w"))

'''
------------------------------------------------------------------------
    Run SS with tax experiment
------------------------------------------------------------------------
'''

# import SS

'''
------------------------------------------------------------------------
    Run TPI for tax experiment
------------------------------------------------------------------------
'''

# import TPI

'''
------------------------------------------------------------------------
Delete all .pyc files that have been generated
------------------------------------------------------------------------
'''

files = glob('*.pyc')
for i in files:
    os.remove(i)
<commit_msg>finished TPI check of resource constraint<commit_after>'''
------------------------------------------------------------------------
Last updated 6/19/2015

This will run the steady state solver as well as time path iteration.

This py-file calls the following other file(s):
            income_polynomials.py
            demographics.py
            wealth_data.py
            labor_data.py
            SS.py
            TPI.py

This py-file creates the following other file(s):
    (make sure that an OUTPUT folder exists)
            OUTPUT/Saved_moments/params_given.pkl
            OUTPUT/Saved_moments/params_changed.pkl
------------------------------------------------------------------------
'''

'''
------------------------------------------------------------------------
Import Packages
------------------------------------------------------------------------
'''

import cPickle as pickle
import os
from glob import glob
from subprocess import call

import numpy as np

import income_polynomials as income
import demographics
import wealth_data
import labor_data

'''
------------------------------------------------------------------------
Setting up the Model
------------------------------------------------------------------------
S            = number of periods an individual lives
J            = number of different ability groups
T            = number of time periods until steady state is reached
lambdas  = desired percentiles of ability groups
scal         = scalar multiplier used in SS files to make the initial value work
starting_age = age of first members of cohort
ending age   = age of the last members of cohort
E            = number of cohorts before S=1
beta_annual  = discount factor for one year
beta         = discount factor for each age cohort
sigma        = coefficient of relative risk aversion
alpha        = capital share of income
Z            = total factor productivity parameter in firms' production
               function
delta_annual = depreciation rate of capital for one year
delta        = depreciation rate of capital for each cohort
ltilde       = measure of time each individual is endowed with each
               period
g_y_annual   = annual growth rate of technology
g_y          = growth rate of technology for one cohort
slow_work    = time at which chi_n starts increasing from 1
chi_n_multiplier = scalar which is increased to force the labor
               distribution to 0
maxiter   = Maximum number of iterations that TPI will undergo
mindist_SS   = Cut-off distance between iterations for SS
mindist_TPI   = Cut-off distance between iterations for TPI
nu           = contraction parameter in steady state iteration process
               representing the weight on the new distribution gamma_nu
b_ellipse    = value of b for elliptical fit of utility function
k_ellipse    = value of k for elliptical fit of utility function
upsilon      = value of omega for elliptical fit of utility function
mean_income_data  = mean income from IRS data file used to calibrate income tax
               (scalar)
a_tax_income = used to calibrate income tax (scalar)
b_tax_income = used to calibrate income tax (scalar)
c_tax_income = used to calibrate income tax (scalar)
d_tax_income = used to calibrate income tax (scalar)
retire       = age in which individuals retire(scalar)
h_wealth     = wealth tax parameter h
m_wealth     = wealth tax parameter m
p_wealth     = wealth tax parameter p
tau_bq       = bequest tax (scalar)
tau_payroll  = payroll tax (scalar)
theta    = payback value for payroll tax (scalar)
e            = S x J matrix of age dependent possible working abilities
               e_s
omega        = T x S x J array of demographics
g_n_ss          = steady state population growth rate
omega_SS     = steady state population distribution
surv_rate    = S x 1 array of survival rates
rho    = S x 1 array of mortality rates
------------------------------------------------------------------------
'''

# Parameters
S = 80
J = 7
T = int(2 * S)
lambdas = np.array([.25, .25, .2, .1, .1, .09, .01])
starting_age = 20
ending_age = 100
E = int(starting_age * (S / float(ending_age-starting_age)))
beta_annual = .96
beta = beta_annual ** (float(ending_age-starting_age) / S)
sigma = 3.0
alpha = .35
Z = 1.0
delta_annual = .05
delta = 1 - ((1-delta_annual) ** (float(ending_age-starting_age) / S))
ltilde = 1.0
g_y_annual = 0.03
g_y = (1 + g_y_annual)**(float(ending_age-starting_age)/S) - 1
# TPI parameters
maxiter = 250
mindist_SS = 1e-9
mindist_TPI = 1e-6
nu = .40
# Ellipse parameters
b_ellipse = 25.6594
k_ellipse = -26.4902
upsilon = 3.0542
# Tax parameters:
mean_income_data = 84377.0
a_tax_income = 3.03452713268985e-06
b_tax_income = .222
c_tax_income = 133261.0
d_tax_income = .219
retire = np.round(9.0 * S / 16.0) - 1
# Wealth tax params
# These won't be used for the wealth tax, h and m just need
# need to be nonzero to avoid errors
h_wealth = 0.1
m_wealth = 1.0
p_wealth = 0.0
# Tax parameters that are zeroed out for SS
# Initial taxes below
tau_bq = np.zeros(J)
tau_payroll = 0.15
# Flag to prevent graphing from occuring in demographic, income, wealth, and labor files
flag_graphs = False
# Generate Income and Demographic parameters
omega, g_n_ss, omega_SS, surv_rate, rho, g_n_vector = demographics.get_omega(
    S, T, starting_age, ending_age, E, flag_graphs)

e = income.get_e(S, J, starting_age, ending_age, lambdas, omega_SS, flag_graphs)

# Calibration parameters
calibrate_model = False
chi_b_guess = np.array([2, 10, 90, 350, 1700, 22000, 120000])
chi_n_guess = np.array([47.12000874 , 22.22762421 , 14.34842241 , 10.67954008 ,  8.41097278
                         ,  7.15059004 ,  6.46771332 ,  5.85495452 ,  5.46242013 ,  5.00364263
                         ,  4.57322063 ,  4.53371545 ,  4.29828515 ,  4.10144524 ,  3.8617942  ,  3.57282
                         ,  3.47473172 ,  3.31111347 ,  3.04137299 ,  2.92616951 ,  2.58517969
                         ,  2.48761429 ,  2.21744847 ,  1.9577682  ,  1.66931057 ,  1.6878927
                         ,  1.63107201 ,  1.63390543 ,  1.5901486  ,  1.58143606 ,  1.58005578
                         ,  1.59073213 ,  1.60190899 ,  1.60001831 ,  1.67763741 ,  1.70451784
                         ,  1.85430468 ,  1.97291208 ,  1.97017228 ,  2.25518398 ,  2.43969757
                         ,  3.21870602 ,  4.18334822 ,  4.97772026 ,  6.37663164 ,  8.65075992
                         ,  9.46944758 , 10.51634777 , 12.13353793 , 11.89186997 , 12.07083882
                         , 13.2992811  , 14.07987878 , 14.19951571 , 14.97943562 , 16.05601334
                         , 16.42979341 , 16.91576867 , 17.62775142 , 18.4885405  , 19.10609921
                         , 20.03988031 , 20.86564363 , 21.73645892 , 22.6208256  , 23.37786072
                         , 24.38166073 , 25.22395387 , 26.21419653 , 27.05246704 , 27.86896121
                         , 28.90029708 , 29.83586775 , 30.87563699 , 31.91207845 , 33.07449767
                         , 34.27919965 , 35.57195873 , 36.95045988 , 38.62308152])

# Generate Wealth data moments
wealth_data.get_wealth_data(lambdas, J, flag_graphs)

# Generate labor data moments

labor_data.labor_data_moments(flag_graphs)

get_baseline = True

# List of parameter names that will not be changing (unless we decide to
# change them for a tax experiment.
param_names = ['S', 'J', 'T', 'lambdas', 'starting_age', 'ending_age',
             'beta', 'sigma', 'alpha', 'nu', 'Z', 'delta', 'E',
             'ltilde', 'g_y', 'maxiter', 'mindist_SS', 'mindist_TPI',
             'b_ellipse', 'k_ellipse', 'upsilon',
             'a_tax_income', 'chi_b_guess', 'chi_n_guess',
             'b_tax_income', 'c_tax_income', 'd_tax_income',
             'tau_payroll', 'tau_bq', 'calibrate_model',
             'retire', 'mean_income_data', 'g_n_vector',
             'h_wealth', 'p_wealth', 'm_wealth', 'get_baseline',
             'omega', 'g_n_ss', 'omega_SS', 'surv_rate', 'e', 'rho']

'''
------------------------------------------------------------------------
    Run SS with minimization to fit chi_b and chi_n
------------------------------------------------------------------------
'''

# This is the simulation before getting the replacement rate values

dictionary = {}
for key in param_names:
    dictionary[key] = globals()[key]
pickle.dump(dictionary, open("OUTPUT/Saved_moments/params_given.pkl", "w"))

import SS

'''
------------------------------------------------------------------------
    Run the baseline TPI simulation
------------------------------------------------------------------------
'''

# import TPI

'''
------------------------------------------------------------------------
    Alter desired tax parameters
------------------------------------------------------------------------
'''

# New Tax Parameters

# get_baseline = False
# d_tax_income = .42

# var_names = ['get_baseline', 'd_tax_income']
# dictionary = {}
# for key in var_names:
#     dictionary[key] = globals()[key]
# pickle.dump(dictionary, open("OUTPUT/Saved_moments/params_changed.pkl", "w"))

'''
------------------------------------------------------------------------
    Run SS with tax experiment
------------------------------------------------------------------------
'''

# import SS

'''
------------------------------------------------------------------------
    Run TPI for tax experiment
------------------------------------------------------------------------
'''

# import TPI

'''
------------------------------------------------------------------------
Delete all .pyc files that have been generated
------------------------------------------------------------------------
'''

files = glob('*.pyc')
for i in files:
    os.remove(i)
<|endoftext|>


<commit_before>import scrapy

class Bonn2011Spider(scrapy.Spider):
    name = "bonn2011"
    start_urls = ['http://bonn-packts-an-2011.de/www.bonn-packts-an.de/dito/forumc0d2.html']
    # TODO: parse the list_pages for next page/last -> maybe in start_requests
    #response.css('div.list_pages a').extract()
    def parse(self, response):
        for thread in response.css('div.vorschlag.buergervorschlag'):
            # TODO: Follow each link
            yield {
                # TODO: Parse id
                'id' : thread.css('h2::text').extract_first(),
                'title' : thread.css('div.col_01 h3 a::text').extract_first(),
                'link' : thread.css('div.col_01 h3 a::attr(href)').extract_first()
            }

        # Here: Parse next Site
        next_page = response.xpath('//div[@class="list_pages"]/a[contains(.,"vor")]/@href').extract_first()
        if next_page:
            yield scrapy.Request(
                response.urljoin(next_page),
                callback=self.parse
            )
<commit_msg>Updated comments<commit_after>import scrapy

class Bonn2011Spider(scrapy.Spider):
    name = "bonn2011"
    start_urls = ['http://bonn-packts-an-2011.de/www.bonn-packts-an.de/dito/forumc0d2.html']

    def parse(self, response):
        for thread in response.css('div.vorschlag.buergervorschlag'):
            yield {
                # TODO: Parse id
                'id' : thread.css('h2::text').extract_first(),
                'title' : thread.css('div.col_01 h3 a::text').extract_first(),
                'link' : thread.css('div.col_01 h3 a::attr(href)').extract_first()
                # yield request?
            }

        # Here: Parse next Site
        next_page = response.xpath('//div[@class="list_pages"]/a[contains(.,"vor")]/@href').extract_first()
        if next_page:
            yield scrapy.Request(
                response.urljoin(next_page),
                callback=self.parse
            )
        #Here: Parse threads with parse2?
<|endoftext|>


<commit_before>"""
Pre build plugin which injects custom yum repository in dockerfile.
"""
import os
import re
import uuid
import sys
from dock.plugin import PreBuildPlugin

def alter_yum_commands(df, wrap_str):
    regex = re.compile(r"RUN (?P<yum_command>yum((\s.+\\\n)+)?(.+))", re.MULTILINE)
    sub_func = lambda match: wrap_str % {'yum_command': match.group('yum_command').rstrip()}
    return regex.sub(sub_func, df)

class InjectYumRepoPlugin(PreBuildPlugin):
    key = "inject_yum_repo"

    def __init__(self, tasker, workflow):
        """
        constructor

        :param tasker: DockerTasker instance
        :param workflow: DockerBuildWorkflow instance
        """
        # call parent constructor
        super(InjectYumRepoPlugin, self).__init__(tasker, workflow)
        self.yum_repos_dir = '/etc/yum.repos.d/'
        self.repofile_name = 'dock-injected.repo'
        self.repo_path = os.path.join(self.yum_repos_dir, self.repofile_name)

    def run(self):
        """
        run the plugin
        """
        rendered_repos = ''
        for repo in self.workflow.repos.get('yum', []):
            repo.setdefault("name", str(uuid.uuid4().get_hex()[:6]))
            rendered_repo = ''
            for key, value in repo.items():
                rendered_repo += r"%s=%s\n" % (key, value)
            rendered_repo = r'[%(name)s]\n' % repo + rendered_repo
            rendered_repos += rendered_repo

        wrap_cmd = 'RUN printf "%s"' % rendered_repos + \
            ' >%(repo_path)s && %%(yum_command)s && yum clean all && rm -f %(repo_path)s' \
            % {'repo_path': self.repo_path}

        with open(self.workflow.builder.df_path, "r+") as fd:
            df = fd.read()
            out = alter_yum_commands(df, wrap_cmd)
            fd.seek(0)
            fd.truncate()
            fd.write(out)
<commit_msg>inject_yum_repo: handle 'RUN  yum ...' (two spaces).<commit_after>"""
Pre build plugin which injects custom yum repository in dockerfile.
"""
import os
import re
import uuid
import sys
from dock.plugin import PreBuildPlugin

def alter_yum_commands(df, wrap_str):
    regex = re.compile(r"RUN\s+(?P<yum_command>yum((\s.+\\\n)+)?(.+))", re.MULTILINE)
    sub_func = lambda match: wrap_str % {'yum_command': match.group('yum_command').rstrip()}
    return regex.sub(sub_func, df)

class InjectYumRepoPlugin(PreBuildPlugin):
    key = "inject_yum_repo"

    def __init__(self, tasker, workflow):
        """
        constructor

        :param tasker: DockerTasker instance
        :param workflow: DockerBuildWorkflow instance
        """
        # call parent constructor
        super(InjectYumRepoPlugin, self).__init__(tasker, workflow)
        self.yum_repos_dir = '/etc/yum.repos.d/'
        self.repofile_name = 'dock-injected.repo'
        self.repo_path = os.path.join(self.yum_repos_dir, self.repofile_name)

    def run(self):
        """
        run the plugin
        """
        rendered_repos = ''
        for repo in self.workflow.repos.get('yum', []):
            repo.setdefault("name", str(uuid.uuid4().get_hex()[:6]))
            rendered_repo = ''
            for key, value in repo.items():
                rendered_repo += r"%s=%s\n" % (key, value)
            rendered_repo = r'[%(name)s]\n' % repo + rendered_repo
            rendered_repos += rendered_repo

        wrap_cmd = 'RUN printf "%s"' % rendered_repos + \
            ' >%(repo_path)s && %%(yum_command)s && yum clean all && rm -f %(repo_path)s' \
            % {'repo_path': self.repo_path}

        with open(self.workflow.builder.df_path, "r+") as fd:
            df = fd.read()
            out = alter_yum_commands(df, wrap_cmd)
            fd.seek(0)
            fd.truncate()
            fd.write(out)
<|endoftext|>


<commit_before># coding=utf-8
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import errno
import fcntl
import os
import psutil
import select
import subprocess
import time

_DELAY_STEP = 1

def _set_ononblock(fd):
    curr_flags = fcntl.fcntl(fd.fileno(), fcntl.F_GETFL)
    fcntl.fcntl(
        fd.fileno(),
        fcntl.F_SETFL,
        curr_flags | os.O_NONBLOCK
    )

def _read_async(fd):
    output, eof = '', False
    try:
        output = fd.read()
    except IOError as e:
        if e.errno != errno.EAGAIN:
            raise
    else:
        eof = (output == '')
    return output, eof

def _read_fds(ctx, process, timeout):
    ctx.logger.info('Waiting for subprocess to finish...')
    fds = {
        process.stdout.fileno(): {
            'file': process.stdout,
            'output': '',
            'eof': False
        },
        process.stderr.fileno(): {
            'file': process.stderr,
            'output': '',
            'eof': False
        }
    }
    while True:
        read_fds, _, _ = select.select(
            [fds[fd]['file'] for fd in fds if not fds[fd]['eof']],
            [],
            [],
            timeout
        )
        if not read_fds:
            ctx.logger.error('Subprocess hung up')
            hung_up = True
            break
        for fd in read_fds:
            output, fds[fd.fileno()]['eof'] = _read_async(fd)
            fds[fd.fileno()]['output'] += output
        if all(fds[fd]['eof'] for fd in fds):
            ctx.logger.info('Subprocess finished')
            hung_up = False
            break
    return (
        fds[process.stdout.fileno()]['output'],
        fds[process.stderr.fileno()]['output'],
        not hung_up
    )

def _manually_clean_up(ctx, process, waiting_for_output, timeout_terminate):
    ctx.logger.info('Terminating process')
    process.terminate()
    time_no_terminate = 0
    p = psutil.Process(process.pid)
    while (
            time_no_terminate < timeout_terminate and
            p.status() != psutil.STATUS_ZOMBIE
    ):
        time.sleep(_DELAY_STEP)
        time_no_terminate += 1
        p = psutil.Process(process.pid)

    stdout, stderr, success = _read_fds(ctx, process, waiting_for_output)
    if p.status() == psutil.STATUS_ZOMBIE:
        ctx.logger.info('Process terminated')
    else:
        ctx.logger.info('Killing process')
        process.kill()
        output, _ = _read_async(process.stdout)
        stdout += output
        output, _ = _read_async(process.stderr)
        stderr += output
    process.wait()
    return stdout, stderr

def _clean_up(ctx, process, success, waiting_for_output, timeout_terminate):
    ctx.logger.info('Cleaning up')
    stdout, stderr = '', ''
    if success:
        process.wait()
    else:
        stdout, stderr = _manually_clean_up(
            ctx,
            process,
            waiting_for_output,
            timeout_terminate,
        )
    process.stdout.close()
    process.stderr.close()
    ctx.logger.info('Cleaned up')
    return stdout, stderr

def run_process(
        ctx,
        command,
        waiting_for_output,
        timeout_terminate
):
    ctx.logger.info('Starting process')
    process = subprocess.Popen(
        command,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
    )
    for fd in process.stdout, process.stderr:
        _set_ononblock(fd)
    stdout, stderr, success = _read_fds(ctx, process, waiting_for_output)
    new_stdout, new_stderr = _clean_up(
        ctx,
        process,
        success,
        waiting_for_output,
        timeout_terminate
    )
    stdout += new_stdout
    stderr += new_stderr
    ctx.logger.info('Finishing process')
    return process.returncode, stdout, stderr
<commit_msg>Import fix<commit_after># coding=utf-8
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import errno
import fcntl
import os
import select
import subprocess
import time

import psutil

_DELAY_STEP = 1

def _set_ononblock(fd):
    curr_flags = fcntl.fcntl(fd.fileno(), fcntl.F_GETFL)
    fcntl.fcntl(
        fd.fileno(),
        fcntl.F_SETFL,
        curr_flags | os.O_NONBLOCK
    )

def _read_async(fd):
    output, eof = '', False
    try:
        output = fd.read()
    except IOError as e:
        if e.errno != errno.EAGAIN:
            raise
    else:
        eof = (output == '')
    return output, eof

def _read_fds(ctx, process, timeout):
    ctx.logger.info('Waiting for subprocess to finish...')
    fds = {
        process.stdout.fileno(): {
            'file': process.stdout,
            'output': '',
            'eof': False
        },
        process.stderr.fileno(): {
            'file': process.stderr,
            'output': '',
            'eof': False
        }
    }
    while True:
        read_fds, _, _ = select.select(
            [fds[fd]['file'] for fd in fds if not fds[fd]['eof']],
            [],
            [],
            timeout
        )
        if not read_fds:
            ctx.logger.error('Subprocess hung up')
            hung_up = True
            break
        for fd in read_fds:
            output, fds[fd.fileno()]['eof'] = _read_async(fd)
            fds[fd.fileno()]['output'] += output
        if all(fds[fd]['eof'] for fd in fds):
            ctx.logger.info('Subprocess finished')
            hung_up = False
            break
    return (
        fds[process.stdout.fileno()]['output'],
        fds[process.stderr.fileno()]['output'],
        not hung_up
    )

def _manually_clean_up(ctx, process, waiting_for_output, timeout_terminate):
    ctx.logger.info('Terminating process')
    process.terminate()
    time_no_terminate = 0
    p = psutil.Process(process.pid)
    while (
            time_no_terminate < timeout_terminate and
            p.status() != psutil.STATUS_ZOMBIE
    ):
        time.sleep(_DELAY_STEP)
        time_no_terminate += 1
        p = psutil.Process(process.pid)

    stdout, stderr, success = _read_fds(ctx, process, waiting_for_output)
    if p.status() == psutil.STATUS_ZOMBIE:
        ctx.logger.info('Process terminated')
    else:
        ctx.logger.info('Killing process')
        process.kill()
        output, _ = _read_async(process.stdout)
        stdout += output
        output, _ = _read_async(process.stderr)
        stderr += output
    process.wait()
    return stdout, stderr

def _clean_up(ctx, process, success, waiting_for_output, timeout_terminate):
    ctx.logger.info('Cleaning up')
    stdout, stderr = '', ''
    if success:
        process.wait()
    else:
        stdout, stderr = _manually_clean_up(
            ctx,
            process,
            waiting_for_output,
            timeout_terminate,
        )
    process.stdout.close()
    process.stderr.close()
    ctx.logger.info('Cleaned up')
    return stdout, stderr

def run_process(
        ctx,
        command,
        waiting_for_output,
        timeout_terminate
):
    ctx.logger.info('Starting process')
    process = subprocess.Popen(
        command,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
    )
    for fd in process.stdout, process.stderr:
        _set_ononblock(fd)
    stdout, stderr, success = _read_fds(ctx, process, waiting_for_output)
    new_stdout, new_stderr = _clean_up(
        ctx,
        process,
        success,
        waiting_for_output,
        timeout_terminate
    )
    stdout += new_stdout
    stderr += new_stderr
    ctx.logger.info('Finishing process')
    return process.returncode, stdout, stderr
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from openerp import api, fields, models

class Session(models.Model):
    _name = 'openacademy.session'

    name = fields.Char(required=True)
    start_date = fields.Date(default=fields.Date.today)
    duration = fields.Float(digits=(6, 2), help="Duration in days")
    seats = fields.Integer(string="Number of seats")
    active = fields.Boolean(default=True)

    instructor_id = fields.Many2one('res.partner', string="Instructor",
        domain=['|', ('instructor', '=', True),
                     ('category_id.name', 'ilike', "Teacher")])
    course_id = fields.Many2one('openacademy.course',
        ondelete='cascade', string="Course", required=True)
    attendee_ids = fields.Many2many('res.partner', string="Attendees")

    taken_seats = fields.Float(string="Taken seats", compute='_taken_seats')

    @api.one
    @api.depends('seats', 'attendee_ids')
    def _taken_seats(self):
        if not self.seats:
            self.taken_seats = 0.0
        else:
            self.taken_seats = 100.0 * len(self.attendee_ids) / self.seats

    @api.onchange('seats', 'attendee_ids')
    def _verify_valid_seats(self):
        if self.seats < 0:
            return {
                'warning': {
                    'title': "Incorrect 'seats' value",
                    'message': "The number of available seats may not be negative",
                },
            }
        if self.seats < len(self.attendee_ids):
            return {
                'warning': {
                    'title': "Too many attendees",
                    'message': "Increase seats or remove excess attendees",
                },
            }
<commit_msg>[REF] openacademy: Adding constrain in session<commit_after># -*- coding: utf-8 -*-

from openerp import api, exceptions, fields, models

class Session(models.Model):
    _name = 'openacademy.session'

    name = fields.Char(required=True)
    start_date = fields.Date(default=fields.Date.today)
    duration = fields.Float(digits=(6, 2), help="Duration in days")
    seats = fields.Integer(string="Number of seats")
    active = fields.Boolean(default=True)

    instructor_id = fields.Many2one('res.partner', string="Instructor",
        domain=['|', ('instructor', '=', True),
                     ('category_id.name', 'ilike', "Teacher")])
    course_id = fields.Many2one('openacademy.course',
        ondelete='cascade', string="Course", required=True)
    attendee_ids = fields.Many2many('res.partner', string="Attendees")

    taken_seats = fields.Float(string="Taken seats", compute='_taken_seats')

    @api.one
    @api.depends('seats', 'attendee_ids')
    def _taken_seats(self):
        if not self.seats:
            self.taken_seats = 0.0
        else:
            self.taken_seats = 100.0 * len(self.attendee_ids) / self.seats

    @api.onchange('seats', 'attendee_ids')
    def _verify_valid_seats(self):
        if self.seats < 0:
            return {
                'warning': {
                    'title': "Incorrect 'seats' value",
                    'message': "The number of available seats may not be negative",
                },
            }
        if self.seats < len(self.attendee_ids):
            return {
                'warning': {
                    'title': "Too many attendees",
                    'message': "Increase seats or remove excess attendees",
                },
            }

    @api.one
    @api.constrains('instructor_id', 'attendee_ids')
    def _check_instructor_not_in_attendees(self):
        if self.instructor_id and self.instructor_id in self.attendee_ids:
            raise exceptions.ValidationError("A session's instructor can't be an attendee")

<|endoftext|>


<commit_before>import json
import datetime
import uuid
import decimal

from django.http import HttpResponse

from .exceptions import BinderRequestError

# Default Binder serializers; override these by doing
# json.SERIALIZERS.update({}) in settings.py
SERIALIZERS = {
	set:                 list,
	datetime.datetime:   lambda v: v.strftime('%Y-%m-%dT%H:%M:%S.%f%z'),   # .isoformat() can omit microseconds
	datetime.date:       lambda v: v.isoformat(),
	uuid.UUID:           str,
	decimal.Decimal:     str,
}

# dateutil.relativedelta serializer, if available
try:
	from dateutil.relativedelta import relativedelta
	from relativedeltafield import format_relativedelta
	SERIALIZERS[relativedelta] = lambda e, v: format_relativedelta(v)
except ImportError:
	pass

# Converts values json.dumps can't convert itself.
def default(value):
	# Find a serializer in the Method Resolution Order
	for cls in type(value).mro():
		if cls in SERIALIZERS:
			return SERIALIZERS[cls](value)

	raise TypeError('{} is not JSON serializable'.format(repr(value)))

def jsondumps(o, default=default, indent=None):
	return json.dumps(o, default=default, indent=indent)

def jsonloads(data):
	try:
		return json.loads(data.decode())
	except ValueError as e:
		raise BinderRequestError('JSON parse error: {}.'.format(str(e)))

def JsonResponse(data):
	return HttpResponse(jsondumps(data), content_type='application/json')
<commit_msg>Fix optional relative delta serializer for new api<commit_after>import json
import datetime
import uuid
import decimal

from django.http import HttpResponse

from .exceptions import BinderRequestError

# Default Binder serializers; override these by doing
# json.SERIALIZERS.update({}) in settings.py
SERIALIZERS = {
	set:                 list,
	datetime.datetime:   lambda v: v.strftime('%Y-%m-%dT%H:%M:%S.%f%z'),   # .isoformat() can omit microseconds
	datetime.date:       lambda v: v.isoformat(),
	uuid.UUID:           str,
	decimal.Decimal:     str,
}

# dateutil.relativedelta serializer, if available
try:
	from dateutil.relativedelta import relativedelta
	from relativedeltafield import format_relativedelta
	SERIALIZERS[relativedelta] = format_relativedelta
except ImportError:
	pass

# Converts values json.dumps can't convert itself.
def default(value):
	# Find a serializer in the Method Resolution Order
	for cls in type(value).mro():
		if cls in SERIALIZERS:
			return SERIALIZERS[cls](value)

	raise TypeError('{} is not JSON serializable'.format(repr(value)))

def jsondumps(o, default=default, indent=None):
	return json.dumps(o, default=default, indent=indent)

def jsonloads(data):
	try:
		return json.loads(data.decode())
	except ValueError as e:
		raise BinderRequestError('JSON parse error: {}.'.format(str(e)))

def JsonResponse(data):
	return HttpResponse(jsondumps(data), content_type='application/json')
<|endoftext|>


<commit_before><commit_msg>pw_env_setup: Remove newlines from python path<commit_after><|endoftext|>


<commit_before>import json
import sys
import shutil
import os
import resource
import subprocess
import stat
from os import listdir
from os.path import isfile, join
import xmltodict

def copytree(src, dst, symlinks=False, ignore=None):
    """ Custom copy tree to allow to copy into existing directories """
    for item in os.listdir(src):
        s = os.path.join(src, item)
        d = os.path.join(dst, item)
        if os.path.isdir(s):
            shutil.copytree(s, d, symlinks, ignore)
        else:
            shutil.copy2(s, d)

def setlimits():
    resource.setrlimit(resource.RLIMIT_CPU, (60, 60)) #TODO: set real limits

def setExecutable(filename):
    st = os.stat(filename)
    os.chmod(filename, st.st_mode | stat.S_IEXEC)

os.mkdir("/tmp/work")

#Copy /ro/task (which is read-only) in /job. Everything will be executed there
shutil.copytree("/ro/task","/job")

if not os.path.exists("/job/input"):
    os.mkdir("/job/input")
os.symlink("/job/input","/tmp/work/input")

os.mkdir("/tmp/work/output")
os.mkdir("/tmp/log")

#Copy the libs needed by the tools
if os.path.exists("/job/lib"):
    copytree("/job/lib","/job/input/lib")
else:
    os.mkdir("/job/input/lib")
copytree("/pythia/lib","/job/input/lib")
#copytree("/pythia/lib","/tmp/work/lib")

#Launch everything
stdOutputData={"stdout":"","stderr":""}

#Parse the files
files = [ join("/job/input",f) for f in listdir("/job/input") if isfile(join("/job/input",f)) ]
cmd = ["python", "/pythia/pythia_input.py"]+files
p = subprocess.Popen(cmd, preexec_fn=setlimits, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
p.wait()
stdout, stderr = p.communicate()
stdOutputData["stdout"] = stdOutputData["stdout"]+"PARSE: "+stdout+"\n"
stdOutputData["stderr"] = stdOutputData["stderr"]+"PARSE: "+stderr+"\n"

if os.path.exists("/job/dataset.sh"):
    setExecutable("/job/dataset.sh")
    os.chdir("/job")
    p = subprocess.Popen(["/job/dataset.sh"], preexec_fn=setlimits, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    p.wait()
    stdout, stderr = p.communicate()
    stdOutputData["stdout"] = stdOutputData["stdout"]+"DATASET: "+stdout+"\n"
    stdOutputData["stderr"] = stdOutputData["stderr"]+"DATASET: "+stderr+"\n"

os.chdir("/tmp/work")
setExecutable("/job/run.sh")
p = subprocess.Popen(["/job/run.sh"], preexec_fn=setlimits, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
p.wait()
stdout, stderr = p.communicate()
stdOutputData["stdout"] = stdOutputData["stdout"]+"RUN: "+stdout+"\n"
stdOutputData["stderr"] = stdOutputData["stderr"]+"RUN: "+stderr+"\n"

#Move some files
shutil.copytree("/tmp/work/output","/job/output/files")
open("/job/output/status","w").write("done")

os.chdir("/job")
setExecutable("/job/feedback.sh")
p = subprocess.Popen(["/job/feedback.sh"], preexec_fn=setlimits, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
p.wait()
stdout, stderr = p.communicate()
stdOutputData["stdout"] = stdOutputData["stdout"]+"FEEDBACK: "+stdout+"\n"
stdOutputData["stderr"] = stdOutputData["stderr"]+"FEEDBACK: "+stderr+"\n"

if os.path.exists("feedback.xml"):
    file = open("feedback.xml","r")
    feedback = xmltodict.parse(file.read())['feedback']
    text = (feedback["#text"] if "#text" in feedback else "") + (feedback["general"] if "general" in feedback else "")
    problems = {}
    if "question" in feedback:
        problems = {feedback["question"]["@id"]: feedback["question"]["#text"]}
    print json.dumps({"result":("success" if feedback["verdict"] == "OK" else "failed"),"text":text,"problems":problems,"v0out":stdOutputData})
else:
    print json.dumps({"result":"crash","text":"The grader did not give any input","problems":{},"v0out":stdOutputData})

<commit_msg>Improve handling of feedback and put some file at the right place<commit_after>import json
import sys
import shutil
import os
import resource
import subprocess
import stat
from os import listdir
from os.path import isfile, join
import xmltodict
import StringIO

def copytree(src, dst, symlinks=False, ignore=None):
    """ Custom copy tree to allow to copy into existing directories """
    for item in os.listdir(src):
        s = os.path.join(src, item)
        d = os.path.join(dst, item)
        if os.path.isdir(s):
            shutil.copytree(s, d, symlinks, ignore)
        else:
            shutil.copy2(s, d)

def setlimits():
    resource.setrlimit(resource.RLIMIT_CPU, (60, 60)) #TODO: set real limits

def setExecutable(filename):
    st = os.stat(filename)
    os.chmod(filename, st.st_mode | stat.S_IEXEC)

#get input data
stdin = sys.stdin.read().strip('\0').strip()
input_data = json.loads(stdin)

os.mkdir("/tmp/work")

#Copy /ro/task (which is read-only) in /job. Everything will be executed there
shutil.copytree("/ro/task","/job")

if not os.path.exists("/job/input"):
    os.mkdir("/job/input")
os.symlink("/job/input","/tmp/work/input")

os.mkdir("/tmp/work/output")
os.mkdir("/tmp/log")

#Copy the libs needed by the tools
if os.path.exists("/job/lib"):
    copytree("/job/lib","/job/input/lib")
else:
    os.mkdir("/job/input/lib")
copytree("/pythia/lib","/job/input/lib")

#Launch everything
stdOutputData={"stdout":"","stderr":""}

#Parse the files
files = [ join("/job/input",f) for f in listdir("/job/input") if isfile(join("/job/input",f)) ]
cmd = ["python", "/pythia/pythia_input.py"]+files
p = subprocess.Popen(cmd, preexec_fn=setlimits, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate(stdin)
stdOutputData["stdout"] = stdOutputData["stdout"]+"PARSE: "+stdout+"\n"
stdOutputData["stderr"] = stdOutputData["stderr"]+"PARSE: "+stderr+"\n"

#Put the input in the .out files (...)
for question in input_data:
    open("/tmp/work/output/"+question+".out","w").write(input_data[question])

if os.path.exists("/job/dataset.sh"):
    setExecutable("/job/dataset.sh")
    os.chdir("/job")
    p = subprocess.Popen(["/job/dataset.sh"], preexec_fn=setlimits, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    p.wait()
    stdout, stderr = p.communicate()
    stdOutputData["stdout"] = stdOutputData["stdout"]+"DATASET: "+stdout+"\n"
    stdOutputData["stderr"] = stdOutputData["stderr"]+"DATASET: "+stderr+"\n"

os.chdir("/tmp/work")
if os.path.exists("/job/run.sh"):
    setExecutable("/job/run.sh")
    p = subprocess.Popen(["/job/run.sh"], preexec_fn=setlimits, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    p.wait()
    stdout, stderr = p.communicate()
    stdOutputData["stdout"] = stdOutputData["stdout"]+"RUN: "+stdout+"\n"
    stdOutputData["stderr"] = stdOutputData["stderr"]+"RUN: "+stderr+"\n"

#Move some files
shutil.copytree("/tmp/work/output","/job/output/files")
open("/job/output/status","w").write("done")

os.chdir("/job")
setExecutable("/job/feedback.sh")
p = subprocess.Popen(["/job/feedback.sh"], preexec_fn=setlimits, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
p.wait()
stdout, stderr = p.communicate()
stdOutputData["stdout"] = stdOutputData["stdout"]+"FEEDBACK: "+stdout+"\n"
stdOutputData["stderr"] = stdOutputData["stderr"]+"FEEDBACK: "+stderr+"\n"

if os.path.exists("feedback.xml"):
    fileF = open("feedback.xml","r")
    feedback = xmltodict.parse(fileF.read())['feedback']
    text = (feedback["#text"] if "#text" in feedback else "") + (feedback["general"] if "general" in feedback else "")
    problems = {}
    if "question" in feedback and isinstance(feedback["question"],list):
        for question in feedback["question"]:
            if "#text" in question:
                problems = {question["@id"]: question["#text"]}
    elif "question" in feedback: #ordered dict
        if "#text" in feedback["question"]:
            problems = {feedback["question"]["@id"]: feedback["question"]["#text"]}
    print json.dumps({"result":("success" if feedback["verdict"] == "OK" else "failed"),"text":text,"problems":problems,"v0out":stdOutputData})
else:
    print json.dumps({"result":"crash","text":"The grader did not give any input","problems":{},"v0out":stdOutputData})

<|endoftext|>


<commit_before>from __future__ import division, absolute_import, print_function

__all__ = ['Punic']

import os
import punic.shshutil as shutil
from copy import copy
from pathlib2 import Path
import affirm
from .xcode import *
from .basic_types import *
from .runner import *
from .resolver import *
from .config import *
from .repository import *
from .logger import *
from .cartfile import *

########################################################################################################################

class Punic(object):

    __slots__ = ['root_path', 'config', 'all_repositories', 'root_project']

    def __init__(self, root_path=None):

        if not root_path:
            root_path = Path.cwd()

        self.config = config

        root_project_identifier = ProjectIdentifier(overrides=None, project_name=self.config.root_path.name)

        self.all_repositories = {
            root_project_identifier: Repository(punic=self, identifier=root_project_identifier,
                repo_path=self.config.root_path),
        }

        self.root_project = self._repository_for_identifier(root_project_identifier)

    def _resolver(self):
        return Resolver(root = Node(self.root_project.identifier, None), dependencies_for_node = self._dependencies_for_node)

    def _dependencies_for_node(self, node):
        assert not node.version or isinstance(node.version, Revision)
        dependencies = self.dependencies_for_project_and_tag(identifier=node.identifier, tag=node.version)
        return dependencies

    def resolve(self):
        # type: (bool)
        resolver = self._resolver()
        build_order = resolver.resolve_build_order()

        for index, value in enumerate(build_order[:-1]):
            dependency, version = value
            logger.debug(
                '{} <ref>{}</ref> <rev>{}</rev> <ref>{}</ref>'.format(index + 1, dependency,
                    version.revision if version else '', dependency.remote_url))

        specifications = [
            Specification(identifier=dependency, predicate=VersionPredicate('"{}"'.format(version.revision))) for
            dependency, version in build_order[:-1]]
        logger.debug("Saving <ref>Cartfile.resolved</ref>")

        cartfile = Cartfile(specifications=specifications)
        cartfile.write((self.config.root_path / 'Cartfile.resolved').open('w'))

    def graph(self):
        # type: (bool) -> DiGraph
        return self._resolver().resolve()

    # TODO: This can be deprecated and the can_fetch flag relied on instead
    def fetch(self, dependencies = None):

        configuration, platforms = self.config.configuration, self.config.platforms

        if not self.config.build_path.exists():
            self.config.build_path.mkdir(parents=True)

        self._ordered_dependencies(name_filter=dependencies)

    @property
    def xcode_arguments(self):
        # type: () -> dict
        return {
            'ONLY_ACTIVE_ARCH': 'NO',
            'BITCODE_GENERATION_MODE': 'bitcode',
            'CODE_SIGNING_REQUIRED': 'NO',
            'CODE_SIGN_IDENTITY': '',
            'CARTHAGE': 'YES',
        }

    def build(self, dependencies):
        # type: ([str])

        logger.info('Using xcode version: {}'.format(self.config.xcode))

        configuration, platforms = self.config.configuration, self.config.platforms

        if not self.config.build_path.exists():
            self.config.build_path.mkdir(parents=True)

        filtered_dependencies = self._ordered_dependencies(name_filter=dependencies)

        checkouts = [Checkout(punic = self, identifier = identifier, revision = revision) for identifier, revision in filtered_dependencies]

        for platform in platforms:
            for checkout in checkouts:
                checkout.prepare()
                for project in checkout.projects:
                    schemes = project.schemes
                    schemes = [scheme for scheme in schemes if platform.device_sdk in scheme.support_platform_names]
                    schemes = [scheme for scheme in schemes if scheme.product_is_framework]
                    for scheme in schemes:
                        self._build_one(platform, project, scheme.name, configuration)

    def _ordered_dependencies(self, name_filter=None):
        # type: (bool, [str]) -> [(ProjectIdentifier, Revision)]

        cartfile = Cartfile(overrides=config.repo_overrides)
        cartfile.read(self.config.root_path / 'Cartfile.resolved')

        def _predicate_to_revision(spec):
            repository = self._repository_for_identifier(spec.identifier)
            if spec.predicate.operator == VersionOperator.commitish:
                return Revision(repository=repository, revision=spec.predicate.value, revision_type=Revision.Type.commitish)
            else:
                raise Exception("Cannot convert spec to revision: {}".format(spec))

        dependencies = [(spec.identifier, _predicate_to_revision(spec)) for spec in cartfile.specifications]
        resolved_dependencies = self._resolver().resolve_versions(dependencies)
        resolved_dependencies = [dependency for dependency in resolved_dependencies if
            dependency[0].matches(name_filter)]
        return resolved_dependencies

    def _repository_for_identifier(self, identifier):
        # type: (ProjectIdentifier) -> Repository
        if identifier in self.all_repositories:
            return self.all_repositories[identifier]
        else:
            repository = Repository(self, identifier=identifier)
            if self.config.can_fetch:
                repository.fetch()
            self.all_repositories[identifier] = repository
            return repository

    def dependencies_for_project_and_tag(self, identifier, tag):
        # type: (ProjectIdentifier, Revision) -> [ProjectIdentifier, [Revision]]

        assert isinstance(identifier, ProjectIdentifier)
        assert not tag or isinstance(tag, Revision)

        repository = self._repository_for_identifier(identifier)
        specifications = repository.specifications_for_revision(tag)

        def make(specification):
            repository = self._repository_for_identifier(specification.identifier)
            tags = repository.revisions_for_predicate(specification.predicate)
            if specification.predicate.operator == VersionOperator.commitish:
                tags.append(Revision(repository=repository, revision=specification.predicate.value, revision_type=Revision.Type.commitish))
                tags.sort()
            assert len(tags)
            return repository.identifier, tags

        return [make(specification) for specification in specifications]

    def _prepare_dependency(self, identifier, revision):
        project = self._repository_for_identifier(identifier)
        checkout_path = self.config.checkouts_path / identifier.project_name

        # TODO: This isn't really 'can_fetch'
        if self.config.can_fetch:
            project.checkout(revision)
            logger.debug('Copying project to <ref>Carthage/Checkouts</ref>')
            if checkout_path.exists():
                shutil.rmtree(checkout_path)
            shutil.copytree(project.path, checkout_path, ignore=shutil.ignore_patterns('.git'))

        if not checkout_path.exists():
            raise Exception('No checkout at path: {}'.format(checkout_path))

        # We only need to bother making a symlink to <root>/Carthage/Build if dependency also has dependencies.
        if len(self.dependencies_for_project_and_tag(identifier, revision)):
            # Make a Carthage/Build symlink inside checked out project.
            carthage_path = checkout_path / 'Carthage'
            if not carthage_path.exists():
                carthage_path.mkdir()

            carthage_symlink_path = carthage_path / 'Build'
            if carthage_symlink_path.exists():
                carthage_symlink_path.unlink()
            logger.debug('Creating symlink: <ref>{}</ref> to <ref>{}</ref>'.format(
                carthage_symlink_path.relative_to(self.root_path), self.build_path.relative_to(self.root_path)))
            assert self.build_path.exists()
            os.symlink(str(self.build_path), str(carthage_symlink_path))

    def _build_one(self, platform, project, scheme, configuration):
        products = dict()

        # Build device & simulator (if sim exists)
        for sdk in platform.sdks:
            logger.info('Building <ref>{}</ref> ({}, {}, {})'.format(project.path.name, scheme, sdk,
                configuration))

            derived_data_path = self.config.derived_data_path

            arguments = XcodeBuildArguments(scheme=scheme, configuration=configuration, sdk=sdk, derived_data_path=derived_data_path, arguments=self.xcode_arguments)

            product = project.build(arguments=arguments)
            products[sdk] = product

        self._post_process(platform, products)

    def _post_process(self, platform, products):

        ########################################################################################################

        logger.debug("Post processing")

        # By convention sdk[0] is always the device sdk (e.g. 'iphoneos' and not 'iphonesimulator')
        device_sdk = platform.device_sdk
        device_product = products[device_sdk]

        ########################################################################################################

        output_product = copy(device_product)
        output_product.target_build_dir = self.config.build_path / platform.output_directory_name

        ########################################################################################################

        logger.debug('Copying binary')
        if output_product.product_path.exists():
            shutil.rmtree(output_product.product_path)
        shutil.copytree(device_product.product_path, output_product.product_path)

        ########################################################################################################

        if len(products) > 1:
            logger.debug('Lipo-ing')
            executable_paths = [product.executable_path for product in products.values()]
            command = ['/usr/bin/xcrun', 'lipo', '-create'] + executable_paths + ['-output',
                output_product.executable_path]
            runner.check_run(command)
            mtime = executable_paths[0].stat().st_mtime
            os.utime(str(output_product.executable_path), (mtime, mtime))

        ########################################################################################################

        logger.debug('Copying swiftmodule files')
        for product in products.values():
            for path in product.module_paths:
                relative_path = path.relative_to(product.product_path)
                shutil.copyfile(path, output_product.product_path / relative_path)

        ########################################################################################################

        logger.debug('Copying bcsymbolmap files')
        for product in products.values():
            for path in product.bcsymbolmap_paths:
                shutil.copy(path, output_product.target_build_dir)

        ########################################################################################################

        logger.debug('Producing dSYM files')
        command = ['/usr/bin/xcrun', 'dsymutil', str(output_product.executable_path), '-o',
            str(output_product.target_build_dir / (output_product.executable_name + '.dSYM'))]
        runner.check_run(command)

        ########################################################################################################

class Checkout(object):
    def __init__(self, punic, identifier, revision):
        self.punic = punic
        self.identifier = identifier
        self.repository = self.punic._repository_for_identifier(self.identifier)
        self.revision = revision
        self.checkout_path = self.punic.config.checkouts_path / self.identifier.project_name

    def prepare(self):

        # TODO: This isn't really 'can_fetch'
        if self.punic.config.can_fetch:
            self.repository.checkout(self.revision)
            logger.debug('Copying project to <ref>Carthage/Checkouts</ref>')
            if self.checkout_path.exists():
                shutil.rmtree(self.checkout_path)
            shutil.copytree(self.repository.path, self.checkout_path, ignore=shutil.ignore_patterns('.git'))

        if not self.checkout_path.exists():
            raise Exception('No checkout at path: {}'.format(self.checkout_path))

        # We only need to bother making a symlink to <root>/Carthage/Build if dependency also has dependencies.
        if len(self.punic.dependencies_for_project_and_tag(self.identifier, self.revision)):
            # Make a Carthage/Build symlink inside checked out project.
            carthage_path = self.checkout_path / 'Carthage'
            if not carthage_path.exists():
                carthage_path.mkdir()

            carthage_symlink_path = carthage_path / 'Build'
            if carthage_symlink_path.exists():
                carthage_symlink_path.unlink()
            logger.debug('Creating symlink: <ref>{}</ref> to <ref>{}</ref>'.format(
                carthage_symlink_path.relative_to(self.punic.config.root_path), self.punic.config.build_path.relative_to(self.punic.config.root_path)))
            assert self.punic.config.build_path.exists()
            os.symlink(str(self.punic.config.build_path), str(carthage_symlink_path))

    @property
    def projects(self):
        def _make_cache_identifier(project_path):
            rev = self.repository.rev_parse(self.revision)
            cache_identifier = '{},{}'.format(str(rev), project_path.relative_to(self.checkout_path))
            return cache_identifier

        project_paths = self.checkout_path.glob("*.xcodeproj")
        projects = [XcodeProject(self, config.xcode, project_path, _make_cache_identifier(project_path)) for project_path
            in
            project_paths]
        return projects
<commit_msg>Put a colon on it.<commit_after>from __future__ import division, absolute_import, print_function

__all__ = ['Punic']

import os
import punic.shshutil as shutil
from copy import copy
from pathlib2 import Path
import affirm
from .xcode import *
from .basic_types import *
from .runner import *
from .resolver import *
from .config import *
from .repository import *
from .logger import *
from .cartfile import *

########################################################################################################################

class Punic(object):

    __slots__ = ['root_path', 'config', 'all_repositories', 'root_project']

    def __init__(self, root_path=None):

        if not root_path:
            root_path = Path.cwd()

        self.config = config

        root_project_identifier = ProjectIdentifier(overrides=None, project_name=self.config.root_path.name)

        self.all_repositories = {
            root_project_identifier: Repository(punic=self, identifier=root_project_identifier,
                repo_path=self.config.root_path),
        }

        self.root_project = self._repository_for_identifier(root_project_identifier)

    def _resolver(self):
        return Resolver(root = Node(self.root_project.identifier, None), dependencies_for_node = self._dependencies_for_node)

    def _dependencies_for_node(self, node):
        assert not node.version or isinstance(node.version, Revision)
        dependencies = self.dependencies_for_project_and_tag(identifier=node.identifier, tag=node.version)
        return dependencies

    def resolve(self):
        # type: (bool)
        resolver = self._resolver()
        build_order = resolver.resolve_build_order()

        for index, value in enumerate(build_order[:-1]):
            dependency, version = value
            logger.debug(
                '{} <ref>{}</ref> <rev>{}</rev> <ref>{}</ref>'.format(index + 1, dependency,
                    version.revision if version else '', dependency.remote_url))

        specifications = [
            Specification(identifier=dependency, predicate=VersionPredicate('"{}"'.format(version.revision))) for
            dependency, version in build_order[:-1]]
        logger.debug("Saving <ref>Cartfile.resolved</ref>")

        cartfile = Cartfile(specifications=specifications)
        cartfile.write((self.config.root_path / 'Cartfile.resolved').open('w'))

    def graph(self):
        # type: (bool) -> DiGraph
        return self._resolver().resolve()

    # TODO: This can be deprecated and the can_fetch flag relied on instead
    def fetch(self, dependencies = None):

        configuration, platforms = self.config.configuration, self.config.platforms

        if not self.config.build_path.exists():
            self.config.build_path.mkdir(parents=True)

        self._ordered_dependencies(name_filter=dependencies)

    @property
    def xcode_arguments(self):
        # type: () -> dict
        return {
            'ONLY_ACTIVE_ARCH': 'NO',
            'BITCODE_GENERATION_MODE': 'bitcode',
            'CODE_SIGNING_REQUIRED': 'NO',
            'CODE_SIGN_IDENTITY': '',
            'CARTHAGE': 'YES',
        }

    def build(self, dependencies):
        # type: ([str])

        logger.info('Using xcode version: {}'.format(self.config.xcode))

        configuration, platforms = self.config.configuration, self.config.platforms

        if not self.config.build_path.exists():
            self.config.build_path.mkdir(parents=True)

        filtered_dependencies = self._ordered_dependencies(name_filter=dependencies)

        checkouts = [Checkout(punic = self, identifier = identifier, revision = revision) for identifier, revision in filtered_dependencies]

        for platform in platforms:
            for checkout in checkouts:
                checkout.prepare()
                for project in checkout.projects:
                    schemes = project.schemes
                    schemes = [scheme for scheme in schemes if platform.device_sdk in scheme.support_platform_names]
                    schemes = [scheme for scheme in schemes if scheme.product_is_framework]
                    for scheme in schemes:
                        self._build_one(platform, project, scheme.name, configuration)

    def _ordered_dependencies(self, name_filter=None):
        # type: (bool, [str]) -> [(ProjectIdentifier, Revision)]

        cartfile = Cartfile(overrides=config.repo_overrides)
        cartfile.read(self.config.root_path / 'Cartfile.resolved')

        def _predicate_to_revision(spec):
            repository = self._repository_for_identifier(spec.identifier)
            if spec.predicate.operator == VersionOperator.commitish:
                return Revision(repository=repository, revision=spec.predicate.value, revision_type=Revision.Type.commitish)
            else:
                raise Exception("Cannot convert spec to revision: {}".format(spec))

        dependencies = [(spec.identifier, _predicate_to_revision(spec)) for spec in cartfile.specifications]
        resolved_dependencies = self._resolver().resolve_versions(dependencies)
        resolved_dependencies = [dependency for dependency in resolved_dependencies if
            dependency[0].matches(name_filter)]
        return resolved_dependencies

    def _repository_for_identifier(self, identifier):
        # type: (ProjectIdentifier) -> Repository
        if identifier in self.all_repositories:
            return self.all_repositories[identifier]
        else:
            repository = Repository(self, identifier=identifier)
            if self.config.can_fetch:
                repository.fetch()
            self.all_repositories[identifier] = repository
            return repository

    def dependencies_for_project_and_tag(self, identifier, tag):
        # type: (ProjectIdentifier, Revision) -> [ProjectIdentifier, [Revision]]

        assert isinstance(identifier, ProjectIdentifier)
        assert not tag or isinstance(tag, Revision)

        repository = self._repository_for_identifier(identifier)
        specifications = repository.specifications_for_revision(tag)

        def make(specification):
            repository = self._repository_for_identifier(specification.identifier)
            tags = repository.revisions_for_predicate(specification.predicate)
            if specification.predicate.operator == VersionOperator.commitish:
                tags.append(Revision(repository=repository, revision=specification.predicate.value, revision_type=Revision.Type.commitish))
                tags.sort()
            assert len(tags)
            return repository.identifier, tags

        return [make(specification) for specification in specifications]

    def _prepare_dependency(self, identifier, revision):
        project = self._repository_for_identifier(identifier)
        checkout_path = self.config.checkouts_path / identifier.project_name

        # TODO: This isn't really 'can_fetch'
        if self.config.can_fetch:
            project.checkout(revision)
            logger.debug('Copying project to <ref>Carthage/Checkouts</ref>')
            if checkout_path.exists():
                shutil.rmtree(checkout_path)
            shutil.copytree(project.path, checkout_path, ignore=shutil.ignore_patterns('.git'))

        if not checkout_path.exists():
            raise Exception('No checkout at path: {}'.format(checkout_path))

        # We only need to bother making a symlink to <root>/Carthage/Build if dependency also has dependencies.
        if len(self.dependencies_for_project_and_tag(identifier, revision)):
            # Make a Carthage/Build symlink inside checked out project.
            carthage_path = checkout_path / 'Carthage'
            if not carthage_path.exists():
                carthage_path.mkdir()

            carthage_symlink_path = carthage_path / 'Build'
            if carthage_symlink_path.exists():
                carthage_symlink_path.unlink()
            logger.debug('Creating symlink: <ref>{}</ref> to <ref>{}</ref>'.format(
                carthage_symlink_path.relative_to(self.root_path), self.build_path.relative_to(self.root_path)))
            assert self.build_path.exists()
            os.symlink(str(self.build_path), str(carthage_symlink_path))

    def _build_one(self, platform, project, scheme, configuration):
        products = dict()

        # Build device & simulator (if sim exists)
        for sdk in platform.sdks:
            logger.info('Building: <ref>{}</ref> ({}, {}, {})'.format(project.path.name, scheme, sdk,
                configuration))

            derived_data_path = self.config.derived_data_path

            arguments = XcodeBuildArguments(scheme=scheme, configuration=configuration, sdk=sdk, derived_data_path=derived_data_path, arguments=self.xcode_arguments)

            product = project.build(arguments=arguments)
            products[sdk] = product

        self._post_process(platform, products)

    def _post_process(self, platform, products):

        ########################################################################################################

        logger.debug("Post processing")

        # By convention sdk[0] is always the device sdk (e.g. 'iphoneos' and not 'iphonesimulator')
        device_sdk = platform.device_sdk
        device_product = products[device_sdk]

        ########################################################################################################

        output_product = copy(device_product)
        output_product.target_build_dir = self.config.build_path / platform.output_directory_name

        ########################################################################################################

        logger.debug('Copying binary')
        if output_product.product_path.exists():
            shutil.rmtree(output_product.product_path)
        shutil.copytree(device_product.product_path, output_product.product_path)

        ########################################################################################################

        if len(products) > 1:
            logger.debug('Lipo-ing')
            executable_paths = [product.executable_path for product in products.values()]
            command = ['/usr/bin/xcrun', 'lipo', '-create'] + executable_paths + ['-output',
                output_product.executable_path]
            runner.check_run(command)
            mtime = executable_paths[0].stat().st_mtime
            os.utime(str(output_product.executable_path), (mtime, mtime))

        ########################################################################################################

        logger.debug('Copying swiftmodule files')
        for product in products.values():
            for path in product.module_paths:
                relative_path = path.relative_to(product.product_path)
                shutil.copyfile(path, output_product.product_path / relative_path)

        ########################################################################################################

        logger.debug('Copying bcsymbolmap files')
        for product in products.values():
            for path in product.bcsymbolmap_paths:
                shutil.copy(path, output_product.target_build_dir)

        ########################################################################################################

        logger.debug('Producing dSYM files')
        command = ['/usr/bin/xcrun', 'dsymutil', str(output_product.executable_path), '-o',
            str(output_product.target_build_dir / (output_product.executable_name + '.dSYM'))]
        runner.check_run(command)

        ########################################################################################################

class Checkout(object):
    def __init__(self, punic, identifier, revision):
        self.punic = punic
        self.identifier = identifier
        self.repository = self.punic._repository_for_identifier(self.identifier)
        self.revision = revision
        self.checkout_path = self.punic.config.checkouts_path / self.identifier.project_name

    def prepare(self):

        # TODO: This isn't really 'can_fetch'
        if self.punic.config.can_fetch:
            self.repository.checkout(self.revision)
            logger.debug('Copying project to <ref>Carthage/Checkouts</ref>')
            if self.checkout_path.exists():
                shutil.rmtree(self.checkout_path)
            shutil.copytree(self.repository.path, self.checkout_path, ignore=shutil.ignore_patterns('.git'))

        if not self.checkout_path.exists():
            raise Exception('No checkout at path: {}'.format(self.checkout_path))

        # We only need to bother making a symlink to <root>/Carthage/Build if dependency also has dependencies.
        if len(self.punic.dependencies_for_project_and_tag(self.identifier, self.revision)):
            # Make a Carthage/Build symlink inside checked out project.
            carthage_path = self.checkout_path / 'Carthage'
            if not carthage_path.exists():
                carthage_path.mkdir()

            carthage_symlink_path = carthage_path / 'Build'
            if carthage_symlink_path.exists():
                carthage_symlink_path.unlink()
            logger.debug('Creating symlink: <ref>{}</ref> to <ref>{}</ref>'.format(
                carthage_symlink_path.relative_to(self.punic.config.root_path), self.punic.config.build_path.relative_to(self.punic.config.root_path)))
            assert self.punic.config.build_path.exists()
            os.symlink(str(self.punic.config.build_path), str(carthage_symlink_path))

    @property
    def projects(self):
        def _make_cache_identifier(project_path):
            rev = self.repository.rev_parse(self.revision)
            cache_identifier = '{},{}'.format(str(rev), project_path.relative_to(self.checkout_path))
            return cache_identifier

        project_paths = self.checkout_path.glob("*.xcodeproj")
        projects = [XcodeProject(self, config.xcode, project_path, _make_cache_identifier(project_path)) for project_path
            in
            project_paths]
        return projects
<|endoftext|>