<commit_before>from __future__ import print_function
import sys
from chi2_plots import *
import random
import ast

"""
This script can be used to get the p value for the Miranda method (=chi squared). It takes input files with column vectors corresponding to
features and lables.
"""

print(__doc__)
import sys
sys.path.insert(0,'../..')
import os
from scipy import stats
import numpy as np
import matplotlib.pyplot as plt
import numpy.matlib
from matplotlib.colors import Normalize

from sklearn.preprocessing import StandardScaler

##############################################################################
# Setting parameters
#
#orig_name= sys.argv[1]
#number_of_splits_list= ast.literal_eval(sys.argv[2])
#print("number_of_splits_list : ", number_of_splits_list)
#dim_list = ast.literal_eval(sys.argv[3])
#comp_file_list_list = ast.literal_eval(sys.argv[4])

def chi2_adaptive_binning_systematics(orig_name, dim_list, comp_file_list_list,number_of_splits_list,systematics_fraction):
	sample1_name="original"
	sample2_name="modified"

	shuffling_seed = 100

	DEBUG = False

	##############################################################################
	max_number_of_splits = np.max(number_of_splits_list)
	for dim_index, dim_data in enumerate(dim_list):
		print("We are now in "+str(dim_data) + " Dimensions")
		#comp_file_list=[]
		comp_file_list = comp_file_list_list[dim_index]
		#for i in range(2):

			#comp_file_list.append((os.environ['MLToolsDir']+"/Dalitz/gaussian_samples/higher_dimensional_gauss/gauss_data/data_high" +str(dim_data)+"Dgauss_10000_0.5_0.1_0.0_{0}.txt".format(i),os.environ['MLToolsDir']+"/Dalitz/gaussian_samples/higher_dimensional_gauss/gauss_data/data_high"+str(dim_data)+"Dgauss_10000_0.5_0.1_0.01_{0}.txt".format(i)))
			#comp_file_list.append((os.environ['MLToolsDir']+"/Dalitz/gaussian_samples/legendre/legendre_data/data_legendre_contrib0__1__10__sample_{0}.txt".format(i),os.environ['MLToolsDir']+"/Dalitz/gaussian_samples/legendre/legendre_data/data_legendre_contrib0__1__9__sample_{0}.txt".format(i)))
			#comp_file_list.append((os.environ['MLToolsDir']+"/Dalitz/gaussian_samples/gaussian_same_projection_on_each_axis/gauss_data/gaussian_same_projection_on_each_axis_redefined_{1}D_1000_0.6_0.2_0.1_{0}.txt".format(i,dim_data),os.environ['MLToolsDir']+"/Dalitz/gaussian_samples/gaussian_same_projection_on_each_axis/gauss_data/gaussian_same_projection_on_each_axis_redefined_{1}D_1000_0.6_0.2_0.075_{0}.txt".format(i,dim_data)))
		#print(comp_file_list)

		score_dict = {}
		for number_of_splits in number_of_splits_list:
			score_dict[str(number_of_splits)]=[]

		for comp_file_0,comp_file_1 in comp_file_list:
			print("Operating of files :"+comp_file_0+"   "+comp_file_1)

			#extracts data from the files
			features_0=np.loadtxt(comp_file_0,dtype='d')
			features_1=np.loadtxt(comp_file_1,dtype='d')

			#determine how many data points are in each sample
			no_0=features_0.shape[0]
			no_1=features_1.shape[0]

			#Give all samples in file 0 the label 0 and in file 1 the feature 1
			label_0=np.zeros((no_0,1))
			label_1=np.ones((no_1,1))

			#Create an array containing samples and features.
			data_0=np.c_[features_0,label_0]
			data_1=np.c_[features_1,label_1]

			data=np.r_[data_0,data_1]

			no_dim = data.shape[1]-1
			assert (no_dim ==dim_data), 	"The dimension of the data is inconsistent {} vs {}".format(no_dim,dim_data)

			np.random.shuffle(data)

			#print("data.shape : ", data.shape)
			labels=data[:,-1]

			X_values= data[:,:-1]
			X_max   = np.amax(data,axis=0)[:-1]
			X_min   = np.amin(data,axis=0)[:-1]
			X_total_width = (np.subtract(X_max,X_min))
			del data

			#Scaling
			X_values = X_values - X_min[None,:]
			X_values = X_values / X_total_width[None,:]

			data = np.concatenate((X_values, labels[:,None]), axis=1)

			#print("X_values : ",X_values)
			starting_boundary = []
			for i in range(dim_data):
				starting_boundary.append([0.0,1.0])
			#Each key has the following stricture: # of splits and for each split if it was closer (a) or further away from (b) the origin. The original bin is "0"
			#For example "2ab" means it is the bin that was closer to the origin for the first split and further away for the second one.
			bin_boundaries_dict = {'0' : np.array(starting_boundary)}
			bin_points_dict = {'0' : data}

			for split_number in range(1,1+max_number_of_splits):
				for bin_key, bin_boundary in bin_boundaries_dict.items():
					if str(split_number-1) in bin_key:
						variances= np.var(bin_points_dict[bin_key][:,:-1], axis=0)
						#print("\nvariances : ", variances)
						dim_to_be_sliced = np.argmax(variances)
						#print("dim_to_be_sliced : ",dim_to_be_sliced)
						#print("bin_points_dict[bin_key] : ",bin_points_dict[bin_key])
						#print("bin_points_dict[bin_key][:,dim_to_be_sliced] : ",bin_points_dict[bin_key][:,dim_to_be_sliced])
						median = np.median(bin_points_dict[bin_key][:,dim_to_be_sliced])
						#print("median : ",median)

						a_bin_boundary, b_bin_boundary = bin_boundary.copy(), bin_boundary.copy()
						#print("a_bin_boundary : ",a_bin_boundary)
						a_bin_boundary[dim_to_be_sliced,1] = median
						b_bin_boundary[dim_to_be_sliced,0] = median
						bin_boundaries_dict[str(split_number)+bin_key[1:]+'a'] = a_bin_boundary
						bin_boundaries_dict[str(split_number)+bin_key[1:]+'b'] = b_bin_boundary

						a_points, b_points = [],[]
						for event_number in range(bin_points_dict[bin_key].shape[0]):
							if bin_points_dict[bin_key][event_number,dim_to_be_sliced] < median: a_points.append(bin_points_dict[bin_key][event_number,:].tolist())
							else: b_points.append(bin_points_dict[bin_key][event_number,:].tolist())

						bin_points_dict[str(split_number)+bin_key[1:]+'a'] = np.array(a_points)
						bin_points_dict[str(split_number)+bin_key[1:]+'b'] = np.array(b_points)
						#If a bin contains no particles it should be deleted
						if len(a_points)==0:
							del bin_points_dict[str(split_number)+bin_key[1:]+'a']
							del bin_boundaries_dict[str(split_number)+bin_key[1:]+'a']
                                                if len(b_points)==0:
                                                        del bin_points_dict[str(split_number)+bin_key[1:]+'b']
                                                        del bin_boundaries_dict[str(split_number)+bin_key[1:]+'b']

			check_key = '3aaa'
			#check_key = str(max_number_of_splits)
			#for i in range(max_number_of_splits): check_key += random.choice(['a','b'])
			#print("bin_boundaries_dict.keys() : ", bin_boundaries_dict.keys())
			#print("bin_boundaries_dict[check_key] : ",bin_boundaries_dict[check_key])
			#print("bin_points_dict.keys(): ", bin_points_dict.keys())
			#print("bin_points_dict[check_key][0,:] : ",bin_points_dict[check_key][0,:])

			bins_sample01_dict= {}
			signed_Scp2_dict= {}

			for number_of_splits in number_of_splits_list:
				print("\nnumber_of_splits : ",number_of_splits)
				bins_sample0, bins_sample1 = [] , []
				for bin_key, bin_points in bin_points_dict.items():
					if str(number_of_splits) in bin_key:
						labels_in_bin = bin_points[:,-1]
						#print("labels_in_bin : ",labels_in_bin)
						bin_sample0 = np.count_nonzero( labels_in_bin == 0)
						bin_sample1 = np.count_nonzero( labels_in_bin == 1)
						#simulate uncertainties
						bin_sample0 += int(round(np.random.normal(0,systematics_fraction*bin_sample0)))
                                                bin_sample1 += int(round(np.random.normal(0,systematics_fraction*bin_sample1)))
						bins_sample01_dict[bin_key]=[bin_sample0,bin_sample1]
						signed_Scp2_dict[bin_key] = np.square(float(bin_sample1-bin_sample0))/(float(bin_sample1)+float(bin_sample0)+np.square(float(bin_sample1)*systematics_fraction)+np.square(float(bin_sample1)*systematics_fraction))*np.sign(bin_sample1-bin_sample0)
						#print("\n\nbin_sample0 : ",bin_sample0, "\n bins_sample0 : ", bins_sample0 )
						#print("type(bin_sample0) : ",type(bin_sample0), "  type(bins_sample0) : ",type(bins_sample0))
						bins_sample0.append(bin_sample0)
						#print("  bins_sample0 : ", bins_sample0, "\n\n" )
						bins_sample1.append(bin_sample1)
				bins_sample0, bins_sample1 = np.array(bins_sample0,dtype=float), np.array(bins_sample1, dtype=float)
				print("bins_sample0 : ",bins_sample0,"\n bins_sample1 : ",bins_sample1)
				#element wise subtraction and division
				Scp2 = ((bins_sample1-bins_sample0)**2)/ (bins_sample1+bins_sample0+(systematics_fraction*bins_sample1)**2+(systematics_fraction*bins_sample0)**2 )
				#Scp2 =  np.divide(np.square(np.subtract(bins_sample1,bins_sample0)),np.add(bins_sample1,bins_sample0))
				if DEBUG:
					print(Scp2)

				#nansum ignores all the contributions that are Not A Number (NAN)
				Chi2 = np.nansum(Scp2)
				if DEBUG:
					print("Chi2")
					print(Chi2)
				dof=bins_sample0.shape[0]-1

				pvalue= 1 - stats.chi2.cdf(Chi2,dof)

				print("\nThe p value for Scp2 = ",Scp2," and Chi2 = ", Chi2, " is ",pvalue,"\n\n")
				if DEBUG:
					print(bins_sample0)
					print(bins_sample1)
					print("Chi2/dof : {0}".format(str(Chi2/dof)))

					print("pvalue : {0}".format(str(pvalue)))
				score_dict[str(number_of_splits)].append(pvalue)

		for number_of_splits in number_of_splits_list:
			name = orig_name + "_" +str(dim_data) + "D_" + str(number_of_splits) + "_splits"
			print("score_dict[{}] : ".format(number_of_splits), score_dict[str(number_of_splits)])
			with open(name+"_adaptive_binning_p_values", "wb") as test_statistics_file:
				for score in score_dict[str(number_of_splits)]:
					test_statistics_file.write(str(score)+"\n")

			if X_values.shape[1]==2: adaptive_binning_plot(bin_boundaries_dict,signed_Scp2_dict,number_of_splits,X_values,name)
			histo_plot_pvalue(score_dict[str(number_of_splits)],50,"p value","Frequency","p value distribution "+ str(number_of_splits) + " splits",name+"_adaptive_binning")

<commit_msg>added random fluctuations of bin contents properly<commit_after>from __future__ import print_function
import sys
from chi2_plots import *
import random
import ast

"""
This script can be used to get the p value for the Miranda method (=chi squared). It takes input files with column vectors corresponding to
features and lables.
"""

print(__doc__)
import sys
sys.path.insert(0,'../..')
import os
from scipy import stats
import numpy as np
import matplotlib.pyplot as plt
import numpy.matlib
from matplotlib.colors import Normalize

from sklearn.preprocessing import StandardScaler

##############################################################################
# Setting parameters
#
#orig_name= sys.argv[1]
#number_of_splits_list= ast.literal_eval(sys.argv[2])
#print("number_of_splits_list : ", number_of_splits_list)
#dim_list = ast.literal_eval(sys.argv[3])
#comp_file_list_list = ast.literal_eval(sys.argv[4])

def chi2_adaptive_binning_systematics(orig_name, dim_list, comp_file_list_list,number_of_splits_list,systematics_fraction):
	sample1_name="original"
	sample2_name="modified"

	shuffling_seed = 100

	DEBUG = False

	##############################################################################
	max_number_of_splits = np.max(number_of_splits_list)
	for dim_index, dim_data in enumerate(dim_list):
		print("We are now in "+str(dim_data) + " Dimensions")
		#comp_file_list=[]
		comp_file_list = comp_file_list_list[dim_index]
		#for i in range(2):

			#comp_file_list.append((os.environ['MLToolsDir']+"/Dalitz/gaussian_samples/higher_dimensional_gauss/gauss_data/data_high" +str(dim_data)+"Dgauss_10000_0.5_0.1_0.0_{0}.txt".format(i),os.environ['MLToolsDir']+"/Dalitz/gaussian_samples/higher_dimensional_gauss/gauss_data/data_high"+str(dim_data)+"Dgauss_10000_0.5_0.1_0.01_{0}.txt".format(i)))
			#comp_file_list.append((os.environ['MLToolsDir']+"/Dalitz/gaussian_samples/legendre/legendre_data/data_legendre_contrib0__1__10__sample_{0}.txt".format(i),os.environ['MLToolsDir']+"/Dalitz/gaussian_samples/legendre/legendre_data/data_legendre_contrib0__1__9__sample_{0}.txt".format(i)))
			#comp_file_list.append((os.environ['MLToolsDir']+"/Dalitz/gaussian_samples/gaussian_same_projection_on_each_axis/gauss_data/gaussian_same_projection_on_each_axis_redefined_{1}D_1000_0.6_0.2_0.1_{0}.txt".format(i,dim_data),os.environ['MLToolsDir']+"/Dalitz/gaussian_samples/gaussian_same_projection_on_each_axis/gauss_data/gaussian_same_projection_on_each_axis_redefined_{1}D_1000_0.6_0.2_0.075_{0}.txt".format(i,dim_data)))
		#print(comp_file_list)

		score_dict = {}
		for number_of_splits in number_of_splits_list:
			score_dict[str(number_of_splits)]=[]

		for comp_file_0,comp_file_1 in comp_file_list:
			print("Operating of files :"+comp_file_0+"   "+comp_file_1)

			#extracts data from the files
			features_0=np.loadtxt(comp_file_0,dtype='d')
			features_1=np.loadtxt(comp_file_1,dtype='d')

			#determine how many data points are in each sample
			no_0=features_0.shape[0]
			no_1=features_1.shape[0]

			#Give all samples in file 0 the label 0 and in file 1 the feature 1
			label_0=np.zeros((no_0,1))
			label_1=np.ones((no_1,1))

			#Create an array containing samples and features.
			data_0=np.c_[features_0,label_0]
			data_1=np.c_[features_1,label_1]

			data=np.r_[data_0,data_1]

			no_dim = data.shape[1]-1
			assert (no_dim ==dim_data), 	"The dimension of the data is inconsistent {} vs {}".format(no_dim,dim_data)

			np.random.shuffle(data)

			#print("data.shape : ", data.shape)
			labels=data[:,-1]

			X_values= data[:,:-1]
			X_max   = np.amax(data,axis=0)[:-1]
			X_min   = np.amin(data,axis=0)[:-1]
			X_total_width = (np.subtract(X_max,X_min))
			del data

			#Scaling
			X_values = X_values - X_min[None,:]
			X_values = X_values / X_total_width[None,:]

			data = np.concatenate((X_values, labels[:,None]), axis=1)

			#print("X_values : ",X_values)
			starting_boundary = []
			for i in range(dim_data):
				starting_boundary.append([0.0,1.0])
			#Each key has the following stricture: # of splits and for each split if it was closer (a) or further away from (b) the origin. The original bin is "0"
			#For example "2ab" means it is the bin that was closer to the origin for the first split and further away for the second one.
			bin_boundaries_dict = {'0' : np.array(starting_boundary)}
			bin_points_dict = {'0' : data}

			for split_number in range(1,1+max_number_of_splits):
				for bin_key, bin_boundary in bin_boundaries_dict.items():
					if str(split_number-1) in bin_key:
						variances= np.var(bin_points_dict[bin_key][:,:-1], axis=0)
						#print("\nvariances : ", variances)
						dim_to_be_sliced = np.argmax(variances)
						#print("dim_to_be_sliced : ",dim_to_be_sliced)
						#print("bin_points_dict[bin_key] : ",bin_points_dict[bin_key])
						#print("bin_points_dict[bin_key][:,dim_to_be_sliced] : ",bin_points_dict[bin_key][:,dim_to_be_sliced])
						median = np.median(bin_points_dict[bin_key][:,dim_to_be_sliced])
						#print("median : ",median)

						a_bin_boundary, b_bin_boundary = bin_boundary.copy(), bin_boundary.copy()
						#print("a_bin_boundary : ",a_bin_boundary)
						a_bin_boundary[dim_to_be_sliced,1] = median
						b_bin_boundary[dim_to_be_sliced,0] = median
						bin_boundaries_dict[str(split_number)+bin_key[1:]+'a'] = a_bin_boundary
						bin_boundaries_dict[str(split_number)+bin_key[1:]+'b'] = b_bin_boundary

						a_points, b_points = [],[]
						for event_number in range(bin_points_dict[bin_key].shape[0]):
							if bin_points_dict[bin_key][event_number,dim_to_be_sliced] < median: a_points.append(bin_points_dict[bin_key][event_number,:].tolist())
							else: b_points.append(bin_points_dict[bin_key][event_number,:].tolist())

						bin_points_dict[str(split_number)+bin_key[1:]+'a'] = np.array(a_points)
						bin_points_dict[str(split_number)+bin_key[1:]+'b'] = np.array(b_points)
						#If a bin contains no particles it should be deleted
						if len(a_points)==0:
							del bin_points_dict[str(split_number)+bin_key[1:]+'a']
							del bin_boundaries_dict[str(split_number)+bin_key[1:]+'a']
                                                if len(b_points)==0:
                                                        del bin_points_dict[str(split_number)+bin_key[1:]+'b']
                                                        del bin_boundaries_dict[str(split_number)+bin_key[1:]+'b']

			check_key = '3aaa'
			#check_key = str(max_number_of_splits)
			#for i in range(max_number_of_splits): check_key += random.choice(['a','b'])
			#print("bin_boundaries_dict.keys() : ", bin_boundaries_dict.keys())
			#print("bin_boundaries_dict[check_key] : ",bin_boundaries_dict[check_key])
			#print("bin_points_dict.keys(): ", bin_points_dict.keys())
			#print("bin_points_dict[check_key][0,:] : ",bin_points_dict[check_key][0,:])

			bins_sample01_dict= {}
			signed_Scp2_dict= {}

			for number_of_splits in number_of_splits_list:
				print("\nnumber_of_splits : ",number_of_splits)
				bins_sample0, bins_sample1 = [] , []
				for bin_key, bin_points in bin_points_dict.items():
					if str(number_of_splits) in bin_key:
						labels_in_bin = bin_points[:,-1]
						#print("labels_in_bin : ",labels_in_bin)
						bin_sample0 = np.count_nonzero( labels_in_bin == 0)
						bin_sample1 = np.count_nonzero( labels_in_bin == 1)
						print("systematics_fraction : ",systematics_fraction,"\nbin_sample0 : ",bin_sample0)
						#simulate uncertainties
						if(bin_sample0!=0):bin_sample0 += int(round(np.random.normal(0.,systematics_fraction*bin_sample0)))
                                                if(bin_sample1!=0):bin_sample1 += int(round(np.random.normal(0.,systematics_fraction*bin_sample1)))
						bins_sample01_dict[bin_key]=[bin_sample0,bin_sample1]
						signed_Scp2_dict[bin_key] = np.square(float(bin_sample1-bin_sample0))/(float(bin_sample1)+float(bin_sample0)+np.square(float(bin_sample1)*systematics_fraction)+np.square(float(bin_sample1)*systematics_fraction))*np.sign(bin_sample1-bin_sample0)
						#print("\n\nbin_sample0 : ",bin_sample0, "\n bins_sample0 : ", bins_sample0 )
						#print("type(bin_sample0) : ",type(bin_sample0), "  type(bins_sample0) : ",type(bins_sample0))
						bins_sample0.append(bin_sample0)
						#print("  bins_sample0 : ", bins_sample0, "\n\n" )
						bins_sample1.append(bin_sample1)
				bins_sample0, bins_sample1 = np.array(bins_sample0,dtype=float), np.array(bins_sample1, dtype=float)
				print("bins_sample0 : ",bins_sample0,"\n bins_sample1 : ",bins_sample1)
				#element wise subtraction and division
				Scp2 = ((bins_sample1-bins_sample0)**2)/ (bins_sample1+bins_sample0+(systematics_fraction*bins_sample1)**2+(systematics_fraction*bins_sample0)**2 )
				#Scp2 =  np.divide(np.square(np.subtract(bins_sample1,bins_sample0)),np.add(bins_sample1,bins_sample0))
				if DEBUG:
					print(Scp2)

				#nansum ignores all the contributions that are Not A Number (NAN)
				Chi2 = np.nansum(Scp2)
				if DEBUG:
					print("Chi2")
					print(Chi2)
				dof=bins_sample0.shape[0]-1

				pvalue= 1 - stats.chi2.cdf(Chi2,dof)

				print("\nThe p value for Scp2 = ",Scp2," and Chi2 = ", Chi2, " is ",pvalue,"\n\n")
				if DEBUG:
					print(bins_sample0)
					print(bins_sample1)
					print("Chi2/dof : {0}".format(str(Chi2/dof)))

					print("pvalue : {0}".format(str(pvalue)))
				score_dict[str(number_of_splits)].append(pvalue)

		for number_of_splits in number_of_splits_list:
			name = orig_name + "_" +str(dim_data) + "D_" + str(number_of_splits) + "_splits"
			print("score_dict[{}] : ".format(number_of_splits), score_dict[str(number_of_splits)])
			with open(name+"_adaptive_binning_p_values", "wb") as test_statistics_file:
				for score in score_dict[str(number_of_splits)]:
					test_statistics_file.write(str(score)+"\n")

			if X_values.shape[1]==2: adaptive_binning_plot(bin_boundaries_dict,signed_Scp2_dict,number_of_splits,X_values,name)
			histo_plot_pvalue(score_dict[str(number_of_splits)],50,"p value","Frequency","p value distribution "+ str(number_of_splits) + " splits",name+"_adaptive_binning")

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

from odoo import fields, models, tools

class ProfitabilityAnalysis(models.Model):

    _name = "project.profitability.report"
    _description = "Project Profitability Analysis"
    _order = 'project_id, sale_line_id'
    _auto = False

    analytic_account_id = fields.Many2one('account.analytic.account', string='Analytic Account', readonly=True)
    project_id = fields.Many2one('project.project', string='Project', readonly=True)
    currency_id = fields.Many2one('res.currency', string='Project Currency', readonly=True)
    company_id = fields.Many2one('res.company', string='Project Company', readonly=True)
    user_id = fields.Many2one('res.users', string='Project Manager', readonly=True)
    partner_id = fields.Many2one('res.partner', string='Customer', readonly=True)
    # cost
    timesheet_unit_amount = fields.Float("Timesheet Unit Amount", digits=(16, 2), readonly=True, group_operator="sum")
    timesheet_cost = fields.Float("Timesheet Cost", digits=(16, 2), readonly=True, group_operator="sum")
    # sale revenue
    order_confirmation_date = fields.Datetime('Sales Order Confirmation Date', readonly=True)
    sale_line_id = fields.Many2one('sale.order.line', string='Sale Order Line', readonly=True)
    sale_order_id = fields.Many2one('sale.order', string='Sale Order', readonly=True)
    product_id = fields.Many2one('product.product', string='Product', readonly=True)
    amount_untaxed_to_invoice = fields.Float("Untaxed Amout To Invoice", digits=(16, 2), readonly=True, group_operator="sum")
    amount_untaxed_invoiced = fields.Float("Untaxed Amout Invoiced", digits=(16, 2), readonly=True, group_operator="sum")

    def init(self):
        tools.drop_view_if_exists(self._cr, self._table)
        query = """
            CREATE VIEW %s AS (
                WITH currency_rate as (%s)
                SELECT
                    ROW_NUMBER() OVER (ORDER BY P.id, SOL.id) AS id,
                    P.user_id AS user_id,
                    TIMESHEET_AMOUNT.project_id AS project_id,
                    TIMESHEET_AMOUNT.sale_line_id AS sale_line_id,
                    AA.id AS analytic_account_id,
                    AA.partner_id AS partner_id,
                    C.id AS company_id,
                    C.currency_id AS currency_id,
                    S.id AS sale_order_id,
                    S.date_order AS order_confirmation_date,
                    SOL.product_id AS product_id,
                    (SOL.price_reduce / COALESCE(CR.rate, 1.0)) * SOL.qty_to_invoice AS amount_untaxed_to_invoice,
                    (SOL.price_reduce / COALESCE(CR.rate, 1.0)) * SOL.qty_invoiced AS amount_untaxed_invoiced,
                    TIMESHEET_AMOUNT.timesheet_unit_amount AS timesheet_unit_amount,
                    TIMESHEET_AMOUNT.timesheet_cost AS timesheet_cost
                FROM project_project P
                    LEFT JOIN account_analytic_account AA ON P.analytic_account_id = AA.id
                    JOIN res_company C ON C.id = AA.company_id
                    LEFT JOIN
                        (
                            SELECT T.sale_line_id AS sale_line_id, T.project_id AS project_id
                            FROM project_task T
                            WHERE T.sale_line_id IS NOT NULL
                            UNION
                            SELECT P.sale_line_id AS sale_line_id, P.id AS project_id
                            FROM project_project P
                            WHERE P.sale_line_id IS NOT NULL
                        ) SOL_PER_PROJECT ON P.id = SOL_PER_PROJECT.project_id
                    RIGHT OUTER JOIN
                        (
                            SELECT
                                project_id AS project_id,
                                so_line AS sale_line_id,
                                array_agg(id),
                                SUM(TS.unit_amount) AS timesheet_unit_amount,
                                SUM(TS.amount) AS timesheet_cost
                            FROM account_analytic_line TS
                            WHERE project_id IS NOT NULL
                            GROUP BY project_id, so_line
                        ) TIMESHEET_AMOUNT ON TIMESHEET_AMOUNT.project_id = SOL_PER_PROJECT.project_id AND TIMESHEET_AMOUNT.sale_line_id = SOL_PER_PROJECT.sale_line_id
                    LEFT JOIN sale_order_line SOL ON SOL_PER_PROJECT.sale_line_id = SOL.id
                    LEFT JOIN sale_order S ON SOL.order_id = S.id
                    LEFT JOIN currency_rate CR ON (CR.currency_id = SOL.currency_id
                        AND CR.currency_id != C.currency_id
                        AND CR.company_id = SOL.company_id
                        AND CR.date_start <= COALESCE(S.date_order, now())
                        AND (CR.date_end IS NULL OR cr.date_end > COALESCE(S.date_order, now())))
            )
        """ % (self._table, self.env['res.currency']._select_companies_rates())
        self._cr.execute(query)
<commit_msg>[IMP] sale_timesheet: simplify the profitability report SQL query<commit_after># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

from odoo import fields, models, tools

class ProfitabilityAnalysis(models.Model):

    _name = "project.profitability.report"
    _description = "Project Profitability Analysis"
    _order = 'project_id, sale_line_id'
    _auto = False

    analytic_account_id = fields.Many2one('account.analytic.account', string='Analytic Account', readonly=True)
    project_id = fields.Many2one('project.project', string='Project', readonly=True)
    currency_id = fields.Many2one('res.currency', string='Project Currency', readonly=True)
    company_id = fields.Many2one('res.company', string='Project Company', readonly=True)
    user_id = fields.Many2one('res.users', string='Project Manager', readonly=True)
    partner_id = fields.Many2one('res.partner', string='Customer', readonly=True)
    # cost
    timesheet_unit_amount = fields.Float("Timesheet Unit Amount", digits=(16, 2), readonly=True, group_operator="sum")
    timesheet_cost = fields.Float("Timesheet Cost", digits=(16, 2), readonly=True, group_operator="sum")
    # sale revenue
    order_confirmation_date = fields.Datetime('Sales Order Confirmation Date', readonly=True)
    sale_line_id = fields.Many2one('sale.order.line', string='Sale Order Line', readonly=True)
    sale_order_id = fields.Many2one('sale.order', string='Sale Order', readonly=True)
    product_id = fields.Many2one('product.product', string='Product', readonly=True)
    amount_untaxed_to_invoice = fields.Float("Untaxed Amout To Invoice", digits=(16, 2), readonly=True, group_operator="sum")
    amount_untaxed_invoiced = fields.Float("Untaxed Amout Invoiced", digits=(16, 2), readonly=True, group_operator="sum")

    def init(self):
        tools.drop_view_if_exists(self._cr, self._table)
        query = """
            CREATE VIEW %s AS (
                WITH currency_rate as (%s)
                SELECT
                    ROW_NUMBER() OVER (ORDER BY P.id, SOL.id) AS id,
                    P.user_id AS user_id,
                    TIMESHEET_AMOUNT.project_id AS project_id,
                    TIMESHEET_AMOUNT.sale_line_id AS sale_line_id,
                    P.analytic_account_id AS analytic_account_id,
                    P.partner_id AS partner_id,
                    C.id AS company_id,
                    C.currency_id AS currency_id,
                    S.id AS sale_order_id,
                    S.date_order AS order_confirmation_date,
                    SOL.product_id AS product_id,
                    (SOL.price_reduce / COALESCE(CR.rate, 1.0)) * SOL.qty_to_invoice AS amount_untaxed_to_invoice,
                    (SOL.price_reduce / COALESCE(CR.rate, 1.0)) * SOL.qty_invoiced AS amount_untaxed_invoiced,
                    TIMESHEET_AMOUNT.timesheet_unit_amount AS timesheet_unit_amount,
                    TIMESHEET_AMOUNT.timesheet_cost AS timesheet_cost
                FROM project_project P
                    JOIN res_company C ON C.id = P.company_id
                    LEFT JOIN
                        (
                            SELECT T.sale_line_id AS sale_line_id, T.project_id AS project_id
                            FROM project_task T
                            WHERE T.sale_line_id IS NOT NULL
                            UNION
                            SELECT P.sale_line_id AS sale_line_id, P.id AS project_id
                            FROM project_project P
                            WHERE P.sale_line_id IS NOT NULL
                        ) SOL_PER_PROJECT ON P.id = SOL_PER_PROJECT.project_id
                    RIGHT OUTER JOIN
                        (
                            SELECT
                                project_id AS project_id,
                                so_line AS sale_line_id,
                                SUM(TS.unit_amount) AS timesheet_unit_amount,
                                SUM(TS.amount) AS timesheet_cost
                            FROM account_analytic_line TS
                            WHERE project_id IS NOT NULL
                            GROUP BY project_id, so_line
                        ) TIMESHEET_AMOUNT ON TIMESHEET_AMOUNT.project_id = SOL_PER_PROJECT.project_id AND TIMESHEET_AMOUNT.sale_line_id = SOL_PER_PROJECT.sale_line_id
                    LEFT JOIN sale_order_line SOL ON SOL_PER_PROJECT.sale_line_id = SOL.id
                    LEFT JOIN sale_order S ON SOL.order_id = S.id
                    LEFT JOIN currency_rate CR ON (CR.currency_id = SOL.currency_id
                        AND CR.currency_id != C.currency_id
                        AND CR.company_id = SOL.company_id
                        AND CR.date_start <= COALESCE(S.date_order, now())
                        AND (CR.date_end IS NULL OR cr.date_end > COALESCE(S.date_order, now())))
            )
        """ % (self._table, self.env['res.currency']._select_companies_rates())
        self._cr.execute(query)
<|endoftext|>


<commit_before>from django.db import models
from market import Market
from stock import Stock
from trader import Trader
from datetime import datetime

class Order(models.Model):
    market = models.ForeignKey(Market)
    stock = models.ForeignKey(Stock)
    trader = models.ForeignKey(Trader)

    ORDER_CHOICES = (
            ('B', 'Buy'),
            ('S', 'Sell'),
        )
    order = models.CharField(max_length=1, choices=ORDER_CHOICES)
    creation_time = models.DateTimeField(default=datetime.now())
    volume = models.IntegerField()
    price = models.DecimalField(max_digits=8, decimal_places=2)

    completed = models.BooleanField()
    completion_time = models.DateTimeField(blank=True, null=True)

    def __unicode__(self):
        ### MAKE THIS BETTER
        return unicode(self.order) + ': ' + unicode(self.stock.name)

    class Meta:
        app_label = 'market'
<commit_msg>fixed order unicode method<commit_after>from django.db import models
from market import Market
from stock import Stock
from trader import Trader
from datetime import datetime

class Order(models.Model):
    market = models.ForeignKey(Market)
    stock = models.ForeignKey(Stock)
    trader = models.ForeignKey(Trader)

    ORDER_CHOICES = (
            ('B', 'BUY'),
            ('S', 'SELL'),
        )
    order = models.CharField(max_length=1, choices=ORDER_CHOICES)
    creation_time = models.DateTimeField(default=datetime.now())
    volume = models.IntegerField()
    price = models.DecimalField(max_digits=8, decimal_places=2)

    completed = models.BooleanField()
    completion_time = models.DateTimeField(blank=True, null=True)

    def __unicode__(self):
        output = [self.get_order_display(), unicode(self.volume), 'of', unicode(self.stock.name), 'at', unicode(self.price)]
        return ' '.join(output)

    class Meta:
        app_label = 'market'
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
   Copyright 2014 The Trustees of Princeton University

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
"""

import ConfigParser

import os
import argparse
import sys
import StringIO
import traceback
import argparse
import logging

logging.basicConfig( format='[%(levelname)s] [%(module)s:%(lineno)d] %(message)s' )

log = logging.getLogger()

CONFIG_DIR = os.path.expanduser( "~/.syndicate" )
CONFIG_FILENAME = os.path.join(CONFIG_DIR, "syndicate.conf")

CONFIG_DESCRIPTION = "Syndicate Administration Tool"

OBJECT_DIR_NAMES = {
   "volume": "volumes",
   "user": "users",
   "gateway": "gateways",
   "syndicate": "syndicate"
}

CONFIG_DEFAULTS = {
   "MS_url": "http://127.0.0.1:8080",
   'trust_public_key': False,
   'debug': False
}

# 'syndicate' section of the config file
CONFIG_OPTIONS = {
   "MS_url":            ("-m", 1, "URL to your Syndicate MS"),
   "username":          ("-u", 1, "The ID of the Syndicate user to run as."),
   "volume_keys":       ("-V", 1, "Path to the directory holding the Volume public keys."),
   "gateway_keys":      ("-G", 1, "Path to the directory holding the Gateway private keys."),
   "user_keys":         ("-K", 1, "Path to the directory holding the User private keys."),
   "syndicate_keys":    ("-S", 1, "Path to the directory holding the Syndicate public keys."),
   "helpers":           ("-l", 1, "Path to the directory holding the Syndicate helper programs."),
   "config":            ("-c", 1, "Path to your config file (default is %s)." % CONFIG_FILENAME),
   "debug":             ("-d", 0, "Verbose debugging output"),
   "trust_public_key":  ("-t", 0, "If set, automatically trust the Syndicate public key if it is not yet trusted."),
   "params":            (None, "+", "Method name, followed by parameters (positional and keyword supported)."),
}

# 'helpers' section of the config file
HELPER_OPTIONS = {
   "fetch_user_cert":       (None, 0, "Program to fetch a user's certificate"),
   "fetch_gateway_cert":    (None, 0, "Program to fetch a gateway's certificate"),
   "fetch_volume_cert":     (None, 0, "Program to fetch a volume's certificate"),
   "fetch_cert_bundle":     (None, 0, "Program to fetch a volume's certificate bundle"),
   "fetch_driver":          (None, 0, "Program to fetch a gateway's driver"),
   "fetch_syndicate_pubkey":   (None, 0, "Program to fetch a Syndicate instance's signing key"),
   "validate_user_cert":    (None, 0, "Program to validate and authenticate a user's certificate")
}

class ArgLib(object):
   """
   Generic object to store temporary
   data between argument pre- and post-processing.
   """
   def __init__(self):
      pass

def serialize_config( config_data ):
   """
   Generate a .ini-formatted configuration string to e.g. save to disk.
   """
   return "[syndicate]\n" + "\n".join( ["%s=%s" % (config_key, config_value) for (config_key, config_value) in config_data.items()] )

def load_config( config_path, config_str, opts, config_header, config_options ):
   """
   Load configuration options from an ini-formatted string and from parsed command-line options.
   Prefer command-line options to config str options.

   Return a dict with the data.
   """

   config = None

   if config_str:
      config = ConfigParser.SafeConfigParser()
      config_fd = StringIO.StringIO( config_str )
      config_fd.seek( 0 )

      try:
         config.readfp( config_fd )
      except Exception, e:
         log.exception( e )
         return None

   ret = {}
   ret["_in_argv"] = []
   ret["_in_config"] = []

   # convert to dictionary, merging in argv opts
   for arg_opt in config_options.keys():
      if hasattr(opts, arg_opt) and getattr(opts, arg_opt) != None:
         ret[arg_opt] = getattr(opts, arg_opt)

         # force singleton...
         if isinstance(ret[arg_opt], list) and len(ret[arg_opt]) == 1 and config_options[arg_opt][1] == 1:
            ret[arg_opt] = ret[arg_opt][0]

         ret["_in_argv"].append( arg_opt )

      elif config is not None and config.has_option( config_header, arg_opt):
         ret[arg_opt] = config.get( config_header, arg_opt)

         ret["_in_config"].append( arg_opt )

   conf_dir = os.path.dirname( config_path )
   extend_key_paths( ret, conf_dir )
   return ret

def build_parser( progname, description, config_options ):
   """
   Make an argument parser for the program, given the name, description, and list of configuraiton options.
   """
   parser = argparse.ArgumentParser( prog=progname, description=description )

   for (config_option, (short_option, nargs, config_help)) in config_options.items():
      if not isinstance(nargs, int) or nargs >= 1:
         if short_option:
            # short option means 'typical' argument
            parser.add_argument( "--" + config_option, short_option, metavar=config_option, nargs=nargs, help=config_help)
         else:
            # no short option (no option in general) means accumulate
            parser.add_argument( config_option, metavar=config_option, type=str, nargs=nargs, help=config_help)
      else:
         if nargs >= 1:
            parser.add_argument( config_option, metavar=config_option, type=str, nargs=nargs, help=config_help)

         else:
            # no argument, but mark its existence
            parser.add_argument( "--" + config_option, short_option, action="store_true", help=config_help)

   return parser

def build_config( argv, description, config_header, config_options, conf_validator=None, opt_handlers={}, config_opt=None, allow_none=False ):
   """
   Construct and populate a config dict, using argv, a program description, a designated ini header that contains
   the requested fields, a description of the config fields.

   Return the dict with the options loaded.
   """
   parser = build_parser( argv[0], description, config_options )
   opts = parser.parse_args( argv[1:] )

   for opt, opt_cb in opt_handlers.items():
      if config_options.has_key( opt ) and hasattr( opts, opt ):
         # generate this option argument
         result = opt_cb( getattr(opts, opt) )
         setattr( opts, opt, result )

   config_text = None
   if config_opt is not None and hasattr(opts, config_opt):
      config_text = getattr(opts, config_opt)

   config = load_config( config_text, opts, config_header, config_options )

   if config == None:
      print >> sys.stderr, "Failed to load configuration"
      parser.print_help()
      return None

   if not allow_none:
      # remove any instances of None from config
      ks = config.keys()
      for k in ks:
         if config[k] is None:
            del config[k]

   if conf_validator is not None:
      rc = conf_validator( config )
      if rc != 0:
         print >> sys.stderr, "Invalid arguments"
         parser.print_help()
         return None

   return config

def object_file_path( config, object_type, object_id ):
   """
   Get the path to an object's file.
   """
   dirname = config.get( OBJECT_DIR_NAMES.get( object_type, None ) )
   if dirname == None:
      raise Exception("Could not get directory for type '%s'" % object_type)

   return os.path.join( dirname, str(object_id) )

def object_key_path( config, object_type, object_id, public=False, no_suffix=False ):
   """
   Get the path to an object's public or private key.
   Add default suffixes, unless directed otherwise.
   Return the path to the key file.
   """
   suffix = ".pkey"
   if public:
      suffix = ".pub"

   if no_suffix:
      suffix = ""

   return object_file_path( config, object_type, object_id ) + suffix

def parse_args( config, method_name, args, kw, lib ):
   """
   Parse API method arguments.
   Look up the method in the API, get its list of arguments and its
   argument parser, and go forth and load them.
   Load positional arguments in order, and then load keyword arguments
   in lexical order by argument name.
   Accumulate ancillary data from the parsing process as well, i.e.
   to be used for subsequent processing.

   Return (parsed argument list, parsed keyword dictionary, ancillary data dictionary)
   Raise an exception if we failed somehow.
   """

   import syndicate.ms.api as api
   method = api.get_method( method_name )
   try:
      if method.parse_args:
         args, kw, extras = method.parse_args( config, method_name, method.argspec, args, kw, lib )
         return args, kw, extras
      else:
         return args, kw, {}
   except Exception, e:
      log.exception(e)
      log.error("Failed to parse method arguments for '%s'" % method_name)
      raise

def validate_args( method_name, args, kw ):
   """
   Validate an API method's arguments.
   * verify that we got the right number of positional arguments.

   Return True if valid
   Raise an exception on error
   """
   import syndicate.ms.api as api
   method = api.get_method( method_name )

   arg_len = len(method.argspec.args)
   def_len = 0

   if method.argspec.defaults is not None:
      def_len = len(method.argspec.defaults)

   if len(args) != arg_len - def_len:
      raise Exception("Method '%s' expects %s arguments; got %s (%s)" % (method_name, arg_len - def_len, len(args), args))

   return True

def extend_key_paths( config, base_dir ):
   """
   Extend all key paths for all object types by ensuring
   that they are all prefixed with base_dir.
   """

   for (_, keydir) in OBJECT_DIR_NAMES.items():
      path = config.get( keydir, None )
      if path == None:
         path = keydir

      if not os.path.isabs( path ):
         # not absolute, so append with base dir
         path = os.path.join( base_dir, path )
         config[keydir] = path

def fill_defaults( config ):
   """
   Fill a config dict with default values.
   """
   global OBJECT_DIR_NAMES, CONFIG_DEFAULTS

   config.update( CONFIG_DEFAULTS )

   extend_key_paths( config, CONFIG_DIR )
   config['_in_argv'] = []
   config['_in_config'] = []

   return

def usage( progname ):
   """
   Print usage and exit.
   """
   parser = build_parser( progname, CONFIG_DESCRIPTION )
   parser.print_help()
   sys.exit(1)
<commit_msg>configure default helpers<commit_after>#!/usr/bin/env python

"""
   Copyright 2014 The Trustees of Princeton University

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
"""

import ConfigParser

import os
import argparse
import sys
import StringIO
import traceback
import argparse
import logging

logging.basicConfig( format='[%(levelname)s] [%(module)s:%(lineno)d] %(message)s' )

log = logging.getLogger()

CONFIG_DIR = "~/.syndicate"
CONFIG_FILENAME = "~/.syndicate/syndicate.conf"

def default_config_dir():
    global CONFIG_DIR
    return os.path.expanduser(CONFIG_DIR)

def default_config_path():
    global CONFIG_FILENAME
    return os.path.expanduser(CONFIG_FILENAME)

CONFIG_DESCRIPTION = "Syndicate Administration Tool"

OBJECT_DIR_NAMES = {
   "volume": "volumes",
   "user": "users",
   "gateway": "gateways",
   "syndicate": "syndicate",
   "certs": "certs",
   "data": "data",
   "drivers": "drivers",
   "logs": "logs"
}

CONFIG_SYNDICATE_KEYS = [
   "MS_url",
   "username",
   "volumes",
   "users",
   "gateways",
   "syndicate",
   "drivers",
   "logs",
   "data",
   "certs"
]

CONFIG_DEFAULTS = {
   "MS_url": "http://127.0.0.1:8080",
   'trust_public_key': False,
   'debug': False,
   'helpers': {
       'fetch_user_cert': "/usr/local/lib/syndicate/fetch_user_cert",
       'fetch_gateway_cert': "/usr/local/lib/syndicate/fetch_gateway_cert",
       'fetch_volume_cert': "/usr/local/lib/syndicate/fetch_volume_cert",
       'fetch_cert_bundle': "/usr/local/lib/syndicate/fetch_cert_bundle",
       'fetch_driver': "/usr/local/lib/syndicate/fetch_driver",
       'fetch_syndicate_pubkey': "/usr/local/lib/syndicate/fetch_syndicate_pubkey",
       'validate_user_cert': "/usr/local/lib/syndicate/validate_user_cert",
       'certs_reload': "/usr/local/lib/syndicate/certs_reload",
       'driver_reload': "/usr/local/lib/syndicate/driver_reload"
    }
}

# 'syndicate' section of the config file
CONFIG_OPTIONS = {
   "MS_url":            ("-m", 1, "URL to your Syndicate MS"),
   "username":          ("-u", 1, "The ID of the Syndicate user to run as."),
   "volume_keys":       ("-V", 1, "Path to the directory holding the Volume public keys."),
   "gateway_keys":      ("-G", 1, "Path to the directory holding the Gateway private keys."),
   "user_keys":         ("-K", 1, "Path to the directory holding the User private keys."),
   "syndicate_keys":    ("-S", 1, "Path to the directory holding the Syndicate public keys."),
   "certs_dir":         ("-C", 1, "Path to the directory to store remotely-fetched certificates."),
   "helpers":           ("-l", 1, "Path to the directory holding the Syndicate helper programs."),
   "config":            ("-c", 1, "Path to your config file (default is %s)." % CONFIG_FILENAME),
   "debug":             ("-d", 0, "Verbose debugging output"),
   "trust_public_key":  ("-t", 0, "If set, automatically trust the Syndicate public key if it is not yet trusted."),
   "params":            (None, "+", "Method name, followed by parameters (positional and keyword supported)."),
}

# 'helpers' section of the config file
HELPER_OPTIONS = {
   "fetch_user_cert":       (None, 0, "Program to fetch a user's certificate"),
   "fetch_gateway_cert":    (None, 0, "Program to fetch a gateway's certificate"),
   "fetch_volume_cert":     (None, 0, "Program to fetch a volume's certificate"),
   "fetch_cert_bundle":     (None, 0, "Program to fetch a volume's certificate bundle"),
   "fetch_driver":          (None, 0, "Program to fetch a gateway's driver"),
   "fetch_syndicate_pubkey":   (None, 0, "Program to fetch a Syndicate instance's signing key"),
   "validate_user_cert":    (None, 0, "Program to validate and authenticate a user's certificate"),
   "certs_reload":          (None, 0, "Program to repopulate the certificate cache"),
   "driver_reload":         (None, 0, "Program to repopulate the driver cache")
}

class ArgLib(object):
   """
   Generic object to store temporary
   data between argument pre- and post-processing.
   """
   def __init__(self):
      pass

def serialize_config( config_data ):
   """
   Generate a .ini-formatted configuration string to e.g. save to disk.
   """
   return "[syndicate]\n" + "\n".join( ["%s=%s" % (config_key, config_value) for (config_key, config_value) in config_data.items()] )

def load_config( config_path, config_str, opts, config_header, config_options ):
   """
   Load configuration options from an ini-formatted string and from parsed command-line options.
   Prefer command-line options to config str options.

   Return a dict with the data.
   """

   config = None

   if config_str:
      config = ConfigParser.SafeConfigParser()
      config_fd = StringIO.StringIO( config_str )
      config_fd.seek( 0 )

      try:
         config.readfp( config_fd )
      except Exception, e:
         log.exception( e )
         return None

   ret = {}
   ret["_in_argv"] = []
   ret["_in_config"] = []

   # convert to dictionary, merging in argv opts
   for arg_opt in config_options.keys():
      if hasattr(opts, arg_opt) and getattr(opts, arg_opt) != None:
         ret[arg_opt] = getattr(opts, arg_opt)

         # force singleton...
         if isinstance(ret[arg_opt], list) and len(ret[arg_opt]) == 1 and config_options[arg_opt][1] == 1:
            ret[arg_opt] = ret[arg_opt][0]

         ret["_in_argv"].append( arg_opt )

      elif config is not None and config.has_option( config_header, arg_opt):
         ret[arg_opt] = config.get( config_header, arg_opt)

         ret["_in_config"].append( arg_opt )

   conf_dir = os.path.dirname( config_path )
   extend_key_paths( ret, conf_dir )
   return ret

def build_parser( progname, description, config_options ):
   """
   Make an argument parser for the program, given the name, description, and list of configuraiton options.
   """
   parser = argparse.ArgumentParser( prog=progname, description=description )

   for (config_option, (short_option, nargs, config_help)) in config_options.items():
      if not isinstance(nargs, int) or nargs >= 1:
         if short_option:
            # short option means 'typical' argument
            parser.add_argument( "--" + config_option, short_option, metavar=config_option, nargs=nargs, help=config_help)
         else:
            # no short option (no option in general) means accumulate
            parser.add_argument( config_option, metavar=config_option, type=str, nargs=nargs, help=config_help)
      else:
         if nargs >= 1:
            parser.add_argument( config_option, metavar=config_option, type=str, nargs=nargs, help=config_help)

         else:
            # no argument, but mark its existence
            parser.add_argument( "--" + config_option, short_option, action="store_true", help=config_help)

   return parser

def build_config( argv, description, config_header, config_options, conf_validator=None, opt_handlers={}, config_opt=None, allow_none=False ):
   """
   Construct and populate a config dict, using argv, a program description, a designated ini header that contains
   the requested fields, a description of the config fields.

   Return the dict with the options loaded.
   """
   parser = build_parser( argv[0], description, config_options )
   opts = parser.parse_args( argv[1:] )

   for opt, opt_cb in opt_handlers.items():
      if config_options.has_key( opt ) and hasattr( opts, opt ):
         # generate this option argument
         result = opt_cb( getattr(opts, opt) )
         setattr( opts, opt, result )

   config_text = None
   if config_opt is not None and hasattr(opts, config_opt):
      config_text = getattr(opts, config_opt)

   config = load_config( config_text, opts, config_header, config_options )

   if config == None:
      print >> sys.stderr, "Failed to load configuration"
      parser.print_help()
      return None

   if not allow_none:
      # remove any instances of None from config
      ks = config.keys()
      for k in ks:
         if config[k] is None:
            del config[k]

   if conf_validator is not None:
      rc = conf_validator( config )
      if rc != 0:
         print >> sys.stderr, "Invalid arguments"
         parser.print_help()
         return None

   return config

def object_file_path( config, object_type, object_id ):
   """
   Get the path to an object's file.
   """
   dirname = config.get( OBJECT_DIR_NAMES.get( object_type, None ) )
   if dirname == None:
      raise Exception("Could not get directory for type '%s' (config = %s, object_dir_names = %s)" % (object_type, config, OBJECT_DIR_NAMES))

   return os.path.join( dirname, str(object_id) )

def object_key_path( config, object_type, object_id, public=False, no_suffix=False ):
   """
   Get the path to an object's public or private key.
   Add default suffixes, unless directed otherwise.
   Return the path to the key file.
   """
   suffix = ".pkey"
   if public:
      suffix = ".pub"

   if no_suffix:
      suffix = ""

   return object_file_path( config, object_type, object_id ) + suffix

def parse_args( config, method_name, args, kw, lib ):
   """
   Parse API method arguments.
   Look up the method in the API, get its list of arguments and its
   argument parser, and go forth and load them.
   Load positional arguments in order, and then load keyword arguments
   in lexical order by argument name.
   Accumulate ancillary data from the parsing process as well, i.e.
   to be used for subsequent processing.

   Return (parsed argument list, parsed keyword dictionary, ancillary data dictionary)
   Raise an exception if we failed somehow.
   """

   import syndicate.ms.api as api
   method = api.get_method( method_name )
   try:
      if method.parse_args:
         args, kw, extras = method.parse_args( config, method_name, method.argspec, args, kw, lib )
         return args, kw, extras
      else:
         return args, kw, {}
   except Exception, e:
      log.exception(e)
      log.error("Failed to parse method arguments for '%s'" % method_name)
      raise

def validate_args( method_name, args, kw ):
   """
   Validate an API method's arguments.
   * verify that we got the right number of positional arguments.

   Return True if valid
   Raise an exception on error
   """
   import syndicate.ms.api as api
   method = api.get_method( method_name )

   arg_len = len(method.argspec.args)
   def_len = 0

   if method.argspec.defaults is not None:
      def_len = len(method.argspec.defaults)

   if len(args) != arg_len - def_len:
      raise Exception("Method '%s' expects %s arguments; got %s (%s)" % (method_name, arg_len - def_len, len(args), args))

   return True

def extend_key_paths( config, base_dir ):
   """
   Extend all key paths for all object types by ensuring
   that they are all prefixed with base_dir.
   """

   for (_, keydir) in OBJECT_DIR_NAMES.items():
      path = config.get( keydir, None )
      if path == None:
         path = keydir

      if not os.path.isabs( path ):
         # not absolute, so append with base dir
         path = os.path.join( base_dir, path )
         config[keydir] = path

def fill_defaults( config ):
   """
   Fill a config dict with default values.
   """
   global CONFIG_DEFAULTS

   config.update( CONFIG_DEFAULTS )

   extend_key_paths( config, default_config_path())
   config['_in_argv'] = []
   config['_in_config'] = []

   return

def usage( progname ):
   """
   Print usage and exit.
   """
   parser = build_parser( progname, CONFIG_DESCRIPTION )
   parser.print_help()
   sys.exit(1)
<|endoftext|>


<commit_before>### This script assigns annual above and belowground biomass gain rates (in the units of IPCC Table 4.9 (currently tonnes
### biomass/ha/yr)) to non-mangrove natural forest pixels.
### The inputs are continent-ecozone tiles and natural forest age category tiles, as well as IPCC Table 4.9, formatted
### for easy ingestion by pandas.
### Essentially, this does some processing of the IPCC gain rate table, then uses it as a dictionary that it applies
### to every pixel in every tile.
### Each continent-ecozone-forest age category combination gets its own code, which matches the codes in the
### processed IPCC table.
### Belowground biomass gain rate is a constant proportion of aboveground biomass gain rate, again according to IPCC tables.

import datetime
import numpy as np
import rasterio
import subprocess
import os
import sys
sys.path.append('../')
import constants_and_names as cn
import universal_util as uu

# Necessary to suppress a pandas error later on
np.set_printoptions(threshold=np.nan)

def annual_gain_rate(tile_id, gain_table_dict):

    # Converts the forest age category decision tree output values to the three age categories--
    # 10000: primary forest; 20000: secondary forest > 20 years; 30000: secondary forest <= 20 years
    # These are five digits so they can easily be added to the four digits of the continent-ecozone code to make unique codes
    # for each continent-ecozone-age combination.
    # The key in the dictionary is the forest age category decision tree endpoints.
    age_dict = {0: 0, 1: 20000, 2: 20000, 3: 10000, 4: 30000, 5: 20000, 6: 10000, 7: 30000, 8: 30000, 9: 30000, 10: 30000}

    print "Processing:", tile_id

    # Start time
    start = datetime.datetime.now()

    # Names of the forest age category, continent-ecozone, and mangrove biomass tiles
    age_cat = '{0}_{1}.tif'.format(tile_id, cn.pattern_age_cat_natrl_forest)
    cont_eco = '{0}_{1}.tif'.format(tile_id, cn.pattern_cont_eco_processed)

    # Name of the output natural forest gain rate tile, before mangroves are masked out
    AGB_natrl_forest_gain_rate = '{0}_{1}.tif'.format(tile_id, cn.pattern_annual_gain_AGB_natrl_forest)

    BGB_natrl_forest_gain_rate = '{0}_{1}.tif'.format(tile_id, cn.pattern_annual_gain_BGB_natrl_forest)

    # Aboveground mangrove biomass tile
    mangrove_biomass = '{0}_{1}.tif'.format(tile_id, cn.pattern_mangrove_biomass_2000)

    # Planted forest gain rate tile
    planted_forest_gain = '{0}_{1}.tif'.format(tile_id, cn.pattern_annual_gain_AGC_planted_forest_full_extent)

    print "  Reading input files and creating aboveground and belowground biomass gain rates for {}".format(tile_id)

    cont_eco_src = rasterio.open(cont_eco)

    # Grabs metadata about the tif, like its location/projection/cellsize
    kwargs = cont_eco_src.meta

    # Grabs the windows of the tile (stripes) to iterate over the entire tif without running out of memory
    windows = cont_eco_src.block_windows(1)

    age_cat_src = rasterio.open(age_cat)

    try:
        mangrove_src = rasterio.open(mangrove_biomass)
    except:
        print "No mangrove tile for {}".format(tile_id)

    try:
        planted_forest_src = rasterio.open(planted_forest_gain)
    except:
        print "No planted forest tile for {}".format(tile_id)

    # Updates kwargs for the output dataset.
    # Need to update data type to float 32 so that it can handle fractional gain rates
    kwargs.update(
        driver='GTiff',
        count=1,
        compress='lzw',
        nodata=0,
        dtype='float32'
    )

    dst_above = rasterio.open(AGB_natrl_forest_gain_rate, 'w', **kwargs)

    dst_below = rasterio.open(BGB_natrl_forest_gain_rate, 'w', **kwargs)

    # Iterates across the windows (1 pixel strips) of the input tile
    for idx, window in windows:

        # Creates windows for each input raster
        cont_eco = cont_eco_src.read(1, window=window)
        age_cat = age_cat_src.read(1, window=window)

        # Recodes the input forest age category array with 10 different values into the 3 actual age categories
        age_recode = np.vectorize(age_dict.get)(age_cat)

        # Adds the age category codes to the continent-ecozone codes to create an array of unique continent-ecozone-age codes
        cont_eco_age = cont_eco + age_recode

        # Converts the continent-ecozone array to float so that the values can be replaced with fractional gain rates
        gain_rate_AGB = cont_eco_age.astype('float32')

        # Applies the dictionary of continent-ecozone-age gain rates to the continent-ecozone-age array to
        # get annual gain rates (metric tons aboveground biomass/yr) for each pixel
        for key, value in gain_table_dict.iteritems():
            gain_rate_AGB[gain_rate_AGB == key] = value

        if os.path.exists(mangrove_biomass):

            mangrove_AGB = mangrove_src.read(1, window=window)

            nodata = uu.get_raster_nodata_value(mangrove_biomass)

            # Reclassifies mangrove biomass to 1 or 0 to make a mask of mangrove pixels.
            # Ultimately, only these pixels (ones with mangrove biomass) will get values.
            mangrove_AGB[mangrove_AGB != nodata] = 0

            gain_rate_AGB = gain_rate_AGB * mangrove_AGB

        if os.path.exists(planted_forest_gain):

            planted_forest = planted_forest_src.read(1, window=window)

            nodata = uu.get_raster_nodata_value(planted_forest_gain)

            # Reclassifies mangrove biomass to 1 or 0 to make a mask of mangrove pixels.
            # Ultimately, only these pixels (ones with mangrove biomass) will get values.
            planted_forest[planted_forest != nodata] = 0

            gain_rate_AGB = gain_rate_AGB * planted_forest

        # Writes the output window to the output file
        dst_above.write_band(1, gain_rate_AGB, window=window)

        gain_rate_BGB = gain_rate_AGB * cn.below_to_above_natrl_forest

        # Writes the output window to the output file
        dst_below.write_band(1, gain_rate_BGB, window=window)

    # # Opens continent-ecozone tile
    # with rasterio.open(cont_eco) as cont_eco_src:
    #
    #     # Grabs metadata about the tif, like its location/projection/cellsize
    #     kwargs = cont_eco_src.meta
    #
    #     # Grabs the windows of the tile (stripes) to iterate over the entire tif without running out of memory
    #     windows = cont_eco_src.block_windows(1)
    #
    #     # Opens age category tile
    #     with rasterio.open(age_cat) as age_cat_src:
    #
    #         # Updates kwargs for the output dataset.
    #         # Need to update data type to float 32 so that it can handle fractional gain rates
    #         kwargs.update(
    #             driver='GTiff',
    #             count=1,
    #             compress='lzw',
    #             nodata=0,
    #             dtype='float32'
    #         )
    #
    #         # Opens the output aboveground biomass gain rate tile, giving it the arguments of the input tiles
    #         with rasterio.open(AGB_natrl_forest_gain_rate_unmasked, 'w', **kwargs) as dst_above:
    #
    #             # Iterates across the windows (1 pixel strips) of the input tile
    #             for idx, window in windows:
    #
    #                 # Creates windows for each input raster
    #                 cont_eco = cont_eco_src.read(1, window=window)
    #                 age_cat = age_cat_src.read(1, window=window)
    #
    #                 # Recodes the input forest age category array with 10 different values into the 3 actual age categories
    #                 age_recode = np.vectorize(age_dict.get)(age_cat)
    #
    #                 # Adds the age category codes to the continent-ecozone codes to create an array of unique continent-ecozone-age codes
    #                 cont_eco_age = cont_eco + age_recode
    #
    #                 # Converts the continent-ecozone-age array to float so that the values can be replaced with fractional gain rates
    #                 cont_eco_age = cont_eco_age.astype('float32')
    #
    #                 # Creates a new array which will have the annual aboveground gain rates
    #                 gain_rate_AGB = cont_eco_age
    #
    #                 # Applies the dictionary of continent-ecozone-age gain rates to the continent-ecozone-age array to
    #                 # get annual gain rates (metric tons aboveground biomass/yr) for each pixel
    #                 for key, value in gain_table_dict.iteritems():
    #                     gain_rate_AGB[gain_rate_AGB  == key] = value
    #
    #                 # Writes the output window to the output
    #                 dst_above.write_band(1, gain_rate_AGB , window=window)
    #
    #
    # # Checks if there are both mangroves and planted forests and masks out both
    # if os.path.exists(mangrove_biomass) & os.path.exists(planted_forest_gain):
    #
    #     # Mangrove biomass tiles that have the nodata pixels removed
    #     mangrove_reclass = '{0}_reclass_{1}.tif'.format(tile_id, cn.pattern_mangrove_biomass_2000)
    #
    #     # Removes the nodata values in the mangrove biomass rasters because having nodata values in the mangroves didn't work
    #     # in gdal_calc. The gdal_calc expression didn't know how to evaluate nodata values, so I had to remove them.
    #     print "  Removing nodata values in mangrove biomass raster {}".format(tile_id)
    #     cmd = ['gdal_translate', '-a_nodata', 'none', mangrove_biomass, mangrove_reclass]
    #     subprocess.check_call(cmd)
    #
    #     # Masks out the mangroves and the planted forests from the natural forest gain rate
    #     # Ideally this would be part of the rasterio/numpy operation (not its own gdal operation later) but I couldn't
    #     # figure out how to get the conditional mask working in numpy
    #     print "  Masking mangroves and planted forests from aboveground gain rate for tile {}".format(tile_id)
    #     mask_calc = '--calc=A*(B==0)*(C==0)'
    #     mask_outfilename = AGB_natrl_forest_gain_rate_masked
    #     mask_outfilearg = '--outfile={}'.format(mask_outfilename)
    #     cmd = ['gdal_calc.py', '-A', AGB_natrl_forest_gain_rate_unmasked, '-B', mangrove_reclass, '-C', planted_forest_gain,
    #            mask_calc, mask_outfilearg, '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW']
    #     subprocess.check_call(cmd)
    #
    # # Checks if the tile has mangrove pixels and masks them out if there are any
    # elif os.path.exists(mangrove_biomass):
    #
    #     # Mangrove biomass tiles that have the nodata pixels removed
    #     mangrove_reclass = '{0}_reclass_{1}.tif'.format(tile_id, cn.pattern_mangrove_biomass_2000)
    #
    #     # Removes the nodata values in the mangrove biomass rasters because having nodata values in the mangroves didn't work
    #     # in gdal_calc. The gdal_calc expression didn't know how to evaluate nodata values, so I had to remove them.
    #     print "  Removing nodata values in mangrove biomass raster {}".format(tile_id)
    #     cmd = ['gdal_translate', '-a_nodata', 'none', mangrove_biomass, mangrove_reclass]
    #     subprocess.check_call(cmd)
    #
    #     # Masks out the mangrove biomass from the natural forest gain rate
    #     # Ideally this would be part of the rasterio/numpy operation (not its own gdal operation later) but I couldn't
    #     # figure out how to get the mask working in numpy for some reason
    #     print "  Masking mangroves from aboveground gain rate for tile {}".format(tile_id)
    #     mask_calc = '--calc=A*(B==0)'
    #     mask_outfilename = AGB_natrl_forest_gain_rate_masked
    #     mask_outfilearg = '--outfile={}'.format(mask_outfilename)
    #     cmd = ['gdal_calc.py', '-A', AGB_natrl_forest_gain_rate_unmasked, '-B', mangrove_reclass, mask_calc, mask_outfilearg,
    #            '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW']
    #     subprocess.check_call(cmd)
    #
    # # Checks if the tile has planted forest pixels and masks them out if there are any
    # elif os.path.exists(planted_forest_gain):
    #
    #     # Masks out the mangrove biomass from the natural forest gain rate
    #     # Ideally this would be part of the rasterio/numpy operation (not its own gdal operation later) but I couldn't
    #     # figure out how to get the mask working in numpy for some reason
    #     print "  Masking planted forests from aboveground gain rate for tile {}".format(tile_id)
    #     mask_calc = '--calc=A*(B==0)'
    #     mask_outfilename = AGB_natrl_forest_gain_rate_masked
    #     mask_outfilearg = '--outfile={}'.format(mask_outfilename)
    #     cmd = ['gdal_calc.py', '-A', AGB_natrl_forest_gain_rate_unmasked, '-B', planted_forest_gain, mask_calc, mask_outfilearg,
    #            '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW']
    #     subprocess.check_call(cmd)
    #
    # else:
    #
    #     os.rename(AGB_natrl_forest_gain_rate_unmasked, AGB_natrl_forest_gain_rate_masked)

    # # Calculates belowground biomass rate from aboveground biomass rate
    # print "  Creating belowground biomass gain rate for tile {}".format(tile_id)
    # above_to_below_calc = '--calc=(A>0)*A*{}'.format(cn.below_to_above_natrl_forest)
    # below_outfilename = '{0}_{1}.tif'.format(tile_id, cn.pattern_annual_gain_BGB_natrl_forest)
    # below_outfilearg = '--outfile={}'.format(below_outfilename)
    # cmd = ['gdal_calc.py', '-A', AGB_natrl_forest_gain_rate_unmasked, above_to_below_calc, below_outfilearg,
    #        '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW']
    # subprocess.check_call(cmd)

    end = datetime.datetime.now()
    elapsed_time = end-start

    print "  Processing time for tile", tile_id, ":", elapsed_time
<commit_msg>Attemping to make natural forest gain rate entirely rasterio based and not use gdal.<commit_after>### This script assigns annual above and belowground biomass gain rates (in the units of IPCC Table 4.9 (currently tonnes
### biomass/ha/yr)) to non-mangrove natural forest pixels.
### The inputs are continent-ecozone tiles and natural forest age category tiles, as well as IPCC Table 4.9, formatted
### for easy ingestion by pandas.
### Essentially, this does some processing of the IPCC gain rate table, then uses it as a dictionary that it applies
### to every pixel in every tile.
### Each continent-ecozone-forest age category combination gets its own code, which matches the codes in the
### processed IPCC table.
### Belowground biomass gain rate is a constant proportion of aboveground biomass gain rate, again according to IPCC tables.

import datetime
import numpy as np
import rasterio
import subprocess
import os
import sys
sys.path.append('../')
import constants_and_names as cn
import universal_util as uu

# Necessary to suppress a pandas error later on
np.set_printoptions(threshold=np.nan)

def annual_gain_rate(tile_id, gain_table_dict):

    # Converts the forest age category decision tree output values to the three age categories--
    # 10000: primary forest; 20000: secondary forest > 20 years; 30000: secondary forest <= 20 years
    # These are five digits so they can easily be added to the four digits of the continent-ecozone code to make unique codes
    # for each continent-ecozone-age combination.
    # The key in the dictionary is the forest age category decision tree endpoints.
    age_dict = {0: 0, 1: 20000, 2: 20000, 3: 10000, 4: 30000, 5: 20000, 6: 10000, 7: 30000, 8: 30000, 9: 30000, 10: 30000}

    print "Processing:", tile_id

    # Start time
    start = datetime.datetime.now()

    # Names of the forest age category, continent-ecozone, and mangrove biomass tiles
    age_cat = '{0}_{1}.tif'.format(tile_id, cn.pattern_age_cat_natrl_forest)
    cont_eco = '{0}_{1}.tif'.format(tile_id, cn.pattern_cont_eco_processed)

    # Name of the output natural forest gain rate tile, before mangroves are masked out
    AGB_natrl_forest_gain_rate = '{0}_{1}.tif'.format(tile_id, cn.pattern_annual_gain_AGB_natrl_forest)

    BGB_natrl_forest_gain_rate = '{0}_{1}.tif'.format(tile_id, cn.pattern_annual_gain_BGB_natrl_forest)

    # Aboveground mangrove biomass tile
    mangrove_biomass = '{0}_{1}.tif'.format(tile_id, cn.pattern_mangrove_biomass_2000)

    # Planted forest gain rate tile
    planted_forest_gain = '{0}_{1}.tif'.format(tile_id, cn.pattern_annual_gain_AGC_planted_forest_full_extent)

    print "  Reading input files and creating aboveground and belowground biomass gain rates for {}".format(tile_id)

    cont_eco_src = rasterio.open(cont_eco)

    # Grabs metadata about the tif, like its location/projection/cellsize
    kwargs = cont_eco_src.meta

    # Grabs the windows of the tile (stripes) to iterate over the entire tif without running out of memory
    windows = cont_eco_src.block_windows(1)

    age_cat_src = rasterio.open(age_cat)

    try:
        mangrove_src = rasterio.open(mangrove_biomass)
    except:
        print "No mangrove tile for {}".format(tile_id)

    try:
        planted_forest_src = rasterio.open(planted_forest_gain)
    except:
        print "No planted forest tile for {}".format(tile_id)

    # Updates kwargs for the output dataset.
    # Need to update data type to float 32 so that it can handle fractional gain rates
    kwargs.update(
        driver='GTiff',
        count=1,
        compress='lzw',
        nodata=0,
        dtype='float32'
    )

    dst_above = rasterio.open(AGB_natrl_forest_gain_rate, 'w', **kwargs)

    dst_below = rasterio.open(BGB_natrl_forest_gain_rate, 'w', **kwargs)

    # Iterates across the windows (1 pixel strips) of the input tile
    for idx, window in windows:

        # Creates windows for each input raster
        cont_eco = cont_eco_src.read(1, window=window)
        age_cat = age_cat_src.read(1, window=window)

        # Recodes the input forest age category array with 10 different values into the 3 actual age categories
        age_recode = np.vectorize(age_dict.get)(age_cat)

        # Adds the age category codes to the continent-ecozone codes to create an array of unique continent-ecozone-age codes
        cont_eco_age = cont_eco + age_recode

        # Converts the continent-ecozone array to float so that the values can be replaced with fractional gain rates
        gain_rate_AGB = cont_eco_age.astype('float32')

        # Applies the dictionary of continent-ecozone-age gain rates to the continent-ecozone-age array to
        # get annual gain rates (metric tons aboveground biomass/yr) for each pixel
        for key, value in gain_table_dict.iteritems():
            gain_rate_AGB[gain_rate_AGB == key] = value

        if os.path.exists(mangrove_biomass):

            mangrove_AGB = mangrove_src.read(1, window=window)

            nodata = uu.get_raster_nodata_value(mangrove_biomass)

            # Reclassifies mangrove biomass to 1 or 0 to make a mask of mangrove pixels.
            # Ultimately, only these pixels (ones with mangrove biomass) will get values.
            mangrove_AGB[mangrove_AGB > 0] = 0

            gain_rate_AGB = gain_rate_AGB * mangrove_AGB

        if os.path.exists(planted_forest_gain):

            planted_forest = planted_forest_src.read(1, window=window)

            nodata = uu.get_raster_nodata_value(planted_forest_gain)

            # Reclassifies mangrove biomass to 1 or 0 to make a mask of mangrove pixels.
            # Ultimately, only these pixels (ones with mangrove biomass) will get values.
            planted_forest[planted_forest > 0] = 0

            gain_rate_AGB = gain_rate_AGB * planted_forest

        # Writes the output window to the output file
        dst_above.write_band(1, gain_rate_AGB, window=window)

        gain_rate_BGB = gain_rate_AGB * cn.below_to_above_natrl_forest

        # Writes the output window to the output file
        dst_below.write_band(1, gain_rate_BGB, window=window)

    # # Opens continent-ecozone tile
    # with rasterio.open(cont_eco) as cont_eco_src:
    #
    #     # Grabs metadata about the tif, like its location/projection/cellsize
    #     kwargs = cont_eco_src.meta
    #
    #     # Grabs the windows of the tile (stripes) to iterate over the entire tif without running out of memory
    #     windows = cont_eco_src.block_windows(1)
    #
    #     # Opens age category tile
    #     with rasterio.open(age_cat) as age_cat_src:
    #
    #         # Updates kwargs for the output dataset.
    #         # Need to update data type to float 32 so that it can handle fractional gain rates
    #         kwargs.update(
    #             driver='GTiff',
    #             count=1,
    #             compress='lzw',
    #             nodata=0,
    #             dtype='float32'
    #         )
    #
    #         # Opens the output aboveground biomass gain rate tile, giving it the arguments of the input tiles
    #         with rasterio.open(AGB_natrl_forest_gain_rate_unmasked, 'w', **kwargs) as dst_above:
    #
    #             # Iterates across the windows (1 pixel strips) of the input tile
    #             for idx, window in windows:
    #
    #                 # Creates windows for each input raster
    #                 cont_eco = cont_eco_src.read(1, window=window)
    #                 age_cat = age_cat_src.read(1, window=window)
    #
    #                 # Recodes the input forest age category array with 10 different values into the 3 actual age categories
    #                 age_recode = np.vectorize(age_dict.get)(age_cat)
    #
    #                 # Adds the age category codes to the continent-ecozone codes to create an array of unique continent-ecozone-age codes
    #                 cont_eco_age = cont_eco + age_recode
    #
    #                 # Converts the continent-ecozone-age array to float so that the values can be replaced with fractional gain rates
    #                 cont_eco_age = cont_eco_age.astype('float32')
    #
    #                 # Creates a new array which will have the annual aboveground gain rates
    #                 gain_rate_AGB = cont_eco_age
    #
    #                 # Applies the dictionary of continent-ecozone-age gain rates to the continent-ecozone-age array to
    #                 # get annual gain rates (metric tons aboveground biomass/yr) for each pixel
    #                 for key, value in gain_table_dict.iteritems():
    #                     gain_rate_AGB[gain_rate_AGB  == key] = value
    #
    #                 # Writes the output window to the output
    #                 dst_above.write_band(1, gain_rate_AGB , window=window)
    #
    #
    # # Checks if there are both mangroves and planted forests and masks out both
    # if os.path.exists(mangrove_biomass) & os.path.exists(planted_forest_gain):
    #
    #     # Mangrove biomass tiles that have the nodata pixels removed
    #     mangrove_reclass = '{0}_reclass_{1}.tif'.format(tile_id, cn.pattern_mangrove_biomass_2000)
    #
    #     # Removes the nodata values in the mangrove biomass rasters because having nodata values in the mangroves didn't work
    #     # in gdal_calc. The gdal_calc expression didn't know how to evaluate nodata values, so I had to remove them.
    #     print "  Removing nodata values in mangrove biomass raster {}".format(tile_id)
    #     cmd = ['gdal_translate', '-a_nodata', 'none', mangrove_biomass, mangrove_reclass]
    #     subprocess.check_call(cmd)
    #
    #     # Masks out the mangroves and the planted forests from the natural forest gain rate
    #     # Ideally this would be part of the rasterio/numpy operation (not its own gdal operation later) but I couldn't
    #     # figure out how to get the conditional mask working in numpy
    #     print "  Masking mangroves and planted forests from aboveground gain rate for tile {}".format(tile_id)
    #     mask_calc = '--calc=A*(B==0)*(C==0)'
    #     mask_outfilename = AGB_natrl_forest_gain_rate_masked
    #     mask_outfilearg = '--outfile={}'.format(mask_outfilename)
    #     cmd = ['gdal_calc.py', '-A', AGB_natrl_forest_gain_rate_unmasked, '-B', mangrove_reclass, '-C', planted_forest_gain,
    #            mask_calc, mask_outfilearg, '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW']
    #     subprocess.check_call(cmd)
    #
    # # Checks if the tile has mangrove pixels and masks them out if there are any
    # elif os.path.exists(mangrove_biomass):
    #
    #     # Mangrove biomass tiles that have the nodata pixels removed
    #     mangrove_reclass = '{0}_reclass_{1}.tif'.format(tile_id, cn.pattern_mangrove_biomass_2000)
    #
    #     # Removes the nodata values in the mangrove biomass rasters because having nodata values in the mangroves didn't work
    #     # in gdal_calc. The gdal_calc expression didn't know how to evaluate nodata values, so I had to remove them.
    #     print "  Removing nodata values in mangrove biomass raster {}".format(tile_id)
    #     cmd = ['gdal_translate', '-a_nodata', 'none', mangrove_biomass, mangrove_reclass]
    #     subprocess.check_call(cmd)
    #
    #     # Masks out the mangrove biomass from the natural forest gain rate
    #     # Ideally this would be part of the rasterio/numpy operation (not its own gdal operation later) but I couldn't
    #     # figure out how to get the mask working in numpy for some reason
    #     print "  Masking mangroves from aboveground gain rate for tile {}".format(tile_id)
    #     mask_calc = '--calc=A*(B==0)'
    #     mask_outfilename = AGB_natrl_forest_gain_rate_masked
    #     mask_outfilearg = '--outfile={}'.format(mask_outfilename)
    #     cmd = ['gdal_calc.py', '-A', AGB_natrl_forest_gain_rate_unmasked, '-B', mangrove_reclass, mask_calc, mask_outfilearg,
    #            '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW']
    #     subprocess.check_call(cmd)
    #
    # # Checks if the tile has planted forest pixels and masks them out if there are any
    # elif os.path.exists(planted_forest_gain):
    #
    #     # Masks out the mangrove biomass from the natural forest gain rate
    #     # Ideally this would be part of the rasterio/numpy operation (not its own gdal operation later) but I couldn't
    #     # figure out how to get the mask working in numpy for some reason
    #     print "  Masking planted forests from aboveground gain rate for tile {}".format(tile_id)
    #     mask_calc = '--calc=A*(B==0)'
    #     mask_outfilename = AGB_natrl_forest_gain_rate_masked
    #     mask_outfilearg = '--outfile={}'.format(mask_outfilename)
    #     cmd = ['gdal_calc.py', '-A', AGB_natrl_forest_gain_rate_unmasked, '-B', planted_forest_gain, mask_calc, mask_outfilearg,
    #            '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW']
    #     subprocess.check_call(cmd)
    #
    # else:
    #
    #     os.rename(AGB_natrl_forest_gain_rate_unmasked, AGB_natrl_forest_gain_rate_masked)

    # # Calculates belowground biomass rate from aboveground biomass rate
    # print "  Creating belowground biomass gain rate for tile {}".format(tile_id)
    # above_to_below_calc = '--calc=(A>0)*A*{}'.format(cn.below_to_above_natrl_forest)
    # below_outfilename = '{0}_{1}.tif'.format(tile_id, cn.pattern_annual_gain_BGB_natrl_forest)
    # below_outfilearg = '--outfile={}'.format(below_outfilename)
    # cmd = ['gdal_calc.py', '-A', AGB_natrl_forest_gain_rate_unmasked, above_to_below_calc, below_outfilearg,
    #        '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW']
    # subprocess.check_call(cmd)

    end = datetime.datetime.now()
    elapsed_time = end-start

    print "  Processing time for tile", tile_id, ":", elapsed_time
<|endoftext|>


<commit_before>class Team:

    def __init__(self, name):
        self.name = name
        self.wins = 0
        self.losses = 0
        self.draws = 0

    def matches_played(self):
        return self.wins + self.losses + self.draws

    def points(self):
        return (self.wins * 3) + self.draws

    def __str__(self):
        return '{:<30} | {:^3}| {:^3}| {:^3}| {:^3}| {:>2}'.format(self.name, self.matches_played(), self.wins, self.draws, self.losses, self.points())

class Tournament:

    WIN = 'win'
    DRAW = 'draw'
    LOSS = 'loss'
    RESULT_SEPERATOR = ';'
    COLUMN_HEADERS = ['Team', 'MP', 'W', 'D', 'L', 'P']

    def __init__(self, results):
        self.teams = {}
        if results:
            self.parse(results)

    def header(self):
        return '{:<30} | {:^3}| {:^3}| {:^3}| {:^3}| {:>2}'.format(*self.COLUMN_HEADERS)

    def results_table(self):
        table = [self.header()]
        for team in self.sorted_teams():
            table.append(str(team))
        return "\n".join(table)

    def sorted_teams(self):
        alphabetic = sorted(self.teams.values(), key=lambda team: team.name)
        alphabetic_descending_points = sorted(alphabetic, key=lambda team: team.points(), reverse=True)
        return alphabetic_descending_points

    def parse(self, results):
        for result in results.split("\n"):
            team_a, team_b, outcome = result.split(self.RESULT_SEPERATOR)
            self.maybe_initialize_teams(team_a, team_b)
            self.tally_outcome(team_a, team_b, outcome)

    def tally_outcome(self, team_a, team_b, outcome):
        if outcome == self.WIN:
            self.teams[team_a].wins += 1
            self.teams[team_b].losses += 1
        if outcome == self.LOSS:
            self.teams[team_a].losses += 1
            self.teams[team_b].wins += 1
        if outcome == self.DRAW:
            self.teams[team_a].draws += 1
            self.teams[team_b].draws += 1

    def maybe_initialize_teams(self, team_a, team_b):
        self.maybe_initialize_team(team_a)
        self.maybe_initialize_team(team_b)

    def maybe_initialize_team(self, name):
        if name not in self.teams:
            self.teams[name] = Team(name)

def tally(tournament_results):
    tournament = Tournament(tournament_results)
    return tournament.results_table()
<commit_msg>Extract class constants for points per win / draw<commit_after>class Team:

    POINTS_PER_WIN = 3
    POINTS_PER_DRAW = 1

    def __init__(self, name):
        self.name = name
        self.wins = 0
        self.draws = 0
        self.losses = 0

    def matches_played(self):
        return self.wins + self.losses + self.draws

    def points(self):
        return (self.wins * self.POINTS_PER_WIN) + (self.draws * self.POINTS_PER_DRAW)

    def __str__(self):
        return '{:<30} | {:^3}| {:^3}| {:^3}| {:^3}| {:>2}'.format(self.name, self.matches_played(), self.wins, self.draws, self.losses, self.points())

class Tournament:

    WIN = 'win'
    DRAW = 'draw'
    LOSS = 'loss'
    RESULT_SEPERATOR = ';'
    COLUMN_HEADERS = ['Team', 'MP', 'W', 'D', 'L', 'P']

    def __init__(self, results):
        self.teams = {}
        if results:
            self.parse(results)

    def results_table(self):
        table = [self.header()]
        for team in self.sorted_teams():
            table.append(str(team))
        return "\n".join(table)

    def sorted_teams(self):
        alphabetic = sorted(self.teams.values(), key=lambda team: team.name)
        alphabetic_descending_points = sorted(alphabetic, key=lambda team: team.points(), reverse=True)
        return alphabetic_descending_points

    def parse(self, results):
        for result in results.split("\n"):
            team_a, team_b, outcome = result.split(self.RESULT_SEPERATOR)
            self.maybe_initialize_teams(team_a, team_b)
            self.tally_outcome(team_a, team_b, outcome)

    def tally_outcome(self, team_a, team_b, outcome):
        if outcome == self.WIN:
            self.teams[team_a].wins += 1
            self.teams[team_b].losses += 1
        if outcome == self.LOSS:
            self.teams[team_a].losses += 1
            self.teams[team_b].wins += 1
        if outcome == self.DRAW:
            self.teams[team_a].draws += 1
            self.teams[team_b].draws += 1

    def maybe_initialize_teams(self, team_a, team_b):
        self.maybe_initialize_team(team_a)
        self.maybe_initialize_team(team_b)

    def maybe_initialize_team(self, name):
        if name not in self.teams:
            self.teams[name] = Team(name)

    def header(self):
        return '{:<30} | {:^3}| {:^3}| {:^3}| {:^3}| {:>2}'.format(*self.COLUMN_HEADERS)

def tally(tournament_results):
    tournament = Tournament(tournament_results)
    return tournament.results_table()
<|endoftext|>