<commit_before>import numpy as np
import math
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

import thermo
import problem
import solvers
import examples

def make_figure2():
    plt.rc('font', size = 15)
    # Values for the pathological problem
    y0 = 0.5
    y1 = 0
    T0 = 335        # temperature at 1 bar
    T1 = 375        # temperature at 1 bar
    s0 = thermo.entropy(y0, thermo.bar, T0)
    s1 = thermo.entropy(y1, thermo.bar, T1)

    print ('value of w for wet parcel', thermo.compute_w(y0))

    n = 10000
    p = np.linspace(4e4, 6.5e4, n)
    dhdp0 = np.zeros((n,))
    dhdp1 = np.zeros((n,))
    Tv0 = np.zeros((n,))
    Tv1 = np.zeros((n,))

    for i in range(n):
        dhdp0[i] = thermo.enthalpy_dp(y0, p[i], s0)
        dhdp1[i] = thermo.enthalpy_dp(y1, p[i], s1)
        Tv0[i] = thermo.virtual_temperature(y0, thermo.bar, s0)
        Tv1[i] = thermo.virtual_temperature(y1, thermo.bar, s1)

    plt.clf()
    plt.plot(dhdp1, p / 100, '-', label = 'dry parcel')
    plt.plot(dhdp0, p / 100, '--', label = 'wet parcel')
    plt.xlabel("$\partial_p h$ (J kg$^{-1}$ Pa$^{-1}$)")
    plt.ylabel("Pressure (hPa)")
    plt.ylim(670, 380)
    plt.legend(loc = 'lower right', frameon = False)

    ax = plt.gca()
    ax.tick_params(direction='out', right=False, top = False)
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)
    plt.subplots_adjust(bottom = 0.15, left = 0.125)
    plt.savefig('../output/Stansifer_Fig2.png')
    plt.close()

    # plt.clf()
    # plt.plot(p / 100, Tv1, '-', label = 'dry air')
    # plt.plot(p / 100, Tv0, '--', label = 'wet air')
    # plt.xlabel("Pressure (hPa)")
    # plt.ylabel("virtual potential temperature (K)")
    # plt.xlim(380, 670)
    # plt.legend()
    # plt.savefig('../output/Stansifer_Fig_Other3.png')
    # plt.close()

def make_figure1():
    plt.rc('font', size = 13)
    p1 = examples.problem1
    # sol = solvers.divide_and_conquer(p1)
    sol = solvers.munkres(p1)

    y = np.array(p1.y)
    s = np.array(p1.s)
    p = np.zeros((len(y),))
    pt = np.zeros((len(y),))

    for i in range(len(y)):
        p[i] = p1.p[sol.j2k[i]]
        pt[i] = thermo.temperature(y[i], thermo.bar, s[i])

    # cmap = plt.get_cmap('hot')
    # cmap = plt.get_cmap('gray')

    # plt.clf()
    # sc = plt.scatter(pt, y, c = p / 100, cmap = cmap)
    # plt.colorbar(sc)
    # plt.xlabel("Potential temperature (K at 1000 hPa)")
    # plt.ylabel("Water content (molar fraction)")
    # plt.title("Pressure (in mbar) distribution at minimum enthalpy")
    # plt.xlim(270, 500)
    # plt.ylim(0, 0.025)
    # plt.savefig('../output/Stansifer_Fig_Other.png')
    # plt.close()

    # plt.clf()
    # sc = plt.scatter(pt, y, c = p / 100, cmap = cmap)
    # plt.colorbar(sc)
    # plt.xlabel("Potential temperature (K at 1000 hPa)")
    # plt.ylabel("Water content (molar fraction)")
    # plt.title("Pressure (in mbar) distribution at minimum enthalpy")
    # plt.xlim(270, 350)
    # plt.ylim(0, 0.025)
    # plt.savefig('../output/Stansifer_Fig_Other2.png')
    # plt.close()

    pt_ = pt.reshape(40, 40)
    y_ = y.reshape(40, 40)
    p_ = p.reshape(40, 40)

    p_ = np.array(p_, copy = True)
    p_[pt_ > 330] = np.nan

    plt.clf()
    fig = plt.figure()
    ax = fig.gca(projection='3d')
    ax.plot_wireframe(pt_, y_ * thermo.epsilon, p_ / 100, rstride = 1, cstride = 1)
    ax.set_xlim(275, 321)
    ax.set_ylim(0.003 * thermo.epsilon, 0.022 * thermo.epsilon)
    ax.set_zlim(1000, 200)
    ax.view_init(elev=30, azim=160)
    ax.set_xlabel("Temperature (K) at 1000 hPa")
    ax.set_ylabel("Water content (kg per kg)")
    ax.set_zlabel("Pressure (hPa)")
    plt.subplots_adjust(bottom = 0.1, right = 1, left = 0)
    plt.savefig('../output/Stansifer_Fig1.png')
    plt.close()

def compute(problems = None):
    if problems is None:
        problems = [examples.problem2.lin_interpolate(1000),
                examples.problem1,
                examples.pathological_problem(100, True)]

    runs = [examples.Results(p) for p in problems]
    for r in runs:
        print ('Running all solvers on a problem...', end = '')
        r.run()
        print (' done.')

    # print ('Initial')
    # print (runs[0].results['initial'].solution)
    # print ('R&W Hard-coded')
    # print (runs[0].results['randallwangsolution'].solution)
    # print ('R&W')
    # print (runs[0].results['randallwang'].solution)
    # print ('Lorenz')
    # print (runs[0].results['lorenz'].solution)
    # print ('Munkres')
    # print (runs[0].results['munkres'].solution)
    # print ('Divide and conquer')
    # print (runs[0].results['divide-and-conquer'].solution)

    return runs

def print_results(runs, prec = 5):
    import sys
    import numpy.version
    import scipy.version

    print ("Python version ", sys.version)
    print ("Numpy version ", numpy.version.full_version)
    print ("Scipy version ", scipy.version.full_version)
    # Thermodynamic data
    print ('R = {:.5g} J / mol / K'.format(thermo.R))
    print ('cd * R = {:.5g} J / mol / K'.format(thermo.cd_))
    print ('cv * R = {:.5g} J / mol / K'.format(thermo.cv_))
    print ('cl * R = {:.5g} J / mol / K'.format(thermo.cl_))
    print ('Md * R = {:.5g} J / mol / K'.format(thermo.Md_))
    print ('Mw * R = {:.5g} J / mol / K'.format(thermo.Mw_))
    print ('pc = {:.5g} Pa'.format(thermo.pc))
    print ('Tc = {:.5g} K'.format(thermo.Tc))
    print ('Lc = {:.5g} J / mol'.format(thermo.Lc * thermo.R))

    algs = ['munkres', 'divide-and-conquer', 'randallwang', 'greedy-from-top', 'lorenz', 'initial', 'munkres-worst']

    # Results
    for name in algs:
        s = '{: <24}'.format(name)

        for r in runs:
            h = r.enthalpy_range
            s += ' & '
            if name in r.results:
                s += '{0:<9.{prec}g}'.format(r.results[name].score * h, prec = prec)
            else:
                s += '{: <9}'.format('-')

        for r in runs:
            s += ' & '
            if name in r.results:
                s += '{:4.2f}'.format(r.results[name].time)
            else:
                s += '{: <5}'.format('-')
        s += ' \\\\'
        print (s)

if __name__ == "__main__":
    make_figure1()
    make_figure2()
    r = compute()
    print_results(r)
<commit_msg>Flushed output and added timing information<commit_after>import numpy as np
import math
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

import thermo
import problem
import solvers
import examples

def make_figure2():
    plt.rc('font', size = 15)
    # Values for the pathological problem
    y0 = 0.5
    y1 = 0
    T0 = 335        # temperature at 1 bar
    T1 = 375        # temperature at 1 bar
    s0 = thermo.entropy(y0, thermo.bar, T0)
    s1 = thermo.entropy(y1, thermo.bar, T1)

    print ('value of w for wet parcel', thermo.compute_w(y0))

    n = 10000
    p = np.linspace(4e4, 6.5e4, n)
    dhdp0 = np.zeros((n,))
    dhdp1 = np.zeros((n,))
    Tv0 = np.zeros((n,))
    Tv1 = np.zeros((n,))

    for i in range(n):
        dhdp0[i] = thermo.enthalpy_dp(y0, p[i], s0)
        dhdp1[i] = thermo.enthalpy_dp(y1, p[i], s1)
        Tv0[i] = thermo.virtual_temperature(y0, thermo.bar, s0)
        Tv1[i] = thermo.virtual_temperature(y1, thermo.bar, s1)

    plt.clf()
    plt.plot(dhdp1, p / 100, '-', label = 'dry parcel')
    plt.plot(dhdp0, p / 100, '--', label = 'wet parcel')
    plt.xlabel("$\partial_p h$ (J kg$^{-1}$ Pa$^{-1}$)")
    plt.ylabel("Pressure (hPa)")
    plt.ylim(670, 380)
    plt.legend(loc = 'lower right', frameon = False)

    ax = plt.gca()
    ax.tick_params(direction='out', right=False, top = False)
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)
    plt.subplots_adjust(bottom = 0.15, left = 0.125)
    plt.savefig('../output/Stansifer_Fig2.png')
    plt.close()

    # plt.clf()
    # plt.plot(p / 100, Tv1, '-', label = 'dry air')
    # plt.plot(p / 100, Tv0, '--', label = 'wet air')
    # plt.xlabel("Pressure (hPa)")
    # plt.ylabel("virtual potential temperature (K)")
    # plt.xlim(380, 670)
    # plt.legend()
    # plt.savefig('../output/Stansifer_Fig_Other3.png')
    # plt.close()

def make_figure1():
    plt.rc('font', size = 13)
    p1 = examples.problem1
    # sol = solvers.divide_and_conquer(p1)
    sol = solvers.munkres(p1)

    y = np.array(p1.y)
    s = np.array(p1.s)
    p = np.zeros((len(y),))
    pt = np.zeros((len(y),))

    for i in range(len(y)):
        p[i] = p1.p[sol.j2k[i]]
        pt[i] = thermo.temperature(y[i], thermo.bar, s[i])

    # cmap = plt.get_cmap('hot')
    # cmap = plt.get_cmap('gray')

    # plt.clf()
    # sc = plt.scatter(pt, y, c = p / 100, cmap = cmap)
    # plt.colorbar(sc)
    # plt.xlabel("Potential temperature (K at 1000 hPa)")
    # plt.ylabel("Water content (molar fraction)")
    # plt.title("Pressure (in mbar) distribution at minimum enthalpy")
    # plt.xlim(270, 500)
    # plt.ylim(0, 0.025)
    # plt.savefig('../output/Stansifer_Fig_Other.png')
    # plt.close()

    # plt.clf()
    # sc = plt.scatter(pt, y, c = p / 100, cmap = cmap)
    # plt.colorbar(sc)
    # plt.xlabel("Potential temperature (K at 1000 hPa)")
    # plt.ylabel("Water content (molar fraction)")
    # plt.title("Pressure (in mbar) distribution at minimum enthalpy")
    # plt.xlim(270, 350)
    # plt.ylim(0, 0.025)
    # plt.savefig('../output/Stansifer_Fig_Other2.png')
    # plt.close()

    pt_ = pt.reshape(40, 40)
    y_ = y.reshape(40, 40)
    p_ = p.reshape(40, 40)

    p_ = np.array(p_, copy = True)
    p_[pt_ > 330] = np.nan

    plt.clf()
    fig = plt.figure()
    ax = fig.gca(projection='3d')
    ax.plot_wireframe(pt_, y_ * thermo.epsilon, p_ / 100, rstride = 1, cstride = 1)
    ax.set_xlim(275, 321)
    ax.set_ylim(0.003 * thermo.epsilon, 0.022 * thermo.epsilon)
    ax.set_zlim(1000, 200)
    ax.view_init(elev=30, azim=160)
    ax.set_xlabel("Temperature (K) at 1000 hPa")
    ax.set_ylabel("Water content (kg per kg)")
    ax.set_zlabel("Pressure (hPa)")
    plt.subplots_adjust(bottom = 0.1, right = 1, left = 0)
    plt.savefig('../output/Stansifer_Fig1.png')
    plt.close()

def compute(problems = None):
    if problems is None:
        problems = [examples.problem2.lin_interpolate(1000),
                examples.problem1,
                examples.pathological_problem(100, True)]

    runs = [examples.Results(p) for p in problems]
    for r in runs:
        print ('Running all solvers on a problem...', end = '', flush = True)
        r.run()
        print (' done.')

    # print ('Initial')
    # print (runs[0].results['initial'].solution)
    # print ('R&W Hard-coded')
    # print (runs[0].results['randallwangsolution'].solution)
    # print ('R&W')
    # print (runs[0].results['randallwang'].solution)
    # print ('Lorenz')
    # print (runs[0].results['lorenz'].solution)
    # print ('Munkres')
    # print (runs[0].results['munkres'].solution)
    # print ('Divide and conquer')
    # print (runs[0].results['divide-and-conquer'].solution)

    return runs

def print_results(runs, prec = 5):
    import sys
    import numpy.version
    import scipy.version

    print ("Python version ", sys.version)
    print ("Numpy version ", numpy.version.full_version)
    print ("Scipy version ", scipy.version.full_version)
    # Thermodynamic data
    print ('R = {:.5g} J / mol / K'.format(thermo.R))
    print ('cd * R = {:.5g} J / mol / K'.format(thermo.cd_))
    print ('cv * R = {:.5g} J / mol / K'.format(thermo.cv_))
    print ('cl * R = {:.5g} J / mol / K'.format(thermo.cl_))
    print ('Md * R = {:.5g} J / mol / K'.format(thermo.Md_))
    print ('Mw * R = {:.5g} J / mol / K'.format(thermo.Mw_))
    print ('pc = {:.5g} Pa'.format(thermo.pc))
    print ('Tc = {:.5g} K'.format(thermo.Tc))
    print ('Lc = {:.5g} J / mol'.format(thermo.Lc * thermo.R))

    algs = ['munkres', 'divide-and-conquer', 'randallwang', 'greedy-from-top', 'lorenz', 'initial', 'munkres-worst']

    # Results
    for name in algs:
        s = '{: <24}'.format(name)

        for r in runs:
            h = r.enthalpy_range
            s += ' & '
            if name in r.results:
                s += '{0:<9.{prec}g}'.format(r.results[name].score * h, prec = prec)
            else:
                s += '{: <9}'.format('-')

        for r in runs:
            s += ' & '
            if name in r.results:
                s += '{:4.2f}'.format(r.results[name].time)
            else:
                s += '{: <5}'.format('-')
        s += ' \\\\'
        print (s)

if __name__ == "__main__":
    # Total execution time about 7 minutes on my machine
    make_figure1()
    make_figure2()
    r = compute()
    print_results(r)
<|endoftext|>


<commit_before>from logConstructs import *

class SAT_solver:

    @staticmethod
    def up(formula):
        #forall Or nodes with size of clause == 1 add variable name and value to dictionary
        #if variable has True and False add special value
        #return dictionary
        pass

    @staticmethod
    def purge(formula):
        pass

    def solve(formula):
        #convert to NNF
        #convert to CNF
        #call solve_cnf
        pass

    def solve_cnf(formula):
        #while call up != []
            #dictinary = up(formula)
            #if dictionary contains special value return False
            #formula = formula.evaluate(dictionary)

        #while call purge != []
            #dictionary = purge(formula)
            #formula = formula.evaluate(dictionary)

        #scan for remaining variables

        #=============================================
        #multithreading - spliting search
        #=============================================
        #heuristic - some heuristic for variable selection
        #=============================================
        #select a variable v
        #set v to false, evaluate and call solve_cnf
        #if return == True return True
        #set v to true, evaluate and call solve_cnf
        #if return == True return True
        #return False

        pass

def evaluateFormula(expr, varlist, assignments={}):
    if len(varlist) == 0:

        value = expr.value(assignments)

        if value:
            return True

        return False

    r_value = False
    for i in [False, True]:
        assignments[varlist[0]] = i
        r_value = r_value or evaluateFormula(expr, varlist[1:], assignments)

    return r_value
<commit_msg>Implement purge and up methods.<commit_after>from logConstructs import *

class SAT_solver:

    @staticmethod
    def up(formula):
        """Return None if there is contradiction and dictionary otherwise."""
        #filter cluses which contains one element
        fformula = filter(lambda x: len(x.clause) == 1, formula)

        result = {}
        #if there are elements in formula then they are Variable or
        #Not class
        for element in fformula:
            if element.clause.__class__.__name__ == "Var":
                name = element.clause.name
                eresult = True
            else:
                name = element.clause.clause.name
                eresult = False

            #check if formula contains contradiction
            if name in result and result[name] != eresult:
                return None
            else:
                result[name] = eresult

        return result

    #return all elements values which can be purged
    def filterElements(element):
        result = {}
        for el in element:
            if el.__class__.__name__ == "Var":
                name = el.name
                eresult = True
            else:
                name = el.clause.name
                eresult = False

            if name in result[name] and result[name] != eresult:
                result[name] = None
            else:
                result[name] = eresult

        return {i: j for i, j in result.items() if j is not None}

    @staticmethod
    def purge(formula):
        """Return None if there is no elements which can be"""
        """purged or dictionary otherwise."""
        #get all variables in clauses
        assignments = filterElements(reduce(lambda x, y: x + y,
                                            map(lambda x: x.clause, formula)))

        #if there is no element for purging return None
        if assignments == {}:
            return None

        #return purged formula
        return formula.evaluate(assignments)

    def solve(formula):
        #convert to NNF
        #convert to CNF
        #call solve_cnf
        pass

    def solve_cnf(formula):
        #while call values = up != {}
            #if values = None return False
            #formula = formula.evaluate(values)

        #while call purge != None

        #=============================================
        #multithreading - spliting search
        #=============================================
        #heuristic - some heuristic for variable selection
        #=============================================
        #select a variable v
        #set v to false, evaluate and call solve_cnf
        #if return == True return True
        #set v to true, evaluate and call solve_cnf
        #if return == True return True
        #return False

        pass

def evaluateFormula(expr, varlist, assignments={}):
    if len(varlist) == 0:

        value = expr.value(assignments)

        if value:
            return True

        return False

    r_value = False
    for i in [False, True]:
        assignments[varlist[0]] = i
        r_value = r_value or evaluateFormula(expr, varlist[1:], assignments)

    return r_value
<|endoftext|>


<commit_before>#!/usr/bin/env python
from optparse import OptionParser
import sys

import numpy as np
import pandas as pd
import pysam

from dna_io import dna_one_hot

################################################################################
# vcf.py
#
# Methods and classes to support .vcf SNP analysis.
################################################################################

def cap_allele(allele, cap=5):
    ''' Cap the length of an allele in the figures '''
    if len(allele) > cap:
        allele = allele[:cap] + '*'
    return allele

def snps_seq1(snps, genome_fasta, seq_len):
    ''' Produce an array of one hot coded sequences for a list of SNPs.

    Attrs:
        snps [SNP] : list of SNPs
        genome_fasta (str) : genome FASTA file
        seq_len (int) : sequence length to code

    Return:
        seq_vecs (array) : one hot coded sequences surrounding the SNPs
        seq_headers [str] : headers for sequences
    '''
    left_len = seq_len/2 - 1
    right_len = seq_len/2

    # open genome FASTA
    genome = pysam.Fastafile(genome_fasta)

    # initialize one hot coded vector list
    seq_vecs_list = []

    # save successful SNPs
    seq_snps = []

    # save sequence strings, too
    seqs = []

    # name sequences
    seq_headers = []

    for snp in snps:
        # specify positions in GFF-style 1-based
        seq_start = snp.pos - left_len
        seq_end = snp.pos + right_len + len(snp.ref_allele) - snp.longest_alt()

        # extract sequence as BED style
        if seq_start < 0:
            seq = 'N'*(-seq_start) + genome.fetch(snp.chrom, 0, seq_end).upper()
        else:
            seq = genome.fetch(snp.chrom, seq_start-1, seq_end).upper()

        if len(seq) < seq_end - seq_start:
            seq += 'N'*(seq_end-seq_start-len(seq))

        # verify that ref allele matches ref sequence
        seq_ref = seq[left_len:left_len+len(snp.ref_allele)]
        if seq_ref != snp.ref_allele:
            print >> sys.stderr, 'WARNING: skipping %s because reference allele does not match reference genome: %s vs %s' % (snp.rsid, snp.ref_allele, seq_ref)
            continue
        else:
            seq_snps.append(snp)

        # one hot code ref allele
        seq_vecs_list.append(dna_one_hot(seq[:seq_len], seq_len))
        seqs.append(seq[:seq_len])

        # name ref allele
        seq_headers.append('%s_%s' % (snp.rsid, cap_allele(snp.ref_allele)))

        for alt_al in snp.alt_alleles:
            # remove ref allele and include alt allele
            seq_alt = seq[:left_len] + alt_al + seq[left_len+len(snp.ref_allele):]

            # one hot code
            seq_vecs_list.append(dna_one_hot(seq_alt, seq_len))
            seqs.append(seq_alt)

            # name
            seq_headers.append('%s_%s' % (snp.rsid, cap_allele(alt_al)))

    # stack
    seq_vecs = np.vstack(seq_vecs_list)

    return seq_vecs, seqs, seq_headers, seq_snps

def vcf_snps(vcf_file, index_snp=False, score=False):
    ''' Load SNPs from a VCF file '''
    vcf_in = open(vcf_file)

    # read through header
    line = vcf_in.readline()
    while line[0] == '#':
        line = vcf_in.readline()

    # read in SNPs
    snps = []
    while line:
        snps.append(SNP(line, index_snp, score))
        line = vcf_in.readline()

    return snps

class SNP:
    ''' SNP

    Represent SNPs read in from a VCF file

    Attributes:
        vcf_line (str)
    '''
    def __init__(self, vcf_line, index_snp=False, score=False):
        a = vcf_line.split()
        if a[0].startswith('chr'):
            self.chrom = a[0]
        else:
            self.chrom = 'chr%s' % a[0]
        self.pos = int(a[1])
        self.rsid = a[2]
        self.ref_allele = a[3]
        self.alt_alleles = a[4].split(',')

        self.index_snp = '.'
        if index_snp:
            self.index_snp = a[5]

        self.score = None
        if score:
            self.score = float(a[6])

    def get_alleles(self):
        ''' Return a list of all alleles '''
        alleles = [self.ref_allele] + self.alt_alleles
        return alleles

    def longest_alt(self):
        ''' Return the longest alt allele. '''
        return max([len(al) for al in self.alt_alleles])

    def __str__(self):
        return 'SNP(%s, %s:%d, %s/%s)' % (self.rsid, self.chrom, self.pos, self.ref_allele, ','.join(self.alt_alleles))
<commit_msg>exiting when ref allele doesnt match<commit_after>#!/usr/bin/env python
from optparse import OptionParser
import sys

import numpy as np
import pandas as pd
import pysam

from dna_io import dna_one_hot

################################################################################
# vcf.py
#
# Methods and classes to support .vcf SNP analysis.
################################################################################

def cap_allele(allele, cap=5):
    ''' Cap the length of an allele in the figures '''
    if len(allele) > cap:
        allele = allele[:cap] + '*'
    return allele

def snps_seq1(snps, genome_fasta, seq_len):
    ''' Produce an array of one hot coded sequences for a list of SNPs.

    Attrs:
        snps [SNP] : list of SNPs
        genome_fasta (str) : genome FASTA file
        seq_len (int) : sequence length to code

    Return:
        seq_vecs (array) : one hot coded sequences surrounding the SNPs
        seq_headers [str] : headers for sequences
    '''
    left_len = seq_len/2 - 1
    right_len = seq_len/2

    # open genome FASTA
    genome = pysam.Fastafile(genome_fasta)

    # initialize one hot coded vector list
    seq_vecs_list = []

    # save successful SNPs
    seq_snps = []

    # save sequence strings, too
    seqs = []

    # name sequences
    seq_headers = []

    for snp in snps:
        # specify positions in GFF-style 1-based
        seq_start = snp.pos - left_len
        seq_end = snp.pos + right_len + len(snp.ref_allele) - snp.longest_alt()

        # extract sequence as BED style
        if seq_start < 0:
            seq = 'N'*(-seq_start) + genome.fetch(snp.chrom, 0, seq_end).upper()
        else:
            seq = genome.fetch(snp.chrom, seq_start-1, seq_end).upper()

        if len(seq) < seq_end - seq_start:
            seq += 'N'*(seq_end-seq_start-len(seq))

        # verify that ref allele matches ref sequence
        seq_ref = seq[left_len:left_len+len(snp.ref_allele)]
        if seq_ref != snp.ref_allele:
            print >> sys.stderr, 'ERROR: %s - reference allele does not match reference genome: %s vs %s' % (snp.rsid, snp.ref_allele, seq_ref)
            continue
        else:
            seq_snps.append(snp)

        # one hot code ref allele
        seq_vecs_list.append(dna_one_hot(seq[:seq_len], seq_len))
        seqs.append(seq[:seq_len])

        # name ref allele
        seq_headers.append('%s_%s' % (snp.rsid, cap_allele(snp.ref_allele)))

        for alt_al in snp.alt_alleles:
            # remove ref allele and include alt allele
            seq_alt = seq[:left_len] + alt_al + seq[left_len+len(snp.ref_allele):]

            # one hot code
            seq_vecs_list.append(dna_one_hot(seq_alt, seq_len))
            seqs.append(seq_alt)

            # name
            seq_headers.append('%s_%s' % (snp.rsid, cap_allele(alt_al)))

    # stack
    seq_vecs = np.vstack(seq_vecs_list)

    return seq_vecs, seqs, seq_headers, seq_snps

def vcf_snps(vcf_file, index_snp=False, score=False):
    ''' Load SNPs from a VCF file '''
    vcf_in = open(vcf_file)

    # read through header
    line = vcf_in.readline()
    while line[0] == '#':
        line = vcf_in.readline()

    # read in SNPs
    snps = []
    while line:
        snps.append(SNP(line, index_snp, score))
        line = vcf_in.readline()

    return snps

class SNP:
    ''' SNP

    Represent SNPs read in from a VCF file

    Attributes:
        vcf_line (str)
    '''
    def __init__(self, vcf_line, index_snp=False, score=False):
        a = vcf_line.split()
        if a[0].startswith('chr'):
            self.chrom = a[0]
        else:
            self.chrom = 'chr%s' % a[0]
        self.pos = int(a[1])
        self.rsid = a[2]
        self.ref_allele = a[3]
        self.alt_alleles = a[4].split(',')

        self.index_snp = '.'
        if index_snp:
            self.index_snp = a[5]

        self.score = None
        if score:
            self.score = float(a[6])

    def get_alleles(self):
        ''' Return a list of all alleles '''
        alleles = [self.ref_allele] + self.alt_alleles
        return alleles

    def longest_alt(self):
        ''' Return the longest alt allele. '''
        return max([len(al) for al in self.alt_alleles])

    def __str__(self):
        return 'SNP(%s, %s:%d, %s/%s)' % (self.rsid, self.chrom, self.pos, self.ref_allele, ','.join(self.alt_alleles))
<|endoftext|>


<commit_before>import collections
import datetime
import re

class Burndown():

    def __init__(self, sprint, commitment, report, issues):

        self.sprint = sprint
        self.commitment = commitment
        self.report = report
        self.issues = issues
        self.timeline = collections.OrderedDict()

        self.start = datetime.datetime.strptime(sprint['startDate'], '%d/%b/%y %I:%M %p')
        self.end = datetime.datetime.strptime(sprint['endDate'], '%d/%b/%y %I:%M %p')

        start = self.start.replace(hour=0, minute=0, second=0, microsecond=0)
        day = datetime.timedelta(days=1)

        while start <= self.end:
            self.timeline[start.strftime('%Y-%m-%d')] = {
                'date': start,
                'ideal': commitment,
                'completed': 0,
                'unplanned': 0
            }
            start += day

        self.calculate()

    def _calculate_completed(self):

        changes = {}
        completed = self.commitment

        # closed issues
        for key in self.report.completed:

            if 'Story' != self.issues[key].fields.issuetype.name:
                continue

            date = self._get_resolution_date(self.issues[key].changelog.histories)
            if not date:
                continue

            if date not in changes:
                changes[date] = 0
            changes[date] += int(self.issues[key].fields.customfield_10002)

        # decreased story points
        for key in self.report.all:

            if 'Story' != self.issues[key].fields.issuetype.name:
                continue

            status = 'Open'

            for history in self.issues[key].changelog.histories:
                for item in history.items:

                    created = datetime.datetime.strptime(re.sub(r'\..*$', '', history.created), '%Y-%m-%dT%H:%M:%S')
                    if not (self.start <= created <= self.end):
                        continue

                    if 'status' == item.field:
                        status = item.toString

                    if 'Open' == status:
                        continue

                    if 'Story Points' == item.field and item.fromString is not None:

                        diff = int(item.toString) - int(item.fromString)
                        if diff > 0:
                            continue
                        print('%s - %s' % (key, diff))
                        change = created.strftime('%Y-%m-%d')
                        if change not in changes:
                            changes[change] = 0
                        changes[change] += abs(diff)

        for date, entry in self.timeline.items():
            if date in changes:
                completed -= changes[date]
            entry['completed'] = completed

    def _calculate_ideal(self):

        items = self.timeline.items()
        ideal = self.commitment
        step = ideal / (len(items) - 1)

        for date, entry in self.timeline.items():
            entry['ideal'] = ideal
            ideal -= step
        entry['ideal'] = 0

    def _calculate_unplanned(self):

        changes = {}
        unplanned = self.commitment

        # added
        for key in self.report.added:

            if 'Story' != self.issues[key].fields.issuetype.name:
                continue

            for date in self._get_added_dates(self.issues[key].changelog.histories):

                estimation = self._get_estimation_from_date(date, self.issues[key].changelog.histories)
                if estimation is None:
                    estimation = int(self.issues[key].fields.customfield_10002)

                change = date.strftime('%Y-%m-%d')
                if change not in changes:
                    changes[change] = 0
                changes[change] += estimation

        # punted
        for key in self.report.punted:

            if 'Story' != self.issues[key].fields.issuetype.name:
                continue

            for date in self._get_punted_dates(self.issues[key].changelog.histories):

                estimation = self._get_estimation_from_date(date, self.issues[key].changelog.histories)
                if estimation is None:
                    estimation = int(self.issues[key].fields.customfield_10002)

                change = date.strftime('%Y-%m-%d')
                if change not in changes:
                    changes[change] = 0
                changes[change] -= estimation

        # increased story points
        for key in self.report.all:

            if 'Story' != self.issues[key].fields.issuetype.name:
                continue

            for history in self.issues[key].changelog.histories:
                for item in history.items:

                    created = datetime.datetime.strptime(re.sub(r'\..*$', '', history.created), '%Y-%m-%dT%H:%M:%S')
                    if not (self.start <= created <= self.end):
                        continue

                    if 'Story Points' == item.field and item.fromString is not None:

                        diff = int(item.toString) - int(item.fromString)
                        if diff < 0:
                            continue

                        change = created.strftime('%Y-%m-%d')
                        if change not in changes:
                            changes[change] = 0
                        changes[change] += diff

        for date, entry in self.timeline.items():
            if date in changes:
                unplanned += changes[date]
            entry['unplanned'] = unplanned

    def _get_added_dates(self, histories):

        dates = []

        for history in histories:
            for item in history.items:

                created = datetime.datetime.strptime(re.sub(r'\..*$', '', history.created), '%Y-%m-%dT%H:%M:%S')
                if not (self.start <= created <= self.end):
                    continue

                if 'Sprint' == item.field and str(self.sprint['id']) == item.to:
                    dates.append(created)

        return dates

    def _get_estimation_from_date(self, date, histories):

        before = None
        after = None

        for history in histories:
            for item in history.items:

                created = datetime.datetime.strptime(re.sub(r'\..*$', '', history.created), '%Y-%m-%dT%H:%M:%S')

                if 'Story Points' == item.field:

                    if created <= date:
                        before = int(item.toString)
                    if created > date:
                        after = int(item.toString)

                if None != after:
                    return after

        return before

    def _get_punted_dates(self, histories):

        dates = []

        for history in histories:
            for item in history.items:

                created = datetime.datetime.strptime(re.sub(r'\..*$', '', history.created), '%Y-%m-%dT%H:%M:%S')
                if not (self.start <= created <= self.end):
                    continue

                if 'Sprint' == item.field and str(self.sprint['id']) not in item.to:
                    dates.append(created)

        return dates

    def _get_resolution_date(self, histories):

        for history in reversed(histories):
            for item in history.items:

                if 'resolution' == item.field and 'Done' == item.toString:
                    created = datetime.datetime.strptime(re.sub(r'\..*$', '', history.created), '%Y-%m-%dT%H:%M:%S')
                    if self.start <= created <= self.end:
                        return created.strftime('%Y-%m-%d')

    def calculate(self):

        self._calculate_ideal()
        self._calculate_completed()
        self._calculate_unplanned()

    def get_timeline(self):

        dates = []
        ideal = []
        completed = []
        unplanned = []

        now = datetime.datetime.now()

        for date, entry in self.timeline.items():
            dates.append(entry['date'].strftime('%Y-%m-%d'))
            ideal.append(entry['ideal'])
            if entry['date'] <= now:
                completed.append(entry['completed'])
                unplanned.append(entry['unplanned'])

        return {
            'dates': dates,
            'ideal': ideal,
            'completed': completed,
            'unplanned': unplanned
        }<commit_msg>use only opened issues which are decreased for 'unplanned'<commit_after>import collections
import datetime
import re

class Burndown():

    def __init__(self, sprint, commitment, report, issues):

        self.sprint = sprint
        self.commitment = commitment
        self.report = report
        self.issues = issues
        self.timeline = collections.OrderedDict()

        self.start = datetime.datetime.strptime(sprint['startDate'], '%d/%b/%y %I:%M %p')
        self.end = datetime.datetime.strptime(sprint['endDate'], '%d/%b/%y %I:%M %p')

        start = self.start.replace(hour=0, minute=0, second=0, microsecond=0)
        day = datetime.timedelta(days=1)

        while start <= self.end:
            self.timeline[start.strftime('%Y-%m-%d')] = {
                'date': start,
                'ideal': commitment,
                'completed': 0,
                'unplanned': 0
            }
            start += day

        self.calculate()

    def _calculate_completed(self):

        changes = {}
        completed = self.commitment

        # closed issues
        for key in self.report.completed:

            if 'Story' != self.issues[key].fields.issuetype.name:
                continue

            date = self._get_resolution_date(self.issues[key].changelog.histories)
            if not date:
                continue

            if date not in changes:
                changes[date] = 0
            changes[date] += int(self.issues[key].fields.customfield_10002)

        # decreased story points
        for key in self.report.all:

            if 'Story' != self.issues[key].fields.issuetype.name:
                continue

            status = 'Open'

            for history in self.issues[key].changelog.histories:
                for item in history.items:

                    created = datetime.datetime.strptime(re.sub(r'\..*$', '', history.created), '%Y-%m-%dT%H:%M:%S')
                    if not (self.start <= created <= self.end):
                        continue

                    if 'status' == item.field:
                        status = item.toString

                    if 'Open' == status:
                        continue

                    if 'Story Points' == item.field and item.fromString is not None:

                        diff = int(item.toString) - int(item.fromString)
                        if diff > 0:
                            continue
                        print('completed: %s - %s' % (key, diff))
                        change = created.strftime('%Y-%m-%d')
                        if change not in changes:
                            changes[change] = 0
                        changes[change] += abs(diff)

        for date, entry in self.timeline.items():
            if date in changes:
                completed -= changes[date]
            entry['completed'] = completed

    def _calculate_ideal(self):

        items = self.timeline.items()
        ideal = self.commitment
        step = ideal / (len(items) - 1)

        for date, entry in self.timeline.items():
            entry['ideal'] = ideal
            ideal -= step
        entry['ideal'] = 0

    def _calculate_unplanned(self):

        changes = {}
        unplanned = self.commitment

        # added
        for key in self.report.added:

            if 'Story' != self.issues[key].fields.issuetype.name:
                continue

            for date in self._get_added_dates(self.issues[key].changelog.histories):

                estimation = self._get_estimation_from_date(date, self.issues[key].changelog.histories)
                if estimation is None:
                    estimation = int(self.issues[key].fields.customfield_10002)

                change = date.strftime('%Y-%m-%d')
                if change not in changes:
                    changes[change] = 0
                changes[change] += estimation

        # punted
        for key in self.report.punted:

            if 'Story' != self.issues[key].fields.issuetype.name:
                continue

            for date in self._get_punted_dates(self.issues[key].changelog.histories):

                estimation = self._get_estimation_from_date(date, self.issues[key].changelog.histories)
                if estimation is None:
                    estimation = int(self.issues[key].fields.customfield_10002)

                change = date.strftime('%Y-%m-%d')
                if change not in changes:
                    changes[change] = 0
                changes[change] -= estimation

        # decreased/increased story points
        for key in self.report.all:

            if 'Story' != self.issues[key].fields.issuetype.name:
                continue

            status = 'Open'

            for history in self.issues[key].changelog.histories:
                for item in history.items:

                    created = datetime.datetime.strptime(re.sub(r'\..*$', '', history.created), '%Y-%m-%dT%H:%M:%S')
                    if not (self.start <= created <= self.end):
                        continue

                    if 'status' == item.field:
                        status = item.toString

                    if 'Story Points' == item.field and item.fromString is not None:

                        diff = int(item.toString) - int(item.fromString)
                        if 'Open' != status and diff < 0:
                            continue
                        print('unplanned: %s - %s' % (key, diff))
                        change = created.strftime('%Y-%m-%d')
                        if change not in changes:
                            changes[change] = 0
                        changes[change] += diff

        for date, entry in self.timeline.items():
            if date in changes:
                unplanned += changes[date]
            entry['unplanned'] = unplanned

    def _get_added_dates(self, histories):

        dates = []

        for history in histories:
            for item in history.items:

                created = datetime.datetime.strptime(re.sub(r'\..*$', '', history.created), '%Y-%m-%dT%H:%M:%S')
                if not (self.start <= created <= self.end):
                    continue

                if 'Sprint' == item.field and str(self.sprint['id']) == item.to:
                    dates.append(created)

        return dates

    def _get_estimation_from_date(self, date, histories):

        before = None
        after = None

        for history in histories:
            for item in history.items:

                created = datetime.datetime.strptime(re.sub(r'\..*$', '', history.created), '%Y-%m-%dT%H:%M:%S')

                if 'Story Points' == item.field:

                    if created <= date:
                        before = int(item.toString)
                    if created > date:
                        after = int(item.toString)

                if None != after:
                    return after

        return before

    def _get_punted_dates(self, histories):

        dates = []

        for history in histories:
            for item in history.items:

                created = datetime.datetime.strptime(re.sub(r'\..*$', '', history.created), '%Y-%m-%dT%H:%M:%S')
                if not (self.start <= created <= self.end):
                    continue

                if 'Sprint' == item.field and str(self.sprint['id']) not in item.to:
                    dates.append(created)

        return dates

    def _get_resolution_date(self, histories):

        for history in reversed(histories):
            for item in history.items:

                if 'resolution' == item.field and 'Done' == item.toString:
                    created = datetime.datetime.strptime(re.sub(r'\..*$', '', history.created), '%Y-%m-%dT%H:%M:%S')
                    if self.start <= created <= self.end:
                        return created.strftime('%Y-%m-%d')

    def calculate(self):

        self._calculate_ideal()
        self._calculate_completed()
        self._calculate_unplanned()

    def get_timeline(self):

        dates = []
        ideal = []
        completed = []
        unplanned = []

        now = datetime.datetime.now()

        for date, entry in self.timeline.items():
            dates.append(entry['date'].strftime('%Y-%m-%d'))
            ideal.append(entry['ideal'])
            if entry['date'] <= now:
                completed.append(entry['completed'])
                unplanned.append(entry['unplanned'])

        return {
            'dates': dates,
            'ideal': ideal,
            'completed': completed,
            'unplanned': unplanned
        }<|endoftext|>


<commit_before>"""A script to generate a list of products we are interested in found in job descriptions.

The script takes a list of urls in stdin and returns a |-separated output for each url.
The urls that led to exceptions are writte to a separate errors.txt file
"""

import time
import threading
import sys
import random
import requests
import multiprocessing as mp
from urllib.parse import urlparse
import logging

from jobs.common import DEFAULT_HEADERS, iter_good_lines
from jobs.fetcher import ThrottledFetcher
from jobs.parser import PageParser, IndeedParser, TermsExtractor, NewtonSoftwareParser, GreenHouseParser

G_LOG = logging.getLogger(__name__)

URLS = [
'https://newton.newtonsoftware.com/career/JobIntroduction.action?clientId=8aa0050632afa2010132b69b35493eab&id=8a7880665cb19ccc015cc1f77b0a6410&source=Indeed',
'https://boards.greenhouse.io/embed/job_app?for=pantheon&token=619056&b=https://pantheon.io/careers/apply',
'https://www.indeed.com/viewjob?jk=8539193e3a45a062&q=sales+operations&l=San+Francisco%2C+CA&tk=1aevt39cab9rbe1v&from=web',
'http://www.indeed.com/cmp/Exablox/jobs/Devop-Cloud-Management-System-053c56b763ebd5c5?sjdu=QwrRXKrqZ3CNX5W-O9jEvVKh33UeeQGaPWHksPPR7jOVeKpHgZBD8uj-JYSLVQQdOzuvtcRZqR1VUgvQgUCLrlSAZIc8VspZ09RpfJLmZ1g',
'http://www.indeed.com/cmp/The-Resource-Corner,-LLC/jobs/Bookkeeper-ce09ccbdef05dafc?sjdu=QwrRXKrqZ3CNX5W-O9jEvZYUjcFz8G6VtThA0LDUaBBKkXOI7HyNUFAgnmvj10geaet8H1fzoalk9SEj0AgHMA',
]

def main1():
    start = time.time()
    for url in URLS:
       res = requests.get(url, headers=DEFAULT_HEADERS)
       print(url, 'length', len(res.text))
    end = time.time()
    print('took {:0.3f}'.format(end-start))

def init_fetchers(q_out, q_err, save_page=False, terms_path='techs.txt'):
    terms_extractor = TermsExtractor(terms_path)
    fetchers = {
        'www.indeed.com':
            ThrottledFetcher(
                parser=IndeedParser(save_page=save_page),
                terms_extractor=terms_extractor,
                q_out=q_out, q_err=q_err,
                max_workers=5),
        'newton.newtonsoftware.com':
            ThrottledFetcher(
                parser=NewtonSoftwareParser(save_page=save_page),
                terms_extractor=terms_extractor,
                q_out=q_out, q_err=q_err,
                max_workers=5),
        'boards.greenhouse.io':
            ThrottledFetcher(
                parser=GreenHouseParser(save_page=save_page),
                terms_extractor=terms_extractor,
                q_out=q_out, q_err=q_err,
                max_workers=5),
        'default':
            ThrottledFetcher(
                parser=PageParser(save_page=save_page),
                terms_extractor=terms_extractor,
                q_out=q_out, q_err=q_err,
                max_workers=5, max_rps=0),

    }
    return fetchers

def main2():
    import argparse
    parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__)
    args = parser.parse_args()

    start = time.time()
    q_out = mp.Queue()
    q_err = mp.Queue()
    fetchers = init_fetchers(q_out, q_err, save_page=False)
    default_fetcher = fetchers['default']
    for fetcher in fetchers.values():
        fetcher.start()

    def write_results(q_out):
        while True:
            result = q_out.get()
            if not result:
                break
            print(result)

    def write_errors(q_err):
        with open('errors.txt', 'w') as f1:
            while True:
                result = q_err.get()
                if not result:
                    break
                print(*result, sep='\t', file=f1)

    writer1 = threading.Thread(target=write_results, args=(q_out,))
    writer2 = threading.Thread(target=write_errors, args=(q_err,))
    writer1.start()
    writer2.start()

    #for url in URLS:
    for url in iter_good_lines(sys.stdin):
       urlp = urlparse(url)
       fetcher = fetchers.get(urlp.netloc, default_fetcher)
       fetcher.q_in.put(url)

    for fetcher in fetchers.values():
        fetcher.q_in.put(None)
        fetcher.q_in.join()
    end = time.time()
    q_out.put(None)
    q_err.put(None)
    G_LOG.info('took {:0.3f}'.format(end-start))

if __name__ == '__main__':
    import sys
    #logging.basicConfig(level=logging.ERROR, stream=sys.stderr)
    logging.basicConfig(filename='log.txt')
    #main1()
    main2()
<commit_msg>Enhanced configuration using argparse<commit_after>"""A script to generate a list of products we are interested in found in job descriptions.

It takes a list of urls from infile (stdin by default) and returns a |-separated output for each url.
The script requires a file with techs we are searching for, the file is specified by --techs-file option.
The urls that led to exceptions are written to a separate file, configured by --errors-file.
"""

import time
import threading
import sys
import random
import requests
import multiprocessing as mp
from urllib.parse import urlparse
import logging

from jobs.common import DEFAULT_HEADERS, iter_good_lines
from jobs.fetcher import ThrottledFetcher
from jobs.parser import PageParser, IndeedParser, TermsExtractor, NewtonSoftwareParser, GreenHouseParser

G_LOG = logging.getLogger(__name__)

URLS = [
'https://newton.newtonsoftware.com/career/JobIntroduction.action?clientId=8aa0050632afa2010132b69b35493eab&id=8a7880665cb19ccc015cc1f77b0a6410&source=Indeed',
'https://boards.greenhouse.io/embed/job_app?for=pantheon&token=619056&b=https://pantheon.io/careers/apply',
'https://www.indeed.com/viewjob?jk=8539193e3a45a062&q=sales+operations&l=San+Francisco%2C+CA&tk=1aevt39cab9rbe1v&from=web',
'http://www.indeed.com/cmp/Exablox/jobs/Devop-Cloud-Management-System-053c56b763ebd5c5?sjdu=QwrRXKrqZ3CNX5W-O9jEvVKh33UeeQGaPWHksPPR7jOVeKpHgZBD8uj-JYSLVQQdOzuvtcRZqR1VUgvQgUCLrlSAZIc8VspZ09RpfJLmZ1g',
'http://www.indeed.com/cmp/The-Resource-Corner,-LLC/jobs/Bookkeeper-ce09ccbdef05dafc?sjdu=QwrRXKrqZ3CNX5W-O9jEvZYUjcFz8G6VtThA0LDUaBBKkXOI7HyNUFAgnmvj10geaet8H1fzoalk9SEj0AgHMA',
]

def main1():
    start = time.time()
    for url in URLS:
       res = requests.get(url, headers=DEFAULT_HEADERS)
       print(url, 'length', len(res.text))
    end = time.time()
    print('took {:0.3f}'.format(end-start))

def init_fetchers(q_out, q_err, save_page=False, terms_path='techs.txt'):
    terms_extractor = TermsExtractor(terms_path)
    fetchers = {
        'www.indeed.com':
            ThrottledFetcher(
                parser=IndeedParser(save_page=save_page),
                terms_extractor=terms_extractor,
                q_out=q_out, q_err=q_err,
                max_workers=5),
        'newton.newtonsoftware.com':
            ThrottledFetcher(
                parser=NewtonSoftwareParser(save_page=save_page),
                terms_extractor=terms_extractor,
                q_out=q_out, q_err=q_err,
                max_workers=5),
        'boards.greenhouse.io':
            ThrottledFetcher(
                parser=GreenHouseParser(save_page=save_page),
                terms_extractor=terms_extractor,
                q_out=q_out, q_err=q_err,
                max_workers=5),
        'default':
            ThrottledFetcher(
                parser=PageParser(save_page=save_page),
                terms_extractor=terms_extractor,
                q_out=q_out, q_err=q_err,
                max_workers=5, max_rps=0),

    }
    return fetchers

def main2():
    import argparse
    parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__)
    parser.add_argument('--techs-file', default='techs.txt',
                        help='A file where the searched techs are listed: each tech on a separate line.')
    parser.add_argument('infile', nargs='?', type=argparse.FileType('r'), default=sys.stdin,
                        help='A file with a list of urls. Each url is supposed to contain a job description.')
    parser.add_argument('--errors-file', type=argparse.FileType('w'), default='failed_urls.txt',
                        help=('A tab-separated file to which we are going to dump urls requesting or parsing '
                              'of which resulted in an error. The error message is dumped after the url.'))
    parser.add_argument('--log-file', type=argparse.FileType('a'), default='extract_techs.log',
                        help='A file where we write logs to')
    args = parser.parse_args()

    logging.basicConfig(level=logging.INFO, stream=args.log_file)

    start = time.time()
    q_out = mp.Queue()
    q_err = mp.Queue()
    fetchers = init_fetchers(q_out, q_err, save_page=False, terms_path=args.techs_file)
    default_fetcher = fetchers['default']
    for fetcher in fetchers.values():
        fetcher.start()

    def write_results(q_out):
        while True:
            result = q_out.get()
            if not result:
                break
            print(result)

    def write_errors(q_err, errors_file):
        while True:
            result = q_err.get()
            if not result:
                break
            print(*result, sep='\t', file=errors_file)

    writer1 = threading.Thread(target=write_results, args=(q_out,))
    writer2 = threading.Thread(target=write_errors, args=(q_err, args.errors_file))
    writer1.start()
    writer2.start()

    #for url in URLS:
    for url in iter_good_lines(args.infile):
       urlp = urlparse(url)
       fetcher = fetchers.get(urlp.netloc, default_fetcher)
       fetcher.q_in.put(url)

    for fetcher in fetchers.values():
        fetcher.q_in.put(None)
        fetcher.q_in.join()
    end = time.time()
    q_out.put(None)
    q_err.put(None)
    G_LOG.info('took {:0.3f}'.format(end-start))

if __name__ == '__main__':
    main2()
<|endoftext|>


<commit_before>import sys
from collections import defaultdict

from django.core.management.base import BaseCommand

from wazimap.data.utils import get_session
from wazimap.data.tables import get_datatable, get_table_id, DATA_TABLES, FIELD_TABLES, FieldTable, SimpleTable
from wazimap.geo import geo_data

import logging

logging.basicConfig()
logging.getLogger('sqlalchemy.engine').setLevel(logging.WARN)

"""
This is a helper script that checks the tables in the DB for missing geo entries,
and missing keys for fields. The tables with missing values will be output after completion.

If the store_missing_entries flag is passed, the missing items will be populated in the DB.
Missing geos are populated with null values for all keys,
and missing key values are populated with 0.

"""

class Command(BaseCommand):
    help = ("Checks the database for completeness (or a single table if passed)." +
        "Populates the missing entries if populate_missing_values=True")

    def add_arguments(self, parser):
        parser.add_argument(
            '--table',
            action='store',
            dest='table',
            default=None,
            help='The name of the database table to check'
        )
        parser.add_argument(
            '--geo_code',
            action='store',
            dest='geo_code',
            default='ZA',
            help='The geo_code for the base geo that contains entries for all keys, for all fields e.g. ZA'
        )
        parser.add_argument(
            '--geo_level',
            action='store',
            dest='geo_level',
            default='country',
            help='The geo_level for the base geo that contains entries for all keys, for all fields e.g. country'
        )
        parser.add_argument(
            '--geo_version',
            action='store',
            dest='geo_version',
            default='2011',
            help='The geo_version of the data which should be checked.'
        )
        parser.add_argument(
            '--store_missing_entries',
            action='store_true',
            dest='store_missing_entries',
            default=False,
            help="Write data for missing entries to DB",
        )
        parser.add_argument(
            '--dry-run',
            action='store_true',
            dest='dryrun',
            default=False,
            help="Dry-run, don't actuall write any data.",
        )

    def debug(self, msg):
        if self.verbosity >= 2:
            self.stdout.write(str(msg))

    def handle(self, *args, **options):
        self.session = get_session()
        self.verbosity = options.get('verbosity', 0)
        self.table_id = options.get('table')
        self.geo_version = options.get('geo_version')
        self.store_missing_entries = options.get('store_missing_entries', False)
        self.dryrun = options.get('dryrun')

        self.geos = self.get_geos(self.geo_version)

        self.db_tables = []
        self.fields_by_table = {}
        self.keys_by_table = {}
        self.missing_keys_by_table = {}
        self.missing_geos_by_table = {}
        self.missing_geos_by_simple_table = {}

        if self.table_id:
            table = get_datatable(self.table_id)
            if type(table) == FieldTable:
                self.field_tables = {table.id:table}
            else:
                self.simple_tables = {table.id:table}
        else:
            self.tables = DATA_TABLES
            self.field_tables = FIELD_TABLES
            self.simple_tables = {k: v for k, v in DATA_TABLES.iteritems() if k not in FIELD_TABLES.keys()}

        for table_id, table in self.field_tables.iteritems():
            if table.db_table in self.db_tables:
                # Multiple field tables can refer to the same underlying db table
                continue
            self.db_tables.append(table.db_table)
            self.stdout.write("Checking table: %s" % (table.id))

            self.fields_by_table[table.id] = table.fields
            rows = self.session.query(table.model).filter(table.model.geo_version == self.geo_version).all()

            self.keys_by_table[table_id] = self.get_table_keys(table, table.fields)

            missing_keys = self.get_missing_keys(table, rows, table.fields)
            if missing_keys:
                self.missing_keys_by_table[table.id] = missing_keys

            missing_geos = self.get_missing_geos(table, rows)
            if missing_geos:
                self.missing_geos_by_table[table.id] = missing_geos

        if self.missing_keys_by_table:
            self.stdout.write("Missing keys for tables:")
            for table in self.missing_keys_by_table.iterkeys():
                self.stdout.write("%s" % (table))

        if self.missing_geos_by_table:
            self.stdout.write("Missing geos for tables:")
            for table in self.missing_geos_by_table.iterkeys():
                self.stdout.write("%s" % table)

        if self.store_missing_entries:
            if self.missing_keys_by_table:
                self.store_missing_keys()
                self.stdout.write("Missing keys stored in database.")
            if self.missing_geos_by_table:
                self.store_missing_geos()
                self.stdout.write("Missing geos stored in database.")
        else:
            self.stdout.write("Run command with --store_missing_entries=True to populate missing keys with 0 and missing geos with null")

        for table_id, table in self.simple_tables.iteritems():
            self.stdout.write("Checking table: %s" % (table.id))
            rows = self.session.query(table.model).filter(table.model.geo_version == self.geo_version).all()
            missing_geos = self.get_missing_geos(table, rows)
            if missing_geos:
                self.missing_geos_by_simple_table[table.id] = missing_geos

        if self.missing_geos_by_simple_table:
            self.stdout.write("Missing geos for Simple tables:")
            for table in self.missing_geos_by_simple_table.iterkeys():
                self.stdout.write("%s" % table)

            if self.store_missing_entries:
                self.store_simple_table_missing_geos()
                self.stdout.write("Missing geos for Simple tables stored in database.")

        self.session.close()

    def get_geos(self, geo_version):
        return geo_data.geo_model.objects.filter(version=geo_version)

    def get_table_keys(self, table, fields):
        # Return a list with all permuations of the keys for all fields
        keys = []
        for obj in self.session.query(table.model).distinct(*fields):
            keys.append([obj.__getattribute__(field) for field in fields])

        return keys

    def get_missing_keys(self, table, rows, fields):
        keys_by_geo = defaultdict(list)
        for row in rows:
            # Get existing key combinations by geo_id
            keys_by_geo[row.geo_level + '-' + row.geo_code].append(
                [row.__getattribute__(field) for field in fields])

        missing_keys_by_geo = {}
        for geo_id, geo_keys in keys_by_geo.iteritems():
            if len(geo_keys) != len(self.keys_by_table[table.id]):
                missing_keys_by_geo[geo_id] = [
                    x for x in self.keys_by_table[table.id] if x not in geo_keys]

        return missing_keys_by_geo

    def get_missing_geos(self, table, rows):
        table_geos = set((row.geo_level, row.geo_code) for row in rows)
        if not table_geos:
            sys.exit("Empty table: %s" % (table.id))

        all_geos = set((g.geo_level, g.geo_code) for g in self.geos)
        missing_geos = [g for g in all_geos if g not in table_geos]

        if missing_geos:
            self.missing_geos_by_table[table.id] = missing_geos

        return missing_geos

    def store_missing_keys(self):
        count = 0
        for table_id, missing_keys_by_geo in self.missing_keys_by_table.iteritems():
            table = self.field_tables[table_id]
            self.stdout.write("Storing missing keys for : %s" % (table_id))
            for geo, missing_keys in missing_keys_by_geo.iteritems():
                for key_combo in missing_keys:
                    count += 1
                    geo_level, geo_code = geo.split('-')
                    entry = {
                        'geo_level': geo_level,
                        'geo_code': geo_code,
                        'total': 0,
                        'geo_version': self.geo_version}
                    for i, field in enumerate(self.fields_by_table[table_id]):
                        entry[field] = key_combo[i]

                    entry = table.model(**entry)

                    if not self.dryrun:
                        self.session.add(entry)

                    if count % 100 == 0:
                        self.session.flush()

        if not self.dryrun:
            self.session.commit()

    def store_missing_geos(self):
        count = 0
        for table_id, missing_geos in self.missing_geos_by_table.iteritems():
            table = self.field_tables[table_id]
            self.stdout.write("Storing missing geos for : %s" % (table_id))
            for geo in missing_geos:
                # Entry for each possible key combination for each missing geo
                for keys in self.keys_by_table[table_id]:
                    count += 1
                    row = {
                        'geo_level': geo[0],
                        'geo_code': geo[1],
                        'total': None,
                        'geo_version': self.geo_version}
                    for i, field in enumerate(self.fields_by_table[table_id]):
                        row[field] = keys[i]

                    entry = table.model(**row)

                    if not self.dryrun:
                        self.session.add(entry)

                    if count % 100 == 0:
                        self.session.flush()

        if not self.dryrun:
            self.session.commit()

    def store_simple_table_missing_geos(self):
        count = 0
        for table_id, missing_geos in self.missing_geos_by_simple_table.iteritems():
            table = self.simple_tables[table_id]
            self.stdout.write("Storing missing geos for : %s" % (table_id))
            for geo in missing_geos:
                count += 1
                row = {
                    'geo_level': geo[0],
                    'geo_code': geo[1],
                    'geo_version': self.geo_version}

                entry = table.model(**row)

                if not self.dryrun:
                    self.session.add(entry)

                if count % 100 == 0:
                    self.session.flush()

        if not self.dryrun:
            self.session.commit()
<commit_msg>Only check for WC geographies for certain tables<commit_after>import sys
from collections import defaultdict

from django.core.management.base import BaseCommand

from wazimap.data.utils import get_session
from wazimap.data.tables import get_datatable, get_table_id, DATA_TABLES, FIELD_TABLES, FieldTable, SimpleTable
from wazimap.geo import geo_data

import logging

logging.basicConfig()
logging.getLogger('sqlalchemy.engine').setLevel(logging.WARN)

"""
This is a helper script that checks the tables in the DB for missing geo entries,
and missing keys for fields. The tables with missing values will be output after completion.

If the store_missing_entries flag is passed, the missing items will be populated in the DB.
Missing geos are populated with null values for all keys,
and missing key values are populated with 0.

"""

WC_ONLY_TABLES = [
    # Education
    'youth_average_mean_score_by_year',
    'youth_average_language_score_by_year',
    'youth_language_outcome_by_year',
    'youth_average_maths_score_by_year',
    'youth_maths_outcome_by_year',
    'youth_matric_outcome_by_year',
    'youth_matric_passes_as_percentage_of_grade8_enrolment_by_year',
    'youth_bachelor_passes_as_percentage_of_grade8_enrolment_by_year',
    'youth_student_dropout_rate_by_year',
    # Crime
    'crimes_victims_age_group',
    'crimes_accused_age_group',
    'youth_victims_offence_type',
    'youth_accused_offence_type',
    'youth_victims_population_group',
    'youth_accused_population_group',
    'youth_victims_gender',
    'youth_accused_gender',
    'youth_victims_year',
    'youth_accused_year',
    'crimes_type_of_crime_year',
    # Health
    'youth_pregnancy_rate_year',
    'youth_delivery_rate_year',
    'youth_causes_of_death_female',
    'youth_causes_of_death_male',
]

class Command(BaseCommand):
    help = ("Checks the database for completeness (or a single table if passed)." +
        "Populates the missing entries if populate_missing_values=True")

    def add_arguments(self, parser):
        parser.add_argument(
            '--table',
            action='store',
            dest='table',
            default=None,
            help='The name of the database table to check'
        )
        parser.add_argument(
            '--geo_code',
            action='store',
            dest='geo_code',
            default='ZA',
            help='The geo_code for the base geo that contains entries for all keys, for all fields e.g. ZA'
        )
        parser.add_argument(
            '--geo_level',
            action='store',
            dest='geo_level',
            default='country',
            help='The geo_level for the base geo that contains entries for all keys, for all fields e.g. country'
        )
        parser.add_argument(
            '--geo_version',
            action='store',
            dest='geo_version',
            default='2011',
            help='The geo_version of the data which should be checked.'
        )
        parser.add_argument(
            '--store_missing_entries',
            action='store_true',
            dest='store_missing_entries',
            default=False,
            help="Write data for missing entries to DB",
        )
        parser.add_argument(
            '--dry-run',
            action='store_true',
            dest='dryrun',
            default=False,
            help="Dry-run, don't actuall write any data.",
        )

    def debug(self, msg):
        if self.verbosity >= 2:
            self.stdout.write(str(msg))

    def handle(self, *args, **options):
        self.session = get_session()
        self.verbosity = options.get('verbosity', 0)
        self.table_id = options.get('table')
        self.geo_version = options.get('geo_version')
        self.store_missing_entries = options.get('store_missing_entries', False)
        self.dryrun = options.get('dryrun')

        self.geos = self.get_geos(self.geo_version)

        self.db_tables = []
        self.fields_by_table = {}
        self.keys_by_table = {}
        self.missing_keys_by_table = {}
        self.missing_geos_by_table = {}
        self.missing_geos_by_simple_table = {}

        if self.table_id:
            table = get_datatable(self.table_id)
            if type(table) == FieldTable:
                self.field_tables = {table.id:table}
            else:
                self.simple_tables = {table.id:table}
        else:
            self.tables = DATA_TABLES
            self.field_tables = FIELD_TABLES
            self.simple_tables = {k: v for k, v in DATA_TABLES.iteritems() if k not in FIELD_TABLES.keys()}

        for table_id, table in self.field_tables.iteritems():
            if table.db_table in self.db_tables:
                # Multiple field tables can refer to the same underlying db table
                continue
            self.db_tables.append(table.db_table)
            self.stdout.write("Checking table: %s" % (table.id))

            self.fields_by_table[table.id] = table.fields
            rows = self.session.query(table.model).filter(table.model.geo_version == self.geo_version).all()

            self.keys_by_table[table_id] = self.get_table_keys(table, table.fields)

            missing_keys = self.get_missing_keys(table, rows, table.fields)
            if missing_keys:
                self.missing_keys_by_table[table.id] = missing_keys

            missing_geos = self.get_missing_geos(table, rows)
            if missing_geos:
                self.missing_geos_by_table[table.id] = missing_geos

        if self.missing_keys_by_table:
            self.stdout.write("Missing keys for tables:")
            for table in self.missing_keys_by_table.iterkeys():
                self.stdout.write("%s" % (table))

        if self.missing_geos_by_table:
            self.stdout.write("Missing geos for tables:")
            for table in self.missing_geos_by_table.iterkeys():
                self.stdout.write("%s" % table)

        if self.store_missing_entries:
            if self.missing_keys_by_table:
                self.store_missing_keys()
                self.stdout.write("Missing keys stored in database.")
            if self.missing_geos_by_table:
                self.store_missing_geos()
                self.stdout.write("Missing geos stored in database.")
        else:
            self.stdout.write("Run command with --store_missing_entries=True to populate missing keys with 0 and missing geos with null")

        for table_id, table in self.simple_tables.iteritems():
            self.stdout.write("Checking table: %s" % (table.id))
            rows = self.session.query(table.model).filter(table.model.geo_version == self.geo_version).all()
            missing_geos = self.get_missing_geos(table, rows)
            if missing_geos:
                self.missing_geos_by_simple_table[table.id] = missing_geos

        if self.missing_geos_by_simple_table:
            self.stdout.write("Missing geos for Simple tables:")
            for table in self.missing_geos_by_simple_table.iterkeys():
                self.stdout.write("%s" % table)

            if self.store_missing_entries:
                self.store_simple_table_missing_geos()
                self.stdout.write("Missing geos for Simple tables stored in database.")

        self.session.close()

    def get_geos(self, geo_version):
        return geo_data.geo_model.objects.filter(version=geo_version)

    def get_table_keys(self, table, fields):
        # Return a list with all permuations of the keys for all fields
        keys = []
        for obj in self.session.query(table.model).distinct(*fields):
            keys.append([obj.__getattribute__(field) for field in fields])

        return keys

    def get_missing_keys(self, table, rows, fields):
        keys_by_geo = defaultdict(list)
        for row in rows:
            # Get existing key combinations by geo_id
            keys_by_geo[row.geo_level + '-' + row.geo_code].append(
                [row.__getattribute__(field) for field in fields])

        missing_keys_by_geo = {}
        for geo_id, geo_keys in keys_by_geo.iteritems():
            if len(geo_keys) != len(self.keys_by_table[table.id]):
                missing_keys_by_geo[geo_id] = [
                    x for x in self.keys_by_table[table.id] if x not in geo_keys]

        return missing_keys_by_geo

    def get_missing_geos(self, table, rows):
        table_geos = set((row.geo_level, row.geo_code) for row in rows)
        if not table_geos:
            sys.exit("Empty table: %s" % (table.id))

        if table.id.lower() in WC_ONLY_TABLES:
            all_geos = set(
                (g.geo_level, g.geo_code) for g in self.geos
                if g.geo_code == 'WC'
                or 'WC' in [cg.geo_code for cg in geo_data.get_comparative_geos(g)])
        else:
            all_geos = set((g.geo_level, g.geo_code) for g in self.geos)

        missing_geos = [g for g in all_geos if g not in table_geos]

        if missing_geos:
            self.missing_geos_by_table[table.id] = missing_geos

        return missing_geos

    def store_missing_keys(self):
        count = 0
        for table_id, missing_keys_by_geo in self.missing_keys_by_table.iteritems():
            table = self.field_tables[table_id]
            self.stdout.write("Storing missing keys for : %s" % (table_id))
            for geo, missing_keys in missing_keys_by_geo.iteritems():
                for key_combo in missing_keys:
                    count += 1
                    geo_level, geo_code = geo.split('-')
                    entry = {
                        'geo_level': geo_level,
                        'geo_code': geo_code,
                        'total': 0,
                        'geo_version': self.geo_version}
                    for i, field in enumerate(self.fields_by_table[table_id]):
                        entry[field] = key_combo[i]

                    entry = table.model(**entry)

                    if not self.dryrun:
                        self.session.add(entry)

                    if count % 100 == 0:
                        self.session.flush()

        if not self.dryrun:
            self.session.commit()

    def store_missing_geos(self):
        count = 0
        for table_id, missing_geos in self.missing_geos_by_table.iteritems():
            table = self.field_tables[table_id]
            self.stdout.write("Storing missing geos for : %s" % (table_id))
            for geo in missing_geos:
                # Entry for each possible key combination for each missing geo
                for keys in self.keys_by_table[table_id]:
                    count += 1
                    row = {
                        'geo_level': geo[0],
                        'geo_code': geo[1],
                        'total': None,
                        'geo_version': self.geo_version}
                    for i, field in enumerate(self.fields_by_table[table_id]):
                        row[field] = keys[i]

                    entry = table.model(**row)

                    if not self.dryrun:
                        self.session.add(entry)

                    if count % 100 == 0:
                        self.session.flush()

        if not self.dryrun:
            self.session.commit()

    def store_simple_table_missing_geos(self):
        count = 0
        for table_id, missing_geos in self.missing_geos_by_simple_table.iteritems():
            table = self.simple_tables[table_id]
            self.stdout.write("Storing missing geos for : %s" % (table_id))
            for geo in missing_geos:
                count += 1
                row = {
                    'geo_level': geo[0],
                    'geo_code': geo[1],
                    'geo_version': self.geo_version}

                entry = table.model(**row)

                if not self.dryrun:
                    self.session.add(entry)

                if count % 100 == 0:
                    self.session.flush()

        if not self.dryrun:
            self.session.commit()
<|endoftext|>


<commit_before># Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
# Copyright 2012 Red Hat, Inc.
# Copyright 2013 NTT corp.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Command-line flag library.

Emulates gflags by wrapping cfg.ConfigOpts.

The idea is to move fully to cfg eventually, and this wrapper is a
stepping stone.

"""

import socket

from oslo_config import cfg
from oslo_log import log as logging
from oslo_middleware import cors
from oslo_utils import netutils

CONF = cfg.CONF
logging.register_options(CONF)

core_opts = [
    cfg.StrOpt('state_path',
               default='/var/lib/cinder',
               deprecated_name='pybasedir',
               help="Top-level directory for maintaining cinder's state"), ]

CONF.register_cli_opts(core_opts)

global_opts = [
    cfg.StrOpt('my_ip',
               default=netutils.get_my_ipv4(),
               help='IP address of this host'),
    cfg.ListOpt('glance_api_servers',
                default=None,
                help='A list of the URLs of glance API servers available to '
                     'cinder ([http[s]://][hostname|ip]:port). If protocol '
                     'is not specified it defaults to http.'),
    cfg.IntOpt('glance_api_version',
               default=1,
               help='Version of the glance API to use'),
    cfg.IntOpt('glance_num_retries',
               min=0,
               default=0,
               help='Number retries when downloading an image from glance'),
    cfg.BoolOpt('glance_api_insecure',
                default=False,
                help='Allow to perform insecure SSL (https) requests to '
                     'glance (https will be used but cert validation will '
                     'not be performed).'),
    cfg.BoolOpt('glance_api_ssl_compression',
                default=False,
                help='Enables or disables negotiation of SSL layer '
                     'compression. In some cases disabling compression '
                     'can improve data throughput, such as when high '
                     'network bandwidth is available and you use '
                     'compressed image formats like qcow2.'),
    cfg.StrOpt('glance_ca_certificates_file',
               help='Location of ca certificates file to use for glance '
                    'client requests.'),
    cfg.IntOpt('glance_request_timeout',
               help='http/https timeout value for glance operations. If no '
                    'value (None) is supplied here, the glanceclient default '
                    'value is used.'),
    cfg.BoolOpt('enable_v1_api',
                default=False,
                deprecated_for_removal=True,
                help="DEPRECATED: Deploy v1 of the Cinder API."),
    cfg.BoolOpt('enable_v2_api',
                default=True,
                deprecated_for_removal=True,
                help="DEPRECATED: Deploy v2 of the Cinder API."),
    cfg.BoolOpt('enable_v3_api',
                default=True,
                help="Deploy v3 of the Cinder API."),
    cfg.BoolOpt('api_rate_limit',
                default=True,
                help='Enables or disables rate limit of the API.'),
    cfg.ListOpt('osapi_volume_ext_list',
                default=[],
                help='Specify list of extensions to load when using osapi_'
                     'volume_extension option with cinder.api.contrib.'
                     'select_extensions'),
    cfg.MultiStrOpt('osapi_volume_extension',
                    default=['cinder.api.contrib.standard_extensions'],
                    help='osapi volume extension to load'),
    cfg.StrOpt('volume_manager',
               default='cinder.volume.manager.VolumeManager',
               help='Full class name for the Manager for volume'),
    cfg.StrOpt('backup_manager',
               default='cinder.backup.manager.BackupManager',
               help='Full class name for the Manager for volume backup'),
    cfg.StrOpt('scheduler_manager',
               default='cinder.scheduler.manager.SchedulerManager',
               help='Full class name for the Manager for scheduler'),
    cfg.StrOpt('host',
               default=socket.gethostname(),
               help='Name of this node.  This can be an opaque identifier. '
                    'It is not necessarily a host name, FQDN, or IP address.'),
    # NOTE(vish): default to nova for compatibility with nova installs
    cfg.StrOpt('storage_availability_zone',
               default='nova',
               help='Availability zone of this node. Can be overridden per '
                    'volume backend with the option '
                    '"backend_availability_zone".'),
    cfg.StrOpt('default_availability_zone',
               help='Default availability zone for new volumes. If not set, '
                    'the storage_availability_zone option value is used as '
                    'the default for new volumes.'),
    cfg.BoolOpt('allow_availability_zone_fallback',
                default=False,
                help='If the requested Cinder availability zone is '
                     'unavailable, fall back to the value of '
                     'default_availability_zone, then '
                     'storage_availability_zone, instead of failing.'),
    cfg.StrOpt('default_volume_type',
               help='Default volume type to use'),
    cfg.StrOpt('default_group_type',
               help='Default group type to use'),
    cfg.StrOpt('volume_usage_audit_period',
               default='month',
               help='Time period for which to generate volume usages. '
                    'The options are hour, day, month, or year.'),
    cfg.StrOpt('rootwrap_config',
               default='/etc/cinder/rootwrap.conf',
               help='Path to the rootwrap configuration file to use for '
                    'running commands as root'),
    cfg.BoolOpt('monkey_patch',
                default=False,
                help='Enable monkey patching'),
    cfg.ListOpt('monkey_patch_modules',
                default=[],
                help='List of modules/decorators to monkey patch'),
    cfg.IntOpt('service_down_time',
               default=60,
               help='Maximum time since last check-in for a service to be '
                    'considered up'),
    cfg.StrOpt('volume_api_class',
               default='cinder.volume.api.API',
               help='The full class name of the volume API class to use'),
    cfg.StrOpt('backup_api_class',
               default='cinder.backup.api.API',
               help='The full class name of the volume backup API class'),
    cfg.StrOpt('auth_strategy',
               default='keystone',
               choices=['noauth', 'keystone'],
               help='The strategy to use for auth. Supports noauth or '
                    'keystone.'),
    cfg.ListOpt('enabled_backends',
                help='A list of backend names to use. These backend names '
                     'should be backed by a unique [CONFIG] group '
                     'with its options'),
    cfg.BoolOpt('no_snapshot_gb_quota',
                default=False,
                help='Whether snapshots count against gigabyte quota'),
    cfg.StrOpt('transfer_api_class',
               default='cinder.transfer.api.API',
               help='The full class name of the volume transfer API class'),
    cfg.StrOpt('consistencygroup_api_class',
               default='cinder.consistencygroup.api.API',
               help='The full class name of the consistencygroup API class'),
    cfg.StrOpt('group_api_class',
               default='cinder.group.api.API',
               help='The full class name of the group API class'),
    cfg.StrOpt('os_privileged_user_name',
               help='OpenStack privileged account username. Used for requests '
                    'to other services (such as Nova) that require an account '
                    'with special rights.'),
    cfg.StrOpt('os_privileged_user_password',
               help='Password associated with the OpenStack privileged '
                    'account.',
               secret=True),
    cfg.StrOpt('os_privileged_user_tenant',
               help='Tenant name associated with the OpenStack privileged '
                    'account.'),
    cfg.URIOpt('os_privileged_user_auth_url',
               help='Auth URL associated with the OpenStack privileged '
                    'account.'),
]

CONF.register_opts(core_opts)
CONF.register_opts(global_opts)

def set_middleware_defaults():
    """Update default configuration options for oslo.middleware."""
    # CORS Defaults
    # TODO(krotscheck): Update with https://review.openstack.org/#/c/285368/
    cfg.set_defaults(cors.CORS_OPTS,
                     allow_headers=['X-Auth-Token',
                                    'X-Identity-Status',
                                    'X-Roles',
                                    'X-Service-Catalog',
                                    'X-User-Id',
                                    'X-Tenant-Id',
                                    'X-OpenStack-Request-ID',
                                    'X-Trace-Info',
                                    'X-Trace-HMAC',
                                    'OpenStack-API-Version'],
                     expose_headers=['X-Auth-Token',
                                     'X-Subject-Token',
                                     'X-Service-Token',
                                     'X-OpenStack-Request-ID',
                                     'OpenStack-API-Version'],
                     allow_methods=['GET',
                                    'PUT',
                                    'POST',
                                    'DELETE',
                                    'PATCH',
                                    'HEAD']
                     )
<commit_msg>cors: update default configuration<commit_after># Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
# Copyright 2012 Red Hat, Inc.
# Copyright 2013 NTT corp.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Command-line flag library.

Emulates gflags by wrapping cfg.ConfigOpts.

The idea is to move fully to cfg eventually, and this wrapper is a
stepping stone.

"""

import socket

from oslo_config import cfg
from oslo_log import log as logging
from oslo_middleware import cors
from oslo_utils import netutils

CONF = cfg.CONF
logging.register_options(CONF)

core_opts = [
    cfg.StrOpt('state_path',
               default='/var/lib/cinder',
               deprecated_name='pybasedir',
               help="Top-level directory for maintaining cinder's state"), ]

CONF.register_cli_opts(core_opts)

global_opts = [
    cfg.StrOpt('my_ip',
               default=netutils.get_my_ipv4(),
               help='IP address of this host'),
    cfg.ListOpt('glance_api_servers',
                default=None,
                help='A list of the URLs of glance API servers available to '
                     'cinder ([http[s]://][hostname|ip]:port). If protocol '
                     'is not specified it defaults to http.'),
    cfg.IntOpt('glance_api_version',
               default=1,
               help='Version of the glance API to use'),
    cfg.IntOpt('glance_num_retries',
               min=0,
               default=0,
               help='Number retries when downloading an image from glance'),
    cfg.BoolOpt('glance_api_insecure',
                default=False,
                help='Allow to perform insecure SSL (https) requests to '
                     'glance (https will be used but cert validation will '
                     'not be performed).'),
    cfg.BoolOpt('glance_api_ssl_compression',
                default=False,
                help='Enables or disables negotiation of SSL layer '
                     'compression. In some cases disabling compression '
                     'can improve data throughput, such as when high '
                     'network bandwidth is available and you use '
                     'compressed image formats like qcow2.'),
    cfg.StrOpt('glance_ca_certificates_file',
               help='Location of ca certificates file to use for glance '
                    'client requests.'),
    cfg.IntOpt('glance_request_timeout',
               help='http/https timeout value for glance operations. If no '
                    'value (None) is supplied here, the glanceclient default '
                    'value is used.'),
    cfg.BoolOpt('enable_v1_api',
                default=False,
                deprecated_for_removal=True,
                help="DEPRECATED: Deploy v1 of the Cinder API."),
    cfg.BoolOpt('enable_v2_api',
                default=True,
                deprecated_for_removal=True,
                help="DEPRECATED: Deploy v2 of the Cinder API."),
    cfg.BoolOpt('enable_v3_api',
                default=True,
                help="Deploy v3 of the Cinder API."),
    cfg.BoolOpt('api_rate_limit',
                default=True,
                help='Enables or disables rate limit of the API.'),
    cfg.ListOpt('osapi_volume_ext_list',
                default=[],
                help='Specify list of extensions to load when using osapi_'
                     'volume_extension option with cinder.api.contrib.'
                     'select_extensions'),
    cfg.MultiStrOpt('osapi_volume_extension',
                    default=['cinder.api.contrib.standard_extensions'],
                    help='osapi volume extension to load'),
    cfg.StrOpt('volume_manager',
               default='cinder.volume.manager.VolumeManager',
               help='Full class name for the Manager for volume'),
    cfg.StrOpt('backup_manager',
               default='cinder.backup.manager.BackupManager',
               help='Full class name for the Manager for volume backup'),
    cfg.StrOpt('scheduler_manager',
               default='cinder.scheduler.manager.SchedulerManager',
               help='Full class name for the Manager for scheduler'),
    cfg.StrOpt('host',
               default=socket.gethostname(),
               help='Name of this node.  This can be an opaque identifier. '
                    'It is not necessarily a host name, FQDN, or IP address.'),
    # NOTE(vish): default to nova for compatibility with nova installs
    cfg.StrOpt('storage_availability_zone',
               default='nova',
               help='Availability zone of this node. Can be overridden per '
                    'volume backend with the option '
                    '"backend_availability_zone".'),
    cfg.StrOpt('default_availability_zone',
               help='Default availability zone for new volumes. If not set, '
                    'the storage_availability_zone option value is used as '
                    'the default for new volumes.'),
    cfg.BoolOpt('allow_availability_zone_fallback',
                default=False,
                help='If the requested Cinder availability zone is '
                     'unavailable, fall back to the value of '
                     'default_availability_zone, then '
                     'storage_availability_zone, instead of failing.'),
    cfg.StrOpt('default_volume_type',
               help='Default volume type to use'),
    cfg.StrOpt('default_group_type',
               help='Default group type to use'),
    cfg.StrOpt('volume_usage_audit_period',
               default='month',
               help='Time period for which to generate volume usages. '
                    'The options are hour, day, month, or year.'),
    cfg.StrOpt('rootwrap_config',
               default='/etc/cinder/rootwrap.conf',
               help='Path to the rootwrap configuration file to use for '
                    'running commands as root'),
    cfg.BoolOpt('monkey_patch',
                default=False,
                help='Enable monkey patching'),
    cfg.ListOpt('monkey_patch_modules',
                default=[],
                help='List of modules/decorators to monkey patch'),
    cfg.IntOpt('service_down_time',
               default=60,
               help='Maximum time since last check-in for a service to be '
                    'considered up'),
    cfg.StrOpt('volume_api_class',
               default='cinder.volume.api.API',
               help='The full class name of the volume API class to use'),
    cfg.StrOpt('backup_api_class',
               default='cinder.backup.api.API',
               help='The full class name of the volume backup API class'),
    cfg.StrOpt('auth_strategy',
               default='keystone',
               choices=['noauth', 'keystone'],
               help='The strategy to use for auth. Supports noauth or '
                    'keystone.'),
    cfg.ListOpt('enabled_backends',
                help='A list of backend names to use. These backend names '
                     'should be backed by a unique [CONFIG] group '
                     'with its options'),
    cfg.BoolOpt('no_snapshot_gb_quota',
                default=False,
                help='Whether snapshots count against gigabyte quota'),
    cfg.StrOpt('transfer_api_class',
               default='cinder.transfer.api.API',
               help='The full class name of the volume transfer API class'),
    cfg.StrOpt('consistencygroup_api_class',
               default='cinder.consistencygroup.api.API',
               help='The full class name of the consistencygroup API class'),
    cfg.StrOpt('group_api_class',
               default='cinder.group.api.API',
               help='The full class name of the group API class'),
    cfg.StrOpt('os_privileged_user_name',
               help='OpenStack privileged account username. Used for requests '
                    'to other services (such as Nova) that require an account '
                    'with special rights.'),
    cfg.StrOpt('os_privileged_user_password',
               help='Password associated with the OpenStack privileged '
                    'account.',
               secret=True),
    cfg.StrOpt('os_privileged_user_tenant',
               help='Tenant name associated with the OpenStack privileged '
                    'account.'),
    cfg.URIOpt('os_privileged_user_auth_url',
               help='Auth URL associated with the OpenStack privileged '
                    'account.'),
]

CONF.register_opts(core_opts)
CONF.register_opts(global_opts)

def set_middleware_defaults():
    """Update default configuration options for oslo.middleware."""

    cors.set_defaults(
        allow_headers=['X-Auth-Token',
                       'X-Identity-Status',
                       'X-Roles',
                       'X-Service-Catalog',
                       'X-User-Id',
                       'X-Tenant-Id',
                       'X-OpenStack-Request-ID',
                       'X-Trace-Info',
                       'X-Trace-HMAC',
                       'OpenStack-API-Version'],
        expose_headers=['X-Auth-Token',
                        'X-Subject-Token',
                        'X-Service-Token',
                        'X-OpenStack-Request-ID',
                        'OpenStack-API-Version'],
        allow_methods=['GET',
                       'PUT',
                       'POST',
                       'DELETE',
                       'PATCH',
                       'HEAD']
    )
<|endoftext|>