<commit_before>"""
Count occurrence of bases in a set of sequences
"""

from Bio import SeqIO
import csv

def register_arguments(parser):
    parser.add_argument('--sequences', '-s', required=True, help="sequences in fasta or VCF format")
    parser.add_argument('--output', '-o', help="output file", required=True)

def run(args):
    '''
    counts the number of a,c,t,g,n and other IUPAC nucleotides in a set of sequences
    '''
    #read in files
    try:
        seqs = SeqIO.parse(args.sequences, 'fasta')
    except ValueError as error:
        print("ERROR: Problem reading in {}:".format(args.sequences))
        print(error)
        return 1

    good_chars = {'a', 'c', 'g', 't', 'n', '-', 'r', 'y', 's', 'w', 'k', 'm', 'd', 'h', 'b', 'v','?'}
    tot_length = 0
    num_of_seqs = 0
    #characters to count as list of strings
    other_IUPAC = ['r', 'y', 's', 'w', 'k', 'm', 'd', 'h', 'b', 'v']
    values = [['a'],['c'],['g'],['t'],['n'],other_IUPAC]
    #labels for header in tsv output file
    labels = ['A','C','G','T','N','other_IUPAC']

    #print a warning if labels do not match characters to count
    if len(labels)-len(values)!=0:
        print('Warning: labels do not  match length of values to count')

    #count occurences and write to output file
    with open(args.output, 'wt') as out_file:
        tsv_writer = csv.writer(out_file, delimiter = '\t')

        #write header
        header = ['sequence_name', 'length']+labels
        tsv_writer.writerow(header)

        for record in seqs:
            #check if sequence contains non nucleotide characters
            if len(set(str(record.seq)).difference(good_chars))==0:

                #change sequences to lower case and remove gaps
                seq = record.seq.ungap(gap='-').ungap(gap='?').lower()
                l = len(seq)
                tot_length += l
                num_of_seqs += 1

                #count occurrence for each value
                counts = []
                for v in values:
                    counts.append(sum(map(lambda x: seq.count(x), v)))
                #write row for each sequence
                row = [record.id, l]+counts
                tsv_writer.writerow(row)

            else:
                #skip sequences with non nucleotide characters
                print("sequence contains non nucleotide characters, skipping...")

    print("Analysed %i sequences with an average length of %i nucleotides."%(num_of_seqs,int(tot_length/num_of_seqs)))
    seqs.close()<commit_msg>Fixes parsing of upper case sequences.<commit_after>"""
Count occurrence of bases in a set of sequences
"""

from Bio import SeqIO
import csv

def register_arguments(parser):
    parser.add_argument('--sequences', '-s', required=True, help="sequences in fasta or VCF format")
    parser.add_argument('--output', '-o', help="output file", required=True)

def run(args):
    '''
    counts the number of a,c,t,g,n and other IUPAC nucleotides in a set of sequences
    '''
    #read in files
    try:
        seqs = SeqIO.parse(args.sequences, 'fasta')
    except ValueError as error:
        print("ERROR: Problem reading in {}:".format(args.sequences))
        print(error)
        return 1

    good_chars = {'a', 'c', 'g', 't', 'n', '-', 'r', 'y', 's', 'w', 'k', 'm', 'd', 'h', 'b', 'v','?'}
    tot_length = 0
    num_of_seqs = 0
    #characters to count as list of strings
    other_IUPAC = ['r', 'y', 's', 'w', 'k', 'm', 'd', 'h', 'b', 'v']
    values = [['a'],['c'],['g'],['t'],['n'],other_IUPAC]
    #labels for header in tsv output file
    labels = ['A','C','G','T','N','other_IUPAC']

    #print a warning if labels do not match characters to count
    if len(labels)-len(values)!=0:
        print('Warning: labels do not  match length of values to count')

    #count occurences and write to output file
    with open(args.output, 'wt') as out_file:
        tsv_writer = csv.writer(out_file, delimiter = '\t')

        #write header
        header = ['sequence_name', 'length']+labels
        tsv_writer.writerow(header)

        for record in seqs:
            #check if sequence contains non nucleotide characters
            if len(set(str(record.seq.lower())).difference(good_chars))==0:

                #change sequences to lower case and remove gaps
                seq = record.seq.ungap(gap='-').ungap(gap='?').lower()
                l = len(seq)
                tot_length += l
                num_of_seqs += 1

                #count occurrence for each value
                counts = []
                for v in values:
                    counts.append(sum(map(lambda x: seq.count(x), v)))
                #write row for each sequence
                row = [record.id, l]+counts
                tsv_writer.writerow(row)

            else:
                #skip sequences with non nucleotide characters
                print("sequence contains non nucleotide characters, skipping...")

    if num_of_seqs==0:
        print("No valid sequences found.")
    else:
        print("Analysed %i sequences with an average length of %i nucleotides."%(num_of_seqs,int(tot_length/num_of_seqs)))

    seqs.close()<|endoftext|>


<commit_before>import VCF
import unittest
from collections import OrderedDict

class TestSlicing(unittest.TestCase):

    def setUp(self):
        self.bgzip_path = "test_data/butterfly.vcf.gz"

    def test_make_slices_default_settings(self):
        """Test slicing function with default settings: 500 bp slices"""

        for count, i in enumerate(VCF.get_slice_indicies(self.bgzip_path, regions=None, window_size=500)):
            if count > 10: break

        self.assertEqual(i, ('Chr01', 5501, 6000))

    def test_make_slices_default_with_params_set(self):
        """Test slicing function with window_size set"""

        for count, i in enumerate(VCF.get_slice_indicies(self.bgzip_path, regions=None, window_size=1008)):
            if count > 10: break

        self.assertEqual(i, ('Chr01', 11089, 12096))

    # def test_make_vcf_slices(self):
    #     print VCF.slice_vcf(self.bgzip_path, 'Chr01', 5501, 6000)

    def test_check_length_of_first_ten_slices(self):
        pass

class TestVCFInfoParsing(unittest.TestCase):

    def setUp(self):
        self.bgzip_path = "test_data/butterfly.vcf.gz"

        self.populations_list = [
            "cydno:c511,c512,c513,c514,c515,c563,c614,c630,c639,c640",
            "outgroups:h665,i02-210",
            "melpo:m523,m524,m525,m589,m675,m676,m682,m683,m687,m689",
            "pachi:p516,p517,p518,p519,p520,p591,p596,p690,p694,p696"]

        self.header_dict = OrderedDict([('CHROM', None), ('POS', None),\
            ('ID', None), ('REF', None), ('ALT', None), ('QUAL', None), \
            ('FILTER', None), ('INFO', None), ('FORMAT', None), ('c511', None), \
            ('c512', None), ('c513', None), ('c514', None), ('c515', None), \
            ('c563', None), ('c614', None), ('c630', None), ('c639', None), \
            ('c640', None), ('h665', None), ('i02-210', None), ('m523', None), \
            ('m524', None), ('m525', None), ('m589', None), ('m675', None), \
            ('m676', None), ('m682', None), ('m683', None), ('m687', None), \
            ('m689', None), ('p516', None), ('p517', None), ('p518', None), \
            ('p519', None), ('p520', None), ('p591', None), ('p596', None), \
            ('p690', None), ('p694', None), ('p696', None)])

        self.allle_counts = {'melpo': {0: 12.0, 1: 0.0, 2: 0.0, 3: 0.0, 4: 0.0},
                            'cydno': {0: 20.0, 1: 0.0, 2: 0.0, 3: 0.0, 4: 0.0},
                            'pachi': {0: 18.0, 1: 0.0, 2: 0.0, 3: 0.0, 4: 0.0},
                            'outgroups': {0: 2.0, 1: 2.0, 2: 0.0, 3: 0.0, 4: 0.0}}

    def test_population_string_parsing(self):
        populations = VCF.parse_populations_list(self.populations_list)

        self.assertEqual(populations, {'melpo': ['m523', 'm524', 'm525',
            'm589', 'm675', 'm676', 'm682', 'm683', 'm687', 'm689'],
            'pachi': ['p516', 'p517', 'p518', 'p519', 'p520', 'p591',
            'p596', 'p690', 'p694', 'p696'], 'cydno': ['c511',
            'c512', 'c513', 'c514', 'c515', 'c563', 'c614', 'c630',
            'c639', 'c640'], 'outgroups': ['h665', 'i02-210']})

    def test_header_to_ordered_dict_parsing(self):
        header = VCF.make_empty_vcf_ordered_dict(self.bgzip_path)

        self.assertEqual(header,self.header_dict)

    def test_header_vs_population_sample_ids(self):
        """Check that the sample IDs parsed from the population arguement
            match those in the VCF file.

            NOTE: In practice the populations arguement can contain fewer
            samples and populations than actually contained in the VCF file.
        """

        header = VCF.make_empty_vcf_ordered_dict(self.bgzip_path, )
        header_sample_ids = [item for count, item in enumerate(header) if count >= 9]

        populations_dict  = VCF.parse_populations_list(self.populations_list)
        populations_sample_ids = [i for l in populations_dict.values() for i in l]

        # Check both unique IDs and equal length
        self.assertEqual(set(header_sample_ids), set(populations_sample_ids))
        self.assertEqual(len(header_sample_ids), len(populations_sample_ids))

class TestGenoTypeParsing(unittest.TestCase):

    def setUp(self):
        self.vcf_line = OrderedDict([
                        ('CHROM', 'Chr01'),
                        ('POS', 4984),
                        ('ID', '.'),
                        ('REF', 'C'),
                        ('ALT', 'T'),
                        ('QUAL', 235.91),
                        ('FILTER', 'PASS'),
                        ('INFO', {'AC': '1'}),
                        ('FORMAT', 'GT:AD:DP:GQ:PL'),
                        ('out1', {'GT': '0/0', 'GQ': 51.17, 'AD': (17, 0), 'DP': 17, 'PL': (0, 51, 669)}),
                        ('out2', {'GT': '0/0', 'GQ': 45.15, 'AD': (16, 0), 'DP': 16, 'PL': (0, 45, 600)})])

        self.populations = {'Outgroup': ['out1', 'out2']}

    #### WITH AMBIGUITY CODES ####
    def test_homo_ref_genotype_calling(self):
        homo_ref = VCF.process_snp_call('0/0:10,9:19:99:254,0,337', 'A', 'T', IUPAC_ambiguities=True)
        self.assertEqual(homo_ref, 'A')

    def test_heterozygote_genotype_calling(self):
        heterozygote = VCF.process_snp_call('0/1:10,9:19:99:254,0,337', 'A', 'T', IUPAC_ambiguities=True)
        self.assertEqual(heterozygote, 'W')

    def test_homo_alt_genotype_calling(self):
        homo_alt =  VCF.process_snp_call('1/1:10,9:19:99:254,0,337', 'A', 'T', IUPAC_ambiguities=True)
        self.assertEqual(homo_alt, 'T')

    def test_second_alt_genotype_calling(self):
        second_alt = VCF.process_snp_call('0/2:10,9:19:99:254,0,337', 'A', 'T,G', IUPAC_ambiguities=True)
        self.assertEqual(second_alt, 'R')

    def test_double_alt_genotype_calling(self):
        double_alt = VCF.process_snp_call('1/2:10,9:19:99:254,0,337', 'A', 'T,G', IUPAC_ambiguities=True)
        self.assertEqual(double_alt, 'K')

    #### WITHOUT AMBIGUITY CODES ####
    def test_heterozygote_as_N_genotype_calling(self):
        heterozygote_as_N = VCF.process_snp_call('0/1:10,9:19:99:254,0,337', 'A', 'T', IUPAC_ambiguities=False)
        self.assertEqual(heterozygote_as_N, 'N')

    def test_double_alt_het_as_N_genotype_calling(self):
        double_alt_het_as_N = VCF.process_snp_call('1/2:10,9:19:99:254,0,337', 'A', 'T,G', IUPAC_ambiguities=False)
        self.assertEqual(double_alt_het_as_N, 'N')

    #### TEST OUTGROUP CALLING ####
    def test_home_ref_outgroup_calling(self):
        homo_ref = VCF.process_outgroup(self.vcf_line, self.populations)
        self.assertEqual(homo_ref, '0')

    def test_home_alt_outgroup_calling(self):
        vcf_line = self.vcf_line

        vcf_line['out1']['GT'] = '1/1'
        vcf_line['out2']['GT'] = '1/1'

        homo_alt = VCF.process_outgroup(vcf_line, self.populations)
        self.assertEqual(homo_alt, '1')

    def test_heterozyogote_outgroup_calling(self):
        vcf_line = self.vcf_line

        vcf_line['out1']['GT'] = '1/1'
        vcf_line['out2']['GT'] = '1/0'

        het = VCF.process_outgroup(vcf_line, self.populations)
        self.assertEqual(het, None)

    def test_diff_homozygotes_outgroups_calling(self):
        vcf_line = self.vcf_line

        vcf_line['out1']['GT'] = '1/1'
        vcf_line['out2']['GT'] = '0/0'

        diff = VCF.process_outgroup(vcf_line, self.populations)
        self.assertEqual(diff, None)

class TestFstatsCalculations(unittest.TestCase):

    def setUp(self):
        self.trivial_allele_counts = {'pop1': {0: 2.0, 1: 8.0, },
                                      'pop2': {0: 8.0, 1: 2.0, }}

        self.normal_allele_counts= {'melpo': {0: 18.0, 1: 2.0, 2: 0.0, 3: 0.0, 4: 0.0},
                                    'cydno': {0: 16.0, 1: 4.0, 2: 0.0, 3: 0.0, 4: 0.0},
                                    'pachi': {0: 4.0, 1: 16.0, 2: 0.0, 3: 0.0, 4: 0.0},
                                    'outgroups': {0: 4.0, 1: 0.0, 2: 0.0, 3: 0.0, 4: 0.0}}

    def test_calc_fstats_trivial_allele_counts(self):
        f_statistics = VCF.calc_fstats(self.trivial_allele_counts)
        self.assertEqual(
            {('pop2', 'pop1'):
                {'G_prime_st_est': 0.6803049722304385,
                'D_est': 0.517460317460318,
                'G_double_prime_st_est': 0.7609710550887027,
                'Gst_est': 0.33747412008281613,
                'Hs_est': 0.3368421052631577,
                'Ht_est': 0.508421052631579}},
            f_statistics)

    def test_calc_fstats_normal_allele_counts(self):
        f_statistics = VCF.calc_fstats(self.normal_allele_counts)
        self.assertEqual(
            {'G_prime_st_est': 0.9140159767610748,
            'D_est': 0.7581699346405228,
            'G_double_prime_st_est': 0.9477124183006534,
            'Gst_est': 0.6444444444444445,
            'Hs_est': 0.1729729729729729,
            'Ht_est': 0.48648648648648646},
            f_statistics[('pachi', 'outgroups')])

    def test_calculate_multilocus_f_statistics(self):

        import numpy as np

        Hs_est_dict = {('pop1','pop2'):[0.2,0.5]}
        Ht_est_dict = {('pop1','pop2'):[np.nan,0.5]}
        print VCF.calc_multilocus_f_statistics(Hs_est_dict, Ht_est_dict)

if __name__ == '__main__':
    unittest.main()

<commit_msg>Cosmetic changes.<commit_after>import VCF
import unittest
from collections import OrderedDict

class TestSlicing(unittest.TestCase):

    def setUp(self):
        self.bgzip_path = "test_data/butterfly.vcf.gz"

    def test_make_slices_default_settings(self):
        """Test slicing function with default settings: 500 bp slices"""

        for count, i in enumerate(VCF.get_slice_indicies(self.bgzip_path, regions=None, window_size=500)):
            if count > 10: break

        self.assertEqual(i, ('Chr01', 5501, 6000))

    def test_make_slices_default_with_params_set(self):
        """Test slicing function with window_size set"""

        for count, i in enumerate(VCF.get_slice_indicies(self.bgzip_path, regions=None, window_size=1008)):
            if count > 10: break

        self.assertEqual(i, ('Chr01', 11089, 12096))

    # def test_make_vcf_slices(self):
    #     print VCF.slice_vcf(self.bgzip_path, 'Chr01', 5501, 6000)

    def test_check_length_of_first_ten_slices(self):
        pass

class TestVCFInfoParsing(unittest.TestCase):

    def setUp(self):
        self.bgzip_path = "test_data/butterfly.vcf.gz"

        self.populations_list = [
            "cydno:c511,c512,c513,c514,c515,c563,c614,c630,c639,c640",
            "outgroups:h665,i02-210",
            "melpo:m523,m524,m525,m589,m675,m676,m682,m683,m687,m689",
            "pachi:p516,p517,p518,p519,p520,p591,p596,p690,p694,p696"]

        self.header_dict = OrderedDict([('CHROM', None), ('POS', None),\
            ('ID', None), ('REF', None), ('ALT', None), ('QUAL', None), \
            ('FILTER', None), ('INFO', None), ('FORMAT', None), ('c511', None), \
            ('c512', None), ('c513', None), ('c514', None), ('c515', None), \
            ('c563', None), ('c614', None), ('c630', None), ('c639', None), \
            ('c640', None), ('h665', None), ('i02-210', None), ('m523', None), \
            ('m524', None), ('m525', None), ('m589', None), ('m675', None), \
            ('m676', None), ('m682', None), ('m683', None), ('m687', None), \
            ('m689', None), ('p516', None), ('p517', None), ('p518', None), \
            ('p519', None), ('p520', None), ('p591', None), ('p596', None), \
            ('p690', None), ('p694', None), ('p696', None)])

        self.allle_counts = {'melpo': {0: 12.0, 1: 0.0, 2: 0.0, 3: 0.0, 4: 0.0},
                            'cydno': {0: 20.0, 1: 0.0, 2: 0.0, 3: 0.0, 4: 0.0},
                            'pachi': {0: 18.0, 1: 0.0, 2: 0.0, 3: 0.0, 4: 0.0},
                            'outgroups': {0: 2.0, 1: 2.0, 2: 0.0, 3: 0.0, 4: 0.0}}

    def test_population_string_parsing(self):
        populations = VCF.parse_populations_list(self.populations_list)

        self.assertEqual(populations, {'melpo': ['m523', 'm524', 'm525',
            'm589', 'm675', 'm676', 'm682', 'm683', 'm687', 'm689'],
            'pachi': ['p516', 'p517', 'p518', 'p519', 'p520', 'p591',
            'p596', 'p690', 'p694', 'p696'], 'cydno': ['c511',
            'c512', 'c513', 'c514', 'c515', 'c563', 'c614', 'c630',
            'c639', 'c640'], 'outgroups': ['h665', 'i02-210']})

    def test_header_to_ordered_dict_parsing(self):
        header = VCF.make_empty_vcf_ordered_dict(self.bgzip_path)

        self.assertEqual(header,self.header_dict)

    def test_header_vs_population_sample_ids(self):
        """Check that the sample IDs parsed from the population arguement
            match those in the VCF file.

            NOTE: In practice the populations arguement can contain fewer
            samples and populations than actually contained in the VCF file.
        """

        header = VCF.make_empty_vcf_ordered_dict(self.bgzip_path, )
        header_sample_ids = [item for count, item in enumerate(header) if count >= 9]

        populations_dict  = VCF.parse_populations_list(self.populations_list)
        populations_sample_ids = [i for l in populations_dict.values() for i in l]

        # Check both unique IDs and equal length
        self.assertEqual(set(header_sample_ids), set(populations_sample_ids))
        self.assertEqual(len(header_sample_ids), len(populations_sample_ids))

class TestGenoTypeParsing(unittest.TestCase):

    def setUp(self):
        self.vcf_line = OrderedDict([
                        ('CHROM', 'Chr01'),
                        ('POS', 4984),
                        ('ID', '.'),
                        ('REF', 'C'),
                        ('ALT', 'T'),
                        ('QUAL', 235.91),
                        ('FILTER', 'PASS'),
                        ('INFO', {'AC': '1'}),
                        ('FORMAT', 'GT:AD:DP:GQ:PL'),
                        ('out1', {'GT': '0/0', 'GQ': 51.17, 'AD': (17, 0), 'DP': 17, 'PL': (0, 51, 669)}),
                        ('out2', {'GT': '0/0', 'GQ': 45.15, 'AD': (16, 0), 'DP': 16, 'PL': (0, 45, 600)})])

        self.populations = {'Outgroup': ['out1', 'out2']}

    #### WITH AMBIGUITY CODES ####
    def test_homo_ref_genotype_calling(self):
        homo_ref = VCF.process_snp_call('0/0:10,9:19:99:254,0,337', 'A', 'T', IUPAC_ambiguities=True)
        self.assertEqual(homo_ref, 'A')

    def test_heterozygote_genotype_calling(self):
        heterozygote = VCF.process_snp_call('0/1:10,9:19:99:254,0,337', 'A', 'T', IUPAC_ambiguities=True)
        self.assertEqual(heterozygote, 'W')

    def test_homo_alt_genotype_calling(self):
        homo_alt =  VCF.process_snp_call('1/1:10,9:19:99:254,0,337', 'A', 'T', IUPAC_ambiguities=True)
        self.assertEqual(homo_alt, 'T')

    def test_second_alt_genotype_calling(self):
        second_alt = VCF.process_snp_call('0/2:10,9:19:99:254,0,337', 'A', 'T,G', IUPAC_ambiguities=True)
        self.assertEqual(second_alt, 'R')

    def test_double_alt_genotype_calling(self):
        double_alt = VCF.process_snp_call('1/2:10,9:19:99:254,0,337', 'A', 'T,G', IUPAC_ambiguities=True)
        self.assertEqual(double_alt, 'K')

    #### WITHOUT AMBIGUITY CODES ####
    def test_heterozygote_as_N_genotype_calling(self):
        heterozygote_as_N = VCF.process_snp_call('0/1:10,9:19:99:254,0,337', 'A', 'T', IUPAC_ambiguities=False)
        self.assertEqual(heterozygote_as_N, 'N')

    def test_double_alt_het_as_N_genotype_calling(self):
        double_alt_het_as_N = VCF.process_snp_call('1/2:10,9:19:99:254,0,337', 'A', 'T,G', IUPAC_ambiguities=False)
        self.assertEqual(double_alt_het_as_N, 'N')

    #### TEST OUTGROUP CALLING ####
    def test_home_ref_outgroup_calling(self):
        homo_ref = VCF.process_outgroup(self.vcf_line, self.populations)
        self.assertEqual(homo_ref, '0')

    def test_home_alt_outgroup_calling(self):
        vcf_line = self.vcf_line

        vcf_line['out1']['GT'] = '1/1'
        vcf_line['out2']['GT'] = '1/1'

        homo_alt = VCF.process_outgroup(vcf_line, self.populations)
        self.assertEqual(homo_alt, '1')

    def test_heterozyogote_outgroup_calling(self):
        vcf_line = self.vcf_line

        vcf_line['out1']['GT'] = '1/1'
        vcf_line['out2']['GT'] = '1/0'

        het = VCF.process_outgroup(vcf_line, self.populations)
        self.assertEqual(het, None)

    def test_diff_homozygotes_outgroups_calling(self):
        vcf_line = self.vcf_line

        vcf_line['out1']['GT'] = '1/1'
        vcf_line['out2']['GT'] = '0/0'

        diff = VCF.process_outgroup(vcf_line, self.populations)
        self.assertEqual(diff, None)

class TestFstatsCalculations(unittest.TestCase):

    def setUp(self):
        self.trivial_allele_counts = {'pop1': {0: 2.0, 1: 8.0, },
                                      'pop2': {0: 8.0, 1: 2.0, }}

        self.normal_allele_counts= {'melpo': {0: 18.0, 1: 2.0, 2: 0.0, 3: 0.0, 4: 0.0},
                                    'cydno': {0: 16.0, 1: 4.0, 2: 0.0, 3: 0.0, 4: 0.0},
                                    'pachi': {0: 4.0, 1: 16.0, 2: 0.0, 3: 0.0, 4: 0.0},
                                    'outgroups': {0: 4.0, 1: 0.0, 2: 0.0, 3: 0.0, 4: 0.0}}

    def test_calc_fstats_trivial_allele_counts(self):
        f_statistics = VCF.calc_fstats(self.trivial_allele_counts)
        self.assertEqual(
            {('pop2', 'pop1'):
                {'G_prime_st_est': 0.6803049722304385,
                'D_est': 0.517460317460318,
                'G_double_prime_st_est': 0.7609710550887027,
                'Gst_est': 0.33747412008281613,
                'Hs_est': 0.3368421052631577,
                'Ht_est': 0.508421052631579}},
            f_statistics)

    def test_calc_fstats_normal_allele_counts(self):
        f_statistics = VCF.calc_fstats(self.normal_allele_counts)
        self.assertEqual(
            {'G_prime_st_est': 0.9140159767610748,
            'D_est': 0.7581699346405228,
            'G_double_prime_st_est': 0.9477124183006534,
            'Gst_est': 0.6444444444444445,
            'Hs_est': 0.1729729729729729,
            'Ht_est': 0.48648648648648646},
            f_statistics[('pachi', 'outgroups')])

if __name__ == '__main__':
    unittest.main()

<|endoftext|>


<commit_before>"""
Django settings for aretheyok project.

Generated by 'django-admin startproject' using Django 1.9.6.

For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""

import os

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'lm%a6@r92=xa%*3_t%&zye*wvej=d9bymryapvynr!ccyj_7pm'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = [
	'stream_django',
	'feed.apps.FeedConfig',
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
]

STREAM_API_KEY = 'pjtmz6uavyrh'
STREAM_API_SECRET = ''
# no secret key for you
MIDDLEWARE_CLASSES = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'aretheyok.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'aretheyok.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.9/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.9/howto/static-files/

STATIC_URL = '/static/'
<commit_msg>No more secret keys, now to bury these commits<commit_after>"""
Django settings for aretheyok project.

Generated by 'django-admin startproject' using Django 1.9.6.

For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""

import os

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'no'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = [
	'stream_django',
	'feed.apps.FeedConfig',
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
]

STREAM_API_KEY = 'pjtmz6uavyrh'
STREAM_API_SECRET = 'no'

MIDDLEWARE_CLASSES = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'aretheyok.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'aretheyok.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.9/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.9/howto/static-files/

STATIC_URL = '/static/'
<|endoftext|>


<commit_before>from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import classification_report,confusion_matrix, accuracy_score, precision_score, f1_score, recall_score
import pandas
from pandas import DataFrame
import pandas.io.sql as psql
import KnowledgeBase
import random

class NeuralNetworkController:

	#Creates and processes a neural_network with defined architecture, or a random architecture
	#Preconditions:
	# * X - attributes as retrieved from the DATA table
	# * Y - classes as retrieved from the the DATA table
	# * layers - architecture, may be none
	#Postconditions: returns performance from the neural network
	#NOTE: Code from kdnuggets
	def __init__(self,layers=None):
		self.algorithm_name = "Neural Network"
		self.algorithm_id = "ANN"
		random.seed()
		for i in range(1,10):
			self.algorithm_id = self.algorithm_id + str(random.randint(1,9))
		self.algorithm_id = self.algorithm_id + "( " + str(layers).strip('[]') + ")"

		#initialize remaining instance variables
		self.X_train = []
		self.X_test = []
		self.y_train = []
		self.y_test = []
		self.accuracy = None
		self.precision = None
		self.recall = None
		self.f1 = None
		self.cm = None
		self.layerslist = layers

	def createModel(self, x, y):
		print "X length " + str(len(x))
		print "Y length " + str(len(y))

		self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(x,y)

		scaler = StandardScaler()
		scaler.fit(self.X_train)
		self.X_train = scaler.transform(self.X_train)
		self.X_test = scaler.transform(self.X_test)

		if self.layerslist is None:
			random.seed()
			num_layers = random.randint(1,25)
			self.layerslist = []
			for i in range(0,num_layers):
				self.layerslist.append(random.randint(1,100))

		self.mlp = MLPClassifier(hidden_layer_sizes=self.layerslist)
		self.mlp.fit(self.X_train, self.y_train)

	def runModel(self):
		predictions = self.mlp.predict(self.X_test)

		print(confusion_matrix(self.y_test,predictions))
		print(classification_report(self.y_test,predictions))

		self.accuracy = accuracy_score(self.y_test,predictions)
		self.precision = precision_score(self.y_test,predictions)
		self.recall = recall_score(self.y_test, predictions)
		self.f1 = f1_score(self.y_test,predictions)
		self.cm = confusion_matrix(self.y_test,predictions)

		to_return =  (self.algorithm_id, self.algorithm_name, self.accuracy, self.precision, self.recall, self.f1, self.cm)
		return to_return
		<commit_msg>More bug fixes<commit_after>from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import classification_report,confusion_matrix, accuracy_score, precision_score, f1_score, recall_score
import pandas
from pandas import DataFrame
import pandas.io.sql as psql
import KnowledgeBase
import random

class NeuralNetworkController:

	#Creates and processes a neural_network with defined architecture, or a random architecture
	#Preconditions:
	# * X - attributes as retrieved from the DATA table
	# * Y - classes as retrieved from the the DATA table
	# * layers - architecture, may be none
	#Postconditions: returns performance from the neural network
	#NOTE: Code from kdnuggets
	def __init__(self,layers=None):
		self.algorithm_name = "Neural Network"
		self.algorithm_id = "ANN"
		random.seed()
		for i in range(1,10):
			self.algorithm_id = self.algorithm_id + str(random.randint(1,9))
		self.algorithm_id = self.algorithm_id + "( " + str(layers).strip('[]') + ")"

		#initialize remaining instance variables
		self.X_train = []
		self.X_test = []
		self.y_train = []
		self.y_test = []
		self.accuracy = None
		self.precision = None
		self.recall = None
		self.f1 = None
		self.cm = None
		self.layerslist = layers

	def createModel(self, x, y):
		print "X length " + str(len(x))
		print "Y length " + str(len(y))

		self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(x,y)

		scaler = StandardScaler()
		scaler.fit(self.X_train)
		self.X_train = scaler.transform(self.X_train)
		self.X_test = scaler.transform(self.X_test)

		if self.layerslist is None:
			random.seed()
			num_layers = random.randint(1,25)
			self.layerslist = []
			for i in range(0,num_layers):
				self.layerslist.append(random.randint(1,100))

		self.algorithm_id = self.algorithm_id + "( " + str(layers).strip('[]') + ")"
		self.mlp = MLPClassifier(hidden_layer_sizes=self.layerslist)
		self.mlp.fit(self.X_train, self.y_train)

	def runModel(self):
		predictions = self.mlp.predict(self.X_test)

		print(confusion_matrix(self.y_test,predictions))
		print(classification_report(self.y_test,predictions))

		self.accuracy = accuracy_score(self.y_test,predictions)
		self.precision = precision_score(self.y_test,predictions)
		self.recall = recall_score(self.y_test, predictions)
		self.f1 = f1_score(self.y_test,predictions)
		self.cm = confusion_matrix(self.y_test,predictions)

		to_return =  (self.algorithm_id, self.algorithm_name, self.accuracy, self.precision, self.recall, self.f1, self.cm)
		return to_return
		<|endoftext|>


<commit_before>"""
 Monthly download of the MERRA data, website suggests that the previous
 month is available around the 2-3 week of the next month, so this is run
 from RUN_MIDNIGHT.sh, but only on the 21rst of each month
"""
import datetime
import urllib2
import os
import sys
import subprocess

def trans(now):
    """ Hacky hack hack """
    if now.year < 1993:
        return '101'
    if now.year < 2001:
        return '201'
    return '300'

def do_month(sts):
    """ Run for a given month """

    ets = sts + datetime.timedelta(days=35)
    ets = ets.replace(day=1)

    interval = datetime.timedelta(days=1)
    now = sts
    while now < ets:
        uri = now.strftime(
            ("http://goldsmr2.sci.gsfc.nasa.gov/daac-bin/OTF/"
             "HTTP_services.cgi?FILENAME=%%2Fdata%%2Fs4pa%%2FMERRA%%2"
             "FMAT1NXRAD.5.2.0%%2F%Y%%2F%m%%2FMERRA"+trans(now)+".prod.assim."
             "tavg1_2d_rad_Nx.%Y%m%d.hdf&FORMAT=TmV0Q0RGL2d6aXAv&"
             "BBOX=22.852,-129.727,52.383,-60.117&LABEL=MERRA" +
             trans(now) + ".prod.assim.tavg1_2d_rad_Nx.%Y%m%d.SUB.nc.gz&"
             "FLAGS=&SHORTNAME=MAT1NXRAD&SERVICE=SUBSET_LATS4D&LAYERS=&"
             "VERSION=1.02&VARIABLES=swtdn,swgdn,swgdnclr,tauhgh,taulow,"
             "taumid,tautot,cldhgh,cldlow,cldmid,cldtot"))

        try:
            req = urllib2.Request(uri)
            data = urllib2.urlopen(req).read()
        except:
            uri = uri.replace("MERRA300", "MERRA301")
            req = urllib2.Request(uri)
            data = urllib2.urlopen(req).read()
        dirname = now.strftime("/mesonet/merra/%Y")
        if not os.path.isdir(dirname):
            os.makedirs(dirname)
        fp = open(now.strftime("/mesonet/merra/%Y/%Y%m%d.nc.gz"), 'w')
        fp.write(data)
        fp.close()
        now += interval

    os.chdir(dirname)
    subprocess.call("gunzip *.gz", shell=True)

def main():
    """ Run for last month month """
    now = datetime.datetime.now()
    if len(sys.argv) == 3:
        now = datetime.datetime(int(sys.argv[1]), int(sys.argv[2]), 1)
    else:
        now = now - datetime.timedelta(days=35)
        now = now.replace(day=1)
    do_month(now)

if __name__ == '__main__':
    main()
<commit_msg>MERRA dl script should clean up after itself<commit_after>"""
 Monthly download of the MERRA data, website suggests that the previous
 month is available around the 2-3 week of the next month, so this is run
 from RUN_MIDNIGHT.sh, but only on the 21rst of each month
"""
import datetime
import urllib2
import os
import sys
import subprocess
import glob

def trans(now):
    """ Hacky hack hack """
    if now.year < 1993:
        return '101'
    if now.year < 2001:
        return '201'
    return '300'

def do_month(sts):
    """ Run for a given month """

    ets = sts + datetime.timedelta(days=35)
    ets = ets.replace(day=1)

    interval = datetime.timedelta(days=1)
    now = sts
    while now < ets:
        uri = now.strftime(
            ("http://goldsmr2.sci.gsfc.nasa.gov/daac-bin/OTF/"
             "HTTP_services.cgi?FILENAME=%%2Fdata%%2Fs4pa%%2FMERRA%%2"
             "FMAT1NXRAD.5.2.0%%2F%Y%%2F%m%%2FMERRA"+trans(now)+".prod.assim."
             "tavg1_2d_rad_Nx.%Y%m%d.hdf&FORMAT=TmV0Q0RGL2d6aXAv&"
             "BBOX=22.852,-129.727,52.383,-60.117&LABEL=MERRA" +
             trans(now) + ".prod.assim.tavg1_2d_rad_Nx.%Y%m%d.SUB.nc.gz&"
             "FLAGS=&SHORTNAME=MAT1NXRAD&SERVICE=SUBSET_LATS4D&LAYERS=&"
             "VERSION=1.02&VARIABLES=swtdn,swgdn,swgdnclr,tauhgh,taulow,"
             "taumid,tautot,cldhgh,cldlow,cldmid,cldtot"))

        try:
            req = urllib2.Request(uri)
            data = urllib2.urlopen(req).read()
        except:
            uri = uri.replace("MERRA300", "MERRA301")
            req = urllib2.Request(uri)
            data = urllib2.urlopen(req).read()
        dirname = now.strftime("/mesonet/merra/%Y")
        if not os.path.isdir(dirname):
            os.makedirs(dirname)
        fp = open(now.strftime("/mesonet/merra/%Y/%Y%m%d.nc.gz"), 'w')
        fp.write(data)
        fp.close()
        now += interval

    os.chdir(dirname)
    for fn in glob.glob("*.gz"):
        fn2 = fn[:-3]
        if os.path.isfile(fn2):
            os.unlink(fn2)
        subprocess.call("gunzip %s" % (fn, ), shell=True)
        os.unlink(fn)

def main():
    """ Run for last month month """
    now = datetime.datetime.now()
    if len(sys.argv) == 3:
        now = datetime.datetime(int(sys.argv[1]), int(sys.argv[2]), 1)
    else:
        now = now - datetime.timedelta(days=35)
        now = now.replace(day=1)
    do_month(now)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#! /usr/bin/python
from __future__ import absolute_import as _absolute_import
from __future__ import print_function

'''
Copyright 2015 Tim Nonner

Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
'''

"""
a simple interface to pulp which is also an interface blueprint for potential other mip-interfaces
"""

import pulp as pl
import time

class MIP(object):
	"""
	Interface to pulp mip solver
	"""

	def __init__(self,name,kind='Minimize'):
		kinds = {'Minimize':pl.LpMinimize, 'Maximize':pl.LpMaximize}
		self.mip = pl.LpProblem(name, kinds[kind])

	def var(self,name,low=0,up=0,cat='Binary'):
		return pl.LpVariable(name, low, up, cat=cat)

	def con(self,affine,sense=0,rhs=0):
		con = pl.LpConstraint(pl.LpAffineExpression(affine),sense=sense,rhs=rhs)
		self.mip += con
		return con

	def obj(self,affine):
		self.mip += pl.LpAffineExpression(affine)

	def solve(self,msg=0,**kwarg):
		kind = 'CBC'
		if 'kind' in kwarg:
			kind = kwarg['kind']
		time_limit = None
		if 'time_limit' in kwarg:
			time_limit = kwarg['time_limit']
		random_seed = None
		if 'random_seed' in kwarg:
			random_seed = kwarg['random_seed']
		ratio_gap = None
		if 'ratio_gap' in kwarg:
			ratio_gap = kwarg['ratio_gap']
		start_time = time.time()
		# select solver for pl
		if kind == 'CPLEX':
			if time_limit is not None:
				# pulp does currently not support a timelimit in 1.5.9
				self.mip.solve(pl.CPLEX_CMD(msg=msg, timelimit=time_limit))
			else:
				self.mip.solve(pl.CPLEX_CMD(msg=msg))
		elif kind == 'GLPK':
			self.mip.solve(pl.GLPK_CMD(msg=msg))
		elif kind == 'CBC':
			options = []
			if time_limit is not None:
				options.extend(['sec', str(time_limit)])
			if random_seed is not None:
				options.extend(['randomSeed', str(random_seed)])
				options.extend(['randomCbcSeed', str(random_seed)])
			if ratio_gap is not None:
				options.extend(['ratio', str(ratio_gap)])
			self.mip.solve(pl.PULP_CBC_CMD(msg=msg, options=options))
		else:
			raise Exception('ERROR: solver ' + kind + ' not known')

		if msg:
			print('INFO: execution time for solving mip (sec) = ' + str(time.time() - start_time))
		if self.mip.status == 1 and msg:
			print('INFO: objective = ' + str(pl.value(self.mip.objective)))

	def status(self):
		return self.mip.status

	def value(self,var):
		return var.varValue
<commit_msg>spacing<commit_after>#! /usr/bin/python
from __future__ import absolute_import as _absolute_import
from __future__ import print_function

'''
Copyright 2015 Tim Nonner

Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
'''

"""
a simple interface to pulp which is also an interface blueprint for potential other mip-interfaces
"""

import pulp as pl
import time

from pyschedule.solvers.pulp_scip import SCIP_CMD

class MIP(object):
    """
    Interface to pulp mip solver
    """

    def __init__(self,name,kind='Minimize'):
        kinds = {'Minimize':pl.LpMinimize, 'Maximize':pl.LpMaximize}
        self.mip = pl.LpProblem(name, kinds[kind])

    def var(self,name,low=0,up=0,cat='Binary'):
        return pl.LpVariable(name, low, up, cat=cat)

    def con(self,affine,sense=0,rhs=0):
        con = pl.LpConstraint(pl.LpAffineExpression(affine),sense=sense,rhs=rhs)
        self.mip += con
        return con

    def obj(self,affine):
        self.mip += pl.LpAffineExpression(affine)

    def solve(self,msg=0,**kwarg):
        kind = 'CBC'
        if 'kind' in kwarg:
            kind = kwarg['kind']
        time_limit = None
        if 'time_limit' in kwarg:
            time_limit = kwarg['time_limit']
        random_seed = None
        if 'random_seed' in kwarg:
            random_seed = kwarg['random_seed']
        ratio_gap = None
        if 'ratio_gap' in kwarg:
            ratio_gap = kwarg['ratio_gap']
        start_time = time.time()
        # select solver for pl
        if kind == 'CPLEX':
            if time_limit is not None:
                # pulp does currently not support a timelimit in 1.5.9
                self.mip.solve(pl.CPLEX_CMD(msg=msg, timelimit=time_limit))
            else:
                self.mip.solve(pl.CPLEX_CMD(msg=msg))
        elif kind == 'GLPK':
            self.mip.solve(pl.GLPK_CMD(msg=msg))
        elif kind == 'SCIP':
                self.mip.solve(SCIP_CMD(msg=msg))
        elif kind == 'CBC':
            options = []
            if time_limit is not None:
                options.extend(['sec', str(time_limit)])
            if random_seed is not None:
                options.extend(['randomSeed', str(random_seed)])
                options.extend(['randomCbcSeed', str(random_seed)])
            if ratio_gap is not None:
                options.extend(['ratio', str(ratio_gap)])
            self.mip.solve(pl.PULP_CBC_CMD(msg=msg, options=options))
        else:
            raise Exception('ERROR: solver ' + kind + ' not known')

        if msg:
            print('INFO: execution time for solving mip (sec) = ' + str(time.time() - start_time))
        if self.mip.status == 1 and msg:
            print('INFO: objective = ' + str(pl.value(self.mip.objective)))

    def status(self):
        return self.mip.status

    def value(self,var):
        return var.varValue
<|endoftext|>


<commit_before>from flask import Flask, request, jsonify, abort, json, make_response
import re
import time
from sqltools import *
from math import ceil
app = Flask(__name__)
app.debug = True

def fixDecimal(value):
    try:
      return str(ceil(float(value)*(1e8))/1e8)
    except Exception as e:
      print "couldn't convert ",value,"got error: ",e

#@app.route('/book')
def getOrderbook(lasttrade=0):
    #use for websocket to load/broadcast updated book
    book={}
    trade=0
    updated=False

    #find last know DEx2.0 trade and see if it's newer than what we have
    trades=dbSelect("select max(txdbserialnum) from transactions where txtype >24 and txtype<29 and txstate='valid'")
    if len(trades) > 0 and len(trades[0]) > 0:
      trade=int(trades[0][0])

    if (trade > lasttrade):
      AO=dbSelect("select distinct propertyiddesired, propertyidselling from activeoffers "
                  "where offerstate='active' order by propertyiddesired")
      if len(AO) > 0:
        for pair in AO:
          pd=int(pair[0])
          ps=int(pair[1])
          data = get_orders_by_market(pd,ps)
          data2 = get_orders_by_market(ps,pd)
          try:
            book[pd][ps]=data
          except KeyError:
            book[pd]={ps: data}
          try:
            book[ps][pd]=data2
          except KeyError:
            book[ps]={pd: data2}
        updated=True

    ret={"updated":updated ,"book":book, "lasttrade":trade}
    return ret

@app.route('/designatingcurrencies', methods=['POST'])
def getDesignatingCurrencies():
    try:
        value = int(re.sub(r'\D+', '', request.form['ecosystem']))
        valid_values = [1,2]
        if value not in valid_values:
            abort(make_response('Field \'ecosystem\' invalid value, request failed', 400))

        ecosystem = "Production" if value == 1 else "Test"
    except KeyError:
        abort(make_response('No field \'ecosystem\' in request, request failed', 400))
    except ValueError:
        abort(make_response('Field \'ecosystem\' invalid value, request failed', 400))

    #designating_currencies = dbSelect("select distinct ao.propertyiddesired as propertyid, sp.propertyname from activeoffers ao "
    #                                  "inner join SmartProperties sp on ao.propertyiddesired = sp.propertyid and sp.ecosystem = %s "
                                      #"where (ao.propertyidselling not in (1, 2, 31)) or (ao.propertyidselling = 1 and ao.propertyiddesired = 31) "
    #                                  "where ao.offerstate='active' "
    #                                  "order by ao.propertyiddesired ",[ecosystem])
    designating_currencies = dbSelect("select distinct propertyiddesired,desiredname from markets where supply > 0 and "
                                      "CASE WHEN %s='Production' THEN "
                                      "propertyiddesired > 0 and propertyiddesired < 2147483648 and propertyiddesired !=2 "
                                      "ELSE propertyiddesired > 2147483650 or propertyiddesired=2 END "
                                      "order by propertyiddesired",[ecosystem])
    return jsonify({"status" : 200, "currencies": [
	{
	 "propertyid":currency[0], "propertyname" : currency[1], "displayname" : str(currency[1])+" #"+str(currency[0])
	} for currency in designating_currencies]})

@app.route('/<int:denominator>')
def get_markets_by_denominator(denominator):
    markets = dbSelect("select propertyidselling as marketid, sellingname as marketname, unitprice, supply, lastprice, marketpropertytype "
                         "from markets where propertyiddesired=%s and ( supply>0 or propertyidselling in "
                           "(select propertyiddesired as marketid from markets where propertyidselling=%s and supply>0) "
                         ") order by propertyidselling",(denominator,denominator))
    return jsonify({"status" : 200, "markets": [
	{
	 "propertyid":currency[0],
	 "propertyname" : currency[1],
	 "price" : float(currency[2]),
	 "supply" : currency[3],
	 "change" : float(currency[2]-currency[4]),
         "propertytype" : currency[5]
	} for currency in markets]})

@app.route('/ohlcv/<int:propertyid_desired>/<int:propertyid_selling>')
def get_OHLCV(propertyid_desired, propertyid_selling):
    orderbook = dbSelect("SELECT timeframe.date,FIRST(offers.unitprice) ,MAX(offers.unitprice), MIN(offers.unitprice), "
                         "LAST(offers.unitprice), SUM(offers.totalselling) FROM generate_series('2016-01-01 00:00'::timestamp,current_date, '1 day') "
                         "timeframe(date) INNER JOIN (SELECT ao.totalselling, ao.unitprice, createtx.TXRecvTime as createdate, "
                         "COALESCE(lasttx.TXRecvTime,createtx.TXRecvTime) as solddate from ActiveOffers ao inner join Transactions createtx "
                         "on ao.CreateTXDBSerialNum = createtx.TxDBSerialNum left outer join Transactions lasttx on ao.LastTXDBSerialNum = lasttx.TxDBSerialNum "
                         "where (ao.OfferState = 'sold' or ao.OfferState = 'active')  and ao.unitprice > 0 and ao.PropertyIdSelling = %s and "
                         "ao.PropertyIdDesired = %s ORDER BY createtx.TXRecvTime DESC) offers on DATE(offers.createdate) <= timeframe.date and "
                         "DATE(offers.solddate) >= timeframe.date group by timeframe.date",[propertyid_selling, propertyid_desired])
    return jsonify({"status" : 200, "orderbook": [
        {
            "date":int((time.mktime(order[0].timetuple()) + order[0].microsecond/1000000.0)/86400),
            "open":order[1], #if order[1] is not None else 160 - (0.01 * orderbook.index(order)),
            "high" : order[2], #if order[2] is not None else 160 + (0.01 * orderbook.index(order)),
            "low" : order[3], #if order[3] is not None else 160 - (0.01 * orderbook.index(order)),
            "close" : order[4], #if order[4] is not None else 160 + (0.01 * orderbook.index(order)),
            "volume": order[5], #if order[5] is not None else 34.5 + (11.2 * orderbook.index(order)),
            "adjustment":(order[2] + order[3]) /2
        } for order in orderbook]})

@app.route('/<int:propertyid_desired>/<int:propertyid_selling>')
def get_orders_by_market_json(propertyid_desired, propertyid_selling):
    return jsonify(get_orders_by_market(propertyid_desired, propertyid_selling))

def get_orders_by_market(propertyid_desired, propertyid_selling):
    orderbook = dbSelect("SELECT ao.propertyiddesired, ao.propertyidselling, ao.AmountAvailable, ao.AmountDesired, ao.TotalSelling, ao.AmountAccepted, "
                         "cast(txj.txdata->>'unitprice' as numeric), ao.Seller, tx.TxRecvTime, 'active', tx.txhash from activeoffers ao, transactions tx, txjson txj "
                         "where ao.CreateTxDBSerialNum = txj.TxDBSerialNum and ao.CreateTxDBSerialNum = tx.TxDBSerialNum and ao.propertyiddesired = %s and "
                         "ao.propertyidselling = %s and ao.OfferState = 'active' union all select cast(txj.txdata->>'propertyiddesired' as bigint), "
                         "cast(txj.txdata->>'propertyidforsale' as bigint),CASE WHEN txj.txdata->>'propertyidforsaleisdivisible' = 'true' THEN "
                         "round(cast(txj.txdata->>'amountforsale' as numeric) * 100000000) ELSE cast(txj.txdata->>'amountforsale' as numeric) END, "
                         "CASE WHEN txj.txdata->>'propertyiddesiredisdivisible' = 'true' THEN round(cast(txj.txdata->>'amountdesired' as numeric) * 100000000) "
                         "ELSE cast(txj.txdata->>'amountdesired' as numeric) END,CASE WHEN txj.txdata->>'propertyidforsaleisdivisible' = 'true' THEN "
                         "round(cast(txj.txdata->>'amountforsale' as numeric) * 100000000) ELSE cast(txj.txdata->>'amountforsale' as numeric) END,0, "
                         "cast(txj.txdata->>'unitprice' as numeric),txj.txdata->>'sendingaddress', tx.TxRecvTime, 'pending' from transactions tx inner join txjson txj "
                         "on tx.txdbserialnum = txj.txdbserialnum where tx.txdbserialnum < 0 and tx.txtype = 25 and cast(txj.txdata->>'propertyidforsale' as numeric) = %s "
                         "and cast(txj.txdata->>'propertyiddesired' as numeric) = %s",[propertyid_desired,propertyid_selling,propertyid_selling,propertyid_desired])

    cancels = dbSelect("SELECT cast(txj.txdata->>'propertyiddesired' as bigint),cast(txj.txdata->>'propertyidforsale' as bigint),CASE WHEN "
                       "txj.txdata->>'propertyiddesiredisdivisible' = 'true' THEN round(cast(txj.txdata->>'amountdesired' as numeric) * 100000000) "
                       "ELSE cast(txj.txdata->>'amountdesired' as numeric) END,CASE WHEN txj.txdata->>'propertyidforsaleisdivisible' = 'true' THEN "
                       "round(cast(txj.txdata->>'amountforsale' as numeric) * 100000000) ELSE cast(txj.txdata->>'amountforsale' as numeric) END, "
                       "cast(txj.txdata->>'unitprice' as numeric),txj.txdata->>'sendingaddress', tx.TxRecvTime, 'pending', tx.txhash from transactions tx "
                       "inner join txjson txj on tx.txdbserialnum = txj.txdbserialnum where tx.txdbserialnum < 0 and tx.txtype = 26 and "
                       "cast(txj.txdata->>'propertyidforsale' as numeric) = %s and cast(txj.txdata->>'propertyiddesired' as numeric) = %s",
                       [propertyid_selling,propertyid_desired])

    return {"status" : 200, "orderbook": [
        {
            "propertyid_desired":order[0],
            "propertyid_selling":order[1],
            "available_amount" : str(order[2]),
            "desired_amount" : str(order[3]),
            "total_amount" : str(order[4]),
            "accepted_amount": str(order[5]),
            "unit_price" : fixDecimal(order[6]),
            "seller" : str(order[7]),
            "time" : str(order[8]),
            "status" : order[9],
            "txhash" : str(order[10])
        } for order in orderbook], "cancels":[
        {
            "propertyid_desired":cancel[0],
            "propertyid_selling":cancel[1],
            "desired_amount" : str(cancel[2]),
            "total_amount" : str(cancel[3]),
            "unit_price" : str(cancel[4]),
            "seller" : str(cancel[5]),
            "time" : str(cancel[6])
            "txhash" : str(order[7])
        } for cancel in cancels]}
<commit_msg>syntax typo<commit_after>from flask import Flask, request, jsonify, abort, json, make_response
import re
import time
from sqltools import *
from math import ceil
app = Flask(__name__)
app.debug = True

def fixDecimal(value):
    try:
      return str(ceil(float(value)*(1e8))/1e8)
    except Exception as e:
      print "couldn't convert ",value,"got error: ",e

#@app.route('/book')
def getOrderbook(lasttrade=0):
    #use for websocket to load/broadcast updated book
    book={}
    trade=0
    updated=False

    #find last know DEx2.0 trade and see if it's newer than what we have
    trades=dbSelect("select max(txdbserialnum) from transactions where txtype >24 and txtype<29 and txstate='valid'")
    if len(trades) > 0 and len(trades[0]) > 0:
      trade=int(trades[0][0])

    if (trade > lasttrade):
      AO=dbSelect("select distinct propertyiddesired, propertyidselling from activeoffers "
                  "where offerstate='active' order by propertyiddesired")
      if len(AO) > 0:
        for pair in AO:
          pd=int(pair[0])
          ps=int(pair[1])
          data = get_orders_by_market(pd,ps)
          data2 = get_orders_by_market(ps,pd)
          try:
            book[pd][ps]=data
          except KeyError:
            book[pd]={ps: data}
          try:
            book[ps][pd]=data2
          except KeyError:
            book[ps]={pd: data2}
        updated=True

    ret={"updated":updated ,"book":book, "lasttrade":trade}
    return ret

@app.route('/designatingcurrencies', methods=['POST'])
def getDesignatingCurrencies():
    try:
        value = int(re.sub(r'\D+', '', request.form['ecosystem']))
        valid_values = [1,2]
        if value not in valid_values:
            abort(make_response('Field \'ecosystem\' invalid value, request failed', 400))

        ecosystem = "Production" if value == 1 else "Test"
    except KeyError:
        abort(make_response('No field \'ecosystem\' in request, request failed', 400))
    except ValueError:
        abort(make_response('Field \'ecosystem\' invalid value, request failed', 400))

    #designating_currencies = dbSelect("select distinct ao.propertyiddesired as propertyid, sp.propertyname from activeoffers ao "
    #                                  "inner join SmartProperties sp on ao.propertyiddesired = sp.propertyid and sp.ecosystem = %s "
                                      #"where (ao.propertyidselling not in (1, 2, 31)) or (ao.propertyidselling = 1 and ao.propertyiddesired = 31) "
    #                                  "where ao.offerstate='active' "
    #                                  "order by ao.propertyiddesired ",[ecosystem])
    designating_currencies = dbSelect("select distinct propertyiddesired,desiredname from markets where supply > 0 and "
                                      "CASE WHEN %s='Production' THEN "
                                      "propertyiddesired > 0 and propertyiddesired < 2147483648 and propertyiddesired !=2 "
                                      "ELSE propertyiddesired > 2147483650 or propertyiddesired=2 END "
                                      "order by propertyiddesired",[ecosystem])
    return jsonify({"status" : 200, "currencies": [
	{
	 "propertyid":currency[0], "propertyname" : currency[1], "displayname" : str(currency[1])+" #"+str(currency[0])
	} for currency in designating_currencies]})

@app.route('/<int:denominator>')
def get_markets_by_denominator(denominator):
    markets = dbSelect("select propertyidselling as marketid, sellingname as marketname, unitprice, supply, lastprice, marketpropertytype "
                         "from markets where propertyiddesired=%s and ( supply>0 or propertyidselling in "
                           "(select propertyiddesired as marketid from markets where propertyidselling=%s and supply>0) "
                         ") order by propertyidselling",(denominator,denominator))
    return jsonify({"status" : 200, "markets": [
	{
	 "propertyid":currency[0],
	 "propertyname" : currency[1],
	 "price" : float(currency[2]),
	 "supply" : currency[3],
	 "change" : float(currency[2]-currency[4]),
         "propertytype" : currency[5]
	} for currency in markets]})

@app.route('/ohlcv/<int:propertyid_desired>/<int:propertyid_selling>')
def get_OHLCV(propertyid_desired, propertyid_selling):
    orderbook = dbSelect("SELECT timeframe.date,FIRST(offers.unitprice) ,MAX(offers.unitprice), MIN(offers.unitprice), "
                         "LAST(offers.unitprice), SUM(offers.totalselling) FROM generate_series('2016-01-01 00:00'::timestamp,current_date, '1 day') "
                         "timeframe(date) INNER JOIN (SELECT ao.totalselling, ao.unitprice, createtx.TXRecvTime as createdate, "
                         "COALESCE(lasttx.TXRecvTime,createtx.TXRecvTime) as solddate from ActiveOffers ao inner join Transactions createtx "
                         "on ao.CreateTXDBSerialNum = createtx.TxDBSerialNum left outer join Transactions lasttx on ao.LastTXDBSerialNum = lasttx.TxDBSerialNum "
                         "where (ao.OfferState = 'sold' or ao.OfferState = 'active')  and ao.unitprice > 0 and ao.PropertyIdSelling = %s and "
                         "ao.PropertyIdDesired = %s ORDER BY createtx.TXRecvTime DESC) offers on DATE(offers.createdate) <= timeframe.date and "
                         "DATE(offers.solddate) >= timeframe.date group by timeframe.date",[propertyid_selling, propertyid_desired])
    return jsonify({"status" : 200, "orderbook": [
        {
            "date":int((time.mktime(order[0].timetuple()) + order[0].microsecond/1000000.0)/86400),
            "open":order[1], #if order[1] is not None else 160 - (0.01 * orderbook.index(order)),
            "high" : order[2], #if order[2] is not None else 160 + (0.01 * orderbook.index(order)),
            "low" : order[3], #if order[3] is not None else 160 - (0.01 * orderbook.index(order)),
            "close" : order[4], #if order[4] is not None else 160 + (0.01 * orderbook.index(order)),
            "volume": order[5], #if order[5] is not None else 34.5 + (11.2 * orderbook.index(order)),
            "adjustment":(order[2] + order[3]) /2
        } for order in orderbook]})

@app.route('/<int:propertyid_desired>/<int:propertyid_selling>')
def get_orders_by_market_json(propertyid_desired, propertyid_selling):
    return jsonify(get_orders_by_market(propertyid_desired, propertyid_selling))

def get_orders_by_market(propertyid_desired, propertyid_selling):
    orderbook = dbSelect("SELECT ao.propertyiddesired, ao.propertyidselling, ao.AmountAvailable, ao.AmountDesired, ao.TotalSelling, ao.AmountAccepted, "
                         "cast(txj.txdata->>'unitprice' as numeric), ao.Seller, tx.TxRecvTime, 'active', tx.txhash from activeoffers ao, transactions tx, txjson txj "
                         "where ao.CreateTxDBSerialNum = txj.TxDBSerialNum and ao.CreateTxDBSerialNum = tx.TxDBSerialNum and ao.propertyiddesired = %s and "
                         "ao.propertyidselling = %s and ao.OfferState = 'active' union all select cast(txj.txdata->>'propertyiddesired' as bigint), "
                         "cast(txj.txdata->>'propertyidforsale' as bigint),CASE WHEN txj.txdata->>'propertyidforsaleisdivisible' = 'true' THEN "
                         "round(cast(txj.txdata->>'amountforsale' as numeric) * 100000000) ELSE cast(txj.txdata->>'amountforsale' as numeric) END, "
                         "CASE WHEN txj.txdata->>'propertyiddesiredisdivisible' = 'true' THEN round(cast(txj.txdata->>'amountdesired' as numeric) * 100000000) "
                         "ELSE cast(txj.txdata->>'amountdesired' as numeric) END,CASE WHEN txj.txdata->>'propertyidforsaleisdivisible' = 'true' THEN "
                         "round(cast(txj.txdata->>'amountforsale' as numeric) * 100000000) ELSE cast(txj.txdata->>'amountforsale' as numeric) END,0, "
                         "cast(txj.txdata->>'unitprice' as numeric),txj.txdata->>'sendingaddress', tx.TxRecvTime, 'pending' from transactions tx inner join txjson txj "
                         "on tx.txdbserialnum = txj.txdbserialnum where tx.txdbserialnum < 0 and tx.txtype = 25 and cast(txj.txdata->>'propertyidforsale' as numeric) = %s "
                         "and cast(txj.txdata->>'propertyiddesired' as numeric) = %s",[propertyid_desired,propertyid_selling,propertyid_selling,propertyid_desired])

    cancels = dbSelect("SELECT cast(txj.txdata->>'propertyiddesired' as bigint),cast(txj.txdata->>'propertyidforsale' as bigint),CASE WHEN "
                       "txj.txdata->>'propertyiddesiredisdivisible' = 'true' THEN round(cast(txj.txdata->>'amountdesired' as numeric) * 100000000) "
                       "ELSE cast(txj.txdata->>'amountdesired' as numeric) END,CASE WHEN txj.txdata->>'propertyidforsaleisdivisible' = 'true' THEN "
                       "round(cast(txj.txdata->>'amountforsale' as numeric) * 100000000) ELSE cast(txj.txdata->>'amountforsale' as numeric) END, "
                       "cast(txj.txdata->>'unitprice' as numeric),txj.txdata->>'sendingaddress', tx.TxRecvTime, 'pending', tx.txhash from transactions tx "
                       "inner join txjson txj on tx.txdbserialnum = txj.txdbserialnum where tx.txdbserialnum < 0 and tx.txtype = 26 and "
                       "cast(txj.txdata->>'propertyidforsale' as numeric) = %s and cast(txj.txdata->>'propertyiddesired' as numeric) = %s",
                       [propertyid_selling,propertyid_desired])

    return {"status" : 200, "orderbook": [
        {
            "propertyid_desired":order[0],
            "propertyid_selling":order[1],
            "available_amount" : str(order[2]),
            "desired_amount" : str(order[3]),
            "total_amount" : str(order[4]),
            "accepted_amount": str(order[5]),
            "unit_price" : fixDecimal(order[6]),
            "seller" : str(order[7]),
            "time" : str(order[8]),
            "status" : order[9],
            "txhash" : str(order[10])
        } for order in orderbook], "cancels":[
        {
            "propertyid_desired":cancel[0],
            "propertyid_selling":cancel[1],
            "desired_amount" : str(cancel[2]),
            "total_amount" : str(cancel[3]),
            "unit_price" : str(cancel[4]),
            "seller" : str(cancel[5]),
            "time" : str(cancel[6]),
            "txhash" : str(order[7])
        } for cancel in cancels]}
<|endoftext|>


<commit_before>from pathlib import PureWindowsPath, PurePosixPath
from buildbot.status.results import SUCCESS
from buildbot.steps.source.git import Git
from buildbot.steps.shell import ShellCommand
from buildbot.steps.transfer import FileUpload
from buildbot.process.factory import BuildFactory
from buildbot.process.properties import Property, Interpolate, renderer
from buildbot.steps.master import MasterShellCommand

# Constants
python_exe_property_name = 'python_exe'
dir_command_property_name = 'dir_command'
run_cpp_builds_property_name = 'run_cpp_builds'
run_release_builds_property_name = 'run_release_builds'
hide_cpp_builds_property_name = 'hide_cpp_builds'
hide_release_builds_property_name = 'hide_release_builds'

python_exe = Property('python_exe', default='python')

def step_has_property(property_name, default=None, takesResults=False):
    def check_for_property(step):
        result = step.getProperty(property_name)
        if result is None:
            return default
        else:
            return result
    if takesResults:
        return lambda results, s: check_for_property(s)
    else:
        return check_for_property

def step_has_properties(property_names, default=None, takesResults=False,
                        sentinal=None):
    def check_for_property(step):
        for name in property_names:
            result = step.getProperty(name)
            if result is not sentinal:
                return result
        return default
    if takesResults:
        return lambda results, s: check_for_property(s)
    else:
        return check_for_property

def gen_dest_filename(s):
    return '{1}-{0}.{2}'.format('{buildnumber[0]}', *s.rsplit('.'))

# Git Repositories
nim_git_url = 'https://github.com/Araq/Nim'
csources_git_url = 'https://github.com/nimrod-code/csources'
scripts_git_url = 'https://github.com/nimrod-code/nim-buildbot'

repositories = {
    nim_git_url: 'nim',
    csources_git_url: 'csources',
    scripts_git_url: 'scripts'
}

def get_codebase(change_dict):
    return repositories[change_dict['repository']]

# Resource directories
resource_dirs = {
    'current_dir': './',
    'nim_dir': 'build/',
    'csources_dir': 'build/csources/',
    'scripts_dir': 'scripts/',
    'tester_dir': 'tests/testament/',
    'compiler_dir': 'compiler'
}

class PlatformPaths:
    pass
windows_directories = PlatformPaths()
posix_directories = PlatformPaths()
for key, value in resource_dirs.iteritems():
    setattr(windows_directories, key, PureWindowsPath(value))
    setattr(posix_directories, key, PurePosixPath(value))

# Utility Functions

def inject_paths(func):
    def wrapper(platform, *args, **kwargs):
        platform_directories = posix_directories
        if platform == 'windows':
            platform_directories = windows_directories
        return func(platform_directories, *args, **kwargs)
    return wrapper

has_passed = lambda results, step: results == SUCCESS

# Build Steps

# Common build step parameters
common_git_parameters = {
    'haltOnFailure': True,
    'description': 'Updating',
    'descriptionDone': 'Updated',

    'mode': 'incremental',
    'retry': (10, 20),

    'progress': True,
    'clobberOnFailure': True
}

# Steps

@inject_paths
def update_utility_scripts(platform):
    """
    Updates the utility scripts used by other steps.
    """
    common_parameters = common_git_parameters.copy()
    common_parameters['haltOnFailure'] = False

    return [
        Git(
            name="Update Utility Scripts",
            descriptionSuffix=' Utility Scripts',

            repourl=scripts_git_url,
            codebase=repositories[scripts_git_url],
            alwaysUseLatest=True,

            workdir=str(platform.scripts_dir),
            hideStepIf=False,
            **common_parameters
        )
    ]

@inject_paths
def update_repositories(platform):
    """
    Adds the steps needed to update the csources and Nimrod repositories.
    """
    return [
        Git(
            name="Update Local Nim Repository",
            descriptionSuffix=' Local Nim Repository',

            repourl=nim_git_url,
            codebase=repositories[nim_git_url],

            workdir=str(platform.nim_dir),
            **common_git_parameters
        ),

        Git(
            name="Update Local CSources Repository",
            descriptionSuffix=' Local CSources Repository',

            repourl=csources_git_url,
            alwaysUseLatest=True,
            codebase=repositories[csources_git_url],

            workdir=str(platform.csources_dir),
            **common_git_parameters
        )
    ]

@inject_paths
def clean_repositories(platform):
    """
    Adds the steps needed to clean the Nimrod and CSources repositories.
    Requires that the repositories already exist, in the layout done by
    update_repositories.
    """
    csources_filter = str(platform.csources_dir / '*')
    return [
        ShellCommand(
            name='Clean Local Nim Repository',
            description='Cleaning',
            descriptionDone='Cleaned',
            descriptionSuffix=' Local Nim Repository',

            command=[
                'git', 'clean', '-x', '-f', '-e', csources_filter, '-d'],
            workdir=str(platform.nim_dir),
            haltOnFailure=True,
        ),

        ShellCommand(
            name='Clean Local CSources Repository',
            description='Cleaning',
            descriptionDone='Cleaned',
            descriptionSuffix=' Local CSources Repository',

            command=['git', 'clean', '-x', '-f', '-d'],
            workdir=str(platform.nim_dir),
            haltOnFailure=True,
        )
    ]

@inject_paths
def build_csources(platform, csources_script_cmd):
    """
    Builds the csources binary. Requires that the csources repository be
    present and that a suitable C compiler be present on the system path.
    """
    return [
        ShellCommand(
            name='Build Basic Nim Binary',
            description='Building',
            descriptionDone='Built',
            descriptionSuffix=' Basic CSources Binary',

            command=csources_script_cmd,
            workdir=str(platform.csources_dir),
            haltOnFailure=True,
        )
    ]

@inject_paths
def normalize_nim_names(platform):
    """
    Makes sure that both a 'nim' and 'nimrod' binary are present.
    """
    bin_dir = str(platform.nim_dir / 'bin')
    return [
        ShellCommand(
            name='Normalizing Binary Names',
            description='Normalizing',
            descriptionDone='Normalized',
            descriptionSuffix=' Binary Names',

            command=[python_exe, str(
                platform.scripts_dir / 'normalize_nim.py'), bin_dir],
            workdir=str(platform.current_dir),
            hideStepIf=False
        )
    ]

@inject_paths
def compile_koch(platform):
    """
    Compiles the koch utility.
    """
    bin_dir = str(platform.nim_dir / 'bin')
    base_env = {
        'PATH': [
            str(platform.current_dir),
            bin_dir,
            'bin',
            "${PATH}"
        ]
    }

    return [
        ShellCommand(
            name='Compile Koch',
            description='Compiling',
            descriptionDone='Compiled',
            descriptionSuffix=' Koch Binary',

            command=['nim', 'c', 'koch.nim'],
            env=base_env,
            workdir=str(platform.nim_dir),
            haltOnFailure=True,
        )
    ]

@inject_paths
def boot_nimrod(platform):
    nimfile_dir = str(platform.compiler_dir / 'nim.nim')
    base_env = {
        'PATH': [
            str(platform.current_dir),
            str(platform.current_dir / 'bin'),
            'bin',
            "${PATH}"
        ]
    }

    return [
        ShellCommand(
            name='Bootstrap Debug Version of Nim Compiler (With C Backend)',
            description='Booting',
            descriptionDone='Booted',
            descriptionSuffix=' Debug Nim Compiler (With C Backend)',

            command=['koch', 'boot'],
            workdir=str(platform.nim_dir),
            env=base_env,
            haltOnFailure=True,
        ),

        ShellCommand(
            name='Bootstrap Release Version of Nim Compiler (With C Backend)',
            description='Booting',
            descriptionDone='Booted',
            descriptionSuffix=' Release Nim Compiler (With C Backend)',

            command=['nim', 'c', '-d:release', nimfile_dir],
            workdir=str(platform.nim_dir),
            env=base_env,

            doStepIf=step_has_property(
                property_name=run_release_builds_property_name,
                default=True),
            hideStepIf=step_has_property(
                property_name=hide_release_builds_property_name,
                default=False,
                takesResults=True),
            haltOnFailure=False
        ),

        ShellCommand(
            name='Bootstrap Debug Version of Nim Compiler (With C++ Backend)',
            description='Booting',
            descriptionDone='Booted',
            descriptionSuffix=' Debug Nim Compiler (With C++ Backend)',

            command=['nim', 'cpp', nimfile_dir],
            workdir=str(platform.nim_dir),
            env=base_env,

            doStepIf=step_has_property(
                property_name=run_cpp_builds_property_name,
                default=True
            ),
            hideStepIf=step_has_property(
                property_name=hide_cpp_builds_property_name,
                default=False,
                takesResults=True
            ),
            haltOnFailure=True
        ),

        ShellCommand(
            name='Bootstrap Release Version of Nim Compiler (With C++ Backend)',
            description='Booting',
            descriptionDone='Booted',
            descriptionSuffix=' Release Nim Compiler (With C++ Backend)',

            command=['nim', 'cpp', '-d:release', nimfile_dir],
            workdir=str(platform.nim_dir),
            env=base_env,

            doStepIf=step_has_properties(
                property_names=[
                    run_cpp_builds_property_name,
                    run_release_builds_property_name
                ],
                default=True
            ),
            hideStepIf=step_has_properties(
                property_names=[
                    hide_cpp_builds_property_name,
                    hide_release_builds_property_name
                ],
                default=False,
                takesResults=True
            ),
            haltOnFailure=False,
        )
    ]

def FormatInterpolate(format_string):
    @renderer
    def render_revision(props):
        return format_string.format(**props.properties)
    return render_revision

@inject_paths
def run_testament(platform):
    base_env = {
        'PATH': [
            str(platform.current_dir),
            str(platform.current_dir / 'bin'),
            'bin',
            "${PATH}"
        ]
    }
    test_url = "test-data/{buildername[0]}/{got_revision[0][nim]}/"
    test_directory = 'public_html/' + test_url

    html_test_results = 'testresults.html'
    html_test_results_dest = gen_dest_filename(html_test_results)
    db_test_results = 'testament.db'
    db_test_results_dest = gen_dest_filename(db_test_results)

    return [
        ShellCommand(
            name='Run Testament',
            description='Running',
            descriptionDone='Run',
            descriptionSuffix=' Testament',

            command=['koch', 'test'],
            workdir=str(platform.nim_dir),
            env=base_env,
            haltOnFailure=True,
            timeout=None
        ),

        MasterShellCommand(
            command=['mkdir', '-p', Interpolate(test_directory)],
            path="public_html",
            hideStepIf=True
        ),

        FileUpload(
            slavesrc=html_test_results,
            workdir=str(platform.nim_dir),
            masterdest=FormatInterpolate(
                test_directory + html_test_results_dest),
            url=FormatInterpolate(test_url + html_test_results_dest)
        ),

        FileUpload(
            slavesrc=db_test_results,
            workdir=str(platform.nim_dir),
            masterdest=FormatInterpolate(
                test_directory + db_test_results_dest),
            url=FormatInterpolate(test_url + db_test_results_dest)
        )
    ]

@inject_paths
def generate_csources(platform):
    base_env = {
        'PATH': [
            str(platform.current_dir),
            str(platform.current_dir / 'bin'),
            'bin',
            "${PATH}"
        ]
    }

    return [
        ShellCommand(
            name='Generate CSources',
            description='Generating',
            descriptionDone='Generated',
            descriptionSuffix=' CSources',

            command=['koch', 'csources'],
            workdir=str(platform.nim_dir),
            env=base_env,
            haltOnFailure=True,
        )
    ]

@inject_paths
def generate_zip(platform):
    base_env = {
        'PATH': [
            str(platform.current_dir),
            str(platform.current_dir / 'bin'),
            'bin',
            "${PATH}"
        ]
    }

    return [
        ShellCommand(
            name='Generate CSources',
            description='Generating',
            descriptionDone='Generated',
            descriptionSuffix=' CSources',

            command=['koch', 'csources'],
            workdir=str(platform.nim_dir),
            env=base_env,
            haltOnFailure=True,
        )
    ]

@inject_paths
def generate_installer(platform):
    base_env = {
        'PATH': [
            str(platform.current_dir),
            str(platform.current_dir / 'bin'),
            'bin',
            "${PATH}"
        ]
    }

    return [
        ShellCommand(
            name='Generate CSources',
            description='Generating',
            descriptionDone='Generated',
            descriptionSuffix=' CSources',

            command=['koch', 'nsis'],
            workdir=str(platform.nim_dir),
            env=base_env,
            haltOnFailure=True,
        )
    ]

# Build Configurations
def construct_nim_build(platform, csources_script_cmd, f=None):
    if f is None:
        f = BuildFactory()

    steps = []
    steps.extend(update_utility_scripts(platform))
    steps.extend(update_repositories(platform))
    steps.extend(clean_repositories(platform))
    steps.extend(build_csources(platform, csources_script_cmd))
    steps.extend(normalize_nim_names(platform))
    steps.extend(compile_koch(platform))
    steps.extend(boot_nimrod(platform))
    steps.extend(run_testament(platform))
    for step in steps:
        f.addStep(step)

    return f
<commit_msg>Aligned and re-styled buildsteps<commit_after>from pathlib import PureWindowsPath, PurePosixPath
from buildbot.steps.source.git import Git
from buildbot.steps.shell import ShellCommand
from buildbot.steps.transfer import FileUpload
from buildbot.process.factory import BuildFactory
from buildbot.process.properties import Property, Interpolate, renderer
from buildbot.steps.master import MasterShellCommand

# Constants

# Properties
python_exe_prop          = Property('python_exe', default='python')
dir_command_prop         = Property('dir_command')
run_cpp_builds_prop      = Property('run_cpp_builds')
run_release_builds_prop  = Property('run_release_builds')
hide_cpp_builds_prop     = Property('hide_cpp_builds')
hide_release_builds_prop = Property('hide_release_builds')

# Git Repositories
nim_git_url      = 'https://github.com/Araq/Nim'
csources_git_url = 'https://github.com/nimrod-code/csources'
scripts_git_url  = 'https://github.com/nimrod-code/nim-buildbot'

repositories = {
    nim_git_url      : 'nim',
    csources_git_url : 'csources',
    scripts_git_url  : 'scripts'
}

# Resource Directories
resource_dirs = {
    'current_dir'  : './',
    'nim_dir'      : 'build/',
    'csources_dir' : 'build/csources/',
    'scripts_dir'  : 'scripts/',
    'tester_dir'   : 'tests/testament/',
    'compiler_dir' : 'compiler'
}

# Common Build Step Parameters
common_git_parameters = {
    'haltOnFailure'     : True,
    'description'      : 'Updating',
    'descriptionDone'  : 'Updated',
    'mode'             : 'incremental',
    'retry'            : (10, 20),
    'progress'         : True,
    'clobberOnFailure' : True
}

# Utility Functions
def step_has_property(name, default=None, takesResults=False):
    def check_for_property(step):
        result = step.getProperty(name)
        if result is None:
            return default
        else:
            return result
    if takesResults:
        return lambda results, s: check_for_property(s)
    else:
        return check_for_property

def step_has_properties(names, default=None, giveResults=False, sentinal=None):
    def check_for_property(step):
        for name in names:
            result = step.getProperty(name)
            if result is not sentinal:
                return result
        return default
    if giveResults:
        return lambda results, s: check_for_property(s)
    else:
        return check_for_property

def gen_dest_filename(s):
    return '{1}-{0}.{2}'.format('{buildnumber[0]}', *s.rsplit('.'))

def get_codebase(change_dict):
    return repositories[change_dict['repository']]

# Cross-Platform Environment Calculation
class PlatformPaths:
    pass
windows_directories = PlatformPaths()
posix_directories = PlatformPaths()
for key, value in resource_dirs.iteritems():
    setattr(windows_directories, key, PureWindowsPath(value))
    setattr(posix_directories, key, PurePosixPath(value))
for platform in [windows_directories, posix_directories]:
    platform.base_env = {
        'PATH': [
            str(platform.current_dir),
            str(platform.current_dir / 'bin'),
            'bin',
            "${PATH}"
        ]
    }

def inject_paths(func):
    def wrapper(platform, *args, **kwargs):
        platform_directories = posix_directories
        if platform == 'windows':
            platform_directories = windows_directories
        return func(platform_directories, *args, **kwargs)
    return wrapper

# Build Steps

@inject_paths
def update_utility_scripts(platform):
    """
    Updates the utility scripts used by other steps.
    """
    common_parameters = common_git_parameters.copy()
    common_parameters['haltOnFailure'] = False

    return [
        Git(
            name              = "Update Utility Scripts",
            descriptionSuffix = ' Utility Scripts',
            repourl           = scripts_git_url,
            codebase          = repositories[scripts_git_url],
            workdir           = str(platform.scripts_dir),
            alwaysUseLatest   = True,
            hideStepIf        = False,
            **common_parameters
        )
    ]

@inject_paths
def update_repositories(platform):
    """
    Adds the steps needed to update the csources and Nimrod repositories.
    """
    return [
        Git(
            name              = "Update Local Nim Repository",
            descriptionSuffix = ' Local Nim Repository',
            repourl           = nim_git_url,
            codebase          = repositories[nim_git_url],
            workdir           = str(platform.nim_dir),
            **common_git_parameters
        ),

        Git(
            name              = "Update Local CSources Repository",
            descriptionSuffix = ' Local CSources Repository',
            repourl           = csources_git_url,
            codebase          = repositories[csources_git_url],
            workdir           = str(platform.csources_dir),
            alwaysUseLatest   = True,
            **common_git_parameters
        )
    ]

@inject_paths
def clean_repositories(platform):
    """
    Adds the steps needed to clean the Nimrod and CSources repositories.
    Requires that the repositories already exist, in the layout done by
    update_repositories.
    """
    csources_filter = str(platform.csources_dir / '*')
    nim_clean_cmd = ['git', 'clean', '-x', '-f', '-e', csources_filter, '-d']

    return [
        ShellCommand(
            name              = 'Clean Local Nim Repository',
            description       = 'Cleaning',
            descriptionDone   = 'Cleaned',
            descriptionSuffix = ' Local Nim Repository',
            command           = nim_clean_cmd,
            workdir           = str(platform.nim_dir),
            haltOnFailure     = True,
        ),

        ShellCommand(
            name              = 'Clean Local CSources Repository',
            description       = 'Cleaning',
            descriptionDone   = 'Cleaned',
            descriptionSuffix = ' Local CSources Repository',
            command           = ['git', 'clean', '-x', '-f', '-d'],
            workdir           = str(platform.nim_dir),
            haltOnFailure     = True,
        )
    ]

@inject_paths
def build_csources(platform, csources_script_cmd):
    """
    Builds the csources binary. Requires that the csources repository be
    present and that a suitable C compiler be present on the system path.
    """
    return [
        ShellCommand(
            name              = 'Build Basic Nim Binary',
            description       = 'Building',
            descriptionDone   = 'Built',
            descriptionSuffix = ' Basic CSources Binary',
            command           = csources_script_cmd,
            workdir           = str(platform.csources_dir),
            haltOnFailure     = True,
        )
    ]

@inject_paths
def normalize_nim_names(platform):
    """
    Makes sure that both a 'nim' and 'nimrod' binary are present.
    """
    bin_dir = str(platform.nim_dir / 'bin')
    script_path = str(platform.scripts_dir / 'normalize_nim.py')

    return [
        ShellCommand(
            name              = 'Normalizing Binary Names',
            description       = 'Normalizing',
            descriptionDone   = 'Normalized',
            descriptionSuffix = ' Binary Names',
            command           = [python_exe_prop, script_path, bin_dir],
            workdir           = str(platform.current_dir),
            hideStepIf        = False
        )
    ]

@inject_paths
def compile_koch(platform):
    """
    Compiles the koch utility.
    """
    return [
        ShellCommand(
            name              = 'Compile Koch',
            description       = 'Compiling',
            descriptionDone   = 'Compiled',
            descriptionSuffix = ' Koch Binary',
            command           = ['nim', 'c', 'koch.nim'],
            workdir           = str(platform.nim_dir),
            env               = platform.base_env,
            haltOnFailure     = True,
        )
    ]

@inject_paths
def boot_nimrod(platform):
    nimfile_dir = str(platform.compiler_dir / 'nim.nim')

    return [
        ShellCommand(
            name              = 'Bootstrap Debug Version of Nim Compiler '
                                '(With C Backend)',
            description       = 'Booting',
            descriptionDone   = 'Booted',
            descriptionSuffix = ' Debug Nim Compiler (With C Backend)',
            command           = ['koch', 'boot'],
            workdir           = str(platform.nim_dir),
            env               = platform.base_env,
            haltOnFailure     = True,
        ),

        ShellCommand(
            name              = 'Bootstrap Release Version of Nim Compiler '
                                '(With C Backend)',
            description       = 'Booting',
            descriptionDone   = 'Booted',
            descriptionSuffix = ' Release Nim Compiler (With C Backend)',
            command           = ['nim', 'c', '-d:release', nimfile_dir],
            workdir           = str(platform.nim_dir),
            env               = platform.base_env,
            haltOnFailure     = False,

            doStepIf=step_has_property(
                property_name = run_release_builds_prop.key,
                default       = True
            ),
            hideStepIf=step_has_property(
                property_name = hide_release_builds_prop.key,
                default       = False,
                takesResults  = True
            ),
        ),

        ShellCommand(
            name              = 'Bootstrap Debug Version of Nim Compiler '
                                '(With C++ Backend)',
            description       = 'Booting',
            descriptionDone   = 'Booted',
            descriptionSuffix = ' Debug Nim Compiler (With C++ Backend)',

            command           = ['nim', 'cpp', nimfile_dir],
            workdir           = str(platform.nim_dir),
            env               = platform.base_env,
            haltOnFailure     = True,

            doStepIf=step_has_property(
                property_name = run_cpp_builds_prop.key,
                default       = True
            ),
            hideStepIf=step_has_property(
                property_name = hide_cpp_builds_prop.key,
                default       = False,
                takesResults  = True
            ),
        ),

        ShellCommand(
            name              = 'Bootstrap Release Version of Nim Compiler'
                                '(With C++ Backend)',
            description       = 'Booting',
            descriptionDone   = 'Booted',
            descriptionSuffix = ' Release Nim Compiler (With C++ Backend)',
            command           = ['nim', 'cpp', '-d:release', nimfile_dir],
            workdir           = str(platform.nim_dir),
            env               = platform.base_env,
            haltOnFailure     = False,

            doStepIf=step_has_properties(
                property_names=[
                    run_cpp_builds_prop.key,
                    run_release_builds_prop.key
                ],
                default = True
            ),
            hideStepIf=step_has_properties(
                property_names=[
                    hide_cpp_builds_prop.key,
                    hide_release_builds_prop.key
                ],
                default      = False,
                takesResults = True
            ),
        )
    ]

def FormatInterpolate(format_string):
    @renderer
    def render_revision(props):
        return format_string.format(**props.properties)
    return render_revision

@inject_paths
def run_testament(platform):
    test_url = "test-data/{buildername[0]}/{got_revision[0][nim]}/"
    test_directory = 'public_html/' + test_url

    html_test_results = 'testresults.html'
    html_test_results_dest = gen_dest_filename(html_test_results)
    db_test_results = 'testament.db'
    db_test_results_dest = gen_dest_filename(db_test_results)

    return [
        ShellCommand(
            name              = 'Run Testament',
            description       = 'Running',
            descriptionDone   = 'Run',
            descriptionSuffix = ' Testament',
            command           = ['koch', 'test'],
            workdir           = str(platform.nim_dir),
            env               = platform.base_env,
            haltOnFailure     = True,
            timeout           = None
        ),

        MasterShellCommand(
            command    = ['mkdir', '-p', Interpolate(test_directory)],
            path       = "public_html",
            hideStepIf = True
        ),

        FileUpload(
            slavesrc   = html_test_results,
            workdir    = str(platform.nim_dir),
            url        = FormatInterpolate(test_url + html_test_results_dest),
            masterdest = FormatInterpolate(
                test_directory + html_test_results_dest
            ),
        ),

        FileUpload(
            slavesrc   = db_test_results,
            workdir    = str(platform.nim_dir),
            url        = FormatInterpolate(test_url + db_test_results_dest),
            masterdest = FormatInterpolate(
                test_directory + db_test_results_dest
            ),
        )
    ]

@inject_paths
def generate_csources(platform):
    return [
        ShellCommand(
            name              = 'Generate CSources',
            description       = 'Generating',
            descriptionDone   = 'Generated',
            descriptionSuffix = ' CSources',
            command           = ['koch', 'csources'],
            workdir           = str(platform.nim_dir),
            env               = platform.base_env,
            haltOnFailure     = True,
        )
    ]

@inject_paths
def generate_zip(platform):
    return [
        ShellCommand(
            name              = 'Generate CSources',
            description       = 'Generating',
            descriptionDone   = 'Generated',
            descriptionSuffix = ' CSources',
            command           = ['koch', 'csources'],
            workdir           = str(platform.nim_dir),
            env               = platform.base_env,
            haltOnFailure     = True,
        )
    ]

@inject_paths
def generate_installer(platform):
    return [
        ShellCommand(
            name              = 'Generate CSources',
            description       = 'Generating',
            descriptionDone   = 'Generated',
            descriptionSuffix = ' CSources',
            command           = ['koch', 'nsis'],
            workdir           = str(platform.nim_dir),
            env               = platform.base_env,
            haltOnFailure     = True,
        )
    ]

# Build Configurations
def construct_nim_build(platform, csources_script_cmd, f=None):
    if f is None:
        f = BuildFactory()

    steps = []
    steps.extend(update_utility_scripts(platform))
    steps.extend(update_repositories(platform))
    steps.extend(clean_repositories(platform))
    steps.extend(build_csources(platform, csources_script_cmd))
    steps.extend(normalize_nim_names(platform))
    steps.extend(compile_koch(platform))
    steps.extend(boot_nimrod(platform))
    steps.extend(run_testament(platform))
    for step in steps:
        f.addStep(step)

    return f
<|endoftext|>