<commit_before>#!/usr/bin/env python
# Built-in python packages
from __future__ import division # This modifies Python 2.7 so that any expression of the form int/int returns a float
import getopt
import sys
import os
from itertools import combinations
from multiprocessing.pool import Pool
from contextlib import closing
# Project-specific packages
from queries_with_ref_bases import query_contains_ref_bases

def get_accession_map(fasta_path):
    '''
    Suppose in the FASTA file used as reference for makeblastdb, there are n sequences.
    Magic-BLAST labels these sequences as the order in which they appear when outputting in tabulated format.
    For example, suppose that the SNP rs0001 appears second from the top in the FASTA file.
    Then Magic-BLAST assigns the label '1' to rs0001 whenever it appears in an alignment.
    This function returns a dictionary that serves as a map from integers (in string datatype) to accessions
    e.g. accession_map['1'] == rs0001 in our above example.
    Inputs
    - (str) fasta_path: path to the FASTA file used as reference for makeblastdb
    Outputs
    - (dict) accession_map: the map from integers to accessions
    '''
    accession_map = {}
    with open(fasta_path,'r') as fasta:
        id_number = 0
        for line in fasta:
            if line[0] == ">":
                accession = line[1:].rstrip()
                accession_map[str(id_number)] = accession
                id_number += 1
    return accession_map

def get_mbo_paths(directory):
    '''
    Given a directory, retrieves the paths of all files within the directory whose extension is .mbo
    Inputs
    - (str) directory: directory to be scoured for mbo files
    Outputs
    - paths: a dictionary where the keys are accessions and the values are paths
    '''
    paths = {}
    for file in os.listdir(directory):
        if file.endswith(".mbo"):
            # We only want the accession number, not the extension as well
            accession = os.path.basename(file).split('.')[0]
            path = os.path.join(directory,file)
            paths[accession] = path
    return paths

def get_sra_alignments(paths,accession_map):
    '''
    Given a list of paths as described in the function get_mbo_paths, retrieves the BTOP string for each
    alignment.
    Inputs
    - paths: a list of pairs where the first entry of the pair is the accession and the second is the path
    - (dict) accession_map: the map between integers and accessions
    Outputs
    - a dictionary where keys are SRA accessions and the values are alignment dictionaries
    '''
    sra_alignments = {}
    for accession in paths:
        path = paths[accession]
        alignments = []
        with open(path,'r') as mbo:
            for line in mbo:
                tokens = line.split()
                # Skip the line if it is commented, the number of fields isn't equal to 25 or
                # the query read was not aligned
                if line[0] != "#" and len(tokens) == 25 and tokens[1] != "-":
                    var_acc = accession_map[ tokens[1] ]
                    ref_start = int(tokens[8])
                    ref_stop = int(tokens[9])
                    if ref_start > ref_stop:
                        temp = ref_start
                        ref_start = ref_stop
                        ref_stop = temp
                    btop = tokens[16]
                    alignment = { 'var_acc': var_acc, 'ref_start': ref_start,\
                              'ref_stop': ref_stop, 'btop': btop }
                    alignments.append( alignment )
        sra_alignments[accession] = alignments
    return sra_alignments

def get_var_info(path):
    '''
    Retrieves the flanking sequence lengths for the SNP sequences
    Inputs
    - path: path to the file that contains the flanking sequence lengths
    Outputs
    - var_info: a dictionary where the keys are SNP accessions and the values are tuples where the first entry\
                    is the start position of the variant, the second is the stop position of the variant and \
                    the third is the length of the variant sequence
    '''
    var_info = {}
    with open(path,'r') as input_file:
        for line in input_file:
            tokens = line.split()
            if len(tokens) == 4:
                accession = tokens[0]
                start = int(tokens[1])
                stop = int(tokens[2])
                length = int(tokens[3])
                var_info[accession] = {'start':start,'stop':stop,'length':length}
    return var_info

def call_variants(var_freq):
    '''
    Determines which variants exist in a given SRA dataset given the number of reads that do and do not contain
    the var
    Inputs
    - var_freq: a dict where the keys are SNP accessions and the values are dicts which contain the frequency of
            reads that do and reads that do not contain the SNP
    Outputs
    - variants: dict where the keys are SRA accessions and the value is another dict that contains the homozgyous and
                heterozygous variants in separate lists
    '''
    variants = {'heterozygous':[],'homozygous':[]}
    for var_acc in var_freq:
        frequencies = var_freq[var_acc]
        true = frequencies['true']
        false = frequencies['false']
        try:
            percentage = true/(true+false)
            if percentage > 0.8: # For now, we use this simple heuristic.
                variants['homozygous'].append(var_acc)
            elif percentage > 0.3:
                variants['heterozygous'].append(var_acc)
        except ZeroDivisionError: # We ignore division errors because they correspond to no mapped reads
            pass
    return variants

def call_sra_variants(alignments_and_info):
    '''
    For all SRA accession, determines which variants exist in the SRA dataset
    Inputs
    - alignments_and_info: a dict which contains
        - sra_alignments: dict where the keys are SRA accessions and the values are lists of alignment dicts
        - var_info: dict where the keys are variant accessions and the values are information concerning the variants
        - keys: list which contains the keys of the SRA accessions to analyze
    Outputs
    - variants: dict where the keys are SRA accessions and the value is another dict that contains the homozgyous and
                heterozygous variants in separate lists
    '''
    variants = {}
    for sra_acc in keys:
        alignments = sra_alignments[sra_acc]
        var_freq = {}
        for alignment in alignments:
            var_acc = alignment['var_acc']
            # Get the flank information
            info = var_info[var_acc]
            if var_acc not in var_freq:
                var_freq[var_acc] = {'true':0,'false':0}
            # Determine whether the variant exists in the particular SRA dataset
            var_called = query_contains_ref_bases(alignment,info)
            if var_called == True:
                var_freq[var_acc]['true'] += 1
            elif var_called == False:
                var_freq[var_acc]['false'] += 1
        sra_variants = call_variants(var_freq)
        variants[sra_acc] = sra_variants
    return variants

def create_tsv(variants,output_path):
    '''
    Creates a TSV file containing the set of variants each SRA dataset contains.
    Inputs
    - variants: dict where the keys are SRA accessions and the value is another dict that contains the homozgyous and
                heterozygous variants in separate lists
    - output_path: path to where to construct the output file
    '''
    with open(output_path,'w') as tsv:
        header = "SRA\tHeterozygous SNPs\tHomozygous SNPs\n"
        tsv.write(header)
        for sra_acc in variants:
            line = "%s" % (sra_acc)
            sra_variants = variants[sra_acc]
            line = line + "\t"
            for var_acc in sra_variants['heterozygous']:
                line = line + var_acc + ","
            line = line + "\t"
            for var_acc in sra_variants['homozygous']:
                line = line + var_acc + ","
            tsv.write( line.rstrip() )
            tsv.write('\n')

def create_variant_matrix(variants):
    '''
    Returns an adjacency matrix that represents the graph constructed out of the variants such that:
    1. Every vertex represents a variant and every variant is represented by a vertex
    2. An edge (line) connects two vertices if and only if there exists an SRA dataset that contains both of the
       corresponding variants
    The matrix is represented by a dictionary of dictionaries. To save memory, we do not store 0 entries or variants
    without any incident edges.
    Inputs
    - variants: dict where the keys are SRA accessions and the value is another dict that contains the homozgyous and
                heterozygous variants in separate lists
    Outputs
    - matrix: a dict which, for any two keys (variants) variant_1 and variant_2, satisfies the following -
              1. type(matrix[variant_1]) is DictType
              2. type(matrix[variant_1][variant_2]) is IntType and matrix[variant_1][variant_2] >= 1
              3. matrix[variant_1][variant_2] == matrix[variant_2][variant_1]
              all of which holds if and only if variant_1 and variant_2 exist in the dictionaries as keys
    '''
    matrix = {}
    for sra_acc in variants:
        sra_variants = variants[sra_acc]
        all_variants = []
        if 'homozygous' in sra_variants:
            all_variants += sra_variants['homozygous']
        if 'heterozygous' in sra_variants:
            all_variants += sra_variants['heterozygous']
        # Get all of the unique 2-combinations of the variants
        two_combinations = list( combinations(all_variants,2) )
        for pair in two_combinations:
            variant_1 = pair[0]
            variant_2 = pair[1]
            if variant_1 not in matrix:
                matrix[variant_1] = {}
            if variant_2 not in matrix:
                matrix[variant_2] = {}
            if variant_2 not in matrix[variant_1]:
                matrix[variant_1][variant_2] = 0
            if variant_1 not in matrix[variant_2]:
                matrix[variant_2][variant_1] = 0
            matrix[variant_1][variant_2] += 1
            matrix[variant_2][variant_1] = matrix[variant_1][variant_2]
    return matrix

def partition(lst,n):
    '''
    Partitions a list into n lists
    Inputs
    - (list) lst
    Outputs
    - partitioned_lists: a list of lists
    '''
    division = len(lst)/float(n)
    return [ lst[int(round(division * i)): int(round(division * (i + 1)))] for i in xrange(n) ]

def combine_list_of_dicts(list_of_dicts):
    '''
    Given a list of dicts, returns a single dict such that (key,value) pairs from each original dict exists in the
    new single dict
    Inputs
    - list_of_dicts: a list of dicts
    Outputs
    - combined_dict
    '''
    combined_dict = list_of_dicts[0]
    for i in range(1,len(list_of_dicts)):
        combined_dict.update( list_of_dicts[i] )
    return combined_dict

def unit_tests():
    variants = {}
    variants['sra_1'] = {'homozygous':['a','b'],'heterozygous':['c','e']}
    variants['sra_2'] = {'homozygous':['a','c','d']}
    variants['sra_3'] = {'heterozygous':['b','d','e']}
    matrix = create_variant_matrix(variants)
    print(matrix)
    for variant_1 in matrix:
        for variant_2 in matrix[variant_1]:
            left_hand_side = matrix[variant_1][variant_2]
            right_hand_side = matrix[variant_2][variant_1]
            assert( left_hand_side >= 1 )
            assert( right_hand_side >= 1 )
            assert( left_hand_side == right_hand_side )
    print("All unit tests passed!")

if __name__ == "__main__":
    help_message = "Description: Given a directory with Magic-BLAST output files where each output file\n" \
                     + "             contains the alignment between an SRA dataset and known variants in a human\n" \
                     + "             genome, this script determines which variants each SRA dataset contains\n" \
                     + "             using a heuristic."
    usage_message = "Usage: %s\n[-h (help and usage)]\n[-m <directory containing .mbo files>]\n" % (sys.argv[0]) \
                      + "[-v <path to variant info file>]\n[-f <path to the reference FASTA file>]\n"\
                      + "[-o <output path for TSV file>]\n[-p <num of threads>]\n[-t <unit tests>]"
    options = "htm:v:f:o:p:"

    try:
        opts,args = getopt.getopt(sys.argv[1:],options)
    except getopt.GetoptError:
        print("Error: unable to read command line arguments.")
        sys.exit(1)

    if len(sys.argv) == 1:
        print(help_message)
        print(usage_message)
        sys.exit()

    mbo_directory = None
    var_info_path = None
    output_path = None
    fasta_path = None
    threads = 1

    for opt, arg in opts:
        if opt == '-h':
            print(help_message)
            print(usage_message)
            sys.exit(0)
        elif opt == '-m':
            mbo_directory = arg
        elif opt == '-v':
            var_info_path = arg
        elif opt == '-o':
            output_path = arg
        elif opt == '-f':
            fasta_path = arg
        elif opt == '-p':
            threads = arg
        elif opt == '-t':
            unit_tests()
            sys.exit(0)

    opts_incomplete = False

    if mbo_directory == None:
        print("Error: please provide the directory containing your Magic-BLAST output files.")
        opts_incomplete = True
    if var_info_path == None:
        print("Error: please provide the path to the file containing flanking sequence information.")
        opts_incomplete = True
    if output_path == None:
        print("Error: please provide an output path for the TSV file.")
        opts_incomplete = True
    if fasta_path == None:
        print("Error: please provide the path to the FASTA file used as reference for makeblastdb")
        opts_incomplete = True
    if opts_incomplete:
        print(usage_message)
        sys.exit(1)

    var_info = get_var_info(var_info_path)
    accession_map = get_accession_map(fasta_path)
    paths = get_mbo_paths(mbo_directory)

    # Retrieve the alignments concurrently
    get_alignments_threads = min(threads,len(paths))
    paths_partitions = partition( paths, get_alignments_threads )
    with closing(Pool(processes=get_alignments_threads)) as pool:
        sra_alignments_pool = pool.map(get_sra_alignments,paths_partitions)
    pool.terminate()
    sra_alignments = combine_list_of_dicts(sra_alignments_pool)

    # Call variants concurrently
    sra_keys = sra_alignments.keys()
    variant_call_threads = min( threads, len(sra_keys) )
    keys_partitions = partition(sra_keys, variant_call_threads)
    # alignments, info and key partitions
    alignments_and_info_part = [{'alignments':sra_alignments,'keys':keys,'info':var_info} for keys in keys_partitions]
    with closing(Pool(processes=variant_call_threads)) as pool:
        variants_pool = pool.map(call_sra_variants,alignments_and_info_part)
	pool.terminate()
    called_variants = combine_list_of_dicts(variants_pool)

    create_tsv(called_variants,output_path)
    matrix = create_variant_matrix(called_variants)
<commit_msg>fixed bug in call_variants.py<commit_after>#!/usr/bin/env python
# Built-in python packages
from __future__ import division # This modifies Python 2.7 so that any expression of the form int/int returns a float
import getopt
import sys
import os
from itertools import combinations
from multiprocessing.pool import Pool
from contextlib import closing
# Project-specific packages
from queries_with_ref_bases import query_contains_ref_bases

def get_accession_map(fasta_path):
    '''
    Suppose in the FASTA file used as reference for makeblastdb, there are n sequences.
    Magic-BLAST labels these sequences as the order in which they appear when outputting in tabulated format.
    For example, suppose that the SNP rs0001 appears second from the top in the FASTA file.
    Then Magic-BLAST assigns the label '1' to rs0001 whenever it appears in an alignment.
    This function returns a dictionary that serves as a map from integers (in string datatype) to accessions
    e.g. accession_map['1'] == rs0001 in our above example.
    Inputs
    - (str) fasta_path: path to the FASTA file used as reference for makeblastdb
    Outputs
    - (dict) accession_map: the map from integers to accessions
    '''
    accession_map = {}
    with open(fasta_path,'r') as fasta:
        id_number = 0
        for line in fasta:
            if line[0] == ">":
                accession = line[1:].rstrip()
                accession_map[str(id_number)] = accession
                id_number += 1
    return accession_map

def get_mbo_paths(directory):
    '''
    Given a directory, retrieves the paths of all files within the directory whose extension is .mbo
    Inputs
    - (str) directory: directory to be scoured for mbo files
    Outputs
    - paths: a dictionary where the keys are accessions and the values are paths
    '''
    paths = {}
    for file in os.listdir(directory):
        if file.endswith(".mbo"):
            # We only want the accession number, not the extension as well
            accession = os.path.basename(file).split('.')[0]
            path = os.path.join(directory,file)
            paths[accession] = path
    return paths

def get_sra_alignments(map_paths_and_partition):
    '''
    Given a list of paths as described in the function get_mbo_paths, retrieves the BTOP string for each
    alignment.
    Inputs
    - map_paths_and_partition: a dict which contains the following:
        - partition: the list of paths to .mbo files to read
        - paths: a list of pairs where the first entry of the pair is the accession and the second is the path
        - (dict) accession_map: the map between integers and accessions
    Outputs
    - a dictionary where keys are SRA accessions and the values are alignment dictionaries
    '''
    accession_map = map_paths_and_partition['map']
    paths = map_paths_and_partition['paths']
    partition = map_paths_and_partition['partition']
    sra_alignments = {}
    for accession in partition:
        path = paths[accession]
        alignments = []
        with open(path,'r') as mbo:
            for line in mbo:
                tokens = line.split()
                # Skip the line if it is commented, the number of fields isn't equal to 25 or
                # the query read was not aligned
                if line[0] != "#" and len(tokens) == 25 and tokens[1] != "-":
                    var_acc = accession_map[ tokens[1] ]
                    ref_start = int(tokens[8])
                    ref_stop = int(tokens[9])
                    if ref_start > ref_stop:
                        temp = ref_start
                        ref_start = ref_stop
                        ref_stop = temp
                    btop = tokens[16]
                    alignment = { 'var_acc': var_acc, 'ref_start': ref_start,\
                              'ref_stop': ref_stop, 'btop': btop }
                    alignments.append( alignment )
        sra_alignments[accession] = alignments
    return sra_alignments

def get_var_info(path):
    '''
    Retrieves the flanking sequence lengths for the SNP sequences
    Inputs
    - path: path to the file that contains the flanking sequence lengths
    Outputs
    - var_info: a dictionary where the keys are SNP accessions and the values are tuples where the first entry\
                    is the start position of the variant, the second is the stop position of the variant and \
                    the third is the length of the variant sequence
    '''
    var_info = {}
    with open(path,'r') as input_file:
        for line in input_file:
            tokens = line.split()
            if len(tokens) == 4:
                accession = tokens[0]
                start = int(tokens[1])
                stop = int(tokens[2])
                length = int(tokens[3])
                var_info[accession] = {'start':start,'stop':stop,'length':length}
    return var_info

def call_variants(var_freq):
    '''
    Determines which variants exist in a given SRA dataset given the number of reads that do and do not contain
    the var
    Inputs
    - var_freq: a dict where the keys are SNP accessions and the values are dicts which contain the frequency of
            reads that do and reads that do not contain the SNP
    Outputs
    - variants: dict where the keys are SRA accessions and the value is another dict that contains the homozgyous and
                heterozygous variants in separate lists
    '''
    variants = {'heterozygous':[],'homozygous':[]}
    for var_acc in var_freq:
        frequencies = var_freq[var_acc]
        true = frequencies['true']
        false = frequencies['false']
        try:
            percentage = true/(true+false)
            if percentage > 0.8: # For now, we use this simple heuristic.
                variants['homozygous'].append(var_acc)
            elif percentage > 0.3:
                variants['heterozygous'].append(var_acc)
        except ZeroDivisionError: # We ignore division errors because they correspond to no mapped reads
            pass
    return variants

def call_sra_variants(alignments_and_info):
    '''
    For all SRA accession, determines which variants exist in the SRA dataset
    Inputs
    - alignments_and_info: a dict which contains
        - sra_alignments: dict where the keys are SRA accessions and the values are lists of alignment dicts
        - var_info: dict where the keys are variant accessions and the values are information concerning the variants
        - keys: list which contains the keys of the SRA accessions to analyze
    Outputs
    - variants: dict where the keys are SRA accessions and the value is another dict that contains the homozgyous and
                heterozygous variants in separate lists
    '''
    variants = {}
    for sra_acc in keys:
        alignments = sra_alignments[sra_acc]
        var_freq = {}
        for alignment in alignments:
            var_acc = alignment['var_acc']
            # Get the flank information
            info = var_info[var_acc]
            if var_acc not in var_freq:
                var_freq[var_acc] = {'true':0,'false':0}
            # Determine whether the variant exists in the particular SRA dataset
            var_called = query_contains_ref_bases(alignment,info)
            if var_called == True:
                var_freq[var_acc]['true'] += 1
            elif var_called == False:
                var_freq[var_acc]['false'] += 1
        sra_variants = call_variants(var_freq)
        variants[sra_acc] = sra_variants
    return variants

def create_tsv(variants,output_path):
    '''
    Creates a TSV file containing the set of variants each SRA dataset contains.
    Inputs
    - variants: dict where the keys are SRA accessions and the value is another dict that contains the homozgyous and
                heterozygous variants in separate lists
    - output_path: path to where to construct the output file
    '''
    with open(output_path,'w') as tsv:
        header = "SRA\tHeterozygous SNPs\tHomozygous SNPs\n"
        tsv.write(header)
        for sra_acc in variants:
            line = "%s" % (sra_acc)
            sra_variants = variants[sra_acc]
            line = line + "\t"
            for var_acc in sra_variants['heterozygous']:
                line = line + var_acc + ","
            line = line + "\t"
            for var_acc in sra_variants['homozygous']:
                line = line + var_acc + ","
            tsv.write( line.rstrip() )
            tsv.write('\n')

def create_variant_matrix(variants):
    '''
    Returns an adjacency matrix that represents the graph constructed out of the variants such that:
    1. Every vertex represents a variant and every variant is represented by a vertex
    2. An edge (line) connects two vertices if and only if there exists an SRA dataset that contains both of the
       corresponding variants
    The matrix is represented by a dictionary of dictionaries. To save memory, we do not store 0 entries or variants
    without any incident edges.
    Inputs
    - variants: dict where the keys are SRA accessions and the value is another dict that contains the homozgyous and
                heterozygous variants in separate lists
    Outputs
    - matrix: a dict which, for any two keys (variants) variant_1 and variant_2, satisfies the following -
              1. type(matrix[variant_1]) is DictType
              2. type(matrix[variant_1][variant_2]) is IntType and matrix[variant_1][variant_2] >= 1
              3. matrix[variant_1][variant_2] == matrix[variant_2][variant_1]
              all of which holds if and only if variant_1 and variant_2 exist in the dictionaries as keys
    '''
    matrix = {}
    for sra_acc in variants:
        sra_variants = variants[sra_acc]
        all_variants = []
        if 'homozygous' in sra_variants:
            all_variants += sra_variants['homozygous']
        if 'heterozygous' in sra_variants:
            all_variants += sra_variants['heterozygous']
        # Get all of the unique 2-combinations of the variants
        two_combinations = list( combinations(all_variants,2) )
        for pair in two_combinations:
            variant_1 = pair[0]
            variant_2 = pair[1]
            if variant_1 not in matrix:
                matrix[variant_1] = {}
            if variant_2 not in matrix:
                matrix[variant_2] = {}
            if variant_2 not in matrix[variant_1]:
                matrix[variant_1][variant_2] = 0
            if variant_1 not in matrix[variant_2]:
                matrix[variant_2][variant_1] = 0
            matrix[variant_1][variant_2] += 1
            matrix[variant_2][variant_1] = matrix[variant_1][variant_2]
    return matrix

def partition(lst,n):
    '''
    Partitions a list into n lists
    Inputs
    - (list) lst
    Outputs
    - partitioned_lists: a list of lists
    '''
    division = len(lst)/float(n)
    return [ lst[int(round(division * i)): int(round(division * (i + 1)))] for i in xrange(n) ]

def combine_list_of_dicts(list_of_dicts):
    '''
    Given a list of dicts, returns a single dict such that (key,value) pairs from each original dict exists in the
    new single dict
    Inputs
    - list_of_dicts: a list of dicts
    Outputs
    - combined_dict
    '''
    combined_dict = list_of_dicts[0]
    for i in range(1,len(list_of_dicts)):
        combined_dict.update( list_of_dicts[i] )
    return combined_dict

def unit_tests():
    variants = {}
    variants['sra_1'] = {'homozygous':['a','b'],'heterozygous':['c','e']}
    variants['sra_2'] = {'homozygous':['a','c','d']}
    variants['sra_3'] = {'heterozygous':['b','d','e']}
    matrix = create_variant_matrix(variants)
    print(matrix)
    for variant_1 in matrix:
        for variant_2 in matrix[variant_1]:
            left_hand_side = matrix[variant_1][variant_2]
            right_hand_side = matrix[variant_2][variant_1]
            assert( left_hand_side >= 1 )
            assert( right_hand_side >= 1 )
            assert( left_hand_side == right_hand_side )
    print("All unit tests passed!")

if __name__ == "__main__":
    help_message = "Description: Given a directory with Magic-BLAST output files where each output file\n" \
                     + "             contains the alignment between an SRA dataset and known variants in a human\n" \
                     + "             genome, this script determines which variants each SRA dataset contains\n" \
                     + "             using a heuristic."
    usage_message = "Usage: %s\n[-h (help and usage)]\n[-m <directory containing .mbo files>]\n" % (sys.argv[0]) \
                      + "[-v <path to variant info file>]\n[-f <path to the reference FASTA file>]\n"\
                      + "[-o <output path for TSV file>]\n[-p <num of threads>]\n[-t <unit tests>]"
    options = "htm:v:f:o:p:"

    try:
        opts,args = getopt.getopt(sys.argv[1:],options)
    except getopt.GetoptError:
        print("Error: unable to read command line arguments.")
        sys.exit(1)

    if len(sys.argv) == 1:
        print(help_message)
        print(usage_message)
        sys.exit()

    mbo_directory = None
    var_info_path = None
    output_path = None
    fasta_path = None
    threads = 1

    for opt, arg in opts:
        if opt == '-h':
            print(help_message)
            print(usage_message)
            sys.exit(0)
        elif opt == '-m':
            mbo_directory = arg
        elif opt == '-v':
            var_info_path = arg
        elif opt == '-o':
            output_path = arg
        elif opt == '-f':
            fasta_path = arg
        elif opt == '-p':
            threads = arg
        elif opt == '-t':
            unit_tests()
            sys.exit(0)

    opts_incomplete = False

    if mbo_directory == None:
        print("Error: please provide the directory containing your Magic-BLAST output files.")
        opts_incomplete = True
    if var_info_path == None:
        print("Error: please provide the path to the file containing flanking sequence information.")
        opts_incomplete = True
    if output_path == None:
        print("Error: please provide an output path for the TSV file.")
        opts_incomplete = True
    if fasta_path == None:
        print("Error: please provide the path to the FASTA file used as reference for makeblastdb")
        opts_incomplete = True
    if opts_incomplete:
        print(usage_message)
        sys.exit(1)

    var_info = get_var_info(var_info_path)
    accession_map = get_accession_map(fasta_path)
    paths = get_mbo_paths(mbo_directory)

    # Retrieve the alignments concurrently
    get_alignments_threads = min(threads,len(paths.keys()))
    paths_partitions = partition( paths.keys(), get_alignments_threads )
    map_paths_and_partitions = [{'map':accession_map,'paths':paths,'partition':partition} \
                                for partition in paths_partitions]
    with closing(Pool(processes=get_alignments_threads)) as pool:
        sra_alignments_pool = pool.map(get_sra_alignments,map_paths_and_partitions)
    pool.terminate()
    sra_alignments = combine_list_of_dicts(sra_alignments_pool)

    # Call variants concurrently
    sra_keys = sra_alignments.keys()
    variant_call_threads = min( threads, len(sra_keys) )
    keys_partitions = partition(sra_keys, variant_call_threads)
    # alignments, info and key partitions
    alignments_and_info_part = [{'alignments':sra_alignments,'keys':keys,'info':var_info} for keys in keys_partitions]
    with closing(Pool(processes=variant_call_threads)) as pool:
        variants_pool = pool.map(call_sra_variants,alignments_and_info_part)
	pool.terminate()
    called_variants = combine_list_of_dicts(variants_pool)

    create_tsv(called_variants,output_path)
    matrix = create_variant_matrix(called_variants)
<|endoftext|>


<commit_before># coding: utf-8
import os
import urllib
from logging import error

from django.db.models.fields.files import ImageFieldFile, ImageField
from django.conf import settings
from PIL import Image, ImageEnhance

from watermarker import models

def reduce_opacity(im, opacity):
    """Returns an image with reduced opacity."""

    assert opacity >= 0 and opacity <= 1
    if im.mode != 'RGBA':
        im = im.convert('RGBA')
    else:
        im = im.copy()
    alpha = im.split()[3]
    alpha = ImageEnhance.Brightness(alpha).enhance(opacity)
    im.putalpha(alpha)
    return im

def make_watermark(im, mark, position, opacity=1, shift=(0, 0)):
    """Adds a watermark to an image."""

    if opacity < 1:
        mark = reduce_opacity(mark, opacity)
    if im.mode != 'RGBA':
        im = im.convert('RGBA')
    # create a transparent layer the size of the image and draw the
    # watermark in that layer.
    layer = Image.new('RGBA', im.size, (0, 0, 0, 0))
    if position == 'tile':
        for y in range(0, im.size[1], mark.size[1]):
            for x in range(0, im.size[0], mark.size[0]):
                layer.paste(mark, (x, y))
    elif position == 'scale':
        # scale, but preserve the aspect ratio
        ratio = min(
            float(im.size[0]) / mark.size[0], float(im.size[1]) / mark.size[1])
        w = int(mark.size[0] * ratio)
        h = int(mark.size[1] * ratio)
        mark = mark.resize((w, h))
        layer.paste(mark, ((im.size[0] - w) / 2, (im.size[1] - h) / 2))
    elif position == 'br':
        x = max(im.size[0] - mark.size[0], 0) - shift[0]
        y = max(im.size[1] - mark.size[1], 0) - shift[1]
        layer.paste(mark, (x, y))
    elif position == 'tr':
        x = max(im.size[0] - mark.size[0], 0) - shift[0]
        y = 0 + shift[1]
        layer.paste(mark, (x, y))
    elif position == 'bl':
        x = 0 + shift[0]
        y = max(im.size[1] - mark.size[1], 0) - shift[1]
        layer.paste(mark, (x, y))
    elif position == 'tl':
        x = 0 + shift[0]
        y = 0 + shift[1]
        layer.paste(mark, (x, y))
    else:
        layer.paste(mark, (0, 0))
    # composite the watermark with the layer
    return Image.composite(layer, im, layer)

def get_path(url):
    """Makes a filesystem path from the specified URL"""

    if url.startswith(settings.MEDIA_URL):
        root = settings.MEDIA_ROOT
    else:
        root = settings.STATIC_ROOT

    url = url.replace(settings.MEDIA_URL, '').replace(settings.STATIC_URL, '')
    url = urllib.unquote(url).decode("utf8")
    return os.path.normpath(os.path.join(root, url))

def make_iff_instance(path):
    """ Returns ImageFieldFile for provided image path """

    return ImageFieldFile(instance=None, field=ImageField(), name=path)

def watermark(url, wm):
    # return usual url(string) or ImageFieldFile
    result_as_iff = False

    if isinstance(wm, models.Watermark):
        watermark = wm
    else:
        try:
            watermark = models.Watermark.objects.get(title=wm, is_active=True)
        except (models.Watermark.DoesNotExist, models.Watermark.MultipleObjectsReturned):
            return url

    # to work not only with strings
    if isinstance(url, ImageFieldFile):
        result_as_iff = True
        if hasattr(url, 'url'):
            url = url.url
        else:
            return url

    basedir = '%s/watermarked' % os.path.dirname(url)
    base, ext = os.path.splitext(os.path.basename(url))

    # watermarked url for template
    wm_url = os.path.join(basedir, '%s%s' % (base, ext))
    # path to save watermarked img
    wm_path = get_path(wm_url)

    # not to do more than we need
    if os.path.exists(wm_path) and not watermark.update_hard:
        if result_as_iff:
            return make_iff_instance(wm_path)
        return wm_url

    # create a folder for watermarks if needed
    wm_dir = os.path.dirname(wm_path)
    if not os.path.exists(wm_dir):
        os.makedirs(wm_dir)

    img_path = get_path(url)

    if not os.path.exists(img_path):
        return url

    try:
        img = Image.open(img_path)

        mark = Image.open(watermark.mark.path)
        x = watermark.x or 0
        y = watermark.y or 0
        shift = (x, y)
        position = watermark.position

        opacity = watermark.opacity

        marked_img = make_watermark(img, mark, position, opacity, shift)
        marked_img.save(wm_path)
    except Exception, e:
        error("Cant create watermark for %s. Error type: %s. Msg: %s" % (img_path, type(e), e))
        return url

    if result_as_iff:
        return make_iff_instance(wm_path)

    return wm_url

def get_url_safe(path):
    """ If we save wm with result_as_iff == True, we thumbnail returns us full path to picture (not valid url)
    may be there is prettier way to manage this

    """

    if not path.startswith(settings.MEDIA_URL):
        return os.path.join(settings.MEDIA_URL, path.split(settings.MEDIA_URL)[-1])
    return path
<commit_msg>is_acrive tic works norm<commit_after># coding: utf-8
import os
import urllib
from logging import error

from django.db.models.fields.files import ImageFieldFile, ImageField
from django.conf import settings
from PIL import Image, ImageEnhance

from watermarker import models

def reduce_opacity(im, opacity):
    """Returns an image with reduced opacity."""

    assert opacity >= 0 and opacity <= 1
    if im.mode != 'RGBA':
        im = im.convert('RGBA')
    else:
        im = im.copy()
    alpha = im.split()[3]
    alpha = ImageEnhance.Brightness(alpha).enhance(opacity)
    im.putalpha(alpha)
    return im

def make_watermark(im, mark, position, opacity=1, shift=(0, 0)):
    """Adds a watermark to an image."""

    if opacity < 1:
        mark = reduce_opacity(mark, opacity)
    if im.mode != 'RGBA':
        im = im.convert('RGBA')
    # create a transparent layer the size of the image and draw the
    # watermark in that layer.
    layer = Image.new('RGBA', im.size, (0, 0, 0, 0))
    if position == 'tile':
        for y in range(0, im.size[1], mark.size[1]):
            for x in range(0, im.size[0], mark.size[0]):
                layer.paste(mark, (x, y))
    elif position == 'scale':
        # scale, but preserve the aspect ratio
        ratio = min(
            float(im.size[0]) / mark.size[0], float(im.size[1]) / mark.size[1])
        w = int(mark.size[0] * ratio)
        h = int(mark.size[1] * ratio)
        mark = mark.resize((w, h))
        layer.paste(mark, ((im.size[0] - w) / 2, (im.size[1] - h) / 2))
    elif position == 'br':
        x = max(im.size[0] - mark.size[0], 0) - shift[0]
        y = max(im.size[1] - mark.size[1], 0) - shift[1]
        layer.paste(mark, (x, y))
    elif position == 'tr':
        x = max(im.size[0] - mark.size[0], 0) - shift[0]
        y = 0 + shift[1]
        layer.paste(mark, (x, y))
    elif position == 'bl':
        x = 0 + shift[0]
        y = max(im.size[1] - mark.size[1], 0) - shift[1]
        layer.paste(mark, (x, y))
    elif position == 'tl':
        x = 0 + shift[0]
        y = 0 + shift[1]
        layer.paste(mark, (x, y))
    else:
        layer.paste(mark, (0, 0))
    # composite the watermark with the layer
    return Image.composite(layer, im, layer)

def get_path(url):
    """Makes a filesystem path from the specified URL"""

    if url.startswith(settings.MEDIA_URL):
        root = settings.MEDIA_ROOT
    else:
        root = settings.STATIC_ROOT

    url = url.replace(settings.MEDIA_URL, '').replace(settings.STATIC_URL, '')
    url = urllib.unquote(url).decode("utf8")
    return os.path.normpath(os.path.join(root, url))

def make_iff_instance(path):
    """ Returns ImageFieldFile for provided image path """

    return ImageFieldFile(instance=None, field=ImageField(), name=path)

def watermark(url, wm):
    # return usual url(string) or ImageFieldFile
    result_as_iff = False

    if isinstance(wm, models.Watermark):
        watermark = wm
    else:
        try:
            watermark = models.Watermark.objects.get(title=wm, is_active=True)
        except (models.Watermark.DoesNotExist, models.Watermark.MultipleObjectsReturned):
            return url

    if not watermark.is_active:
        return url

    # to work not only with strings
    if isinstance(url, ImageFieldFile):
        result_as_iff = True
        if hasattr(url, 'url'):
            url = url.url
        else:
            return url

    basedir = '%s/watermarked' % os.path.dirname(url)
    base, ext = os.path.splitext(os.path.basename(url))

    # watermarked url for template
    wm_url = os.path.join(basedir, '%s%s' % (base, ext))
    # path to save watermarked img
    wm_path = get_path(wm_url)

    # not to do more than we need
    if os.path.exists(wm_path) and not watermark.update_hard:
        if result_as_iff:
            return make_iff_instance(wm_path)
        return wm_url

    # create a folder for watermarks if needed
    wm_dir = os.path.dirname(wm_path)
    if not os.path.exists(wm_dir):
        os.makedirs(wm_dir)

    img_path = get_path(url)

    if not os.path.exists(img_path):
        return url

    try:
        img = Image.open(img_path)

        mark = Image.open(watermark.mark.path)
        x = watermark.x or 0
        y = watermark.y or 0
        shift = (x, y)
        position = watermark.position

        opacity = watermark.opacity

        marked_img = make_watermark(img, mark, position, opacity, shift)
        marked_img.save(wm_path)
    except Exception, e:
        error("Cant create watermark for %s. Error type: %s. Msg: %s" % (img_path, type(e), e))
        return url

    if result_as_iff:
        return make_iff_instance(wm_path)

    return wm_url

def get_url_safe(path):
    """ If we save wm with result_as_iff == True, we thumbnail returns us full path to picture (not valid url)
    may be there is prettier way to manage this

    """

    if not path.startswith(settings.MEDIA_URL):
        return os.path.join(settings.MEDIA_URL, path.split(settings.MEDIA_URL)[-1])
    return path
<|endoftext|>


<commit_before>import copy
import fractions
import functools
import math
import numbers
import re
import sys
import weakref

class UnitArithmetic:
  @classmethod
  def multiply(cls, first, second):
    result = first.copy()
    for unit, power in second.items():
      result[unit] = result.get(unit, 0) + power
    return cls.clean(result)

  @classmethod
  def divide(cls, first, second):
    result = first.copy()
    for unit, power in second.items():
      result[unit] = result.get(unit, 0) - power
    return cls.clean(result)

  @classmethod
  def power(cls, units, other):
    if other == 0:
      return {}
    else:
      return cls.clean({unit: power * other for unit, power in units.items()})

  @staticmethod
  def clean(units):
    result = {}
    for unit, power in units.items():
      if isinstance(power, fractions.Fraction):
        if power.denominator == 1:
          if not power.numerator == 0:
            result[unit] = power.numerator
        else:
          result[unit] = power
      else:
        rounded = round(power)
        # each operation contributes at most one epsilon of relative error;
        # assuming limited number of operations and powers close to unity,
        # this is a reasonable upper bound for the absolute error
        if abs(power - rounded) < 16 * sys.float_info.epsilon:
          if rounded != 0:
            result[unit] = rounded
        else:
          result[unit] = power
    return result

class UnitSystem:
  def __init__(self):
    self.units = {}
    self.constants = {}

  def add_unit(self, unit, symbol, expansion=None):
    if expansion is not None and expansion.system() is not self:
      raise TypeError('unit expansion is not in terms of system units')
    self.units[unit] = {'symbol': symbol, 'expansion': expansion}

  def add_constant(self, constant, symbol, definition):
    if definition.system() is not self:
      raise TypeError('constant definition is not in terms of system units')
    self.constants[constant] = {'symbol': symbol, 'definition': definition}

  def get_constant(self, arg):
    if isinstance(arg, dict):
      # argument is a dictionary of constants and their powers
      return functools.reduce(Quantity.__mul__,
        (self.constants[constant]['definition'] ** power
          for constant, power in arg.items()))
    else:
      # argument is the name of a constant
      return self.constants[arg]['definition']

  def expand_quantity(self, quantity):
    return (Quantity(quantity.value, quantity.error, {}, self) *
      self.expand_units(quantity.units))

  def expand_units(self, units):
    if not units:
      return Quantity(1, 0, {}, self)
    units = list(units.items())
    expansions = []
    while units:
      unit, power = units.pop()
      expansion = self.units.get(unit, {'expansion': None})['expansion']
      if expansion is None:
        expansions.append(Quantity(1, 0, {unit: power}, self))
      else:
        temp = expansion ** power
        units += list(temp.units.items())
        expansions.append(Quantity(temp.value, temp.error, {}, self))
    return functools.reduce(Quantity.__mul__, expansions)

  def format_quantity(self, quantity, format_spec):
    if quantity.units:
      units_string = self.format_units(quantity.units)
      if quantity.error == 0:
        return '{} {}'.format(quantity.value, units_string)
      else:
        value_error_string = self.format_value_error(
          quantity.value, quantity.error, format_spec, parens=True)
        return '{} {}'.format(value_error_string, units_string)
    else:
      if quantity.error == 0:
        return str(quantity.value)
      else:
        return self.format_value_error(
          quantity.value, quantity.error, format_spec)

  @staticmethod
  def format_value_error(value, error, format_spec, parens=False):
    # options: number of significant digits displayed for error
    #          separate value and error ('s') or parenthesize error ('p')
    match = re.fullmatch(r'(\d+)?([sp])?', format_spec)
    if not match:
      raise ValueError('invalid format specifier')
    error_sigfig, mode = 2, 's'
    temp = match.group(1)
    if temp is not None:
      temp = int(temp)
      if temp > 0:
        error_sigfig = temp
    temp = match.group(2)
    if temp is not None:
      mode = temp
    if not math.isfinite(value) or not math.isfinite(error):
      if mode == 's':
        result = '{} \xb1 {}'.format(value, error)
      else:
        result = '{}({})'.format(value, error)
      return '({})'.format(result) if parens else result
    # 'head' and 'tail' refer to the places (in positional notation) of the
    # highest and lowest displayed digits of a number
    error_head = math.floor(math.log10(error))
    if value != 0:
      value_head = math.floor(math.log10(abs(value)))
    else:
      value_head = error_head
    error_tail = error_head - error_sigfig + 1
    # handle the case when rounding changes the number of significant digits
    if round(abs(value), -error_tail) >= 10**(value_head+1):
      value_head += 1
    if mode == 's':
      dp = max(value_head - error_tail, 0)
      if value_head == 0:
        result = '{:.{dp}f} \xb1 {:.{dp}f}'.format(value, error, dp=dp)
        return '({})'.format(result) if parens else result
      else:
        temp1 = value / 10**value_head
        temp2 = error / 10**value_head
        return '({:.{dp}f} \xb1 {:.{dp}f})e{:+03}'.format(temp1, temp2,
          value_head, dp=dp)
    else:
      if value_head >= error_head:
        formatted_head = value_head
        dp = value_head - error_tail
      else:
        formatted_head = error_head
        dp = error_sigfig - 1
      temp1 = value / 10**formatted_head
      temp2 = round(error / 10**error_tail)
      if formatted_head == 0:
        return '{:.{dp}f}({})'.format(temp1, temp2, dp=dp)
      else:
        return '{:.{dp}f}({})e{}'.format(temp1, temp2, formatted_head, dp=dp)

  def format_units(self, units):
    return ' '.join(self.format_unit(unit, power)
      for unit, power in sorted(units.items(),
      key=lambda arg: (math.copysign(1, -arg[1]), arg[0])))

  def format_unit(self, unit, power):
    symbol = self.units.get(unit, {'symbol': unit})['symbol']
    if power == 1:
      return symbol
    elif power < 0 or isinstance(power, fractions.Fraction):
      return '{}^({})'.format(symbol, power)
    else:
      return '{}^{}'.format(symbol, power)

  def copy(self):
    result = copy.deepcopy(self)
    # copy.deepcopy updates the circular reference automatically, but not
    # circular weak references
    for data in result.units.values():
      if data['expansion'] is not None:
        data['expansion'].system = weakref.ref(result)
    for data in result.constants.values():
      data['definition'].system = weakref.ref(result)
    return result

class Quantity:
  def __init__(self, value, error, units, system):
    self.value = value
    self.error = error
    self.units = units
    if isinstance(system, weakref.ref):
      self.system = system
    else:
      self.system = weakref.ref(system)

  def __pos__(self):
    return Quantity(self.value, self.error, self.units, self.system)

  def __neg__(self):
    return Quantity(-self.value, self.error, self.units, self.system)

  def __abs__(self):
    return Quantity(abs(self.value), self.error, self.units, self.system)

  def __add__(self, other):
    if isinstance(other, Quantity) and self.system is other.system:
      first, second = self.expand(), other.expand()
      if first.units == second.units:
        value = first.value + second.value
        error = math.hypot(first.error, second.error)
        return Quantity(value, error, first.units, first.system)
    elif isinstance(other, numbers.Real):
      if other == 0:
        return +self
      else:
        first = self.expand()
        if not first.units:
          return Quantity(first.value + other, first.error, {}, first.system)
    return NotImplemented

  def __sub__(self, other):
    if isinstance(other, Quantity) and self.system is other.system:
      first, second = self.expand(), other.expand()
      if first.units == second.units:
        value = first.value - second.value
        error = math.hypot(first.error, second.error)
        return Quantity(value, error, first.units, first.system)
    elif isinstance(other, numbers.Real):
      if other == 0:
        return +self
      else:
        first = self.expand()
        if not first.units:
          return Quantity(first.value - other, first.error, {}, first.system)
    return NotImplemented

  def __mul__(self, other):
    if isinstance(other, Quantity) and self.system is other.system:
      value = self.value * other.value
      error = math.hypot(self.error * other.value, other.error * self.value)
      units = UnitArithmetic.multiply(self.units, other.units)
      return Quantity(value, error, units, self.system)
    elif isinstance(other, numbers.Real):
      value = self.value * other
      error = abs(self.error * other)
      return Quantity(value, error, self.units, self.system)
    else:
      return NotImplemented

  def __truediv__(self, other):
    if isinstance(other, Quantity) and self.system is other.system:
      value = self.value / other.value
      error = math.hypot(self.error / other.value,
        other.error * self.value / other.value**2)
      units = UnitArithmetic.divide(self.units, other.units)
      return Quantity(value, error, units, self.system)
    elif isinstance(other, numbers.Real):
      value = self.value / other
      error = abs(self.error / other)
      return Quantity(value, error, self.units, self.system)
    else:
      return NotImplemented

  def __pow__(self, other):
    if isinstance(other, Quantity) and self.system is other.system:
      first, second = self.expand(), other.expand()
      if not first.units and not second.units:
        value = first.value ** second.value
        error = value * math.hypot(
          second.value / first.value * first.error,
          math.log(first.value) * second.error)
        return Quantity(value, error, {}, first.system)
      elif first.units and second.error == 0 and not second.units:
        return first ** second.value
    elif isinstance(other, numbers.Real):
      value = self.value ** other
      error = abs(other * value / self.value * self.error)
      units = UnitArithmetic.power(self.units, other)
      return Quantity(value, error, units, self.system)
    return NotImplemented

  def __radd__(self, other):
    if isinstance(other, numbers.Real):
      return self + other
    else:
      return NotImplemented

  def __rsub__(self, other):
    if isinstance(other, numbers.Real):
      return -self + other
    else:
      return NotImplemented

  def __rmul__(self, other):
    if isinstance(other, numbers.Real):
      return self * other
    else:
      return NotImplemented

  def __rtruediv__(self, other):
    if isinstance(other, numbers.Real):
      return self**-1 * other
    else:
      return NotImplemented

  def __rpow__(self, other):
    if isinstance(other, numbers.Real):
      second = self.expand()
      if not second.units:
        return Quantity(other, 0, {}, second.system) ** second
    return NotImplemented

  def __eq__(self, other):
    return (isinstance(other, Quantity) and
      self.value == other.value and self.error == other.error and
      self.units == other.units and self.system is other.system)

  def __ne__(self, other):
    return not self == other

  def __hash__(self):
    return hash((self.value, self.error,
      tuple(sorted(self.units.items())), self.system))

  def almost_equals(self, other):
    if isinstance(other, Quantity) and self.system is other.system:
      first, second = self.expand(), other.expand()
      if first.units == second.units:
        return abs(first.value - second.value) <= first.error + second.error
    elif isinstance(other, numbers.Real):
      first = self.expand()
      if not first.units:
        return abs(first.value - other) < first.error
    raise TypeError('quantities are incomparable: {} and {}'
      .format(self, other))

  def __float__(self):
    first = self.expand()
    if first.error == 0 and not first.units:
      return float(first.value)
    else:
      raise TypeError('quantity has either error or units')

  def expand(self):
    if self.system is not None:
      return self.system().expand_quantity(self)
    else:
      return self

  def copy(self):
    return Quantity(self.value, self.error, self.units.copy(), self.system)

  def __repr__(self):
    kwargs = ', '.join('{}={}'.format(key, repr(self.__dict__[key]))
      for key in ['value', 'error', 'units', 'system'])
    return '{}({})'.format(self.__class__.__name__, kwargs)

  def __str__(self):
    return format(self, '2s')

  def __format__(self, format_spec):
    return self.system().format_quantity(self, format_spec)

  @staticmethod
  def make_same_system(args):
    if all(isinstance(arg, numbers.Real) for arg in args):
      return args
    if not all(isinstance(arg, (Quantity, numbers.Real)) for arg in args):
      raise TypeError('arguments are not all quantities or numbers')
    systems = [arg.system for arg in args if isinstance(arg, Quantity)]
    if not all(system is systems[0] for system in systems):
      raise TypeError('arguments do not have the same system')
    return [Quantity(arg, 0, {}, systems[0]) if isinstance(arg, numbers.Real)
      else arg for arg in args]
<commit_msg>Fix error propagation when value is zero<commit_after>import copy
import fractions
import functools
import math
import numbers
import re
import sys
import weakref

class UnitArithmetic:
  @classmethod
  def multiply(cls, first, second):
    result = first.copy()
    for unit, power in second.items():
      result[unit] = result.get(unit, 0) + power
    return cls.clean(result)

  @classmethod
  def divide(cls, first, second):
    result = first.copy()
    for unit, power in second.items():
      result[unit] = result.get(unit, 0) - power
    return cls.clean(result)

  @classmethod
  def power(cls, units, other):
    if other == 0:
      return {}
    else:
      return cls.clean({unit: power * other for unit, power in units.items()})

  @staticmethod
  def clean(units):
    result = {}
    for unit, power in units.items():
      if isinstance(power, fractions.Fraction):
        if power.denominator == 1:
          if not power.numerator == 0:
            result[unit] = power.numerator
        else:
          result[unit] = power
      else:
        rounded = round(power)
        # each operation contributes at most one epsilon of relative error;
        # assuming limited number of operations and powers close to unity,
        # this is a reasonable upper bound for the absolute error
        if abs(power - rounded) < 16 * sys.float_info.epsilon:
          if rounded != 0:
            result[unit] = rounded
        else:
          result[unit] = power
    return result

class UnitSystem:
  def __init__(self):
    self.units = {}
    self.constants = {}

  def add_unit(self, unit, symbol, expansion=None):
    if expansion is not None and expansion.system() is not self:
      raise TypeError('unit expansion is not in terms of system units')
    self.units[unit] = {'symbol': symbol, 'expansion': expansion}

  def add_constant(self, constant, symbol, definition):
    if definition.system() is not self:
      raise TypeError('constant definition is not in terms of system units')
    self.constants[constant] = {'symbol': symbol, 'definition': definition}

  def get_constant(self, arg):
    if isinstance(arg, dict):
      # argument is a dictionary of constants and their powers
      return functools.reduce(Quantity.__mul__,
        (self.constants[constant]['definition'] ** power
          for constant, power in arg.items()))
    else:
      # argument is the name of a constant
      return self.constants[arg]['definition']

  def expand_quantity(self, quantity):
    return (Quantity(quantity.value, quantity.error, {}, self) *
      self.expand_units(quantity.units))

  def expand_units(self, units):
    if not units:
      return Quantity(1, 0, {}, self)
    units = list(units.items())
    expansions = []
    while units:
      unit, power = units.pop()
      expansion = self.units.get(unit, {'expansion': None})['expansion']
      if expansion is None:
        expansions.append(Quantity(1, 0, {unit: power}, self))
      else:
        temp = expansion ** power
        units += list(temp.units.items())
        expansions.append(Quantity(temp.value, temp.error, {}, self))
    return functools.reduce(Quantity.__mul__, expansions)

  def format_quantity(self, quantity, format_spec):
    if quantity.units:
      units_string = self.format_units(quantity.units)
      if quantity.error == 0:
        return '{} {}'.format(quantity.value, units_string)
      else:
        value_error_string = self.format_value_error(
          quantity.value, quantity.error, format_spec, parens=True)
        return '{} {}'.format(value_error_string, units_string)
    else:
      if quantity.error == 0:
        return str(quantity.value)
      else:
        return self.format_value_error(
          quantity.value, quantity.error, format_spec)

  @staticmethod
  def format_value_error(value, error, format_spec, parens=False):
    # options: number of significant digits displayed for error
    #          separate value and error ('s') or parenthesize error ('p')
    match = re.fullmatch(r'(\d+)?([sp])?', format_spec)
    if not match:
      raise ValueError('invalid format specifier')
    error_sigfig, mode = 2, 's'
    temp = match.group(1)
    if temp is not None:
      temp = int(temp)
      if temp > 0:
        error_sigfig = temp
    temp = match.group(2)
    if temp is not None:
      mode = temp
    if not math.isfinite(value) or not math.isfinite(error):
      if mode == 's':
        result = '{} \xb1 {}'.format(value, error)
      else:
        result = '{}({})'.format(value, error)
      return '({})'.format(result) if parens else result
    # 'head' and 'tail' refer to the places (in positional notation) of the
    # highest and lowest displayed digits of a number
    error_head = math.floor(math.log10(error))
    if value != 0:
      value_head = math.floor(math.log10(abs(value)))
    else:
      value_head = error_head
    error_tail = error_head - error_sigfig + 1
    # handle the case when rounding changes the number of significant digits
    if round(abs(value), -error_tail) >= 10**(value_head+1):
      value_head += 1
    if mode == 's':
      dp = max(value_head - error_tail, 0)
      if value_head == 0:
        result = '{:.{dp}f} \xb1 {:.{dp}f}'.format(value, error, dp=dp)
        return '({})'.format(result) if parens else result
      else:
        temp1 = value / 10**value_head
        temp2 = error / 10**value_head
        return '({:.{dp}f} \xb1 {:.{dp}f})e{:+03}'.format(temp1, temp2,
          value_head, dp=dp)
    else:
      if value_head >= error_head:
        formatted_head = value_head
        dp = value_head - error_tail
      else:
        formatted_head = error_head
        dp = error_sigfig - 1
      temp1 = value / 10**formatted_head
      temp2 = round(error / 10**error_tail)
      if formatted_head == 0:
        return '{:.{dp}f}({})'.format(temp1, temp2, dp=dp)
      else:
        return '{:.{dp}f}({})e{}'.format(temp1, temp2, formatted_head, dp=dp)

  def format_units(self, units):
    return ' '.join(self.format_unit(unit, power)
      for unit, power in sorted(units.items(),
      key=lambda arg: (math.copysign(1, -arg[1]), arg[0])))

  def format_unit(self, unit, power):
    symbol = self.units.get(unit, {'symbol': unit})['symbol']
    if power == 1:
      return symbol
    elif power < 0 or isinstance(power, fractions.Fraction):
      return '{}^({})'.format(symbol, power)
    else:
      return '{}^{}'.format(symbol, power)

  def copy(self):
    result = copy.deepcopy(self)
    # copy.deepcopy updates the circular reference automatically, but not
    # circular weak references
    for data in result.units.values():
      if data['expansion'] is not None:
        data['expansion'].system = weakref.ref(result)
    for data in result.constants.values():
      data['definition'].system = weakref.ref(result)
    return result

class Quantity:
  def __init__(self, value, error, units, system):
    self.value = value
    self.error = error
    self.units = units
    if isinstance(system, weakref.ref):
      self.system = system
    else:
      self.system = weakref.ref(system)

  def __pos__(self):
    return Quantity(self.value, self.error, self.units, self.system)

  def __neg__(self):
    return Quantity(-self.value, self.error, self.units, self.system)

  def __abs__(self):
    return Quantity(abs(self.value), self.error, self.units, self.system)

  def __add__(self, other):
    if isinstance(other, Quantity) and self.system is other.system:
      first, second = self.expand(), other.expand()
      if first.units == second.units:
        value = first.value + second.value
        error = math.hypot(first.error, second.error)
        return Quantity(value, error, first.units, first.system)
    elif isinstance(other, numbers.Real):
      if other == 0:
        return +self
      else:
        first = self.expand()
        if not first.units:
          return Quantity(first.value + other, first.error, {}, first.system)
    return NotImplemented

  def __sub__(self, other):
    if isinstance(other, Quantity) and self.system is other.system:
      first, second = self.expand(), other.expand()
      if first.units == second.units:
        value = first.value - second.value
        error = math.hypot(first.error, second.error)
        return Quantity(value, error, first.units, first.system)
    elif isinstance(other, numbers.Real):
      if other == 0:
        return +self
      else:
        first = self.expand()
        if not first.units:
          return Quantity(first.value - other, first.error, {}, first.system)
    return NotImplemented

  def __mul__(self, other):
    if isinstance(other, Quantity) and self.system is other.system:
      value = self.value * other.value
      error = math.hypot(self.error * other.value, other.error * self.value)
      units = UnitArithmetic.multiply(self.units, other.units)
      return Quantity(value, error, units, self.system)
    elif isinstance(other, numbers.Real):
      value = self.value * other
      error = abs(self.error * other)
      return Quantity(value, error, self.units, self.system)
    else:
      return NotImplemented

  def __truediv__(self, other):
    if isinstance(other, Quantity) and self.system is other.system:
      value = self.value / other.value
      error = math.hypot(self.error / other.value,
        other.error * self.value / other.value**2)
      units = UnitArithmetic.divide(self.units, other.units)
      return Quantity(value, error, units, self.system)
    elif isinstance(other, numbers.Real):
      value = self.value / other
      error = abs(self.error / other)
      return Quantity(value, error, self.units, self.system)
    else:
      return NotImplemented

  def __pow__(self, other):
    if isinstance(other, Quantity) and self.system is other.system:
      first, second = self.expand(), other.expand()
      if not first.units and not second.units:
        value = first.value ** second.value
        if first.value != 0:
          error = value * math.hypot(
            second.value / first.value * first.error,
            math.log(first.value) * second.error)
        else:
          error = first.error ** second.value
        return Quantity(value, error, {}, first.system)
      elif first.units and second.error == 0 and not second.units:
        return first ** second.value
    elif isinstance(other, numbers.Real):
      value = self.value ** other
      if self.value != 0:
        error = abs(other * value / self.value * self.error)
      else:
        error = self.error ** other
      units = UnitArithmetic.power(self.units, other)
      return Quantity(value, error, units, self.system)
    return NotImplemented

  def __radd__(self, other):
    if isinstance(other, numbers.Real):
      return self + other
    else:
      return NotImplemented

  def __rsub__(self, other):
    if isinstance(other, numbers.Real):
      return -self + other
    else:
      return NotImplemented

  def __rmul__(self, other):
    if isinstance(other, numbers.Real):
      return self * other
    else:
      return NotImplemented

  def __rtruediv__(self, other):
    if isinstance(other, numbers.Real):
      return self**-1 * other
    else:
      return NotImplemented

  def __rpow__(self, other):
    if isinstance(other, numbers.Real):
      second = self.expand()
      if not second.units:
        return Quantity(other, 0, {}, second.system) ** second
    return NotImplemented

  def __eq__(self, other):
    return (isinstance(other, Quantity) and
      self.value == other.value and self.error == other.error and
      self.units == other.units and self.system is other.system)

  def __ne__(self, other):
    return not self == other

  def __hash__(self):
    return hash((self.value, self.error,
      tuple(sorted(self.units.items())), self.system))

  def almost_equals(self, other):
    if isinstance(other, Quantity) and self.system is other.system:
      first, second = self.expand(), other.expand()
      if first.units == second.units:
        return abs(first.value - second.value) <= first.error + second.error
    elif isinstance(other, numbers.Real):
      first = self.expand()
      if not first.units:
        return abs(first.value - other) < first.error
    raise TypeError('quantities are incomparable: {} and {}'
      .format(self, other))

  def __float__(self):
    first = self.expand()
    if first.error == 0 and not first.units:
      return float(first.value)
    else:
      raise TypeError('quantity has either error or units')

  def expand(self):
    if self.system is not None:
      return self.system().expand_quantity(self)
    else:
      return self

  def copy(self):
    return Quantity(self.value, self.error, self.units.copy(), self.system)

  def __repr__(self):
    kwargs = ', '.join('{}={}'.format(key, repr(self.__dict__[key]))
      for key in ['value', 'error', 'units', 'system'])
    return '{}({})'.format(self.__class__.__name__, kwargs)

  def __str__(self):
    return format(self, '2s')

  def __format__(self, format_spec):
    return self.system().format_quantity(self, format_spec)

  @staticmethod
  def make_same_system(args):
    if all(isinstance(arg, numbers.Real) for arg in args):
      return args
    if not all(isinstance(arg, (Quantity, numbers.Real)) for arg in args):
      raise TypeError('arguments are not all quantities or numbers')
    systems = [arg.system for arg in args if isinstance(arg, Quantity)]
    if not all(system is systems[0] for system in systems):
      raise TypeError('arguments do not have the same system')
    return [Quantity(arg, 0, {}, systems[0]) if isinstance(arg, numbers.Real)
      else arg for arg in args]
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- encoding: utf-8 -*-
import json
import requests

from decimal import Decimal
from bs4 import BeautifulSoup
from datetime import datetime

def decimal_default(obj):
    if isinstance(obj, Decimal):
        return float(obj)
    raise TypeError

def chaco():
    try:
        soup = BeautifulSoup(
            requests.get('http://www.cambioschaco.com.py/', timeout=10).text, "html.parser")
        compra = soup.find_all(class_='purchase')[0].contents[0].replace('.','')
        venta = soup.find_all(class_='sale')[0].contents[0].replace('.','')
    except requests.ConnectionError:
        compra, venta = 0, 0
    except:
        compra, venta = 0, 0

    return Decimal(compra), Decimal(venta)

def maxi():
    try:
        soup = BeautifulSoup(
            requests.get('http://www.maxicambios.com.py/', timeout=10).text, "html.parser")
        compra = soup.find_all(class_='lineas1')[0].contents[
            7].string.replace('.', '')
        venta = soup.find_all(class_='lineas1')[0].contents[
            5].string.replace('.', '')
    except requests.ConnectionError:
        compra, venta = 0, 0
    except:
        compra, venta = 0, 0

    return Decimal(compra), Decimal(venta)

def alberdi():
    try:
        soup = json.loads(
            requests.get('http://www.cambiosalberdi.com/customscripts/ajax/getCotizaciones.php', timeout=10).text)
        compra =  soup['asuncion'][0]['compra'].replace('.','')
        venta = soup['asuncion'][0]['venta'].replace('.','')
    except requests.ConnectionError:
        compra, venta = 0, 0
    except:
        compra, venta = 0, 0

    return Decimal(compra), Decimal(venta)

def bcp():
    try:
        soup = BeautifulSoup(
            requests.get('https://www.bcp.gov.py/webapps/web/cotizacion/monedas', timeout=10,
                         headers={'user-agent': 'Mozilla/5.0'}).text, "html.parser")
        ref = soup.select(
            '#cotizacion-interbancaria > tbody > tr > td:nth-of-type(4)')[0].get_text()
        ref = ref.replace('.', '').replace(',', '.')
        soup = BeautifulSoup(
            requests.get('https://www.bcp.gov.py/webapps/web/cotizacion/referencial-fluctuante', timeout=10,
                         headers={'user-agent': 'Mozilla/5.0'}).text, "html.parser")
        compra_array = soup.find(
            class_="table table-striped table-bordered table-condensed").select('tr > td:nth-of-type(4)')
        venta_array = soup.find(
            class_="table table-striped table-bordered table-condensed").select('tr > td:nth-of-type(5)')
        posicion = len(compra_array) - 1
        compra = compra_array[posicion].get_text(
        ).replace('.', '').replace(',', '.')
        venta = venta_array[posicion].get_text().replace(
            '.', '').replace(',', '.')
    except requests.ConnectionError:
        compra, venta, ref = 0, 0, 0
    except:
        compra, venta, ref = 0, 0, 0

    return Decimal(compra), Decimal(venta), Decimal(ref)

def setgov():
    try:
        soup = BeautifulSoup(
            requests.get('http://www.set.gov.py/portal/PARAGUAY-SET', timeout=10).text, "html.parser")
        compra = soup.find_all(
            'span', style="font-family:arial,helvetica,sans-serif;").pop(0).select('span')[0].contents[
                     0].string.replace('.', '')[2::]
        venta = soup.find_all(
            'span', style="font-family:arial,helvetica,sans-serif;").pop(1).select('span')[0].contents[
                    0].string.replace('.', '')[2::]
    except requests.ConnectionError:
        compra, venta = 0, 0
    except:
        compra, venta = 0, 0

    return Decimal(compra), Decimal(venta)

def create_json():
    mcompra, mventa = maxi()
    ccompra, cventa = chaco()
    acompra, aventa = alberdi()
    bcpcompra, bcpventa, bcpref = bcp()
    setcompra, setventa = setgov()
    respjson = {
        'dolarpy': {
            'cambiosalberdi': {
                'compra': acompra,
                'venta': aventa
            },
            'cambioschaco': {
                'compra': ccompra,
                'venta': cventa
            },
            'maxicambios': {
                'compra': mcompra,
                'venta': mventa
            },
            'bcp': {
                'compra': bcpcompra,
                'venta': bcpventa,
                'referencial_diario': bcpref
            },
            'set': {
                'compra': setcompra,
                'venta': setventa
            },
        },
        "updated": datetime.now().strftime('%Y-%m-%d %H:%M:%S')
    }
    return json.dumps(respjson, indent=4, sort_keys=True, separators=(',', ': '), default=decimal_default)

def get_output():
    with open('dolar.json', 'r') as f:
        response = f.read()
    return response

def write_output():
    response = create_json()
    with open('dolar.json', 'w') as f:
        f.write(response)

write_output()
<commit_msg>update to use cambios chaco 'API'<commit_after>#!/usr/bin/env python
# -*- encoding: utf-8 -*-
import json
import requests

from decimal import Decimal
from bs4 import BeautifulSoup
from datetime import datetime

def decimal_default(obj):
    if isinstance(obj, Decimal):
        return float(obj)
    raise TypeError

def chaco():
    try:
        soup = json.loads(
            requests.get('http://www.cambioschaco.com.py/api/branch_office/1/exchange', timeout=10).text)
        compra = soup['items'][0]['purchasePrice']
        venta =  soup['items'][0]['salePrice']
    except requests.ConnectionError:
        compra, venta = 0, 0
    except:
        compra, venta = 0, 0

    return Decimal(compra), Decimal(venta)

def maxi():
    try:
        soup = BeautifulSoup(
            requests.get('http://www.maxicambios.com.py/', timeout=10).text, "html.parser")
        compra = soup.find_all(class_='lineas1')[0].contents[
            7].string.replace('.', '')
        venta = soup.find_all(class_='lineas1')[0].contents[
            5].string.replace('.', '')
    except requests.ConnectionError:
        compra, venta = 0, 0
    except:
        compra, venta = 0, 0

    return Decimal(compra), Decimal(venta)

def alberdi():
    try:
        soup = json.loads(
            requests.get('http://www.cambiosalberdi.com/customscripts/ajax/getCotizaciones.php', timeout=10).text)
        compra =  soup['asuncion'][0]['compra'].replace('.','')
        venta = soup['asuncion'][0]['venta'].replace('.','')
    except requests.ConnectionError:
        compra, venta = 0, 0
    except:
        compra, venta = 0, 0

    return Decimal(compra), Decimal(venta)

def bcp():
    try:
        soup = BeautifulSoup(
            requests.get('https://www.bcp.gov.py/webapps/web/cotizacion/monedas', timeout=10,
                         headers={'user-agent': 'Mozilla/5.0'}).text, "html.parser")
        ref = soup.select(
            '#cotizacion-interbancaria > tbody > tr > td:nth-of-type(4)')[0].get_text()
        ref = ref.replace('.', '').replace(',', '.')
        soup = BeautifulSoup(
            requests.get('https://www.bcp.gov.py/webapps/web/cotizacion/referencial-fluctuante', timeout=10,
                         headers={'user-agent': 'Mozilla/5.0'}).text, "html.parser")
        compra_array = soup.find(
            class_="table table-striped table-bordered table-condensed").select('tr > td:nth-of-type(4)')
        venta_array = soup.find(
            class_="table table-striped table-bordered table-condensed").select('tr > td:nth-of-type(5)')
        posicion = len(compra_array) - 1
        compra = compra_array[posicion].get_text(
        ).replace('.', '').replace(',', '.')
        venta = venta_array[posicion].get_text().replace(
            '.', '').replace(',', '.')
    except requests.ConnectionError:
        compra, venta, ref = 0, 0, 0
    except:
        compra, venta, ref = 0, 0, 0

    return Decimal(compra), Decimal(venta), Decimal(ref)

def setgov():
    try:
        soup = BeautifulSoup(
            requests.get('http://www.set.gov.py/portal/PARAGUAY-SET', timeout=10).text, "html.parser")
        compra = soup.find_all(
            'span', style="font-family:arial,helvetica,sans-serif;").pop(0).select('span')[0].contents[
                     0].string.replace('.', '')[2::]
        venta = soup.find_all(
            'span', style="font-family:arial,helvetica,sans-serif;").pop(1).select('span')[0].contents[
                    0].string.replace('.', '')[2::]
    except requests.ConnectionError:
        compra, venta = 0, 0
    except:
        compra, venta = 0, 0

    return Decimal(compra), Decimal(venta)

def create_json():
    mcompra, mventa = maxi()
    ccompra, cventa = chaco()
    acompra, aventa = alberdi()
    bcpcompra, bcpventa, bcpref = bcp()
    setcompra, setventa = setgov()
    respjson = {
        'dolarpy': {
            'cambiosalberdi': {
                'compra': acompra,
                'venta': aventa
            },
            'cambioschaco': {
                'compra': ccompra,
                'venta': cventa
            },
            'maxicambios': {
                'compra': mcompra,
                'venta': mventa
            },
            'bcp': {
                'compra': bcpcompra,
                'venta': bcpventa,
                'referencial_diario': bcpref
            },
            'set': {
                'compra': setcompra,
                'venta': setventa
            },
        },
        "updated": datetime.now().strftime('%Y-%m-%d %H:%M:%S')
    }
    return json.dumps(respjson, indent=4, sort_keys=True, separators=(',', ': '), default=decimal_default)

def get_output():
    with open('dolar.json', 'r') as f:
        response = f.read()
    return response

def write_output():
    response = create_json()
    with open('dolar.json', 'w') as f:
        f.write(response)

write_output()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# ---------------------
# The MIT License (MIT)
#
# Copyright (c) 2013 Francis Hart
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
# the Software, and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
# ---------------------
#
# Written by Francis Hart
# http://hartcw.com
# http://github.com/hartcw/cppy

import io
import os
import re
import sys
import argparse
import platform
import builtins

class options:
    input  = ''
    output = ''
    type   = 'auto'
    debug  = False

class context:
    indent     = ''
    buffer     = None
    first      = True
    variables  = dict()
    colour_log = platform.system() != 'Windows'

def parse_options():
    parser = argparse.ArgumentParser('cppy')
    parser.add_argument('-i', '--input',
                        help='Specify the input file (defaults to stdin)')
    parser.add_argument('-o', '--output',
                        help='Specify the output file (defaults to stdout)')
    parser.add_argument('-t', '--type',
                        choices=['c', 'xml', 'auto'],
                        help='Set the type of embedded comment to match and expand (defaults to auto)')
    parser.add_argument('-d', '--debug', action='store_true',
                        help='Enable debug information')
    parser.parse_args(namespace=options)

def log_set_colour(code):
    if context.colour_log:
        print('\033[' + str(code) + ';1m', end='', file=sys.stdout)

def log_unset_colour():
    if context.colour_log:
        print('\033[0m', end='', file=sys.stdout)
        sys.stdout.flush()

def log_error(message):
    log_set_colour(31)
    print('ERROR: ', message, file=sys.stdout)
    log_unset_colour()
    exit()

def log_warning(message):
    log_set_colour(33)
    print('WARNING: ', message, file=sys.stdout)
    log_unset_colour()

def log_message(message):
    log_set_colour(32)
    print(message, file=sys.stdout)
    log_unset_colour()

def log_debug(message):
    if options.debug:
        log_set_colour(32)
        print('DEBUG:', message, file=sys.stdout)
        log_unset_colour()

def print(*objects, sep=' ', end='\n', file=None, flush=False):
    if file == None:
        file = context.buffer
    if context.first:
        context.first = False
        text = sep.join(objects)
    else:
        text = context.indent + sep.join(objects)
    builtins.print(text, end=end, file=file)

def execute(code, indent, first):
    buffer = io.StringIO()
    indent = re.sub(r'[^\t]', ' ', indent)
    code = re.sub(r'^' + indent, '', code, flags=re.MULTILINE)
    log_debug('INDENT'+indent+'INDENTEND')
    log_debug('CODE\n' + code + '\nCODEEND')
    prev_buffer = context.buffer
    prev_indent = context.indent
    prev_first = context.first
    context.buffer = buffer
    context.indent = indent
    context.first = first
    exec(code, globals(), context.variables)
    context.buffer = prev_buffer
    context.indent = prev_indent
    context.first = prev_first
    output = buffer.getvalue().rstrip('\n')
    log_debug('OUTPUT\n' + output + '\nOUTPUTEND')
    return output

def match_with_indent(match):
    matches = list(match.groups())
    indent = matches[0]
    i = 1
    code = ''
    while i < len(matches):
        if matches[i] != None:
            code += matches[i]
        i += 1
    return execute(code, indent, False)

def match_without_indent(match):
    prefix = match.string[0:match.start()]
    indent = prefix[prefix.rfind('\n') + 1:]
    code = ''
    for m in list(match.groups()):
        if m != None:
            code += m
    return execute(code, indent, True)

def expand(input='', output='', type='auto', newline=None):
    log_debug('expand input=' + input + ' output=' + output)
    cwd = os.getcwd()
    if newline == None:
        newline = os.linesep
    if output == '':
        if context.buffer != None:
            dst = context.buffer
        else:
            dst = sys.stdout
    else:
        if not os.path.exists(os.path.dirname(output)):
            os.mkdir(os.path.dirname(output))
        dst = open(output, 'wt', newline=newline)
    if input == '':
        src = sys.stdin
    else:
        src = open(input, 'rt')
        dirname = os.path.dirname(input)
        if dirname != '':
            os.chdir(dirname)
    contents = src.read()
    regexes = [
        (re.compile(r'^([ \t]*)#[ \t]*py[ \t]*(?:(.*)\\(\n))*(.*)$', re.MULTILINE), match_with_indent),
        (re.compile(r'<!--[ \t]*py[ \t]*(.*?)[ \t]*-->', re.DOTALL | re.MULTILINE), match_without_indent),
        (re.compile(r'/\*\**[ \t]*py[ \t]*(.*?)[ \t]*\*/', re.DOTALL | re.MULTILINE), match_without_indent),
        (re.compile(r'//[ \t]*py[ \t]*(.*)$', re.MULTILINE), match_without_indent)
    ]
    for regex, match_function in regexes:
        contents = regex.sub(match_function, contents)
    dst.write(contents)
    os.chdir(cwd)
    if output != '':
        dst.close()
    if input != '':
        src.close()

def main():
    parse_options()
    expand(options.input, options.output, options.type)

if __name__ == '__main__':
    main()
<commit_msg>Add ability to query current src and dst dirs<commit_after>#!/usr/bin/env python3
#
# ---------------------
# The MIT License (MIT)
#
# Copyright (c) 2013 Francis Hart
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
# the Software, and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
# ---------------------
#
# Written by Francis Hart
# http://hartcw.com
# http://github.com/hartcw/cppy

import io
import os
import re
import sys
import argparse
import platform
import builtins

class options:
    input  = ''
    output = ''
    type   = 'auto'
    debug  = False

class context:
    indent     = ''
    srcdir     = ''
    dstdir     = ''
    buffer     = None
    first      = True
    variables  = dict()
    colour_log = platform.system() != 'Windows'

def parse_options():
    parser = argparse.ArgumentParser('cppy')
    parser.add_argument('-i', '--input',
                        help='Specify the input file (defaults to stdin)')
    parser.add_argument('-o', '--output',
                        help='Specify the output file (defaults to stdout)')
    parser.add_argument('-t', '--type',
                        choices=['c', 'xml', 'auto'],
                        help='Set the type of embedded comment to match and expand (defaults to auto)')
    parser.add_argument('-d', '--debug', action='store_true',
                        help='Enable debug information')
    parser.parse_args(namespace=options)

def log_set_colour(code):
    if context.colour_log:
        print('\033[' + str(code) + ';1m', end='', file=sys.stdout)

def log_unset_colour():
    if context.colour_log:
        print('\033[0m', end='', file=sys.stdout)
        sys.stdout.flush()

def log_error(message):
    log_set_colour(31)
    print('ERROR: ', message, file=sys.stdout)
    log_unset_colour()
    exit()

def log_warning(message):
    log_set_colour(33)
    print('WARNING: ', message, file=sys.stdout)
    log_unset_colour()

def log_message(message):
    log_set_colour(32)
    print(message, file=sys.stdout)
    log_unset_colour()

def log_debug(message):
    if options.debug:
        log_set_colour(32)
        print('DEBUG:', message, file=sys.stdout)
        log_unset_colour()

def print(*objects, sep=' ', end='\n', file=None, flush=False):
    if file == None:
        file = context.buffer
    if context.first:
        context.first = False
        text = sep.join(objects)
    else:
        text = context.indent + sep.join(objects)
    builtins.print(text, end=end, file=file)

def execute(code, indent, first):
    buffer = io.StringIO()
    indent = re.sub(r'[^\t]', ' ', indent)
    code = re.sub(r'^' + indent, '', code, flags=re.MULTILINE)
    log_debug('INDENT'+indent+'INDENTEND')
    log_debug('CODE\n' + code + '\nCODEEND')
    prev_buffer = context.buffer
    prev_indent = context.indent
    prev_first = context.first
    context.buffer = buffer
    context.indent = indent
    context.first = first
    exec(code, globals(), context.variables)
    context.buffer = prev_buffer
    context.indent = prev_indent
    context.first = prev_first
    output = buffer.getvalue().rstrip('\n')
    log_debug('OUTPUT\n' + output + '\nOUTPUTEND')
    return output

def match_with_indent(match):
    matches = list(match.groups())
    indent = matches[0]
    i = 1
    code = ''
    while i < len(matches):
        if matches[i] != None:
            code += matches[i]
        i += 1
    return execute(code, indent, False)

def match_without_indent(match):
    prefix = match.string[0:match.start()]
    indent = prefix[prefix.rfind('\n') + 1:]
    code = ''
    for m in list(match.groups()):
        if m != None:
            code += m
    return execute(code, indent, True)

def srcdir():
    return context.srcdir

def dstdir():
    return context.dstdir

def expand(input='', output='', type='auto', newline=None):
    log_debug('expand input=' + input + ' output=' + output)
    cwd = os.getcwd()
    prev_srcdir = context.srcdir
    prev_dstdir = context.dstdir
    if newline == None:
        newline = os.linesep
    if output == '':
        if context.buffer != None:
            dst = context.buffer
        else:
            dst = sys.stdout
    else:
        if not os.path.exists(os.path.dirname(output)):
            os.mkdir(os.path.dirname(output))
        dst = open(output, 'wt', newline=newline)
        context.dstdir = os.path.abspath(os.path.dirname(output))
    if input == '':
        src = sys.stdin
    else:
        src = open(input, 'rt')
        context.srcdir = os.path.abspath(os.path.dirname(input))
        dirname = os.path.dirname(input)
        if dirname != '':
            os.chdir(dirname)
    contents = src.read()
    regexes = [
        (re.compile(r'^([ \t]*)#[ \t]*py[ \t]*(?:(.*)\\(\n))*(.*)$', re.MULTILINE), match_with_indent),
        (re.compile(r'<!--[ \t]*py[ \t]*(.*?)[ \t]*-->', re.DOTALL | re.MULTILINE), match_without_indent),
        (re.compile(r'/\*\**[ \t]*py[ \t]*(.*?)[ \t]*\*/', re.DOTALL | re.MULTILINE), match_without_indent),
        (re.compile(r'//[ \t]*py[ \t]*(.*)$', re.MULTILINE), match_without_indent)
    ]
    for regex, match_function in regexes:
        contents = regex.sub(match_function, contents)
    dst.write(contents)
    os.chdir(cwd)
    context.srcdir = prev_srcdir
    context.dstdir = prev_dstdir
    if output != '':
        dst.close()
    if input != '':
        src.close()

def main():
    parse_options()
    expand(options.input, options.output, options.type)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#! /usr/bin/python
import logging
from datetime import datetime
from itertools import ifilter

import settings
from model import User, Task
from library.template import Template
from library.mailclient import MailClient, Subjects

class Cron(object):
    def __init__(self):
        self.now = datetime.now()
        self.template = Template()
        self.mailclient = MailClient()

        if settings.DEBUG:
            self.mail_buffer = []

    def job(self):
        users = User.select().where(User.is_active == True)
        print "Users: " + [user.name for user in users]
        for user in ifilter(lambda user: user.is_due(self.now), users):
            print "Creating digest for {0}".format(user.name)
            try:
                digest = self.create_digest(user)
                print "Successfully created digest for {0}".format(user.name)
                email = self.mailclient.send(user, digest, Subjects.DIGEST)
                print "Sent digest to {0}".format(user.name)
            except:
                print "Couldn't send digest to {0}".format(user.name)
                logging.error("Failed to send email to {0}".format(user.name))
                if settings.DEBUG:
                    raise

            if settings.DEBUG:
                print user.name
                print email, '\n\n'
            else:
                user.last_sent = self.now
                user.save()

    def create_digest(self, user):
        last_sent = user.last_sent or user.created
        tasks = Task.select()\
            .where((Task.date > last_sent))\
            .order_by(Task.user.name, Task.date)
        email_params = {
            'user': user,
            'start': last_sent,
            'end': self.now,
            'tasks': tasks,
            'sender': settings.SENDER
        }
        return self.template.render('email_digest.tpl', **email_params)

# Cron for creating digest emails
if __name__ == '__main__':
    try:
        print "Starting cron"
        cron = Cron()
        cron.job()
        print "Finished cron"
    except Exception as e:
        logging.error(e)
        if settings.DEBUG:
            raise
else:
    print "Cron should be run as __main__."
    print "Ran as {0} instead.".format(__name__)
<commit_msg>Added more debugging:<commit_after>#! /usr/bin/python
import logging
from datetime import datetime
from itertools import ifilter

import settings
from model import User, Task
from library.template import Template
from library.mailclient import MailClient, Subjects

class Cron(object):
    def __init__(self):
        print "Construction cron object"
        self.now = datetime.now()
        print "Determined current time"
        self.template = Template()
        print "Created template manager"
        self.mailclient = MailClient()
        print "Created mail client"
        print "Finished constructing cron Object"

    def job(self):
        pring "Getting users"
        users = User.select().where(User.is_active == True)
        print "Users: " + [user.name for user in users]
        for user in ifilter(lambda user: user.is_due(self.now), users):
            print "Creating digest for {0}".format(user.name)
            try:
                digest = self.create_digest(user)
                print "Successfully created digest for {0}".format(user.name)
                email = self.mailclient.send(user, digest, Subjects.DIGEST)
                print "Sent digest to {0}".format(user.name)
            except:
                print "Couldn't send digest to {0}".format(user.name)
                logging.error("Failed to send email to {0}".format(user.name))
                if settings.DEBUG:
                    raise

            if settings.DEBUG:
                print user.name
                print email, '\n\n'
            else:
                user.last_sent = self.now
                user.save()

    def create_digest(self, user):
        last_sent = user.last_sent or user.created
        tasks = Task.select()\
            .where((Task.date > last_sent))\
            .order_by(Task.user.name, Task.date)
        email_params = {
            'user': user,
            'start': last_sent,
            'end': self.now,
            'tasks': tasks,
            'sender': settings.SENDER
        }
        return self.template.render('email_digest.tpl', **email_params)

# Cron for creating digest emails
if __name__ == '__main__':
    try:
        print "Starting cron"
        cron = Cron()
        cron.job()
        print "Finished cron"
    except Exception as e:
        logging.error(e)
        if settings.DEBUG:
            raise
else:
    print "Cron should be run as __main__."
    print "Ran as {0} instead.".format(__name__)
<|endoftext|>