<commit_before>#!/usr/bin/env python

import sys
import re
import argparse
from itertools import izip_longest, izip
import gzip
import contextlib

def reverse_complement(seq):
    """Return reverse complement of a dna sequence."""
    bases_dict = {
        'A': 'T', 'a': 't',
        'C': 'G', 'g': 'c',
        'G': 'C', 'c': 'g',
        'T': 'A', 't': 'a'}
    return "".join([bases_dict[base] for base in reversed(seq)])

def parse_design(design_file):
    """Parse design file and return mips dictonary."""
    mips = {}
    with open(design_file, 'r') as f:
        header = f.readline().strip('\n').split('\t')

        # Check header
        if header[6] != 'ext_probe_sequence':
            print "Error: column 7 in design file should be: ext_probe_sequence"
            sys.exit()
        elif header[10] != 'lig_probe_sequence':
            print "Error: column 11 in design file should be: lig_probe_sequence"
            sys.exit()
        elif header[19] != 'mip_name':
            print "Error: column 20 in design file should be: mip_name"
            sys.exit()

        # Parse MIPS
        for line in f:
            line = line.strip('\n').split('\t')

            mip_name = line[19]
            ext_probe = line[6]
            ext_probe_revcom = reverse_complement(ext_probe)
            lig_probe = line[10]
            lig_probe_revcom = reverse_complement(lig_probe)

            mips[mip_name] = {
                'ext_probe': ext_probe, 'ext_probe_revcom': ext_probe_revcom,
                'lig_probe': lig_probe, 'lig_probe_revcom': lig_probe_revcom,
                'uuids': set({}),
                'count': 0, 'dup_count': 0
            }

    return mips

def grouper(iterable, n, fillvalue=None):
    """Helper function to read in fasta file per 4 lines."""
    args = [iter(iterable)] * n
    return izip_longest(*args, fillvalue=fillvalue)

class FixedGzip(gzip.GzipFile):
    """Fix gzip class to work with contextlib.nested in python 2.6."""

    def __enter__(self):
        if self.fileobj is None:
            raise ValueError("I/O operation on closed GzipFile object")
        return self

    def __exit__(self, *args):
        self.close()

if __name__ == "__main__":
    # Parse arguments
    parser = argparse.ArgumentParser(formatter_class=lambda prog: argparse.HelpFormatter(prog, max_help_position=100, width=200),
    description='Trim, merge and dedup fastq files from mips experiment. Assumes fastq naming convention: sample_flowcell_index_lane_R[12]_tag.fastq.gz and fastq files from one sample.')
    parser.add_argument('-d', '--design_file', type=str, help='Mips design file', required=True)
    parser.add_argument('-r1', '--r1_fastq', type=str, help='R1 fastq', required=True, nargs='*')
    parser.add_argument('-r2', '--r2_fastq', type=str, help='R2 fastq', required=True, nargs='*')
    parser.add_argument('-l', '--uuid_length', type=int, help='UUID length', required=True)
    parser.add_argument('-ur', '--uuid_read', type=str, help='Read containing UUID', choices=['R1', 'R2'], required=True)
    args = parser.parse_args()

    # Check input fastq's
    if len(args.r1_fastq) != len(args.r2_fastq):
        parser.error("Arguments -r1/--r1_fastq and -r2/--r2_fastq should be of equal length.")

    mips = parse_design(args.design_file)
    unique_uuids = set({})

    # Output files
    fastq_1_file_out = "{sample_name}-trimmed-dedup".format(sample_name=args.r1_fastq[0].split('/')[-1])
    fastq_2_file_out = "{sample_name}-trimmed-dedup".format(sample_name=args.r2_fastq[0].split('/')[-1])

    if len(args.r1_fastq) > 1 and len(args.r2_fastq) > 1:  # Multiple fastq's -> merge
        fastq_1_file_out = re.sub('_L\d{3}_', '_LMerged_', fastq_1_file_out)
        fastq_2_file_out = re.sub('_L\d{3}_', '_LMerged_', fastq_2_file_out)

    with contextlib.nested(
        FixedGzip(fastq_1_file_out, 'w'),
        FixedGzip(fastq_2_file_out, 'w')
    ) as (write_f1, write_f2):

        # Statistics variables
        total = 0
        match = 0
        n_count = 0
        duplicate = 0

        # Loop over fastq files
        for fastq_1_file, fastq_2_file in zip(args.r1_fastq, args.r2_fastq):

            # Open input files
            with contextlib.nested(
                FixedGzip(fastq_1_file, 'r'),
                FixedGzip(fastq_2_file, 'r'),
            ) as (f1, f2):
                # Read in both fastq files per 4 lines [id, seq, +, qual]
                for fastq_1_lines, fastq_2_lines in izip(grouper(f1, 4, ''), grouper(f2, 4, '')):
                    total += 1
                    for mip in mips:

                        if args.uuid_read == 'R1':
                            if fastq_2_lines[1].startswith(mips[mip]['ext_probe']) and fastq_1_lines[1].startswith(mips[mip]['lig_probe_revcom'], args.uuid_length):
                                match += 1
                                uuid = fastq_1_lines[1][0:args.uuid_length]
                                # skip uuid containing 'N'
                                if "N" in uuid.upper():
                                    n_count += 1
                                    break
                                # Check duplicate reads, uuid must be unique per mip.
                                elif uuid in mips[mip]['uuids']:
                                    duplicate += 1
                                    mips[mip]['dup_count'] += 1
                                else:
                                    mips[mip]['uuids'].add(uuid)
                                    mips[mip]['count'] += 1
                                    # Trim fastq
                                    fastq_1_lines = list(fastq_1_lines)
                                    fastq_2_lines = list(fastq_2_lines)

                                    fastq_1_lines[1] = fastq_1_lines[1][len(mips[mip]['lig_probe_revcom'])+args.uuid_length:]  # seq
                                    fastq_1_lines[3] = fastq_1_lines[3][len(mips[mip]['lig_probe_revcom'])+args.uuid_length:]  # qual

                                    fastq_2_lines[1] = fastq_2_lines[1][len(mips[mip]['ext_probe']):]  # seq
                                    fastq_2_lines[3] = fastq_2_lines[3][len(mips[mip]['ext_probe']):]  # qual

                                    # Print fastq to new trimmed and dedupped fastq's.
                                    write_f1.write(''.join(fastq_1_lines))
                                    write_f2.write(''.join(fastq_2_lines))

                                # Track unique uuids in sample
                                if uuid not in unique_uuids:
                                    unique_uuids.add(uuid)
                                break  # A read can only belong to one mip thus break.

                        if args.uuid_read == 'R2':
                            if fastq_2_lines[1].startswith(mips[mip]['ext_probe'], args.uuid_length) and fastq_1_lines[1].startswith(mips[mip]['lig_probe_revcom']):
                                match += 1
                                uuid = fastq_2_lines[1][0:args.uuid_length]
                                # skip uuid containing 'N' or 'n'
                                if "N" in uuid.upper():
                                    n_count += 1
                                    break
                                # Check duplicate reads, uuid must be unique per mip.
                                elif uuid in mips[mip]['uuids']:
                                    duplicate += 1
                                    mips[mip]['dup_count'] += 1
                                else:
                                    mips[mip]['uuids'].add(uuid)
                                    mips[mip]['count'] += 1
                                    # Trim fastq
                                    fastq_1_lines = list(fastq_1_lines)
                                    fastq_2_lines = list(fastq_2_lines)

                                    fastq_1_lines[1] = fastq_1_lines[1][len(mips[mip]['lig_probe_revcom']):]  # seq
                                    fastq_1_lines[3] = fastq_1_lines[3][len(mips[mip]['lig_probe_revcom']):]  # qual

                                    fastq_2_lines[1] = fastq_2_lines[1][len(mips[mip]['ext_probe'])+args.uuid_length:]  # seq
                                    fastq_2_lines[3] = fastq_2_lines[3][len(mips[mip]['ext_probe'])+args.uuid_length:]  # qual

                                    # Print fastq to new trimmed and dedupped fastq's.
                                    write_f1.write(''.join(fastq_1_lines))
                                    write_f2.write(''.join(fastq_2_lines))

                                # Track unique uuids in sample
                                if uuid not in unique_uuids:
                                    unique_uuids.add(uuid)
                                break  # A read can only belong to one mip thus break.

    print 'Match with mip:', match
    print 'Reads with N in uuid', n_count
    print 'Duplicate reads', duplicate
    print 'total reads', total
    print 'sample_unique_uuid_count', len(unique_uuids)

    print 'mip\tuniqe_read_count\tdup_count\tuuids'
    for mip in mips:
        print '{0}\t{1}\t{2}\t{3}'.format(mip, mips[mip]['count'], mips[mip]['dup_count'], ','.join(mips[mip]['uuids']))
<commit_msg>Fix fastq.gz name<commit_after>#!/usr/bin/env python

import sys
import re
import argparse
from itertools import izip_longest, izip
import gzip
import contextlib

def reverse_complement(seq):
    """Return reverse complement of a dna sequence."""
    bases_dict = {
        'A': 'T', 'a': 't',
        'C': 'G', 'g': 'c',
        'G': 'C', 'c': 'g',
        'T': 'A', 't': 'a'}
    return "".join([bases_dict[base] for base in reversed(seq)])

def parse_design(design_file):
    """Parse design file and return mips dictonary."""
    mips = {}
    with open(design_file, 'r') as f:
        header = f.readline().strip('\n').split('\t')

        # Check header
        if header[6] != 'ext_probe_sequence':
            print "Error: column 7 in design file should be: ext_probe_sequence"
            sys.exit()
        elif header[10] != 'lig_probe_sequence':
            print "Error: column 11 in design file should be: lig_probe_sequence"
            sys.exit()
        elif header[19] != 'mip_name':
            print "Error: column 20 in design file should be: mip_name"
            sys.exit()

        # Parse MIPS
        for line in f:
            line = line.strip('\n').split('\t')

            mip_name = line[19]
            ext_probe = line[6]
            ext_probe_revcom = reverse_complement(ext_probe)
            lig_probe = line[10]
            lig_probe_revcom = reverse_complement(lig_probe)

            mips[mip_name] = {
                'ext_probe': ext_probe, 'ext_probe_revcom': ext_probe_revcom,
                'lig_probe': lig_probe, 'lig_probe_revcom': lig_probe_revcom,
                'uuids': set({}),
                'count': 0, 'dup_count': 0
            }

    return mips

def grouper(iterable, n, fillvalue=None):
    """Helper function to read in fasta file per 4 lines."""
    args = [iter(iterable)] * n
    return izip_longest(*args, fillvalue=fillvalue)

class FixedGzip(gzip.GzipFile):
    """Fix gzip class to work with contextlib.nested in python 2.6."""

    def __enter__(self):
        if self.fileobj is None:
            raise ValueError("I/O operation on closed GzipFile object")
        return self

    def __exit__(self, *args):
        self.close()

if __name__ == "__main__":
    # Parse arguments
    parser = argparse.ArgumentParser(formatter_class=lambda prog: argparse.HelpFormatter(prog, max_help_position=100, width=200),
    description='Trim, merge and dedup fastq files from mips experiment. Assumes fastq naming convention: sample_flowcell_index_lane_R[12]_tag.fastq.gz and fastq files from one sample.')
    parser.add_argument('-d', '--design_file', type=str, help='Mips design file', required=True)
    parser.add_argument('-r1', '--r1_fastq', type=str, help='R1 fastq', required=True, nargs='*')
    parser.add_argument('-r2', '--r2_fastq', type=str, help='R2 fastq', required=True, nargs='*')
    parser.add_argument('-l', '--uuid_length', type=int, help='UUID length', required=True)
    parser.add_argument('-ur', '--uuid_read', type=str, help='Read containing UUID', choices=['R1', 'R2'], required=True)
    args = parser.parse_args()

    # Check input fastq's
    if len(args.r1_fastq) != len(args.r2_fastq):
        parser.error("Arguments -r1/--r1_fastq and -r2/--r2_fastq should be of equal length.")

    mips = parse_design(args.design_file)
    unique_uuids = set({})

    # Output files
    fastq_1_file_out = args.r1_fastq[0].split('/')[-1]
    fastq_2_file_out = args.r2_fastq[0].split('/')[-1]

    if len(args.r1_fastq) > 1 and len(args.r2_fastq) > 1:  # Multiple fastq's -> merge
        fastq_1_file_out = re.sub('_L\d{3}_', '_LMergedTrimmedDedup_', fastq_1_file_out)
        fastq_2_file_out = re.sub('_L\d{3}_', '_LMergedTrimmedDedup_', fastq_2_file_out)

    with contextlib.nested(
        FixedGzip(fastq_1_file_out, 'w'),
        FixedGzip(fastq_2_file_out, 'w')
    ) as (write_f1, write_f2):

        # Statistics variables
        total = 0
        match = 0
        n_count = 0
        duplicate = 0

        # Loop over fastq files
        for fastq_1_file, fastq_2_file in zip(args.r1_fastq, args.r2_fastq):

            # Open input files
            with contextlib.nested(
                FixedGzip(fastq_1_file, 'r'),
                FixedGzip(fastq_2_file, 'r'),
            ) as (f1, f2):
                # Read in both fastq files per 4 lines [id, seq, +, qual]
                for fastq_1_lines, fastq_2_lines in izip(grouper(f1, 4, ''), grouper(f2, 4, '')):
                    total += 1
                    for mip in mips:

                        if args.uuid_read == 'R1':
                            if fastq_2_lines[1].startswith(mips[mip]['ext_probe']) and fastq_1_lines[1].startswith(mips[mip]['lig_probe_revcom'], args.uuid_length):
                                match += 1
                                uuid = fastq_1_lines[1][0:args.uuid_length]
                                # skip uuid containing 'N'
                                if "N" in uuid.upper():
                                    n_count += 1
                                    break
                                # Check duplicate reads, uuid must be unique per mip.
                                elif uuid in mips[mip]['uuids']:
                                    duplicate += 1
                                    mips[mip]['dup_count'] += 1
                                else:
                                    mips[mip]['uuids'].add(uuid)
                                    mips[mip]['count'] += 1
                                    # Trim fastq
                                    fastq_1_lines = list(fastq_1_lines)
                                    fastq_2_lines = list(fastq_2_lines)

                                    fastq_1_lines[1] = fastq_1_lines[1][len(mips[mip]['lig_probe_revcom'])+args.uuid_length:]  # seq
                                    fastq_1_lines[3] = fastq_1_lines[3][len(mips[mip]['lig_probe_revcom'])+args.uuid_length:]  # qual

                                    fastq_2_lines[1] = fastq_2_lines[1][len(mips[mip]['ext_probe']):]  # seq
                                    fastq_2_lines[3] = fastq_2_lines[3][len(mips[mip]['ext_probe']):]  # qual

                                    # Print fastq to new trimmed and dedupped fastq's.
                                    write_f1.write(''.join(fastq_1_lines))
                                    write_f2.write(''.join(fastq_2_lines))

                                # Track unique uuids in sample
                                if uuid not in unique_uuids:
                                    unique_uuids.add(uuid)
                                break  # A read can only belong to one mip thus break.

                        if args.uuid_read == 'R2':
                            if fastq_2_lines[1].startswith(mips[mip]['ext_probe'], args.uuid_length) and fastq_1_lines[1].startswith(mips[mip]['lig_probe_revcom']):
                                match += 1
                                uuid = fastq_2_lines[1][0:args.uuid_length]
                                # skip uuid containing 'N' or 'n'
                                if "N" in uuid.upper():
                                    n_count += 1
                                    break
                                # Check duplicate reads, uuid must be unique per mip.
                                elif uuid in mips[mip]['uuids']:
                                    duplicate += 1
                                    mips[mip]['dup_count'] += 1
                                else:
                                    mips[mip]['uuids'].add(uuid)
                                    mips[mip]['count'] += 1
                                    # Trim fastq
                                    fastq_1_lines = list(fastq_1_lines)
                                    fastq_2_lines = list(fastq_2_lines)

                                    fastq_1_lines[1] = fastq_1_lines[1][len(mips[mip]['lig_probe_revcom']):]  # seq
                                    fastq_1_lines[3] = fastq_1_lines[3][len(mips[mip]['lig_probe_revcom']):]  # qual

                                    fastq_2_lines[1] = fastq_2_lines[1][len(mips[mip]['ext_probe'])+args.uuid_length:]  # seq
                                    fastq_2_lines[3] = fastq_2_lines[3][len(mips[mip]['ext_probe'])+args.uuid_length:]  # qual

                                    # Print fastq to new trimmed and dedupped fastq's.
                                    write_f1.write(''.join(fastq_1_lines))
                                    write_f2.write(''.join(fastq_2_lines))

                                # Track unique uuids in sample
                                if uuid not in unique_uuids:
                                    unique_uuids.add(uuid)
                                break  # A read can only belong to one mip thus break.

    print 'Match with mip:', match
    print 'Reads with N in uuid', n_count
    print 'Duplicate reads', duplicate
    print 'total reads', total
    print 'sample_unique_uuid_count', len(unique_uuids)

    print 'mip\tuniqe_read_count\tdup_count\tuuids'
    for mip in mips:
        print '{0}\t{1}\t{2}\t{3}'.format(mip, mips[mip]['count'], mips[mip]['dup_count'], ','.join(mips[mip]['uuids']))
<|endoftext|>


<commit_before>import re
import os
import gzip
import string
import multiprocessing
import argparse
import logging

import pandas as pd

from utils import timeit, backup_file

logging.basicConfig(
    level=logging.DEBUG, format='%(asctime)s|%(levelname)s|%(message)s')

def parse_args():
    parser = argparse.ArgumentParser(
        description=('This script converts NCBI taxonomy (taxdump) into '
                     'lineages, save the information in csv.gz format'))

    parser.add_argument(
        '--nodes-file', required=True,
        help='NCBI taxonomy path/to/taxdump/nodes.dmp')

    parser.add_argument(
        '--names-file', required=True,
        help='NCBI taxonomy path/to/taxdump/names.dmp')

    parser.add_argument(
        '-o', '--output-prefix', default='ncbi_lineages',
        help='will output both lineage information in output_prefix.csv.gz')

    args = parser.parse_args()
    return args

@timeit
def load_nodes(nodes_file):
    '''
    load nodes.dmp and convert it into a pandas.DataFrame
    '''
    df = pd.read_csv(nodes_file, sep='|', header=None, index_col=False,
                     names=[
                         'tax_id',
                         'parent_tax_id',
                         'rank',
                         'embl_code',
                         'division_id',
                         'inherited_div_flag',
                         'genetic_code_id',
                         'inherited_GC__flag',
                         'mitochondrial_genetic_code_id',
                         'inherited_MGC_flag',
                         'GenBank_hidden_flag',
                         'hidden_subtree_root_flag',
                         'comments'
                     ])

    # To get rid of flanking tab characters
    df['rank'] = df['rank'].apply(string.strip)
    df['embl_code'] = df['embl_code'].apply(string.strip)
    df['comments'] = df['comments'].apply(string.strip)
    return df

@timeit
def load_names(names_file):
    '''
    load names.dmp and convert it into a pandas.DataFrame
    '''
    df = pd.read_csv(names_file, sep='|', header=None, index_col=False,
                     names=[
                         'tax_id',
                         'name_txt',
                         'unique_name',
                         'name_class'
                     ])
    df['name_txt'] = df['name_txt'].apply(string.strip)
    df['unique_name'] = df['unique_name'].apply(string.strip)
    df['name_class'] = df['name_class'].apply(string.strip)

    sci_df = df[df['name_class'] == 'scientific name']
    sci_df.reset_index(drop=True, inplace=True)
    return sci_df

def to_dict(lineage):
    """
    convert the lineage into a list of tuples in the form of

    [
        (tax_id1, rank1, name_txt1),
        (tax_id2, rank2, name_txt2),
        ...
    ]

    to a dict
    """
    dd = {}
    num_re = re.compile('[0-9]+')
    len_lineage = len(lineage)
    for k, __ in enumerate(lineage):
        tax_id, rank, name_txt = __
        # use the last rank as the tax_id, whatever it is, genus or species.
        if k == len_lineage - 1:
            dd['tax_id'] = tax_id

        # e.g. there could be multiple 'no rank'
        numbered_rank = rank
        while numbered_rank in dd:
            # print __, numbered_rank
            search = num_re.search(numbered_rank)
            if search is None:
                count = 1
            else:
                count = int(search.group()) + 1
            numbered_rank = '{0}{1}'.format(rank, count)
        dd[numbered_rank] = name_txt
    return dd

def find_lineage(tax_id):
    if tax_id % 50000 == 0:
        logging.debug('working on tax_id: {0}'.format(tax_id))
    lineage = []
    while True:
        rec = TAXONOMY_DICT[tax_id]
        lineage.append((rec['tax_id'], rec['rank'], rec['name_txt']))
        tax_id = rec['parent_tax_id']

        if tax_id == 1:
            break

    # reverse results in lineage of Kingdom => species, this is helpful for
    # to_dict when there are multiple "no rank"s
    lineage.reverse()
    return to_dict(lineage)

def main():
    # data downloaded from ftp://ftp.ncbi.nih.gov/pub/taxonomy/
    args = parse_args()
    nodes_df = load_nodes(args.nodes_file)
    names_df = load_names(args.names_file)
    df = nodes_df.merge(names_df, on='tax_id')
    df = df[['tax_id', 'parent_tax_id', 'rank', 'name_txt']]
    df.reset_index(drop=True, inplace=True)
    logging.info('# of tax ids: {0}'.format(df.shape[0]))
    # log summary info about the dataframe
    df.info()

    # force to use global variable TAXONOMY_DICT because map doesn't allow
    # passing extra args easily

    # TAXONOMY_DICT: a dict with tax_id as the key and each record as a value.
    # example tuple items:
    # (1,
    #  {'parent_tax_id': 1, 'name_txt': 'root',
    #   'rank': 'no rank', 'tax_id': 1}
    # )

    # (16,
    #  {'parent_tax_id': 32011, 'name_txt': 'Methylophilus',
    #   'rank': 'genus', 'tax_id': 16}
    # )

    global TAXONOMY_DICT
    logging.info('generating TAXONOMY_DICT...')
    TAXONOMY_DICT = dict(zip(df.tax_id.values, df.to_dict('records')))

    ncpus = multiprocessing.cpu_count()
    logging.info('found {0} cpus, and will use all of them to find lineages '
                 'for all tax ids'.format(ncpus))
    pool = multiprocessing.Pool(ncpus)
    # take about 18G memory
    lineages_dd = pool.map(find_lineage, df.tax_id.values)
    pool.close()

    logging.info('generating a dictionary of lineages information...')
    dd_for_df = dict(zip(range(len(lineages_dd)), lineages_dd))

    logging.info('generating lineages_df...')
    lineages_df = pd.DataFrame.from_dict(dd_for_df, orient='index')
    lineages_df.sort_values('tax_id', inplace=True)
    # # alternatively, but less useful, sort by ranks
    # lineages_df.sort_values(['superkingdom',
    #                          'phylum',
    #                          'class',
    #                          'order',
    #                          'family',
    #                          'genus',
    #                          'species'], inplace=True)

    lineages_csv_output = os.path.join('{0}.csv.gz'.format(args.output_prefix))
    backup_file(lineages_csv_output)
    logging.info("writing lineages to {0}".format(lineages_csv_output))
    with open(lineages_csv_output, 'wb') as opf:
        # make sure the name and timestamp are not gzipped, (like gzip -n)
        opf_gz = gzip.GzipFile('', 'wb', 9, opf, 0.)
        cols = ['tax_id',
                'superkingdom',
                'phylum',
                'class',
                'order',
                'family',
                'genus',
                'species']

        other_cols = sorted([__ for __ in lineages_df.columns
                             if __ not in cols])
        output_cols = cols + other_cols
        lineages_df.to_csv(opf_gz, index=False, columns=output_cols)

if __name__ == "__main__":
    main()
<commit_msg>replaced string.strip for future py3 compatibility<commit_after>import re
import os
import gzip
import multiprocessing
import argparse
import logging

import pandas as pd

from utils import timeit, backup_file

logging.basicConfig(
    level=logging.DEBUG, format='%(asctime)s|%(levelname)s|%(message)s')

def parse_args():
    parser = argparse.ArgumentParser(
        description=('This script converts NCBI taxonomy (taxdump) into '
                     'lineages, save the information in csv.gz format'))

    parser.add_argument(
        '--nodes-file', required=True,
        help='NCBI taxonomy path/to/taxdump/nodes.dmp')

    parser.add_argument(
        '--names-file', required=True,
        help='NCBI taxonomy path/to/taxdump/names.dmp')

    parser.add_argument(
        '-o', '--output-prefix', default='ncbi_lineages',
        help='will output both lineage information in output_prefix.csv.gz')

    args = parser.parse_args()
    return args

def strip(str_):
    '''
    :param str_: a string
    '''
    return str_.strip()

@timeit
def load_nodes(nodes_file):
    '''
    load nodes.dmp and convert it into a pandas.DataFrame
    '''
    df = pd.read_csv(nodes_file, sep='|', header=None, index_col=False,
                     names=[
                         'tax_id',
                         'parent_tax_id',
                         'rank',
                         'embl_code',
                         'division_id',
                         'inherited_div_flag',
                         'genetic_code_id',
                         'inherited_GC__flag',
                         'mitochondrial_genetic_code_id',
                         'inherited_MGC_flag',
                         'GenBank_hidden_flag',
                         'hidden_subtree_root_flag',
                         'comments'
                     ])

    # To get rid of flanking tab characters
    df['rank'] = df['rank'].apply(strip)
    df['embl_code'] = df['embl_code'].apply(strip)
    df['comments'] = df['comments'].apply(strip)
    return df

@timeit
def load_names(names_file):
    '''
    load names.dmp and convert it into a pandas.DataFrame
    '''
    df = pd.read_csv(names_file, sep='|', header=None, index_col=False,
                     names=[
                         'tax_id',
                         'name_txt',
                         'unique_name',
                         'name_class'
                     ])
    df['name_txt'] = df['name_txt'].apply(strip)
    df['unique_name'] = df['unique_name'].apply(strip)
    df['name_class'] = df['name_class'].apply(strip)

    sci_df = df[df['name_class'] == 'scientific name']
    sci_df.reset_index(drop=True, inplace=True)
    return sci_df

def to_dict(lineage):
    """
    convert the lineage into a list of tuples in the form of

    [
        (tax_id1, rank1, name_txt1),
        (tax_id2, rank2, name_txt2),
        ...
    ]

    to a dict
    """
    dd = {}
    num_re = re.compile('[0-9]+')
    len_lineage = len(lineage)
    for k, __ in enumerate(lineage):
        tax_id, rank, name_txt = __
        # use the last rank as the tax_id, whatever it is, genus or species.
        if k == len_lineage - 1:
            dd['tax_id'] = tax_id

        # e.g. there could be multiple 'no rank'
        numbered_rank = rank
        while numbered_rank in dd:
            # print __, numbered_rank
            search = num_re.search(numbered_rank)
            if search is None:
                count = 1
            else:
                count = int(search.group()) + 1
            numbered_rank = '{0}{1}'.format(rank, count)
        dd[numbered_rank] = name_txt
    return dd

def find_lineage(tax_id):
    if tax_id % 50000 == 0:
        logging.debug('working on tax_id: {0}'.format(tax_id))
    lineage = []
    while True:
        rec = TAXONOMY_DICT[tax_id]
        lineage.append((rec['tax_id'], rec['rank'], rec['name_txt']))
        tax_id = rec['parent_tax_id']

        if tax_id == 1:
            break

    # reverse results in lineage of Kingdom => species, this is helpful for
    # to_dict when there are multiple "no rank"s
    lineage.reverse()
    return to_dict(lineage)

def main():
    # data downloaded from ftp://ftp.ncbi.nih.gov/pub/taxonomy/
    args = parse_args()
    nodes_df = load_nodes(args.nodes_file)
    names_df = load_names(args.names_file)
    df = nodes_df.merge(names_df, on='tax_id')
    df = df[['tax_id', 'parent_tax_id', 'rank', 'name_txt']]
    df.reset_index(drop=True, inplace=True)
    logging.info('# of tax ids: {0}'.format(df.shape[0]))
    # log summary info about the dataframe
    df.info()

    # force to use global variable TAXONOMY_DICT because map doesn't allow
    # passing extra args easily

    # TAXONOMY_DICT: a dict with tax_id as the key and each record as a value.
    # example tuple items:
    # (1,
    #  {'parent_tax_id': 1, 'name_txt': 'root',
    #   'rank': 'no rank', 'tax_id': 1}
    # )

    # (16,
    #  {'parent_tax_id': 32011, 'name_txt': 'Methylophilus',
    #   'rank': 'genus', 'tax_id': 16}
    # )

    global TAXONOMY_DICT
    logging.info('generating TAXONOMY_DICT...')
    TAXONOMY_DICT = dict(zip(df.tax_id.values, df.to_dict('records')))

    ncpus = multiprocessing.cpu_count()
    logging.info('found {0} cpus, and will use all of them to find lineages '
                 'for all tax ids'.format(ncpus))
    pool = multiprocessing.Pool(ncpus)
    # take about 18G memory
    lineages_dd = pool.map(find_lineage, df.tax_id.values)
    pool.close()

    logging.info('generating a dictionary of lineages information...')
    dd_for_df = dict(zip(range(len(lineages_dd)), lineages_dd))

    logging.info('generating lineages_df...')
    lineages_df = pd.DataFrame.from_dict(dd_for_df, orient='index')
    lineages_df.sort_values('tax_id', inplace=True)
    # # alternatively, but less useful, sort by ranks
    # lineages_df.sort_values(['superkingdom',
    #                          'phylum',
    #                          'class',
    #                          'order',
    #                          'family',
    #                          'genus',
    #                          'species'], inplace=True)

    lineages_csv_output = os.path.join('{0}.csv.gz'.format(args.output_prefix))
    backup_file(lineages_csv_output)
    logging.info("writing lineages to {0}".format(lineages_csv_output))
    with open(lineages_csv_output, 'wb') as opf:
        # make sure the name and timestamp are not gzipped, (like gzip -n)
        opf_gz = gzip.GzipFile('', 'wb', 9, opf, 0.)
        cols = ['tax_id',
                'superkingdom',
                'phylum',
                'class',
                'order',
                'family',
                'genus',
                'species']

        other_cols = sorted([__ for __ in lineages_df.columns
                             if __ not in cols])
        output_cols = cols + other_cols
        lineages_df.to_csv(opf_gz, index=False, columns=output_cols)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python

# Copyright 2008-2010 WebDriver committers
# Copyright 2008-2010 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import pytest
import re
import tempfile
import time
import shutil
import unittest
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import NoSuchFrameException

def not_available_on_remote(func):
    def testMethod(self):
        print self.driver
        if type(self.driver) == 'remote':
            return lambda x: None
        else:
            return func(self)
    return testMethod

class FrameSwitchingTest(unittest.TestCase):

    @pytest.mark.xskipif("self.driver.capabilities['version'].startswith('3')")
    def testShouldBeAbleToSwitchToAFrameByItsIndex(self):
      self._loadPage("frameset")
      self.driver.switch_to_frame(2)
      element = self.driver.find_element_by_id("email")
      self.assertEquals("email", element.get_attribute("type"))

    def testShouldBeAbleToSwitchToAnIframeByItsIndex(self):
      self._loadPage("iframes")
      self.driver.switch_to_frame(0)
      element = self.driver.find_element_by_id("id-name1")
      self.assertEquals("id", element.get_attribute("value"))

    def testShouldBeAbleToSwitchToAFrameByItsName(self):
      self._loadPage("frameset")
      self.driver.switch_to_frame("fourth")
      element = self.driver.find_element_by_tag_name("frame")
      self.assertEquals("child1", element.get_attribute("name"))

    def testShouldBeAbleToSwitchToAnIframeByItsName(self):
      self._loadPage("iframes")
      self.driver.switch_to_frame("iframe1-name");
      element = self.driver.find_element_by_name("id-name1")
      self.assertEquals("name", element.get_attribute("value"))

    def testShouldBeAbleToSwitchToAFrameByItsID(self):
      self._loadPage("frameset")
      self.driver.switch_to_frame("fifth")
      element = self.driver.find_element_by_name("windowOne")
      self.assertEquals("Open new window", element.text)

    def testShouldBeAbleToSwitchToAnIframeByItsID(self):
      self._loadPage("iframes")
      self.driver.switch_to_frame("iframe1");
      element = self.driver.find_element_by_name("id-name1")
      self.assertEquals("name", element.get_attribute("value"))

    @pytest.mark.skipif("self.driver.capabilities['version'].startswith('3')")
    def testShouldBeAbleToSwitchToAFrameUsingAPreviouslyLocatedWebElement(self):
      self._loadPage("frameset")
      frame = self.driver.find_element_by_name("third")
      self.driver.switch_to_frame(frame)
      element = self.driver.find_element_by_id("email")
      self.assertEquals("email", element.get_attribute("type"))

    def testShouldBeAbleToSwitchToAnIFrameUsingAPreviouslyLocatedWebElement(self):
      self._loadPage("iframes")
      frame = self.driver.find_element_by_tag_name("iframe")
      self.driver.switch_to_frame(frame)
      element = self.driver.find_element_by_name("id-name1")
      self.assertEquals("name", element.get_attribute("value"))

    def testShouldEnsureElementIsAFrameBeforeSwitching(self):
      self._loadPage("frameset")
      frame = self.driver.find_element_by_tag_name("frameset")
      try:
        self.driver.switch_to_frame(frame)
        self.fail()
      except NoSuchFrameException:
        pass

    def testFrameSearchesShouldBeRelativeToTheCurrentlySelectedFrame(self):
      self._loadPage("frameset")
      self.driver.switch_to_frame("sixth")
      element = self.driver.find_element_by_id("iframe_page_heading")
      self.assertEquals("This is the heading", element.text)

      try:
        self.driver.switch_to_frame("third")
        self.fail()
      except NoSuchFrameException:
        pass

      self.driver.switch_to_default_content()
      self.driver.switch_to_frame("third")

      try:
        self.driver.switch_to_frame("third")
        self.fail()
      except NoSuchFrameException:
        pass

      # Now make sure we can go back.
      self.driver.switch_to_default_content()
      self.driver.switch_to_frame("sixth")
      element = self.driver.find_element_by_id("iframe_page_heading")
      self.assertEquals("This is the heading", element.text)

    def testShouldBeAbleToSelectChildFrames(self):
      self._loadPage("frameset")
      self.driver.switch_to_frame("sixth")
      self.driver.switch_to_frame(0)
      element = self.driver.find_element_by_id("id-name1")
      self.assertEquals("id", element.get_attribute("value"))

    def testShouldThrowFrameNotFoundExceptionLookingUpSubFramesWithSuperFrameNames(self):
      self._loadPage("frameset")
      self.driver.switch_to_frame("fourth")

      try:
        self.driver.switch_to_frame("second")
        self.fail("Expected NoSuchFrameException")
      except NoSuchFrameException:
        pass

    def testShouldThrowAnExceptionWhenAFrameCannotBeFound(self):
      self._loadPage("xhtmlTest")
      try:
        self.driver.switch_to_frame("nothing here")
        self.fail("Should not have been able to switch")
      except NoSuchFrameException:
        pass

    def testShouldThrowAnExceptionWhenAFrameCannotBeFoundByIndex(self):
      self._loadPage("xhtmlTest")
      try:
        self.driver.switch_to_frame(27)
        self.fail("Should not have been able to switch")
      except NoSuchFrameException:
        pass

    # disabled till we use the Java Webserver
    #def testThatWeStillReferToTheSameFrameOnceItHasBeenSelected(self):
    #    self._loadPage("frameset")
    #    self.driver.switch_to_frame(2)
    #    checkbox = self.driver.find_element_by_xpath("//input[@name='cheeky']")
    #    checkbox.toggle()
    #    checkbox.submit()
    #    self.assertEqual(self.driver.find_element_by_xpath("//p").text, "Success")

    # Disabled till we use the Java WebServer
    #def testThatWeShouldUseTheFirstFrameOnAPage(self):
    #    self._loadPage("frameset")
    #    time.sleep(1)
    #    pageNumber = self.driver.find_element_by_xpath('//span[@id="pageNumber"]')
    #    self.assertEqual(pageNumber.text.strip(), "1")

    #Disabled till we use the Java WebServer
    #def testThatWeFocusOnTheReplacementWhenAFrameFollowsALinkToATopTargettedPage(self):
    #    self._loadPage("frameset")
    #    time.sleep(1)
    #    self.driver.switch_to_frame(0)
    #    self.driver.find_element_by_link_text("top").click()
    #    time.sleep(1)
    #    self.assertEqual("XHTML Test Page", self.driver.title)
    #    self.assertEqual("XHTML Test Page",
    #    self.driver.find_element_by_xpath("/html/head/title").text)

    def testThatWeShouldNotAutoSwitchFocusToAnIFrameWhenAPageContainingThemIsLoaded(self):
        self._loadPage("iframes")
        time.sleep(0.5)
        self.driver.find_element_by_id("iframe_page_heading")

    def testShouldAllowAUserToSwitchFromAnIframeBackToTheMainContentOfThePage(self):
        self._loadPage("iframes")
        self.driver.switch_to_frame(0)
        self.driver.switch_to_default_content()
        self.driver.find_element_by_id('iframe_page_heading')

    # Disabled till we use the Java WebServer
    #def testShouldAllowTheUserToSwitchToAnIFrameAndRemainFocusedOnIt(self):
    #    self._loadPage("iframes")
    #    self.driver.switch_to_frame(0)
    #    submitButton = self.driver.find_element_by_id("greeting")
    #    submitButton.click()

    #    time.sleep(1)
    #    hello = self.driver.find_element_by_id('greeting')
    #    self.assertEqual(hello.text, "Success!")

    # Disalbled till we used the Java Webserver
    #def testShouldBeAbleToClickInAFrame(self):
    #    self._loadPage("frameset")
    #    self.driver.switch_to_frame("third")
    #    time.sleep(1)
    #    submitButton = self.driver.find_element_by_id("greeting")
    #    submitButton.click()

    #    time.sleep(0.5)
    #    hello = self.driver.find_element_by_id('greeting')
    #    self.assertEqual(hello.text, "Success!")
    #    self.driver.switch_to_default_content()

    def testShouldReturnFrameTitleNotWindowTitle(self):
        self._loadPage("frameset")
        self.driver.switch_to_frame("third")
        self.assertEqual("Unique title", self.driver.title)

    def _pageURL(self, name):
        return "http://localhost:%d/%s.html" % (self.webserver.port, name)

    def _loadSimplePage(self):
        self._loadPage("simpleTest")

    def _loadPage(self, name):
        self.driver.get(self._pageURL(name))
<commit_msg>LukeIS: fixing wrong style of skip in python for this test checking the browser version<commit_after>#!/usr/bin/python

# Copyright 2008-2010 WebDriver committers
# Copyright 2008-2010 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import pytest
import re
import tempfile
import time
import shutil
import unittest
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import NoSuchFrameException

def not_available_on_remote(func):
    def testMethod(self):
        print self.driver
        if type(self.driver) == 'remote':
            return lambda x: None
        else:
            return func(self)
    return testMethod

class FrameSwitchingTest(unittest.TestCase):

    def testShouldBeAbleToSwitchToAFrameByItsIndex(self):
      if self.driver.capabilities['browserName'] == 'firefox' and self.driver.capabilities['version'].startswith('3'):
        pytest.skip()
      self._loadPage("frameset")
      self.driver.switch_to_frame(2)
      element = self.driver.find_element_by_id("email")
      self.assertEquals("email", element.get_attribute("type"))

    def testShouldBeAbleToSwitchToAnIframeByItsIndex(self):
      self._loadPage("iframes")
      self.driver.switch_to_frame(0)
      element = self.driver.find_element_by_id("id-name1")
      self.assertEquals("id", element.get_attribute("value"))

    def testShouldBeAbleToSwitchToAFrameByItsName(self):
      self._loadPage("frameset")
      self.driver.switch_to_frame("fourth")
      element = self.driver.find_element_by_tag_name("frame")
      self.assertEquals("child1", element.get_attribute("name"))

    def testShouldBeAbleToSwitchToAnIframeByItsName(self):
      self._loadPage("iframes")
      self.driver.switch_to_frame("iframe1-name");
      element = self.driver.find_element_by_name("id-name1")
      self.assertEquals("name", element.get_attribute("value"))

    def testShouldBeAbleToSwitchToAFrameByItsID(self):
      self._loadPage("frameset")
      self.driver.switch_to_frame("fifth")
      element = self.driver.find_element_by_name("windowOne")
      self.assertEquals("Open new window", element.text)

    def testShouldBeAbleToSwitchToAnIframeByItsID(self):
      self._loadPage("iframes")
      self.driver.switch_to_frame("iframe1");
      element = self.driver.find_element_by_name("id-name1")
      self.assertEquals("name", element.get_attribute("value"))

    def testShouldBeAbleToSwitchToAFrameUsingAPreviouslyLocatedWebElement(self):
      if self.driver.capabilities['browserName'] == 'firefox' and self.driver.capabilities['version'].startswith('3'):
        pytest.skip()
      self._loadPage("frameset")
      frame = self.driver.find_element_by_name("third")
      self.driver.switch_to_frame(frame)
      element = self.driver.find_element_by_id("email")
      self.assertEquals("email", element.get_attribute("type"))

    def testShouldBeAbleToSwitchToAnIFrameUsingAPreviouslyLocatedWebElement(self):
      self._loadPage("iframes")
      frame = self.driver.find_element_by_tag_name("iframe")
      self.driver.switch_to_frame(frame)
      element = self.driver.find_element_by_name("id-name1")
      self.assertEquals("name", element.get_attribute("value"))

    def testShouldEnsureElementIsAFrameBeforeSwitching(self):
      self._loadPage("frameset")
      frame = self.driver.find_element_by_tag_name("frameset")
      try:
        self.driver.switch_to_frame(frame)
        self.fail()
      except NoSuchFrameException:
        pass

    def testFrameSearchesShouldBeRelativeToTheCurrentlySelectedFrame(self):
      self._loadPage("frameset")
      self.driver.switch_to_frame("sixth")
      element = self.driver.find_element_by_id("iframe_page_heading")
      self.assertEquals("This is the heading", element.text)

      try:
        self.driver.switch_to_frame("third")
        self.fail()
      except NoSuchFrameException:
        pass

      self.driver.switch_to_default_content()
      self.driver.switch_to_frame("third")

      try:
        self.driver.switch_to_frame("third")
        self.fail()
      except NoSuchFrameException:
        pass

      # Now make sure we can go back.
      self.driver.switch_to_default_content()
      self.driver.switch_to_frame("sixth")
      element = self.driver.find_element_by_id("iframe_page_heading")
      self.assertEquals("This is the heading", element.text)

    def testShouldBeAbleToSelectChildFrames(self):
      self._loadPage("frameset")
      self.driver.switch_to_frame("sixth")
      self.driver.switch_to_frame(0)
      element = self.driver.find_element_by_id("id-name1")
      self.assertEquals("id", element.get_attribute("value"))

    def testShouldThrowFrameNotFoundExceptionLookingUpSubFramesWithSuperFrameNames(self):
      self._loadPage("frameset")
      self.driver.switch_to_frame("fourth")

      try:
        self.driver.switch_to_frame("second")
        self.fail("Expected NoSuchFrameException")
      except NoSuchFrameException:
        pass

    def testShouldThrowAnExceptionWhenAFrameCannotBeFound(self):
      self._loadPage("xhtmlTest")
      try:
        self.driver.switch_to_frame("nothing here")
        self.fail("Should not have been able to switch")
      except NoSuchFrameException:
        pass

    def testShouldThrowAnExceptionWhenAFrameCannotBeFoundByIndex(self):
      self._loadPage("xhtmlTest")
      try:
        self.driver.switch_to_frame(27)
        self.fail("Should not have been able to switch")
      except NoSuchFrameException:
        pass

    # disabled till we use the Java Webserver
    #def testThatWeStillReferToTheSameFrameOnceItHasBeenSelected(self):
    #    self._loadPage("frameset")
    #    self.driver.switch_to_frame(2)
    #    checkbox = self.driver.find_element_by_xpath("//input[@name='cheeky']")
    #    checkbox.toggle()
    #    checkbox.submit()
    #    self.assertEqual(self.driver.find_element_by_xpath("//p").text, "Success")

    # Disabled till we use the Java WebServer
    #def testThatWeShouldUseTheFirstFrameOnAPage(self):
    #    self._loadPage("frameset")
    #    time.sleep(1)
    #    pageNumber = self.driver.find_element_by_xpath('//span[@id="pageNumber"]')
    #    self.assertEqual(pageNumber.text.strip(), "1")

    #Disabled till we use the Java WebServer
    #def testThatWeFocusOnTheReplacementWhenAFrameFollowsALinkToATopTargettedPage(self):
    #    self._loadPage("frameset")
    #    time.sleep(1)
    #    self.driver.switch_to_frame(0)
    #    self.driver.find_element_by_link_text("top").click()
    #    time.sleep(1)
    #    self.assertEqual("XHTML Test Page", self.driver.title)
    #    self.assertEqual("XHTML Test Page",
    #    self.driver.find_element_by_xpath("/html/head/title").text)

    def testThatWeShouldNotAutoSwitchFocusToAnIFrameWhenAPageContainingThemIsLoaded(self):
        self._loadPage("iframes")
        time.sleep(0.5)
        self.driver.find_element_by_id("iframe_page_heading")

    def testShouldAllowAUserToSwitchFromAnIframeBackToTheMainContentOfThePage(self):
        self._loadPage("iframes")
        self.driver.switch_to_frame(0)
        self.driver.switch_to_default_content()
        self.driver.find_element_by_id('iframe_page_heading')

    # Disabled till we use the Java WebServer
    #def testShouldAllowTheUserToSwitchToAnIFrameAndRemainFocusedOnIt(self):
    #    self._loadPage("iframes")
    #    self.driver.switch_to_frame(0)
    #    submitButton = self.driver.find_element_by_id("greeting")
    #    submitButton.click()

    #    time.sleep(1)
    #    hello = self.driver.find_element_by_id('greeting')
    #    self.assertEqual(hello.text, "Success!")

    # Disalbled till we used the Java Webserver
    #def testShouldBeAbleToClickInAFrame(self):
    #    self._loadPage("frameset")
    #    self.driver.switch_to_frame("third")
    #    time.sleep(1)
    #    submitButton = self.driver.find_element_by_id("greeting")
    #    submitButton.click()

    #    time.sleep(0.5)
    #    hello = self.driver.find_element_by_id('greeting')
    #    self.assertEqual(hello.text, "Success!")
    #    self.driver.switch_to_default_content()

    def testShouldReturnFrameTitleNotWindowTitle(self):
        self._loadPage("frameset")
        self.driver.switch_to_frame("third")
        self.assertEqual("Unique title", self.driver.title)

    def _pageURL(self, name):
        return "http://localhost:%d/%s.html" % (self.webserver.port, name)

    def _loadSimplePage(self):
        self._loadPage("simpleTest")

    def _loadPage(self, name):
        self.driver.get(self._pageURL(name))
<|endoftext|>


<commit_before>#!/usr/bin/python

import sys

def asdot2plain( asdot ):
	"This returns an ASPLAIN formated ASN given an ASDOT+ format"
	left,right= asdot.split('.')
	ret = int(left) * 65536 + int(right)
	return ret

def asplain2asdot( asplain ):
	"This returns an ASDOT+ formated ASN given an ASPLAIN format"
	begin = int(asplain)
	current = begin
	new = current
	counter = 0
	while current > 65535:
		new = current - 65536;
		counter = counter + 1
		current = new
	ret = str(counter) + "." + str(new)
	return ret

length = len(sys.argv)

if length == 2:
	start = sys.argv[1]
	if "." not in start:
		print "Start is ASPLAIN: ", start
		print asplain2asdot( start )
	else:
		print "Start is ASDOT+: ", start
		print asdot2plain( start )

<commit_msg>add if __name__ == "__main__":<commit_after>#!/usr/bin/python

import sys

def asdot2plain( asdot ):
	"This returns an ASPLAIN formated ASN given an ASDOT+ format"
	left,right= asdot.split('.')
	ret = int(left) * 65536 + int(right)
	return ret

def asplain2asdot( asplain ):
	"This returns an ASDOT+ formated ASN given an ASPLAIN format"
	begin = int(asplain)
	current = begin
	new = current
	counter = 0
	while current > 65535:
		new = current - 65536;
		counter = counter + 1
		current = new
	ret = str(counter) + "." + str(new)
	return ret

def main():
	length = len(sys.argv)

	if length == 2:
		start = sys.argv[1]
		if "." not in start:
			print "Start is ASPLAIN: ", start
			print asplain2asdot( start )
		else:
			print "Start is ASDOT+: ", start
			print asdot2plain( start )
	else:
		print "Usage:"
		print sys.argv[0] + " <asn>"
		print ""
		print "    <asn> - ASN to convert in ASPLAIN or ASDOT format"
		print ""

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import os
import re

from django.conf import settings
from django.template.loader import render_to_string
from django.utils import timezone
from django.db.models import Q
from enum import Enum
from numpy import linspace
from eventkit_cloud.ui.helpers import logger, cd
from eventkit_cloud.utils import auth_requests
from eventkit_cloud.utils.gdalutils import get_band_statistics
import logging
from time import sleep
import signal

logger = logging.getLogger()

class Directory(Enum):
    ARCGIS = 'arcgis'
    DATA = 'data'
    TEMPLATES = 'templates'

def get_run_staging_dir(run_uid):
    """
    The run staging dir is where all files are stored while they are being processed.
    It is a unique space to ensure that files aren't being improperly modified.
    :param run_uid: The unique value to store the directory for the run data.
    :return: The path to the run directory.
    """
    return os.path.join(settings.EXPORT_STAGING_ROOT.rstrip('\/'), str(run_uid))

def get_run_download_dir(run_uid):
    """
    The run download dir is where all files are stored after they are processed.
    It is a unique space to ensure that files aren't being improperly modified.
    :param run_uid: The unique value to store the directory for the run data.
    :return: The path to the run directory.
    """
    return os.path.join(settings.EXPORT_DOWNLOAD_ROOT.rstrip('\/'), str(run_uid))

def get_run_download_url(run_uid):
    """
    A URL path to the run data
    :param run_uid: The unique identifier for the run data.
    :return: The url context. (e.g. /downloads/123e4567-e89b-12d3-a456-426655440000)
    """
    return "{0}/{1}".format(settings.EXPORT_MEDIA_ROOT.rstrip('\/'), str(run_uid))

def get_provider_staging_dir(run_uid, provider_slug):
    """
    The provider staging dir is where all files are stored while they are being processed.
    It is a unique space to ensure that files aren't being improperly modified.
    :param provider_slug: The unique value to store the directory for the provider data.
    :return: The path to the provider directory.
    """
    run_staging_dir = get_run_staging_dir(run_uid)
    return os.path.join(run_staging_dir, provider_slug)

def get_download_filename(name, time, ext, additional_descriptors=None):
    """
    This provides specific formatting for the names of the downloadable files.
    :param name: A name for the file, typically the job name.
    :param additional_descriptors: Additional descriptors, typically the provider slug or project name or any list of items.
    :param time:  A python datetime object.
    :param ext: The file extension (e.g. .gpkg)
    :return: The formatted file name (e.g. Boston-example-20180711.gpkg)
    """
    # Allow numbers or strings.
    if not isinstance(additional_descriptors, (list, tuple)):
        additional_descriptors = [str(additional_descriptors)]
    return '{0}-{1}-{2}{3}'.format(
        name,
        '-'.join(additional_descriptors),
        default_format_time(time),
        ext
    )

def get_archive_data_path(provider_slug=None, file_name=None):
    """
    Gets a datapath for the files to be placed in the zip file.
    :param provider_slug: An optional unique value to store files.
    :param file_name: The name of a file.
    :return:
    """
    file_path = Directory.DATA.value
    if provider_slug:
        file_path = os.path.join(file_path, provider_slug)
    if file_name:
        file_path = os.path.join(file_path, file_name)
    return file_path

def default_format_time(date_time):
    return date_time.strftime("%Y%m%d")

def normalize_name(name):
    # Remove all non-word characters
    s = re.sub(r"[^\w\s]", '', name)
    # Replace all whitespace with a single underscore
    s = re.sub(r"\s+", '_', s)
    return s.lower()

def get_style_files():
    """

    :return: A list of all of the static files used for styles (e.g. icons)
    """
    style_dir = os.path.join(os.path.dirname(__file__), 'static', 'tasks', 'styles')
    files = get_file_paths(style_dir)
    arcgis_dir = os.path.join(os.path.dirname(__file__), 'arcgis')
    files = get_file_paths(arcgis_dir, files)
    return files

def create_license_file(provider_task):
    # checks a DataProviderTaskRecord's license file and adds it to the file list if it exists
    from eventkit_cloud.jobs.models import DataProvider
    from eventkit_cloud.tasks.task_runners import normalize_name
    data_provider_license = DataProvider.objects.get(slug=provider_task.slug).license

    # DataProviders are not required to have a license
    if data_provider_license is None:
        return

    license_file_path = os.path.join(get_provider_staging_dir(provider_task.run.uid, provider_task.slug),
                                     '{0}.txt'.format(normalize_name(data_provider_license.name)))

    with open(license_file_path, 'w') as license_file:
        license_file.write(data_provider_license.text)

    return license_file_path

def generate_qgs_style(run_uid=None, data_provider_task_record=None):
    """
    Task to create QGIS project file with styles for osm.

    If a data_provider_task_record is provided a style file will be generated only for that, otherwise all of the
    data providers in the run will be added to the style file.
    :param run_uid: A uid representing the run.
    :param data_provider_task_record: A DataProviderTaskRecord model.
    :return: The path to the generated qgs file.
    """
    from eventkit_cloud.tasks.models import ExportRun
    from eventkit_cloud.tasks.export_tasks import TaskStates
    from eventkit_cloud.tasks.helpers import normalize_name
    run = ExportRun.objects.get(uid=run_uid)
    stage_dir = os.path.join(settings.EXPORT_STAGING_ROOT, str(run_uid))

    job_name = run.job.name.lower()

    provider_tasks = run.provider_tasks.filter(~Q(name='run'))

    # A dict is used here to ensure that just one file per provider is added,
    # this should be updated when multiple formats are supported.
    provider_details = {}
    has_raster = False
    has_elevation = False
    # This collecting of metadata should be generalized and used for both QGS styles and arcmap styles.
    if data_provider_task_record:
        provider_tasks = [data_provider_task_record]

    for provider_task in provider_tasks:
        if TaskStates[provider_task.status] not in TaskStates.get_incomplete_states():
            for export_task in provider_task.tasks.all():
                try:
                    filename = export_task.result.filename
                except Exception:
                    continue
                full_file_path = os.path.join(settings.EXPORT_STAGING_ROOT, str(run_uid),
                                              provider_task.slug, filename)
                if not os.path.isfile(full_file_path):
                    logger.error("Could not find file {0} for export {1}.".format(full_file_path,
                                                                                export_task.name))
                    continue
                # Exclude zip files created by prepare_file_for zip and the selection geojson
                # also within the QGIS style sheet it is currently assumed that GPKG files are Imagery and
                # GeoTIFF are elevation.  This will need to be updated in the future.
                file_ext = os.path.splitext(full_file_path)[1]
                if file_ext not in [".zip", ".geojson"]:
                    provider_details[provider_task.slug] = {'provider_slug': provider_task.slug, 'file_path': full_file_path,
                                       'provider_name': provider_task.name,
                                       'file_type': file_ext}
                    if provider_task.slug not in ['osm', 'nome']:
                        if file_ext == '.gpkg':
                            has_raster = True
                        if file_ext == '.tif':
                            has_elevation = True
                    if os.path.splitext(full_file_path)[1] == '.tif':
                        # Get statistics to update ranges in template.
                        band_stats = get_band_statistics(full_file_path)
                        logger.info("Band Stats {0}: {1}".format(full_file_path, band_stats))
                        provider_details[provider_task.slug]["band_stats"] = band_stats
                        # Calculate the value for each elevation step (of 16)
                        steps = linspace(band_stats[0], band_stats[1], num=16)
                        provider_details[provider_task.slug]["ramp_shader_steps"] = map(int, steps)

    if data_provider_task_record:
        style_file_name =  '{0}-{1}-{2}.qgs'.format(normalize_name(job_name), normalize_name(data_provider_task_record.slug),
                                                         default_format_time(timezone.now()))
    else:
        style_file_name = '{0}-{1}.qgs'.format(normalize_name(job_name), default_format_time(timezone.now()))
    style_file = os.path.join(stage_dir, style_file_name)

    provider_details = [provider_detail for provider_slug, provider_detail in provider_details.iteritems()]
    logger.error(provider_details)

    with open(style_file, 'w') as open_file:
        open_file.write(render_to_string('styles/Style.qgs', context={'job_name': normalize_name(job_name),
                                                                      'job_date_time': '{0}'.format(
                                                                          timezone.now().strftime("%Y%m%d%H%M%S%f")[
                                                                          :-3]),
                                                                      'provider_details': provider_details,
                                                                      'bbox': run.job.extents,
                                                                      'has_raster': has_raster,
                                                                      'has_elevation': has_elevation}))
    return style_file

def get_human_readable_metadata_document(run_uid):
    """

    :param run_uid: A UID for the export run.
    :return: A filepath to a txt document.
    """
    from eventkit_cloud.tasks.models import ExportRun
    from eventkit_cloud.jobs.models import DataProvider
    from eventkit_cloud.tasks.helpers import normalize_name
    run = ExportRun.objects.get(uid=run_uid)
    stage_dir = os.path.join(settings.EXPORT_STAGING_ROOT, str(run_uid))

    data_providers = []
    for provider_task in run.provider_tasks.filter(~Q(name='run')):
        data_provider = DataProvider.objects.get(slug=provider_task.slug)
        provider_type = data_provider.export_provider_type.type_name
        data_provider_metadata = {'name': data_provider.name,
                                  'description': data_provider.service_description.replace('\r\n', '\n').replace('\n', '\r\n\t'),
                                  'last_update': get_last_update(data_provider.url,
                                                                 provider_type,
                                                                 slug=data_provider.slug),
                                  'metadata': get_metadata_url(data_provider.url, provider_type),
                                  'copyright': data_provider.service_copyright}
        data_providers += [data_provider_metadata]

    metadata = {'name': run.job.name,
                'url': "{0}/status/{1}".format(getattr(settings, "SITE_URL"), run.job.uid),
                'description': run.job.description,
                'project': run.job.event,
                'date': timezone.now().strftime("%Y%m%d"),
                'run_uid': run.uid,
                'data_providers': data_providers,
                'aoi': run.job.bounds_geojson}

    metadata_file = os.path.join(stage_dir, '{0}_ReadMe.txt'.format(normalize_name(run.job.name)))

    with open(metadata_file, 'w') as open_file:
        open_file.write(render_to_string('styles/metadata.txt', context={'metadata': metadata}).replace('\r\n', '\n').replace('\n', '\r\n'))

    return metadata_file

def get_file_paths(directory, paths=None):
    paths = paths or dict()
    with cd(directory):
        for dirpath, _, filenames in os.walk('./'):
            for f in filenames:
                paths[os.path.abspath(os.path.join(dirpath, f))] = os.path.join(dirpath, f)
    return paths

def get_last_update(url, type, slug=None):
    """
    A wrapper to get different timestamps.
    :param url: The url to get the timestamp
    :param type: The type of services (e.g. osm)
    :param slug: Optionally a slug if the service requires credentials.
    :return: The timestamp as a string.
    """
    if type == 'osm':
        return get_osm_last_update(url, slug=slug)

def get_metadata_url(url, type):
    """
    A wrapper to get different timestamps.
    :param url: The url to get the timestamp
    :param type: The type of services (e.g. osm)
    :param slug: Optionally a slug if the service requires credentials.
    :return: The timestamp as a string.
    """
    if type in ['wcs', 'wms', 'wmts']:
        return "{0}?request=GetCapabilities".format(url.split('?')[0])
    else:
        return url

def get_osm_last_update(url, slug=None):
    """

    :param url: A path to the overpass api.
    :param slug: Optionally a slug if credentials are needed
    :return: The default timestamp as a string (2018-06-18T13:09:59Z)
    """

    timestamp_url = "{0}timestamp".format(url.rstrip('/').rstrip('interpreter'))
    try:
        response = auth_requests.get(timestamp_url, slug=slug)
        if response:
            return response.content
        raise Exception("Get OSM last update failed with {0}: {1}".format(response.status_code, response.content))
    except Exception as e:
        logger.warning(e)
        logger.warning("Could not get the timestamp from the overpass url.")
        return None

def progressive_kill(pid):
    """
    Tries to kill first with TERM and then with KILL.
    :param pid: The process ID to kill
    :return: None.
    """
    try:
        logger.info("Trying to kill pid {0} with SIGTERM.".format(pid))
        os.kill(pid, signal.SIGTERM)
        sleep(5)

        logger.info("Trying to kill pid {0} with SIGKILL.".format(pid))
        os.kill(pid, signal.SIGKILL)
        sleep(1)

    except OSError:
        logger.info("{0} PID no longer exists.".format(pid))
<commit_msg>fixed broken import<commit_after>import os
import re

from django.conf import settings
from django.template.loader import render_to_string
from django.utils import timezone
from django.db.models import Q
from enum import Enum
from numpy import linspace
from eventkit_cloud.ui.helpers import logger, cd
from eventkit_cloud.utils import auth_requests
from eventkit_cloud.utils.gdalutils import get_band_statistics
import logging
from time import sleep
import signal

logger = logging.getLogger()

class Directory(Enum):
    ARCGIS = 'arcgis'
    DATA = 'data'
    TEMPLATES = 'templates'

def get_run_staging_dir(run_uid):
    """
    The run staging dir is where all files are stored while they are being processed.
    It is a unique space to ensure that files aren't being improperly modified.
    :param run_uid: The unique value to store the directory for the run data.
    :return: The path to the run directory.
    """
    return os.path.join(settings.EXPORT_STAGING_ROOT.rstrip('\/'), str(run_uid))

def get_run_download_dir(run_uid):
    """
    The run download dir is where all files are stored after they are processed.
    It is a unique space to ensure that files aren't being improperly modified.
    :param run_uid: The unique value to store the directory for the run data.
    :return: The path to the run directory.
    """
    return os.path.join(settings.EXPORT_DOWNLOAD_ROOT.rstrip('\/'), str(run_uid))

def get_run_download_url(run_uid):
    """
    A URL path to the run data
    :param run_uid: The unique identifier for the run data.
    :return: The url context. (e.g. /downloads/123e4567-e89b-12d3-a456-426655440000)
    """
    return "{0}/{1}".format(settings.EXPORT_MEDIA_ROOT.rstrip('\/'), str(run_uid))

def get_provider_staging_dir(run_uid, provider_slug):
    """
    The provider staging dir is where all files are stored while they are being processed.
    It is a unique space to ensure that files aren't being improperly modified.
    :param provider_slug: The unique value to store the directory for the provider data.
    :return: The path to the provider directory.
    """
    run_staging_dir = get_run_staging_dir(run_uid)
    return os.path.join(run_staging_dir, provider_slug)

def get_download_filename(name, time, ext, additional_descriptors=None):
    """
    This provides specific formatting for the names of the downloadable files.
    :param name: A name for the file, typically the job name.
    :param additional_descriptors: Additional descriptors, typically the provider slug or project name or any list of items.
    :param time:  A python datetime object.
    :param ext: The file extension (e.g. .gpkg)
    :return: The formatted file name (e.g. Boston-example-20180711.gpkg)
    """
    # Allow numbers or strings.
    if not isinstance(additional_descriptors, (list, tuple)):
        additional_descriptors = [str(additional_descriptors)]
    return '{0}-{1}-{2}{3}'.format(
        name,
        '-'.join(additional_descriptors),
        default_format_time(time),
        ext
    )

def get_archive_data_path(provider_slug=None, file_name=None):
    """
    Gets a datapath for the files to be placed in the zip file.
    :param provider_slug: An optional unique value to store files.
    :param file_name: The name of a file.
    :return:
    """
    file_path = Directory.DATA.value
    if provider_slug:
        file_path = os.path.join(file_path, provider_slug)
    if file_name:
        file_path = os.path.join(file_path, file_name)
    return file_path

def default_format_time(date_time):
    return date_time.strftime("%Y%m%d")

def normalize_name(name):
    # Remove all non-word characters
    s = re.sub(r"[^\w\s]", '', name)
    # Replace all whitespace with a single underscore
    s = re.sub(r"\s+", '_', s)
    return s.lower()

def get_style_files():
    """

    :return: A list of all of the static files used for styles (e.g. icons)
    """
    style_dir = os.path.join(os.path.dirname(__file__), 'static', 'tasks', 'styles')
    files = get_file_paths(style_dir)
    arcgis_dir = os.path.join(os.path.dirname(__file__), 'arcgis')
    files = get_file_paths(arcgis_dir, files)
    return files

def create_license_file(provider_task):
    # checks a DataProviderTaskRecord's license file and adds it to the file list if it exists
    from eventkit_cloud.jobs.models import DataProvider
    from eventkit_cloud.tasks.helpers import normalize_name
    data_provider_license = DataProvider.objects.get(slug=provider_task.slug).license

    # DataProviders are not required to have a license
    if data_provider_license is None:
        return

    license_file_path = os.path.join(get_provider_staging_dir(provider_task.run.uid, provider_task.slug),
                                     '{0}.txt'.format(normalize_name(data_provider_license.name)))

    with open(license_file_path, 'w') as license_file:
        license_file.write(data_provider_license.text)

    return license_file_path

def generate_qgs_style(run_uid=None, data_provider_task_record=None):
    """
    Task to create QGIS project file with styles for osm.

    If a data_provider_task_record is provided a style file will be generated only for that, otherwise all of the
    data providers in the run will be added to the style file.
    :param run_uid: A uid representing the run.
    :param data_provider_task_record: A DataProviderTaskRecord model.
    :return: The path to the generated qgs file.
    """
    from eventkit_cloud.tasks.models import ExportRun
    from eventkit_cloud.tasks.export_tasks import TaskStates
    from eventkit_cloud.tasks.helpers import normalize_name
    run = ExportRun.objects.get(uid=run_uid)
    stage_dir = os.path.join(settings.EXPORT_STAGING_ROOT, str(run_uid))

    job_name = run.job.name.lower()

    provider_tasks = run.provider_tasks.filter(~Q(name='run'))

    # A dict is used here to ensure that just one file per provider is added,
    # this should be updated when multiple formats are supported.
    provider_details = {}
    has_raster = False
    has_elevation = False
    # This collecting of metadata should be generalized and used for both QGS styles and arcmap styles.
    if data_provider_task_record:
        provider_tasks = [data_provider_task_record]

    for provider_task in provider_tasks:
        if TaskStates[provider_task.status] not in TaskStates.get_incomplete_states():
            for export_task in provider_task.tasks.all():
                try:
                    filename = export_task.result.filename
                except Exception:
                    continue
                full_file_path = os.path.join(settings.EXPORT_STAGING_ROOT, str(run_uid),
                                              provider_task.slug, filename)
                if not os.path.isfile(full_file_path):
                    logger.error("Could not find file {0} for export {1}.".format(full_file_path,
                                                                                export_task.name))
                    continue
                # Exclude zip files created by prepare_file_for zip and the selection geojson
                # also within the QGIS style sheet it is currently assumed that GPKG files are Imagery and
                # GeoTIFF are elevation.  This will need to be updated in the future.
                file_ext = os.path.splitext(full_file_path)[1]
                if file_ext not in [".zip", ".geojson"]:
                    provider_details[provider_task.slug] = {'provider_slug': provider_task.slug, 'file_path': full_file_path,
                                       'provider_name': provider_task.name,
                                       'file_type': file_ext}
                    if provider_task.slug not in ['osm', 'nome']:
                        if file_ext == '.gpkg':
                            has_raster = True
                        if file_ext == '.tif':
                            has_elevation = True
                    if os.path.splitext(full_file_path)[1] == '.tif':
                        # Get statistics to update ranges in template.
                        band_stats = get_band_statistics(full_file_path)
                        logger.info("Band Stats {0}: {1}".format(full_file_path, band_stats))
                        provider_details[provider_task.slug]["band_stats"] = band_stats
                        # Calculate the value for each elevation step (of 16)
                        steps = linspace(band_stats[0], band_stats[1], num=16)
                        provider_details[provider_task.slug]["ramp_shader_steps"] = map(int, steps)

    if data_provider_task_record:
        style_file_name =  '{0}-{1}-{2}.qgs'.format(normalize_name(job_name), normalize_name(data_provider_task_record.slug),
                                                         default_format_time(timezone.now()))
    else:
        style_file_name = '{0}-{1}.qgs'.format(normalize_name(job_name), default_format_time(timezone.now()))
    style_file = os.path.join(stage_dir, style_file_name)

    provider_details = [provider_detail for provider_slug, provider_detail in provider_details.iteritems()]
    logger.error(provider_details)

    with open(style_file, 'w') as open_file:
        open_file.write(render_to_string('styles/Style.qgs', context={'job_name': normalize_name(job_name),
                                                                      'job_date_time': '{0}'.format(
                                                                          timezone.now().strftime("%Y%m%d%H%M%S%f")[
                                                                          :-3]),
                                                                      'provider_details': provider_details,
                                                                      'bbox': run.job.extents,
                                                                      'has_raster': has_raster,
                                                                      'has_elevation': has_elevation}))
    return style_file

def get_human_readable_metadata_document(run_uid):
    """

    :param run_uid: A UID for the export run.
    :return: A filepath to a txt document.
    """
    from eventkit_cloud.tasks.models import ExportRun
    from eventkit_cloud.jobs.models import DataProvider
    from eventkit_cloud.tasks.helpers import normalize_name
    run = ExportRun.objects.get(uid=run_uid)
    stage_dir = os.path.join(settings.EXPORT_STAGING_ROOT, str(run_uid))

    data_providers = []
    for provider_task in run.provider_tasks.filter(~Q(name='run')):
        data_provider = DataProvider.objects.get(slug=provider_task.slug)
        provider_type = data_provider.export_provider_type.type_name
        data_provider_metadata = {'name': data_provider.name,
                                  'description': data_provider.service_description.replace('\r\n', '\n').replace('\n', '\r\n\t'),
                                  'last_update': get_last_update(data_provider.url,
                                                                 provider_type,
                                                                 slug=data_provider.slug),
                                  'metadata': get_metadata_url(data_provider.url, provider_type),
                                  'copyright': data_provider.service_copyright}
        data_providers += [data_provider_metadata]

    metadata = {'name': run.job.name,
                'url': "{0}/status/{1}".format(getattr(settings, "SITE_URL"), run.job.uid),
                'description': run.job.description,
                'project': run.job.event,
                'date': timezone.now().strftime("%Y%m%d"),
                'run_uid': run.uid,
                'data_providers': data_providers,
                'aoi': run.job.bounds_geojson}

    metadata_file = os.path.join(stage_dir, '{0}_ReadMe.txt'.format(normalize_name(run.job.name)))

    with open(metadata_file, 'w') as open_file:
        open_file.write(render_to_string('styles/metadata.txt', context={'metadata': metadata}).replace('\r\n', '\n').replace('\n', '\r\n'))

    return metadata_file

def get_file_paths(directory, paths=None):
    paths = paths or dict()
    with cd(directory):
        for dirpath, _, filenames in os.walk('./'):
            for f in filenames:
                paths[os.path.abspath(os.path.join(dirpath, f))] = os.path.join(dirpath, f)
    return paths

def get_last_update(url, type, slug=None):
    """
    A wrapper to get different timestamps.
    :param url: The url to get the timestamp
    :param type: The type of services (e.g. osm)
    :param slug: Optionally a slug if the service requires credentials.
    :return: The timestamp as a string.
    """
    if type == 'osm':
        return get_osm_last_update(url, slug=slug)

def get_metadata_url(url, type):
    """
    A wrapper to get different timestamps.
    :param url: The url to get the timestamp
    :param type: The type of services (e.g. osm)
    :param slug: Optionally a slug if the service requires credentials.
    :return: The timestamp as a string.
    """
    if type in ['wcs', 'wms', 'wmts']:
        return "{0}?request=GetCapabilities".format(url.split('?')[0])
    else:
        return url

def get_osm_last_update(url, slug=None):
    """

    :param url: A path to the overpass api.
    :param slug: Optionally a slug if credentials are needed
    :return: The default timestamp as a string (2018-06-18T13:09:59Z)
    """

    timestamp_url = "{0}timestamp".format(url.rstrip('/').rstrip('interpreter'))
    try:
        response = auth_requests.get(timestamp_url, slug=slug)
        if response:
            return response.content
        raise Exception("Get OSM last update failed with {0}: {1}".format(response.status_code, response.content))
    except Exception as e:
        logger.warning(e)
        logger.warning("Could not get the timestamp from the overpass url.")
        return None

def progressive_kill(pid):
    """
    Tries to kill first with TERM and then with KILL.
    :param pid: The process ID to kill
    :return: None.
    """
    try:
        logger.info("Trying to kill pid {0} with SIGTERM.".format(pid))
        os.kill(pid, signal.SIGTERM)
        sleep(5)

        logger.info("Trying to kill pid {0} with SIGKILL.".format(pid))
        os.kill(pid, signal.SIGKILL)
        sleep(1)

    except OSError:
        logger.info("{0} PID no longer exists.".format(pid))
<|endoftext|>


<commit_before>'''Specification of the asq version'''

__version__ = '1.0'
<commit_msg>Bump to version 1.2<commit_after>'''Specification of the asq version'''

__version__ = '1.2'
<|endoftext|>