<commit_before>#!/usr/bin/env python

import argparse
from datetime import datetime
import os
import json
import requests
from bs4 import BeautifulSoup

SPAMHAUS_ROOT = "https://www.spamhaus.org/sbl/listings/{0}"

class Listing(object):
    def __init__(self, incident_id, ip_address, domain_name, timestamp, description):
        self.incident_id = incident_id
        self.ip_address = ip_address
        self.domain_name = domain_name
        self.timestamp = datetime.strptime(timestamp, "%d-%b-%Y %H:%M %Z")
        self.description = description

    def __lt__(self, other):
        return self.timestamp < other.timestamp

    def __repr__(self):
        return "{incident_id:<12}{ip_address:<22}{domain_name} \t {timestamp}\t {description}".format(**self.__dict__)

    def to_json(self):
        return json.dumps({
            "incident_id": self.incident_id,
            "ip_address": self.ip_address,
            "domain_name": self.domain_name,
            "timestamp": self.timestamp.isoformat(),
            "description": self.description
        }, sort_keys=True, indent=4)

def parse_spamhaus(domain, format):
    uri = SPAMHAUS_ROOT.format(domain)

    request = requests.get(uri)
    soup = BeautifulSoup(request.text, "html.parser")
    listings = []
    for incident in soup.find_all("table", border="0", cellpadding="4", cellspacing="0", width="100%"):
        spans = incident.findAll("span")
        listing = Listing(incident_id=spans[0].b.text,
                          ip_address=spans[1].b.text,
                          domain_name=spans[2].b.text,
                          timestamp=spans[3].text,
                          description=' '.join(s.strip() for s in spans[4].text.strip().splitlines()))
        listings.append(listing)

    if format == 'json':
        print (",%s" % os.linesep).join(l.to_json() for l in sorted(listings, reverse=True))
    else:
        print os.linesep.join(str(l) for l in sorted(listings, reverse=True))

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Parse the Spamhaus DB for your domain")
    parser.add_argument("-d", "--domain", help="The domain you want to query")
    parser.add_argument("-f", "--format", type=str, choices=["json", "tab"],
                        default="tab", help="Output format")
    args = parser.parse_args()

    if args.domain:
        parse_spamhaus(args.domain, args.format)
    else:
        parser.print_help()
<commit_msg>Fix JSON output<commit_after>#!/usr/bin/env python

import argparse
from datetime import datetime
import os
import json
import requests
from bs4 import BeautifulSoup

SPAMHAUS_ROOT = "https://www.spamhaus.org/sbl/listings/{0}"

class Listing(object):
    def __init__(self, incident_id, ip_address, domain_name, timestamp, description):
        self.incident_id = incident_id
        self.ip_address = ip_address
        self.domain_name = domain_name
        self.timestamp = datetime.strptime(timestamp, "%d-%b-%Y %H:%M %Z")
        self.description = description

    def __lt__(self, other):
        return self.timestamp < other.timestamp

    def __repr__(self):
        return "{incident_id:<12}{ip_address:<22}{domain_name} \t {timestamp}\t {description}".format(**self.__dict__)

    def to_json(self):
        return json.dumps({
            "incident_id": self.incident_id,
            "ip_address": self.ip_address,
            "domain_name": self.domain_name,
            "timestamp": self.timestamp.isoformat(),
            "description": self.description
        }, sort_keys=True, indent=4)

def parse_spamhaus(domain, format):
    uri = SPAMHAUS_ROOT.format(domain)

    request = requests.get(uri)
    soup = BeautifulSoup(request.text, "html.parser")
    listings = []
    for incident in soup.find_all("table", border="0", cellpadding="4", cellspacing="0", width="100%"):
        spans = incident.findAll("span")
        listing = Listing(incident_id=spans[0].b.text,
                          ip_address=spans[1].b.text,
                          domain_name=spans[2].b.text,
                          timestamp=spans[3].text,
                          description=' '.join(s.strip() for s in spans[4].text.strip().splitlines()))
        listings.append(listing)

    if format == 'json':
        print "[" + (",%s" % os.linesep).join(l.to_json() for l in sorted(listings, reverse=True)) + "]"
    else:
        print os.linesep.join(str(l) for l in sorted(listings, reverse=True))

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Parse the Spamhaus DB for your domain")
    parser.add_argument("-d", "--domain", help="The domain you want to query")
    parser.add_argument("-f", "--format", type=str, choices=["json", "tab"],
                        default="tab", help="Output format")
    args = parser.parse_args()

    if args.domain:
        parse_spamhaus(args.domain, args.format)
    else:
        parser.print_help()
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Look at reads mapping to decoy STR chromosomes and use their pairs to
determine which STR locus they originated from.
Generates counts of reads mapping to the decoy originating each STR locus.
"""

import argparse
import pysam
#from insert_size import parse_bed, span_intervals
import pybedtools as bt
import pandas as pd
import numpy as np
import os
import random
import string
from decoy_STR import normalise_str
import sys

__author__ = "Harriet Dashnow"
__credits__ = ["Harriet Dashnow"]
__license__ = "MIT"
__version__ = "0.1.0"
__email__ = "h.dashnow@gmail.com"

def parse_args():
    """Parse the input arguments, use '-h' for help"""
    parser = argparse.ArgumentParser(description='Reports counts of reads mapping to the STR decoy originating each STR locus.')
    parser.add_argument(
        '--bam', type=str, required=True, nargs='+',
        help='One or more BAM files of mapped reads e.g. produced by Bowtie2 or BWA. If multiple BAMs all are assumed to come from the same sample.')
    parser.add_argument(
        '--bed', type=str, required=True,
        help='bed file containing genomic locations of STR loci. Genomic locations should be relative to the fasta reference used to create the bam')
    parser.add_argument(
        '--output', type=str, required=False,
        help='Output file name. Defaults to stdout.')
    parser.add_argument(
        '--dist', type=int, required=False, default=500,
        help='Counts are only assigned to an STR locus that is at most this many bp away. The best choice is probably the insert size. (default: %(default)s)')
    return parser.parse_args()

def randomletters(length):
   return ''.join(random.choice(string.ascii_lowercase) for i in range(length))

def detect_readlen(bamfile, sample = 100):
    """Samples reads from a bam file to detect the read length. Assumes uniform
    read lengths and that any shorter reads are due to trimming or clipping.
    Args:
        bamfile (str): Location of bamfile (will be opened and closed, so don't
            use an existing open bamfile handle).
        sample (int): Number of reads to sample.

    Returns:
        int: The maximum read length observed.
    """
    bam = pysam.Samfile(bamfile, 'rb')
    maxlen = 0
    count = 0
    count_noCIGAR = 0

    for read in bam.fetch():
        count += 1
        readlen = read.infer_query_length(always=False)
        if readlen:
            if readlen > maxlen:
                maxlen = readlen
        else:
            count_noCIGAR += 1

        if count >= sample:
            bam.close()
            return maxlen, count_noCIGAR

def spans_region(test_region, target_region):
    """Check if test_region spans target region.
    Regions are inclusive of both endpoints.

    Args:
        test_region tuple/list (int, int)
        target_region tuple/list (int, int)
    Returns:
        bool
    Raises:
        TypeError: Incorrectly defined regions.
    """
    try:
        positions = [test_region[0], target_region[0], target_region[1], test_region[1]]
    except TypeError:
        raise TypeError( ('Possible cause: expecting 2 positions per region, '
            'regions specified were {} and {}').format(
            test_region, target_region))
    if len(test_region) != 2 or len(target_region) != 2:
        raise TypeError( ('Possible cause: expecting 2 positions per region, '
            'regions specified were {} and {}').format(
            test_region, target_region))

    if positions == sorted(positions): # read spans position
        return True
    else:
        return False

def in_region(position, region):
    """Check if position is in the region.
    Regions are inclusive of both endpoints.

    Args:
        position int
        region tuple/list (int, int)
    Returns:
        bool: True if position is within region, else False
    """
    return spans_region(region, (position, position) )

def indel_size(read, region, chrom = None):
    """
    Use the CIGAR string to report the total size of all indels in the read, if that read spans the region specified
    Args:
        read (pysam alignment object)
        region (start,stop): tuple containing two integer coordinates of the
            target on the reference genome
        chrom (str): chromosome name of the target coordinates.
            Optional, if provided will check read aligned to that chromosome
    Returns:
        indel_size (int): Total size of indel(s) over the STR region of the read
            (+ive for insertion -ive for deletion)
    Raises:
        TypeError: CIGAR string missing
        ValueError: Read does not span the region specified
    """
    ref_start, ref_stop = region
    cigar = read.cigar
    read_start = read.pos
    #XXX throw exception if the read does not align to that chromosome
    if chrom:
        pass

    # Iterate through the CIGAR string to extract indel info and the 'footprint' of read on the ref
    # footprint = matches + deletions
    total_deletion = 0
    total_insertion = 0
    footprint = 0
    repeat_indel = 0
    try:
        for field in cigar:
            this_insertion = 0
            this_deletion = 0
            if field[0] == 0: # matches
                footprint += field[1]
            elif field[0] == 1: # insertions
                this_insertion = field[1]
                total_insertion += this_insertion
            elif field[0] == 2: # deletions
                this_deletion = field[1]
                total_deletion += this_deletion
                footprint += this_deletion

            # test if current position is in the repeat
            current_pos = read_start + footprint

            if in_region(current_pos, (ref_start, ref_stop)):
                repeat_indel += this_insertion
                repeat_indel -= this_deletion

    except TypeError:
        raise TypeError("CIGAR string missing")

    # check if the read covers the entire repeat
    read_end = read_start + footprint
    if spans_region((read_start, read_end), (ref_start, ref_stop)): # read spans position
        return repeat_indel
    else:
        raise ValueError("Read does not span the region specified") # repeat not contained in read

def main():
    # Parse command line arguments
    args = parse_args()
    bamfiles = args.bam
    bedfile = args.bed
    outfile = args.output

    max_distance = args.dist

    # Check bamfiles have unique names
    if len(set(bamfiles)) < len(bamfiles):
        sys.exit('ERROR: There were multiple bamfiles with the same filename. Please check your input')

    #STR_bed = parse_bed(args.bed, position_base=0)
    STR_bed = bt.BedTool(bedfile).sort()

    all_results = []
    for bamfile in bamfiles:

        readlen, count_noCIGAR = detect_readlen(bamfile)

        # Read bam
        bam = pysam.Samfile(bamfile, 'rb')

        # Get relevant chromosomes
        required_chroms = []
        unpaired = 0
        total = 0
        for chrom in bam.references:
            if chrom.startswith('STR-'):
                required_chroms.append(chrom)
        # Check if any STR- chromosomes
        if len(required_chroms) == 0:
           sys.exit('ERROR: There were no reads mapping to chromosomes with names starting with "STR-" in {0}. Are you sure this data is mapped to a reference genome with STR decoy chromosomes?'.format(bamfile))

        for chrom in required_chroms:
            motif = chrom.split('-')[1]
            all_positions = []
            all_segments = bam.fetch(reference=chrom)

            for read in all_segments:
                #if read.is_secondary:
                #    continue
                total += 1
                try:
                    mate_chr = read.next_reference_name
                except ValueError:
                    unpaired += 1
                    continue
                mate_start = read.next_reference_start
                mate_stop = mate_start + readlen
                all_positions.append([mate_chr, mate_start, mate_stop])

            # Strategy:
            # Merge all overlapping intervals
            # Keep the count of reads corresponding to each merged interval (i.e. 1 for each read contained in it)
            # Assign each interval to the closest STR (within 500 bp? - the insert size) with the correct motif, adding together the count of reads
            # Check motif: == normalise_str()
            # There should be two 1-2 intervals per STR, likely one for each flank.
            # Report the read count for each STR

            if len(all_positions) > 0:
                motif_bed = bt.BedTool(all_positions).sort()
                # Merge all the intervals, then count how many of the original intervals overlap the merged ones (4th column)
                motif_coverage = motif_bed.merge(stream=True).coverage(b=motif_bed, counts=True,
                    nonamecheck=True)

                tmp_bed = 'tmp-' + randomletters(8) + '.bed' #create temporary file for bedtools to write to and pandas to read since streams don't seem to work
                closest_STR = motif_coverage.closest(STR_bed, d=True, stream=True,
                    nonamecheck=True).saveas(tmp_bed)
                colnames = ['chr', 'start', 'stop', 'count', 'STR_chr', 'STR_start',
                'STR_stop', 'motif', 'reflen', 'distance']
                df = pd.read_csv(tmp_bed, sep='\t', header=None, names=colnames)
                os.remove(tmp_bed) #delete temporary file

                # Filter out loci that are too far away
                df = df.loc[df['distance'] <= max_distance, :]
                df['motif'] = df['motif'].map(normalise_str) # Normalise the STR motif to enable comparisons
                # Remove STRs that don't match the motif
                df = df.loc[df['motif'] == normalise_str(motif), :]
                df = df.loc[:, ['STR_chr', 'STR_start', 'STR_stop', 'motif', 'count', 'reflen']]

                all_results.append(df)

        if total == 0:
            sys.exit('ERROR: there were no reads overlapping the target STR regions. This may indicate a problem with the input file.\n')
        elif unpaired == total:
            sys.exit('ERROR: all {0} reads overlapping the target STR regions appear to be unpaired. You may wish to check your bam file is paired-end and correctly formed.\n'.format(total))
        elif unpaired > 0:
            sys.stderr.write('WARNING: it appears that {0} of the {1} reads overlapping the target STR regions were unpaired and so no useful data could be obtained from them.\n'.format(unpaired, total))

    # Sum counts from multiple bam files and multiple rows
    if len(all_results) == 1:
        df_total = all_results[0]
    else:
        df_total = pd.concat(all_results, ignore_index=True)

    summed = df_total.groupby(['STR_chr', 'STR_start', 'STR_stop', 'motif', 'reflen'], as_index=False).aggregate(np.sum)

    # Print a warning message in case of reads without a CIGAR string
    if count_noCIGAR > 0:
        sys.stderr.write('WARNING: ' + str(count_noCIGAR) + ' read(s) in ' + bamfile + ' file had no CIGAR string.\n')

    if total == 0:
        sys.exit('ERROR: there were no reads overlapping the target STR regions. This may indicate a problem with the input file.\n')
    elif unpaired == total:
        sys.exit('ERROR: all {0} reads overlapping the target STR regions appear to be unpaired. You may wish to check your bam file is paired-end and correctly formed.\n'.format(total))
    elif unpaired > 0:
        sys.stderr.write('WARNING: it appears that {0} of the {1} reads overlapping the target STR regions were unpaired and so no useful data could be obtained from them.\n'.format(unpaired, total))

    # Write results
    if outfile:
        outstream = open(outfile, 'w')
    else:
        outstream = sys.stdout

    out_header = '\t'.join(['STR_chr', 'STR_start', 'STR_stop', 'motif', 'reflen', 'count'])
    outstream.write(out_header + '\n')
    outstring = summed.to_csv(sep='\t', header=False, index=False)
    outstream.write(outstring)
    outstream.close()

if __name__ == '__main__':
    main()
<commit_msg>use all bam cigar strings that consume the reference to calculate the read footprint<commit_after>#!/usr/bin/env python
"""Look at reads mapping to decoy STR chromosomes and use their pairs to
determine which STR locus they originated from.
Generates counts of reads mapping to the decoy originating each STR locus.
"""

import argparse
import pysam
#from insert_size import parse_bed, span_intervals
import pybedtools as bt
import pandas as pd
import numpy as np
import os
import random
import string
from decoy_STR import normalise_str
import sys

__author__ = "Harriet Dashnow"
__credits__ = ["Harriet Dashnow"]
__license__ = "MIT"
__version__ = "0.1.0"
__email__ = "h.dashnow@gmail.com"

def parse_args():
    """Parse the input arguments, use '-h' for help"""
    parser = argparse.ArgumentParser(description='Reports counts of reads mapping to the STR decoy originating each STR locus.')
    parser.add_argument(
        '--bam', type=str, required=True, nargs='+',
        help='One or more BAM files of mapped reads e.g. produced by Bowtie2 or BWA. If multiple BAMs all are assumed to come from the same sample.')
    parser.add_argument(
        '--bed', type=str, required=True,
        help='bed file containing genomic locations of STR loci. Genomic locations should be relative to the fasta reference used to create the bam')
    parser.add_argument(
        '--output', type=str, required=False,
        help='Output file name. Defaults to stdout.')
    parser.add_argument(
        '--dist', type=int, required=False, default=500,
        help='Counts are only assigned to an STR locus that is at most this many bp away. The best choice is probably the insert size. (default: %(default)s)')
    return parser.parse_args()

def randomletters(length):
   return ''.join(random.choice(string.ascii_lowercase) for i in range(length))

def detect_readlen(bamfile, sample = 100):
    """Samples reads from a bam file to detect the read length. Assumes uniform
    read lengths and that any shorter reads are due to trimming or clipping.
    Args:
        bamfile (str): Location of bamfile (will be opened and closed, so don't
            use an existing open bamfile handle).
        sample (int): Number of reads to sample.

    Returns:
        int: The maximum read length observed.
    """
    bam = pysam.Samfile(bamfile, 'rb')
    maxlen = 0
    count = 0
    count_noCIGAR = 0

    for read in bam.fetch():
        count += 1
        readlen = read.infer_query_length(always=False)
        if readlen:
            if readlen > maxlen:
                maxlen = readlen
        else:
            count_noCIGAR += 1

        if count >= sample:
            bam.close()
            return maxlen, count_noCIGAR

def spans_region(test_region, target_region):
    """Check if test_region spans target region.
    Regions are inclusive of both endpoints.

    Args:
        test_region tuple/list (int, int)
        target_region tuple/list (int, int)
    Returns:
        bool
    Raises:
        TypeError: Incorrectly defined regions.
    """
    try:
        positions = [test_region[0], target_region[0], target_region[1], test_region[1]]
    except TypeError:
        raise TypeError( ('Possible cause: expecting 2 positions per region, '
            'regions specified were {} and {}').format(
            test_region, target_region))
    if len(test_region) != 2 or len(target_region) != 2:
        raise TypeError( ('Possible cause: expecting 2 positions per region, '
            'regions specified were {} and {}').format(
            test_region, target_region))

    if positions == sorted(positions): # read spans position
        return True
    else:
        return False

def in_region(position, region):
    """Check if position is in the region.
    Regions are inclusive of both endpoints.

    Args:
        position int
        region tuple/list (int, int)
    Returns:
        bool: True if position is within region, else False
    """
    return spans_region(region, (position, position) )

def indel_size(read, region, chrom = None):
    """
    Use the CIGAR string to report the total size of all indels in the read, if that read spans the region specified
    Args:
        read (pysam alignment object)
        region (start,stop): tuple containing two integer coordinates of the
            target on the reference genome
        chrom (str): chromosome name of the target coordinates.
            Optional, if provided will check read aligned to that chromosome
    Returns:
        indel_size (int): Total size of indel(s) over the STR region of the read
            (+ive for insertion -ive for deletion)
    Raises:
        TypeError: CIGAR string missing
        ValueError: Read does not span the region specified
    """
    ref_start, ref_stop = region
    cigar = read.cigartuples
    read_start = read.pos
    #XXX throw exception if the read does not align to that chromosome
    if chrom:
        pass

    # Iterate through the CIGAR string to extract indel info and the 'footprint' of read on the ref
    # footprint = alignenment match + deletion + skipped + sequence match + sequence mismatch
    # i.e. anything that consumes reference in the CIGAR
    consume_ref_cigar = (0,2,3,7,8)
    total_deletion = 0
    total_insertion = 0
    footprint = 0
    repeat_indel = 0
    try:
        for field in cigar:
            this_insertion = 0
            this_deletion = 0
            if field[0] == 1: # insertions
                this_insertion = field[1]
                total_insertion += this_insertion
            elif field[0] == 2: # deletions
                this_deletion = field[1]
                total_deletion += this_deletion
            if field[0] in consume_ref_cigar:
                footprint += field[1]

            # test if current position is in the repeat
            current_pos = read_start + footprint

            if in_region(current_pos, (ref_start, ref_stop)):
                repeat_indel += this_insertion
                repeat_indel -= this_deletion

    except TypeError:
        raise TypeError("CIGAR string missing")

    # check if the read covers the entire repeat
    read_end = read_start + footprint
    if spans_region((read_start, read_end), (ref_start, ref_stop)): # read spans position
        return repeat_indel
    else:
        raise ValueError("Read does not span the region specified") # repeat not contained in read

def main():
    # Parse command line arguments
    args = parse_args()
    bamfiles = args.bam
    bedfile = args.bed
    outfile = args.output

    max_distance = args.dist

    # Check bamfiles have unique names
    if len(set(bamfiles)) < len(bamfiles):
        sys.exit('ERROR: There were multiple bamfiles with the same filename. Please check your input')

    #STR_bed = parse_bed(args.bed, position_base=0)
    STR_bed = bt.BedTool(bedfile).sort()

    all_results = []
    for bamfile in bamfiles:

        readlen, count_noCIGAR = detect_readlen(bamfile)

        # Read bam
        bam = pysam.Samfile(bamfile, 'rb')

        # Get relevant chromosomes
        required_chroms = []
        unpaired = 0
        total = 0
        for chrom in bam.references:
            if chrom.startswith('STR-'):
                required_chroms.append(chrom)
        # Check if any STR- chromosomes
        if len(required_chroms) == 0:
           sys.exit('ERROR: There were no reads mapping to chromosomes with names starting with "STR-" in {0}. Are you sure this data is mapped to a reference genome with STR decoy chromosomes?'.format(bamfile))

        for chrom in required_chroms:
            motif = chrom.split('-')[1]
            all_positions = []
            all_segments = bam.fetch(reference=chrom)

            for read in all_segments:
                #if read.is_secondary:
                #    continue
                total += 1
                try:
                    mate_chr = read.next_reference_name
                except ValueError:
                    unpaired += 1
                    continue
                mate_start = read.next_reference_start
                mate_stop = mate_start + readlen
                all_positions.append([mate_chr, mate_start, mate_stop])

            # Strategy:
            # Merge all overlapping intervals
            # Keep the count of reads corresponding to each merged interval (i.e. 1 for each read contained in it)
            # Assign each interval to the closest STR (within 500 bp? - the insert size) with the correct motif, adding together the count of reads
            # Check motif: == normalise_str()
            # There should be two 1-2 intervals per STR, likely one for each flank.
            # Report the read count for each STR

            if len(all_positions) > 0:
                motif_bed = bt.BedTool(all_positions).sort()
                # Merge all the intervals, then count how many of the original intervals overlap the merged ones (4th column)
                motif_coverage = motif_bed.merge(stream=True).coverage(b=motif_bed, counts=True,
                    nonamecheck=True)

                tmp_bed = 'tmp-' + randomletters(8) + '.bed' #create temporary file for bedtools to write to and pandas to read since streams don't seem to work
                closest_STR = motif_coverage.closest(STR_bed, d=True, stream=True,
                    nonamecheck=True).saveas(tmp_bed)
                colnames = ['chr', 'start', 'stop', 'count', 'STR_chr', 'STR_start',
                'STR_stop', 'motif', 'reflen', 'distance']
                df = pd.read_csv(tmp_bed, sep='\t', header=None, names=colnames)
                os.remove(tmp_bed) #delete temporary file

                # Filter out loci that are too far away
                df = df.loc[df['distance'] <= max_distance, :]
                df['motif'] = df['motif'].map(normalise_str) # Normalise the STR motif to enable comparisons
                # Remove STRs that don't match the motif
                df = df.loc[df['motif'] == normalise_str(motif), :]
                df = df.loc[:, ['STR_chr', 'STR_start', 'STR_stop', 'motif', 'count', 'reflen']]

                all_results.append(df)

        if total == 0:
            sys.exit('ERROR: there were no reads overlapping the target STR regions. This may indicate a problem with the input file.\n')
        elif unpaired == total:
            sys.exit('ERROR: all {0} reads overlapping the target STR regions appear to be unpaired. You may wish to check your bam file is paired-end and correctly formed.\n'.format(total))
        elif unpaired > 0:
            sys.stderr.write('WARNING: it appears that {0} of the {1} reads overlapping the target STR regions were unpaired and so no useful data could be obtained from them.\n'.format(unpaired, total))

    # Sum counts from multiple bam files and multiple rows
    if len(all_results) == 1:
        df_total = all_results[0]
    else:
        df_total = pd.concat(all_results, ignore_index=True)

    summed = df_total.groupby(['STR_chr', 'STR_start', 'STR_stop', 'motif', 'reflen'], as_index=False).aggregate(np.sum)

    # Print a warning message in case of reads without a CIGAR string
    if count_noCIGAR > 0:
        sys.stderr.write('WARNING: ' + str(count_noCIGAR) + ' read(s) in ' + bamfile + ' file had no CIGAR string.\n')

    if total == 0:
        sys.exit('ERROR: there were no reads overlapping the target STR regions. This may indicate a problem with the input file.\n')
    elif unpaired == total:
        sys.exit('ERROR: all {0} reads overlapping the target STR regions appear to be unpaired. You may wish to check your bam file is paired-end and correctly formed.\n'.format(total))
    elif unpaired > 0:
        sys.stderr.write('WARNING: it appears that {0} of the {1} reads overlapping the target STR regions were unpaired and so no useful data could be obtained from them.\n'.format(unpaired, total))

    # Write results
    if outfile:
        outstream = open(outfile, 'w')
    else:
        outstream = sys.stdout

    out_header = '\t'.join(['STR_chr', 'STR_start', 'STR_stop', 'motif', 'reflen', 'count'])
    outstream.write(out_header + '\n')
    outstring = summed.to_csv(sep='\t', header=False, index=False)
    outstream.write(outstring)
    outstream.close()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># python3
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Class that handles API calls to Spanner."""

from __future__ import annotations

import abc
import logging
from typing import Any, Callable, Iterable, List, Optional, TypeVar

from spanner_orm import error

from google.auth import credentials as auth_credentials
from google.cloud import spanner
from google.cloud.spanner_v1 import database as spanner_database
from google.cloud.spanner_v1 import transaction as spanner_transaction

_logger = logging.getLogger(__name__)

CallableReturn = TypeVar('CallableReturn')

class SpannerReadApi(abc.ABC):
  """Handles sending read requests to Spanner."""

  @property
  @abc.abstractmethod
  def _connection(self) -> spanner_database.Database:
    raise NotImplementedError

  def run_read_only(self, method: Callable[..., CallableReturn], *args: Any,
                    **kwargs: Any) -> CallableReturn:
    """Wraps read-only queries in a read transaction."""
    with self._connection.snapshot(multi_use=True) as snapshot:
      return method(snapshot, *args, **kwargs)

  # Read methods
  def find(self, transaction: spanner_transaction.Transaction, table_name: str,
           columns: Iterable[str],
           keyset: spanner.KeySet) -> List[Iterable[Any]]:
    """Obtains rows with primary_keys from the given table."""
    _logger.debug('Find table=%s columns=%s keys=%s', table_name, columns,
                  keyset.keys)
    stream_results = transaction.read(
        table=table_name, columns=columns, keyset=keyset)
    return list(stream_results)

  def sql_query(self, transaction: spanner_transaction.Transaction, query: str,
                parameters: Iterable[str],
                parameter_types: Iterable[Any]) -> List[Iterable[Any]]:
    """Runs a read only SQL query."""
    _logger.debug('Executing SQL:\n%s\n%s\n%s', query, parameters,
                  parameter_types)
    stream_results = transaction.execute_sql(
        query, params=parameters, param_types=parameter_types)
    return list(stream_results)

class SpannerWriteApi(abc.ABC):
  """Handles sending write requests to Spanner."""

  @property
  @abc.abstractmethod
  def _connection(self) -> spanner_database.SpannerDatabase:
    raise NotImplementedError

  def run_write(self, method: Callable[..., CallableReturn], *args: Any,
                **kwargs: Any) -> CallableReturn:
    """Wraps write and read-write queries in a transaction."""
    return self._connection.run_in_transaction(method, *args, **kwargs)

  def delete(self, transaction: spanner_transaction.Transaction,
             table_name: str, keyset: spanner.KeySet) -> None:
    _logger.debug('Delete table=%s keys=%s', table_name, keyset.keys)
    transaction.delete(table=table_name, keyset=keyset)

  def insert(self, transaction: spanner_transaction.Transaction,
             table_name: str, columns: Iterable[str],
             values: Iterable[Iterable[Any]]) -> None:
    """Add rows to a table."""
    _logger.debug('Insert table=%s columns=%s values=%s', table_name, columns,
                  values)
    transaction.insert(table=table_name, columns=columns, values=values)

  def update(self, transaction: spanner_transaction.Transaction,
             table_name: str, columns: Iterable[str],
             values: Iterable[Iterable[Any]]) -> None:
    """Updates rows of a table."""
    _logger.debug('Update table=%s columns=%s values=%s', table_name, columns,
                  values)
    transaction.update(table=table_name, columns=columns, values=values)

  def upsert(self, transaction: spanner_transaction.Transaction,
             table_name: str, columns: Iterable[str],
             values: Iterable[Iterable[Any]]) -> None:
    """Updates existing rows of a table or adds rows if they don't exist."""
    _logger.debug('Upsert table=%s columns=%s values=%s', table_name, columns,
                  values)
    transaction.insert_or_update(
        table=table_name, columns=columns, values=values)

class SpannerConnection:
  """Class that handles connecting to a Spanner database."""

  def __init__(self,
               instance: str,
               database: str,
               project: Optional[str] = None,
               credentials: Optional[auth_credentials.Credentials] = None,
               pool: Optional[spanner.Pool] = None,
               create_ddl: Optional[Iterable[str]] = None):
    """Connects to the specified Spanner database."""
    client = spanner.Client(project=project, credentials=credentials)
    instance = client.instance(instance)
    self.database = instance.database(
        database, pool=pool, ddl_statements=create_ddl or ())

class SpannerApi(SpannerReadApi, SpannerWriteApi):
  """Class that handles reading from and writing to Spanner tables."""

  def __init__(self, connection: SpannerConnection):
    self._spanner_connection = connection

  @property
  def _connection(self):
    return self._spanner_connection.database

_api = None  # type: Optional[SpannerApi]

def connect(instance: str,
            database: str,
            project: Optional[str] = None,
            credentials: Optional[auth_credentials.Credentials] = None,
            pool: Optional[spanner.Pool] = None) -> SpannerApi:
  connection = SpannerConnection(
      instance, database, project=project, credentials=credentials, pool=pool)
  return from_connection(connection)

def from_connection(connection: SpannerConnection) -> SpannerApi:
  global _api
  _api = SpannerApi(connection)
  return _api

def hangup() -> None:
  global _api
  _api = None

def spanner_api() -> SpannerApi:
  if not _api:
    raise error.SpannerError('Must connect to Spanner before calling APIs')
  return _api
<commit_msg>Add docstrings to API methods<commit_after># python3
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Class that handles API calls to Spanner."""

from __future__ import annotations

import abc
import logging
from typing import Any, Callable, Dict, Iterable, List, Optional, TypeVar

from spanner_orm import error

from google.auth import credentials as auth_credentials
from google.cloud import spanner
from google.cloud.spanner_v1 import database as spanner_database
from google.cloud.spanner_v1 import transaction as spanner_transaction
from google.cloud.spanner_v1.proto import type_pb2

_logger = logging.getLogger(__name__)

CallableReturn = TypeVar('CallableReturn')

class SpannerReadApi(abc.ABC):
  """Handles sending read requests to Spanner."""

  @property
  @abc.abstractmethod
  def _connection(self) -> spanner_database.Database:
    raise NotImplementedError

  def run_read_only(self, method: Callable[..., CallableReturn], *args: Any,
                    **kwargs: Any) -> CallableReturn:
    """Wraps read-only queries in a read transaction.

    The callable will be executed with the read transaction (Snapshot from
    the Spanner client library)passed to it as the first argument.

    Args:
        method: The method that will be run in the transaction
        *args: Positional arguments that will be passed to `method`
        **kwargs: Keyword arguments that will be passed to `method`

    Returns:
        The return value from `method` will be returned from this method
    """
    with self._connection.snapshot(multi_use=True) as snapshot:
      return method(snapshot, *args, **kwargs)

  # Read methods
  def find(self, transaction: spanner_transaction.Transaction, table_name: str,
           columns: Iterable[str],
           keyset: spanner.KeySet) -> List[Iterable[Any]]:
    """Retrieves rows from the given table based on the provided KeySet.

    Args:
        transaction: The Spanner transaction to execute the request on
        table_name: The Spanner table being queried
        columns: Which columns to retrieve from the Spanner table
        keyset: Contains a list of primary keys that indicates which rows to
          retrieve from the Spanner table

    Returns:
        A list of lists. Each sublist is the set of `columns` requested from
        a row in the Spanner table whose primary key matches one of the
        primary keys in the `keyset`. The order of the values in the sublist
        matches the order of the columns from the `columns` parameter.
    """
    _logger.debug('Find table=%s columns=%s keys=%s', table_name, columns,
                  keyset.keys)
    stream_results = transaction.read(
        table=table_name, columns=columns, keyset=keyset)
    return list(stream_results)

  def sql_query(
      self, transaction: spanner_transaction.Transaction, query: str,
      parameters: Dict[str, Any],
      parameter_types: Dict[str, type_pb2.Type]) -> List[Iterable[Any]]:
    """Executes a given SQL query against the Spanner database.

    This isn't technically read-only, but it's necessary to implement the read-
    only features of the ORM

    Args:
        transaction: The Spanner transaction to execute the request on
        query: The SQL query to run
        parameters: A mapping from the names of the parameters used in the SQL
            query to the value to be substituted in for that parameter
        parameter_types: A mapping from the names of the parameters used in the
            SQL query to the type of the value being substituted in for that
            parameter

    Returns:
        A list of lists. Each sublist is a result row from the SQL query.
        For SELECT queries, the order of values in the sublist matches the
        order of the columns requested from the SELECT clause of the query.
    """
    _logger.debug('Executing SQL:\n%s\n%s\n%s', query, parameters,
                  parameter_types)
    stream_results = transaction.execute_sql(
        query, params=parameters, param_types=parameter_types)
    return list(stream_results)

class SpannerWriteApi(abc.ABC):
  """Handles sending write requests to Spanner."""

  @property
  @abc.abstractmethod
  def _connection(self) -> spanner_database.SpannerDatabase:
    raise NotImplementedError

  def run_write(self, method: Callable[..., CallableReturn], *args: Any,
                **kwargs: Any) -> CallableReturn:
    """Wraps read-write queries in a write transaction.

    The callable will be executed with the write transaction passed to it as
    the first argument. If the transaction aborts (usually because the data
    the write operation depended on has changed since the start of the
    transaction), the callable will continue to be retried until success or
    30 seconds has passed.

    Args:
        method: The method that will be run in the transaction
        *args: Positional arguments that will be passed to `method`
        **kwargs: Keyword arguments that will be passed to `method`

    Returns:
        The return value from `method` will be returned from this method
    """
    return self._connection.run_in_transaction(method, *args, **kwargs)

  def delete(self, transaction: spanner_transaction.Transaction,
             table_name: str, keyset: spanner.KeySet) -> None:
    """Deletes rows from the given table based on the provided KeySet.

    Args:
        transaction: The Spanner transaction to execute the request on
        table_name: The Spanner table being modified
        keyset: Contains a list of primary keys that indicates which rows to
          delete from the Spanner table
    """

    _logger.debug('Delete table=%s keys=%s', table_name, keyset.keys)
    transaction.delete(table=table_name, keyset=keyset)

  def insert(self, transaction: spanner_transaction.Transaction,
             table_name: str, columns: Iterable[str],
             values: Iterable[Iterable[Any]]) -> None:
    """Adds rows to the given table based on the provided values.

    All non-nullable columns must be specified. Note that if a row is specified
    for which the primary key already exists in the table, an exception will
    be thrown and the insert will be aborted.

    Args:
        transaction: The Spanner transaction to execute the request on
        table_name: The Spanner table being modified
        columns: Which columns to write on the Spanner table
        values: A list of rows to write to the table. The order of the values in
          each sublist must match the order of the columns specified in the
          `columns` parameter.
    """
    _logger.debug('Insert table=%s columns=%s values=%s', table_name, columns,
                  values)
    transaction.insert(table=table_name, columns=columns, values=values)

  def update(self, transaction: spanner_transaction.Transaction,
             table_name: str, columns: Iterable[str],
             values: Iterable[Iterable[Any]]) -> None:
    """Updates rows in the given table based on the provided values.

    Note that if a row is specified for which the primary key does not
    exist in the table, an exception will be thrown and the update
    will be aborted.

    Args:
        transaction: The Spanner transaction to execute the request on
        table_name: The Spanner table being modified
        columns: Which columns to write on the Spanner table
        values: A list of rows to write to the table. The order of the values in
          each sublist must match the order of the columns specified in the
          `columns` parameter.
    """
    _logger.debug('Update table=%s columns=%s values=%s', table_name, columns,
                  values)
    transaction.update(table=table_name, columns=columns, values=values)

  def upsert(self, transaction: spanner_transaction.Transaction,
             table_name: str, columns: Iterable[str],
             values: Iterable[Iterable[Any]]) -> None:
    """Inserts or updates rows in the given table based on the provided values.

    All non-nullable columns must be specified, similarly to the insert method.
    The presence or absence of data in the table will not cause an exception
    to be thrown, unlike insert or update.

    Args:
        transaction: The Spanner transaction to execute the request on
        table_name: The Spanner table being modified
        columns: Which columns to write on the Spanner table
        values: A list of rows to write to the table. The order of the values in
          each sublist must match the order of the columns specified in the
          `columns` parameter.
    """
    _logger.debug('Upsert table=%s columns=%s values=%s', table_name, columns,
                  values)
    transaction.insert_or_update(
        table=table_name, columns=columns, values=values)

class SpannerConnection:
  """Class that handles connecting to a Spanner database."""

  def __init__(self,
               instance: str,
               database: str,
               project: Optional[str] = None,
               credentials: Optional[auth_credentials.Credentials] = None,
               pool: Optional[spanner.Pool] = None,
               create_ddl: Optional[Iterable[str]] = None):
    """Connects to the specified Spanner database."""
    client = spanner.Client(project=project, credentials=credentials)
    instance = client.instance(instance)
    self.database = instance.database(
        database, pool=pool, ddl_statements=create_ddl or ())

class SpannerApi(SpannerReadApi, SpannerWriteApi):
  """Class that handles reading from and writing to Spanner tables."""

  def __init__(self, connection: SpannerConnection):
    self._spanner_connection = connection

  @property
  def _connection(self):
    return self._spanner_connection.database

_api = None  # type: Optional[SpannerApi]

def connect(instance: str,
            database: str,
            project: Optional[str] = None,
            credentials: Optional[auth_credentials.Credentials] = None,
            pool: Optional[spanner.Pool] = None) -> SpannerApi:
  """Connects to the Spanner database and sets the global spanner_api."""
  connection = SpannerConnection(
      instance, database, project=project, credentials=credentials, pool=pool)
  return from_connection(connection)

def from_connection(connection: SpannerConnection) -> SpannerApi:
  """Sets the global spanner_api from the provided connection."""
  global _api
  _api = SpannerApi(connection)
  return _api

def hangup() -> None:
  """Clears the global spanner_api."""
  global _api
  _api = None

def spanner_api() -> SpannerApi:
  """Returns the global spanner_api if it has been set."""
  if not _api:
    raise error.SpannerError('Must connect to Spanner before calling APIs')
  return _api
<|endoftext|>


<commit_before>from lib.config import group_id, my_id, achi_is_active, data_path
from lib.guilds import Guild, Player, Avatar, Achi
from lib.commands import database, api, vk

class updateGuild(object):
	def __init__(self, guild_id):
		self.guild = Guild(guild_id)
		if self.guild.exists:
			self.xml = self.guild.xml_element
			self.preparePage()
			self.saveNewPage()

	def preparePage(self):
		self.template = getPageTemplate("guild.txt")
		self.attributes = self.makeAttributes()
		self.page = editPageTemplate(self.attributes, self.template)

	def saveNewPage(self):
		page_id = self.guild.get("page")
		saveWikiPage(self.page, page_id)

	def makeAttributes(self):
		attr = self.getAttributes()
		players = self.getPlayers()
		attr['head'] = self.getHead()
		attr['vice'] = self.getVice()
		attr['stats'] = self.getStats()
		attr['players'] = self.getPlayerList(players)
		attr['numberofplayers'] = len(players)
		if achi_is_active:
			attr['achi'] = self.getAchi()
		else:
			self.template = self.template.replace("[achi]", "")
		return attr

	def getAttributes(self):
		xml_fields = self.xml.iterchildren()
		attributes = {f.tag:f.text for f in xml_fields}
		return attributes

	def getPlayers(self):
		all_players = database.getAll("players")
		guild_id = self.guild.get('id')
		guild_players = []
		for player in all_players:
			if player.find("guild").text == guild_id:
				player_id = player.find("id").text
				guild_players.append(player_id)
		return [Player(p) for p in guild_players]

	def getHead(self):
		heads = self.guild.heads
		return self.makeFancyList(heads)

	def getVice(self):
		vices = self.guild.vices
		if len(vices):
			return self.makeFancyList(vices)
		else:
			self.removeViceField()

	def removeViceField(self):
		template = self.template.splitlines()
		template.pop(6)
		self.template = "\n".join(template)

	def makeFancyList(self, ids):
		players = [Player(p) for p in ids]
		formatted_list = []
		for player in players:
			name = player.get("name")
			id = player.get("id")
			hyperlink = "[[id{}|{}]]".format(id, name)
			formatted_list.append(hyperlink)
		return ", ".join(formatted_list)

	def getStats(self):
		wins = self.guild.get("wins")
		loses = self.guild.get("loses")
		wins, loses = int(wins), int(loses)
		if wins > 0:
			stats = (wins * 100) / (wins + loses)
		else:
			stats = 0
		return round(stats)

	def getPlayerList(self, players):
		""" Составляет список игроков для вики-страницы

			Args: Player[] players
			returns str
		"""
		player_list = "{|\n|-\n"
		new_row = "|-\n"
		avatars = []
		for index, player in enumerate(players):
			if len(avatars) == 4:
				player_list += new_row
				for a in avatars:
					player_list += a
				player_list += new_row
				avatars = []
			id, name = player.get("id"), player.get("name")
			string = "! [[id{}|{}]]\n".format(id, name)
			player_list += string
			avatar = Avatar(player.get("avatar"))
			photo = "| [[{}|125x125px;noborder;nolink| ]]\n".format(avatar)
			avatars.append(photo)
			if index == len(players) - 1:
				player_list += new_row
				for a in avatars:
					player_list += a
		return player_list

	def getAchi(self):
		guild_achi_keys = self.guild.get("achi").split(" ")
		page = "
'''[[page-64867627_49895049|Испытания]]'''"
		for index, result in enumerate(guild_achi_keys):
			achi = Achi(id=index)
			name, icon, waves = achi.get("name", "icon", "waves")
			waves = waves.split(" ")
			wave = waves[result]
			title_line = "\n=={}==".format(name)
			icon_pic = "[[{}|125px;noborder| ]]".format(icon)
			wave_pic = "[[{}|400x70px;noborder;nolink| ]]".format(wave)
			main_line = "\n{}{}".format(icon_pic, wave_pic)
			page += title_line + main_line
		return page

class refreshGuilds(object):
	def __init__(self):
		page_id = 47292063
		self.preparePage()
		saveWikiPage(self.page, page_id)

	def preparePage(self):
		self.template = getPageTemplate("all_guilds.txt")
		self.attributes = self.makeAttributes()
		self.page = editPageTemplate(self.attributes, self.template)

	def makeAttributes(self):
		attr = dict()
		attr['all_guilds'], count = self.getGuildList()
		attr['guildnumber'] = count
		attr['playernumber'] = self.getPlayerCount()
		return attr

	def getGuildList(self):
		guilds = database.getAll("guilds", "id")
		guilds = [Guild(g) for g in guilds]
		return self.makeFancyGuildList(guilds), len(guilds)

	def makeFancyGuildList(self, guilds):
		page, id_line, guild_line = self.getGuildLineTemplates()
		total_waves = self.getTotalAmountOfWaves()
		self.makeGuildPercentages(guilds, total_waves)
		self.sortGuilds(guilds)
		for guild in guilds:
			page += id_line.format(guild.percent)
			banner = guild.get('banner')
			link = guild.get('page')
			page += guild_line.format(banner, link)
		if achi_is_active:
			page += "\n|}"
		return page

	def getGuildLineTemplates(self):
		if achi_is_active:
			page = "{|\n|-\n!Рейтинг\n!Гильдия\n|-"
			id_line = "\n!{}%"
		else:
			page = "{|\n|-\n!ID\n!Гильдия\n|-"
			id_line = "\n!{}"
		guild_line = "\n|[[{}|450px;noborder|page-64867627_{}]]\n|-"
		return page, id_line, guild_line

	def getTotalAmountOfWaves(self):
		if achi_is_active:
			all_achi_waves = database.getAll("achis", "waves")
			all_achi_waves = [len(w.split(" ")) - 1 for w in all_achi_waves]
			return sum(all_achi_waves)

	def makeGuildPercentages(self, guilds, total_waves):
		for guild in guilds:
			if achi_is_active:
				achi_results = guild.get("achi").split(" ")
				achi_results = [int(r) for r in achi_results]
				complete_waves = sum(achi_results)
				percent = complete_waves / total_waves
				guild.percent = round(percent)
			else:
				guild.percent = int(guild.get("id"))

	def sortGuilds(self, guilds):
		guilds.sort(key=lambda g: g.percent)

	def getPlayerCount(self):
		all_players = database.getAll("players")
		counter = 0
		for player in all_players:
			if player.find("guild").text != "0":
				counter += 1
		return counter

def getPageTemplate(file_name):
	folder = data_path + "page_templates/"
	path = folder + file_name
	with open(path) as file:
		return file.read()

def editPageTemplate(attributes, template):
	for key, value in attributes.items():
		key = "[{}]".format(key)
		value = str(value)
		template = template.replace(key, value)
	return template

def saveWikiPage(page, page_id, group=group_id):
	vk(api.pages.save,
		text=page,
		user_id=my_id,
		page_id=page_id,
		group_id=group)
<commit_msg>[add] conditional guild sorting<commit_after>from lib.config import group_id, my_id, achi_is_active, data_path
from lib.guilds import Guild, Player, Avatar, Achi
from lib.commands import database, api, vk

class updateGuild(object):
	def __init__(self, guild_id):
		self.guild = Guild(guild_id)
		if self.guild.exists:
			self.xml = self.guild.xml_element
			self.preparePage()
			self.saveNewPage()

	def preparePage(self):
		self.template = getPageTemplate("guild.txt")
		self.attributes = self.makeAttributes()
		self.page = editPageTemplate(self.attributes, self.template)

	def saveNewPage(self):
		page_id = self.guild.get("page")
		saveWikiPage(self.page, page_id)

	def makeAttributes(self):
		attr = self.getAttributes()
		players = self.getPlayers()
		attr['head'] = self.getHead()
		attr['vice'] = self.getVice()
		attr['stats'] = self.getStats()
		attr['players'] = self.getPlayerList(players)
		attr['numberofplayers'] = len(players)
		if achi_is_active:
			attr['achi'] = self.getAchi()
		else:
			self.template = self.template.replace("[achi]", "")
		return attr

	def getAttributes(self):
		xml_fields = self.xml.iterchildren()
		attributes = {f.tag:f.text for f in xml_fields}
		return attributes

	def getPlayers(self):
		all_players = database.getAll("players")
		guild_id = self.guild.get('id')
		guild_players = []
		for player in all_players:
			if player.find("guild").text == guild_id:
				player_id = player.find("id").text
				guild_players.append(player_id)
		return [Player(p) for p in guild_players]

	def getHead(self):
		heads = self.guild.heads
		return self.makeFancyList(heads)

	def getVice(self):
		vices = self.guild.vices
		if len(vices):
			return self.makeFancyList(vices)
		else:
			self.removeViceField()

	def removeViceField(self):
		template = self.template.splitlines()
		template.pop(6)
		self.template = "\n".join(template)

	def makeFancyList(self, ids):
		players = [Player(p) for p in ids]
		formatted_list = []
		for player in players:
			name = player.get("name")
			id = player.get("id")
			hyperlink = "[[id{}|{}]]".format(id, name)
			formatted_list.append(hyperlink)
		return ", ".join(formatted_list)

	def getStats(self):
		wins = self.guild.get("wins")
		loses = self.guild.get("loses")
		wins, loses = int(wins), int(loses)
		if wins > 0:
			stats = (wins * 100) / (wins + loses)
		else:
			stats = 0
		return round(stats)

	def getPlayerList(self, players):
		""" Составляет список игроков для вики-страницы

			Args: Player[] players
			returns str
		"""
		player_list = "{|\n|-\n"
		new_row = "|-\n"
		avatars = []
		for index, player in enumerate(players):
			if len(avatars) == 4:
				player_list += new_row
				for a in avatars:
					player_list += a
				player_list += new_row
				avatars = []
			id, name = player.get("id"), player.get("name")
			string = "! [[id{}|{}]]\n".format(id, name)
			player_list += string
			avatar = Avatar(player.get("avatar"))
			photo = "| [[{}|125x125px;noborder;nolink| ]]\n".format(avatar)
			avatars.append(photo)
			if index == len(players) - 1:
				player_list += new_row
				for a in avatars:
					player_list += a
		return player_list

	def getAchi(self):
		guild_achi_keys = self.guild.get("achi").split(" ")
		page = "
'''[[page-64867627_49895049|Испытания]]'''"
		for index, result in enumerate(guild_achi_keys):
			achi = Achi(id=index)
			name, icon, waves = achi.get("name", "icon", "waves")
			waves = waves.split(" ")
			wave = waves[result]
			title_line = "\n=={}==".format(name)
			icon_pic = "[[{}|125px;noborder| ]]".format(icon)
			wave_pic = "[[{}|400x70px;noborder;nolink| ]]".format(wave)
			main_line = "\n{}{}".format(icon_pic, wave_pic)
			page += title_line + main_line
		return page

class refreshGuilds(object):
	def __init__(self):
		page_id = 47292063
		self.preparePage()
		saveWikiPage(self.page, page_id)

	def preparePage(self):
		self.template = getPageTemplate("all_guilds.txt")
		self.attributes = self.makeAttributes()
		self.page = editPageTemplate(self.attributes, self.template)

	def makeAttributes(self):
		attr = dict()
		attr['all_guilds'], count = self.getGuildList()
		attr['guildnumber'] = count
		attr['playernumber'] = self.getPlayerCount()
		return attr

	def getGuildList(self):
		guilds = database.getAll("guilds", "id")
		guilds = [Guild(g) for g in guilds]
		return self.makeFancyGuildList(guilds), len(guilds)

	def makeFancyGuildList(self, guilds):
		page, id_line, guild_line = self.getGuildLineTemplates()
		total_waves = self.getTotalAmountOfWaves()
		self.makeGuildPercentages(guilds, total_waves)
		self.sortGuilds(guilds)
		for guild in guilds:
			page += id_line.format(guild.percent)
			banner = guild.get('banner')
			link = guild.get('page')
			page += guild_line.format(banner, link)
		if achi_is_active:
			page += "\n|}"
		return page

	def getGuildLineTemplates(self):
		if achi_is_active:
			page = "{|\n|-\n!Рейтинг\n!Гильдия\n|-"
			id_line = "\n!{}%"
		else:
			page = "{|\n|-\n!ID\n!Гильдия\n|-"
			id_line = "\n!{}"
		guild_line = "\n|[[{}|450px;noborder|page-64867627_{}]]\n|-"
		return page, id_line, guild_line

	def getTotalAmountOfWaves(self):
		if achi_is_active:
			all_achi_waves = database.getAll("achis", "waves")
			all_achi_waves = [len(w.split(" ")) - 1 for w in all_achi_waves]
			return sum(all_achi_waves)

	def makeGuildPercentages(self, guilds, total_waves):
		for guild in guilds:
			if achi_is_active:
				achi_results = guild.get("achi").split(" ")
				achi_results = [int(r) for r in achi_results]
				complete_waves = sum(achi_results)
				percent = complete_waves / total_waves
				guild.percent = round(percent)
			else:
				guild.percent = int(guild.get("id"))

	def sortGuilds(self, guilds):
		reverse = achi_is_active or False
		guilds.sort(key=lambda g: g.percent, reverse=reverse)

	def getPlayerCount(self):
		all_players = database.getAll("players")
		counter = 0
		for player in all_players:
			if player.find("guild").text != "0":
				counter += 1
		return counter

def getPageTemplate(file_name):
	folder = data_path + "page_templates/"
	path = folder + file_name
	with open(path) as file:
		return file.read()

def editPageTemplate(attributes, template):
	for key, value in attributes.items():
		key = "[{}]".format(key)
		value = str(value)
		template = template.replace(key, value)
	return template

def saveWikiPage(page, page_id, group=group_id):
	vk(api.pages.save,
		text=page,
		user_id=my_id,
		page_id=page_id,
		group_id=group)
<|endoftext|>


<commit_before># Copyright 2013 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module containing FormHandler interface to handle POST data."""

class FormHandler(object):
  """Simplified version of request handler that is able to take care of
  the received data.
  """

  def __init__(self, view):
    """Initializes new instance of form handler.

    Args:
      view: callback to implementation of base.RequestHandler
        that creates this object.
    """
    self._view = view

  def handle(self, data, check, mutator):
    """Handles the data that was received in the current request and returns
    an appropriate HTTP response.

    Args:
      data: A soc.views.helper.request_data.RequestData.
      check: A soc.views.helper.access_checker.AccessChecker.
      mutator: A soc.views.helper.access_checker.Mutator.

    Returns:
      An http.HttpResponse appropriate for the given request parameters.
    """
    raise NotImplementedError
<commit_msg>url argument added to the default constructor of FormHandler.<commit_after># Copyright 2013 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module containing FormHandler interface to handle POST data."""

class FormHandler(object):
  """Simplified version of request handler that is able to take care of
  the received data.
  """

  # TODO(daniel): remove view from this list.
  def __init__(self, view, url=None):
    """Initializes new instance of form handler.

    Args:
      view: callback to implementation of base.RequestHandler
        that creates this object.
      url: URL that should be used for redirect after the request is
        handled successfully. If it is not specified, the handler should
        return response with status of 200 or construct a URL to redirect to
        on its own.
    """
    self._view = view
    self._url = url

  def handle(self, data, check, mutator):
    """Handles the data that was received in the current request and returns
    an appropriate HTTP response.

    Args:
      data: A soc.views.helper.request_data.RequestData.
      check: A soc.views.helper.access_checker.AccessChecker.
      mutator: A soc.views.helper.access_checker.Mutator.

    Returns:
      An http.HttpResponse appropriate for the given request parameters.
    """
    raise NotImplementedError
<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2022 The Google Research Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Base class encapsulating processing a subvolume to another subvolume."""

import collections
import dataclasses
import enum
import importlib
from typing import Any, Tuple, Optional, Union

from connectomics.common import array
from connectomics.common import bounding_box
from connectomics.volume import descriptor
from connectomics.volume import subvolume
import dataclasses_json
import numpy as np

ImmutableArray = array.ImmutableArray
MutableArray = array.MutableArray
Subvolume = subvolume.Subvolume
SuggestedXyz = collections.namedtuple('SuggestedXyz', 'x y z')
TupleOrSuggestedXyz = Union['XyzTuple', SuggestedXyz]  # pylint: disable=invalid-name
XyzTuple = array.Tuple3i

@dataclasses.dataclass
class SubvolumeProcessorConfig(dataclasses_json.DataClassJsonMixin):
  """Configuration for a given subvolume processor."""
  # Name of class exposed in module_search_path.
  name: str

  # Arguments to SubvolumeProcessor, passed in as kwargs.
  args: Optional[dict[str, Any]] = None

  # Fully.qualified.python.module to search for SubvolumeProcessor `name`.
  module_search_path: str = 'connectomics.volume.processor'

@dataclasses.dataclass
class ProcessVolumeConfig(dataclasses_json.DataClassJsonMixin):
  """User-supplied configuration."""

  # Input volume to process.
  input_volume: descriptor.VolumeDescriptor

  # Output directory to write the volumetric data, inserted automatically into
  # the output_volume's TensorStore spec.
  output_dir: str

  # Bounding boxes to process.
  bounding_boxes: list[bounding_box.BoundingBox]

  # Processor configuration to apply.
  processor: SubvolumeProcessorConfig

  # Size of each subvolume to process.
  subvolume_size: array.Tuple3i = dataclasses.field(
      metadata=dataclasses_json.config(decoder=tuple))

  # Amount of overlap between processed subvolumes. This is independent of any
  # additional context required by individual processors.
  overlap: array.Tuple3i = dataclasses.field(
      metadata=dataclasses_json.config(decoder=tuple))

  # Number of bounding boxes to batch together per work item during processing.
  batch_size: int = 1

  # Additional Tensorstore context configuration applied to the input. Useful
  # for limiting parallelism.
  input_ts_context: Optional[dict[str, Any]] = None

  # Additional Tensorstore context configuration applied to the output. Useful
  # for limiting parallelism.
  output_ts_context: Optional[dict[str, Any]] = None

  # TODO(timblakely): Support back shifting edge boxes.

class OutputNums(enum.Enum):
  SINGLE = 1
  MULTI = 2

class SubvolumeProcessor:
  """Abstract base class for processors.

  The self.process method does the work.  The rest is for documenting input /
  output requirements and naming.
  """

  # Effective subvolume/overlap configuration as set by the framework within
  # which this processor is being executed. This might include, e.g. user
  # overrides supplied via command-line arguments.
  _context: ImmutableArray
  _subvol_size: ImmutableArray
  _overlap: ImmutableArray

  # Whether the output of this processor will be cropped for subvolumes that
  # are adjacent to the input bounding box(es).
  crop_at_borders = True

  # If true, the actual content of input_ndarray doesn't matter. The processor
  # only uses the type and geometry of the array for further processing.
  ignores_input_data = False

  def output_type(self, input_type: Union[np.uint8, np.uint64, np.float32]):
    """Returns Numpy output type of self.process for given input_type.

    Args:
      input_type: A Numpy type, should be one of np.uint8, np.uint64,
        np.float32.
    """
    return input_type

  @property
  def output_num(self) -> OutputNums:
    """Whether self.process produces single output or multiple per input."""
    return OutputNums.SINGLE

  @property
  def name_parts(self) -> Tuple[str]:
    """Returns Tuple[str] to be used in naming jobs, outputs, etc.

    Often useful to include both the name of the processor as well as relevant
    parameters.  The elements are generally joined with '_' or '-' depending on
    the context.
    """
    return type(self).__name__,

  def pixelsize(self, input_psize: array.ArrayLike3d) -> ImmutableArray:
    return ImmutableArray(input_psize)

  def num_channels(self, input_channels: int) -> int:
    return input_channels

  def process(
      self, subvol: subvolume.Subvolume
  ) -> Union[subvolume.Subvolume, list[subvolume.Subvolume]]:
    """Processes the input subvolume.

    Args:
      subvol: Subvolume to process.

    Returns:
      The processed subvolume. If self.context is > 0, it is expected that the
      returned subvolume will be smaller than the input by the context amount.
    """
    raise NotImplementedError

  def subvolume_size(self) -> Optional[TupleOrSuggestedXyz]:
    """Returns the XYZ subvolume size required by self.process.

    Some processors (e.g. TF inference models) may require specific input size.
    If the input size is just a suggestion, should return SuggestedXyz rather
    than raw tuple.  If there is no suggested input size, return None.
    """
    return None

  def context(self) -> Tuple[TupleOrSuggestedXyz, TupleOrSuggestedXyz]:
    """Returns XYZ front/back context needed for processing.

    It is expected that the return from self.process will be smaller than the
    input by this amount in front and back.
    """
    return SuggestedXyz(0, 0, 0), SuggestedXyz(0, 0, 0)

  def overlap(self) -> TupleOrSuggestedXyz:
    """Keep the type of context and sum front and back context."""
    f, b = self.context()
    overlap = f[0] + b[0], f[1] + b[1], f[2] + b[2]
    if isinstance(f, SuggestedXyz) and isinstance(b, SuggestedXyz):
      return SuggestedXyz(*overlap)
    return overlap

  def set_effective_subvol_and_overlap(self, subvol_size: array.ArrayLike3d,
                                       overlap: array.ArrayLike3d):
    """Assign the effective subvolume and overlap."""
    self._subvol_size = array.ImmutableArray(subvol_size)
    self._overlap = array.ImmutableArray(overlap)
    if np.all(self.overlap() == self._overlap):
      self._context = self.context()  # type: ignore
    else:
      pre = self._overlap // 2
      post = self._overlap - pre
      self._context = pre, post  # type: ignore

  def _context_for_box(
      self, box: bounding_box.BoundingBoxBase) -> Tuple[np.ndarray, np.ndarray]:
    front, back = self._context
    front = np.array(front)
    back = np.array(back)
    if not self.crop_at_borders:
      front *= ~box.is_border_start
      back *= ~box.is_border_end

    return front, back

  def expected_output_box(
      self, box: bounding_box.BoundingBoxBase) -> bounding_box.BoundingBoxBase:
    """Returns the adjusted bounding box after process() is called.

    Args:
        box: Size of the input subvolume passed to process()

    Returns:
        Bounding box for the output volume.
    """
    return box

  def crop_box(
      self, box: bounding_box.BoundingBoxBase) -> bounding_box.BoundingBoxBase:
    """Crop box front/back by self.context.

    Args:
      box: BoundingBox to crop.

    Returns:
      Copy of box with bounds reduced by front/back amount given by
      self.context.
    """
    front, back = self._context_for_box(box)
    return box.adjusted_by(start=front, end=-back)

  def crop_box_and_data(
      self,
      box: bounding_box.BoundingBoxBase,
      # TODO(timblakely): Strongly type this as ArrayCZYX
      data: np.ndarray
  ) -> Subvolume:
    """Crop data front/back by self.context.

    Args:
      box: bounding box corresponding to the data array
      data: 4d Numpy array with dimensions channels, Z, Y, X.

    Returns:
      View of data cropped by front/back amount given by self.context.
    """
    cropped_box = self.crop_box(box)
    front, back = self._context_for_box(box)
    fx, fy, fz = front
    bx, by, bz = np.array(data.shape[:0:-1]) - back
    return Subvolume(data[:, fz:bz, fy:by, fx:bx], cropped_box)

def get_processor(config: SubvolumeProcessorConfig) -> SubvolumeProcessor:
  name = config.name
  package = importlib.import_module(config.module_search_path)
  if not hasattr(package, name):
    raise ValueError(f'No processor named {name} in package {package}')
  processor = getattr(package, name)
  args = {} if not config.args else config.args
  return processor(**args)
<commit_msg>Allow input volumes to be specified as a path.<commit_after># coding=utf-8
# Copyright 2022 The Google Research Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Base class encapsulating processing a subvolume to another subvolume."""

import collections
import dataclasses
import enum
import importlib
from typing import Any, Tuple, Optional, Union

from connectomics.common import array
from connectomics.common import bounding_box
from connectomics.common import file
from connectomics.volume import descriptor
from connectomics.volume import subvolume
import dataclasses_json
import numpy as np

ImmutableArray = array.ImmutableArray
MutableArray = array.MutableArray
Subvolume = subvolume.Subvolume
SuggestedXyz = collections.namedtuple('SuggestedXyz', 'x y z')
TupleOrSuggestedXyz = Union['XyzTuple', SuggestedXyz]  # pylint: disable=invalid-name
XyzTuple = array.Tuple3i

@dataclasses.dataclass
class SubvolumeProcessorConfig(dataclasses_json.DataClassJsonMixin):
  """Configuration for a given subvolume processor."""
  # Name of class exposed in module_search_path.
  name: str

  # Arguments to SubvolumeProcessor, passed in as kwargs.
  args: Optional[dict[str, Any]] = None

  # Fully.qualified.python.module to search for SubvolumeProcessor `name`.
  module_search_path: str = 'connectomics.volume.processor'

@dataclasses.dataclass
class ProcessVolumeConfig(dataclasses_json.DataClassJsonMixin):
  """User-supplied configuration."""

  # Input volume to process.
  input_volume: descriptor.VolumeDescriptor = dataclasses.field(
      metadata=dataclasses_json.config(
          decoder=file.dataclass_loader(descriptor.VolumeDescriptor)))

  # Output directory to write the volumetric data, inserted automatically into
  # the output_volume's TensorStore spec.
  output_dir: str

  # Bounding boxes to process.
  bounding_boxes: list[bounding_box.BoundingBox]

  # Processor configuration to apply.
  processor: SubvolumeProcessorConfig

  # Size of each subvolume to process.
  subvolume_size: array.Tuple3i = dataclasses.field(
      metadata=dataclasses_json.config(decoder=tuple))

  # Amount of overlap between processed subvolumes. This is independent of any
  # additional context required by individual processors.
  overlap: array.Tuple3i = dataclasses.field(
      metadata=dataclasses_json.config(decoder=tuple))

  # Number of bounding boxes to batch together per work item during processing.
  batch_size: int = 1

  # Additional Tensorstore context configuration applied to the input. Useful
  # for limiting parallelism.
  input_ts_context: Optional[dict[str, Any]] = None

  # Additional Tensorstore context configuration applied to the output. Useful
  # for limiting parallelism.
  output_ts_context: Optional[dict[str, Any]] = None

  # TODO(timblakely): Support back shifting edge boxes.

class OutputNums(enum.Enum):
  SINGLE = 1
  MULTI = 2

class SubvolumeProcessor:
  """Abstract base class for processors.

  The self.process method does the work.  The rest is for documenting input /
  output requirements and naming.
  """

  # Effective subvolume/overlap configuration as set by the framework within
  # which this processor is being executed. This might include, e.g. user
  # overrides supplied via command-line arguments.
  _context: ImmutableArray
  _subvol_size: ImmutableArray
  _overlap: ImmutableArray

  # Whether the output of this processor will be cropped for subvolumes that
  # are adjacent to the input bounding box(es).
  crop_at_borders = True

  # If true, the actual content of input_ndarray doesn't matter. The processor
  # only uses the type and geometry of the array for further processing.
  ignores_input_data = False

  def output_type(self, input_type: Union[np.uint8, np.uint64, np.float32]):
    """Returns Numpy output type of self.process for given input_type.

    Args:
      input_type: A Numpy type, should be one of np.uint8, np.uint64,
        np.float32.
    """
    return input_type

  @property
  def output_num(self) -> OutputNums:
    """Whether self.process produces single output or multiple per input."""
    return OutputNums.SINGLE

  @property
  def name_parts(self) -> Tuple[str]:
    """Returns Tuple[str] to be used in naming jobs, outputs, etc.

    Often useful to include both the name of the processor as well as relevant
    parameters.  The elements are generally joined with '_' or '-' depending on
    the context.
    """
    return type(self).__name__,

  def pixelsize(self, input_psize: array.ArrayLike3d) -> ImmutableArray:
    return ImmutableArray(input_psize)

  def num_channels(self, input_channels: int) -> int:
    return input_channels

  def process(
      self, subvol: subvolume.Subvolume
  ) -> Union[subvolume.Subvolume, list[subvolume.Subvolume]]:
    """Processes the input subvolume.

    Args:
      subvol: Subvolume to process.

    Returns:
      The processed subvolume. If self.context is > 0, it is expected that the
      returned subvolume will be smaller than the input by the context amount.
    """
    raise NotImplementedError

  def subvolume_size(self) -> Optional[TupleOrSuggestedXyz]:
    """Returns the XYZ subvolume size required by self.process.

    Some processors (e.g. TF inference models) may require specific input size.
    If the input size is just a suggestion, should return SuggestedXyz rather
    than raw tuple.  If there is no suggested input size, return None.
    """
    return None

  def context(self) -> Tuple[TupleOrSuggestedXyz, TupleOrSuggestedXyz]:
    """Returns XYZ front/back context needed for processing.

    It is expected that the return from self.process will be smaller than the
    input by this amount in front and back.
    """
    return SuggestedXyz(0, 0, 0), SuggestedXyz(0, 0, 0)

  def overlap(self) -> TupleOrSuggestedXyz:
    """Keep the type of context and sum front and back context."""
    f, b = self.context()
    overlap = f[0] + b[0], f[1] + b[1], f[2] + b[2]
    if isinstance(f, SuggestedXyz) and isinstance(b, SuggestedXyz):
      return SuggestedXyz(*overlap)
    return overlap

  def set_effective_subvol_and_overlap(self, subvol_size: array.ArrayLike3d,
                                       overlap: array.ArrayLike3d):
    """Assign the effective subvolume and overlap."""
    self._subvol_size = array.ImmutableArray(subvol_size)
    self._overlap = array.ImmutableArray(overlap)
    if np.all(self.overlap() == self._overlap):
      self._context = self.context()  # type: ignore
    else:
      pre = self._overlap // 2
      post = self._overlap - pre
      self._context = pre, post  # type: ignore

  def _context_for_box(
      self, box: bounding_box.BoundingBoxBase) -> Tuple[np.ndarray, np.ndarray]:
    front, back = self._context
    front = np.array(front)
    back = np.array(back)
    if not self.crop_at_borders:
      front *= ~box.is_border_start
      back *= ~box.is_border_end

    return front, back

  def expected_output_box(
      self, box: bounding_box.BoundingBoxBase) -> bounding_box.BoundingBoxBase:
    """Returns the adjusted bounding box after process() is called.

    Args:
        box: Size of the input subvolume passed to process()

    Returns:
        Bounding box for the output volume.
    """
    return box

  def crop_box(
      self, box: bounding_box.BoundingBoxBase) -> bounding_box.BoundingBoxBase:
    """Crop box front/back by self.context.

    Args:
      box: BoundingBox to crop.

    Returns:
      Copy of box with bounds reduced by front/back amount given by
      self.context.
    """
    front, back = self._context_for_box(box)
    return box.adjusted_by(start=front, end=-back)

  def crop_box_and_data(
      self,
      box: bounding_box.BoundingBoxBase,
      # TODO(timblakely): Strongly type this as ArrayCZYX
      data: np.ndarray
  ) -> Subvolume:
    """Crop data front/back by self.context.

    Args:
      box: bounding box corresponding to the data array
      data: 4d Numpy array with dimensions channels, Z, Y, X.

    Returns:
      View of data cropped by front/back amount given by self.context.
    """
    cropped_box = self.crop_box(box)
    front, back = self._context_for_box(box)
    fx, fy, fz = front
    bx, by, bz = np.array(data.shape[:0:-1]) - back
    return Subvolume(data[:, fz:bz, fy:by, fx:bx], cropped_box)

def get_processor(config: SubvolumeProcessorConfig) -> SubvolumeProcessor:
  name = config.name
  package = importlib.import_module(config.module_search_path)
  if not hasattr(package, name):
    raise ValueError(f'No processor named {name} in package {package}')
  processor = getattr(package, name)
  args = {} if not config.args else config.args
  return processor(**args)
<|endoftext|>