<commit_before>"""
nextgen4b.process.filter
TC, 8/11/16

A collection of functions that read, filter, and output sequence data from
next-generation sequencing experiments.
"""
import gzip
import logging
import os
import re
import sys
import time
import uuid

import numpy as np
import yaml
from Bio import AlignIO, SeqIO
from Bio.Emboss.Applications import NeedleCommandline
from Bio.Seq import Seq
from Bio.SeqRecord import SeqRecord
from tqdm import tqdm

__all__ = ['filter_sample', 'run_all_experiments']

#####################
# File Management
#####################

def load_ngs_file(fpath, ftype='fastq'):
    """
    Load a .fastq file to a SeqIO iterator, un-gzip if necessary.
    """
    if fpath.endswith('.gz'):
        seq_f = gzip.open(fpath, 'rt')
    elif fpath.endswith('.fastq'):
        seq_f = open(fpath, 'rt')
    else:
        raise ValueError('File does not end in .gz or .fastq; confirm file type.')
    f_iter = SeqIO.parse(seq_f, ftype)
    return f_iter

#####################
# Main Filter Code
#####################

def filter_sample(f_name, pe_name, bcs, templates, f_filt_seqs, r_filt_seqs):
    """
    Output filtered sequences as dictionary, indexed by barcode.
    Sequences will be aligned to the provided template.
    Parts of the template not represented will be '-'
    """
    # setup loggers
    text_logger = logging.getLogger(__name__+'.text_logger')
    csv_logger = logging.getLogger(__name__+'.csv_logger')

    text_logger.info('Started filtering routine for %s', f_name)

    # Compile regexes
    f_res = compile_res(f_filt_seqs)
    pe_res = compile_res(r_filt_seqs)

    # Load as generators, then filter
    text_logger.info('Loading Files')
    f_seqs = load_ngs_file(f_name)
    for regex in f_res:
        f_seqs = filter_seqs(f_seqs, regex)
    pe_seqs = load_ngs_file(pe_name)
    for regex in pe_res:
        pe_seqs = filter_seqs(pe_seqs, regex)

    # Barcode Filtering/Demux
    bc_seqs = barcodeDemux(f_seqs, bcs)

    # Sequence-based filtering
    for expt in bcs.keys():
        text_logger.info('Starting post-demux filtering for expt ID %s', expt)
        csv_data = [expt, len(bc_seqs[expt])]
        # Filter based on PE matches, only return the copied sequence
        # Assumes the first RE in f_res will terminate the copied sequence
        # copiedFuncGenerator's output should return all sequence before the adapter
        seqs = filter_pe_mismatch(bc_seqs[expt], pe_seqs,
                                  gen_copied_seq_function(f_res))
        csv_data.append(len(seqs))

        seqs = trim_lig_adapter(seqs, f_res) # Trim CS2 before filtering on quality (bad Qs at end of seqs)

        # Quality filter
        if len(seqs) > 0:
            seqs = quality_filter(seqs) # Quality Filtering (needs to only have copied sequence)
        else:
            text_logger.info("""No sequences left, skipped quality score
                             filtering for expt ID %s.""", expt)
            bc_seqs[expt] = seqs
        csv_data.append(len(seqs))

        # Align filter
        if len(seqs) > 0:
            # Do alignment-based filtering
            seqs = alignment_filter(seqs,
                                    '{}{}'.format(bcs[expt], templates[expt])) # Do alignment-based filtering
        else:
            text_logger.info("""No sequences left, skipped align filtering for
                             expt ID %s.***""", expt)
            bc_seqs[expt] = seqs
        csv_data.append(len(seqs))

        # Length filtering
        if len(seqs) > 0:
            seqs = len_filter(seqs, l_barcode=len(bcs[expt])) # Length Filtering
        else:
            text_logger.info("""No sequences left, skipped length filtering for
                             expt ID %s***""", expt)
            bc_seqs[expt] = seqs
        csv_data.append(len(seqs))

        csv_logger.info(','.join([str(n) for n in csv_data]))
        bc_seqs[expt] = seqs

    return bc_seqs

#####################
# F/R Regex Filtering
#####################

def filter_seqs(seqs, q_re):
    """
    Filter an iterator based on whether items match a regex object.
    """
	# sIO_iterator should be generated by Bio.SeqIO.parse
	# q_re should be a regex object generated by re.compile()
	# Outputs a list of Seq objects that have the desired sequence in them.
    text_logger = logging.getLogger(__name__+'.text_logger')

    text_logger.info('Started regex filter: %s', q_re.pattern)
    out_l = [s for s in seqs if q_re.search(str(s.seq))]
    text_logger.info('Finished regex filter. Kept %i sequences.', len(out_l))
    return out_l

def compile_res(seqs):
    """
    Compile regex for each string in a list, return list of regex objects.
    """
    # Takes a list of sequences you want to filter for
    # Outputs a list of regex objects that you can iterate over
    return [re.compile(s) for s in seqs]

#####################
# Barcode Filtering
#####################

def barcodeDemux(seqs, bcs):
    """
    Takes lists of sequence objects, dict of barcodes (indexed by expt. ID)
    Demuxes based on the barcode the sequences start with
    Discards sequences that don't start with a barcode exact match
    Assumes forward read -> sequences start with a barcode
    """
    text_logger = logging.getLogger(__name__+'.text_logger')
    text_logger.info('Started barcode demuxing.')

    n_seqs = 0
    bc_filtered_data = {}
    for expt in bcs.keys():
        bc_filtered_data[expt] = [s for s in seqs if str(s.seq).startswith(bcs[expt])]
        n_seqs += len(bc_filtered_data[expt])

    text_logger.info('Finished barcode demuxing. Kept %i of %i sequences.', n_seqs, len(seqs))

    return bc_filtered_data

#####################
# Paired End Match Filtering
#####################

def get_coords(s):
    return ':'.join(s.description.split(' ')[0].split(':')[3:])

def get_sense(s):
    return s.description.split(' ')[1].split(':')[0]

def get_copied_seq(s, f_res):
    return s[f_res[0].search(str(s.seq)).end():list(f_res[1].finditer(str(s.seq)))[-1].start()]

def trim_lig_adapter(s, f_res):
    return s[:list(f_res[1].finditer(str(s.seq)))[-1].start()]

def gen_copied_seq_function(f_res):
    return lambda s: get_copied_seq(s, f_res)

def filter_pe_mismatch(f_seqs, pe_seqs, copied_func):
    """
    Args:
        f_seqs - sequences from forward reads. Presumably filtered for the
                 required adatper(s).
        pe_seqs - the paired end sequences of f_seqs. Also presumably filtered
                  for the required adapter(s).
        copied_func - takes a sequence, should ouptut the DNA that we expect
                      to have been copied, i.e. that should be on the paired
                      end read.

    Outputs a list of forward sequences that pass two filters:
        * Have a coordinate match in the paired end reads
        * That coordinate match has the same sequence.
    """
    text_logger = logging.getLogger(__name__+'.text_logger')
    text_logger.info('Started Paired-End Filtering')

    # Some housekeeping stuff
    proc_ct = 0 # number of sequences processed
    co_ct = 0 # number of sequences with coordinate matches
    aln_ct = 0 # number of sequences that have paired end sequence matches
    matched_seq_list = []

    # Get coordinate list
    pe_coord_list = [get_coords(s) for s in pe_seqs]
    for s in f_seqs:
        if get_coords(s) in pe_coord_list: # Filter based on paired-end presence
            co_ct += 1
            copied = copied_func(s) # Get the part of the sequence that was actually copied
            if str(pe_seqs[0].reverse_complement().seq).find(str(copied.seq)): # Filter on PE match
                aln_ct += 1
                matched_seq_list.append(s)

        proc_ct += 1
        if not (proc_ct % 5000):
            text_logger.info("Processed %i out of %i", proc_ct, len(f_seqs))

    text_logger.info("Finished Paired-End Filtering")
    text_logger.info("""Kept %i of %i forward sequences after coordinate
                     filtering""", co_ct, len(f_seqs))
    text_logger.info("""Kept %i of %i forward sequences after paired-end sequence
                     matching""", aln_ct, co_ct)

    return matched_seq_list

#####################
# Q-score Filtering
#####################

def quality_filter(seqs, q_cutoff=20):
    text_logger = logging.getLogger(__name__+'.text_logger')
    text_logger.info('Started Quality Score Filtering')
    out_l = [s for s in seqs
             if not any(s.letter_annotations['phred_quality']
                        < np.ones(len(s.letter_annotations['phred_quality']))*q_cutoff)]
    text_logger.info('Finished Quality Score Filtering. Kept %i of %i sequences.',
                     len(out_l), len(seqs))
    return out_l

#####################
# Length Filtering
#####################

def len_filter(seqs, l_cutoff=40, u_cutoff=200, l_barcode=0):
    """
    Return only sequence objects that have length between l_cutoff and
    u_cutoff
    """
    text_logger = logging.getLogger(__name__+'.text_logger')
    text_logger.info('Started Length Filtering')
    out_l = [s for s in seqs if (len(s.seq) >= (l_cutoff + l_barcode)) and
             (len(s.seq) <= (u_cutoff + l_barcode))]
    text_logger.info('Finished Length Filtering. Kept %i of %i sequences.',
                     len(out_l), len(seqs))
    return out_l

#####################
# Alignment Filtering
#####################

def alignment_filter(seqs, template, gapopen=10, gapextend=0.5, lo_cutoff=0,
                     hi_cutoff=1000, cleanup=True):
    text_logger = logging.getLogger(__name__+'.text_logger')
    text_logger.info('Started alignment-based filtering')
    start_n_seqs = len(seqs)

    # Save the template and sequences as temporary fasta files
    # Probably some hacking that can be done in the NeedleCommandline stuff
    template_f_name = 'temptemplate.fa'
    seqs_f_name = 'tempseq.fa'

    with open(seqs_f_name, 'w') as sh:
        SeqIO.write(seqs, sh, 'fastq')

    with open(template_f_name, 'w') as temp_seq_file:
        # Make temp sequence file for alignment
        temp_seq = SeqRecord(Seq(template), id='template', name='template')
        SeqIO.write(temp_seq, temp_seq_file, 'fasta')

    # Generate alignment command, run the alignment
    text_logger.info("""Began EMBOSS needle routine with settings:\ngapopen:
                    %i\ngapextend: %i\nlo_cutoff: %i\nhi_cutoff: %i""",
                 gapopen, gapextend, lo_cutoff, hi_cutoff)
    ofilen = 'temp_'+str(uuid.uuid4())+'.needle'
    needle_cline = NeedleCommandline(asequence=template_f_name,
                                     bsequence=seqs_f_name, gapopen=gapopen,
                                     gapextend=gapextend, outfile=ofilen)
    needle_cline()
    text_logger.info('Finished EMBOSS needle routine')

    aln_data = AlignIO.parse(open(ofilen), "emboss")
    new_seqs = cull_alignments(aln_data, lo_cutoff=lo_cutoff,
                               hi_cutoff=hi_cutoff)

    # Exit routine
    if cleanup:
        text_logger.info('Cleaning up temp files')
        os.remove(template_f_name)
        os.remove(seqs_f_name)
        os.remove(ofilen)
    text_logger.info("""Finished alignment-based filtering. Kept %i of %i
                     sequences.""", len(new_seqs), start_n_seqs)
    return new_seqs

def cull_alignments(aln_data, lo_cutoff=400, hi_cutoff=650):
    new_seqs = []

    for alignment in aln_data:
        if (alignment.annotations['score'] > lo_cutoff) \
            and (alignment.annotations['score'] < hi_cutoff):
            # Template should have no gaps, and should contain the whole
            # non-template sequence
            if not str(alignment[0].seq).count('-') > 0:
                new_seqs.append(alignment[1])
                new_seqs[-1].annotations['alnscore'] = alignment.annotations['score']
    return new_seqs

#####################
# Main Routines
#####################

def setup_logger(name, file_name, log_format, level=logging.INFO):
    logger = logging.getLogger(name)
    logger.setLevel(level)
    handler = logging.FileHandler(file_name)
    formatter = logging.Formatter(log_format)
    handler.setFormatter(formatter)
    logger.addHandler(handler)

    return logger

def run_all_experiments(yf_name, save_intermediates=True):
    """
    Filters all sequences noted in the passed YAML file.
    """
    # Setup text_logger
    text_logger = setup_logger(__name__+'.text_logger',
                               'ngs_%s.log' % time.strftime("%Y%m%d-%H%M%S"),
                               '%(asctime)s %(message)s')
    csv_logger = setup_logger(__name__+'.csv_logger',
                              'ngs_filter_%s.csv' % time.strftime("%Y%m%d-%H%M%S"),
                              '%(message)s')

    # Load YAML file
    with open(yf_name) as expt_f:
        expt_yaml = yaml.load(expt_f) # Should probably make this a class at some point...
    text_logger.info('Loaded YAML experiment file '+yf_name)

    runs = expt_yaml['ngsruns']
    text_logger.info('Found NGS Runs: '+', '.join(runs))

    for run in tqdm(runs.keys()):
        text_logger.info('Performing routine for NGS Run '+run)
        expts = runs[run]['experiments']
        text_logger.info('Found experiments '+', '.join(expts))

        # Get barcodes, templates for all experiments in the run
        bcs = {}
        templates = {}
        for expt in expts:
            bcs[expt] = expt_yaml['experiments'][expt]['barcode']
            templates[expt] = expt_yaml['experiments'][expt]['template_seq']

        # Do filtering
        text_logger.info('Starting filtering for run %s', run)
        aln_seqs = filter_sample(runs[run]['f_read_name'],
                                 runs[run]['pe_read_name'],
                                 bcs, templates,
                                 runs[run]['filter_seqs']['forward'],
                                 runs[run]['filter_seqs']['reverse'])
        if save_intermediates:
            for expt in aln_seqs.keys():
                with open('aln_seqs_%s_%s.fa' % (run, expt), 'w') as out_f:
                    SeqIO.write(aln_seqs[expt], out_f, 'fasta')
        text_logger.info('Finished filtering for run %s', run)

if __name__ == '__main__':
    if len(sys.argv) > 1:
        yaml_name = sys.argv[1]
    else:
        yaml_name = 'samples.yaml'

    run_all_experiments(yaml_name, save_intermediates=True)
<commit_msg>fixed bug in trim_lig_adapter use, added asis trick<commit_after>"""
nextgen4b.process.filter
TC, 8/11/16

A collection of functions that read, filter, and output sequence data from
next-generation sequencing experiments.
"""
import gzip
import logging
import os
import re
import sys
import time
import uuid

import numpy as np
import yaml
from Bio import AlignIO, SeqIO
from Bio.Emboss.Applications import NeedleCommandline
from Bio.Seq import Seq
from Bio.SeqRecord import SeqRecord
from tqdm import tqdm

__all__ = ['filter_sample', 'run_all_experiments']

#####################
# File Management
#####################

def load_ngs_file(fpath, ftype='fastq'):
    """
    Load a .fastq file to a SeqIO iterator, un-gzip if necessary.
    """
    if fpath.endswith('.gz'):
        seq_f = gzip.open(fpath, 'rt')
    elif fpath.endswith('.fastq'):
        seq_f = open(fpath, 'rt')
    else:
        raise ValueError('File does not end in .gz or .fastq; confirm file type.')
    f_iter = SeqIO.parse(seq_f, ftype)
    return f_iter

#####################
# Main Filter Code
#####################

def filter_sample(f_name, pe_name, bcs, templates, f_filt_seqs, r_filt_seqs):
    """
    Output filtered sequences as dictionary, indexed by barcode.
    Sequences will be aligned to the provided template.
    Parts of the template not represented will be '-'
    """
    # setup loggers
    text_logger = logging.getLogger(__name__+'.text_logger')
    csv_logger = logging.getLogger(__name__+'.csv_logger')

    text_logger.info('Started filtering routine for %s', f_name)

    # Compile regexes
    f_res = compile_res(f_filt_seqs)
    pe_res = compile_res(r_filt_seqs)

    # Load as generators, then filter
    text_logger.info('Loading Files')
    f_seqs = load_ngs_file(f_name)
    for regex in f_res:
        f_seqs = filter_seqs(f_seqs, regex)
    pe_seqs = load_ngs_file(pe_name)
    for regex in pe_res:
        pe_seqs = filter_seqs(pe_seqs, regex)

    # Barcode Filtering/Demux
    bc_seqs = barcodeDemux(f_seqs, bcs)

    # Sequence-based filtering
    for expt in bcs.keys():
        text_logger.info('Starting post-demux filtering for expt ID %s', expt)
        csv_data = [expt, len(bc_seqs[expt])]
        # Filter based on PE matches, only return the copied sequence
        # Assumes the first RE in f_res will terminate the copied sequence
        # copiedFuncGenerator's output should return all sequence before the adapter
        seqs = filter_pe_mismatch(bc_seqs[expt], pe_seqs,
                                  gen_copied_seq_function(f_res))
        csv_data.append(len(seqs))

        seqs = [trim_lig_adapter(s, f_res) for s in seqs] # Trim CS2 before filtering on quality (bad Qs at end of seqs)

        # Quality filter
        if len(seqs) > 0:
            seqs = quality_filter(seqs) # Quality Filtering (needs to only have copied sequence)
        else:
            text_logger.info("""No sequences left, skipped quality score
                             filtering for expt ID %s.""", expt)
            bc_seqs[expt] = seqs
        csv_data.append(len(seqs))

        # Align filter
        if len(seqs) > 0:
            # Do alignment-based filtering
            full_template = '{}{}'.format(bcs[expt], templates[expt])
            seqs = alignment_filter(seqs, full_template) # Do alignment-based filtering
        else:
            text_logger.info("""No sequences left, skipped align filtering for
                             expt ID %s.***""", expt)
            bc_seqs[expt] = seqs
        csv_data.append(len(seqs))

        # Length filtering
        if len(seqs) > 0:
            seqs = len_filter(seqs, l_barcode=len(bcs[expt])) # Length Filtering
        else:
            text_logger.info("""No sequences left, skipped length filtering for
                             expt ID %s***""", expt)
            bc_seqs[expt] = seqs
        csv_data.append(len(seqs))

        csv_logger.info(','.join([str(n) for n in csv_data]))
        bc_seqs[expt] = seqs

    return bc_seqs

#####################
# F/R Regex Filtering
#####################

def filter_seqs(seqs, q_re):
    """
    Filter an iterator based on whether items match a regex object.
    """
	# sIO_iterator should be generated by Bio.SeqIO.parse
	# q_re should be a regex object generated by re.compile()
	# Outputs a list of Seq objects that have the desired sequence in them.
    text_logger = logging.getLogger(__name__+'.text_logger')

    text_logger.info('Started regex filter: %s', q_re.pattern)
    out_l = [s for s in seqs if q_re.search(str(s.seq))]
    text_logger.info('Finished regex filter. Kept %i sequences.', len(out_l))
    return out_l

def compile_res(seqs):
    """
    Compile regex for each string in a list, return list of regex objects.
    """
    # Takes a list of sequences you want to filter for
    # Outputs a list of regex objects that you can iterate over
    return [re.compile(s) for s in seqs]

#####################
# Barcode Filtering
#####################

def barcodeDemux(seqs, bcs):
    """
    Takes lists of sequence objects, dict of barcodes (indexed by expt. ID)
    Demuxes based on the barcode the sequences start with
    Discards sequences that don't start with a barcode exact match
    Assumes forward read -> sequences start with a barcode
    """
    text_logger = logging.getLogger(__name__+'.text_logger')
    text_logger.info('Started barcode demuxing.')

    n_seqs = 0
    bc_filtered_data = {}
    for expt in bcs.keys():
        bc_filtered_data[expt] = [s for s in seqs if str(s.seq).startswith(bcs[expt])]
        n_seqs += len(bc_filtered_data[expt])

    text_logger.info('Finished barcode demuxing. Kept %i of %i sequences.', n_seqs, len(seqs))

    return bc_filtered_data

#####################
# Paired End Match Filtering
#####################

def get_coords(s):
    return ':'.join(s.description.split(' ')[0].split(':')[3:])

def get_sense(s):
    return s.description.split(' ')[1].split(':')[0]

def get_copied_seq(s, f_res):
    return s[f_res[0].search(str(s.seq)).end():list(f_res[1].finditer(str(s.seq)))[-1].start()]

def trim_lig_adapter(s, f_res):
    return s[:list(f_res[1].finditer(str(s.seq)))[-1].start()]

def gen_copied_seq_function(f_res):
    return lambda s: get_copied_seq(s, f_res)

def filter_pe_mismatch(f_seqs, pe_seqs, copied_func):
    """
    Args:
        f_seqs - sequences from forward reads. Presumably filtered for the
                 required adatper(s).
        pe_seqs - the paired end sequences of f_seqs. Also presumably filtered
                  for the required adapter(s).
        copied_func - takes a sequence, should ouptut the DNA that we expect
                      to have been copied, i.e. that should be on the paired
                      end read.

    Outputs a list of forward sequences that pass two filters:
        * Have a coordinate match in the paired end reads
        * That coordinate match has the same sequence.
    """
    text_logger = logging.getLogger(__name__+'.text_logger')
    text_logger.info('Started Paired-End Filtering')

    # Some housekeeping stuff
    proc_ct = 0 # number of sequences processed
    co_ct = 0 # number of sequences with coordinate matches
    aln_ct = 0 # number of sequences that have paired end sequence matches
    matched_seq_list = []

    # Get coordinate list
    pe_coord_list = [get_coords(s) for s in pe_seqs]
    for s in f_seqs:
        if get_coords(s) in pe_coord_list: # Filter based on paired-end presence
            co_ct += 1
            copied = copied_func(s) # Get the part of the sequence that was actually copied
            if str(pe_seqs[0].reverse_complement().seq).find(str(copied.seq)): # Filter on PE match
                aln_ct += 1
                matched_seq_list.append(s)

        proc_ct += 1
        if not (proc_ct % 5000):
            text_logger.info("Processed %i out of %i", proc_ct, len(f_seqs))

    text_logger.info("Finished Paired-End Filtering")
    text_logger.info("""Kept %i of %i forward sequences after coordinate
                     filtering""", co_ct, len(f_seqs))
    text_logger.info("""Kept %i of %i forward sequences after paired-end sequence
                     matching""", aln_ct, co_ct)

    return matched_seq_list

#####################
# Q-score Filtering
#####################

def quality_filter(seqs, q_cutoff=20):
    text_logger = logging.getLogger(__name__+'.text_logger')
    text_logger.info('Started Quality Score Filtering')
    out_l = [s for s in seqs
             if not any(s.letter_annotations['phred_quality']
                        < np.ones(len(s.letter_annotations['phred_quality']))*q_cutoff)]
    text_logger.info('Finished Quality Score Filtering. Kept %i of %i sequences.',
                     len(out_l), len(seqs))
    return out_l

#####################
# Length Filtering
#####################

def len_filter(seqs, l_cutoff=40, u_cutoff=200, l_barcode=0):
    """
    Return only sequence objects that have length between l_cutoff and
    u_cutoff
    """
    text_logger = logging.getLogger(__name__+'.text_logger')
    text_logger.info('Started Length Filtering')
    out_l = [s for s in seqs if (len(s.seq) >= (l_cutoff + l_barcode)) and
             (len(s.seq) <= (u_cutoff + l_barcode))]
    text_logger.info('Finished Length Filtering. Kept %i of %i sequences.',
                     len(out_l), len(seqs))
    return out_l

#####################
# Alignment Filtering
#####################

def alignment_filter(seqs, template, gapopen=10, gapextend=0.5, lo_cutoff=0,
                     hi_cutoff=1000, cleanup=True):
    text_logger = logging.getLogger(__name__+'.text_logger')
    text_logger.info('Started alignment-based filtering')
    start_n_seqs = len(seqs)

    # Save the template and sequences as temporary fasta files
    # Probably some hacking that can be done in the NeedleCommandline stuff
    seqs_f_name = 'tempseq.fa'

    with open(seqs_f_name, 'w') as sh:
        SeqIO.write(seqs, sh, 'fastq')

    # Generate alignment command, run the alignment
    text_logger.info("""Began EMBOSS needle routine with settings:\ngapopen:
                    %i\ngapextend: %i\nlo_cutoff: %i\nhi_cutoff: %i""",
                 gapopen, gapextend, lo_cutoff, hi_cutoff)
    ofilen = 'temp_'+str(uuid.uuid4())+'.needle'
    needle_cline = NeedleCommandline(asequence='asis::{}'.format(template),
                                     bsequence=seqs_f_name, gapopen=gapopen,
                                     gapextend=gapextend, outfile=ofilen)
    needle_cline()
    text_logger.info('Finished EMBOSS needle routine')

    aln_data = AlignIO.parse(open(ofilen), "emboss")
    new_seqs = cull_alignments(aln_data, lo_cutoff=lo_cutoff,
                               hi_cutoff=hi_cutoff)

    # Exit routine
    if cleanup:
        text_logger.info('Cleaning up temp files')
        os.remove(seqs_f_name)
        os.remove(ofilen)
    text_logger.info("""Finished alignment-based filtering. Kept %i of %i
                     sequences.""", len(new_seqs), start_n_seqs)
    return new_seqs

def cull_alignments(aln_data, lo_cutoff=400, hi_cutoff=650):
    new_seqs = []

    for alignment in aln_data:
        if (alignment.annotations['score'] > lo_cutoff) \
            and (alignment.annotations['score'] < hi_cutoff):
            # Template should have no gaps, and should contain the whole
            # non-template sequence
            if not str(alignment[0].seq).count('-') > 0:
                new_seqs.append(alignment[1])
                new_seqs[-1].annotations['alnscore'] = alignment.annotations['score']
    return new_seqs

#####################
# Main Routines
#####################

def setup_logger(name, file_name, log_format, level=logging.INFO):
    logger = logging.getLogger(name)
    logger.setLevel(level)
    handler = logging.FileHandler(file_name)
    formatter = logging.Formatter(log_format)
    handler.setFormatter(formatter)
    logger.addHandler(handler)

    return logger

def run_all_experiments(yf_name, save_intermediates=True):
    """
    Filters all sequences noted in the passed YAML file.
    """
    # Setup text_logger
    text_logger = setup_logger(__name__+'.text_logger',
                               'ngs_%s.log' % time.strftime("%Y%m%d-%H%M%S"),
                               '%(asctime)s %(message)s')
    csv_logger = setup_logger(__name__+'.csv_logger',
                              'ngs_filter_%s.csv' % time.strftime("%Y%m%d-%H%M%S"),
                              '%(message)s')

    # Load YAML file
    with open(yf_name) as expt_f:
        expt_yaml = yaml.load(expt_f) # Should probably make this a class at some point...
    text_logger.info('Loaded YAML experiment file '+yf_name)

    runs = expt_yaml['ngsruns']
    text_logger.info('Found NGS Runs: '+', '.join(runs))

    for run in tqdm(runs.keys()):
        text_logger.info('Performing routine for NGS Run '+run)
        expts = runs[run]['experiments']
        text_logger.info('Found experiments '+', '.join(expts))

        # Get barcodes, templates for all experiments in the run
        bcs = {}
        templates = {}
        for expt in expts:
            bcs[expt] = expt_yaml['experiments'][expt]['barcode']
            templates[expt] = expt_yaml['experiments'][expt]['template_seq']

        # Do filtering
        text_logger.info('Starting filtering for run %s', run)
        aln_seqs = filter_sample(runs[run]['f_read_name'],
                                 runs[run]['pe_read_name'],
                                 bcs, templates,
                                 runs[run]['filter_seqs']['forward'],
                                 runs[run]['filter_seqs']['reverse'])
        if save_intermediates:
            for expt in aln_seqs.keys():
                with open('aln_seqs_%s_%s.fa' % (run, expt), 'w') as out_f:
                    SeqIO.write(aln_seqs[expt], out_f, 'fasta')
        text_logger.info('Finished filtering for run %s', run)

if __name__ == '__main__':
    if len(sys.argv) > 1:
        yaml_name = sys.argv[1]
    else:
        yaml_name = 'samples.yaml'

    run_all_experiments(yaml_name, save_intermediates=True)
<|endoftext|>


<commit_before>#/*******************************************************************************
# * Copyright (c) 2013 GigaSpaces Technologies Ltd. All rights reserved
# *
# * Licensed under the Apache License, Version 2.0 (the "License");
# * you may not use this file except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *       http://www.apache.org/licenses/LICENSE-2.0
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.
# *******************************************************************************/

import logging
import os
from os.path import dirname
import subprocess
import shlex
import ast
import _ast
from os import path

from celery.utils.log import get_task_logger
from cloudify.constants import VIRTUALENV_PATH_KEY
from cloudify.utils import get_cosmo_properties
from cloudify.decorators import operation

logger = get_task_logger(__name__)
logger.level = logging.DEBUG

@operation
def install(plugin, __cloudify_id, **kwargs):

    """
    Installs plugin as celery task according to the provided plugins details.
    plugin parameter is expected to be in the following format: { name: "...", url: "..." }
    The plugin url should be a URL pointing to either a zip or tar.gz file.
    """

    logger.debug("installing plugin [%s] in host [%s]", plugin, __cloudify_id)

    management_ip = get_cosmo_properties()["management_ip"]
    if management_ip:
        plugin["url"] = plugin['url'].replace("#{plugin_repository}", "http://{0}:{1}".format(management_ip, "53229"))

    install_celery_plugin(plugin)

def uninstall(plugin, __cloudify_id, **kwargs):

    logger.debug("uninstalling plugin [%s] in host [%s]", plugin, __cloudify_id)

    uninstall_celery_plugin(plugin['name'])

@operation
def verify_plugin(worker_id, plugin_name, operation, throw_on_failure,  **kwargs):

    """
    Verifies that a plugin and and a specific operation is registered within the celery worker
    """
    out = run_command("{0} inspect registered -d {1} --no-color".format(get_celery(), worker_id))
    lines = out.splitlines()
    registered_operations = []
    for line in lines:
        processed_line = line.strip()
        if processed_line.startswith("*"):
            task_name = processed_line[1:].strip()
            if task_name.startswith(plugin_name):
                registered_operations.append(task_name)
                if task_name == plugin_name + "." + operation:
                    return True
    #Could not locate registered plugin and the specified operation
    if throw_on_failure:
        raise RuntimeError(
"""unable to locate plugin {0} operation {1} in celery registered tasks, make sure the plugin has an implementation of
this operation.
Registered plugin operation are: {2}""".format(plugin_name, operation, registered_operations))
    else:
        return False

@operation
def get_arguments(plugin_name, operation, **kwargs):
    """
    Gets the arguments of an installed plugin operation
    """
    operation_split = operation.split('.')
    module_name = '.'.join(operation_split[:-1])
    method_name = operation.split(".")[-1]
    cosmo_dir = dirname(dirname(os.path.abspath(__file__)))
    plugin_dir = os.path.join(cosmo_dir, plugin_name)
    py_path = _extract_py_path(plugin_dir, module_name, method_name, plugin_name)
    parsed_tasks_file = ast.parse(open(py_path, 'r').read())
    method_description = filter(lambda item: type(item) == _ast.FunctionDef and item.name == method_name,
                                parsed_tasks_file.body)
    if not method_description:
        raise RuntimeError("unable to locate operation {0} inside plugin file {1} [plugin name={2}]"
                           .format(method_name, py_path, plugin_name))
    return map(lambda arg: arg.id, method_description[0].args.args)

def _extract_py_path(plugin_dir, module_name, method_name, plugin_name):
    module_path = module_name.replace('.', '/')
    path1 = path.join(plugin_dir, module_path + '.py')
    if path.isfile(path1):
        return path1
    if module_path != '':
        module_path += '/'
    path2 = path.join(plugin_dir, module_path + '__init__.py')
    if path.isfile(path2):
        return path2
    raise RuntimeError('Unable to locate module containing operation {0}.{1} for plugin {2} tried '
                       '{3} and {4}'.format(module_name, method_name, plugin_name, path1, path2))

def get_pip():
    return get_prefix_for_command("pip")

def get_celery():
    return get_prefix_for_command("celery")

def get_prefix_for_command(command):
    try:
        return os.path.join(os.environ[VIRTUALENV_PATH_KEY], "bin", command)
    except KeyError:
        logger.warning("virtual env does not exist. running regular command {0}".format(command))
        return command

def install_celery_plugin(plugin):

    """
    Installs a plugin from a url as a python library.

    ``plugin['url']`` url to zipped version of the python project.

            - needed for pip installation.

    ``plugin['name']`` is the full name of the plugin.

            - needed for logging.
    """

    plugin_name = plugin["name"]
    plugin_url = plugin["url"]

    # this will install the plugin and its dependencies into the python installation
    command = "{0} install --process-dependency-links {1}".format(get_pip(), plugin_url)
    run_command(command)
    logger.debug("installed plugin {0} and dependencies into python installation".format(plugin_name))

def uninstall_celery_plugin(plugin_name):

    """
    Uninstalls a plugin from a url as a python library.

    ``plugin_name`` is the full name of the plugin.

    """

    # this will install the plugin and its dependencies into the python installation
    command = "{0} uninstall -y {1}".format(get_pip(), plugin_name)
    run_command(command)
    logger.debug("uninstalled plugin {0} from python installation".format(plugin_name))

def get_plugin_simple_name(full_plugin_name):
    return full_plugin_name.split(".")[-1:][0]

def run_command(command):
    shlex_split = shlex.split(command)
    logger.info("Running command {0}".format(command))
    p = subprocess.Popen(shlex_split, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = p.communicate()
    if p.returncode != 0:
        raise RuntimeError("Failed running command {0} [returncode={1}, "
                           "output={2}, error={3}]".format(command, p.returncode, out, err))
    if out is None:
        raise RuntimeError("Running command {0} returned None!".format(command))
    return out
<commit_msg>CLOUDIFY-2370 change log to print<commit_after>#/*******************************************************************************
# * Copyright (c) 2013 GigaSpaces Technologies Ltd. All rights reserved
# *
# * Licensed under the Apache License, Version 2.0 (the "License");
# * you may not use this file except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *       http://www.apache.org/licenses/LICENSE-2.0
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.
# *******************************************************************************/

import logging
import os
from os.path import dirname
import subprocess
import shlex
import ast
import _ast
from os import path

from celery.utils.log import get_task_logger
from cloudify.constants import VIRTUALENV_PATH_KEY
from cloudify.utils import get_cosmo_properties
from cloudify.decorators import operation

logger = get_task_logger(__name__)
logger.level = logging.DEBUG

@operation
def install(plugin, __cloudify_id, **kwargs):

    """
    Installs plugin as celery task according to the provided plugins details.
    plugin parameter is expected to be in the following format: { name: "...", url: "..." }
    The plugin url should be a URL pointing to either a zip or tar.gz file.
    """

    logger.debug("installing plugin [%s] in host [%s]", plugin, __cloudify_id)

    management_ip = get_cosmo_properties()["management_ip"]
    if management_ip:
        plugin["url"] = plugin['url'].replace("#{plugin_repository}", "http://{0}:{1}".format(management_ip, "53229"))

    install_celery_plugin(plugin)

def uninstall(plugin, __cloudify_id, **kwargs):

    logger.debug("uninstalling plugin [%s] in host [%s]", plugin, __cloudify_id)

    uninstall_celery_plugin(plugin['name'])

@operation
def verify_plugin(worker_id, plugin_name, operation, throw_on_failure,  **kwargs):

    """
    Verifies that a plugin and and a specific operation is registered within the celery worker
    """
    out = run_command("{0} inspect registered -d {1} --no-color".format(get_celery(), worker_id))
    lines = out.splitlines()
    registered_operations = []
    for line in lines:
        processed_line = line.strip()
        if processed_line.startswith("*"):
            task_name = processed_line[1:].strip()
            if task_name.startswith(plugin_name):
                registered_operations.append(task_name)
                if task_name == plugin_name + "." + operation:
                    return True
    #Could not locate registered plugin and the specified operation
    if throw_on_failure:
        raise RuntimeError(
"""unable to locate plugin {0} operation {1} in celery registered tasks, make sure the plugin has an implementation of
this operation.
Registered plugin operation are: {2}""".format(plugin_name, operation, registered_operations))
    else:
        return False

@operation
def get_arguments(plugin_name, operation, **kwargs):
    """
    Gets the arguments of an installed plugin operation
    """
    operation_split = operation.split('.')
    module_name = '.'.join(operation_split[:-1])
    method_name = operation.split(".")[-1]
    cosmo_dir = dirname(dirname(os.path.abspath(__file__)))
    plugin_dir = os.path.join(cosmo_dir, plugin_name)
    py_path = _extract_py_path(plugin_dir, module_name, method_name, plugin_name)
    parsed_tasks_file = ast.parse(open(py_path, 'r').read())
    method_description = filter(lambda item: type(item) == _ast.FunctionDef and item.name == method_name,
                                parsed_tasks_file.body)
    if not method_description:
        raise RuntimeError("unable to locate operation {0} inside plugin file {1} [plugin name={2}]"
                           .format(method_name, py_path, plugin_name))
    return map(lambda arg: arg.id, method_description[0].args.args)

def _extract_py_path(plugin_dir, module_name, method_name, plugin_name):
    module_path = module_name.replace('.', '/')
    path1 = path.join(plugin_dir, module_path + '.py')
    if path.isfile(path1):
        return path1
    if module_path != '':
        module_path += '/'
    path2 = path.join(plugin_dir, module_path + '__init__.py')
    if path.isfile(path2):
        return path2
    raise RuntimeError('Unable to locate module containing operation {0}.{1} for plugin {2} tried '
                       '{3} and {4}'.format(module_name, method_name, plugin_name, path1, path2))

def get_pip():
    return get_prefix_for_command("pip")

def get_celery():
    return get_prefix_for_command("celery")

def get_prefix_for_command(command):
    try:
        return os.path.join(os.environ[VIRTUALENV_PATH_KEY], "bin", command)
    except KeyError:
        logger.warning("virtual env does not exist. running regular command {0}".format(command))
        return command

def install_celery_plugin(plugin):

    """
    Installs a plugin from a url as a python library.

    ``plugin['url']`` url to zipped version of the python project.

            - needed for pip installation.

    ``plugin['name']`` is the full name of the plugin.

            - needed for logging.
    """

    plugin_name = plugin["name"]
    plugin_url = plugin["url"]

    # this will install the plugin and its dependencies into the python installation
    command = "{0} install --process-dependency-links {1}".format(get_pip(), plugin_url)
    run_command(command)
    logger.debug("installed plugin {0} and dependencies into python installation".format(plugin_name))

def uninstall_celery_plugin(plugin_name):

    """
    Uninstalls a plugin from a url as a python library.

    ``plugin_name`` is the full name of the plugin.

    """

    # this will install the plugin and its dependencies into the python installation
    command = "{0} uninstall -y {1}".format(get_pip(), plugin_name)
    run_command(command)
    logger.debug("uninstalled plugin {0} from python installation".format(plugin_name))

def get_plugin_simple_name(full_plugin_name):
    return full_plugin_name.split(".")[-1:][0]

def run_command(command):
    shlex_split = shlex.split(command)
    print "Running command {0}".format(command)
    p = subprocess.Popen(shlex_split, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = p.communicate()
    if p.returncode != 0:
        raise RuntimeError("Failed running command {0} [returncode={1}, "
                           "output={2}, error={3}]".format(command, p.returncode, out, err))
    if out is None:
        raise RuntimeError("Running command {0} returned None!".format(command))
    return out
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""Actions and snippets."""

# -----------------------------------------------------------------------------
# Imports
# -----------------------------------------------------------------------------

from functools import partial
import logging
import sys
import traceback

from six import string_types, PY3

from .qt import QKeySequence, QAction, require_qt
from phy.utils import Bunch

logger = logging.getLogger(__name__)

# -----------------------------------------------------------------------------
# Snippet parsing utilities
# -----------------------------------------------------------------------------

def _parse_arg(s):
    """Parse a number or string."""
    try:
        return int(s)
    except ValueError:
        pass
    try:
        return float(s)
    except ValueError:
        pass
    return s

def _parse_list(s):
    """Parse a comma-separated list of values (strings or numbers)."""
    # Range: 'x-y'
    if '-' in s:
        m, M = map(_parse_arg, s.split('-'))
        return list(range(m, M + 1))
    # List of ids: 'x,y,z'
    elif ',' in s:
        return list(map(_parse_arg, s.split(',')))
    else:
        return _parse_arg(s)

def _parse_snippet(s):
    """Parse an entire snippet command."""
    return list(map(_parse_list, s.split(' ')))

# -----------------------------------------------------------------------------
# Show shortcut utility functions
# -----------------------------------------------------------------------------

def _get_shortcut_string(shortcut):
    """Return a string representation of a shortcut."""
    if shortcut is None:
        return ''
    if isinstance(shortcut, (tuple, list)):
        return ', '.join([_get_shortcut_string(s) for s in shortcut])
    if isinstance(shortcut, string_types):
        if hasattr(QKeySequence, shortcut):
            shortcut = QKeySequence(getattr(QKeySequence, shortcut))
        else:
            return shortcut.lower()
    assert isinstance(shortcut, QKeySequence)
    s = shortcut.toString() or ''
    return str(s).lower()

def _get_qkeysequence(shortcut):
    """Return a QKeySequence or list of QKeySequence from a shortcut string."""
    if shortcut is None:
        return []
    if isinstance(shortcut, (tuple, list)):
        return [_get_qkeysequence(s) for s in shortcut]
    assert isinstance(shortcut, string_types)
    if hasattr(QKeySequence, shortcut):
        return QKeySequence(getattr(QKeySequence, shortcut))
    sequence = QKeySequence.fromString(shortcut)
    assert not sequence.isEmpty()
    return sequence

def _show_shortcuts(shortcuts, name=None):
    """Display shortcuts."""
    name = name or ''
    print()
    if name:
        name = ' for ' + name
    print('Keyboard shortcuts' + name)
    for name in sorted(shortcuts):
        shortcut = _get_shortcut_string(shortcuts[name])
        print('{0:<40}: {1:s}'.format(name, shortcut))
    print()

# -----------------------------------------------------------------------------
# Actions
# -----------------------------------------------------------------------------

def _alias(name):
    # Get the alias from the character after & if it exists.
    alias = name[name.index('&') + 1] if '&' in name else name
    return alias

@require_qt
def _create_qaction(gui, name, callback, shortcut):
    # Create the QAction instance.
    action = QAction(name, gui)

    def wrapped(checked, *args, **kwargs):  # pragma: no cover
        return callback(*args, **kwargs)

    action.triggered.connect(wrapped)
    sequence = _get_qkeysequence(shortcut)
    if not isinstance(sequence, (tuple, list)):
        sequence = [sequence]
    action.setShortcuts(sequence)
    return action

class Actions(object):
    """Handle GUI actions.

    This class attaches to a GUI and implements the following features:

    * Add and remove actions
    * Keyboard shortcuts for the actions
    * Display all shortcuts

    """
    def __init__(self, gui, default_shortcuts=None):
        self._actions_dict = {}
        self._aliases = {}
        self._default_shortcuts = default_shortcuts or {}
        self.gui = gui
        gui.actions.append(self)

    def add(self, callback=None, name=None, shortcut=None, alias=None,
            verbose=True):
        """Add an action with a keyboard shortcut."""
        # TODO: add menu_name option and create menu bar
        if callback is None:
            # Allow to use either add(func) or @add or @add(...).
            return partial(self.add, name=name, shortcut=shortcut, alias=alias)
        assert callback

        # Get the name from the callback function if needed.
        name = name or callback.__name__
        alias = alias or _alias(name)
        name = name.replace('&', '')
        shortcut = shortcut or self._default_shortcuts.get(name, None)

        # Skip existing action.
        if name in self._actions_dict:
            return

        # Create and register the action.
        action = _create_qaction(self.gui, name, callback, shortcut)
        action_obj = Bunch(qaction=action, name=name, alias=alias,
                           shortcut=shortcut, callback=callback)
        if verbose and not name.startswith('_'):
            logger.log(5, "Add action `%s` (%s).", name,
                       _get_shortcut_string(action.shortcut()))
        self.gui.addAction(action)
        self._actions_dict[name] = action_obj
        # Register the alias -> name mapping.
        self._aliases[alias] = name

        # Set the callback method.
        if callback:
            setattr(self, name, callback)

    def disable(self, name=None):
        """Disable one or all actions."""
        if name is None:
            for name in self._actions_dict:
                self.disable(name)
            return
        self._actions_dict[name].qaction.setEnabled(False)

    def enable(self, name=None):
        """Enable one or all actions."""
        if name is None:
            for name in self._actions_dict:
                self.enable(name)
            return
        self._actions_dict[name].qaction.setEnabled(True)

    def get(self, name):
        """Get a QAction instance from its name."""
        return self._actions_dict[name].qaction

    def run(self, name, *args):
        """Run an action as specified by its name."""
        assert isinstance(name, string_types)
        # Resolve the alias if it is an alias.
        name = self._aliases.get(name, name)
        # Get the action.
        action = self._actions_dict.get(name, None)
        if not action:
            raise ValueError("Action `{}` doesn't exist.".format(name))
        if not name.startswith('_'):
            logger.debug("Execute action `%s`.", name)
        return action.callback(*args)

    def remove(self, name):
        """Remove an action."""
        self.gui.removeAction(self._actions_dict[name].qaction)
        del self._actions_dict[name]
        delattr(self, name)

    def remove_all(self):
        """Remove all actions."""
        names = sorted(self._actions_dict.keys())
        for name in names:
            self.remove(name)

    @property
    def shortcuts(self):
        """A dictionary of action shortcuts."""
        return {name: action.shortcut
                for name, action in self._actions_dict.items()}

    def show_shortcuts(self):
        """Print all shortcuts."""
        _show_shortcuts(self.shortcuts, self.gui.windowTitle())

    def __contains__(self, name):
        return name in self._actions_dict

    def __repr__(self):
        return '<Actions {}>'.format(sorted(self._actions_dict))

# -----------------------------------------------------------------------------
# Snippets
# -----------------------------------------------------------------------------

class Snippets(object):
    """Provide keyboard snippets to quickly execute actions from a GUI.

    This class attaches to a GUI and an `Actions` instance. To every command
    is associated a snippet with the same name, or with an alias as indicated
    in the action. The arguments of the action's callback functions can be
    provided in the snippet's command with a simple syntax. For example, the
    following command:

    ```
    :my_action string 3-6
    ```

    corresponds to:

    ```python
    my_action('string', (3, 4, 5, 6))
    ```

    The snippet mode is activated with the `:` keyboard shortcut. A snippet
    command is activated with `Enter`, and one can leave the snippet mode
    with `Escape`.

    """

    # HACK: Unicode characters do not seem to work on Python 2
    cursor = '\u200A\u258C' if PY3 else ''

    # Allowed characters in snippet mode.
    # A Qt shortcut will be created for every character.
    _snippet_chars = ("abcdefghijklmnopqrstuvwxyz0123456789"
                      " ,.;?!_-+~=*/\(){}[]")

    def __init__(self, gui):
        self.gui = gui

        self.actions = Actions(gui)

        # Register snippet mode shortcut.
        @self.actions.add(shortcut=':')
        def enable_snippet_mode():
            self.mode_on()

        self._create_snippet_actions()

    @property
    def command(self):
        """This is used to write a snippet message in the status bar.

        A cursor is appended at the end.

        """
        msg = self.gui.status_message
        n = len(msg)
        n_cur = len(self.cursor)
        return msg[:n - n_cur]

    @command.setter
    def command(self, value):
        value += self.cursor
        self.gui.status_message = value

    def _backspace(self):
        """Erase the last character in the snippet command."""
        if self.command == ':':
            return
        logger.log(5, "Snippet keystroke `Backspace`.")
        self.command = self.command[:-1]

    def _enter(self):
        """Disable the snippet mode and execute the command."""
        command = self.command
        logger.log(5, "Snippet keystroke `Enter`.")
        # NOTE: we need to set back the actions (mode_off) before running
        # the command.
        self.mode_off()
        self.run(command)

    def _create_snippet_actions(self):
        """Add mock Qt actions for snippet keystrokes.

        Used to enable snippet mode.

        """
        # One action per allowed character.
        for i, char in enumerate(self._snippet_chars):

            def _make_func(char):
                def callback():
                    logger.log(5, "Snippet keystroke `%s`.", char)
                    self.command += char
                return callback

            self.actions.add(name='_snippet_{}'.format(i),
                             shortcut=char,
                             callback=_make_func(char))

        self.actions.add(name='_snippet_backspace',
                         shortcut='backspace',
                         callback=self._backspace)
        self.actions.add(name='_snippet_activate',
                         shortcut=('enter', 'return'),
                         callback=self._enter)
        self.actions.add(name='_snippet_disable',
                         shortcut='escape',
                         callback=self.mode_off)

    def run(self, snippet):
        """Executes a snippet command.

        May be overridden.

        """
        assert snippet[0] == ':'
        snippet = snippet[1:]
        snippet_args = _parse_snippet(snippet)
        name = snippet_args[0]

        logger.info("Processing snippet `%s`.", snippet)
        try:
            # Try to run the snippet on all attached Actions instances.
            for actions in self.gui.actions:
                try:
                    actions.run(name, *snippet_args[1:])
                    return
                except ValueError:
                    # This Actions instance doesn't contain the requested
                    # snippet, trying the next attached Actions instance.
                    pass
            logger.warn("Couldn't find action `%s`.", name)
        except Exception as e:
            logger.warn("Error when executing snippet: \"%s\".", str(e))
            logger.debug(''.join(traceback.format_exception(*sys.exc_info())))

    def is_mode_on(self):
        return self.command.startswith(':')

    def mode_on(self):
        logger.info("Snippet mode enabled, press `escape` to leave this mode.")

        # Silent all actions except the Snippets actions.
        for actions in self.gui.actions:
            if actions != self.actions:
                actions.disable()
        self.actions.enable()

        self.command = ':'

    def mode_off(self):
        self.gui.status_message = ''

        # Re-enable all actions except the Snippets actions.
        self.actions.disable()
        for actions in self.gui.actions:
            if actions != self.actions:
                actions.enable()

        logger.info("Snippet mode disabled.")
<commit_msg>Enable snippet mode action always enabled<commit_after># -*- coding: utf-8 -*-

"""Actions and snippets."""

# -----------------------------------------------------------------------------
# Imports
# -----------------------------------------------------------------------------

from functools import partial
import logging
import sys
import traceback

from six import string_types, PY3

from .qt import QKeySequence, QAction, require_qt
from phy.utils import Bunch

logger = logging.getLogger(__name__)

# -----------------------------------------------------------------------------
# Snippet parsing utilities
# -----------------------------------------------------------------------------

def _parse_arg(s):
    """Parse a number or string."""
    try:
        return int(s)
    except ValueError:
        pass
    try:
        return float(s)
    except ValueError:
        pass
    return s

def _parse_list(s):
    """Parse a comma-separated list of values (strings or numbers)."""
    # Range: 'x-y'
    if '-' in s:
        m, M = map(_parse_arg, s.split('-'))
        return list(range(m, M + 1))
    # List of ids: 'x,y,z'
    elif ',' in s:
        return list(map(_parse_arg, s.split(',')))
    else:
        return _parse_arg(s)

def _parse_snippet(s):
    """Parse an entire snippet command."""
    return list(map(_parse_list, s.split(' ')))

# -----------------------------------------------------------------------------
# Show shortcut utility functions
# -----------------------------------------------------------------------------

def _get_shortcut_string(shortcut):
    """Return a string representation of a shortcut."""
    if shortcut is None:
        return ''
    if isinstance(shortcut, (tuple, list)):
        return ', '.join([_get_shortcut_string(s) for s in shortcut])
    if isinstance(shortcut, string_types):
        if hasattr(QKeySequence, shortcut):
            shortcut = QKeySequence(getattr(QKeySequence, shortcut))
        else:
            return shortcut.lower()
    assert isinstance(shortcut, QKeySequence)
    s = shortcut.toString() or ''
    return str(s).lower()

def _get_qkeysequence(shortcut):
    """Return a QKeySequence or list of QKeySequence from a shortcut string."""
    if shortcut is None:
        return []
    if isinstance(shortcut, (tuple, list)):
        return [_get_qkeysequence(s) for s in shortcut]
    assert isinstance(shortcut, string_types)
    if hasattr(QKeySequence, shortcut):
        return QKeySequence(getattr(QKeySequence, shortcut))
    sequence = QKeySequence.fromString(shortcut)
    assert not sequence.isEmpty()
    return sequence

def _show_shortcuts(shortcuts, name=None):
    """Display shortcuts."""
    name = name or ''
    print()
    if name:
        name = ' for ' + name
    print('Keyboard shortcuts' + name)
    for name in sorted(shortcuts):
        shortcut = _get_shortcut_string(shortcuts[name])
        print('{0:<40}: {1:s}'.format(name, shortcut))
    print()

# -----------------------------------------------------------------------------
# Actions
# -----------------------------------------------------------------------------

def _alias(name):
    # Get the alias from the character after & if it exists.
    alias = name[name.index('&') + 1] if '&' in name else name
    return alias

@require_qt
def _create_qaction(gui, name, callback, shortcut):
    # Create the QAction instance.
    action = QAction(name, gui)

    def wrapped(checked, *args, **kwargs):  # pragma: no cover
        return callback(*args, **kwargs)

    action.triggered.connect(wrapped)
    sequence = _get_qkeysequence(shortcut)
    if not isinstance(sequence, (tuple, list)):
        sequence = [sequence]
    action.setShortcuts(sequence)
    return action

class Actions(object):
    """Handle GUI actions.

    This class attaches to a GUI and implements the following features:

    * Add and remove actions
    * Keyboard shortcuts for the actions
    * Display all shortcuts

    """
    def __init__(self, gui, default_shortcuts=None):
        self._actions_dict = {}
        self._aliases = {}
        self._default_shortcuts = default_shortcuts or {}
        self.gui = gui
        gui.actions.append(self)

    def add(self, callback=None, name=None, shortcut=None, alias=None,
            verbose=True):
        """Add an action with a keyboard shortcut."""
        # TODO: add menu_name option and create menu bar
        if callback is None:
            # Allow to use either add(func) or @add or @add(...).
            return partial(self.add, name=name, shortcut=shortcut, alias=alias)
        assert callback

        # Get the name from the callback function if needed.
        name = name or callback.__name__
        alias = alias or _alias(name)
        name = name.replace('&', '')
        shortcut = shortcut or self._default_shortcuts.get(name, None)

        # Skip existing action.
        if name in self._actions_dict:
            return

        # Create and register the action.
        action = _create_qaction(self.gui, name, callback, shortcut)
        action_obj = Bunch(qaction=action, name=name, alias=alias,
                           shortcut=shortcut, callback=callback)
        if verbose and not name.startswith('_'):
            logger.log(5, "Add action `%s` (%s).", name,
                       _get_shortcut_string(action.shortcut()))
        self.gui.addAction(action)
        self._actions_dict[name] = action_obj
        # Register the alias -> name mapping.
        self._aliases[alias] = name

        # Set the callback method.
        if callback:
            setattr(self, name, callback)

    def disable(self, name=None):
        """Disable one or all actions."""
        if name is None:
            for name in self._actions_dict:
                self.disable(name)
            return
        self._actions_dict[name].qaction.setEnabled(False)

    def enable(self, name=None):
        """Enable one or all actions."""
        if name is None:
            for name in self._actions_dict:
                self.enable(name)
            return
        self._actions_dict[name].qaction.setEnabled(True)

    def get(self, name):
        """Get a QAction instance from its name."""
        return self._actions_dict[name].qaction

    def run(self, name, *args):
        """Run an action as specified by its name."""
        assert isinstance(name, string_types)
        # Resolve the alias if it is an alias.
        name = self._aliases.get(name, name)
        # Get the action.
        action = self._actions_dict.get(name, None)
        if not action:
            raise ValueError("Action `{}` doesn't exist.".format(name))
        if not name.startswith('_'):
            logger.debug("Execute action `%s`.", name)
        return action.callback(*args)

    def remove(self, name):
        """Remove an action."""
        self.gui.removeAction(self._actions_dict[name].qaction)
        del self._actions_dict[name]
        delattr(self, name)

    def remove_all(self):
        """Remove all actions."""
        names = sorted(self._actions_dict.keys())
        for name in names:
            self.remove(name)

    @property
    def shortcuts(self):
        """A dictionary of action shortcuts."""
        return {name: action.shortcut
                for name, action in self._actions_dict.items()}

    def show_shortcuts(self):
        """Print all shortcuts."""
        _show_shortcuts(self.shortcuts, self.gui.windowTitle())

    def __contains__(self, name):
        return name in self._actions_dict

    def __repr__(self):
        return '<Actions {}>'.format(sorted(self._actions_dict))

# -----------------------------------------------------------------------------
# Snippets
# -----------------------------------------------------------------------------

class Snippets(object):
    """Provide keyboard snippets to quickly execute actions from a GUI.

    This class attaches to a GUI and an `Actions` instance. To every command
    is associated a snippet with the same name, or with an alias as indicated
    in the action. The arguments of the action's callback functions can be
    provided in the snippet's command with a simple syntax. For example, the
    following command:

    ```
    :my_action string 3-6
    ```

    corresponds to:

    ```python
    my_action('string', (3, 4, 5, 6))
    ```

    The snippet mode is activated with the `:` keyboard shortcut. A snippet
    command is activated with `Enter`, and one can leave the snippet mode
    with `Escape`.

    """

    # HACK: Unicode characters do not seem to work on Python 2
    cursor = '\u200A\u258C' if PY3 else ''

    # Allowed characters in snippet mode.
    # A Qt shortcut will be created for every character.
    _snippet_chars = ("abcdefghijklmnopqrstuvwxyz0123456789"
                      " ,.;?!_-+~=*/\(){}[]")

    def __init__(self, gui):
        self.gui = gui

        self.actions = Actions(gui)

        # Register snippet mode shortcut.
        @self.actions.add(shortcut=':')
        def enable_snippet_mode():
            self.mode_on()

        self._create_snippet_actions()
        self.mode_off()

    @property
    def command(self):
        """This is used to write a snippet message in the status bar.

        A cursor is appended at the end.

        """
        msg = self.gui.status_message
        n = len(msg)
        n_cur = len(self.cursor)
        return msg[:n - n_cur]

    @command.setter
    def command(self, value):
        value += self.cursor
        self.gui.status_message = value

    def _backspace(self):
        """Erase the last character in the snippet command."""
        if self.command == ':':
            return
        logger.log(5, "Snippet keystroke `Backspace`.")
        self.command = self.command[:-1]

    def _enter(self):
        """Disable the snippet mode and execute the command."""
        command = self.command
        logger.log(5, "Snippet keystroke `Enter`.")
        # NOTE: we need to set back the actions (mode_off) before running
        # the command.
        self.mode_off()
        self.run(command)

    def _create_snippet_actions(self):
        """Add mock Qt actions for snippet keystrokes.

        Used to enable snippet mode.

        """
        # One action per allowed character.
        for i, char in enumerate(self._snippet_chars):

            def _make_func(char):
                def callback():
                    logger.log(5, "Snippet keystroke `%s`.", char)
                    self.command += char
                return callback

            self.actions.add(name='_snippet_{}'.format(i),
                             shortcut=char,
                             callback=_make_func(char))

        self.actions.add(name='_snippet_backspace',
                         shortcut='backspace',
                         callback=self._backspace)
        self.actions.add(name='_snippet_activate',
                         shortcut=('enter', 'return'),
                         callback=self._enter)
        self.actions.add(name='_snippet_disable',
                         shortcut='escape',
                         callback=self.mode_off)

    def run(self, snippet):
        """Executes a snippet command.

        May be overridden.

        """
        assert snippet[0] == ':'
        snippet = snippet[1:]
        snippet_args = _parse_snippet(snippet)
        name = snippet_args[0]

        logger.info("Processing snippet `%s`.", snippet)
        try:
            # Try to run the snippet on all attached Actions instances.
            for actions in self.gui.actions:
                try:
                    actions.run(name, *snippet_args[1:])
                    return
                except ValueError:
                    # This Actions instance doesn't contain the requested
                    # snippet, trying the next attached Actions instance.
                    pass
            logger.warn("Couldn't find action `%s`.", name)
        except Exception as e:
            logger.warn("Error when executing snippet: \"%s\".", str(e))
            logger.debug(''.join(traceback.format_exception(*sys.exc_info())))

    def is_mode_on(self):
        return self.command.startswith(':')

    def mode_on(self):
        logger.info("Snippet mode enabled, press `escape` to leave this mode.")

        # Silent all actions except the Snippets actions.
        for actions in self.gui.actions:
            if actions != self.actions:
                actions.disable()
        self.actions.enable()

        self.command = ':'

    def mode_off(self):
        self.gui.status_message = ''

        # Re-enable all actions except the Snippets actions.
        self.actions.disable()
        for actions in self.gui.actions:
            if actions != self.actions:
                actions.enable()
        # The `:` shortcut should always be enabled.
        self.actions.enable('enable_snippet_mode')

        logger.info("Snippet mode disabled.")
<|endoftext|>


<commit_before>from functools import partial
import zlib
import json
from io import BytesIO
import sys
import platform

from electrum.plugins import BasePlugin, hook
from electrum_gui.qt.util import WaitingDialog, EnterButton, WindowModalDialog
from electrum.util import print_msg, print_error
from electrum.i18n import _

from PyQt5.QtGui import *
from PyQt5.QtCore import *
from PyQt5.QtWidgets import (QComboBox, QGridLayout, QLabel, QPushButton)

try:
    import amodem.audio
    import amodem.main
    import amodem.config
    print_error('Audio MODEM is available.')
    amodem.log.addHandler(amodem.logging.StreamHandler(sys.stderr))
    amodem.log.setLevel(amodem.logging.INFO)
except ImportError:
    amodem = None
    print_error('Audio MODEM is not found.')

class Plugin(BasePlugin):

    def __init__(self, parent, config, name):
        BasePlugin.__init__(self, parent, config, name)
        if self.is_available():
            self.modem_config = amodem.config.slowest()
            self.library_name = {
                'Linux': 'libportaudio.so'
            }[platform.system()]

    def is_available(self):
        return amodem is not None

    def requires_settings(self):
        return True

    def settings_widget(self, window):
        return EnterButton(_('Settings'), partial(self.settings_dialog, window))

    def settings_dialog(self, window):
        d = WindowModalDialog(window, _("Audio Modem Settings"))

        layout = QGridLayout(d)
        layout.addWidget(QLabel(_('Bit rate [kbps]: ')), 0, 0)

        bitrates = list(sorted(amodem.config.bitrates.keys()))

        def _index_changed(index):
            bitrate = bitrates[index]
            self.modem_config = amodem.config.bitrates[bitrate]

        combo = QComboBox()
        combo.addItems([str(x) for x in bitrates])
        combo.currentIndexChanged.connect(_index_changed)
        layout.addWidget(combo, 0, 1)

        ok_button = QPushButton(_("OK"))
        ok_button.clicked.connect(d.accept)
        layout.addWidget(ok_button, 1, 1)

        return bool(d.exec_())

    @hook
    def transaction_dialog(self, dialog):
        b = QPushButton()
        b.setIcon(QIcon(":icons/speaker.png"))

        def handler():
            blob = json.dumps(dialog.tx.as_dict())
            self._send(parent=dialog, blob=blob)
        b.clicked.connect(handler)
        dialog.sharing_buttons.insert(-1, b)

    @hook
    def scan_text_edit(self, parent):
        parent.addButton(':icons/microphone.png', partial(self._recv, parent),
                         _("Read from microphone"))

    @hook
    def show_text_edit(self, parent):
        def handler():
            blob = str(parent.toPlainText())
            self._send(parent=parent, blob=blob)
        parent.addButton(':icons/speaker.png', handler, _("Send to speaker"))

    def _audio_interface(self):
        interface = amodem.audio.Interface(config=self.modem_config)
        return interface.load(self.library_name)

    def _send(self, parent, blob):
        def sender_thread():
            with self._audio_interface() as interface:
                src = BytesIO(blob)
                dst = interface.player()
                amodem.main.send(config=self.modem_config, src=src, dst=dst)

        print_msg('Sending:', repr(blob))
        blob = zlib.compress(blob)

        kbps = self.modem_config.modem_bps / 1e3
        msg = 'Sending to Audio MODEM ({0:.1f} kbps)...'.format(kbps)
        WaitingDialog(parent, msg, sender_thread)

    def _recv(self, parent):
        def receiver_thread():
            with self._audio_interface() as interface:
                src = interface.recorder()
                dst = BytesIO()
                amodem.main.recv(config=self.modem_config, src=src, dst=dst)
                return dst.getvalue()

        def on_finished(blob):
            if blob:
                blob = zlib.decompress(blob)
                print_msg('Received:', repr(blob))
                parent.setText(blob)

        kbps = self.modem_config.modem_bps / 1e3
        msg = 'Receiving from Audio MODEM ({0:.1f} kbps)...'.format(kbps)
        WaitingDialog(parent, msg, receiver_thread, on_finished)
<commit_msg>amodem: fix str/bytes handling<commit_after>from functools import partial
import zlib
import json
from io import BytesIO
import sys
import platform

from electrum.plugins import BasePlugin, hook
from electrum_gui.qt.util import WaitingDialog, EnterButton, WindowModalDialog
from electrum.util import print_msg, print_error
from electrum.i18n import _

from PyQt5.QtGui import *
from PyQt5.QtCore import *
from PyQt5.QtWidgets import (QComboBox, QGridLayout, QLabel, QPushButton)

try:
    import amodem.audio
    import amodem.main
    import amodem.config
    print_error('Audio MODEM is available.')
    amodem.log.addHandler(amodem.logging.StreamHandler(sys.stderr))
    amodem.log.setLevel(amodem.logging.INFO)
except ImportError:
    amodem = None
    print_error('Audio MODEM is not found.')

class Plugin(BasePlugin):

    def __init__(self, parent, config, name):
        BasePlugin.__init__(self, parent, config, name)
        if self.is_available():
            self.modem_config = amodem.config.slowest()
            self.library_name = {
                'Linux': 'libportaudio.so'
            }[platform.system()]

    def is_available(self):
        return amodem is not None

    def requires_settings(self):
        return True

    def settings_widget(self, window):
        return EnterButton(_('Settings'), partial(self.settings_dialog, window))

    def settings_dialog(self, window):
        d = WindowModalDialog(window, _("Audio Modem Settings"))

        layout = QGridLayout(d)
        layout.addWidget(QLabel(_('Bit rate [kbps]: ')), 0, 0)

        bitrates = list(sorted(amodem.config.bitrates.keys()))

        def _index_changed(index):
            bitrate = bitrates[index]
            self.modem_config = amodem.config.bitrates[bitrate]

        combo = QComboBox()
        combo.addItems([str(x) for x in bitrates])
        combo.currentIndexChanged.connect(_index_changed)
        layout.addWidget(combo, 0, 1)

        ok_button = QPushButton(_("OK"))
        ok_button.clicked.connect(d.accept)
        layout.addWidget(ok_button, 1, 1)

        return bool(d.exec_())

    @hook
    def transaction_dialog(self, dialog):
        b = QPushButton()
        b.setIcon(QIcon(":icons/speaker.png"))

        def handler():
            blob = json.dumps(dialog.tx.as_dict())
            self._send(parent=dialog, blob=blob)
        b.clicked.connect(handler)
        dialog.sharing_buttons.insert(-1, b)

    @hook
    def scan_text_edit(self, parent):
        parent.addButton(':icons/microphone.png', partial(self._recv, parent),
                         _("Read from microphone"))

    @hook
    def show_text_edit(self, parent):
        def handler():
            blob = str(parent.toPlainText())
            self._send(parent=parent, blob=blob)
        parent.addButton(':icons/speaker.png', handler, _("Send to speaker"))

    def _audio_interface(self):
        interface = amodem.audio.Interface(config=self.modem_config)
        return interface.load(self.library_name)

    def _send(self, parent, blob):
        def sender_thread():
            with self._audio_interface() as interface:
                src = BytesIO(blob)
                dst = interface.player()
                amodem.main.send(config=self.modem_config, src=src, dst=dst)

        print_msg('Sending:', repr(blob))
        blob = zlib.compress(blob.encode('ascii'))

        kbps = self.modem_config.modem_bps / 1e3
        msg = 'Sending to Audio MODEM ({0:.1f} kbps)...'.format(kbps)
        WaitingDialog(parent, msg, sender_thread)

    def _recv(self, parent):
        def receiver_thread():
            with self._audio_interface() as interface:
                src = interface.recorder()
                dst = BytesIO()
                amodem.main.recv(config=self.modem_config, src=src, dst=dst)
                return dst.getvalue()

        def on_finished(blob):
            if blob:
                blob = zlib.decompress(blob).decode('ascii')
                print_msg('Received:', repr(blob))
                parent.setText(blob)

        kbps = self.modem_config.modem_bps / 1e3
        msg = 'Receiving from Audio MODEM ({0:.1f} kbps)...'.format(kbps)
        WaitingDialog(parent, msg, receiver_thread, on_finished)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

a = 100

from sys import float_info
import math
from math import ldexp
from pint.roundmode import roundmode
'''
    # Calculate addition, subtraction, multiplication, division and root with rounding mode.

    # using exsample
    from pint import roundfloat as rf
    from pint import roundmode as rdm

    # Calculate 0.25 + 0.1 with rounding to up.
    rf.rdadd(0.25, 0.1, rdm.up)

    # Calculate 0.25 - 0.1 with rounding to down
    rf.rdsub(0.25, 0.1, rdm.down)
'''
def split(a):
    tmp = a * (ldexp(1, 27) + 1)
    x = tmp - (tmp - a)
    y = a - x
    return x, y

# succ and pred by S. M. Rump:
def succ(a):
    abs_a = abs(a)
    if abs_a >= ldexp(1, -969):
        return a + abs_a * (ldexp(1, -53) + ldexp(1, -105))
    if abs_a <= ldexp(1, -1021):
        return a + ldexp(1, -1074)
    c = ldexp(1, 53) * a
    e = (ldexp(1, -53) + ldexp(1, -105)) * abs(c)
    return (c + e) * ldexp(1, -53)

def pred(a):
    abs_a = abs(a)
    if abs_a >= ldexp(1, -969):
        return a - abs_a * (ldexp(1, -53) + ldexp(1, -105))
    if abs_a <= ldexp(1, -1021):
        return a - ldexp(1, -1074)
    c = ldexp(1, 53) * a
    e = (ldexp(1, -53) + ldexp(1, -105)) * abs(c)
    return (c - e) * ldexp(1, -53)

# by Masahide Kashiwagi
# http://verifiedby.me/adiary/029

def twosum(a, b):
    arg1 = a
    arg2 = b
    x = arg1 + arg2
    if abs(arg1) > abs(arg2):
        tmp = x - arg1
        y = arg2 - tmp
    else:
        tmp = x - arg2
        y = arg1 - tmp
    return x, y

def twoproduct(a, b):
    arg1 = a
    arg2 = b
    x = arg1 * arg2
    if abs(arg1) > 2.**996:
        arg1fix = arg1 * 2.**(-28)
        arg2fix = arg2 * 2.**(28)
    elif abs(arg2) > 2.**(996):
        arg1fix = arg1 * 2.**(28)
        arg2fix = arg2 * 2.**(-28)
    else:
        arg1fix = arg1
        arg2fix = arg2
    aH, aL = split(arg1fix)
    bH, bL = split(arg2fix)
    if abs(x) > 2.**1023:
        y = aL * bL - ((((x * 0.5) - (aH * 0.5) * bH) * 2. - aL * bH) - aH * bL)
    else:
        y = aL * bL - (((x - aH * bH) - aL * bH) - aH * bL)
    return x, y

def rdadd(a, b, rmode=roundmode.nearest):
    arg1 = a
    arg2 = b
    x, y = twosum(arg1, arg2)
    if rmode == roundmode.up:
        if x == float('inf'):
            return x
        elif x == -float('inf'):
            if arg1 == -float('inf') or arg2 == -float('inf'):
                return x
            else:
                return -float_info.max
        if y > 0:
            x = succ(x)
    elif rmode == roundmode.down:
        if x == float('inf'):
            if arg1 == float('inf') or arg2 == float('inf'):
                return x
            else:
                return float_info.max
        elif x == -float('inf'):
            return x
        if y < 0:
            x = pred(x)
    return x

def rdsub(a, b, rmode=roundmode.nearest):
    return rdadd(a, -b, rmode)

def rdmul(a, b, rmode=roundmode.nearest):
    arg1 = a
    arg2 = b
    x, y = twoproduct(arg1, arg2)
    if rmode == roundmode.up:
        if x == float('inf'):
            return x
        elif x == -float('inf'):
            if math.fabs(arg1) == float('inf') or math.fabs(arg2) == float('inf'):
                return x
            else:
                return -float_info.max
        if abs(x) >= 2.**(-969):
            if y > 0:
                x = succ(x)
    elif rmode == roundmode.down:
        if x == float('inf'):
            if math.fabs(arg1) == float('inf') or math.fabs(arg2) == float('inf'):
                return x
            else:
                return float_info.max
        elif x == -float('inf'):
            return x

        if abs(x) >= 2.**(-969):
            if y < 0.:
                return pred(x)
        else:
            s1, s2 = twoproduct(arg1 * 2.**537, arg2 * 2.**537)
            t = (x * 2.**537) * 2.**537
            if t > s1 or (t == s1 and s2 < 0.):
                return pred(x)
    return x

def rddiv(a, b, rmode=roundmode.nearest):
    arg1 = a
    arg2 = b
    if rmode == roundmode.up:
        pass
        if (arg1 == 0. or arg2 == 0. or
            math.fabs(arg1) == float('inf') or math.fabs(arg2) == float('inf') or
                arg1 != arg1 or arg2 != arg2):
            return arg1 / arg2
        if arg2 < 0.:
            arg1fix = -arg1
            arg2fix = -arg2
        else:
            arg1fix = arg1
            arg2fix = arg2
        if abs(arg1fix) < 2.**(-969):
            if abs(arg2fix) < 2.**918:
                arg1fix *= 2.**105
                arg2fix *= 2.**105
            else:
                if arg1fix < 0.:
                    return 0.
                else:
                    return 2.**(-1074)
        d = arg1fix / arg2fix
        if d == float('inf'):
            return d
        elif d == -float('inf'):
            return -float_info.max

        x, y = twoproduct(d, arg2fix)
        if x < arg1fix or (x == arg1fix and y < 0.):
            return succ(d)
        return d
    elif rmode == roundmode.down:
        if (arg1 == 0. or arg2 == 0. or
            abs(arg1) == float('inf') or abs(arg2) == float('inf') or
                arg1 != arg1 or arg2 != arg2):
            return arg1 / arg2
        if arg2 < 0.:
            arg1fix = -arg1
            arg2fix = -arg2
        else:
            arg1fix = arg1
            arg2fix = arg2
        if abs(arg1fix) < 2.**(-969):
            if abs(arg2fix) < 2.**918:
                arg1fix *= 2.**105
                arg2fix *= 2.**105
            else:
                if arg1fix < 0.:
                    return -2.**(-1074)
                else:
                    return 0
        d = arg1fix / arg2fix
        if d == float('inf'):
            return float_info.max
        elif d == -float('inf'):
            return d
        x, y = twoproduct(d, arg2fix)
        if x > arg1fix or (x == arg1fix and y > 0.):
            return pred(d)
        return d

def rdsqrt(x, rmode=roundmode.nearest):
    arg = x
    d = math.sqrt(arg)
    if rmode == roundmode.up:
        if arg < 2.**(-969):
            a2 = arg * 2.**106
            d2 = d * 2.**53
            x, y = twoproduct(d2, d2)
            if x < a2 or (x == a2 and y < 0.):
                d = succ(d)
        x, y = twoproduct(d, d)
        if x < arg or (x == arg and y < 0.):
            d = succ(d)
    if rmode == roundmode.down:
        if arg < 2.**(-969):
            a2 = arg * 2.**106
            d2 = d * 2.**53
            x, y = twoproduct(d2, d2)
            if x > a2 or (x == a2 and y > 0.):
                d = pred(d)
        x, y = twoproduct(d, d)
        if x > arg or (x == arg and y > 0.):
            d = pred(d)
    return d
# -*- coding: utf-8 -*-

from sys import float_info
import math
from math import ldexp
from pint.roundmode import roundmode
'''
# Calculate addition, subtraction, multiplication, division and root with rounding mode.

# using exsample
from pint import roundfloat as rf
from pint import roundmode as rdm

# Calculate 0.25 + 0.1 with rounding to up.
rf.rdadd(0.25, 0.1, rdm.up)

# Calculate 0.25 - 0.1 with rounding to down
rf.rdsub(0.25, 0.1, rdm.down)
'''

'''
constants

L = ldexp
M = minus
P = plus
LM969 = ldexp(1, -969)
'''
L27P1 = ldexp(1, 27) + 1
L28 = ldexp(1, 28)
LM28 = ldexp(1, -28)
L53 = ldexp(1, 53)
LM53 = ldexp(1, -53)
L105 = ldexp(1, 105)
LM105 = ldexp(1, -105)
L106 = ldexp(1, 106)
L537 = ldexp(1, 537)
L918 = ldexp(1, 918)
L996 = ldexp(1, 996)
LM969 = ldexp(1, -969)
LM1021 = ldexp(1, -1021)
L1023 = ldexp(1, 1023)
LM1023 = ldexp(1, -1023)
LM1074 = ldexp(1, -1074)
LM53PLM105 = (ldexp(1, -53) + ldexp(1, -105))

def split(a):
    tmp = a * (L27P1)
    x = tmp - (tmp - a)
    y = a - x
    return x, y

# succ and pred by S. M. Rump:
def succ(a):
    abs_a = abs(a)
    if abs_a >= LM969:
        return a + abs_a * LM53PLM105
    if abs_a <= LM1021:
        return a + LM1074
    c = L53 * a
    e = LM53PLM105 * abs(c)
    return (c + e) * LM53

def pred(a):
    abs_a = abs(a)
    if abs_a >= LM969:
        return a - abs_a * LM53PLM105
    if abs_a <= LM1021:
        return a - LM1074
    c = L53 * a
    e = LM53PLM105 * abs(c)
    return (c - e) * LM53

# by Masahide Kashiwagi
# http://verifiedby.me/adiary/029

def fasttwosum(a, b):
    x = a + b
    tmp = x - a
    y = b - tmp
    return x, y

def twosum(a, b):
    arg1 = a
    arg2 = b
    x = arg1 + arg2
    if abs(arg1) > abs(arg2):
        tmp = x - arg1
        y = arg2 - tmp
    else:
        tmp = x - arg2
        y = arg1 - tmp
    return x, y

def twoproduct(a, b):
    arg1 = a
    arg2 = b
    x = arg1 * arg2
    if abs(arg1) > L996:
        arg1fix = arg1 * LM28
        arg2fix = arg2 * L28
    elif abs(arg2) > L996:
        arg1fix = arg1 * L28
        arg2fix = arg2 * LM28
    else:
        arg1fix = arg1
        arg2fix = arg2
    aH, aL = split(arg1fix)
    bH, bL = split(arg2fix)
    if abs(x) > L1023:
        y = aL * bL - ((((x * 0.5) - (aH * 0.5) * bH) * 2. - aL * bH) - aH * bL)
    else:
        y = aL * bL - (((x - aH * bH) - aL * bH) - aH * bL)
    return x, y

def rdadd(a, b, rmode=roundmode.nearest):
    arg1 = a
    arg2 = b
    x, y = twosum(arg1, arg2)
    if rmode == roundmode.up:
        if x == float('inf'):
            return x
        elif x == -float('inf'):
            if arg1 == -float('inf') or arg2 == -float('inf'):
                return x
            else:
                return -float_info.max
        if y > 0:
            x = succ(x)
    elif rmode == roundmode.down:
        if x == float('inf'):
            if arg1 == float('inf') or arg2 == float('inf'):
                return x
            else:
                return float_info.max
        elif x == -float('inf'):
            return x
        if y < 0:
            x = pred(x)
    return x

def rdsub(a, b, rmode=roundmode.nearest):
    return rdadd(a, -b, rmode)

def rdmul(a, b, rmode=roundmode.nearest):
    arg1 = a
    arg2 = b
    x, y = twoproduct(arg1, arg2)
    if rmode == roundmode.up:
        if x == float('inf'):
            return x
        elif x == -float('inf'):
            if math.fabs(arg1) == float('inf') or math.fabs(arg2) == float('inf'):
                return x
            else:
                return -float_info.max
        if abs(x) >= LM969:
            if y > 0:
                x = succ(x)
    elif rmode == roundmode.down:
        if x == float('inf'):
            if math.fabs(arg1) == float('inf') or math.fabs(arg2) == float('inf'):
                return x
            else:
                return float_info.max
        elif x == -float('inf'):
            return x

        if abs(x) >= LM969:
            if y < 0.:
                return pred(x)
        else:
            s1, s2 = twoproduct(arg1 * L537, arg2 * L537)
            t = (x * L537) * L537
            if t > s1 or (t == s1 and s2 < 0.):
                return pred(x)
    return x

def rddiv(a, b, rmode=roundmode.nearest):
    arg1 = a
    arg2 = b
    if rmode == roundmode.up:
        pass
        if (arg1 == 0. or arg2 == 0. or
            math.fabs(arg1) == float('inf') or math.fabs(arg2) == float('inf') or
                arg1 != arg1 or arg2 != arg2):
            return arg1 / arg2
        if arg2 < 0.:
            arg1fix = -arg1
            arg2fix = -arg2
        else:
            arg1fix = arg1
            arg2fix = arg2
        if abs(arg1fix) < LM969:
            if abs(arg2fix) < L918:
                arg1fix *= L105
                arg2fix *= L105
            else:
                if arg1fix < 0.:
                    return 0.
                else:
                    return LM1074
        d = arg1fix / arg2fix
        if d == float('inf'):
            return d
        elif d == -float('inf'):
            return -float_info.max

        x, y = twoproduct(d, arg2fix)
        if x < arg1fix or (x == arg1fix and y < 0.):
            return succ(d)
        return d
    elif rmode == roundmode.down:
        if (arg1 == 0. or arg2 == 0. or
            abs(arg1) == float('inf') or abs(arg2) == float('inf') or
                arg1 != arg1 or arg2 != arg2):
            return arg1 / arg2
        if arg2 < 0.:
            arg1fix = -arg1
            arg2fix = -arg2
        else:
            arg1fix = arg1
            arg2fix = arg2
        if abs(arg1fix) < LM969:
            if abs(arg2fix) < L918:
                arg1fix *= L105
                arg2fix *= L105
            else:
                if arg1fix < 0.:
                    return -LM1074
                else:
                    return 0
        d = arg1fix / arg2fix
        if d == float('inf'):
            return float_info.max
        elif d == -float('inf'):
            return d
        x, y = twoproduct(d, arg2fix)
        if x > arg1fix or (x == arg1fix and y > 0.):
            return pred(d)
        return d

def rdsqrt(x, rmode=roundmode.nearest):
    arg = x
    d = math.sqrt(arg)
    if rmode == roundmode.up:
        if arg < LM969:
            a2 = arg * L106
            d2 = d * L53
            x, y = twoproduct(d2, d2)
            if x < a2 or (x == a2 and y < 0.):
                d = succ(d)
        x, y = twoproduct(d, d)
        if x < arg or (x == arg and y < 0.):
            d = succ(d)
    if rmode == roundmode.down:
        if arg < LM969:
            a2 = arg * L106
            d2 = d * L53
            x, y = twoproduct(d2, d2)
            if x > a2 or (x == a2 and y > 0.):
                d = pred(d)
        x, y = twoproduct(d, d)
        if x > arg or (x == arg and y > 0.):
            d = pred(d)
    return d
<commit_msg>Fix conflict<commit_after># -*- coding: utf-8 -*-

from sys import float_info
import math
from math import ldexp
from pint.roundmode import roundmode
'''
# Calculate addition, subtraction, multiplication, division and root with rounding mode.

# using exsample
from pint import roundfloat as rf
from pint import roundmode as rdm

# Calculate 0.25 + 0.1 with rounding to up.
rf.rdadd(0.25, 0.1, rdm.up)

# Calculate 0.25 - 0.1 with rounding to down
rf.rdsub(0.25, 0.1, rdm.down)
'''

'''
constants

L = ldexp
M = minus
P = plus
LM969 = ldexp(1, -969)
'''
L27P1 = ldexp(1, 27) + 1
L28 = ldexp(1, 28)
LM28 = ldexp(1, -28)
L53 = ldexp(1, 53)
LM53 = ldexp(1, -53)
L105 = ldexp(1, 105)
LM105 = ldexp(1, -105)
L106 = ldexp(1, 106)
L537 = ldexp(1, 537)
L918 = ldexp(1, 918)
L996 = ldexp(1, 996)
LM969 = ldexp(1, -969)
LM1021 = ldexp(1, -1021)
L1023 = ldexp(1, 1023)
LM1023 = ldexp(1, -1023)
LM1074 = ldexp(1, -1074)
LM53PLM105 = (ldexp(1, -53) + ldexp(1, -105))

def split(a):
    tmp = a * (L27P1)
    x = tmp - (tmp - a)
    y = a - x
    return x, y

# succ and pred by S. M. Rump:
def succ(a):
    abs_a = abs(a)
    if abs_a >= LM969:
        return a + abs_a * LM53PLM105
    if abs_a <= LM1021:
        return a + LM1074
    c = L53 * a
    e = LM53PLM105 * abs(c)
    return (c + e) * LM53

def pred(a):
    abs_a = abs(a)
    if abs_a >= LM969:
        return a - abs_a * LM53PLM105
    if abs_a <= LM1021:
        return a - LM1074
    c = L53 * a
    e = LM53PLM105 * abs(c)
    return (c - e) * LM53

# by Masahide Kashiwagi
# http://verifiedby.me/adiary/029

def fasttwosum(a, b):
    x = a + b
    tmp = x - a
    y = b - tmp
    return x, y

def twosum(a, b):
    arg1 = a
    arg2 = b
    x = arg1 + arg2
    if abs(arg1) > abs(arg2):
        tmp = x - arg1
        y = arg2 - tmp
    else:
        tmp = x - arg2
        y = arg1 - tmp
    return x, y

def twoproduct(a, b):
    arg1 = a
    arg2 = b
    x = arg1 * arg2
    if abs(arg1) > L996:
        arg1fix = arg1 * LM28
        arg2fix = arg2 * L28
    elif abs(arg2) > L996:
        arg1fix = arg1 * L28
        arg2fix = arg2 * LM28
    else:
        arg1fix = arg1
        arg2fix = arg2
    aH, aL = split(arg1fix)
    bH, bL = split(arg2fix)
    if abs(x) > L1023:
        y = aL * bL - ((((x * 0.5) - (aH * 0.5) * bH) * 2. - aL * bH) - aH * bL)
    else:
        y = aL * bL - (((x - aH * bH) - aL * bH) - aH * bL)
    return x, y

def rdadd(a, b, rmode=roundmode.nearest):
    arg1 = a
    arg2 = b
    x, y = twosum(arg1, arg2)
    if rmode == roundmode.up:
        if x == float('inf'):
            return x
        elif x == -float('inf'):
            if arg1 == -float('inf') or arg2 == -float('inf'):
                return x
            else:
                return -float_info.max
        if y > 0:
            x = succ(x)
    elif rmode == roundmode.down:
        if x == float('inf'):
            if arg1 == float('inf') or arg2 == float('inf'):
                return x
            else:
                return float_info.max
        elif x == -float('inf'):
            return x
        if y < 0:
            x = pred(x)
    return x

def rdsub(a, b, rmode=roundmode.nearest):
    return rdadd(a, -b, rmode)

def rdmul(a, b, rmode=roundmode.nearest):
    arg1 = a
    arg2 = b
    x, y = twoproduct(arg1, arg2)
    if rmode == roundmode.up:
        if x == float('inf'):
            return x
        elif x == -float('inf'):
            if math.fabs(arg1) == float('inf') or math.fabs(arg2) == float('inf'):
                return x
            else:
                return -float_info.max
        if abs(x) >= LM969:
            if y > 0:
                x = succ(x)
    elif rmode == roundmode.down:
        if x == float('inf'):
            if math.fabs(arg1) == float('inf') or math.fabs(arg2) == float('inf'):
                return x
            else:
                return float_info.max
        elif x == -float('inf'):
            return x

        if abs(x) >= LM969:
            if y < 0.:
                return pred(x)
        else:
            s1, s2 = twoproduct(arg1 * L537, arg2 * L537)
            t = (x * L537) * L537
            if t > s1 or (t == s1 and s2 < 0.):
                return pred(x)
    return x

def rddiv(a, b, rmode=roundmode.nearest):
    arg1 = a
    arg2 = b
    if rmode == roundmode.up:
        pass
        if (arg1 == 0. or arg2 == 0. or
            math.fabs(arg1) == float('inf') or math.fabs(arg2) == float('inf') or
                arg1 != arg1 or arg2 != arg2):
            return arg1 / arg2
        if arg2 < 0.:
            arg1fix = -arg1
            arg2fix = -arg2
        else:
            arg1fix = arg1
            arg2fix = arg2
        if abs(arg1fix) < LM969:
            if abs(arg2fix) < L918:
                arg1fix *= L105
                arg2fix *= L105
            else:
                if arg1fix < 0.:
                    return 0.
                else:
                    return LM1074
        d = arg1fix / arg2fix
        if d == float('inf'):
            return d
        elif d == -float('inf'):
            return -float_info.max

        x, y = twoproduct(d, arg2fix)
        if x < arg1fix or (x == arg1fix and y < 0.):
            return succ(d)
        return d
    elif rmode == roundmode.down:
        if (arg1 == 0. or arg2 == 0. or
            abs(arg1) == float('inf') or abs(arg2) == float('inf') or
                arg1 != arg1 or arg2 != arg2):
            return arg1 / arg2
        if arg2 < 0.:
            arg1fix = -arg1
            arg2fix = -arg2
        else:
            arg1fix = arg1
            arg2fix = arg2
        if abs(arg1fix) < LM969:
            if abs(arg2fix) < L918:
                arg1fix *= L105
                arg2fix *= L105
            else:
                if arg1fix < 0.:
                    return -LM1074
                else:
                    return 0
        d = arg1fix / arg2fix
        if d == float('inf'):
            return float_info.max
        elif d == -float('inf'):
            return d
        x, y = twoproduct(d, arg2fix)
        if x > arg1fix or (x == arg1fix and y > 0.):
            return pred(d)
        return d

def rdsqrt(x, rmode=roundmode.nearest):
    arg = x
    d = math.sqrt(arg)
    if rmode == roundmode.up:
        if arg < LM969:
            a2 = arg * L106
            d2 = d * L53
            x, y = twoproduct(d2, d2)
            if x < a2 or (x == a2 and y < 0.):
                d = succ(d)
        x, y = twoproduct(d, d)
        if x < arg or (x == arg and y < 0.):
            d = succ(d)
    if rmode == roundmode.down:
        if arg < LM969:
            a2 = arg * L106
            d2 = d * L53
            x, y = twoproduct(d2, d2)
            if x > a2 or (x == a2 and y > 0.):
                d = pred(d)
        x, y = twoproduct(d, d)
        if x > arg or (x == arg and y > 0.):
            d = pred(d)
    return d
<|endoftext|>