<commit_before>"""Provide variant calling with VarScan from TGI at Wash U.

http://varscan.sourceforge.net/
"""

from collections import namedtuple
import contextlib
import os
import sys

from bcbio import broad, utils
from bcbio.distributed.transaction import file_transaction, tx_tmpdir
from bcbio.pipeline import config_utils
from bcbio.provenance import do
from bcbio.variation import samtools, vcfutils
from bcbio.variation.vcfutils import (combine_variant_files, write_empty_vcf,
                                      get_paired_bams, bgzip_and_index)

import pysam

def run_varscan(align_bams, items, ref_file, assoc_files,
                region=None, out_file=None):
    paired = get_paired_bams(align_bams, items)
    if paired and paired.normal_bam and paired.tumor_bam:
        call_file = samtools.shared_variantcall(_varscan_paired, "varscan",
                                                align_bams, ref_file, items,
                                                assoc_files, region, out_file)
    else:
        vcfutils.check_paired_problems(items)
        call_file = samtools.shared_variantcall(_varscan_work, "varscan",
                                                align_bams, ref_file,
                                                items, assoc_files,
                                                region, out_file)
    return call_file

def _get_varscan_opts(config, tmp_dir):
    """Retrieve common options for running VarScan.
    Handles jvm_opts, setting user and country to English to avoid issues
    with different locales producing non-compliant VCF.
    """
    resources = config_utils.get_resources("varscan", config)
    jvm_opts = resources.get("jvm_opts", ["-Xmx750m", "-Xmx2g"])
    jvm_opts = config_utils.adjust_opts(jvm_opts,
                                        {"algorithm": {"memory_adjust":
                                                       {"magnitude": 1.1, "direction": "decrease"}}})
    jvm_opts += ["-Duser.language=en", "-Duser.country=US"]
    jvm_opts += broad.get_default_jvm_opts(tmp_dir)
    return " ".join(jvm_opts)

def _safe_to_float(x):
    if x is None:
        return None
    else:
        try:
            return float(x)
        except ValueError:
            return None

def spv_freq_filter(line, tumor_index):
    """Filter VarScan calls based on the SPV value and frequency.

    Removes calls with SPV < 0.05 and a tumor FREQ > 0.35.

    False positives dominate these higher frequency, low SPV calls. They appear
    to be primarily non-somatic/germline variants not removed by other filters.
    """
    if line.startswith("#CHROM"):
        headers = [('##FILTER=<ID=SpvFreq,Description="High frequency (tumor FREQ > 0.35) '
                    'and low p-value for somatic (SPV < 0.05)">')]
        return "\n".join(headers) + "\n" + line
    elif line.startswith("#"):
        return line
    else:
        parts = line.split("\t")
        sample_ft = {a: v for (a, v) in zip(parts[8].split(":"), parts[9 + tumor_index].split(":"))}
        freq = _safe_to_float(sample_ft.get("FREQ"))
        spvs = [x for x in parts[7].split(";") if x.startswith("SPV=")]
        spv = _safe_to_float(spvs[0].split("=")[-1] if spvs else None)
        fname = None
        if spv is not None and freq is not None:
            if spv < 0.05 and freq > 0.35:
                fname = "SpvFreq"
        if fname:
            if parts[6] in set([".", "PASS"]):
                parts[6] = fname
            else:
                parts[6] += ";%s" % fname
        line = "\t".join(parts)
        return line

def _varscan_paired(align_bams, ref_file, items, target_regions, out_file):

    """Run a paired VarScan analysis, also known as "somatic". """

    max_read_depth = "1000"
    config = items[0]["config"]
    paired = get_paired_bams(align_bams, items)
    if not paired.normal_bam:
        affected_batch = items[0]["metadata"]["batch"]
        message = ("Batch {} requires both tumor and normal BAM files for"
                   " VarScan cancer calling").format(affected_batch)
        raise ValueError(message)

    if not utils.file_exists(out_file):
        assert out_file.endswith(".vcf.gz"), "Expect bgzipped output to VarScan"
        normal_mpileup_cl = samtools.prep_mpileup([paired.normal_bam], ref_file,
                                                  config, max_read_depth,
                                                  target_regions=target_regions,
                                                  want_bcf=False)
        tumor_mpileup_cl = samtools.prep_mpileup([paired.tumor_bam], ref_file,
                                                 config, max_read_depth,
                                                 target_regions=target_regions,
                                                 want_bcf=False)
        base, ext = utils.splitext_plus(out_file)
        indel_file = base + "-indel.vcf"
        snp_file = base + "-snp.vcf"
        with file_transaction(config, indel_file, snp_file) as (tx_indel, tx_snp):
            with tx_tmpdir(items[0]) as tmp_dir:
                jvm_opts = _get_varscan_opts(config, tmp_dir)
                remove_zerocoverage = r"grep -v -P '\t0\t\t$'"
                varscan_cmd = ("varscan {jvm_opts} somatic "
                               " <({normal_mpileup_cl} | {remove_zerocoverage}) "
                               "<({tumor_mpileup_cl} | {remove_zerocoverage}) "
                               "--output-snp {tx_snp} --output-indel {tx_indel} "
                               " --output-vcf --min-coverage 5 --p-value 0.98 "
                               "--strand-filter 1 ")
                # add minimum AF
                if "--min-var-freq" not in varscan_cmd:
                    min_af = float(utils.get_in(paired.tumor_config, ("algorithm",
                                                                      "min_allele_fraction"), 10)) / 100.0
                    varscan_cmd += "--min-var-freq {min_af} "
                do.run(varscan_cmd.format(**locals()), "Varscan", None, None)

        to_combine = []
        for fname in [snp_file, indel_file]:
            if utils.file_exists(fname):
                fix_file = "%s-fix.vcf.gz" % (utils.splitext_plus(fname)[0])
                with file_transaction(config, fix_file) as tx_fix_file:
                    fix_ambig_ref = vcfutils.fix_ambiguous_cl()
                    fix_ambig_alt = vcfutils.fix_ambiguous_cl(5)
                    py_cl = os.path.join(os.path.dirname(sys.executable), "py")
                    normal_name = paired.normal_name
                    tumor_name = paired.tumor_name
                    cmd = ("cat {fname} | "
                           "{py_cl} -x 'bcbio.variation.varscan.fix_varscan_output(x,"
                            """ "{normal_name}", "{tumor_name}")' | """
                           "{fix_ambig_ref} | {fix_ambig_alt} | ifne vcfuniqalleles | "
                           """bcftools filter -m + -s REJECT -e "SS != '.' && SS != '2'" 2> /dev/null | """
                           "{py_cl} -x 'bcbio.variation.varscan.spv_freq_filter(x, 1)' | "
                           "bgzip -c > {tx_fix_file}")
                    do.run(cmd.format(**locals()), "Varscan paired fix")
                to_combine.append(fix_file)

        if not to_combine:
            out_file = write_empty_vcf(out_file, config)
        else:
            out_file = combine_variant_files(to_combine,
                                             out_file, ref_file, config,
                                             region=target_regions)
        if os.path.getsize(out_file) == 0:
            write_empty_vcf(out_file)
        if out_file.endswith(".gz"):
            out_file = bgzip_and_index(out_file, config)

def fix_varscan_output(line, normal_name="", tumor_name=""):
    """Fix a varscan VCF line.

    Fixes the ALT column and also fixes floating point values
    output as strings to by Floats: FREQ, SSC.

    This function was contributed by Sean Davis <sdavis2@mail.nih.gov>,
    with minor modifications by Luca Beltrame <luca.beltrame@marionegri.it>.
    """
    line = line.strip()

    tofix = ("##INFO=<ID=SSC", "##FORMAT=<ID=FREQ")
    if(line.startswith("##")):
        if line.startswith(tofix):
            line = line.replace('Number=1,Type=String',
                                'Number=1,Type=Float')
        return line
    line = line.split("\t")

    if line[0].startswith("#CHROM"):
        if tumor_name and normal_name:
            mapping = {"NORMAL": normal_name, "TUMOR": tumor_name}
            base_header = line[:9]
            old_samples = line[9:]

            if len(old_samples) == 0:
                return "\t".join(line)

            samples = [mapping[sample_name] for sample_name in old_samples]

            assert len(old_samples) == len(samples)
            return "\t".join(base_header + samples)
        else:
            return "\t".join(line)

    try:
        REF, ALT = line[3:5]
    except ValueError:
        return "\t".join(line)

    if len(line) > 10:
        #print line
        Ifreq = line[8].split(":").index("FREQ")
        #print repr(Ifreq)
        ndat = line[9].split(":")
        tdat = line[10].split(":")
        #print ndat
        #print tdat
        somatic_status = line[7].split(";")  # SS=<number>
        # HACK: The position of the SS= changes, so we just search for it
        somatic_status = [item for item in somatic_status
                          if item.startswith("SS=")][0]
        somatic_status = int(somatic_status.split("=")[1])  # Get the number

        try:
            ndat[Ifreq] = str(float(ndat[Ifreq].rstrip("%")) / 100)
        except ValueError:  # illegal binary characters -- set frequency to zero
            ndat[Ifreq] = "0.0"
        try:
            tdat[Ifreq] = str(float(tdat[Ifreq].rstrip("%")) / 100)
        except ValueError:  # illegal binary characters -- set frequency to zero
            tdat[Ifreq] = "0.0"
        line[9] = ":".join(ndat)
        line[10] = ":".join(tdat)
        if somatic_status == 5:
            # "Unknown" states are broken in current versions of VarScan
            # so we just bail out here for now
            return

    #FIXME: VarScan also produces invalid REF records (e.g. CAA/A)
    # This is not handled yet.

    if "+" in ALT or "-" in ALT:
        if "/" not in ALT:
            if ALT[0] == "+":
                R = REF
                A = REF + ALT[1:]
            elif ALT[0] == "-":
                R = REF + ALT[1:]
                A = REF
        else:
            Ins = [p[1:] for p in ALT.split("/") if p[0] == "+"]
            Del = [p[1:] for p in ALT.split("/") if p[0] == "-"]

            if len(Del):
                REF += sorted(Del, key=lambda x: len(x))[-1]

            A = ",".join([REF[::-1].replace(p[::-1], "", 1)[::-1]
                          for p in Del] + [REF + p for p in Ins])
            R = REF

        REF = R
        ALT = A
    else:
        ALT = ALT.replace('/', ',')

    line[3] = REF
    line[4] = ALT
    return "\t".join(line)

def _create_sample_list(in_bams, vcf_file):
    """Pull sample names from input BAMs and create input sample list.
    """
    out_file = "%s-sample_list.txt" % os.path.splitext(vcf_file)[0]
    with open(out_file, "w") as out_handle:
        for in_bam in in_bams:
            with contextlib.closing(pysam.Samfile(in_bam, "rb")) as work_bam:
                for rg in work_bam.header.get("RG", []):
                    out_handle.write("%s\n" % rg["SM"])
    return out_file

def _varscan_work(align_bams, ref_file, items, target_regions, out_file):
    """Perform SNP and indel genotyping with VarScan.
    """
    config = items[0]["config"]

    orig_out_file = out_file
    out_file = orig_out_file.replace(".vcf.gz", ".vcf")

    max_read_depth = "1000"
    sample_list = _create_sample_list(align_bams, out_file)
    mpileup = samtools.prep_mpileup(align_bams, ref_file, config, max_read_depth,
                                    target_regions=target_regions, want_bcf=False)
    # VarScan fails to generate a header on files that start with
    # zerocoverage calls; strip these with grep, we're not going to
    # call on them
    remove_zerocoverage = r"grep -v -P '\t0\t\t$'"
    # we use ifne from moreutils to ensure we process only on files with input, skipping otherwise
    # http://manpages.ubuntu.com/manpages/natty/man1/ifne.1.html
    with tx_tmpdir(items[0]) as tmp_dir:
        jvm_opts = _get_varscan_opts(config, tmp_dir)
        fix_ambig_ref = vcfutils.fix_ambiguous_cl()
        fix_ambig_alt = vcfutils.fix_ambiguous_cl(5)
        py_cl = os.path.join(os.path.dirname(sys.executable), "py")
        cmd = ("{mpileup} | {remove_zerocoverage} | "
                "ifne varscan {jvm_opts} mpileup2cns --min-coverage 5 --p-value 0.98 "
                "  --vcf-sample-list {sample_list} --output-vcf --variants | "
               "{py_cl} -x 'bcbio.variation.varscan.fix_varscan_output(x)' | "
                "{fix_ambig_ref} | {fix_ambig_alt} | ifne vcfuniqalleles > {out_file}")
        do.run(cmd.format(**locals()), "Varscan", None,
                [do.file_exists(out_file)])
    os.remove(sample_list)
    # VarScan can create completely empty files in regions without
    # variants, so we create a correctly formatted empty file
    if os.path.getsize(out_file) == 0:
        write_empty_vcf(out_file)

    if orig_out_file.endswith(".gz"):
        vcfutils.bgzip_and_index(out_file, config)
<commit_msg>VarScan: avoid failures with empty mpileup<commit_after>"""Provide variant calling with VarScan from TGI at Wash U.

http://varscan.sourceforge.net/
"""

import contextlib
import os
import sys

from bcbio import broad, utils
from bcbio.distributed.transaction import file_transaction, tx_tmpdir
from bcbio.pipeline import config_utils
from bcbio.provenance import do
from bcbio.variation import samtools, vcfutils
from bcbio.variation.vcfutils import (combine_variant_files, write_empty_vcf,
                                      get_paired_bams, bgzip_and_index)

import pysam

def run_varscan(align_bams, items, ref_file, assoc_files,
                region=None, out_file=None):
    paired = get_paired_bams(align_bams, items)
    if paired and paired.normal_bam and paired.tumor_bam:
        call_file = samtools.shared_variantcall(_varscan_paired, "varscan",
                                                align_bams, ref_file, items,
                                                assoc_files, region, out_file)
    else:
        vcfutils.check_paired_problems(items)
        call_file = samtools.shared_variantcall(_varscan_work, "varscan",
                                                align_bams, ref_file,
                                                items, assoc_files,
                                                region, out_file)
    return call_file

def _get_varscan_opts(config, tmp_dir):
    """Retrieve common options for running VarScan.
    Handles jvm_opts, setting user and country to English to avoid issues
    with different locales producing non-compliant VCF.
    """
    resources = config_utils.get_resources("varscan", config)
    jvm_opts = resources.get("jvm_opts", ["-Xmx750m", "-Xmx2g"])
    jvm_opts = config_utils.adjust_opts(jvm_opts,
                                        {"algorithm": {"memory_adjust":
                                                       {"magnitude": 1.1, "direction": "decrease"}}})
    jvm_opts += ["-Duser.language=en", "-Duser.country=US"]
    jvm_opts += broad.get_default_jvm_opts(tmp_dir)
    return " ".join(jvm_opts)

def _safe_to_float(x):
    if x is None:
        return None
    else:
        try:
            return float(x)
        except ValueError:
            return None

def spv_freq_filter(line, tumor_index):
    """Filter VarScan calls based on the SPV value and frequency.

    Removes calls with SPV < 0.05 and a tumor FREQ > 0.35.

    False positives dominate these higher frequency, low SPV calls. They appear
    to be primarily non-somatic/germline variants not removed by other filters.
    """
    if line.startswith("#CHROM"):
        headers = [('##FILTER=<ID=SpvFreq,Description="High frequency (tumor FREQ > 0.35) '
                    'and low p-value for somatic (SPV < 0.05)">')]
        return "\n".join(headers) + "\n" + line
    elif line.startswith("#"):
        return line
    else:
        parts = line.split("\t")
        sample_ft = {a: v for (a, v) in zip(parts[8].split(":"), parts[9 + tumor_index].split(":"))}
        freq = _safe_to_float(sample_ft.get("FREQ"))
        spvs = [x for x in parts[7].split(";") if x.startswith("SPV=")]
        spv = _safe_to_float(spvs[0].split("=")[-1] if spvs else None)
        fname = None
        if spv is not None and freq is not None:
            if spv < 0.05 and freq > 0.35:
                fname = "SpvFreq"
        if fname:
            if parts[6] in set([".", "PASS"]):
                parts[6] = fname
            else:
                parts[6] += ";%s" % fname
        line = "\t".join(parts)
        return line

def _varscan_paired(align_bams, ref_file, items, target_regions, out_file):

    """Run a paired VarScan analysis, also known as "somatic". """

    max_read_depth = "1000"
    config = items[0]["config"]
    paired = get_paired_bams(align_bams, items)
    if not paired.normal_bam:
        affected_batch = items[0]["metadata"]["batch"]
        message = ("Batch {} requires both tumor and normal BAM files for"
                   " VarScan cancer calling").format(affected_batch)
        raise ValueError(message)

    if not utils.file_exists(out_file):
        assert out_file.endswith(".vcf.gz"), "Expect bgzipped output to VarScan"
        normal_mpileup_cl = samtools.prep_mpileup([paired.normal_bam], ref_file,
                                                  config, max_read_depth,
                                                  target_regions=target_regions,
                                                  want_bcf=False)
        tumor_mpileup_cl = samtools.prep_mpileup([paired.tumor_bam], ref_file,
                                                 config, max_read_depth,
                                                 target_regions=target_regions,
                                                 want_bcf=False)
        base, ext = utils.splitext_plus(out_file)
        indel_file = base + "-indel.vcf"
        snp_file = base + "-snp.vcf"
        with file_transaction(config, indel_file, snp_file) as (tx_indel, tx_snp):
            with tx_tmpdir(items[0]) as tmp_dir:
                jvm_opts = _get_varscan_opts(config, tmp_dir)
                remove_zerocoverage = r"ifne grep -v -P '\t0\t\t$'"
                varscan_cmd = ("varscan {jvm_opts} somatic "
                               " <({normal_mpileup_cl} | {remove_zerocoverage}) "
                               "<({tumor_mpileup_cl} | {remove_zerocoverage}) "
                               "--output-snp {tx_snp} --output-indel {tx_indel} "
                               " --output-vcf --min-coverage 5 --p-value 0.98 "
                               "--strand-filter 1 ")
                # add minimum AF
                if "--min-var-freq" not in varscan_cmd:
                    min_af = float(utils.get_in(paired.tumor_config, ("algorithm",
                                                                      "min_allele_fraction"), 10)) / 100.0
                    varscan_cmd += "--min-var-freq {min_af} "
                do.run(varscan_cmd.format(**locals()), "Varscan", None, None)

        to_combine = []
        for fname in [snp_file, indel_file]:
            if utils.file_exists(fname):
                fix_file = "%s-fix.vcf.gz" % (utils.splitext_plus(fname)[0])
                with file_transaction(config, fix_file) as tx_fix_file:
                    fix_ambig_ref = vcfutils.fix_ambiguous_cl()
                    fix_ambig_alt = vcfutils.fix_ambiguous_cl(5)
                    py_cl = os.path.join(os.path.dirname(sys.executable), "py")
                    normal_name = paired.normal_name
                    tumor_name = paired.tumor_name
                    cmd = ("cat {fname} | "
                           "{py_cl} -x 'bcbio.variation.varscan.fix_varscan_output(x,"
                            """ "{normal_name}", "{tumor_name}")' | """
                           "{fix_ambig_ref} | {fix_ambig_alt} | ifne vcfuniqalleles | "
                           """bcftools filter -m + -s REJECT -e "SS != '.' && SS != '2'" 2> /dev/null | """
                           "{py_cl} -x 'bcbio.variation.varscan.spv_freq_filter(x, 1)' | "
                           "bgzip -c > {tx_fix_file}")
                    do.run(cmd.format(**locals()), "Varscan paired fix")
                to_combine.append(fix_file)

        if not to_combine:
            out_file = write_empty_vcf(out_file, config)
        else:
            out_file = combine_variant_files(to_combine,
                                             out_file, ref_file, config,
                                             region=target_regions)
        if os.path.getsize(out_file) == 0:
            write_empty_vcf(out_file)
        if out_file.endswith(".gz"):
            out_file = bgzip_and_index(out_file, config)

def fix_varscan_output(line, normal_name="", tumor_name=""):
    """Fix a varscan VCF line.

    Fixes the ALT column and also fixes floating point values
    output as strings to by Floats: FREQ, SSC.

    This function was contributed by Sean Davis <sdavis2@mail.nih.gov>,
    with minor modifications by Luca Beltrame <luca.beltrame@marionegri.it>.
    """
    line = line.strip()

    tofix = ("##INFO=<ID=SSC", "##FORMAT=<ID=FREQ")
    if(line.startswith("##")):
        if line.startswith(tofix):
            line = line.replace('Number=1,Type=String',
                                'Number=1,Type=Float')
        return line
    line = line.split("\t")

    if line[0].startswith("#CHROM"):
        if tumor_name and normal_name:
            mapping = {"NORMAL": normal_name, "TUMOR": tumor_name}
            base_header = line[:9]
            old_samples = line[9:]

            if len(old_samples) == 0:
                return "\t".join(line)

            samples = [mapping[sample_name] for sample_name in old_samples]

            assert len(old_samples) == len(samples)
            return "\t".join(base_header + samples)
        else:
            return "\t".join(line)

    try:
        REF, ALT = line[3:5]
    except ValueError:
        return "\t".join(line)

    if len(line) > 10:
        #print line
        Ifreq = line[8].split(":").index("FREQ")
        #print repr(Ifreq)
        ndat = line[9].split(":")
        tdat = line[10].split(":")
        #print ndat
        #print tdat
        somatic_status = line[7].split(";")  # SS=<number>
        # HACK: The position of the SS= changes, so we just search for it
        somatic_status = [item for item in somatic_status
                          if item.startswith("SS=")][0]
        somatic_status = int(somatic_status.split("=")[1])  # Get the number

        try:
            ndat[Ifreq] = str(float(ndat[Ifreq].rstrip("%")) / 100)
        except ValueError:  # illegal binary characters -- set frequency to zero
            ndat[Ifreq] = "0.0"
        try:
            tdat[Ifreq] = str(float(tdat[Ifreq].rstrip("%")) / 100)
        except ValueError:  # illegal binary characters -- set frequency to zero
            tdat[Ifreq] = "0.0"
        line[9] = ":".join(ndat)
        line[10] = ":".join(tdat)
        if somatic_status == 5:
            # "Unknown" states are broken in current versions of VarScan
            # so we just bail out here for now
            return

    #FIXME: VarScan also produces invalid REF records (e.g. CAA/A)
    # This is not handled yet.

    if "+" in ALT or "-" in ALT:
        if "/" not in ALT:
            if ALT[0] == "+":
                R = REF
                A = REF + ALT[1:]
            elif ALT[0] == "-":
                R = REF + ALT[1:]
                A = REF
        else:
            Ins = [p[1:] for p in ALT.split("/") if p[0] == "+"]
            Del = [p[1:] for p in ALT.split("/") if p[0] == "-"]

            if len(Del):
                REF += sorted(Del, key=lambda x: len(x))[-1]

            A = ",".join([REF[::-1].replace(p[::-1], "", 1)[::-1]
                          for p in Del] + [REF + p for p in Ins])
            R = REF

        REF = R
        ALT = A
    else:
        ALT = ALT.replace('/', ',')

    line[3] = REF
    line[4] = ALT
    return "\t".join(line)

def _create_sample_list(in_bams, vcf_file):
    """Pull sample names from input BAMs and create input sample list.
    """
    out_file = "%s-sample_list.txt" % os.path.splitext(vcf_file)[0]
    with open(out_file, "w") as out_handle:
        for in_bam in in_bams:
            with contextlib.closing(pysam.Samfile(in_bam, "rb")) as work_bam:
                for rg in work_bam.header.get("RG", []):
                    out_handle.write("%s\n" % rg["SM"])
    return out_file

def _varscan_work(align_bams, ref_file, items, target_regions, out_file):
    """Perform SNP and indel genotyping with VarScan.
    """
    config = items[0]["config"]

    orig_out_file = out_file
    out_file = orig_out_file.replace(".vcf.gz", ".vcf")

    max_read_depth = "1000"
    sample_list = _create_sample_list(align_bams, out_file)
    mpileup = samtools.prep_mpileup(align_bams, ref_file, config, max_read_depth,
                                    target_regions=target_regions, want_bcf=False)
    # VarScan fails to generate a header on files that start with
    # zerocoverage calls; strip these with grep, we're not going to
    # call on them
    remove_zerocoverage = r"ifne grep -v -P '\t0\t\t$'"
    # we use ifne from moreutils to ensure we process only on files with input, skipping otherwise
    # http://manpages.ubuntu.com/manpages/natty/man1/ifne.1.html
    with tx_tmpdir(items[0]) as tmp_dir:
        jvm_opts = _get_varscan_opts(config, tmp_dir)
        fix_ambig_ref = vcfutils.fix_ambiguous_cl()
        fix_ambig_alt = vcfutils.fix_ambiguous_cl(5)
        py_cl = os.path.join(os.path.dirname(sys.executable), "py")
        cmd = ("{mpileup} | {remove_zerocoverage} | "
                "ifne varscan {jvm_opts} mpileup2cns --min-coverage 5 --p-value 0.98 "
                "  --vcf-sample-list {sample_list} --output-vcf --variants | "
               "{py_cl} -x 'bcbio.variation.varscan.fix_varscan_output(x)' | "
                "{fix_ambig_ref} | {fix_ambig_alt} | ifne vcfuniqalleles > {out_file}")
        do.run(cmd.format(**locals()), "Varscan", None,
                [do.file_exists(out_file)])
    os.remove(sample_list)
    # VarScan can create completely empty files in regions without
    # variants, so we create a correctly formatted empty file
    if os.path.getsize(out_file) == 0:
        write_empty_vcf(out_file)

    if orig_out_file.endswith(".gz"):
        vcfutils.bgzip_and_index(out_file, config)
<|endoftext|>


<commit_before>"""ACME protocol messages."""
import M2Crypto
import zope.interface

from letsencrypt.acme import errors
from letsencrypt.acme import interfaces
from letsencrypt.acme import jose
from letsencrypt.acme import other
from letsencrypt.acme import util

class Message(util.JSONDeSerializable, util.ImmutableMap):
    """ACME message.

    Messages are considered immutable.

    """
    zope.interface.implements(interfaces.IJSONSerializable)

    acme_type = NotImplemented
    """ACME message "type" field. Subclasses must override."""

    TYPES = {}
    """Message types registered for JSON deserialization"""

    @classmethod
    def register(cls, msg_cls):
        """Register class for JSON deserialization."""
        cls.TYPES[msg_cls.acme_type] = msg_cls
        return msg_cls

    def to_json(self):
        """Get JSON serializable object.

        :returns: Serializable JSON object representing ACME message.
            :meth:`validate` will almost certainly not work, due to reasons
            explained in :class:`letsencrypt.acme.interfaces.IJSONSerializable`.
        :rtype: dict

        """
        jobj = self._fields_to_json()
        jobj["type"] = self.acme_type
        return jobj

    def _fields_to_json(self):
        """Prepare ACME message fields for JSON serialiazation.

        Subclasses must override this method.

        :returns: Serializable JSON object containg all ACME message fields
            apart from "type".
        :rtype: dict

        """
        raise NotImplementedError()

    @classmethod
    def get_msg_cls(cls, jobj):
        """Get the registered class for ``jobj``."""
        if cls in cls.TYPES.itervalues():
            # cls is already registered Message type, force to use it
            # so that, e.g Revocation.from_json(jobj) fails if
            # jobj["type"] != "revocation".
            return cls

        if not isinstance(jobj, dict):
            raise errors.ValidationError(
                "{0} is not a dictionary object".format(jobj))
        try:
            msg_type = jobj["type"]
        except KeyError:
            raise errors.ValidationError("missing type field")

        try:
            msg_cls = cls.TYPES[msg_type]
        except KeyError:
            raise errors.UnrecognizedMessageTypeError(msg_type)

        return msg_cls

    @classmethod
    def from_json(cls, jobj, validate=True):
        """Deserialize validated ACME message from JSON string.

        :param str jobj: JSON object.
        :param bool validate: Validate against schema before deserializing.
            Useful if :class:`JWK` is part of already validated json object.

        :raises letsencrypt.acme.errors.ValidationError: if validation
            was unsuccessful

        :returns: Valid ACME message.
        :rtype: subclass of :class:`Message`

        """
        msg_cls = cls.get_msg_cls(jobj)
        if validate:
            msg_cls.validate_json(jobj)
        # pylint: disable=protected-access
        return msg_cls._from_valid_json(jobj)

@Message.register  # pylint: disable=too-few-public-methods
class Challenge(Message):
    """ACME "challenge" message."""
    acme_type = "challenge"
    schema = util.load_schema(acme_type)
    __slots__ = ("session_id", "nonce", "challenges", "combinations")

    def _fields_to_json(self):
        fields = {
            "sessionID": self.session_id,
            "nonce": jose.b64encode(self.nonce),
            "challenges": self.challenges,
        }
        if self.combinations:
            fields["combinations"] = self.combinations
        return fields

    @classmethod
    def _from_valid_json(cls, jobj):
        return cls(session_id=jobj["sessionID"],
                   nonce=jose.b64decode(jobj["nonce"]),
                   challenges=jobj["challenges"],
                   combinations=jobj.get("combinations", []))

@Message.register  # pylint: disable=too-few-public-methods
class ChallengeRequest(Message):
    """ACME "challengeRequest" message.

    :ivar str identifier: Domain name.

    """
    acme_type = "challengeRequest"
    schema = util.load_schema(acme_type)
    __slots__ = ("identifier",)

    def _fields_to_json(self):
        return {
            "identifier": self.identifier,
        }

    @classmethod
    def _from_valid_json(cls, jobj):
        return cls(identifier=jobj["identifier"])

@Message.register  # pylint: disable=too-few-public-methods
class Authorization(Message):
    """ACME "authorization" message."""
    acme_type = "authorization"
    schema = util.load_schema(acme_type)
    __slots__ = ("recovery_token", "identifier", "jwk")

    def _fields_to_json(self):
        fields = {}
        if self.recovery_token is not None:
            fields["recoveryToken"] = self.recovery_token
        if self.identifier is not None:
            fields["identifier"] = self.identifier
        if self.jwk is not None:
            fields["jwk"] = self.jwk
        return fields

    @classmethod
    def _from_valid_json(cls, jobj):
        jwk = jobj.get("jwk")
        if jwk is not None:
            jwk = jose.JWK.from_json(jwk, validate=False)
        return cls(recovery_token=jobj.get("recoveryToken"),
                   identifier=jobj.get("identifier"), jwk=jwk)

@Message.register
class AuthorizationRequest(Message):
    """ACME "authorizationRequest" message.

    :ivar str session_id: "sessionID" from the server challenge
    :ivar str nonce: Nonce from the server challenge
    :ivar list responses: List of completed challenges
    :ivar signature: Signature (:class:`letsencrypt.acme.other.Signature`).
    :ivar contact: TODO

    """
    acme_type = "authorizationRequest"
    schema = util.load_schema(acme_type)
    __slots__ = ("session_id", "nonce", "responses", "signature", "contact")

    @classmethod
    def create(cls, name, key, sig_nonce=None, **kwargs):
        """Create signed "authorizationRequest".

        :param str name: Hostname

        :param key: Key used for signing.
        :type key: :class:`Crypto.PublicKey.RSA`

        :param str sig_nonce: Nonce used for signature. Useful for testing.

        :returns: Signed "authorizationRequest" ACME message.
        :rtype: :class:`AuthorizationRequest`

        """
        # pylint: disable=too-many-arguments
        signature = other.Signature.from_msg(
            name + kwargs["nonce"], key, sig_nonce)
        return cls(
            signature=signature, contact=kwargs.pop("contact", []), **kwargs)

    def verify(self, name):
        """Verify signature.

        :param str name: Hostname

        :returns: True iff ``signature`` can be verified, False otherwise.
        :rtype: bool

        """
        # TODO: must also check that the public key encoded in the JWK object
        # is the correct key for a given context.
        return self.signature.verify(name + self.nonce)

    def _fields_to_json(self):
        fields = {
            "sessionID": self.session_id,
            "nonce": jose.b64encode(self.nonce),
            "responses": self.responses,
            "signature": self.signature,
        }
        if self.contact:
            fields["contact"] = self.contact
        return fields

    @classmethod
    def _from_valid_json(cls, jobj):
        return cls(session_id=jobj["sessionID"],
                   nonce=jose.b64decode(jobj["nonce"]),
                   responses=jobj["responses"],
                   signature=other.Signature.from_json(
                       jobj["signature"], validate=False),
                   contact=jobj.get("contact", []))

@Message.register  # pylint: disable=too-few-public-methods
class Certificate(Message):
    """ACME "certificate" message.

    :ivar certificate: The certificate (:class:`M2Crypto.X509.X509`
        wrapped in :class:`letsencrypt.acme.util.ComparableX509`).

    :ivar list chain: Chain of certificates (:class:`M2Crypto.X509.X509` wrapped
        in :class:`letsencrypt.acme.util.ComparableX509` ).

    """
    acme_type = "certificate"
    schema = util.load_schema(acme_type)
    __slots__ = ("certificate", "chain", "refresh")

    def _fields_to_json(self):
        fields = {"certificate": self._encode_cert(self.certificate)}
        if self.chain:
            fields["chain"] = [self._encode_cert(cert) for cert in self.chain]
        if self.refresh is not None:
            fields["refresh"] = self.refresh
        return fields

    @classmethod
    def _decode_cert(cls, b64der):
        return util.ComparableX509(M2Crypto.X509.load_cert_der_string(
            jose.b64decode(b64der)))

    @classmethod
    def _encode_cert(cls, cert):
        return jose.b64encode(cert.as_der())

    @classmethod
    def _from_valid_json(cls, jobj):
        return cls(certificate=cls._decode_cert(jobj["certificate"]),
                   chain=[cls._decode_cert(cert) for cert in
                          jobj.get("chain", [])],
                   refresh=jobj.get("refresh"))

@Message.register
class CertificateRequest(Message):
    """ACME "certificateRequest" message.

    :ivar csr: Certificate Signing Request (:class:`M2Crypto.X509.Request`
        wrapped in :class:`letsencrypt.acme.util.ComparableX509`.
    :ivar signature: Signature (:class:`letsencrypt.acme.other.Signature`).

    """
    acme_type = "certificateRequest"
    schema = util.load_schema(acme_type)
    __slots__ = ("csr", "signature")

    @classmethod
    def create(cls, key, sig_nonce=None, **kwargs):
        """Create signed "certificateRequest".

        :param key: Key used for signing.
        :type key: :class:`Crypto.PublicKey.RSA`

        :param str sig_nonce: Nonce used for signature. Useful for testing.

        :returns: Signed "certificateRequest" ACME message.
        :rtype: :class:`CertificateRequest`

        """
        return cls(signature=other.Signature.from_msg(
            kwargs["csr"].as_der(), key, sig_nonce), **kwargs)

    def verify(self):
        """Verify signature.

        :returns: True iff ``signature`` can be verified, False otherwise.
        :rtype: bool

        """
        # TODO: must also check that the public key encoded in the JWK object
        # is the correct key for a given context.
        return self.signature.verify(self.csr.as_der())

    @classmethod
    def _decode_csr(cls, b64der):
        return util.ComparableX509(M2Crypto.X509.load_request_der_string(
            jose.b64decode(b64der)))

    @classmethod
    def _encode_csr(cls, csr):
        return jose.b64encode(csr.as_der())

    def _fields_to_json(self):
        return {
            "csr": self._encode_csr(self.csr),
            "signature": self.signature,
        }

    @classmethod
    def _from_valid_json(cls, jobj):
        return cls(csr=cls._decode_csr(jobj["csr"]),
                   signature=other.Signature.from_json(
                       jobj["signature"], validate=False))

@Message.register  # pylint: disable=too-few-public-methods
class Defer(Message):
    """ACME "defer" message."""
    acme_type = "defer"
    schema = util.load_schema(acme_type)
    __slots__ = ("token", "interval", "message")

    def _fields_to_json(self):
        fields = {"token": self.token}
        if self.interval is not None:
            fields["interval"] = self.interval
        if self.message is not None:
            fields["message"] = self.message
        return fields

    @classmethod
    def _from_valid_json(cls, jobj):
        return cls(token=jobj["token"], interval=jobj.get("interval"),
                   message=jobj.get("message"))

@Message.register  # pylint: disable=too-few-public-methods
class Error(Message):
    """ACME "error" message."""
    acme_type = "error"
    schema = util.load_schema(acme_type)
    __slots__ = ("error", "message", "more_info")

    CODES = {
        "malformed": "The request message was malformed",
        "unauthorized": "The client lacks sufficient authorization",
        "serverInternal": "The server experienced an internal error",
        "notSupported": "The request type is not supported",
        "unknown": "The server does not recognize an ID/token in the request",
        "badCSR": "The CSR is unacceptable (e.g., due to a short key)",
    }

    def _fields_to_json(self):
        fields = {"error": self.error}
        if self.message is not None:
            fields["message"] = self.message
        if self.more_info is not None:
            fields["moreInfo"] = self.more_info
        return fields

    @classmethod
    def _from_valid_json(cls, jobj):
        return cls(error=jobj["error"], message=jobj.get("message"),
                   more_info=jobj.get("moreInfo"))

@Message.register  # pylint: disable=too-few-public-methods
class Revocation(Message):
    """ACME "revocation" message."""
    acme_type = "revocation"
    schema = util.load_schema(acme_type)
    __slots__ = ()

    def _fields_to_json(self):
        return {}

    @classmethod
    def _from_valid_json(cls, jobj):
        return cls()

@Message.register
class RevocationRequest(Message):
    """ACME "revocationRequest" message.

    :ivar certificate: Certificate (:class:`M2Crypto.X509.X509`
        wrapped in :class:`letsencrypt.acme.util.ComparableX509`).
    :ivar signature: Signature (:class:`letsencrypt.acme.other.Signature`).

    """
    acme_type = "revocationRequest"
    schema = util.load_schema(acme_type)
    __slots__ = ("certificate", "signature")

    @classmethod
    def create(cls, key, sig_nonce=None, **kwargs):
        """Create signed "revocationRequest".

        :param key: Key used for signing.
        :type key: :class:`Crypto.PublicKey.RSA`

        :param str sig_nonce: Nonce used for signature. Useful for testing.

        :returns: Signed "revocationRequest" ACME message.
        :rtype: :class:`RevocationRequest`

        """
        return cls(signature=other.Signature.from_msg(
            kwargs["certificate"].as_der(), key, sig_nonce), **kwargs)

    def verify(self):
        """Verify signature.

        :returns: True iff ``signature`` can be verified, False otherwise.
        :rtype: bool

        """
        # TODO: must also check that the public key encoded in the JWK object
        # is the correct key for a given context.
        return self.signature.verify(self.certificate.as_der())

    @classmethod
    def _decode_cert(cls, b64der):
        return util.ComparableX509(M2Crypto.X509.load_cert_der_string(
            jose.b64decode(b64der)))

    @classmethod
    def _encode_cert(cls, cert):
        return jose.b64encode(cert.as_der())

    def _fields_to_json(self):
        return {
            "certificate": self._encode_cert(self.certificate),
            "signature": self.signature,
        }

    @classmethod
    def _from_valid_json(cls, jobj):
        return cls(certificate=cls._decode_cert(jobj["certificate"]),
                   signature=other.Signature.from_json(
                       jobj["signature"], validate=False))

@Message.register  # pylint: disable=too-few-public-methods
class StatusRequest(Message):
    """ACME "statusRequest" message.

    :ivar unicode token: Token provided in ACME "defer" message.

    """
    acme_type = "statusRequest"
    schema = util.load_schema(acme_type)
    __slots__ = ("token",)

    def _fields_to_json(self):
        return {"token": self.token}

    @classmethod
    def _from_valid_json(cls, jobj):
        return cls(token=jobj["token"])
<commit_msg>acme.messages: explicit warnings about key verification<commit_after>"""ACME protocol messages."""
import M2Crypto
import zope.interface

from letsencrypt.acme import errors
from letsencrypt.acme import interfaces
from letsencrypt.acme import jose
from letsencrypt.acme import other
from letsencrypt.acme import util

class Message(util.JSONDeSerializable, util.ImmutableMap):
    """ACME message.

    Messages are considered immutable.

    """
    zope.interface.implements(interfaces.IJSONSerializable)

    acme_type = NotImplemented
    """ACME message "type" field. Subclasses must override."""

    TYPES = {}
    """Message types registered for JSON deserialization"""

    @classmethod
    def register(cls, msg_cls):
        """Register class for JSON deserialization."""
        cls.TYPES[msg_cls.acme_type] = msg_cls
        return msg_cls

    def to_json(self):
        """Get JSON serializable object.

        :returns: Serializable JSON object representing ACME message.
            :meth:`validate` will almost certainly not work, due to reasons
            explained in :class:`letsencrypt.acme.interfaces.IJSONSerializable`.
        :rtype: dict

        """
        jobj = self._fields_to_json()
        jobj["type"] = self.acme_type
        return jobj

    def _fields_to_json(self):
        """Prepare ACME message fields for JSON serialiazation.

        Subclasses must override this method.

        :returns: Serializable JSON object containg all ACME message fields
            apart from "type".
        :rtype: dict

        """
        raise NotImplementedError()

    @classmethod
    def get_msg_cls(cls, jobj):
        """Get the registered class for ``jobj``."""
        if cls in cls.TYPES.itervalues():
            # cls is already registered Message type, force to use it
            # so that, e.g Revocation.from_json(jobj) fails if
            # jobj["type"] != "revocation".
            return cls

        if not isinstance(jobj, dict):
            raise errors.ValidationError(
                "{0} is not a dictionary object".format(jobj))
        try:
            msg_type = jobj["type"]
        except KeyError:
            raise errors.ValidationError("missing type field")

        try:
            msg_cls = cls.TYPES[msg_type]
        except KeyError:
            raise errors.UnrecognizedMessageTypeError(msg_type)

        return msg_cls

    @classmethod
    def from_json(cls, jobj, validate=True):
        """Deserialize validated ACME message from JSON string.

        :param str jobj: JSON object.
        :param bool validate: Validate against schema before deserializing.
            Useful if :class:`JWK` is part of already validated json object.

        :raises letsencrypt.acme.errors.ValidationError: if validation
            was unsuccessful

        :returns: Valid ACME message.
        :rtype: subclass of :class:`Message`

        """
        msg_cls = cls.get_msg_cls(jobj)
        if validate:
            msg_cls.validate_json(jobj)
        # pylint: disable=protected-access
        return msg_cls._from_valid_json(jobj)

@Message.register  # pylint: disable=too-few-public-methods
class Challenge(Message):
    """ACME "challenge" message."""
    acme_type = "challenge"
    schema = util.load_schema(acme_type)
    __slots__ = ("session_id", "nonce", "challenges", "combinations")

    def _fields_to_json(self):
        fields = {
            "sessionID": self.session_id,
            "nonce": jose.b64encode(self.nonce),
            "challenges": self.challenges,
        }
        if self.combinations:
            fields["combinations"] = self.combinations
        return fields

    @classmethod
    def _from_valid_json(cls, jobj):
        return cls(session_id=jobj["sessionID"],
                   nonce=jose.b64decode(jobj["nonce"]),
                   challenges=jobj["challenges"],
                   combinations=jobj.get("combinations", []))

@Message.register  # pylint: disable=too-few-public-methods
class ChallengeRequest(Message):
    """ACME "challengeRequest" message.

    :ivar str identifier: Domain name.

    """
    acme_type = "challengeRequest"
    schema = util.load_schema(acme_type)
    __slots__ = ("identifier",)

    def _fields_to_json(self):
        return {
            "identifier": self.identifier,
        }

    @classmethod
    def _from_valid_json(cls, jobj):
        return cls(identifier=jobj["identifier"])

@Message.register  # pylint: disable=too-few-public-methods
class Authorization(Message):
    """ACME "authorization" message."""
    acme_type = "authorization"
    schema = util.load_schema(acme_type)
    __slots__ = ("recovery_token", "identifier", "jwk")

    def _fields_to_json(self):
        fields = {}
        if self.recovery_token is not None:
            fields["recoveryToken"] = self.recovery_token
        if self.identifier is not None:
            fields["identifier"] = self.identifier
        if self.jwk is not None:
            fields["jwk"] = self.jwk
        return fields

    @classmethod
    def _from_valid_json(cls, jobj):
        jwk = jobj.get("jwk")
        if jwk is not None:
            jwk = jose.JWK.from_json(jwk, validate=False)
        return cls(recovery_token=jobj.get("recoveryToken"),
                   identifier=jobj.get("identifier"), jwk=jwk)

@Message.register
class AuthorizationRequest(Message):
    """ACME "authorizationRequest" message.

    :ivar str session_id: "sessionID" from the server challenge
    :ivar str nonce: Nonce from the server challenge
    :ivar list responses: List of completed challenges
    :ivar signature: Signature (:class:`letsencrypt.acme.other.Signature`).
    :ivar contact: TODO

    """
    acme_type = "authorizationRequest"
    schema = util.load_schema(acme_type)
    __slots__ = ("session_id", "nonce", "responses", "signature", "contact")

    @classmethod
    def create(cls, name, key, sig_nonce=None, **kwargs):
        """Create signed "authorizationRequest".

        :param str name: Hostname

        :param key: Key used for signing.
        :type key: :class:`Crypto.PublicKey.RSA`

        :param str sig_nonce: Nonce used for signature. Useful for testing.

        :returns: Signed "authorizationRequest" ACME message.
        :rtype: :class:`AuthorizationRequest`

        """
        # pylint: disable=too-many-arguments
        signature = other.Signature.from_msg(
            name + kwargs["nonce"], key, sig_nonce)
        return cls(
            signature=signature, contact=kwargs.pop("contact", []), **kwargs)

    def verify(self, name):
        """Verify signature.

        .. warning:: Caller must check that the public key encoded in the
            :attr:`signature`'s :class:`letsencrypt.acme.jose.JWK` object
            is the correct key for a given context.

        :param str name: Hostname

        :returns: True iff ``signature`` can be verified, False otherwise.
        :rtype: bool

        """
        return self.signature.verify(name + self.nonce)

    def _fields_to_json(self):
        fields = {
            "sessionID": self.session_id,
            "nonce": jose.b64encode(self.nonce),
            "responses": self.responses,
            "signature": self.signature,
        }
        if self.contact:
            fields["contact"] = self.contact
        return fields

    @classmethod
    def _from_valid_json(cls, jobj):
        return cls(session_id=jobj["sessionID"],
                   nonce=jose.b64decode(jobj["nonce"]),
                   responses=jobj["responses"],
                   signature=other.Signature.from_json(
                       jobj["signature"], validate=False),
                   contact=jobj.get("contact", []))

@Message.register  # pylint: disable=too-few-public-methods
class Certificate(Message):
    """ACME "certificate" message.

    :ivar certificate: The certificate (:class:`M2Crypto.X509.X509`
        wrapped in :class:`letsencrypt.acme.util.ComparableX509`).

    :ivar list chain: Chain of certificates (:class:`M2Crypto.X509.X509` wrapped
        in :class:`letsencrypt.acme.util.ComparableX509` ).

    """
    acme_type = "certificate"
    schema = util.load_schema(acme_type)
    __slots__ = ("certificate", "chain", "refresh")

    def _fields_to_json(self):
        fields = {"certificate": self._encode_cert(self.certificate)}
        if self.chain:
            fields["chain"] = [self._encode_cert(cert) for cert in self.chain]
        if self.refresh is not None:
            fields["refresh"] = self.refresh
        return fields

    @classmethod
    def _decode_cert(cls, b64der):
        return util.ComparableX509(M2Crypto.X509.load_cert_der_string(
            jose.b64decode(b64der)))

    @classmethod
    def _encode_cert(cls, cert):
        return jose.b64encode(cert.as_der())

    @classmethod
    def _from_valid_json(cls, jobj):
        return cls(certificate=cls._decode_cert(jobj["certificate"]),
                   chain=[cls._decode_cert(cert) for cert in
                          jobj.get("chain", [])],
                   refresh=jobj.get("refresh"))

@Message.register
class CertificateRequest(Message):
    """ACME "certificateRequest" message.

    :ivar csr: Certificate Signing Request (:class:`M2Crypto.X509.Request`
        wrapped in :class:`letsencrypt.acme.util.ComparableX509`.
    :ivar signature: Signature (:class:`letsencrypt.acme.other.Signature`).

    """
    acme_type = "certificateRequest"
    schema = util.load_schema(acme_type)
    __slots__ = ("csr", "signature")

    @classmethod
    def create(cls, key, sig_nonce=None, **kwargs):
        """Create signed "certificateRequest".

        :param key: Key used for signing.
        :type key: :class:`Crypto.PublicKey.RSA`

        :param str sig_nonce: Nonce used for signature. Useful for testing.

        :returns: Signed "certificateRequest" ACME message.
        :rtype: :class:`CertificateRequest`

        """
        return cls(signature=other.Signature.from_msg(
            kwargs["csr"].as_der(), key, sig_nonce), **kwargs)

    def verify(self):
        """Verify signature.

        .. warning:: Caller must check that the public key encoded in the
            :attr:`signature`'s :class:`letsencrypt.acme.jose.JWK` object
            is the correct key for a given context.

        :returns: True iff ``signature`` can be verified, False otherwise.
        :rtype: bool

        """
        return self.signature.verify(self.csr.as_der())

    @classmethod
    def _decode_csr(cls, b64der):
        return util.ComparableX509(M2Crypto.X509.load_request_der_string(
            jose.b64decode(b64der)))

    @classmethod
    def _encode_csr(cls, csr):
        return jose.b64encode(csr.as_der())

    def _fields_to_json(self):
        return {
            "csr": self._encode_csr(self.csr),
            "signature": self.signature,
        }

    @classmethod
    def _from_valid_json(cls, jobj):
        return cls(csr=cls._decode_csr(jobj["csr"]),
                   signature=other.Signature.from_json(
                       jobj["signature"], validate=False))

@Message.register  # pylint: disable=too-few-public-methods
class Defer(Message):
    """ACME "defer" message."""
    acme_type = "defer"
    schema = util.load_schema(acme_type)
    __slots__ = ("token", "interval", "message")

    def _fields_to_json(self):
        fields = {"token": self.token}
        if self.interval is not None:
            fields["interval"] = self.interval
        if self.message is not None:
            fields["message"] = self.message
        return fields

    @classmethod
    def _from_valid_json(cls, jobj):
        return cls(token=jobj["token"], interval=jobj.get("interval"),
                   message=jobj.get("message"))

@Message.register  # pylint: disable=too-few-public-methods
class Error(Message):
    """ACME "error" message."""
    acme_type = "error"
    schema = util.load_schema(acme_type)
    __slots__ = ("error", "message", "more_info")

    CODES = {
        "malformed": "The request message was malformed",
        "unauthorized": "The client lacks sufficient authorization",
        "serverInternal": "The server experienced an internal error",
        "notSupported": "The request type is not supported",
        "unknown": "The server does not recognize an ID/token in the request",
        "badCSR": "The CSR is unacceptable (e.g., due to a short key)",
    }

    def _fields_to_json(self):
        fields = {"error": self.error}
        if self.message is not None:
            fields["message"] = self.message
        if self.more_info is not None:
            fields["moreInfo"] = self.more_info
        return fields

    @classmethod
    def _from_valid_json(cls, jobj):
        return cls(error=jobj["error"], message=jobj.get("message"),
                   more_info=jobj.get("moreInfo"))

@Message.register  # pylint: disable=too-few-public-methods
class Revocation(Message):
    """ACME "revocation" message."""
    acme_type = "revocation"
    schema = util.load_schema(acme_type)
    __slots__ = ()

    def _fields_to_json(self):
        return {}

    @classmethod
    def _from_valid_json(cls, jobj):
        return cls()

@Message.register
class RevocationRequest(Message):
    """ACME "revocationRequest" message.

    :ivar certificate: Certificate (:class:`M2Crypto.X509.X509`
        wrapped in :class:`letsencrypt.acme.util.ComparableX509`).
    :ivar signature: Signature (:class:`letsencrypt.acme.other.Signature`).

    """
    acme_type = "revocationRequest"
    schema = util.load_schema(acme_type)
    __slots__ = ("certificate", "signature")

    @classmethod
    def create(cls, key, sig_nonce=None, **kwargs):
        """Create signed "revocationRequest".

        :param key: Key used for signing.
        :type key: :class:`Crypto.PublicKey.RSA`

        :param str sig_nonce: Nonce used for signature. Useful for testing.

        :returns: Signed "revocationRequest" ACME message.
        :rtype: :class:`RevocationRequest`

        """
        return cls(signature=other.Signature.from_msg(
            kwargs["certificate"].as_der(), key, sig_nonce), **kwargs)

    def verify(self):
        """Verify signature.

        .. warning:: Caller must check that the public key encoded in the
            :attr:`signature`'s :class:`letsencrypt.acme.jose.JWK` object
            is the correct key for a given context.

        :returns: True iff ``signature`` can be verified, False otherwise.
        :rtype: bool

        """
        return self.signature.verify(self.certificate.as_der())

    @classmethod
    def _decode_cert(cls, b64der):
        return util.ComparableX509(M2Crypto.X509.load_cert_der_string(
            jose.b64decode(b64der)))

    @classmethod
    def _encode_cert(cls, cert):
        return jose.b64encode(cert.as_der())

    def _fields_to_json(self):
        return {
            "certificate": self._encode_cert(self.certificate),
            "signature": self.signature,
        }

    @classmethod
    def _from_valid_json(cls, jobj):
        return cls(certificate=cls._decode_cert(jobj["certificate"]),
                   signature=other.Signature.from_json(
                       jobj["signature"], validate=False))

@Message.register  # pylint: disable=too-few-public-methods
class StatusRequest(Message):
    """ACME "statusRequest" message.

    :ivar unicode token: Token provided in ACME "defer" message.

    """
    acme_type = "statusRequest"
    schema = util.load_schema(acme_type)
    __slots__ = ("token",)

    def _fields_to_json(self):
        return {"token": self.token}

    @classmethod
    def _from_valid_json(cls, jobj):
        return cls(token=jobj["token"])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Various node-related utilities."""
from django.apps import apps
from modularodm import Q

from website import settings

from keen import KeenClient

# Alias the project serializer

def serialize_node(*args, **kwargs):
    from website.project.views.node import _view_project
    return _view_project(*args, **kwargs)  # Not recommended practice

CONTENT_NODE_QUERY = (
    # Can encompass accessible projects, registrations, or forks
    # Note: is_bookmark collection(s) are implicitly assumed to also be collections; that flag intentionally omitted
    Q('is_deleted', 'eq', False) &
    Q('type', 'ne', 'osf.collection')
)

PROJECT_QUERY = CONTENT_NODE_QUERY

def recent_public_registrations(n=10):
    Registration = apps.get_model('osf.Registration')
    registrations = Registration.find(
        CONTENT_NODE_QUERY &
        Q('is_public', 'eq', True)
    ).get_roots().sort(
        '-registered_date'
    )
    for reg in registrations:
        if not n:
            break
        if reg.is_retracted or reg.is_pending_embargo:
            # Filter based on calculated properties
            continue
        n -= 1
        yield reg

def get_keen_activity():
    client = KeenClient(
        project_id=settings.KEEN['public']['project_id'],
        read_key=settings.KEEN['public']['read_key'],
    )

    node_pageviews = client.count(
        event_collection='pageviews',
        timeframe='this_7_days',
        group_by='node.id',
        filters=[
            {
                'property_name': 'node.id',
                'operator': 'exists',
                'property_value': True
            }
        ]
    )

    node_visits = client.count_unique(
        event_collection='pageviews',
        target_property='anon.id',
        timeframe='this_7_days',
        group_by='node.id',
        filters=[
            {
                'property_name': 'node.id',
                'operator': 'exists',
                'property_value': True
            }
        ]
    )

    return {'node_pageviews': node_pageviews, 'node_visits': node_visits}

def activity():
    """Generate analytics for most popular public projects and registrations.
    Called by `scripts/update_populate_projects_and_registrations`
    """
    Node = apps.get_model('osf.AbstractNode')
    popular_public_projects = []
    popular_public_registrations = []
    max_projects_to_display = settings.MAX_POPULAR_PROJECTS

    if settings.KEEN['public']['read_key']:
        keen_activity = get_keen_activity()
        node_visits = keen_activity['node_visits']

        node_data = [{'node': x['node.id'], 'views': x['result']} for x in node_visits]
        node_data.sort(key=lambda datum: datum['views'], reverse=True)

        node_data = [node_dict['node'] for node_dict in node_data]

        for nid in node_data:
            node = Node.load(nid)
            if node is None:
                continue
            if node.is_public and not node.is_registration and not node.is_deleted:
                if len(popular_public_projects) < max_projects_to_display:
                    popular_public_projects.append(node)
            elif node.is_public and node.is_registration and not node.is_deleted and not node.is_retracted:
                if len(popular_public_registrations) < max_projects_to_display:
                    popular_public_registrations.append(node)
            if len(popular_public_projects) >= max_projects_to_display and len(popular_public_registrations) >= max_projects_to_display:
                break

    # New and Noteworthy projects are updated manually
    new_and_noteworthy_projects = list(Node.find_one(Q('_id', 'eq', settings.NEW_AND_NOTEWORTHY_LINKS_NODE)).nodes_pointer)

    return {
        'new_and_noteworthy_projects': new_and_noteworthy_projects,
        'recent_public_registrations': recent_public_registrations(),
        'popular_public_projects': popular_public_projects,
        'popular_public_registrations': popular_public_registrations
    }
<commit_msg>s/python/sql/g makes finding registrations faster.<commit_after># -*- coding: utf-8 -*-
"""Various node-related utilities."""
from django.apps import apps
from modularodm import Q

from website import settings

from keen import KeenClient

# Alias the project serializer

def serialize_node(*args, **kwargs):
    from website.project.views.node import _view_project
    return _view_project(*args, **kwargs)  # Not recommended practice

CONTENT_NODE_QUERY = (
    # Can encompass accessible projects, registrations, or forks
    # Note: is_bookmark collection(s) are implicitly assumed to also be collections; that flag intentionally omitted
    Q('is_deleted', 'eq', False) &
    Q('type', 'ne', 'osf.collection')
)

PROJECT_QUERY = CONTENT_NODE_QUERY

def recent_public_registrations(n=10):
    from django.db.models import Q as DQ
    Registration = apps.get_model('osf.Registration')

    return Registration.objects.filter(
            is_public=True,
            is_deleted=False,
        ).filter(
            DQ(DQ(embargo__isnull=True) | ~DQ(embargo__state='unapproved')) &
            DQ(DQ(retraction__isnull=True) | ~DQ(retraction__state='approved'))
        ).get_roots().order_by('-registered_date').limit(n)

def get_keen_activity():
    client = KeenClient(
        project_id=settings.KEEN['public']['project_id'],
        read_key=settings.KEEN['public']['read_key'],
    )

    node_pageviews = client.count(
        event_collection='pageviews',
        timeframe='this_7_days',
        group_by='node.id',
        filters=[
            {
                'property_name': 'node.id',
                'operator': 'exists',
                'property_value': True
            }
        ]
    )

    node_visits = client.count_unique(
        event_collection='pageviews',
        target_property='anon.id',
        timeframe='this_7_days',
        group_by='node.id',
        filters=[
            {
                'property_name': 'node.id',
                'operator': 'exists',
                'property_value': True
            }
        ]
    )

    return {'node_pageviews': node_pageviews, 'node_visits': node_visits}

def activity():
    """Generate analytics for most popular public projects and registrations.
    Called by `scripts/update_populate_projects_and_registrations`
    """
    Node = apps.get_model('osf.AbstractNode')
    popular_public_projects = []
    popular_public_registrations = []
    max_projects_to_display = settings.MAX_POPULAR_PROJECTS

    if settings.KEEN['public']['read_key']:
        keen_activity = get_keen_activity()
        node_visits = keen_activity['node_visits']

        node_data = [{'node': x['node.id'], 'views': x['result']} for x in node_visits]
        node_data.sort(key=lambda datum: datum['views'], reverse=True)

        node_data = [node_dict['node'] for node_dict in node_data]

        for nid in node_data:
            node = Node.load(nid)
            if node is None:
                continue
            if node.is_public and not node.is_registration and not node.is_deleted:
                if len(popular_public_projects) < max_projects_to_display:
                    popular_public_projects.append(node)
            elif node.is_public and node.is_registration and not node.is_deleted and not node.is_retracted:
                if len(popular_public_registrations) < max_projects_to_display:
                    popular_public_registrations.append(node)
            if len(popular_public_projects) >= max_projects_to_display and len(popular_public_registrations) >= max_projects_to_display:
                break

    # New and Noteworthy projects are updated manually
    new_and_noteworthy_projects = list(Node.find_one(Q('_id', 'eq', settings.NEW_AND_NOTEWORTHY_LINKS_NODE)).nodes_pointer)

    return {
        'new_and_noteworthy_projects': new_and_noteworthy_projects,
        'recent_public_registrations': recent_public_registrations(),
        'popular_public_projects': popular_public_projects,
        'popular_public_registrations': popular_public_registrations
    }
<|endoftext|>


<commit_before>import glob
import os
import uuid
#os.chdir("/mydir")
nums=[]
files=[]
for file in glob.glob("*.pp"):
    nums.append(int(file[0:-3]))
nums.sort()
for num in nums:
	files.append(str(num)+".pp")
print files
pname=str(uuid.uuid4())+".ppm"
picture = open(pname , 'w+')
picture.close()
picture = open(pname, 'a')
picture.write("P3 \n")
size=str(len(files))
picture.write(size+" "+size+" 1 \n")
for file in files:
	part = open(file, "r")
	picture.write(part.read())
	part.close()
	#os.remove(file) #remove when debugging

picture.close()

<commit_msg>Update collector.py<commit_after>import glob
import os
import uuid
#os.chdir("/mydir")
nums=[]
files=[]
for file in glob.glob("*.pp"):
    nums.append(int(file[0:-3]))
nums.sort()
for num in nums:
	files.append(str(num)+".pp")
print files
pname=str(uuid.uuid4())+".ppm"
picture = open(pname , 'w+')
picture.close()
picture = open(pname, 'a')
picture.write("P3 \n")
size=str(len(files)-1)
picture.write(size+" "+size+" 1 \n")
for file in files:
	part = open(file, "r")
	picture.write(part.read())
	part.close()
	#os.remove(file) #remove when debugging

picture.close()
<|endoftext|>


<commit_before>from nose.tools import assert_raises
from mock import ANY, DEFAULT, MagicMock, mock_open, patch
from bddbot import Dealer
from bddbot.dealer import BotError
from bddbot.dealer import FEATURE_BANK_FILENAME, FEATURES_DIRECTORY, OUTPUT_FEATURES_FILENAME

class BaseDealerTest(object):
    def _mock_dealer_functions(self, content = ""):
        self.mocked_open = mock_open(read_data = content)
        self.mocked_mkdir = MagicMock()
        patcher = patch.multiple("bddbot.dealer",
            open = self.mocked_open,
            mkdir = self.mocked_mkdir)

        patcher.start()

        return patcher

    def _load_dealer(self):
        dealer = Dealer()
        dealer.load()

        # Assert actions during load().
        self.mocked_open.assert_called_once_with(FEATURE_BANK_FILENAME, "rb")
        self.mocked_open().read.assert_called_once_with()
        self.mocked_mkdir.assert_not_called()

        # Reset mocks.
        self.mocked_open.reset_mock()
        self.mocked_mkdir.reset_mock()

        return dealer

class TestLoading(BaseDealerTest):
    CONTENT = "Feature: Some awesome stuff"

    def setup(self):
        self._mock_dealer_functions(content = self.CONTENT)

    def teardown(self):
        patch.stopall()

        self.mocked_mkdir.assert_not_called()

    def test_no_features_bank_file(self):
        self.mocked_open.side_effect = IOError()

        dealer = Dealer()
        with assert_raises(BotError):
            dealer.load()

        self.mocked_open.assert_called_once_with(FEATURE_BANK_FILENAME, "rb")

    def test_successful_call(self):
        dealer = Dealer()
        dealer.load()

        self.mocked_open.assert_called_once_with(FEATURE_BANK_FILENAME, "rb")

    def test_call_load_twice(self):
        dealer = Dealer()
        dealer.load()
        dealer.load()

        self.mocked_open.assert_called_once_with(FEATURE_BANK_FILENAME, "rb")

class TestOutput(BaseDealerTest):
    CONTENT = "Feature: Stuff"

    def setup(self):
        self._mock_dealer_functions(content = self.CONTENT)
        self.dealer = self._load_dealer()

    def teardown(self):
        patch.stopall()

        self.mocked_open.assert_any_call(OUTPUT_FEATURES_FILENAME, "wb")
        self.mocked_mkdir.assert_called_once_with(FEATURES_DIRECTORY)

    def test_features_directory_already_exists(self):
        self.mocked_mkdir.side_effect = OSError()

        self.dealer.deal()

        # If directory already exist, we should proceed as usual.
        self.mocked_open().write.assert_any_call(self.CONTENT)

    def test_cant_open_features_file_for_writing(self):
        self.mocked_open.side_effect = [IOError(), DEFAULT, ]

        with assert_raises(BotError):
            self.dealer.deal()

        # Couldn't open file for writing, so obviously no writes were perfomed.
        self.mocked_open().write.assert_not_called()

    def test_cant_write_to_feature_file(self):
        self.mocked_open().write.side_effect = IOError()

        with assert_raises(BotError):
            self.dealer.deal()

        # First call to write() raised an IOError which was caught and translated.
        self.mocked_open().write.assert_called_once_with(ANY)
<commit_msg>Capture stdout in unit tests<commit_after>from cStringIO import StringIO
from nose.tools import assert_raises, assert_equal
from mock import ANY, DEFAULT, MagicMock, mock_open, patch
from bddbot import Dealer
from bddbot.dealer import BotError
from bddbot.dealer import FEATURE_BANK_FILENAME, FEATURES_DIRECTORY, OUTPUT_FEATURES_FILENAME

class BaseDealerTest(object):
    @property
    def stdout(self):
        try:
            return self.__stdout.getvalue()
        except AttributeError:
            return None

    def _capture_stdout(self):
        self.__stdout = StringIO()
        patcher = patch("sys.stdout", self.__stdout)
        patcher.start()

        return patcher

    def _mock_dealer_functions(self, content = ""):
        self.mocked_open = mock_open(read_data = content)
        self.mocked_mkdir = MagicMock()
        patcher = patch.multiple("bddbot.dealer",
            open = self.mocked_open,
            mkdir = self.mocked_mkdir)

        patcher.start()

        return patcher

    def _load_dealer(self):
        dealer = Dealer()
        dealer.load()

        # Assert actions during load().
        self.mocked_open.assert_called_once_with(FEATURE_BANK_FILENAME, "rb")
        self.mocked_open().read.assert_called_once_with()
        self.mocked_mkdir.assert_not_called()

        # Reset mocks.
        self.mocked_open.reset_mock()
        self.mocked_mkdir.reset_mock()

        return dealer

class TestLoading(BaseDealerTest):
    CONTENT = "Feature: Some awesome stuff"

    def setup(self):
        self._capture_stdout()
        self._mock_dealer_functions(content = self.CONTENT)

    def teardown(self):
        patch.stopall()

        self.mocked_mkdir.assert_not_called()
        assert_equal("", self.stdout)

    def test_no_features_bank_file(self):
        self.mocked_open.side_effect = IOError()

        dealer = Dealer()
        with assert_raises(BotError):
            dealer.load()

        self.mocked_open.assert_called_once_with(FEATURE_BANK_FILENAME, "rb")

    def test_successful_call(self):
        dealer = Dealer()
        dealer.load()

        self.mocked_open.assert_called_once_with(FEATURE_BANK_FILENAME, "rb")

    def test_call_load_twice(self):
        dealer = Dealer()
        dealer.load()
        dealer.load()

        self.mocked_open.assert_called_once_with(FEATURE_BANK_FILENAME, "rb")

class TestOutput(BaseDealerTest):
    CONTENT = "Feature: Stuff"

    def setup(self):
        self._capture_stdout()
        self._mock_dealer_functions(content = self.CONTENT)
        self.dealer = self._load_dealer()

    def teardown(self):
        patch.stopall()

        self.mocked_open.assert_any_call(OUTPUT_FEATURES_FILENAME, "wb")
        self.mocked_mkdir.assert_called_once_with(FEATURES_DIRECTORY)
        assert_equal("", self.stdout)

    def test_features_directory_already_exists(self):
        self.mocked_mkdir.side_effect = OSError()

        self.dealer.deal()

        # If directory already exist, we should proceed as usual.
        self.mocked_open().write.assert_any_call(self.CONTENT)

    def test_cant_open_features_file_for_writing(self):
        self.mocked_open.side_effect = [IOError(), DEFAULT, ]

        with assert_raises(BotError):
            self.dealer.deal()

        # Couldn't open file for writing, so obviously no writes were perfomed.
        self.mocked_open().write.assert_not_called()

    def test_cant_write_to_feature_file(self):
        self.mocked_open().write.side_effect = IOError()

        with assert_raises(BotError):
            self.dealer.deal()

        # First call to write() raised an IOError which was caught and translated.
        self.mocked_open().write.assert_called_once_with(ANY)
<|endoftext|>


<commit_before>
import sys
import simplejson

f = open('tweets.txt', 'r')

followerCounts = {}
favoriteCounts = {}
statusCounts = {}
friendCounts = {}
atmentionCounts = {}

for line in f :
	tweet = simplejson.loads(line)
	retweetCount = 0
	language = 'es'
	if 'retweetCount' and 'twitter_lang' in tweet :
		language = tweet['twitter_lang']
		retweetCount = tweet['retweetCount']

	if language == 'en' :
		for mention in tweet['twitter_entities']['user_mentions'] :
			mid = mention['id']
			if mid in atmentionCounts :
				atmentionCounts[mid] = atmentionCounts[mid] + 1
			else :
				atmentionCounts[mid] = 1

		tweeter = tweet['actor']['id'][len('id:twitter.com:') :]
		followerCounts[tweeter] = tweet['actor']['followersCount']
		statusCounts[tweeter] = tweet['actor']['statusesCount']
		friendCounts[tweeter] = tweet['actor']['friendsCount']
		favoriteCounts[tweeter] = tweet['actor']['favoritesCount']

		if retweetCount > 0 :
			tweeter = tweet['object']['actor']['id'][len('id:twitter.com:') :]
			followerCounts[tweeter] = tweet['object']['actor']['followersCount']
			statusCounts[tweeter] = tweet['object']['actor']['statusesCount']
			friendCounts[tweeter] = tweet['object']['actor']['friendsCount']
			favoriteCounts[tweeter] = tweet['object']['actor']['favoritesCount']

for k, v in followerCounts.items() :
	stat = str(k) + ', folCount, ' + str(v)
	if k in friendCounts :
		stat = stat + ', frenCount, ' + str(friendCounts[k])
	if k in statusCounts :
		stat = stat + ', stCount, ' + str(statusCounts[k])
	if k in favoriteCounts :
		stat = stat + ', favCount, ' + str(favoriteCounts[k])
	if k in atmentionCounts :
		stat = stat + ', @menCount, ' + str(atmentionCounts[k])

	print stat
<commit_msg>fixed bug related to atmentions<commit_after>
import sys
import simplejson

f = open('tweets.txt', 'r')

followerCounts = {}
favoriteCounts = {}
statusCounts = {}
friendCounts = {}
atmentionCounts = {}

for line in f :
	tweet = simplejson.loads(line)
	retweetCount = 0
	language = 'es'
	if 'retweetCount' and 'twitter_lang' in tweet :
		language = tweet['twitter_lang']
		retweetCount = tweet['retweetCount']

	if language == 'en' :
		for mention in tweet['twitter_entities']['user_mentions'] :
			mid = mention['id_str']
			if mid in atmentionCounts :
				atmentionCounts[mid] = atmentionCounts[mid] + 1
			else :
				atmentionCounts[mid] = 1

		tweeter = tweet['actor']['id'][len('id:twitter.com:') :]
		followerCounts[tweeter] = tweet['actor']['followersCount']
		statusCounts[tweeter] = tweet['actor']['statusesCount']
		friendCounts[tweeter] = tweet['actor']['friendsCount']
		favoriteCounts[tweeter] = tweet['actor']['favoritesCount']

		if retweetCount > 0 :
			tweeter = tweet['object']['actor']['id'][len('id:twitter.com:') :]
			followerCounts[tweeter] = tweet['object']['actor']['followersCount']
			statusCounts[tweeter] = tweet['object']['actor']['statusesCount']
			friendCounts[tweeter] = tweet['object']['actor']['friendsCount']
			favoriteCounts[tweeter] = tweet['object']['actor']['favoritesCount']

for k, v in followerCounts.items() :
	stat = str(k) + ', folCount, ' + str(v)
	if k in friendCounts :
		stat = stat + ', frenCount, ' + str(friendCounts[k])
	if k in statusCounts :
		stat = stat + ', stCount, ' + str(statusCounts[k])
	if k in favoriteCounts :
		stat = stat + ', favCount, ' + str(favoriteCounts[k])
	if k in atmentionCounts :
		stat = stat + ', @menCount, ' + str(atmentionCounts[k])

	print stat
<|endoftext|>