<commit_before>#
# ------------------------------------------------------------
# Copyright (c) All rights reserved
# SiLab, Institute of Physics, University of Bonn
# ------------------------------------------------------------
#

import logging

from basil.HL.RegisterHardwareLayer import HardwareLayer

logger = logging.getLogger(__name__)

class debyeflex3003(HardwareLayer):

    """Driver for the ISO-DEBYEFLEX 3003 X-Ray tube.
    A simple protocol via RS 232 serial port is used with 9600 baud rate.
    """

    def __init__(self, intf, conf):
        super(debyeflex3003, self).__init__(intf, conf)

    def init(self):
        super(debyeflex3003, self).init()

    def read(self):
        ret = self._intf.read()
        if ret[-1:] != "\r":
            logger.warning("debyeflex3003.read() termination error")
        return ret[1:-1]

    def write(self, cmd):
        self._intf.write(str(cmd))

    def get_nominal_current(self):
        """Read nominal current in mA.
        """
        self.write("CN")
        curr = self.read()
        return int(curr) / 1000.

    def get_actual_current(self):
        """Read actual current in mA.
        """
        self.write("CA")
        curr = self.read()
        return int(curr) / 1000.

    def set_current(self, curr):
        """Set current in mA
        """
        if curr > 80 or curr < 0:
            raise ValueError("Illegal value for tube current (0 - 80 mA)")
        else:
            self.write("SC:{:02d}".format(int(curr)))
            logger.info("Set tube current to {:.1f} mA".format(self.get_nominal_current()))

    def get_nominal_voltage(self):
        """Read nominal voltage in kV.
        """
        self.write("VN")
        vol = self.read()
        return int(vol) / 1000.

    def get_actual_voltage(self):
        """Read actual voltage in kV.
        """
        self.write("VA")
        vol = self.read()
        return int(vol) / 1000.

    def set_voltage(self, vol):
        """Set high voltage in kV
        """
        if vol > 60 or vol < 0:
            raise ValueError("Illegal value for tube voltage (0 - 60 kV)")
        else:
            self.write("SV:{:02d}".format(vol))
            logger.info("Set tube voltage to {:.1f} kV".format(self.get_nominal_voltage()))

    def set_highvoltage_on(self):
        self.write("HV:1")

    def set_highvoltage_off(self):
        self.write("HV:0")

    def open_shutter(self, shutter=1):
        """Open the shutter with given number. Only shuttter=1 supported from hardware currently
        """
        if not isinstance(shutter, int):
            raise TypeError("Invalid type for shutter number")
        elif shutter > 4 or shutter < 1:
            raise ValueError("Invalid value for shutter number")
        else:
            self.write("OS:{:1d}".format(shutter))
            logger.info("Opened shutter number {:1d}".format(shutter))

    def close_shutter(self, shutter=1):
        """Close the shutter with given number. Only shuttter=1 supported from hardware currently
        """
        if not isinstance(shutter, int):
            raise TypeError("Invalid type for shutter number")
        elif shutter > 4 or shutter < 1:
            raise ValueError("Invalid value for shutter number")
        else:
            self.write("CS:{:1d}".format(shutter))
            logger.info("Closed shutter number {:1d}".format(shutter))

    def activate_timer(self, shutter=1):
        """Activate the timer for a given shutter number
        """
        if not isinstance(shutter, int):
            raise TypeError("Invalid type for shutter number")
        elif shutter > 4 or shutter < 1:
            raise ValueError("Invalid value for shutter number")
        else:
            self.write("TS:{:1d}".format(shutter))
            logger.info("Started timer number {:1d}".format(shutter))

    def deactivate_timer(self, shutter=1):
        """Deactivate the timer for a given shutter number
        """
        if not isinstance(shutter, int):
            raise TypeError("Invalid type for shutter number")
        elif shutter > 4 or shutter < 1:
            raise ValueError("Invalid value for shutter number")
        else:
            self.write("TE:{:1d}".format(shutter))
            logger.info("Stopped timer number {:1d}".format(shutter))

    def set_timer(self, timer=1, dur=3600):
        """Set the timer with the given number (corresponds to shutter number) to the given duration (in s)
        """
        if not isinstance(timer, int):
            raise TypeError("Invalid type for timer number")
        elif not isinstance(dur, int):
            raise TypeError("Illegal type for duration")
        elif timer > 4 or timer < 1:
            raise ValueError("Invalid value for timer number")
        else:
            h = dur // 3600
            m = (dur % 3600) // 60
            s = (dur % 3600) % 60
            self.write("TP:{:1d},{:02d},{:02d},{:02d}".format(timer, h, m, s))
            time = self.get_nominal_time(timer)
            logger.info("Set timer number {:1d} to {:02d}:{:02d}:{:02d} (HH:MM:SS)".format(
                timer, time // 3600, (time % 3600) // 60, (time % 3600) % 60)
            )

    def get_actual_time(self, timer=1):
        """Get the actual time of the given timer in s
        """
        if not isinstance(timer, int):
            raise TypeError("Invalid type for timer number")
        elif timer > 4 or timer < 1:
            raise ValueError("Invalid value for timer number (1, 2, 3, 4)")
        else:
            self.write("TA:{:1d}".format(timer))
            time = self.read()
            return int(time)

    def get_nominal_time(self, timer=1):
        """Get the nominal time of the given timer in s
        """
        if not isinstance(timer, int):
            raise TypeError("Invalid type for timer number")
        elif timer > 4 or timer < 1:
            raise ValueError("Invalid value for timer number (1, 2, 3, 4)")
        else:
            self.write("TN:{:1d}".format(timer))
            time = self.read()
            return int(time)

    def lock_keyboard(self):
        """Locks the hardware keyboard on the device. Only STOP key still works.
        """
        self.write("KB:0")

    def unlock_keyboard(self):
        self.write("KB:1")

    def get_status(self, status_word):
        """ Get a pre-selected range of status parameters
        """
        self.write("SR:{:02d}".format(status_word))
        response = self.read()
        status = bin(int(response[7:10]))[2:].zfill(8)  # Convert response to 8 char long string of binary values
        logger.info("Status word {:02d}: {:8s}".format(status_word, status))
        return status
<commit_msg>FIX: X-ray Debyeflex 3003 driver<commit_after>#
# ------------------------------------------------------------
# Copyright (c) All rights reserved
# SiLab, Institute of Physics, University of Bonn
# ------------------------------------------------------------
#

import logging

from basil.HL.RegisterHardwareLayer import HardwareLayer

logger = logging.getLogger(__name__)

class debyeflex3003(HardwareLayer):

    """Driver for the ISO-DEBYEFLEX 3003 X-Ray tube.
    A simple protocol via RS 232 serial port is used with 9600 baud rate.
    """

    def __init__(self, intf, conf):
        super(debyeflex3003, self).__init__(intf, conf)

    def init(self):
        super(debyeflex3003, self).init()

    def read(self):
        ret = self._intf.read()
        if ret[-1:] != "\r":
            logger.warning("debyeflex3003.read() termination error")
        return ret[1:-1]

    def write(self, cmd):
        self._intf.write(str(cmd))

    def get_nominal_current(self):
        """Read nominal current in mA.
        """
        self.write("CN")
        curr = self.read()
        return int(curr) / 1000.

    def get_actual_current(self):
        """Read actual current in mA.
        """
        self.write("CA")
        curr = self.read()
        return int(curr) / 1000.

    def set_current(self, curr):
        """Set current in mA
        """
        if curr > 80 or curr < 0:
            raise ValueError("Illegal value for tube current (0 - 80 mA)")
        else:
            self.write("SC:{:02d}".format(int(curr)))
            logger.info("Set tube current to {:.1f} mA".format(self.get_nominal_current()))

    def get_nominal_voltage(self):
        """Read nominal voltage in kV.
        """
        self.write("VN")
        vol = self.read()
        return int(vol) / 1000.

    def get_actual_voltage(self):
        """Read actual voltage in kV.
        """
        self.write("VA")
        vol = self.read()
        return int(vol) / 1000.

    def set_voltage(self, vol):
        """Set high voltage in kV
        """
        if vol > 60 or vol < 0:
            raise ValueError("Illegal value for tube voltage (0 - 60 kV)")
        else:
            self.write("SV:{:02d}".format(vol))
            logger.info("Set tube voltage to {:.1f} kV".format(self.get_nominal_voltage()))

    def set_high_voltage_on(self):
        self.write("HV:1")

    def set_high_voltage_off(self):
        self.write("HV:0")

    def open_shutter(self, shutter=1):
        """Open the shutter with given number. Only shuttter=1 supported from hardware currently
        """
        if not isinstance(shutter, int):
            raise TypeError("Invalid type for shutter number")
        elif shutter > 4 or shutter < 1:
            raise ValueError("Invalid value for shutter number")
        else:
            self.write("OS:{:1d}".format(shutter))
            logger.info("Opened shutter number {:1d}".format(shutter))

    def close_shutter(self, shutter=1):
        """Close the shutter with given number. Only shuttter=1 supported from hardware currently
        """
        if not isinstance(shutter, int):
            raise TypeError("Invalid type for shutter number")
        elif shutter > 4 or shutter < 1:
            raise ValueError("Invalid value for shutter number")
        else:
            self.write("CS:{:1d}".format(shutter))
            logger.info("Closed shutter number {:1d}".format(shutter))

    def activate_timer(self, shutter=1):
        """Activate the timer for a given shutter number
        """
        if not isinstance(shutter, int):
            raise TypeError("Invalid type for shutter number")
        elif shutter > 4 or shutter < 1:
            raise ValueError("Invalid value for shutter number")
        else:
            self.write("TS:{:1d}".format(shutter))
            logger.info("Started timer number {:1d}".format(shutter))

    def deactivate_timer(self, shutter=1):
        """Deactivate the timer for a given shutter number
        """
        if not isinstance(shutter, int):
            raise TypeError("Invalid type for shutter number")
        elif shutter > 4 or shutter < 1:
            raise ValueError("Invalid value for shutter number")
        else:
            self.write("TE:{:1d}".format(shutter))
            logger.info("Stopped timer number {:1d}".format(shutter))

    def set_timer(self, timer=1, dur=3600):
        """Set the timer with the given number (corresponds to shutter number) to the given duration (in s)
        """
        if not isinstance(timer, int):
            raise TypeError("Invalid type for timer number")
        elif not isinstance(dur, int):
            raise TypeError("Illegal type for duration")
        elif timer > 4 or timer < 1:
            raise ValueError("Invalid value for timer number")
        else:
            h = dur // 3600
            m = (dur % 3600) // 60
            s = (dur % 3600) % 60
            self.write("TP:{:1d},{:02d},{:02d},{:02d}".format(timer, h, m, s))
            time = self.get_nominal_time(timer)
            logger.info("Set timer number {:1d} to {:02d}:{:02d}:{:02d} (HH:MM:SS)".format(
                timer, time // 3600, (time % 3600) // 60, (time % 3600) % 60)
            )

    def get_actual_time(self, timer=1):
        """Get the actual time of the given timer in s
        """
        if not isinstance(timer, int):
            raise TypeError("Invalid type for timer number")
        elif timer > 4 or timer < 1:
            raise ValueError("Invalid value for timer number (1, 2, 3, 4)")
        else:
            self.write("TA:{:1d}".format(timer))
            time = self.read()
            return int(time)

    def get_nominal_time(self, timer=1):
        """Get the nominal time of the given timer in s
        """
        if not isinstance(timer, int):
            raise TypeError("Invalid type for timer number")
        elif timer > 4 or timer < 1:
            raise ValueError("Invalid value for timer number (1, 2, 3, 4)")
        else:
            self.write("TN:{:1d}".format(timer))
            time = self.read()
            return int(time)

    def lock_keyboard(self):
        """Locks the hardware keyboard on the device. Only STOP key still works.
        """
        self.write("KB:0")

    def unlock_keyboard(self):
        self.write("KB:1")

    def get_status(self, status_word):
        """ Get a pre-selected range of status parameters
        """
        self.write("SR:{:02d}".format(status_word))
        response = self.read()
        status = bin(int(response[7:10]))[2:].zfill(8)  # Convert response to 8 char long string of binary values
        logger.info("Status word {:02d}: {:8s}".format(status_word, status))
        return status
<|endoftext|>


<commit_before>"""Perform regional de-novo assembly calling with cortex_var.

Using a pre-mapped set of reads and BED file of regions, performs de-novo
assembly and variant calling against the reference sequence in each region.
This avoids whole genome costs while gaining the advantage of de-novo
prediction.

http://cortexassembler.sourceforge.net/index_cortex_var.html
"""
import os
import glob
import subprocess
import itertools
from contextlib import closing

import pysam
from Bio import Seq
from Bio.SeqIO.QualityIO import FastqGeneralIterator

from bcbio import broad
from bcbio.distributed.transaction import file_transaction
from bcbio.pipeline.shared import subset_variant_regions
from bcbio.utils import file_exists, safe_makedir
from bcbio.variation.genotype import combine_variant_files, write_empty_vcf

def run_cortex(align_bam, ref_file, config, dbsnp=None, region=None,
               out_file=None):
    """Top level entry to regional de-novo based variant calling with cortex_var.
    """
    broad_runner = broad.runner_from_config(config)
    if out_file is None:
        out_file = "%s-cortex.vcf" % os.path.splitext(align_bam)[0]
    if not file_exists(out_file):
        broad_runner.run_fn("picard_index", align_bam)
        variant_regions = config["algorithm"].get("variant_regions", None)
        if not variant_regions:
            raise ValueError("Only regional variant calling with cortex_var is supported. Set variant_regions")
        target_regions = subset_variant_regions(variant_regions, region, out_file)
        with open(target_regions) as in_handle:
            regional_vcfs = [_run_cortex_on_region(x.strip().split("\t")[:3], align_bam,
                                                   ref_file, out_file, config)
                             for x in in_handle]
        combine_variant_files(regional_vcfs, out_file, ref_file, config)
    return out_file

def _run_cortex_on_region(region, align_bam, ref_file, out_file_base, config):
    """Run cortex on a specified chromosome start/end region.
    """
    kmers = [31]
    min_reads = 700
    cortex_dir = config["program"].get("cortex")
    stampy_dir = config["program"].get("stampy")
    vcftools_dir = config["program"].get("vcftools")
    if cortex_dir is None or stampy_dir is None:
        raise ValueError("cortex_var requires path to pre-built cortex and stampy")
    region_str = apply("{0}-{1}-{2}".format, region)
    base_dir = safe_makedir(os.path.join(os.path.dirname(out_file_base), region_str))
    out_vcf_base = os.path.join(base_dir, "{0}-{1}".format(
            os.path.splitext(os.path.basename(out_file_base))[0], region_str))
    out_file = "{0}.vcf".format(out_vcf_base)
    if not file_exists(out_file):
        fastq = _get_fastq_in_region(region, align_bam, out_vcf_base)
        if _count_fastq_reads(fastq, min_reads) < min_reads:
            write_empty_vcf(out_file)
        else:
            local_ref, genome_size = _get_local_ref(region, ref_file, out_vcf_base)
            indexes = _index_local_ref(local_ref, cortex_dir, stampy_dir, kmers)
            cortex_out = _run_cortex(fastq, indexes, {"kmers": kmers, "genome_size": genome_size,
                                                      "sample": _get_sample_name(align_bam)},
                                     out_vcf_base, {"cortex": cortex_dir, "stampy": stampy_dir,
                                                    "vcftools": vcftools_dir},
                                     config)
            if cortex_out:
                _remap_cortex_out(cortex_out, region, out_file)
            else:
                write_empty_vcf(out_file)
    return out_file

def _remap_cortex_out(cortex_out, region, out_file):
    """Remap coordinates in local cortex variant calls to the original global region.
    """
    def _remap_vcf_line(line, contig, start):
        parts = line.split("\t")
        if parts[0] == "" or parts[1] == "":
            return None
        parts[0] = contig
        try:
            parts[1] = str(int(parts[1]) + start)
        except ValueError:
            raise ValueError("Problem in {0} with \n{1}".format(
                    cortex_out, parts))
        return "\t".join(parts)
    contig, start, _ = region
    start = int(start) - 1
    with open(cortex_out) as in_handle:
        with open(out_file, "w") as out_handle:
            for line in in_handle:
                if line.startswith("##fileDate"):
                    pass
                elif line.startswith("#"):
                    out_handle.write(line)
                else:
                    update_line = _remap_vcf_line(line, contig, start)
                    if update_line:
                        out_handle.write(update_line)

def _run_cortex(fastq, indexes, params, out_base, dirs, config):
    """Run cortex_var run_calls.pl, producing a VCF variant file.
    """
    print out_base
    assert len(params["kmers"]) == 1, "Currently only support single kmer workflow"
    fastaq_index = "{0}.fastaq_index".format(out_base)
    se_fastq_index = "{0}.se_fastq".format(out_base)
    pe_fastq_index = "{0}.pe_fastq".format(out_base)
    reffasta_index = "{0}.list_ref_fasta".format(out_base)
    with open(se_fastq_index, "w") as out_handle:
        out_handle.write(fastq + "\n")
    with open(pe_fastq_index, "w") as out_handle:
        out_handle.write("")
    with open(fastaq_index, "w") as out_handle:
        out_handle.write("{0}\t{1}\t{2}\t{2}\n".format(params["sample"], se_fastq_index,
                                                       pe_fastq_index))
    with open(reffasta_index, "w") as out_handle:
        for x in indexes["fasta"]:
            out_handle.write(x + "\n")
    os.environ["PERL5LIB"] = "{0}:{1}:{2}".format(
        os.path.join(dirs["cortex"], "scripts/calling"),
        os.path.join(dirs["cortex"], "scripts/analyse_variants/bioinf-perl/lib"),
        os.environ.get("PERL5LIB", ""))
    subprocess.check_call(["perl", os.path.join(dirs["cortex"], "scripts", "calling", "run_calls.pl"),
                           "--first_kmer", str(params["kmers"][0]), "--fastaq_index", fastaq_index,
                           "--auto_cleaning", "yes", "--bc", "yes", "--pd", "yes",
                           "--outdir", os.path.dirname(out_base), "--outvcf", os.path.basename(out_base),
                           "--ploidy", str(config["algorithm"].get("ploidy", 2)),
                           "--stampy_hash", indexes["stampy"],
                           "--stampy_bin", os.path.join(dirs["stampy"], "stampy.py"),
                           "--refbindir", os.path.dirname(indexes["cortex"][0]),
                           "--list_ref_fasta",  reffasta_index,
                           "--genome_size", str(params["genome_size"]),
                           "--max_read_len", "250", "--max_var_len", "1000",
                           "--format", "FASTQ", "--qthresh", "5", "--do_union", "yes",
                           "--mem_height", "17", "--mem_width", "100",
                           "--ref", "CoordinatesAndInCalling", "--workflow", "independent",
                           "--vcftools_dir", dirs["vcftools"],
                           "--logfile", "{0}.logfile,f".format(out_base)])
    final = glob.glob(os.path.join(os.path.dirname(out_base), "vcfs",
                                   "{0}*FINALcombined_PD*decomp.vcf".format(os.path.basename(out_base))))
    # No calls, need to setup an empty file
    if len(final) != 1:
        print "Did not find output VCF file for {0}".format(out_base)
        return None
    else:
        return final[0]

def _get_cortex_binary(kmer, cortex_dir):
    cortex_bin = None
    for check_bin in sorted(glob.glob(os.path.join(cortex_dir, "bin", "cortex_var_*"))):
        kmer_check = int(os.path.basename(check_bin).split("_")[2])
        if kmer_check >= kmer:
            cortex_bin = check_bin
            break
    assert cortex_bin is not None, \
        "Could not find cortex_var executable in %s for kmer %s" % (cortex_dir, kmer)
    return cortex_bin

def _index_local_ref(fasta_file, cortex_dir, stampy_dir, kmers):
    """Pre-index a generated local reference sequence with cortex_var and stampy.
    """
    base_out = os.path.splitext(fasta_file)[0]
    cindexes = []
    for kmer in kmers:
        out_file = "{0}.k{1}.ctx".format(base_out, kmer)
        if not file_exists(out_file):
            file_list = "{0}.se_list".format(base_out)
            with open(file_list, "w") as out_handle:
                out_handle.write(fasta_file + "\n")
            subprocess.check_call([_get_cortex_binary(kmer, cortex_dir),
                                   "--kmer_size", str(kmer), "--mem_height", "17",
                                   "--se_list", file_list, "--format", "FASTA",
                                   "--max_read_len", "250", "--sample_id", base_out,
                                   "--dump_binary", out_file])
        cindexes.append(out_file)
    if not file_exists("{0}.stidx".format(base_out)):
        subprocess.check_call([os.path.join(stampy_dir, "stampy.py"), "-G",
                               base_out, fasta_file])
        subprocess.check_call([os.path.join(stampy_dir, "stampy.py"), "-g",
                               base_out, "-H", base_out])
    return {"stampy": base_out,
            "cortex": cindexes,
            "fasta": [fasta_file]}

def _get_local_ref(region, ref_file, out_vcf_base):
    """Retrieve a local FASTA file corresponding to the specified region.
    """
    out_file = "{0}.fa".format(out_vcf_base)
    if not file_exists(out_file):
        with closing(pysam.Fastafile(ref_file)) as in_pysam:
            contig, start, end = region
            seq = in_pysam.fetch(contig, int(start) - 1, int(end))
            with open(out_file, "w") as out_handle:
                out_handle.write(">{0}-{1}-{2}\n{3}".format(contig, start, end,
                                                              str(seq)))
    with open(out_file) as in_handle:
        in_handle.readline()
        size = len(in_handle.readline().strip())
    return out_file, size

def _get_fastq_in_region(region, align_bam, out_base):
    """Retrieve fastq files in region as single end.
    Paired end is more complicated since pairs can map off the region, so focus
    on local only assembly since we've previously used paired information for mapping.
    """
    out_file = "{0}.fastq".format(out_base)
    if not file_exists(out_file):
        with closing(pysam.Samfile(align_bam, "rb")) as in_pysam:
            with file_transaction(out_file) as tx_out_file:
                with open(out_file, "w") as out_handle:
                    contig, start, end = region
                    for read in in_pysam.fetch(contig, int(start) - 1, int(end)):
                        seq = Seq.Seq(read.seq)
                        qual = list(read.qual)
                        if read.is_reverse:
                            seq = seq.reverse_complement()
                            qual.reverse()
                        out_handle.write("@{name}\n{seq}\n+\n{qual}\n".format(
                                name=read.qname, seq=str(seq), qual="".join(qual)))
    return out_file

## Utility functions

def _count_fastq_reads(in_fastq, min_reads):
    """Count the number of fastq reads in a file, stopping after reaching min_reads.
    """
    with open(in_fastq) as in_handle:
        items = list(itertools.takewhile(lambda i : i <= min_reads,
                                         (i for i, _ in enumerate(FastqGeneralIterator(in_handle)))))
    return len(items)

def _get_sample_name(align_bam):
    with closing(pysam.Samfile(align_bam, "rb")) as in_pysam:
        return in_pysam.header["RG"][0]["SM"]
<commit_msg>Increase default max read length to allow greater range of kmers<commit_after>"""Perform regional de-novo assembly calling with cortex_var.

Using a pre-mapped set of reads and BED file of regions, performs de-novo
assembly and variant calling against the reference sequence in each region.
This avoids whole genome costs while gaining the advantage of de-novo
prediction.

http://cortexassembler.sourceforge.net/index_cortex_var.html
"""
import os
import glob
import subprocess
import itertools
from contextlib import closing

import pysam
from Bio import Seq
from Bio.SeqIO.QualityIO import FastqGeneralIterator

from bcbio import broad
from bcbio.distributed.transaction import file_transaction
from bcbio.pipeline.shared import subset_variant_regions
from bcbio.utils import file_exists, safe_makedir
from bcbio.variation.genotype import combine_variant_files, write_empty_vcf

def run_cortex(align_bam, ref_file, config, dbsnp=None, region=None,
               out_file=None):
    """Top level entry to regional de-novo based variant calling with cortex_var.
    """
    broad_runner = broad.runner_from_config(config)
    if out_file is None:
        out_file = "%s-cortex.vcf" % os.path.splitext(align_bam)[0]
    if not file_exists(out_file):
        broad_runner.run_fn("picard_index", align_bam)
        variant_regions = config["algorithm"].get("variant_regions", None)
        if not variant_regions:
            raise ValueError("Only regional variant calling with cortex_var is supported. Set variant_regions")
        target_regions = subset_variant_regions(variant_regions, region, out_file)
        with open(target_regions) as in_handle:
            regional_vcfs = [_run_cortex_on_region(x.strip().split("\t")[:3], align_bam,
                                                   ref_file, out_file, config)
                             for x in in_handle]
        combine_variant_files(regional_vcfs, out_file, ref_file, config)
    return out_file

def _run_cortex_on_region(region, align_bam, ref_file, out_file_base, config):
    """Run cortex on a specified chromosome start/end region.
    """
    kmers = [31]
    min_reads = 700
    cortex_dir = config["program"].get("cortex")
    stampy_dir = config["program"].get("stampy")
    vcftools_dir = config["program"].get("vcftools")
    if cortex_dir is None or stampy_dir is None:
        raise ValueError("cortex_var requires path to pre-built cortex and stampy")
    region_str = apply("{0}-{1}-{2}".format, region)
    base_dir = safe_makedir(os.path.join(os.path.dirname(out_file_base), region_str))
    out_vcf_base = os.path.join(base_dir, "{0}-{1}".format(
            os.path.splitext(os.path.basename(out_file_base))[0], region_str))
    out_file = "{0}.vcf".format(out_vcf_base)
    if not file_exists(out_file):
        fastq = _get_fastq_in_region(region, align_bam, out_vcf_base)
        if _count_fastq_reads(fastq, min_reads) < min_reads:
            write_empty_vcf(out_file)
        else:
            local_ref, genome_size = _get_local_ref(region, ref_file, out_vcf_base)
            indexes = _index_local_ref(local_ref, cortex_dir, stampy_dir, kmers)
            cortex_out = _run_cortex(fastq, indexes, {"kmers": kmers, "genome_size": genome_size,
                                                      "sample": _get_sample_name(align_bam)},
                                     out_vcf_base, {"cortex": cortex_dir, "stampy": stampy_dir,
                                                    "vcftools": vcftools_dir},
                                     config)
            if cortex_out:
                _remap_cortex_out(cortex_out, region, out_file)
            else:
                write_empty_vcf(out_file)
    return out_file

def _remap_cortex_out(cortex_out, region, out_file):
    """Remap coordinates in local cortex variant calls to the original global region.
    """
    def _remap_vcf_line(line, contig, start):
        parts = line.split("\t")
        if parts[0] == "" or parts[1] == "":
            return None
        parts[0] = contig
        try:
            parts[1] = str(int(parts[1]) + start)
        except ValueError:
            raise ValueError("Problem in {0} with \n{1}".format(
                    cortex_out, parts))
        return "\t".join(parts)
    contig, start, _ = region
    start = int(start) - 1
    with open(cortex_out) as in_handle:
        with open(out_file, "w") as out_handle:
            for line in in_handle:
                if line.startswith("##fileDate"):
                    pass
                elif line.startswith("#"):
                    out_handle.write(line)
                else:
                    update_line = _remap_vcf_line(line, contig, start)
                    if update_line:
                        out_handle.write(update_line)

def _run_cortex(fastq, indexes, params, out_base, dirs, config):
    """Run cortex_var run_calls.pl, producing a VCF variant file.
    """
    print out_base
    assert len(params["kmers"]) == 1, "Currently only support single kmer workflow"
    fastaq_index = "{0}.fastaq_index".format(out_base)
    se_fastq_index = "{0}.se_fastq".format(out_base)
    pe_fastq_index = "{0}.pe_fastq".format(out_base)
    reffasta_index = "{0}.list_ref_fasta".format(out_base)
    with open(se_fastq_index, "w") as out_handle:
        out_handle.write(fastq + "\n")
    with open(pe_fastq_index, "w") as out_handle:
        out_handle.write("")
    with open(fastaq_index, "w") as out_handle:
        out_handle.write("{0}\t{1}\t{2}\t{2}\n".format(params["sample"], se_fastq_index,
                                                       pe_fastq_index))
    with open(reffasta_index, "w") as out_handle:
        for x in indexes["fasta"]:
            out_handle.write(x + "\n")
    os.environ["PERL5LIB"] = "{0}:{1}:{2}".format(
        os.path.join(dirs["cortex"], "scripts/calling"),
        os.path.join(dirs["cortex"], "scripts/analyse_variants/bioinf-perl/lib"),
        os.environ.get("PERL5LIB", ""))
    subprocess.check_call(["perl", os.path.join(dirs["cortex"], "scripts", "calling", "run_calls.pl"),
                           "--first_kmer", str(params["kmers"][0]), "--fastaq_index", fastaq_index,
                           "--auto_cleaning", "yes", "--bc", "yes", "--pd", "yes",
                           "--outdir", os.path.dirname(out_base), "--outvcf", os.path.basename(out_base),
                           "--ploidy", str(config["algorithm"].get("ploidy", 2)),
                           "--stampy_hash", indexes["stampy"],
                           "--stampy_bin", os.path.join(dirs["stampy"], "stampy.py"),
                           "--refbindir", os.path.dirname(indexes["cortex"][0]),
                           "--list_ref_fasta",  reffasta_index,
                           "--genome_size", str(params["genome_size"]),
                           "--max_read_len", "1000", "--max_var_len", "1000",
                           "--format", "FASTQ", "--qthresh", "5", "--do_union", "yes",
                           "--mem_height", "17", "--mem_width", "100",
                           "--ref", "CoordinatesAndInCalling", "--workflow", "independent",
                           "--vcftools_dir", dirs["vcftools"],
                           "--logfile", "{0}.logfile,f".format(out_base)])
    final = glob.glob(os.path.join(os.path.dirname(out_base), "vcfs",
                                   "{0}*FINALcombined_PD*decomp.vcf".format(os.path.basename(out_base))))
    # No calls, need to setup an empty file
    if len(final) != 1:
        print "Did not find output VCF file for {0}".format(out_base)
        return None
    else:
        return final[0]

def _get_cortex_binary(kmer, cortex_dir):
    cortex_bin = None
    for check_bin in sorted(glob.glob(os.path.join(cortex_dir, "bin", "cortex_var_*"))):
        kmer_check = int(os.path.basename(check_bin).split("_")[2])
        if kmer_check >= kmer:
            cortex_bin = check_bin
            break
    assert cortex_bin is not None, \
        "Could not find cortex_var executable in %s for kmer %s" % (cortex_dir, kmer)
    return cortex_bin

def _index_local_ref(fasta_file, cortex_dir, stampy_dir, kmers):
    """Pre-index a generated local reference sequence with cortex_var and stampy.
    """
    base_out = os.path.splitext(fasta_file)[0]
    cindexes = []
    for kmer in kmers:
        out_file = "{0}.k{1}.ctx".format(base_out, kmer)
        if not file_exists(out_file):
            file_list = "{0}.se_list".format(base_out)
            with open(file_list, "w") as out_handle:
                out_handle.write(fasta_file + "\n")
            subprocess.check_call([_get_cortex_binary(kmer, cortex_dir),
                                   "--kmer_size", str(kmer), "--mem_height", "17",
                                   "--se_list", file_list, "--format", "FASTA",
                                   "--max_read_len", "1000", "--sample_id", base_out,
                                   "--dump_binary", out_file])
        cindexes.append(out_file)
    if not file_exists("{0}.stidx".format(base_out)):
        subprocess.check_call([os.path.join(stampy_dir, "stampy.py"), "-G",
                               base_out, fasta_file])
        subprocess.check_call([os.path.join(stampy_dir, "stampy.py"), "-g",
                               base_out, "-H", base_out])
    return {"stampy": base_out,
            "cortex": cindexes,
            "fasta": [fasta_file]}

def _get_local_ref(region, ref_file, out_vcf_base):
    """Retrieve a local FASTA file corresponding to the specified region.
    """
    out_file = "{0}.fa".format(out_vcf_base)
    if not file_exists(out_file):
        with closing(pysam.Fastafile(ref_file)) as in_pysam:
            contig, start, end = region
            seq = in_pysam.fetch(contig, int(start) - 1, int(end))
            with open(out_file, "w") as out_handle:
                out_handle.write(">{0}-{1}-{2}\n{3}".format(contig, start, end,
                                                              str(seq)))
    with open(out_file) as in_handle:
        in_handle.readline()
        size = len(in_handle.readline().strip())
    return out_file, size

def _get_fastq_in_region(region, align_bam, out_base):
    """Retrieve fastq files in region as single end.
    Paired end is more complicated since pairs can map off the region, so focus
    on local only assembly since we've previously used paired information for mapping.
    """
    out_file = "{0}.fastq".format(out_base)
    if not file_exists(out_file):
        with closing(pysam.Samfile(align_bam, "rb")) as in_pysam:
            with file_transaction(out_file) as tx_out_file:
                with open(out_file, "w") as out_handle:
                    contig, start, end = region
                    for read in in_pysam.fetch(contig, int(start) - 1, int(end)):
                        seq = Seq.Seq(read.seq)
                        qual = list(read.qual)
                        if read.is_reverse:
                            seq = seq.reverse_complement()
                            qual.reverse()
                        out_handle.write("@{name}\n{seq}\n+\n{qual}\n".format(
                                name=read.qname, seq=str(seq), qual="".join(qual)))
    return out_file

## Utility functions

def _count_fastq_reads(in_fastq, min_reads):
    """Count the number of fastq reads in a file, stopping after reaching min_reads.
    """
    with open(in_fastq) as in_handle:
        items = list(itertools.takewhile(lambda i : i <= min_reads,
                                         (i for i, _ in enumerate(FastqGeneralIterator(in_handle)))))
    return len(items)

def _get_sample_name(align_bam):
    with closing(pysam.Samfile(align_bam, "rb")) as in_pysam:
        return in_pysam.header["RG"][0]["SM"]
<|endoftext|>


<commit_before>"""Synchronization functions.

File- and mutex-based mutual exclusion synchronizers are provided,
as well as a name-based mutex which locks within an application
based on a string name.

"""

import os
import sys
import tempfile

try:
    import threading as _threading
except ImportError:
    import dummy_threading as _threading

# check for fcntl module
try:
    sys.getwindowsversion()
    has_flock = False
except:
    try:
        import fcntl
        has_flock = True
    except ImportError:
        has_flock = False

from beaker import util
from beaker.exceptions import LockError

__all__  = ["file_synchronizer", "mutex_synchronizer", "null_synchronizer",
            "NameLock", "_threading"]

class NameLock(object):
    """a proxy for an RLock object that is stored in a name based
    registry.

    Multiple threads can get a reference to the same RLock based on the
    name alone, and synchronize operations related to that name.

    """
    locks = util.WeakValuedRegistry()

    class NLContainer(object):
        def __init__(self, reentrant):
            if reentrant:
                self.lock = _threading.RLock()
            else:
                self.lock = _threading.Lock()
        def __call__(self):
            return self.lock

    def __init__(self, identifier = None, reentrant = False):
        if identifier is None:
            self._lock = NameLock.NLContainer(reentrant)
        else:
            self._lock = NameLock.locks.get(identifier, NameLock.NLContainer,
                                            reentrant)

    def acquire(self, wait = True):
        return self._lock().acquire(wait)

    def release(self):
        self._lock().release()

_synchronizers = util.WeakValuedRegistry()
def _synchronizer(identifier, cls, **kwargs):
    return _synchronizers.sync_get((identifier, cls), cls, identifier, **kwargs)

def file_synchronizer(identifier, **kwargs):
    if not has_flock or 'lock_dir' not in kwargs:
        return mutex_synchronizer(identifier)
    else:
        return _synchronizer(identifier, FileSynchronizer, **kwargs)

def mutex_synchronizer(identifier, **kwargs):
    return _synchronizer(identifier, ConditionSynchronizer, **kwargs)

class null_synchronizer(object):
    def acquire_write_lock(self, wait=True):
        return True
    def acquire_read_lock(self):
        pass
    def release_write_lock(self):
        pass
    def release_read_lock(self):
        pass
    acquire = acquire_write_lock
    release = release_write_lock

class SynchronizerImpl(object):
    def __init__(self):
        self._state = util.ThreadLocal()

    class SyncState(object):
        __slots__ = 'reentrantcount', 'writing', 'reading'

        def __init__(self):
            self.reentrantcount = 0
            self.writing = False
            self.reading = False

    def state(self):
        if not self._state.has():
            state = SynchronizerImpl.SyncState()
            self._state.put(state)
            return state
        else:
            return self._state.get()
    state = property(state)

    def release_read_lock(self):
        state = self.state

        if state.writing:
            raise LockError("lock is in writing state")
        if not state.reading:
            raise LockError("lock is not in reading state")

        if state.reentrantcount == 1:
            self.do_release_read_lock()
            state.reading = False

        state.reentrantcount -= 1

    def acquire_read_lock(self, wait = True):
        state = self.state

        if state.writing:
            raise LockError("lock is in writing state")

        if state.reentrantcount == 0:
            x = self.do_acquire_read_lock(wait)
            if (wait or x):
                state.reentrantcount += 1
                state.reading = True
            return x
        elif state.reading:
            state.reentrantcount += 1
            return True

    def release_write_lock(self):
        state = self.state

        if state.reading:
            raise LockError("lock is in reading state")
        if not state.writing:
            raise LockError("lock is not in writing state")

        if state.reentrantcount == 1:
            self.do_release_write_lock()
            state.writing = False

        state.reentrantcount -= 1

    release = release_write_lock

    def acquire_write_lock(self, wait  = True):
        state = self.state

        if state.reading:
            raise LockError("lock is in reading state")

        if state.reentrantcount == 0:
            x = self.do_acquire_write_lock(wait)
            if (wait or x):
                state.reentrantcount += 1
                state.writing = True
            return x
        elif state.writing:
            state.reentrantcount += 1
            return True

    acquire = acquire_write_lock

    def do_release_read_lock(self):
        raise NotImplementedError()

    def do_acquire_read_lock(self):
        raise NotImplementedError()

    def do_release_write_lock(self):
        raise NotImplementedError()

    def do_acquire_write_lock(self):
        raise NotImplementedError()

class FileSynchronizer(SynchronizerImpl):
    """a synchronizer which locks using flock().

    Adapted for Python/multithreads from Apache::Session::Lock::File,
    http://search.cpan.org/src/CWEST/Apache-Session-1.81/Session/Lock/File.pm

    This module does not unlink temporary files,
    because it interferes with proper locking.  This can cause
    problems on certain systems (Linux) whose file systems (ext2) do not
    perform well with lots of files in one directory.  To prevent this
    you should use a script to clean out old files from your lock directory.

    """
    def __init__(self, identifier, lock_dir):
        super(FileSynchronizer, self).__init__()
        self._filedescriptor = util.ThreadLocal()

        if lock_dir is None:
            lock_dir = tempfile.gettempdir()
        else:
            lock_dir = lock_dir

        self.filename = util.encoded_path(
                            lock_dir,
                            [identifier],
                            extension='.lock'
                        )

    def _filedesc(self):
        return self._filedescriptor.get()
    _filedesc = property(_filedesc)

    def _open(self, mode):
        filedescriptor = self._filedesc
        if not filedescriptor:
            filedescriptor = os.open(self.filename, mode)
            self._filedescriptor.put(filedescriptor)
        return filedescriptor

    def do_acquire_read_lock(self, wait):
        filedescriptor = self._open(os.O_CREAT | os.O_RDONLY)
        if not wait:
            try:
                fcntl.flock(filedescriptor, fcntl.LOCK_SH | fcntl.LOCK_NB)
                return True
            except IOError:
                os.close(filedescriptor)
                self._filedescriptor.remove()
                return False
        else:
            fcntl.flock(filedescriptor, fcntl.LOCK_SH)
            return True

    def do_acquire_write_lock(self, wait):
        filedescriptor = self._open(os.O_CREAT | os.O_WRONLY)
        if not wait:
            try:
                fcntl.flock(filedescriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
                return True
            except IOError:
                os.close(filedescriptor)
                self._filedescriptor.remove()
                return False
        else:
            fcntl.flock(filedescriptor, fcntl.LOCK_EX)
            return True

    def do_release_read_lock(self):
        self._release_all_locks()

    def do_release_write_lock(self):
        self._release_all_locks()

    def _release_all_locks(self):
        filedescriptor = self._filedesc
        if filedescriptor:
            fcntl.flock(filedescriptor, fcntl.LOCK_UN)
            os.close(filedescriptor)
            self._filedescriptor.remove()

class ConditionSynchronizer(SynchronizerImpl):
    """a synchronizer using a Condition."""

    def __init__(self, identifier):
        super(ConditionSynchronizer, self).__init__()

        # counts how many asynchronous methods are executing
        self.async = 0

        # pointer to thread that is the current sync operation
        self.current_sync_operation = None

        # condition object to lock on
        self.condition = _threading.Condition(_threading.Lock())

    def do_acquire_read_lock(self, wait = True):
        self.condition.acquire()
        try:
            # see if a synchronous operation is waiting to start
            # or is already running, in which case we wait (or just
            # give up and return)
            if wait:
                while self.current_sync_operation is not None:
                    self.condition.wait()
            else:
                if self.current_sync_operation is not None:
                    return False

            self.async += 1
        finally:
            self.condition.release()

        if not wait:
            return True

    def do_release_read_lock(self):
        self.condition.acquire()
        try:
            self.async -= 1

            # check if we are the last asynchronous reader thread
            # out the door.
            if self.async == 0:
                # yes. so if a sync operation is waiting, notifyAll to wake
                # it up
                if self.current_sync_operation is not None:
                    self.condition.notifyAll()
            elif self.async < 0:
                raise LockError("Synchronizer error - too many "
                                "release_read_locks called")
        finally:
            self.condition.release()

    def do_acquire_write_lock(self, wait = True):
        self.condition.acquire()
        try:
            # here, we are not a synchronous reader, and after returning,
            # assuming waiting or immediate availability, we will be.

            if wait:
                # if another sync is working, wait
                while self.current_sync_operation is not None:
                    self.condition.wait()
            else:
                # if another sync is working,
                # we dont want to wait, so forget it
                if self.current_sync_operation is not None:
                    return False

            # establish ourselves as the current sync
            # this indicates to other read/write operations
            # that they should wait until this is None again
            self.current_sync_operation = _threading.currentThread()

            # now wait again for asyncs to finish
            if self.async > 0:
                if wait:
                    # wait
                    self.condition.wait()
                else:
                    # we dont want to wait, so forget it
                    self.current_sync_operation = None
                    return False
        finally:
            self.condition.release()

        if not wait:
            return True

    def do_release_write_lock(self):
        self.condition.acquire()
        try:
            if self.current_sync_operation is not _threading.currentThread():
                raise LockError("Synchronizer error - current thread doesnt "
                                "have the write lock")

            # reset the current sync operation so
            # another can get it
            self.current_sync_operation = None

            # tell everyone to get ready
            self.condition.notifyAll()
        finally:
            # everyone go !!
            self.condition.release()
<commit_msg>Patch to deal with filedescriptor being 0<commit_after>"""Synchronization functions.

File- and mutex-based mutual exclusion synchronizers are provided,
as well as a name-based mutex which locks within an application
based on a string name.

"""

import os
import sys
import tempfile

try:
    import threading as _threading
except ImportError:
    import dummy_threading as _threading

# check for fcntl module
try:
    sys.getwindowsversion()
    has_flock = False
except:
    try:
        import fcntl
        has_flock = True
    except ImportError:
        has_flock = False

from beaker import util
from beaker.exceptions import LockError

__all__  = ["file_synchronizer", "mutex_synchronizer", "null_synchronizer",
            "NameLock", "_threading"]

class NameLock(object):
    """a proxy for an RLock object that is stored in a name based
    registry.

    Multiple threads can get a reference to the same RLock based on the
    name alone, and synchronize operations related to that name.

    """
    locks = util.WeakValuedRegistry()

    class NLContainer(object):
        def __init__(self, reentrant):
            if reentrant:
                self.lock = _threading.RLock()
            else:
                self.lock = _threading.Lock()
        def __call__(self):
            return self.lock

    def __init__(self, identifier = None, reentrant = False):
        if identifier is None:
            self._lock = NameLock.NLContainer(reentrant)
        else:
            self._lock = NameLock.locks.get(identifier, NameLock.NLContainer,
                                            reentrant)

    def acquire(self, wait = True):
        return self._lock().acquire(wait)

    def release(self):
        self._lock().release()

_synchronizers = util.WeakValuedRegistry()
def _synchronizer(identifier, cls, **kwargs):
    return _synchronizers.sync_get((identifier, cls), cls, identifier, **kwargs)

def file_synchronizer(identifier, **kwargs):
    if not has_flock or 'lock_dir' not in kwargs:
        return mutex_synchronizer(identifier)
    else:
        return _synchronizer(identifier, FileSynchronizer, **kwargs)

def mutex_synchronizer(identifier, **kwargs):
    return _synchronizer(identifier, ConditionSynchronizer, **kwargs)

class null_synchronizer(object):
    def acquire_write_lock(self, wait=True):
        return True
    def acquire_read_lock(self):
        pass
    def release_write_lock(self):
        pass
    def release_read_lock(self):
        pass
    acquire = acquire_write_lock
    release = release_write_lock

class SynchronizerImpl(object):
    def __init__(self):
        self._state = util.ThreadLocal()

    class SyncState(object):
        __slots__ = 'reentrantcount', 'writing', 'reading'

        def __init__(self):
            self.reentrantcount = 0
            self.writing = False
            self.reading = False

    def state(self):
        if not self._state.has():
            state = SynchronizerImpl.SyncState()
            self._state.put(state)
            return state
        else:
            return self._state.get()
    state = property(state)

    def release_read_lock(self):
        state = self.state

        if state.writing:
            raise LockError("lock is in writing state")
        if not state.reading:
            raise LockError("lock is not in reading state")

        if state.reentrantcount == 1:
            self.do_release_read_lock()
            state.reading = False

        state.reentrantcount -= 1

    def acquire_read_lock(self, wait = True):
        state = self.state

        if state.writing:
            raise LockError("lock is in writing state")

        if state.reentrantcount == 0:
            x = self.do_acquire_read_lock(wait)
            if (wait or x):
                state.reentrantcount += 1
                state.reading = True
            return x
        elif state.reading:
            state.reentrantcount += 1
            return True

    def release_write_lock(self):
        state = self.state

        if state.reading:
            raise LockError("lock is in reading state")
        if not state.writing:
            raise LockError("lock is not in writing state")

        if state.reentrantcount == 1:
            self.do_release_write_lock()
            state.writing = False

        state.reentrantcount -= 1

    release = release_write_lock

    def acquire_write_lock(self, wait  = True):
        state = self.state

        if state.reading:
            raise LockError("lock is in reading state")

        if state.reentrantcount == 0:
            x = self.do_acquire_write_lock(wait)
            if (wait or x):
                state.reentrantcount += 1
                state.writing = True
            return x
        elif state.writing:
            state.reentrantcount += 1
            return True

    acquire = acquire_write_lock

    def do_release_read_lock(self):
        raise NotImplementedError()

    def do_acquire_read_lock(self):
        raise NotImplementedError()

    def do_release_write_lock(self):
        raise NotImplementedError()

    def do_acquire_write_lock(self):
        raise NotImplementedError()

class FileSynchronizer(SynchronizerImpl):
    """a synchronizer which locks using flock().

    Adapted for Python/multithreads from Apache::Session::Lock::File,
    http://search.cpan.org/src/CWEST/Apache-Session-1.81/Session/Lock/File.pm

    This module does not unlink temporary files,
    because it interferes with proper locking.  This can cause
    problems on certain systems (Linux) whose file systems (ext2) do not
    perform well with lots of files in one directory.  To prevent this
    you should use a script to clean out old files from your lock directory.

    """
    def __init__(self, identifier, lock_dir):
        super(FileSynchronizer, self).__init__()
        self._filedescriptor = util.ThreadLocal()

        if lock_dir is None:
            lock_dir = tempfile.gettempdir()
        else:
            lock_dir = lock_dir

        self.filename = util.encoded_path(
                            lock_dir,
                            [identifier],
                            extension='.lock'
                        )

    def _filedesc(self):
        return self._filedescriptor.get()
    _filedesc = property(_filedesc)

    def _open(self, mode):
        filedescriptor = self._filedesc
        if filedescriptor is None:
            filedescriptor = os.open(self.filename, mode)
            self._filedescriptor.put(filedescriptor)
        return filedescriptor

    def do_acquire_read_lock(self, wait):
        filedescriptor = self._open(os.O_CREAT | os.O_RDONLY)
        if not wait:
            try:
                fcntl.flock(filedescriptor, fcntl.LOCK_SH | fcntl.LOCK_NB)
                return True
            except IOError:
                os.close(filedescriptor)
                self._filedescriptor.remove()
                return False
        else:
            fcntl.flock(filedescriptor, fcntl.LOCK_SH)
            return True

    def do_acquire_write_lock(self, wait):
        filedescriptor = self._open(os.O_CREAT | os.O_WRONLY)
        if not wait:
            try:
                fcntl.flock(filedescriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
                return True
            except IOError:
                os.close(filedescriptor)
                self._filedescriptor.remove()
                return False
        else:
            fcntl.flock(filedescriptor, fcntl.LOCK_EX)
            return True

    def do_release_read_lock(self):
        self._release_all_locks()

    def do_release_write_lock(self):
        self._release_all_locks()

    def _release_all_locks(self):
        filedescriptor = self._filedesc
        if filedescriptor is not None:
            fcntl.flock(filedescriptor, fcntl.LOCK_UN)
            os.close(filedescriptor)
            self._filedescriptor.remove()

class ConditionSynchronizer(SynchronizerImpl):
    """a synchronizer using a Condition."""

    def __init__(self, identifier):
        super(ConditionSynchronizer, self).__init__()

        # counts how many asynchronous methods are executing
        self.async = 0

        # pointer to thread that is the current sync operation
        self.current_sync_operation = None

        # condition object to lock on
        self.condition = _threading.Condition(_threading.Lock())

    def do_acquire_read_lock(self, wait = True):
        self.condition.acquire()
        try:
            # see if a synchronous operation is waiting to start
            # or is already running, in which case we wait (or just
            # give up and return)
            if wait:
                while self.current_sync_operation is not None:
                    self.condition.wait()
            else:
                if self.current_sync_operation is not None:
                    return False

            self.async += 1
        finally:
            self.condition.release()

        if not wait:
            return True

    def do_release_read_lock(self):
        self.condition.acquire()
        try:
            self.async -= 1

            # check if we are the last asynchronous reader thread
            # out the door.
            if self.async == 0:
                # yes. so if a sync operation is waiting, notifyAll to wake
                # it up
                if self.current_sync_operation is not None:
                    self.condition.notifyAll()
            elif self.async < 0:
                raise LockError("Synchronizer error - too many "
                                "release_read_locks called")
        finally:
            self.condition.release()

    def do_acquire_write_lock(self, wait = True):
        self.condition.acquire()
        try:
            # here, we are not a synchronous reader, and after returning,
            # assuming waiting or immediate availability, we will be.

            if wait:
                # if another sync is working, wait
                while self.current_sync_operation is not None:
                    self.condition.wait()
            else:
                # if another sync is working,
                # we dont want to wait, so forget it
                if self.current_sync_operation is not None:
                    return False

            # establish ourselves as the current sync
            # this indicates to other read/write operations
            # that they should wait until this is None again
            self.current_sync_operation = _threading.currentThread()

            # now wait again for asyncs to finish
            if self.async > 0:
                if wait:
                    # wait
                    self.condition.wait()
                else:
                    # we dont want to wait, so forget it
                    self.current_sync_operation = None
                    return False
        finally:
            self.condition.release()

        if not wait:
            return True

    def do_release_write_lock(self):
        self.condition.acquire()
        try:
            if self.current_sync_operation is not _threading.currentThread():
                raise LockError("Synchronizer error - current thread doesnt "
                                "have the write lock")

            # reset the current sync operation so
            # another can get it
            self.current_sync_operation = None

            # tell everyone to get ready
            self.condition.notifyAll()
        finally:
            # everyone go !!
            self.condition.release()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

"""Facilities for automatically determining files' correct metadata.
"""

from __future__ import division, absolute_import, print_function

from beets import logging
from beets import config

# Parts of external interface.
from .hooks import (  # noqa
    AlbumInfo,
    TrackInfo,
    AlbumMatch,
    TrackMatch,
    Distance,
)
from .match import tag_item, tag_album, Proposal  # noqa
from .match import Recommendation  # noqa

# Global logger.
log = logging.getLogger('beets')

# metadata that is already hardcoded
SPECIAL_FIELDS = {
    'album': (
        'va',
        'releasegroup_id',
        'artist_id',
        'album_id',
        'mediums',
        'tracks',
        'year',
        'month',
        'day',
        'artist',
        'artist_credit',
        'artist_sort',
        'data_url'
    ),
    'track': (
        'track_alt',
        'artist_id',
        'release_track_id',
        'medium',
        'index',
        'medium_index',
        'title',
        'artist_credit',
        'artist_sort',
        'artist',
        'track_id',
        'medium_total',
        'data_url',
        'length'
    )
}

# Additional utilities for the main interface.

def apply_item_metadata(item, track_info):
    """Set an item's metadata from its matched TrackInfo object.
    """
    item.artist = track_info.artist
    item.artist_sort = track_info.artist_sort
    item.artist_credit = track_info.artist_credit
    item.title = track_info.title
    item.mb_trackid = track_info.track_id
    item.mb_releasetrackid = track_info.release_track_id
    if track_info.artist_id:
        item.mb_artistid = track_info.artist_id

    for field, value in track_info.items():
        # only overwrite fields that are not already hardcoded
        if field in SPECIAL_FIELDS['track']:
            continue
        if value is None:
            continue
        item[field] = value

    # At the moment, the other metadata is left intact (including album
    # and track number). Perhaps these should be emptied?

def apply_metadata(album_info, mapping):
    """Set the items' metadata to match an AlbumInfo object using a
    mapping from Items to TrackInfo objects.
    """
    for item, track_info in mapping.items():
        # Artist or artist credit.
        if config['artist_credit']:
            item.artist = (track_info.artist_credit or
                           track_info.artist or
                           album_info.artist_credit or
                           album_info.artist)
            item.albumartist = (album_info.artist_credit or
                                album_info.artist)
        else:
            item.artist = (track_info.artist or album_info.artist)
            item.albumartist = album_info.artist

        # Album.
        item.album = album_info.album

        # Artist sort and credit names.
        item.artist_sort = track_info.artist_sort or album_info.artist_sort
        item.artist_credit = (track_info.artist_credit or
                              album_info.artist_credit)
        item.albumartist_sort = album_info.artist_sort
        item.albumartist_credit = album_info.artist_credit

        # Release date.
        for prefix in '', 'original_':
            if config['original_date'] and not prefix:
                # Ignore specific release date.
                continue

            for suffix in 'year', 'month', 'day':
                key = prefix + suffix
                value = getattr(album_info, key) or 0

                # If we don't even have a year, apply nothing.
                if suffix == 'year' and not value:
                    break

                # Otherwise, set the fetched value (or 0 for the month
                # and day if not available).
                item[key] = value

                # If we're using original release date for both fields,
                # also set item.year = info.original_year, etc.
                if config['original_date']:
                    item[suffix] = value

        # Title.
        item.title = track_info.title

        if config['per_disc_numbering']:
            # We want to let the track number be zero, but if the medium index
            # is not provided we need to fall back to the overall index.
            if track_info.medium_index is not None:
                item.track = track_info.medium_index
            else:
                item.track = track_info.index
            item.tracktotal = track_info.medium_total or len(album_info.tracks)
        else:
            item.track = track_info.index
            item.tracktotal = len(album_info.tracks)

        # Disc and disc count.
        item.disc = track_info.medium
        item.disctotal = album_info.mediums

        # MusicBrainz IDs.
        item.mb_trackid = track_info.track_id
        item.mb_releasetrackid = track_info.release_track_id
        item.mb_albumid = album_info.album_id
        if track_info.artist_id:
            item.mb_artistid = track_info.artist_id
        else:
            item.mb_artistid = album_info.artist_id
        item.mb_albumartistid = album_info.artist_id
        item.mb_releasegroupid = album_info.releasegroup_id

        # Compilation flag.
        item.comp = album_info.va

        # Track alt.
        item.track_alt = track_info.track_alt

        # Don't overwrite fields with empty values unless the
        # field is explicitly allowed to be overwritten
        for field, value in album_info.items():
            if field in SPECIAL_FIELDS['album']:
                continue
            clobber = field in config['overwrite_null']['album'].as_str_seq()
            if value is None and not clobber:
                continue
            item[field] = value

        for field, value in track_info.items():
            if field in SPECIAL_FIELDS['track']:
                continue
            clobber = field in config['overwrite_null']['track'].as_str_seq()
            value = getattr(track_info, field)
            if value is None and not clobber:
                continue
            item[field] = value
<commit_msg>Cosmetics<commit_after># -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

"""Facilities for automatically determining files' correct metadata.
"""

from __future__ import division, absolute_import, print_function

from beets import logging
from beets import config

# Parts of external interface.
from .hooks import (  # noqa
    AlbumInfo,
    TrackInfo,
    AlbumMatch,
    TrackMatch,
    Distance,
)
from .match import tag_item, tag_album, Proposal  # noqa
from .match import Recommendation  # noqa

# Global logger.
log = logging.getLogger('beets')

# Metadata that is already hardcoded.
SPECIAL_FIELDS = {
    'album': (
        'va',
        'releasegroup_id',
        'artist_id',
        'album_id',
        'mediums',
        'tracks',
        'year',
        'month',
        'day',
        'artist',
        'artist_credit',
        'artist_sort',
        'data_url'
    ),
    'track': (
        'track_alt',
        'artist_id',
        'release_track_id',
        'medium',
        'index',
        'medium_index',
        'title',
        'artist_credit',
        'artist_sort',
        'artist',
        'track_id',
        'medium_total',
        'data_url',
        'length'
    )
}

# Additional utilities for the main interface.

def apply_item_metadata(item, track_info):
    """Set an item's metadata from its matched TrackInfo object.
    """
    item.artist = track_info.artist
    item.artist_sort = track_info.artist_sort
    item.artist_credit = track_info.artist_credit
    item.title = track_info.title
    item.mb_trackid = track_info.track_id
    item.mb_releasetrackid = track_info.release_track_id
    if track_info.artist_id:
        item.mb_artistid = track_info.artist_id

    for field, value in track_info.items():
        # We only overwrite fields that are not already hardcoded.
        if field in SPECIAL_FIELDS['track']:
            continue
        if value is None:
            continue
        item[field] = value

    # At the moment, the other metadata is left intact (including album
    # and track number). Perhaps these should be emptied?

def apply_metadata(album_info, mapping):
    """Set the items' metadata to match an AlbumInfo object using a
    mapping from Items to TrackInfo objects.
    """
    for item, track_info in mapping.items():
        # Artist or artist credit.
        if config['artist_credit']:
            item.artist = (track_info.artist_credit or
                           track_info.artist or
                           album_info.artist_credit or
                           album_info.artist)
            item.albumartist = (album_info.artist_credit or
                                album_info.artist)
        else:
            item.artist = (track_info.artist or album_info.artist)
            item.albumartist = album_info.artist

        # Album.
        item.album = album_info.album

        # Artist sort and credit names.
        item.artist_sort = track_info.artist_sort or album_info.artist_sort
        item.artist_credit = (track_info.artist_credit or
                              album_info.artist_credit)
        item.albumartist_sort = album_info.artist_sort
        item.albumartist_credit = album_info.artist_credit

        # Release date.
        for prefix in '', 'original_':
            if config['original_date'] and not prefix:
                # Ignore specific release date.
                continue

            for suffix in 'year', 'month', 'day':
                key = prefix + suffix
                value = getattr(album_info, key) or 0

                # If we don't even have a year, apply nothing.
                if suffix == 'year' and not value:
                    break

                # Otherwise, set the fetched value (or 0 for the month
                # and day if not available).
                item[key] = value

                # If we're using original release date for both fields,
                # also set item.year = info.original_year, etc.
                if config['original_date']:
                    item[suffix] = value

        # Title.
        item.title = track_info.title

        if config['per_disc_numbering']:
            # We want to let the track number be zero, but if the medium index
            # is not provided we need to fall back to the overall index.
            if track_info.medium_index is not None:
                item.track = track_info.medium_index
            else:
                item.track = track_info.index
            item.tracktotal = track_info.medium_total or len(album_info.tracks)
        else:
            item.track = track_info.index
            item.tracktotal = len(album_info.tracks)

        # Disc and disc count.
        item.disc = track_info.medium
        item.disctotal = album_info.mediums

        # MusicBrainz IDs.
        item.mb_trackid = track_info.track_id
        item.mb_releasetrackid = track_info.release_track_id
        item.mb_albumid = album_info.album_id
        if track_info.artist_id:
            item.mb_artistid = track_info.artist_id
        else:
            item.mb_artistid = album_info.artist_id
        item.mb_albumartistid = album_info.artist_id
        item.mb_releasegroupid = album_info.releasegroup_id

        # Compilation flag.
        item.comp = album_info.va

        # Track alt.
        item.track_alt = track_info.track_alt

        # Don't overwrite fields with empty values unless the
        # field is explicitly allowed to be overwritten
        for field, value in album_info.items():
            if field in SPECIAL_FIELDS['album']:
                continue
            clobber = field in config['overwrite_null']['album'].as_str_seq()
            if value is None and not clobber:
                continue
            item[field] = value

        for field, value in track_info.items():
            if field in SPECIAL_FIELDS['track']:
                continue
            clobber = field in config['overwrite_null']['track'].as_str_seq()
            value = getattr(track_info, field)
            if value is None and not clobber:
                continue
            item[field] = value
<|endoftext|>


<commit_before>import os
from behave import step
from tempfile import NamedTemporaryFile
from errno import EPIPE
from multiprocessing import Process
from tempfile import mkdtemp

@step(u'an empty file "{name}"')
def empty_file(context, name):
    assert name not in context.files
    f = NamedTemporaryFile()
    assert os.path.exists(f.name), "Path exists"
    assert os.path.isfile(f.name), "Path is file"
    context.files[name] = f

@step(u'a file "{name}" with contents')
def file_with_content(context, name):
    assert name not in context.files
    empty_file(context, name)
    context.files[name].write(context.text)

@step(u'a file "{name}" with {mb} mb of zeroes')
def file_zeroes(context, name, mb):
    assert name not in context.dirs
    assert name not in context.fifo
    fname = os.path.join(mkdtemp(), 'fifo')
    os.mkfifo(fname)
    mb = int(mb)

    def run_fifo():
        import sys
        sys.stdout = sys.stderr = open('/tmp/fiflog', 'w')
        sys.stderr.write("START")
        fifo = open(fname, 'w')
        for _ in xrange(mb*1024):
            try:
                fifo.write('0' * 1024)
                print ("wrote 1k\n")
                sys.stdout.flush()
                fifo.flush()
            except IOError as e:
                if e.errno != EPIPE:
                    raise
        fifo.close()

    proc = Process(target=run_fifo)
    proc.start()
    context.fifo[fname] = proc
    context.files[name] = open(fname, 'r')
<commit_msg>simple variable size file step<commit_after>import os
from behave import step
from tempfile import NamedTemporaryFile
from errno import EPIPE
from multiprocessing import Process
from tempfile import mkdtemp

@step(u'an empty file "{name}"')
def empty_file(context, name):
    assert name not in context.files
    f = NamedTemporaryFile()
    assert os.path.exists(f.name), "Path exists"
    assert os.path.isfile(f.name), "Path is file"
    context.files[name] = f

@step(u'a file "{name}" with contents')
def file_with_content(context, name):
    assert name not in context.files
    empty_file(context, name)
    context.files[name].write(context.text)

@step(u'a file "{name}" with {mb} mb of zeroes')
def file_zeroes(context, name, mb):
    assert name not in context.files
    empty_file(context, name)
    context.files[name].truncate(int(mb) * 1024 * 1024)

@step(u'a fifo "{name}" with {mb} mb of zeroes')
def fifo_zeroes(context, name, mb):
    assert name not in context.dirs
    assert name not in context.fifo
    dname = mkdtemp()
    fname = os.path.join(dname, 'fifo')
    os.mkfifo(fname)
    mb = int(mb)

    def run_fifo():
        import sys
        sys.stdout = sys.stderr = open('/tmp/fiflog', 'w')
        sys.stderr.write("START")
        fifo = open(fname, 'w')
        for _ in xrange(mb*1024):
            try:
                fifo.write('0' * 1024)
                print ("wrote 1k\n")
                sys.stdout.flush()
                fifo.flush()
            except IOError as e:
                if e.errno != EPIPE:
                    raise
        fifo.close()

    proc = Process(target=run_fifo)
    proc.start()
    context.fifo[fname] = proc
    context.dirs[name] = dname
    context.files[name] = open(fname, 'r')
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
BenchExec is a framework for reliable benchmarking.
This file is part of BenchExec.

Copyright (C) 2007-2015  Dirk Beyer
All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import benchexec.result as result
import benchexec.util as util
import benchexec.tools.template

class Tool(benchexec.tools.template.BaseTool):
    """
    Tool wrapper for Ceagle.
    It has additional features such as building CPAchecker before running it
    if executed within a source checkout.
    It also supports extracting data from the statistics output of CPAchecker
    for adding it to the result tables.
    """

    def executable(self):
        return util.find_executable('ceagle.sh')

    def version(self, executable):
        return '1.0'

    def name(self):
        return 'Ceagle'

    def cmdline(self, executable, options, tasks, propertyfile, rlimits):
        return [executable] + tasks

    def determine_result(self, returncode, returnsignal, output, isTimeout):
        """
        @param returncode: code returned by Ceagle
        @param returnsignal: signal, which terminated Ceagle
        @param output: the output of Ceagle
        @return: status of Ceagle after executing a run
        """
        status = result.RESULT_UNKNOWN
        stroutput = str(output)

        if isTimeout:
            status = 'TIMEOUT'
        elif 'TRUE' in stroutput:
            status = result.RESULT_TRUE_PROP
        elif 'FALSE' in stroutput:
            status = result.RESULT_FALSE_REACH
        elif 'UNKNOWN' in stroutput:
            status = result.RESULT_UNKNOWN
        else:
            assert(False)

        return status
<commit_msg>DW: fixed version, option and comments<commit_after>#!/usr/bin/env python
"""
BenchExec is a framework for reliable benchmarking.
This file is part of BenchExec.

Copyright (C) 2007-2015  Dirk Beyer
All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import benchexec.result as result
import benchexec.util as util
import benchexec.tools.template

class Tool(benchexec.tools.template.BaseTool):

    def executable(self):
        return util.find_executable('ceagle.sh')

    def version(self, executable):
        return self._version_from_tool(executable)

    def name(self):
        return 'Ceagle'

    def cmdline(self, executable, options, tasks, propertyfile, rlimits):
        return [executable] + options + tasks

    def determine_result(self, returncode, returnsignal, output, isTimeout):

        status = result.RESULT_UNKNOWN
        stroutput = str(output)

        if isTimeout:
            status = 'TIMEOUT'
        elif 'TRUE' in stroutput:
            status = result.RESULT_TRUE_PROP
        elif 'FALSE' in stroutput:
            status = result.RESULT_FALSE_REACH
        elif 'UNKNOWN' in stroutput:
            status = result.RESULT_UNKNOWN
        else:
            status = result.RESULT_UNKNOWN

        return status
<|endoftext|>


<commit_before>import sys
import code

def components(swc_path):

	f = open(swc_path, 'r')
	lines = f.readlines()
	f.close()

	# make a map from parent to child, determine number of components
	root_indices = []
	parent_to_child = {}
	index_to_type = {}

	for line in lines:
		parent = line.split()[0]

		# check for a root node
		if line.split()[6].strip() == '-1':
			root_indices.append( line.split()[0] )

		# look for some children
		for sub_line in lines:
			if sub_line.split()[6].strip() == parent:
				if parent not in parent_to_child:
					parent_to_child[ parent ] = []
				parent_to_child[ parent ].append( sub_line.split()[0].strip() )

	# dfs to set color of each connected component
	for ii, _ in enumerate(root_indices):
		dfs(ii, root_indices, index_to_type, parent_to_child)

	f = open(swc_path[:-4] + '_components.swc', 'w')

	for line in lines:
		if line.split()[0] in index_to_type:
			f.write(str(line.split()[0]) + ' ' + str(index_to_type[ line.split()[0] ]) + ' ' + str(line.split()[2]) + \
				' ' + str(line.split()[3]) + ' ' + str(line.split()[4]) + ' ' + str(line.split()[5]) + \
				' ' + str(line.split()[6].strip()) + '\n')
	f.close()

def dfs(index, root_indices, index_to_type, parent_to_child):
	dfs_stack = []
	current_index = root_indices[index]
	dfs_stack.append(current_index)

	while dfs_stack:
		current_index = dfs_stack.pop()
		index_to_type[current_index] = index;
		if current_index in parent_to_child:
			for j in parent_to_child[current_index]:
				dfs_stack.append(j)

if __name__ == "__main__":
	if len(sys.argv) != 2:
		print('\nSWC_COMPONENTS -- Nathan Spencer 2017')
		print('Usage: python swc_components.py ["path/to/swc/file.swc"]')
	else:
		components(sys.argv[1])
<commit_msg>removed unused import [swc_components]<commit_after>import sys

def components(swc_path):

	f = open(swc_path, 'r')
	lines = f.readlines()
	f.close()

	# make a map from parent to child, determine number of components
	root_indices = []
	parent_to_child = {}
	index_to_type = {}

	for line in lines:
		parent = line.split()[0]

		# check for a root node
		if line.split()[6].strip() == '-1':
			root_indices.append( line.split()[0] )

		# look for some children
		for sub_line in lines:
			if sub_line.split()[6].strip() == parent:
				if parent not in parent_to_child:
					parent_to_child[ parent ] = []
				parent_to_child[ parent ].append( sub_line.split()[0].strip() )

	# dfs to set color of each connected component
	for ii, _ in enumerate(root_indices):
		dfs(ii, root_indices, index_to_type, parent_to_child)

	f = open(swc_path[:-4] + '_components.swc', 'w')

	for line in lines:
		if line.split()[0] in index_to_type:
			f.write(str(line.split()[0]) + ' ' + str(index_to_type[ line.split()[0] ]) + ' ' + str(line.split()[2]) + \
				' ' + str(line.split()[3]) + ' ' + str(line.split()[4]) + ' ' + str(line.split()[5]) + \
				' ' + str(line.split()[6].strip()) + '\n')
	f.close()

def dfs(index, root_indices, index_to_type, parent_to_child):
	dfs_stack = []
	current_index = root_indices[index]
	dfs_stack.append(current_index)

	while dfs_stack:
		current_index = dfs_stack.pop()
		index_to_type[current_index] = index;
		if current_index in parent_to_child:
			for j in parent_to_child[current_index]:
				dfs_stack.append(j)

if __name__ == "__main__":
	if len(sys.argv) != 2:
		print('\nSWC_COMPONENTS -- Nathan Spencer 2017')
		print('Usage: python swc_components.py ["path/to/swc/file.swc"]')
	else:
		components(sys.argv[1])
<|endoftext|>


<commit_before>
import datetime

from django.forms import fields, widgets
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django_comments.forms import CommentForm
from django.utils.encoding import force_text
from django.utils.safestring import mark_safe

from sweettooth.ratings.models import RatingComment

# Raty inserts its own <input> element, so we don't want to provide
# a widget here. We'll insert a <div> for raty to fill in the template.
class NoOpWidget(widgets.Widget):
    def render(self, *a, **kw):
        return u''

class RatingCommentForm(CommentForm):
    rating = fields.IntegerField(min_value=-1, max_value=5,
                                 required=False, widget=NoOpWidget())

    def clean_rating(self):
        rating = self.cleaned_data["rating"]
        if rating is None:
            rating = -1
        return rating

    def get_comment_model(self):
        return RatingComment

    def get_comment_create_data(self):
        return dict(
            content_type = ContentType.objects.get_for_model(self.target_object),
            object_pk    = force_text(self.target_object._get_pk_val()),
            comment      = self.cleaned_data["comment"],
            rating       = self.cleaned_data["rating"],
            submit_date  = datetime.datetime.now(),
            site_id      = settings.SITE_ID,
            is_public    = True,
            is_removed   = False,
        )

# Remove the URL, name and email fields. We don't want them.
RatingCommentForm.base_fields.pop('url')
RatingCommentForm.base_fields.pop('name')
RatingCommentForm.base_fields.pop('email')
<commit_msg>comments: fixed comment posting after upgrade to new django_comments<commit_after>
from django.forms import fields, widgets
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django_comments.forms import CommentForm
from django.utils.encoding import force_text
from django.utils.safestring import mark_safe
from django.utils import timezone

from sweettooth.ratings.models import RatingComment

# Raty inserts its own <input> element, so we don't want to provide
# a widget here. We'll insert a <div> for raty to fill in the template.
class NoOpWidget(widgets.Widget):
    def render(self, *a, **kw):
        return u''

class RatingCommentForm(CommentForm):
    rating = fields.IntegerField(min_value=-1, max_value=5,
                                 required=False, widget=NoOpWidget())

    def clean_rating(self):
        rating = self.cleaned_data["rating"]
        if rating is None:
            rating = -1
        return rating

    def get_comment_model(self):
        return RatingComment

    def get_comment_create_data(self, site_id=None):
        return dict(
            content_type = ContentType.objects.get_for_model(self.target_object),
            object_pk    = force_text(self.target_object._get_pk_val()),
            comment      = self.cleaned_data["comment"],
            rating       = self.cleaned_data["rating"],
            submit_date  = timezone.now(),
            site_id      = site_id or getattr(settings, "SITE_ID", None),
            is_public    = True,
            is_removed   = False,
        )

# Remove the URL, name and email fields. We don't want them.
RatingCommentForm.base_fields.pop('url')
RatingCommentForm.base_fields.pop('name')
RatingCommentForm.base_fields.pop('email')
<|endoftext|>