<commit_before>import os
import ConfigParser

from sentence_parser import SentenceParser
from lexicon import Lexicon
from spreading_activation import SpreadingActivation
from definition_parser import read as read_defs
from construction import *
from matcher import *
from sup_dic import supplementary_dictionary_reader as sdreader
from avm import AVM

class Wrapper:
    def __init__(self, cf):
        self.cfn = cf
        self.__read_config()

        self.lexicon = Lexicon()
        self.__read_files()
        self.__add_constructions()

    def __read_config(self):
        machinepath = os.path.realpath(__file__).rsplit("/", 2)[0]
        if "MACHINEPATH" in os.environ:
            machinepath = os.environ["MACHINEPATH"]
        config = ConfigParser.SafeConfigParser({"machinepath": machinepath})
        config.read(self.cfn)
        items = dict(config.items("machine"))
        self.def_files = [(s.split(":")[0].strip(), int(s.split(":")[1]))
            for s in items["definitions"].split(",")]
        self.supp_dict_fn = items["supp_dict"]
        self.stations_fn = items["stations"]

    def __read_files(self):
        self.__read_definitions()
        self.__read_supp_dict()

    def __read_definitions(self):
        for file_name, printname_index in self.def_files:
            # TODO HACK makefile needed
            if (file_name.endswith("generated") and
                not os.path.exists(file_name)):
                raise Exception("A definition file that supposed to be " +
                    "generated by pymachine/scripts/generate_translation_dict.sh" +
                    " does not exist: {0}".format(file_name))
            definitions = read_defs(file(file_name), printname_index)
            logging.debug("{0}: {1}".format(file_name, definitions.keys()))
            self.lexicon.add_static(definitions.itervalues())

    def __read_supp_dict(self):
        self.supp_dict = sdreader(file(self.supp_dict_fn))

    def __add_constructions(self):
        # TODO this should be created automatically, maybe from knowing,
        # that "megy" is actually a verb, there is a POS in the definitions
        try:
            megy_construction = VerbConstruction("megy", self.lexicon.static[
                "megy"], self.supp_dict)
            del self.lexicon.static["megy"]
            #self.lexicon.add_construction(megy_construction)
        except KeyError:
            pass
        try:
            tesz_construction = VerbConstruction("tesz", self.lexicon.static[
                "tesz"], self.supp_dict)
            del self.lexicon.static["tesz"]
            self.lexicon.add_construction(tesz_construction)
        except KeyError:
            pass
        #self.lexicon.add_construction(ElviraConstruction())

        self.lexicon.add_construction(DummyNPConstruction())
        self.lexicon.add_construction(TheConstruction())
        self.lexicon.add_construction(MaxNP_InBetweenPostP_Construction())
        self.lexicon.add_construction(PostPConstruction())

        station_matcher = FileContainsMatcher(self.stations_fn)
        ic_name_matcher = EnumMatcher("intercity", self.lexicon)
        src_matcher = AndMatcher(self.supp_dict["@HUN_GO_SRC"],
                                 station_matcher)
        tgt_matcher = AndMatcher(self.supp_dict["@HUN_GO_TGT"],
                                 station_matcher)
        ea = elvira_avm = AVM('ElviraAVM')
        ea.add_attribute("vonat", PrintnameMatcher("vonat"), AVM.RREQ, None)
        ea.add_attribute("menetrend", PrintnameMatcher("menetrend"), AVM.RREQ, None)
        ea.add_attribute("src", src_matcher, AVM.RREQ, "Budapest")
        ea.add_attribute("tgt", tgt_matcher, AVM.RREQ, None)
        ea.add_attribute("date", PosMatcher("\[DATE\]$"), AVM.ROPT, None)
        elvira_const = AVMConstruction(ea)
        self.lexicon.add_avm_construction(elvira_const)

        pta = plain_ticket_avm = AVM('PlainTicketAvm')
        pta.add_attribute("BKSZ", PrintnameMatcher("bksz"), AVM.ROPT, None)
        pta.add_attribute("CLASS", EnumMatcher("class", self.lexicon),
                          AVM.RREQ, "2")
        pta.add_attribute("DATE", PosMatcher("\[DATE\]$"), AVM.ROPT, None)
        pta.add_attribute("DEST", tgt_matcher, AVM.RREQ, None)
        pta.add_attribute("INV", PrintnameMatcher("invoice"), AVM.ROPT, None)
        pta.add_attribute("RED", EnumMatcher("mav_reduction", self.lexicon),
                          AVM.RREQ, "full_price")
        pta.add_attribute("RET", EnumMatcher("ticket_type", self.lexicon),
                          AVM.RREQ, "one_way")
        pta.add_attribute("SRC", src_matcher, AVM.RREQ, "Budapest")
        # Elvira takes precedence
        pta.add_attribute("ELVIRA", AndMatcher(
            PrintnameMatcher('ElviraAVM'),
            SatisfiedAVMMatcher()), AVM.RNEG, None)
        # If there is an invalid seat ticket request, do not return a ticket either
        pta.add_attribute('SEAT_TICKET', AndMatcher(
            PrintnameMatcher('ICTicketAvm'),
            SatisfiedAVMMatcher(False)), AVM.RNEG, None)
        pt_const = AVMConstruction(pta)
        self.lexicon.add_construction(pt_const)

        ita = ic_ticket_avm = AVM('ICTicketAvm')
        ita.add_attribute("CLASS", EnumMatcher("class", self.lexicon), AVM.RREQ, None)
        ita.add_attribute("DATE", PosMatcher("\[DATE\]$"), AVM.ROPT, None)
        ita.add_attribute("DEST", tgt_matcher, AVM.RREQ, None)
        ita.add_attribute("INV", PrintnameMatcher("invoice"), AVM.ROPT, None)
        ita.add_attribute("PLACE", EnumMatcher("seat", self.lexicon), AVM.ROPT, None)
        ita.add_attribute("SRC", src_matcher, AVM.RREQ, "Budapest-Nyugati")
        ita.add_attribute("TIME", AndMatcher(PosMatcher("\[TIME\]$"),
                                             ic_name_matcher), AVM.ROPT, None)
        it_const = AVMConstruction(ita)
        self.lexicon.add_avm_construction(it_const)

        # TODO create shrdlu construction

    def run(self, sentence):
        """Parses a sentence, runs the spreading activation and returns the
        messages that have to be sent to the active plugins."""
        try:
            sp = SentenceParser()
            sa = SpreadingActivation(self.lexicon)
            machines = sp.parse(sentence)

            # results is a list of (url, data) tuples
            results = sa.activation_loop(machines)
            self.lexicon.clear_active()
        except Exception, e:
            import traceback
            traceback.print_exc(e)
            raise(e)

        return results

def test():
    w = Wrapper(config_filename)
    w.run([([("a", "DET"),
             ("kek", "ADJ"),
             ("kockat", "NOUN<CAS<ACC>>")
            ], "ACC")])

if __name__ == "__main__":
    test()
<commit_msg>modified:   wrapper.py - ic_name enum again - OrMatcher for time slot (ic name OR TIME)<commit_after>import os
import ConfigParser

from sentence_parser import SentenceParser
from lexicon import Lexicon
from spreading_activation import SpreadingActivation
from definition_parser import read as read_defs
from construction import *
from matcher import *
from sup_dic import supplementary_dictionary_reader as sdreader
from avm import AVM

class Wrapper:
    def __init__(self, cf):
        self.cfn = cf
        self.__read_config()

        self.lexicon = Lexicon()
        self.__read_files()
        self.__add_constructions()

    def __read_config(self):
        machinepath = os.path.realpath(__file__).rsplit("/", 2)[0]
        if "MACHINEPATH" in os.environ:
            machinepath = os.environ["MACHINEPATH"]
        config = ConfigParser.SafeConfigParser({"machinepath": machinepath})
        config.read(self.cfn)
        items = dict(config.items("machine"))
        self.def_files = [(s.split(":")[0].strip(), int(s.split(":")[1]))
            for s in items["definitions"].split(",")]
        self.supp_dict_fn = items["supp_dict"]
        self.stations_fn = items["stations"]

    def __read_files(self):
        self.__read_definitions()
        self.__read_supp_dict()

    def __read_definitions(self):
        for file_name, printname_index in self.def_files:
            # TODO HACK makefile needed
            if (file_name.endswith("generated") and
                not os.path.exists(file_name)):
                raise Exception("A definition file that supposed to be " +
                    "generated by pymachine/scripts/generate_translation_dict.sh" +
                    " does not exist: {0}".format(file_name))
            definitions = read_defs(file(file_name), printname_index)
            logging.debug("{0}: {1}".format(file_name, definitions.keys()))
            self.lexicon.add_static(definitions.itervalues())

    def __read_supp_dict(self):
        self.supp_dict = sdreader(file(self.supp_dict_fn))

    def __add_constructions(self):
        # TODO this should be created automatically, maybe from knowing,
        # that "megy" is actually a verb, there is a POS in the definitions
        try:
            megy_construction = VerbConstruction("megy", self.lexicon.static[
                "megy"], self.supp_dict)
            del self.lexicon.static["megy"]
            #self.lexicon.add_construction(megy_construction)
        except KeyError:
            pass
        try:
            tesz_construction = VerbConstruction("tesz", self.lexicon.static[
                "tesz"], self.supp_dict)
            del self.lexicon.static["tesz"]
            self.lexicon.add_construction(tesz_construction)
        except KeyError:
            pass
        #self.lexicon.add_construction(ElviraConstruction())

        self.lexicon.add_construction(DummyNPConstruction())
        self.lexicon.add_construction(TheConstruction())
        self.lexicon.add_construction(MaxNP_InBetweenPostP_Construction())
        self.lexicon.add_construction(PostPConstruction())

        station_matcher = FileContainsMatcher(self.stations_fn)
        ic_name_matcher = EnumMatcher("ic_name", self.lexicon)
        src_matcher = AndMatcher(self.supp_dict["@HUN_GO_SRC"],
                                 station_matcher)
        tgt_matcher = AndMatcher(self.supp_dict["@HUN_GO_TGT"],
                                 station_matcher)
        ea = elvira_avm = AVM('ElviraAVM')
        ea.add_attribute("vonat", PrintnameMatcher("vonat"), AVM.RREQ, None)
        ea.add_attribute("menetrend", PrintnameMatcher("menetrend"), AVM.RREQ, None)
        ea.add_attribute("src", src_matcher, AVM.RREQ, "Budapest")
        ea.add_attribute("tgt", tgt_matcher, AVM.RREQ, None)
        ea.add_attribute("date", PosMatcher("\[DATE\]$"), AVM.ROPT, None)
        elvira_const = AVMConstruction(ea)
        self.lexicon.add_avm_construction(elvira_const)

        pta = plain_ticket_avm = AVM('PlainTicketAvm')
        pta.add_attribute("BKSZ", PrintnameMatcher("bksz"), AVM.ROPT, None)
        pta.add_attribute("CLASS", EnumMatcher("class", self.lexicon),
                          AVM.RREQ, "2")
        pta.add_attribute("DATE", PosMatcher("\[DATE\]$"), AVM.ROPT, None)
        pta.add_attribute("DEST", tgt_matcher, AVM.RREQ, None)
        pta.add_attribute("INV", PrintnameMatcher("invoice"), AVM.ROPT, None)
        pta.add_attribute("RED", EnumMatcher("mav_reduction", self.lexicon),
                          AVM.RREQ, "full_price")
        pta.add_attribute("RET", EnumMatcher("ticket_type", self.lexicon),
                          AVM.RREQ, "one_way")
        pta.add_attribute("SRC", src_matcher, AVM.RREQ, "Budapest")
        # Elvira takes precedence
        pta.add_attribute("ELVIRA", AndMatcher(
            PrintnameMatcher('ElviraAVM'),
            SatisfiedAVMMatcher()), AVM.RNEG, None)
        # If there is an invalid seat ticket request, do not return a ticket either
        pta.add_attribute('SEAT_TICKET', AndMatcher(
            PrintnameMatcher('ICTicketAvm'),
            SatisfiedAVMMatcher(False)), AVM.RNEG, None)
        pt_const = AVMConstruction(pta)
        self.lexicon.add_construction(pt_const)

        ita = ic_ticket_avm = AVM('ICTicketAvm')
        ita.add_attribute("CLASS", EnumMatcher("class", self.lexicon), AVM.RREQ, None)
        ita.add_attribute("DATE", PosMatcher("\[DATE\]$"), AVM.ROPT, None)
        ita.add_attribute("DEST", tgt_matcher, AVM.RREQ, None)
        ita.add_attribute("INV", PrintnameMatcher("invoice"), AVM.ROPT, None)
        ita.add_attribute("PLACE", EnumMatcher("seat", self.lexicon), AVM.ROPT, None)
        ita.add_attribute("SRC", src_matcher, AVM.RREQ, "Budapest-Nyugati")
        ita.add_attribute("TIME", OrMatcher(PosMatcher("\[TIME\]$"),
                                             ic_name_matcher), AVM.ROPT, None)
        it_const = AVMConstruction(ita)
        self.lexicon.add_avm_construction(it_const)

        # TODO create shrdlu construction

    def run(self, sentence):
        """Parses a sentence, runs the spreading activation and returns the
        messages that have to be sent to the active plugins."""
        try:
            sp = SentenceParser()
            sa = SpreadingActivation(self.lexicon)
            machines = sp.parse(sentence)

            # results is a list of (url, data) tuples
            results = sa.activation_loop(machines)
            self.lexicon.clear_active()
        except Exception, e:
            import traceback
            traceback.print_exc(e)
            raise(e)

        return results

def test():
    w = Wrapper(config_filename)
    w.run([([("a", "DET"),
             ("kek", "ADJ"),
             ("kockat", "NOUN<CAS<ACC>>")
            ], "ACC")])

if __name__ == "__main__":
    test()
<|endoftext|>


<commit_before>"""Utilities for working with fastq files.
"""

from itertools import izip, product
import os
import random
import gzip

from Bio import SeqIO
from bcbio.distributed import objectstore
from bcbio.distributed.transaction import file_transaction
from bcbio.log import logger
from bcbio import utils
from bcbio.utils import open_possible_gzip
from bcbio.pipeline import config_utils
from bcbio.provenance import do

@utils.memoize_outfile(stem=".groom")
def groom(in_file, data, in_qual="illumina", out_dir=None, out_file=None):
    """
    Grooms a FASTQ file from Illumina 1.3/1.5 quality scores into
    sanger format, if it is not already in that format.
    """
    seqtk = config_utils.get_program("seqtk", data["config"])
    if in_qual == "fastq-sanger":
        logger.info("%s is already in Sanger format." % in_file)
        return out_file
    with file_transaction(out_file) as tmp_out_file:
        cmd = "{seqtk} seq -Q64 {in_file} | gzip > {tmp_out_file}".format(**locals())
        do.run(cmd, "Converting %s to Sanger format." % in_file)
    return out_file

@utils.memoize_outfile(stem=".fixed")
def filter_single_reads_by_length(in_file, quality_format, min_length=20,
                                  out_file=None):
    """
    removes reads from a fastq file which are shorter than a minimum
    length

    """
    logger.info("Removing reads in %s thare are less than %d bases."
                % (in_file, min_length))
    in_iterator = SeqIO.parse(in_file, quality_format)
    out_iterator = (record for record in in_iterator if
                    len(record.seq) > min_length)
    with file_transaction(out_file) as tmp_out_file:
        with open(tmp_out_file, "w") as out_handle:
            SeqIO.write(out_iterator, out_handle, quality_format)
    return out_file

def filter_reads_by_length(fq1, fq2, quality_format, min_length=20):
    """
    removes reads from a pair of fastq files that are shorter than
    a minimum length. removes both ends of a read if one end falls
    below the threshold while maintaining the order of the reads

    """

    logger.info("Removing reads in %s and %s that "
                "are less than %d bases." % (fq1, fq2, min_length))
    fq1_out = utils.append_stem(fq1, ".fixed")
    fq2_out = utils.append_stem(fq2, ".fixed")
    fq1_single = utils.append_stem(fq1, ".singles")
    fq2_single = utils.append_stem(fq2, ".singles")
    if all(map(utils.file_exists, [fq1_out, fq2_out, fq2_single, fq2_single])):
        return [fq1_out, fq2_out]

    fq1_in = SeqIO.parse(fq1, quality_format)
    fq2_in = SeqIO.parse(fq2, quality_format)

    out_files = [fq1_out, fq2_out, fq1_single, fq2_single]

    with file_transaction(out_files) as tmp_out_files:
        fq1_out_handle = open(tmp_out_files[0], "w")
        fq2_out_handle = open(tmp_out_files[1], "w")
        fq1_single_handle = open(tmp_out_files[2], "w")
        fq2_single_handle = open(tmp_out_files[3], "w")

        for fq1_record, fq2_record in izip(fq1_in, fq2_in):
            if len(fq1_record.seq) >= min_length and len(fq2_record.seq) >= min_length:
                fq1_out_handle.write(fq1_record.format(quality_format))
                fq2_out_handle.write(fq2_record.format(quality_format))
            else:
                if len(fq1_record.seq) > min_length:
                    fq1_single_handle.write(fq1_record.format(quality_format))
                if len(fq2_record.seq) > min_length:
                    fq2_single_handle.write(fq2_record.format(quality_format))
        fq1_out_handle.close()
        fq2_out_handle.close()
        fq1_single_handle.close()
        fq2_single_handle.close()

    return [fq1_out, fq2_out]

def rstrip_extra(fname):
    """Strip extraneous, non-discriminative filename info from the end of a file.
    """
    to_strip = ("_R", "_", "fastq", ".", "-")
    while fname.endswith(to_strip):
        for x in to_strip:
            if fname.endswith(x):
                fname = fname[:len(fname) - len(x)]
                break
    return fname

def combine_pairs(input_files):
    """ calls files pairs if they are completely the same except
    for one has _1 and the other has _2 returns a list of tuples
    of pairs or singles.
    From bipy.utils (https://github.com/roryk/bipy/blob/master/bipy/utils.py)
    Adjusted to allow different input paths or extensions for matching files.
    """
    PAIR_FILE_IDENTIFIERS = set(["1", "2"])

    pairs = []
    used = set([])
    for in_file in input_files:
        if in_file in used:
            continue
        for comp_file in input_files:
            if comp_file in used or comp_file == in_file:
                continue
            a = rstrip_extra(utils.splitext_plus(os.path.basename(in_file))[0])
            b = rstrip_extra(utils.splitext_plus(os.path.basename(comp_file))[0])
            if len(a) != len(b):
                continue
            s = dif(a,b)
            if len(s) > 1:
                continue #there is only 1 difference
            if (a[s[0]] in PAIR_FILE_IDENTIFIERS and
                  b[s[0]] in PAIR_FILE_IDENTIFIERS):
                # if the 1/2 isn't the last digit before a separator, skip
                # this skips stuff like 2P 2A, often denoting replicates, not
                # read pairings
                if len(b) > (s[0] + 1):
                    if (b[s[0]+1] not in ("_", "-", ".")):
                        continue
                # if the 1/2 is not a separator or prefaced with R, skip
                if b[s[0]- 1] in ("R", "_", "-", "."):
                    used.add(in_file)
                    used.add(comp_file)
                    if b[s[0]] == "2":
                        pairs.append([in_file, comp_file])
                    else:
                        pairs.append([comp_file, in_file])
                    break
        if in_file not in used:
            pairs.append([in_file])
            used.add(in_file)
    return pairs

def dif(a, b):
    """ copy from http://stackoverflow.com/a/8545526 """
    return [i for i in range(len(a)) if a[i] != b[i]]

def is_fastq(in_file, bzip=True):
    fastq_ends = [".txt", ".fq", ".fastq"]
    zip_ends = [".gzip", ".gz"]
    if bzip:
        zip_ends += [".bz2", ".bzip2"]
    base, first_ext = os.path.splitext(in_file)
    second_ext = os.path.splitext(base)[1]
    if first_ext in fastq_ends:
        return True
    elif (second_ext, first_ext) in product(fastq_ends, zip_ends):
        return True
    else:
        return False

def downsample(f1, f2, data, N, quick=False):
    """ get N random headers from a fastq file without reading the
    whole thing into memory
    modified from: http://www.biostars.org/p/6544/
    quick=True will just grab the first N reads rather than do a true
    downsampling
    """
    if quick:
        rand_records = range(N)
    else:
        records = sum(1 for _ in open(f1)) / 4
        N = records if N > records else N
        rand_records = sorted(random.sample(xrange(records), N))

    fh1 = open_possible_gzip(f1)
    fh2 = open_possible_gzip(f2) if f2 else None
    outf1 = os.path.splitext(f1)[0] + ".subset" + os.path.splitext(f1)[1]
    outf2 = os.path.splitext(f2)[0] + ".subset" + os.path.splitext(f2)[1] if f2 else None

    if utils.file_exists(outf1):
        if not outf2:
            return outf1, outf2
        elif utils.file_exists(outf2):
            return outf1, outf2

    out_files = (outf1, outf2) if outf2 else (outf1)

    with file_transaction(out_files) as tx_out_files:
        if isinstance(tx_out_files, basestring):
            tx_out_f1 = tx_out_files
        else:
            tx_out_f1, tx_out_f2 = tx_out_files
        sub1 = open_possible_gzip(tx_out_f1, "w")
        sub2 = open_possible_gzip(tx_out_f2, "w") if outf2 else None
        rec_no = - 1
        for rr in rand_records:
            while rec_no < rr:
                rec_no += 1
                for i in range(4): fh1.readline()
                if fh2:
                    for i in range(4): fh2.readline()
            for i in range(4):
                sub1.write(fh1.readline())
                if sub2:
                    sub2.write(fh2.readline())
            rec_no += 1
        fh1.close()
        sub1.close()
        if f2:
            fh2.close()
            sub2.close()

    return outf1, outf2

def estimate_read_length(fastq_file, quality_format="fastq-sanger", nreads=1000):
    """
    estimate average read length of a fastq file
    """

    in_handle = SeqIO.parse(open_fastq(fastq_file), quality_format)
    read = in_handle.next()
    average = len(read.seq)
    for _ in range(nreads):
        try:
            average = (average + len(in_handle.next().seq)) / 2
        except StopIteration:
            break
    in_handle.close()
    return average

def estimate_maximum_read_length(fastq_file, quality_format="fastq-sanger",
                                 nreads=1000):
    """
    estimate average read length of a fastq file
    """
    in_handle = SeqIO.parse(open_fastq(fastq_file), quality_format)
    lengths = []
    for _ in range(nreads):
        try:
            lengths.append(len(in_handle.next().seq))
        except StopIteration:
            break
    in_handle.close()
    return max(lengths)

def open_fastq(in_file):
    """ open a fastq file, using gzip if it is gzipped
    """
    if objectstore.is_remote(in_file):
        return objectstore.open(in_file)
    _, ext = os.path.splitext(in_file)
    if ext == ".gz":
        return gzip.open(in_file, 'rb')
    if ext in [".fastq", ".fq"]:
        return open(in_file, 'r')
    # default to just opening it
    return open(in_file, "r")
<commit_msg>Add more helpful error message when two samples are named exactly the same.<commit_after>"""Utilities for working with fastq files.
"""

from itertools import izip, product
import os
import random
import gzip
import sys

from Bio import SeqIO
from bcbio.distributed import objectstore
from bcbio.distributed.transaction import file_transaction
from bcbio.log import logger
from bcbio import utils
from bcbio.utils import open_possible_gzip
from bcbio.pipeline import config_utils
from bcbio.provenance import do

@utils.memoize_outfile(stem=".groom")
def groom(in_file, data, in_qual="illumina", out_dir=None, out_file=None):
    """
    Grooms a FASTQ file from Illumina 1.3/1.5 quality scores into
    sanger format, if it is not already in that format.
    """
    seqtk = config_utils.get_program("seqtk", data["config"])
    if in_qual == "fastq-sanger":
        logger.info("%s is already in Sanger format." % in_file)
        return out_file
    with file_transaction(out_file) as tmp_out_file:
        cmd = "{seqtk} seq -Q64 {in_file} | gzip > {tmp_out_file}".format(**locals())
        do.run(cmd, "Converting %s to Sanger format." % in_file)
    return out_file

@utils.memoize_outfile(stem=".fixed")
def filter_single_reads_by_length(in_file, quality_format, min_length=20,
                                  out_file=None):
    """
    removes reads from a fastq file which are shorter than a minimum
    length

    """
    logger.info("Removing reads in %s thare are less than %d bases."
                % (in_file, min_length))
    in_iterator = SeqIO.parse(in_file, quality_format)
    out_iterator = (record for record in in_iterator if
                    len(record.seq) > min_length)
    with file_transaction(out_file) as tmp_out_file:
        with open(tmp_out_file, "w") as out_handle:
            SeqIO.write(out_iterator, out_handle, quality_format)
    return out_file

def filter_reads_by_length(fq1, fq2, quality_format, min_length=20):
    """
    removes reads from a pair of fastq files that are shorter than
    a minimum length. removes both ends of a read if one end falls
    below the threshold while maintaining the order of the reads

    """

    logger.info("Removing reads in %s and %s that "
                "are less than %d bases." % (fq1, fq2, min_length))
    fq1_out = utils.append_stem(fq1, ".fixed")
    fq2_out = utils.append_stem(fq2, ".fixed")
    fq1_single = utils.append_stem(fq1, ".singles")
    fq2_single = utils.append_stem(fq2, ".singles")
    if all(map(utils.file_exists, [fq1_out, fq2_out, fq2_single, fq2_single])):
        return [fq1_out, fq2_out]

    fq1_in = SeqIO.parse(fq1, quality_format)
    fq2_in = SeqIO.parse(fq2, quality_format)

    out_files = [fq1_out, fq2_out, fq1_single, fq2_single]

    with file_transaction(out_files) as tmp_out_files:
        fq1_out_handle = open(tmp_out_files[0], "w")
        fq2_out_handle = open(tmp_out_files[1], "w")
        fq1_single_handle = open(tmp_out_files[2], "w")
        fq2_single_handle = open(tmp_out_files[3], "w")

        for fq1_record, fq2_record in izip(fq1_in, fq2_in):
            if len(fq1_record.seq) >= min_length and len(fq2_record.seq) >= min_length:
                fq1_out_handle.write(fq1_record.format(quality_format))
                fq2_out_handle.write(fq2_record.format(quality_format))
            else:
                if len(fq1_record.seq) > min_length:
                    fq1_single_handle.write(fq1_record.format(quality_format))
                if len(fq2_record.seq) > min_length:
                    fq2_single_handle.write(fq2_record.format(quality_format))
        fq1_out_handle.close()
        fq2_out_handle.close()
        fq1_single_handle.close()
        fq2_single_handle.close()

    return [fq1_out, fq2_out]

def rstrip_extra(fname):
    """Strip extraneous, non-discriminative filename info from the end of a file.
    """
    to_strip = ("_R", "_", "fastq", ".", "-")
    while fname.endswith(to_strip):
        for x in to_strip:
            if fname.endswith(x):
                fname = fname[:len(fname) - len(x)]
                break
    return fname

def combine_pairs(input_files):
    """ calls files pairs if they are completely the same except
    for one has _1 and the other has _2 returns a list of tuples
    of pairs or singles.
    From bipy.utils (https://github.com/roryk/bipy/blob/master/bipy/utils.py)
    Adjusted to allow different input paths or extensions for matching files.
    """
    PAIR_FILE_IDENTIFIERS = set(["1", "2"])

    pairs = []
    used = set([])
    for in_file in input_files:
        if in_file in used:
            continue
        for comp_file in input_files:
            if comp_file in used or comp_file == in_file:
                continue
            a = rstrip_extra(utils.splitext_plus(os.path.basename(in_file))[0])
            b = rstrip_extra(utils.splitext_plus(os.path.basename(comp_file))[0])
            if len(a) != len(b):
                continue
            s = dif(a,b)
            # no differences, then its the same file stem
            if len(s) == 0:
                logger.error("%s and %s have the same stem, so we don't know "
                             "how to assign it to the sample data in the CSV. To "
                             "get around this you can rename one of the files. "
                             "If they are meant to be the same sample run in two "
                             "lanes, combine them first with the "
                             "bcbio_prepare_samples.py script."
                             "(http://bcbio-nextgen.readthedocs.io/en/latest/contents/configuration.html#multiple-files-per-sample)"
                             % (in_file, comp_file))
                sys.exit(1)
            if len(s) > 1:
                continue #there is only 1 difference
            if (a[s[0]] in PAIR_FILE_IDENTIFIERS and
                  b[s[0]] in PAIR_FILE_IDENTIFIERS):
                # if the 1/2 isn't the last digit before a separator, skip
                # this skips stuff like 2P 2A, often denoting replicates, not
                # read pairings
                if len(b) > (s[0] + 1):
                    if (b[s[0]+1] not in ("_", "-", ".")):
                        continue
                # if the 1/2 is not a separator or prefaced with R, skip
                if b[s[0]- 1] in ("R", "_", "-", "."):
                    used.add(in_file)
                    used.add(comp_file)
                    if b[s[0]] == "2":
                        pairs.append([in_file, comp_file])
                    else:
                        pairs.append([comp_file, in_file])
                    break
        if in_file not in used:
            pairs.append([in_file])
            used.add(in_file)
    return pairs

def dif(a, b):
    """ copy from http://stackoverflow.com/a/8545526 """
    return [i for i in range(len(a)) if a[i] != b[i]]

def is_fastq(in_file, bzip=True):
    fastq_ends = [".txt", ".fq", ".fastq"]
    zip_ends = [".gzip", ".gz"]
    if bzip:
        zip_ends += [".bz2", ".bzip2"]
    base, first_ext = os.path.splitext(in_file)
    second_ext = os.path.splitext(base)[1]
    if first_ext in fastq_ends:
        return True
    elif (second_ext, first_ext) in product(fastq_ends, zip_ends):
        return True
    else:
        return False

def downsample(f1, f2, data, N, quick=False):
    """ get N random headers from a fastq file without reading the
    whole thing into memory
    modified from: http://www.biostars.org/p/6544/
    quick=True will just grab the first N reads rather than do a true
    downsampling
    """
    if quick:
        rand_records = range(N)
    else:
        records = sum(1 for _ in open(f1)) / 4
        N = records if N > records else N
        rand_records = sorted(random.sample(xrange(records), N))

    fh1 = open_possible_gzip(f1)
    fh2 = open_possible_gzip(f2) if f2 else None
    outf1 = os.path.splitext(f1)[0] + ".subset" + os.path.splitext(f1)[1]
    outf2 = os.path.splitext(f2)[0] + ".subset" + os.path.splitext(f2)[1] if f2 else None

    if utils.file_exists(outf1):
        if not outf2:
            return outf1, outf2
        elif utils.file_exists(outf2):
            return outf1, outf2

    out_files = (outf1, outf2) if outf2 else (outf1)

    with file_transaction(out_files) as tx_out_files:
        if isinstance(tx_out_files, basestring):
            tx_out_f1 = tx_out_files
        else:
            tx_out_f1, tx_out_f2 = tx_out_files
        sub1 = open_possible_gzip(tx_out_f1, "w")
        sub2 = open_possible_gzip(tx_out_f2, "w") if outf2 else None
        rec_no = - 1
        for rr in rand_records:
            while rec_no < rr:
                rec_no += 1
                for i in range(4): fh1.readline()
                if fh2:
                    for i in range(4): fh2.readline()
            for i in range(4):
                sub1.write(fh1.readline())
                if sub2:
                    sub2.write(fh2.readline())
            rec_no += 1
        fh1.close()
        sub1.close()
        if f2:
            fh2.close()
            sub2.close()

    return outf1, outf2

def estimate_read_length(fastq_file, quality_format="fastq-sanger", nreads=1000):
    """
    estimate average read length of a fastq file
    """

    in_handle = SeqIO.parse(open_fastq(fastq_file), quality_format)
    read = in_handle.next()
    average = len(read.seq)
    for _ in range(nreads):
        try:
            average = (average + len(in_handle.next().seq)) / 2
        except StopIteration:
            break
    in_handle.close()
    return average

def estimate_maximum_read_length(fastq_file, quality_format="fastq-sanger",
                                 nreads=1000):
    """
    estimate average read length of a fastq file
    """
    in_handle = SeqIO.parse(open_fastq(fastq_file), quality_format)
    lengths = []
    for _ in range(nreads):
        try:
            lengths.append(len(in_handle.next().seq))
        except StopIteration:
            break
    in_handle.close()
    return max(lengths)

def open_fastq(in_file):
    """ open a fastq file, using gzip if it is gzipped
    """
    if objectstore.is_remote(in_file):
        return objectstore.open(in_file)
    _, ext = os.path.splitext(in_file)
    if ext == ".gz":
        return gzip.open(in_file, 'rb')
    if ext in [".fastq", ".fq"]:
        return open(in_file, 'r')
    # default to just opening it
    return open(in_file, "r")
<|endoftext|>


<commit_before>from gherkin.parser import Parser as GherkinParser
import six

from .exceptions import StepException

if six.PY3:
    from importlib.machinery import SourceFileLoader
else:
    from imp import load_source

class Feature:
    def __init__(self, name, tests):
        self.name = name
        self.tests = tests

    def __str__(self):
        return self.name

    def __repr__(self):
        return '<{} ({})>'.format(self.__class__.__name__, str(self))

class Test:
    def __init__(self, name, steps):
        self.name = name
        self.steps = steps

    def __str__(self):
        return self.name

    def __repr__(self):
        return '<{} ({})>'.format(self.__class__.__name__, str(self))

    def __call__(self, *args, **kwargs):
        def output(res, msg):
            if kwargs.get('bdd_verbose'):
                return res, msg
            return res

        context = dict(kwargs)
        if 'bdd_verbose' in context:
            del context['bdd_verbose']

        for step in self.steps:
            result = True
            explain = ''
            try:
                step(*args, **context)
            except StepException as e:
                result = False
                explain = str(e)
            if step.step_type == 'then':
                if result is False:
                    return output(result, explain)
            else:
                if result is False:
                    # failed conditional i.e. not relevant
                    return output(None, explain)
                else:
                    # passed conditional
                    pass
        return output(True, '')

class Step:
    mappings = []

    def __init__(self, step_type, step_text):
        def _find_matching_expr(line):
            for regex, fn in self.mappings:
                r = regex.match(line)
                if r:
                    return fn, r.groups()
            print('I didn\'t understand "{}"'.format(line))

        self.text = step_text
        self.step_type = step_type
        fn, groups = _find_matching_expr(step_text)
        self.expr_fn, self.expr_groups = fn, groups

    def __str__(self):
        return '{} {}'.format(self.step_type.title(), self.text)

    def __repr__(self):
        return '<{} ({})>'.format(self.__class__.__name__,
                                  self.step_type.title())

    def __call__(self, *args, **kwargs):
        if self.expr_groups:
            args = [*args, *self.expr_groups]
        self.expr_fn(*args, **kwargs)

class BDDTester:
    def __init__(self, stepfile_filepath):
        self._load_step_definitions(stepfile_filepath)
        self.gherkinparser = GherkinParser()

    def _load_step_definitions(self, filepath):
        # TODO: This is not right! The mappings array
        # will be overwritten whenever a new step definitions
        # file is loaded.
        Step.mappings = []
        if six.PY3:
            SourceFileLoader('', filepath).load_module()
        else:
            load_source('', filepath)

    def load_feature(self, feature_filepath):
        with open(feature_filepath) as f:
            feature_txt = f.read()
        return self._gherkinify_feature(feature_txt)

    def _gherkinify_feature(self, feature_txt):
        feature = self.gherkinparser.parse(feature_txt)
        feature = feature['feature']
        feature_name = feature['name']
        tests = []
        for test in feature['children']:
            test_name = test['name']
            test_steps = test['steps']
            steps = []
            step_type = 'given'
            for step in test_steps:
                if step['keyword'].lower().strip() == 'then':
                    step_type = 'then'
                steps.append(Step(step_type, step['text']))
            tests.append(Test(test_name, steps))
        return Feature(feature_name, tests)
<commit_msg>python2-related fix<commit_after>from gherkin.parser import Parser as GherkinParser
import six

from .exceptions import StepException

if six.PY3:
    from importlib.machinery import SourceFileLoader
else:
    from imp import load_source

class Feature:
    def __init__(self, name, tests):
        self.name = name
        self.tests = tests

    def __str__(self):
        return self.name

    def __repr__(self):
        return '<{} ({})>'.format(self.__class__.__name__, str(self))

class Test:
    def __init__(self, name, steps):
        self.name = name
        self.steps = steps

    def __str__(self):
        return self.name

    def __repr__(self):
        return '<{} ({})>'.format(self.__class__.__name__, str(self))

    def __call__(self, *args, **kwargs):
        def output(res, msg):
            if kwargs.get('bdd_verbose'):
                return res, msg
            return res

        context = dict(kwargs)
        if 'bdd_verbose' in context:
            del context['bdd_verbose']

        for step in self.steps:
            result = True
            explain = ''
            try:
                step(*args, **context)
            except StepException as e:
                result = False
                explain = str(e)
            if step.step_type == 'then':
                if result is False:
                    return output(result, explain)
            else:
                if result is False:
                    # failed conditional i.e. not relevant
                    return output(None, explain)
                else:
                    # passed conditional
                    pass
        return output(True, '')

class Step:
    mappings = []

    def __init__(self, step_type, step_text):
        def _find_matching_expr(line):
            for regex, fn in self.mappings:
                r = regex.match(line)
                if r:
                    return fn, r.groups()
            print('I didn\'t understand "{}"'.format(line))

        self.text = step_text
        self.step_type = step_type
        fn, groups = _find_matching_expr(step_text)
        self.expr_fn, self.expr_groups = fn, groups

    def __str__(self):
        return '{} {}'.format(self.step_type.title(), self.text)

    def __repr__(self):
        return '<{} ({})>'.format(self.__class__.__name__,
                                  self.step_type.title())

    def __call__(self, *args, **kwargs):
        if self.expr_groups:
            args = args + self.expr_groups
        self.expr_fn(*args, **kwargs)

class BDDTester:
    def __init__(self, stepfile_filepath):
        self._load_step_definitions(stepfile_filepath)
        self.gherkinparser = GherkinParser()

    def _load_step_definitions(self, filepath):
        # TODO: This is not right! The mappings array
        # will be overwritten whenever a new step definitions
        # file is loaded.
        Step.mappings = []
        if six.PY3:
            SourceFileLoader('', filepath).load_module()
        else:
            load_source('', filepath)

    def load_feature(self, feature_filepath):
        with open(feature_filepath) as f:
            feature_txt = f.read()
        return self._gherkinify_feature(feature_txt)

    def _gherkinify_feature(self, feature_txt):
        feature = self.gherkinparser.parse(feature_txt)
        feature = feature['feature']
        feature_name = feature['name']
        tests = []
        for test in feature['children']:
            test_name = test['name']
            test_steps = test['steps']
            steps = []
            step_type = 'given'
            for step in test_steps:
                if step['keyword'].lower().strip() == 'then':
                    step_type = 'then'
                steps.append(Step(step_type, step['text']))
            tests.append(Test(test_name, steps))
        return Feature(feature_name, tests)
<|endoftext|>


<commit_before>"""
Provides classes and methods related to communicating with the remote API
"""

from __future__ import absolute_import, division
from __future__ import unicode_literals, print_function

try:
    from urllib2 import Request as PythonRequest, urlopen, HTTPError
except ImportError:
    from urllib.request import Request as PythonRequest, urlopen
    from urllib.error import HTTPError

import json

from beekeeper.variable_handlers import render
from beekeeper.data_handlers import decode

def download_as_json(url):
    """
    Download the data at the URL and load it as JSON
    """
    return json.loads(request(url=url).read().decode('utf-8'))

def request(*args, **kwargs):
    """
    Make a request with the received arguments and return an
    HTTPResponse object
    """
    req = PythonRequest(*args, **kwargs)
    return urlopen(req)

class Request(object):

    """
    Holds data and provides methods related to building and sending
    an HTTP request with the data passed to it
    """

    def __init__(self, action, variables, verbose=False):
        self.action = action
        self.variables = variables
        self.verbose = verbose
        self.output = {}
        self.output['data'] = None
        self.output['headers'] = {}
        self.output['url'] = self.action.endpoint.url() + '?'
        self.render_variables()

    def render_variables(self):
        """
        Take the variables passed in during init and parse them
        into the base level variables we can actually do stuff with.
        """
        for var_type in self.variables.types():
            for var in render(var_type, **self.variables.vals(var_type)):
                self.set(var)
        self.output['url'] = self.output['url'][:len(self.output['url'])-1]

    def print_out(self):
        """
        Print out the three base level variables of the request.
        """
        print('URL: {}'.format(self.output['url']))
        print('Headers:')
        if self.output['headers']:
            for name, val in self.output['headers'].items():
                print('{}: {}'.format(name, val))
        else:
            print('None')
        print('Data:')
        if self.output['data']:
            print(self.output['data'].decode('utf-8'))

    def send(self):
        """
        Send the request defined by the data stored in the object.
        """
        if self.verbose:
            self.print_out()
            return Response(self.action, request(**self.output))
        else:
            return Response(self.action, request(**self.output)).read()

    def set(self, variable):
        """
        Set the received base-level variable in the object
        """
        method_map = {
            'url_param': self.set_url_param,
            'header': self.set_header,
            'url_replacement': self.set_url_replacement,
            'data': self.set_data
        }
        if variable['type'] in method_map:
            method_map[variable['type']](variable)
        else:
            raise Exception('Cannot handle final variables of type {}'.format(variable['type']))

    def set_header(self, header):
        """
        Set a base-level header variable to have the given value
        """
        self.output['headers'][header['name']] = header['value']

    def set_data(self, data):
        """
        Set the base-level data variable to contain the given
        data, and also set the Content-Type header to the relevant
        value.
        """
        self.output['data'] = data['data']

    def set_url_param(self, param):
        """
        Set the base-level URL parameter variable to have the given value
        """
        self.output['url'] += '{}={}&'.format(param['name'], param['value'])

    def set_url_replacement(self, rep):
        url = self.output['url']
        url = str(rep['value']).join(url.split('{{{}}}'.format(rep['name'])))
        self.output['url'] = url

class Response(object):

    """
    Stores data and provides methods related to the response that
    we get back from the API provider's server
    """

    def __init__(self, action, response):
        self.action = action
        self.headers = response.headers
        self.data = response.read()
        self.code = response.getcode()

    def mimetype(self):
        """
        Get the Content-Type header from the response. Strip
        the ";charset=xxxxx" portion if necessary. If we can't
        find it, use the predefined format.
        """
        if ';' in self.headers.get('Content-Type', ''):
            return self.headers['Content-Type'].split(';')[0]
        return self.headers.get('Content-Type', self.format())

    def format(self):
        """
        Get the hard-defined format of the parent action object
        """
        return self.action.format(direction='returns')

    def encoding(self):
        """
        Look for a "charset=" variable in the Content-Type header;
        if it's not there, just return a default value of UTF-8
        """
        if 'charset=' in self.headers.get('Content-Type', ''):
            return self.headers['Content-Type'].split('charset=')[1].split(';')[0]
        return 'utf-8'

    def cookies(self):
        """
        Get a list from the Set-Cookie header; if there's nothing
        there, return an empty list.
        """
        if self.headers.get('Set-Cookie', False):
            return self.headers.get('Set-Cookie').split('; ')
        return []

    def read(self):
        """
        Parse the body of the response using the Content-Type
        header we pulled from the response, or the hive-defined
        format, if such couldn't be pulled automatically.
        """
        return decode(self.data, self.mimetype(), encoding=self.encoding())
<commit_msg>Using urlencode in params to ensure special characters get properly escaped<commit_after>"""
Provides classes and methods related to communicating with the remote API
"""

from __future__ import absolute_import, division
from __future__ import unicode_literals, print_function

try:
    from urllib2 import Request as PythonRequest, urlopen, HTTPError
    from urllib import urlencode
except ImportError:
    from urllib.request import Request as PythonRequest, urlopen
    from urllib.error import HTTPError
    from urllib.parse import urlencode

import json

from beekeeper.variable_handlers import render
from beekeeper.data_handlers import decode

def download_as_json(url):
    """
    Download the data at the URL and load it as JSON
    """
    return json.loads(request(url=url).read().decode('utf-8'))

def request(*args, **kwargs):
    """
    Make a request with the received arguments and return an
    HTTPResponse object
    """
    req = PythonRequest(*args, **kwargs)
    return urlopen(req)

class Request(object):

    """
    Holds data and provides methods related to building and sending
    an HTTP request with the data passed to it
    """

    def __init__(self, action, variables, verbose=False):
        self.action = action
        self.variables = variables
        self.verbose = verbose
        self.output = {}
        self.output['data'] = None
        self.output['headers'] = {}
        self.output['url'] = self.action.endpoint.url() + '?'
        self.render_variables()

    def render_variables(self):
        """
        Take the variables passed in during init and parse them
        into the base level variables we can actually do stuff with.
        """
        for var_type in self.variables.types():
            for var in render(var_type, **self.variables.vals(var_type)):
                self.set(var)
        self.output['url'] = self.output['url'][:len(self.output['url'])-1]

    def print_out(self):
        """
        Print out the three base level variables of the request.
        """
        print('URL: {}'.format(self.output['url']))
        print('Headers:')
        if self.output['headers']:
            for name, val in self.output['headers'].items():
                print('{}: {}'.format(name, val))
        else:
            print('None')
        print('Data:')
        if self.output['data']:
            print(self.output['data'].decode('utf-8'))

    def send(self):
        """
        Send the request defined by the data stored in the object.
        """
        if self.verbose:
            self.print_out()
            return Response(self.action, request(**self.output))
        else:
            return Response(self.action, request(**self.output)).read()

    def set(self, variable):
        """
        Set the received base-level variable in the object
        """
        method_map = {
            'url_param': self.set_url_param,
            'header': self.set_header,
            'url_replacement': self.set_url_replacement,
            'data': self.set_data
        }
        if variable['type'] in method_map:
            method_map[variable['type']](variable)
        else:
            raise Exception('Cannot handle final variables of type {}'.format(variable['type']))

    def set_header(self, header):
        """
        Set a base-level header variable to have the given value
        """
        self.output['headers'][header['name']] = header['value']

    def set_data(self, data):
        """
        Set the base-level data variable to contain the given
        data, and also set the Content-Type header to the relevant
        value.
        """
        self.output['data'] = data['data']

    def set_url_param(self, param):
        """
        Set the base-level URL parameter variable to have the given value
        """
        self.output['url'] += urlencode({param['name']: param['value']}) + '&'

    def set_url_replacement(self, rep):
        url = self.output['url']
        url = str(rep['value']).join(url.split('{{{}}}'.format(rep['name'])))
        self.output['url'] = url

class Response(object):

    """
    Stores data and provides methods related to the response that
    we get back from the API provider's server
    """

    def __init__(self, action, response):
        self.action = action
        self.headers = response.headers
        self.data = response.read()
        self.code = response.getcode()

    def mimetype(self):
        """
        Get the Content-Type header from the response. Strip
        the ";charset=xxxxx" portion if necessary. If we can't
        find it, use the predefined format.
        """
        if ';' in self.headers.get('Content-Type', ''):
            return self.headers['Content-Type'].split(';')[0]
        return self.headers.get('Content-Type', self.format())

    def format(self):
        """
        Get the hard-defined format of the parent action object
        """
        return self.action.format(direction='returns')

    def encoding(self):
        """
        Look for a "charset=" variable in the Content-Type header;
        if it's not there, just return a default value of UTF-8
        """
        if 'charset=' in self.headers.get('Content-Type', ''):
            return self.headers['Content-Type'].split('charset=')[1].split(';')[0]
        return 'utf-8'

    def cookies(self):
        """
        Get a list from the Set-Cookie header; if there's nothing
        there, return an empty list.
        """
        if self.headers.get('Set-Cookie', False):
            return self.headers.get('Set-Cookie').split('; ')
        return []

    def read(self):
        """
        Parse the body of the response using the Content-Type
        header we pulled from the response, or the hive-defined
        format, if such couldn't be pulled automatically.
        """
        return decode(self.data, self.mimetype(), encoding=self.encoding())
<|endoftext|>


<commit_before>#
# Copyright (c) 2016 Christoph Heiss <me@christoph-heiss.me>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#

import os
import socket
import sqlite3
import threading
from functools import wraps
from watchdog.observers import Observer
from watchdog.events import PatternMatchingEventHandler
from behem0th import utils
from behem0th.RequestHandler import RequestHandler

IGNORE_LIST = [
	'.git',
	'.DS_Store'
]

DEFAULT_PORT = 3078

def synchronized(fn):
	@wraps(fn)
	def wrap(*args, **kwargs):
		lock = args[0]._rlock
		with lock:
			return fn(*args, **kwargs)

	return wrap

class _FsEventHandler(PatternMatchingEventHandler):
	def __init__(self, client):
		super().__init__(ignore_patterns=client._ignore_list)
		self._client = client

	def on_any_event(self, event):
		# Ignore DirModifiedEvent's completely.
		if event.event_type == 'modified' and event.is_directory:
			return

		self._client._handle_fsevent(event)

class _AcceptWorker(threading.Thread):
	def __init__(self, **kwargs):
		super().__init__()
		self.name = 'accept-worker'
		self.kwargs = kwargs
		self.daemon = True

	def run(self):
		client = self.kwargs['client']

		accept_sock = socket.socket()
		accept_sock.bind(self.kwargs['address'])
		accept_sock.listen()

		while 1:
			sock, address = accept_sock.accept()
			RequestHandler(sock=sock, address=address, client=client).start()

class Client:
	"""The main interface for behem0th

	This class can either act as a client which connects to a behem0th server
	or as a server to which behem0th clients can connect.

	Parameters
	----------
	path : :obj:`str`, optional
		The path which should be sync'd.
	folder : :obj:`folder`, optional
		The name of the folder which will contain behem0th-only
		data needed for syncing.

		behem0th will _not_ sync this folder!

	"""

	def __init__(self, path='.', folder='.behem0th'):
		self._sock = None
		self._rlock = threading.RLock()
		self._peers = []

		self._sync_path = os.path.abspath(path)
		self._meta_folder = folder

		path = os.path.join(self._sync_path, self._meta_folder)
		if not os.path.exists(path):
			os.mkdir(path, 0o755)

		self._ignore_list = IGNORE_LIST + [self._meta_folder]

		self._filelist = {}
		self._observer = Observer()
		self._observer.schedule(_FsEventHandler(self), self._sync_path, recursive=True)

		self._fsevent_ignore_list = []

	def connect(self, host, port=DEFAULT_PORT):
		"""Connects to a behem0th server

		Parameters
		----------
		host : :obj:`str`
			Hostname/IP of the behem0th server

		port : :obj:`int`, optional
			Port of the behem0th server

		"""

		self._collect_files()

		address = (host, port)
		self._sock = socket.socket()
		self._sock.connect(address)
		self._server = RequestHandler(sock=self._sock, address=address, client=self)
		self._server.start()
		self._observer.start()

	def listen(self, port=DEFAULT_PORT):
		"""Starts a behem0th server instance

		Parameters
		----------
		port : :obj:`int`, optional
			Port on which the server should be started

		"""

		self._collect_files()

		_AcceptWorker(address=('0.0.0.0', port), client=self).start()
		self._observer.start()

	def close(self):
		"""Closes all request handlers, writes the sync cache and shuts
		down the client.

		"""

		self._run_on_peers('close')

		self._observer.stop()
		self._write_db()

	@synchronized
	def get_files(self):
		"""
		Returns
		-------
		:obj:`list`
			A list of all currently sync'd files.

		"""

		return [
			{'path': f['path'], 'type': f['type']}
				for f in self._filelist.values()
		]

	@synchronized
	def get_peers(self):
		"""
		Returns
		-------
		:obj:`list`
			A list of IP-address of all currently connected devices

		"""
		return [p.address for p in self._peers]

	@synchronized
	def _collect_files(self):
		for root, dirs, files in os.walk(self._sync_path):
			files[:] = [f for f in files if f not in self._ignore_list]
			dirs[:] = [d for d in dirs if d not in self._ignore_list]

			relpath = os.path.relpath(root, self._sync_path)

			for name in files:
				self._add_to_filelist(os.path.join(relpath, name), 'file')

			for name in dirs:
				self._add_to_filelist(os.path.join(relpath, name), 'dir')

	@synchronized
	def _merge_filelist(self, filelist):
		files = []
		events = []

		for file, info in filelist.items():
			if not file in self._filelist:
				self._add_to_filelist(file, info['type'])
				self._ignore_next_fsevent(file)

				if info['type'] == 'file':
					open(file, 'a').close()
					files.append(('request', file))
				else:
					os.mkdir(file, 0o755)

		for file, info in self._filelist.items():
			if not file in filelist:
				if info['type'] == 'file':
					events.append({'type': 'file-created', 'path': file})
					files.append(('send', file))
				else:
					events.append({'type': 'dir-created', 'path': file})

		return (files, events)

	@synchronized
	def _add_to_filelist(self, path, type):
		path = os.path.normpath(path)
		self._filelist[path] = {'path': path, 'type': type}

	@synchronized
	def _remove_from_filelist(self, path):
		del self._filelist[os.path.normpath(path)]

	@synchronized
	def _handle_fsevent(self, evt):
		path = os.path.relpath(evt.src_path, self._sync_path)
		type = 'dir' if evt.is_directory else 'file'

		if path in self._fsevent_ignore_list:
			self._fsevent_ignore_list.remove(path)
			return

		remote_event = {'type': type + '-' + evt.event_type, 'path': path}

		if evt.event_type == 'created':
			self._add_to_filelist(path, type)

		elif evt.event_type == 'deleted':
			self._remove_from_filelist(path)

		elif evt.event_type == 'moved':
			self._remove_from_filelist(path)

			path = os.path.relpath(evt.dest_path, self._sync_path)
			self._add_to_filelist(path, type)
			remote_event['dest'] = path

		if evt.event_type != 'modified':
			self._run_on_peers('queue_event', remote_event)
		elif type == 'file':
			self._run_on_peers('queue_file', 'send', path)

	@synchronized
	def _ignore_next_fsevent(self, path):
		self._fsevent_ignore_list.append(path)

	@synchronized
	def _run_on_peers(self, method, *args, **kwargs):
		for peer in self._peers:
			getattr(peer, method)(*args, **kwargs)

	@synchronized
	def _write_db(self):
		file = os.path.join(self._meta_folder, 'file_index.db')
		try:
			os.remove(file)
		except OSError:
			pass

		conn = sqlite3.connect(file)

		conn.execute('''
			create table files (
				hash text primary key,
				path text not null
			)
		''')

		for file, info in self._filelist.items():
			if info['type'] == 'dir':
				continue

			try:
				conn.execute(
					'insert into files(hash, path) values (?, ?)',
					(utils.hash_file(file), file)
				)
			except FileNotFoundError:
				utils.log(
					"File '{0}' not found - possibly not fully sync'd!\n"
					"(This can normally be safely ignored - as soon as you reconnect, this device will sync up.)",
				file)

		conn.commit()
		conn.close()
<commit_msg>Account for different watchdog-events between Linux and macOS.<commit_after>#
# Copyright (c) 2016 Christoph Heiss <me@christoph-heiss.me>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#

import os
import sys
import socket
import sqlite3
import threading
from functools import wraps
from watchdog.observers import Observer
from watchdog.events import PatternMatchingEventHandler, FileModifiedEvent
from behem0th import utils
from behem0th.RequestHandler import RequestHandler

IGNORE_LIST = [
	'.git',
	'.DS_Store'
]

DEFAULT_PORT = 3078

def synchronized(fn):
	@wraps(fn)
	def wrap(*args, **kwargs):
		lock = args[0]._rlock
		with lock:
			return fn(*args, **kwargs)

	return wrap

class _FsEventHandler(PatternMatchingEventHandler):
	def __init__(self, client):
		super().__init__(ignore_patterns=client._ignore_list)
		self._client = client

	def on_any_event(self, event):
		# Ignore DirModifiedEvent's completely.
		if event.event_type == 'modified' and event.is_directory:
			return

		event_handled = self._client._handle_fsevent(event)

		# On macOS, watchdog only produces a file-created event,
		# on Linux however also a file-modified event is generated.
		if sys.platform == 'darwin':
			if event_handled and event.event_type == 'created' and not event.is_directory:
				self._client._handle_fsevent(FileModifiedEvent(event.src_path))

class _AcceptWorker(threading.Thread):
	def __init__(self, **kwargs):
		super().__init__()
		self.name = 'accept-worker'
		self.kwargs = kwargs
		self.daemon = True

	def run(self):
		client = self.kwargs['client']

		accept_sock = socket.socket()
		accept_sock.bind(self.kwargs['address'])
		accept_sock.listen()

		while 1:
			sock, address = accept_sock.accept()
			RequestHandler(sock=sock, address=address, client=client).start()

class Client:
	"""The main interface for behem0th

	This class can either act as a client which connects to a behem0th server
	or as a server to which behem0th clients can connect.

	Parameters
	----------
	path : :obj:`str`, optional
		The path which should be sync'd.
	folder : :obj:`folder`, optional
		The name of the folder which will contain behem0th-only
		data needed for syncing.

		behem0th will _not_ sync this folder!

	"""

	def __init__(self, path='.', folder='.behem0th'):
		self._sock = None
		self._rlock = threading.RLock()
		self._peers = []

		self._sync_path = os.path.abspath(path)
		self._meta_folder = folder

		path = os.path.join(self._sync_path, self._meta_folder)
		if not os.path.exists(path):
			os.mkdir(path, 0o755)

		self._ignore_list = IGNORE_LIST + [self._meta_folder]

		self._filelist = {}
		self._observer = Observer()
		self._observer.schedule(_FsEventHandler(self), self._sync_path, recursive=True)

		self._fsevent_ignore_list = []

	def connect(self, host, port=DEFAULT_PORT):
		"""Connects to a behem0th server

		Parameters
		----------
		host : :obj:`str`
			Hostname/IP of the behem0th server

		port : :obj:`int`, optional
			Port of the behem0th server

		"""

		self._collect_files()

		address = (host, port)
		self._sock = socket.socket()
		self._sock.connect(address)
		self._server = RequestHandler(sock=self._sock, address=address, client=self)
		self._server.start()
		self._observer.start()

	def listen(self, port=DEFAULT_PORT):
		"""Starts a behem0th server instance

		Parameters
		----------
		port : :obj:`int`, optional
			Port on which the server should be started

		"""

		self._collect_files()

		_AcceptWorker(address=('0.0.0.0', port), client=self).start()
		self._observer.start()

	def close(self):
		"""Closes all request handlers, writes the sync cache and shuts
		down the client.

		"""

		self._run_on_peers('close')

		self._observer.stop()
		self._write_db()

	@synchronized
	def get_files(self):
		"""
		Returns
		-------
		:obj:`list`
			A list of all currently sync'd files.

		"""

		return [
			{'path': f['path'], 'type': f['type']}
				for f in self._filelist.values()
		]

	@synchronized
	def get_peers(self):
		"""
		Returns
		-------
		:obj:`list`
			A list of IP-address of all currently connected devices

		"""
		return [p.address for p in self._peers]

	@synchronized
	def _collect_files(self):
		for root, dirs, files in os.walk(self._sync_path):
			files[:] = [f for f in files if f not in self._ignore_list]
			dirs[:] = [d for d in dirs if d not in self._ignore_list]

			relpath = os.path.relpath(root, self._sync_path)

			for name in files:
				self._add_to_filelist(os.path.join(relpath, name), 'file')

			for name in dirs:
				self._add_to_filelist(os.path.join(relpath, name), 'dir')

	@synchronized
	def _merge_filelist(self, filelist):
		files = []
		events = []

		for file, info in filelist.items():
			if not file in self._filelist:
				self._add_to_filelist(file, info['type'])
				self._ignore_next_fsevent(file)

				if info['type'] == 'file':
					open(file, 'a').close()
					files.append(('request', file))
				else:
					os.mkdir(file, 0o755)

		for file, info in self._filelist.items():
			if not file in filelist:
				if info['type'] == 'file':
					events.append({'type': 'file-created', 'path': file})
					files.append(('send', file))
				else:
					events.append({'type': 'dir-created', 'path': file})

		return (files, events)

	@synchronized
	def _add_to_filelist(self, path, type):
		path = os.path.normpath(path)
		self._filelist[path] = {'path': path, 'type': type}

	@synchronized
	def _remove_from_filelist(self, path):
		del self._filelist[os.path.normpath(path)]

	@synchronized
	def _handle_fsevent(self, evt):
		path = os.path.relpath(evt.src_path, self._sync_path)
		type = 'dir' if evt.is_directory else 'file'

		if path in self._fsevent_ignore_list:
			self._fsevent_ignore_list.remove(path)
			return False

		remote_event = {'type': type + '-' + evt.event_type, 'path': path}

		if evt.event_type == 'created':
			self._add_to_filelist(path, type)

		elif evt.event_type == 'deleted':
			self._remove_from_filelist(path)

		elif evt.event_type == 'moved':
			self._remove_from_filelist(path)

			path = os.path.relpath(evt.dest_path, self._sync_path)
			self._add_to_filelist(path, type)
			remote_event['dest'] = path

		if evt.event_type != 'modified':
			self._run_on_peers('queue_event', remote_event)
		elif type == 'file':
			self._run_on_peers('queue_file', 'send', path)

		return True

	@synchronized
	def _ignore_next_fsevent(self, path):
		self._fsevent_ignore_list.append(path)

	@synchronized
	def _run_on_peers(self, method, *args, **kwargs):
		for peer in self._peers:
			getattr(peer, method)(*args, **kwargs)

	@synchronized
	def _write_db(self):
		file = os.path.join(self._meta_folder, 'file_index.db')
		try:
			os.remove(file)
		except OSError:
			pass

		conn = sqlite3.connect(file)

		conn.execute('''
			create table files (
				hash text primary key,
				path text not null
			)
		''')

		for file, info in self._filelist.items():
			if info['type'] == 'dir':
				continue

			try:
				conn.execute(
					'insert into files(hash, path) values (?, ?)',
					(utils.hash_file(file), file)
				)
			except FileNotFoundError:
				utils.log(
					"File '{0}' not found - possibly not fully sync'd!\n"
					"(This can normally be safely ignored - as soon as you reconnect, this device will sync up.)",
				file)

		conn.commit()
		conn.close()
<|endoftext|>


<commit_before>VERSION="0.4.5.5"
<commit_msg>Version bump<commit_after>VERSION="0.4.6"
<|endoftext|>


<commit_before>#! /usr/bin/env python
from __future__ import print_function
# from bfg.utils import min_lexo
from bfg.utils import seq_to_kmers
from bfg.graph import ProbabilisticMultiColourDeBruijnGraph as Graph
import argparse
import os.path
import time
from Bio import SeqIO
import json
import logging

logger = logging.getLogger(__name__)
from bfg.utils import DEFAULT_LOGGING_LEVEL
logger.setLevel(DEFAULT_LOGGING_LEVEL)

def per(i):
    return float(sum(i))/len(i)

def parse_input(infile):
    gene_to_kmers = {}
    with open(infile, 'r') as inf:
        for record in SeqIO.parse(inf, 'fasta'):
            gene_to_kmers[record.id] = str(record.seq)
            yield (record.id, str(record.seq))
    # return gene_to_kmers

def _search(gene_name, seq, results, threshold, graph, output_format="json", pipe=False):
    if pipe:
        if output_format == "tsv":
            start = time.time()
            result = graph.search(seq, threshold=threshold)
            diff = time.time() - start
            if result:
                for sample_id, percent in result.items():
                    print(
                        "\t".join([gene_name, sample_id, str(percent), str(diff)]))
            else:
                print("\t".join([gene_name, "NA", str(0), str(diff)]))
        elif output_format == "fasta":
            samples = graph.sample_to_colour_lookup.keys()
            print(" ".join(['>', gene_name]))
            print(seq)
            for colour in range(graph.get_num_colours()):
                sample = graph.colour_to_sample_lookup(colour)
                print(
                    " ".join(['>', gene_name, sample, "kmer-%i coverage" % graph.kmer_size]))
                presence = []
                for kmer in seq_to_kmers(seq, graph.kmer_size):
                    kmer_presence = graph.lookup(kmer)[colour]
                    if kmer_presence:
                        presence.append("1")
                    else:
                        presence.append("0")
                print("".join(presence))
        else:
            result = {}
            start = time.time()
            result['results'] = graph.search(seq, threshold=threshold)
            diff = time.time() - start
            result['time'] = diff
            print(json.dumps({gene_name: result}))
    else:
        results[gene_name] = {}
        start = time.time()
        results[gene_name]['results'] = graph.search(seq, threshold=threshold)
        diff = time.time() - start
        results[gene_name]['time'] = diff
    return results

def search(seq, fasta_file, threshold, graph, output_format="json", pipe=False):
    if output_format == "tsv":
        print("\t".join(
            ["gene_name", "sample_id", str("kmer_coverage_percent"), str("time")]))
    results = {}
    if fasta_file is not None:
        for gene, seq in parse_input(fasta_file):
            results = _search(
                gene_name=gene, seq=seq, results=results, threshold=threshold,
                graph=graph, output_format=output_format, pipe=pipe)
    else:
        results = _search(
            gene_name=seq, seq=seq, results=results, threshold=threshold,
            graph=graph, output_format=output_format, pipe=pipe)
    return results
<commit_msg>Don't do look up on fasta output<commit_after>#! /usr/bin/env python
from __future__ import print_function
# from bfg.utils import min_lexo
from bfg.utils import seq_to_kmers
from bfg.graph import ProbabilisticMultiColourDeBruijnGraph as Graph
import argparse
import os.path
import time
from Bio import SeqIO
import json
import logging

logger = logging.getLogger(__name__)
from bfg.utils import DEFAULT_LOGGING_LEVEL
logger.setLevel(DEFAULT_LOGGING_LEVEL)

def per(i):
    return float(sum(i))/len(i)

def parse_input(infile):
    gene_to_kmers = {}
    with open(infile, 'r') as inf:
        for record in SeqIO.parse(inf, 'fasta'):
            gene_to_kmers[record.id] = str(record.seq)
            yield (record.id, str(record.seq))
    # return gene_to_kmers

def _search(gene_name, seq, results, threshold, graph, output_format="json", pipe=False):
    if pipe:
        if output_format == "tsv":
            start = time.time()
            result = graph.search(seq, threshold=threshold)
            diff = time.time() - start
            if result:
                for sample_id, percent in result.items():
                    print(
                        "\t".join([gene_name, sample_id, str(percent), str(diff)]))
            else:
                print("\t".join([gene_name, "NA", str(0), str(diff)]))
        elif output_format == "fasta":
            samples = graph.sample_to_colour_lookup.keys()
            print(" ".join(['>', gene_name]))
            print(seq)
            for colour in range(graph.get_num_colours()):
                sample = graph.colour_to_sample_lookup.get(colour)
                print(
                    " ".join(['>', gene_name, sample, "kmer-%i coverage" % graph.kmer_size]))
                presence = []
                for kmer in seq_to_kmers(seq, graph.kmer_size):
                    kmer_presence = graph.lookup(kmer)[colour]
                    if kmer_presence:
                        presence.append("1")
                    else:
                        presence.append("0")
                print("".join(presence))
        else:
            result = {}
            start = time.time()
            result['results'] = graph.search(seq, threshold=threshold)
            diff = time.time() - start
            result['time'] = diff
            print(json.dumps({gene_name: result}))
    else:
        results[gene_name] = {}
        start = time.time()
        results[gene_name]['results'] = graph.search(seq, threshold=threshold)
        diff = time.time() - start
        results[gene_name]['time'] = diff
    return results

def search(seq, fasta_file, threshold, graph, output_format="json", pipe=False):
    if output_format == "tsv":
        print("\t".join(
            ["gene_name", "sample_id", str("kmer_coverage_percent"), str("time")]))
    results = {}
    if fasta_file is not None:
        for gene, seq in parse_input(fasta_file):
            results = _search(
                gene_name=gene, seq=seq, results=results, threshold=threshold,
                graph=graph, output_format=output_format, pipe=pipe)
    else:
        results = _search(
            gene_name=seq, seq=seq, results=results, threshold=threshold,
            graph=graph, output_format=output_format, pipe=pipe)
    return results
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# BrightstarDB documentation build configuration file, created by
# sphinx-quickstart on Fri May 24 14:04:08 2013.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys, os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.todo', 'sphinx.ext.ifconfig']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'BrightstarDB'
copyright = u'2014, Kal Ahmed, Graham Moore, and other contributors'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '1.12'
# The full version, including alpha/beta/rc tags.
release = '1.12.0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'alabaster'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'BrightstarDBdoc'

# -- Options for LaTeX output --------------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('index', 'BrightstarDB.tex', u'BrightstarDB Documentation',
   u'Kal Ahmed, Graham Moore', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'brightstardb', u'BrightstarDB Documentation',
     [u'Kal Ahmed, Graham Moore'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output ------------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'BrightstarDB', u'BrightstarDB Documentation',
   u'Kal Ahmed, Graham Moore', 'BrightstarDB', 'Native Triple Store for .NET.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

# -- Options for Epub output ---------------------------------------------------

# Bibliographic Dublin Core info.
epub_title = u'BrightstarDB'
epub_author = u'Kal Ahmed, Graham Moore'
epub_publisher = u'Kal Ahmed, Graham Moore'
epub_copyright = u'2015, Kal Ahmed, Graham Moore'

# The language of the text. It defaults to the language option
# or en if the language is not set.
#epub_language = ''

# The scheme of the identifier. Typical schemes are ISBN or URL.
#epub_scheme = ''

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#epub_identifier = ''

# A unique identification for the text.
#epub_uid = ''

# A tuple containing the cover image and cover page html template filenames.
#epub_cover = ()

# A sequence of (type, uri, title) tuples for the guide element of content.opf.
#epub_guide = ()

# HTML files that should be inserted before the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_pre_files = []

# HTML files shat should be inserted after the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_post_files = []

# A list of files that should not be packed into the epub file.
#epub_exclude_files = []

# The depth of the table of contents in toc.ncx.
#epub_tocdepth = 3

# Allow duplicate toc entries.
#epub_tocdup = True

# Fix unsupported image types using the PIL.
#epub_fix_images = False

# Scale large images.
#epub_max_image_width = 0

# If 'no', URL addresses will not be shown.
#epub_show_urls = 'inline'

# If false, no index is generated.
#epub_use_index = True
<commit_msg>Fixed copyright year<commit_after># -*- coding: utf-8 -*-
#
# BrightstarDB documentation build configuration file, created by
# sphinx-quickstart on Fri May 24 14:04:08 2013.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys, os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.todo', 'sphinx.ext.ifconfig']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'BrightstarDB'
copyright = u'2015, Kal Ahmed, Graham Moore, and other contributors'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '1.12'
# The full version, including alpha/beta/rc tags.
release = '1.12.0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'alabaster'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'BrightstarDBdoc'

# -- Options for LaTeX output --------------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('index', 'BrightstarDB.tex', u'BrightstarDB Documentation',
   u'Kal Ahmed, Graham Moore', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'brightstardb', u'BrightstarDB Documentation',
     [u'Kal Ahmed, Graham Moore'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output ------------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'BrightstarDB', u'BrightstarDB Documentation',
   u'Kal Ahmed, Graham Moore', 'BrightstarDB', 'Native Triple Store for .NET.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

# -- Options for Epub output ---------------------------------------------------

# Bibliographic Dublin Core info.
epub_title = u'BrightstarDB'
epub_author = u'Kal Ahmed, Graham Moore'
epub_publisher = u'Kal Ahmed, Graham Moore'
epub_copyright = u'2015, Kal Ahmed, Graham Moore'

# The language of the text. It defaults to the language option
# or en if the language is not set.
#epub_language = ''

# The scheme of the identifier. Typical schemes are ISBN or URL.
#epub_scheme = ''

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#epub_identifier = ''

# A unique identification for the text.
#epub_uid = ''

# A tuple containing the cover image and cover page html template filenames.
#epub_cover = ()

# A sequence of (type, uri, title) tuples for the guide element of content.opf.
#epub_guide = ()

# HTML files that should be inserted before the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_pre_files = []

# HTML files shat should be inserted after the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_post_files = []

# A list of files that should not be packed into the epub file.
#epub_exclude_files = []

# The depth of the table of contents in toc.ncx.
#epub_tocdepth = 3

# Allow duplicate toc entries.
#epub_tocdup = True

# Fix unsupported image types using the PIL.
#epub_fix_images = False

# Scale large images.
#epub_max_image_width = 0

# If 'no', URL addresses will not be shown.
#epub_show_urls = 'inline'

# If false, no index is generated.
#epub_use_index = True
<|endoftext|>