<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-

"""
Convert delly output to BED format.
"""

import os.path as op
import sys
import logging

from jcvi.formats.base import BaseFile, read_until, must_open
from jcvi.apps.base import OptionParser, ActionDispatcher, sh, need_update

class DelLine (object):

    def __init__(self, line):
        args = line.strip().split("\t")
        self.seqid = args[0]
        self.start = int(args[1]) + 1
        self.end = int(args[2])
        self.size = int(args[3])
        assert self.size == self.end - self.start + 1
        self.supporting_pairs = int(args[4])
        self.avg_mapping_quality = float(args[5])
        self.accn = args[6]

    @property
    def bedline(self):
        return "\t".join(str(x) for x in (self.seqid,
                          self.start - 1, self.end, self.accn,
                          self.supporting_pairs, "+"))

class Delly (BaseFile):

    def __init__(self, filename):
        super(Delly, self).__init__(filename)

    def __iter__(self):
        fp = must_open(self.filename)
        while True:
            read_until(fp, "-----")
            nextline = fp.readline()
            nextline = fp.readline()
            if not nextline.strip():
                break
            d = DelLine(nextline)
            yield d

    def write_bed(self, bedfile="stdout"):
        fw = must_open(bedfile, "w")
        for d in self:
            print >> fw, d.bedline
        logging.debug("File written to `{0}`.".format(bedfile))

def main():

    actions = (
        ('bed', 'Convert del.txt to del.bed'),
        ('mito', 'Find mito deletions in BAM'),
        ('mitocompile', 'Compile mito deletions from multiple VCF files'),
            )
    p = ActionDispatcher(actions)
    p.dispatch(globals())

def bed(args):
    """
    %prog bed del.txt

    Convert `del.txt` to BED format. DELLY manual here:
    <http://www.embl.de/~rausch/delly.html>

    Deletion:
    chr, start, end, size, #supporting_pairs, avg._mapping_quality, deletion_id
    chr1, 10180, 10509, 329, 75, 15.8667, Deletion_Sample_00000000
    """
    p = OptionParser(bed.__doc__)
    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(not p.print_help())

    delt, = args
    dt = Delly(delt)
    dt.write_bed("del.bed")

def mitocompile(args):
    """
    %prog mitcompile *.vcf.gz

    Extract information about deletions in vcf file.
    """
    from jcvi.formats.vcf import VcfLine
    from urlparse import parse_qsl

    p = OptionParser(mitocompile.__doc__)
    opts, args = p.parse_args(args)

    if len(args) < 1:
        sys.exit(not p.print_help())

    vcfs = args
    print "\t".join("vcf seqid pos alt pe sr".split())
    for vcf in vcfs:
        fp = must_open(vcf)
        for row in fp:
            if row[0] == '#':
                continue
            v = VcfLine(row)
            info = dict(parse_qsl(v.info))
            print "\t".join(str(x) for x in (vcf, v.seqid, v.pos, v.alt,
                        info["PE"], info["SR"]))

def mito(args):
    """
    %prog mito chrM.fa input.bam

    Identify mitochondrial deletions.
    """
    p = OptionParser(mito.__doc__)
    p.set_cpus()
    opts, args = p.parse_args(args)

    if len(args) != 2:
        sys.exit(not p.print_help())

    chrMfa, bamfile = args
    if not bamfile.endswith(".bam"):
        bamfiles = [x.strip() for x in open(bamfile)]
    else:
        bamfiles = [bamfile]

    for bamfile in bamfiles:
        run_mito(chrMfa, bamfile, opts)

def run_mito(chrMfa, bamfile, opts):
    from jcvi.formats.sam import get_minibam
    region = "chrM"
    minibam = op.basename(bamfile).replace(".bam", ".{}.bam".format(region))
    if not op.exists(minibam):
        get_minibam(bamfile, region)

    realign = minibam.rsplit(".", 1)[0] + ".realign"
    margs = " -v -t {} -o {}".format(opts.cpus, realign)
    if need_update(minibam, realign):
        cmd = "speedseq realign"
        cmd += margs
        cmd += " {} {}".format(chrMfa, minibam)
        sh(cmd)

    vcffile = realign + ".sv.vcf.gz"
    if need_update(realign, vcffile):
        cmd = "speedseq sv"
        cmd += margs
        cmd += " -g -R {}".format(chrMfa)
        cmd += " -B {} -D {} -S {}".format(realign + ".bam",
                        realign + ".discordants.bam", realign + ".splitters.bam")
        sh(cmd)

if __name__ == '__main__':
    main()
<commit_msg>[variation] add SVLEN field in delly<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-

"""
Convert delly output to BED format.
"""

import os.path as op
import sys
import logging

from jcvi.formats.base import BaseFile, read_until, must_open
from jcvi.apps.base import OptionParser, ActionDispatcher, sh, need_update

class DelLine (object):

    def __init__(self, line):
        args = line.strip().split("\t")
        self.seqid = args[0]
        self.start = int(args[1]) + 1
        self.end = int(args[2])
        self.size = int(args[3])
        assert self.size == self.end - self.start + 1
        self.supporting_pairs = int(args[4])
        self.avg_mapping_quality = float(args[5])
        self.accn = args[6]

    @property
    def bedline(self):
        return "\t".join(str(x) for x in (self.seqid,
                          self.start - 1, self.end, self.accn,
                          self.supporting_pairs, "+"))

class Delly (BaseFile):

    def __init__(self, filename):
        super(Delly, self).__init__(filename)

    def __iter__(self):
        fp = must_open(self.filename)
        while True:
            read_until(fp, "-----")
            nextline = fp.readline()
            nextline = fp.readline()
            if not nextline.strip():
                break
            d = DelLine(nextline)
            yield d

    def write_bed(self, bedfile="stdout"):
        fw = must_open(bedfile, "w")
        for d in self:
            print >> fw, d.bedline
        logging.debug("File written to `{0}`.".format(bedfile))

def main():

    actions = (
        ('bed', 'Convert del.txt to del.bed'),
        ('mito', 'Find mito deletions in BAM'),
        ('mitocompile', 'Compile mito deletions from multiple VCF files'),
            )
    p = ActionDispatcher(actions)
    p.dispatch(globals())

def bed(args):
    """
    %prog bed del.txt

    Convert `del.txt` to BED format. DELLY manual here:
    <http://www.embl.de/~rausch/delly.html>

    Deletion:
    chr, start, end, size, #supporting_pairs, avg._mapping_quality, deletion_id
    chr1, 10180, 10509, 329, 75, 15.8667, Deletion_Sample_00000000
    """
    p = OptionParser(bed.__doc__)
    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(not p.print_help())

    delt, = args
    dt = Delly(delt)
    dt.write_bed("del.bed")

def mitocompile(args):
    """
    %prog mitcompile *.vcf.gz

    Extract information about deletions in vcf file.
    """
    from jcvi.formats.vcf import VcfLine
    from urlparse import parse_qsl

    p = OptionParser(mitocompile.__doc__)
    opts, args = p.parse_args(args)

    if len(args) < 1:
        sys.exit(not p.print_help())

    vcfs = args
    print "\t".join("vcf seqid pos alt svlen pe sr".split())
    for vcf in vcfs:
        fp = must_open(vcf)
        for row in fp:
            if row[0] == '#':
                continue
            v = VcfLine(row)
            info = dict(parse_qsl(v.info))
            print "\t".join(str(x) for x in (vcf, v.seqid, v.pos, v.alt,
                        info.get("SVLEN"), info["PE"], info["SR"]))

def mito(args):
    """
    %prog mito chrM.fa input.bam

    Identify mitochondrial deletions.
    """
    p = OptionParser(mito.__doc__)
    p.set_cpus()
    opts, args = p.parse_args(args)

    if len(args) != 2:
        sys.exit(not p.print_help())

    chrMfa, bamfile = args
    if not bamfile.endswith(".bam"):
        bamfiles = [x.strip() for x in open(bamfile)]
    else:
        bamfiles = [bamfile]

    for bamfile in bamfiles:
        run_mito(chrMfa, bamfile, opts)

def run_mito(chrMfa, bamfile, opts):
    from jcvi.formats.sam import get_minibam
    region = "chrM"
    minibam = op.basename(bamfile).replace(".bam", ".{}.bam".format(region))
    if not op.exists(minibam):
        get_minibam(bamfile, region)

    realign = minibam.rsplit(".", 1)[0] + ".realign"
    margs = " -v -t {} -o {}".format(opts.cpus, realign)
    if need_update(minibam, realign):
        cmd = "speedseq realign"
        cmd += margs
        cmd += " {} {}".format(chrMfa, minibam)
        sh(cmd)

    vcffile = realign + ".sv.vcf.gz"
    if need_update(realign, vcffile):
        cmd = "speedseq sv"
        cmd += margs
        cmd += " -g -R {}".format(chrMfa)
        cmd += " -B {} -D {} -S {}".format(realign + ".bam",
                        realign + ".discordants.bam", realign + ".splitters.bam")
        sh(cmd)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import dictdiffer
import motor

import virtool.utils
import virtool.gen
import virtool.database
import virtool.viruses

class Collection(virtool.database.Collection):

    def __init__(self, dispatcher):
        super(Collection, self).__init__("history", dispatcher)

        self.sync_projector.update({key: True for key in [
            "operation",
            "method_name",
            "changes",
            "timestamp",
            "entry_id",
            "entry_version",
            "username",
            "annotation",
            "index",
            "index_version"
        ]})

        self.sequences_collection = motor.MotorClient()[self.settings.get("db_name")]["sequences"]

    @virtool.gen.coroutine
    def sync_processor(self, documents):
        documents = virtool.database.coerce_list(documents)

        for document in documents:
            if document["operation"] in ["insert", "remove"]:
                document["virus"] = document["changes"]["name"]
            else:
                _, patched, _ = yield self.get_versioned_document(document["entry_id"], document["entry_version"])
                document["virus"] = patched["name"]

        return documents

    @virtool.gen.exposed_method([])
    def sync(self, transaction):
        """
        Syncs documents between the server and client. This exposed method will be requested by the client soon after
        its connection is authorized. The client supplies a dictionary of document ids and their version
        numbers. This manifest is used to calculate a list of updates and removals required to bring the client's local
        collection in sync with the one on the server. The calculation is performed by :meth:`.Collection.prepare_sync`.

        The passed ``transaction`` is then updated with the number of update and remove operation that will be
        dispatched to the client. This allows the client to display the progress of the sync operation as it receives
        updates and removals.

        :param transaction: the transaction generated from the request.
        :type transaction: :class:`.Transaction`

        :return: a boolean indicating success and the total number of operations performed.
        :rtype: tuple

        """
        manifest = transaction.data

        if manifest:
            database_ids = yield self.db.distinct("_id")
            database_ids = set(database_ids)

            manifest_ids = set(manifest.keys())

            to_update = list(database_ids - manifest_ids)

            updates = list()

            if len(to_update) > 0:
                updates = yield self.find({"_id": {
                    "$in": to_update
                }}, self.sync_projector).to_list(None)

                updates = yield self.sync_processor(updates)

            removes = list(manifest_ids - database_ids)
        else:
            updates = updates = yield self.find({}, self.sync_projector).to_list(None)
            updates = yield self.sync_processor(updates)
            removes = list()

        expected_operation_count = len(updates) + len(removes)

        transaction.update(expected_operation_count)

        for i in range(0, len(updates), 10):
            yield self.dispatch("update", updates[i: i + 10], connections=[transaction.connection], sync=True)

        # All remaining documents should be deleted by the client since they no longer exist on the server.
        for i in range(0, len(removes), 10):
            yield self.dispatch("remove", removes[i: i + 10], connections=[transaction.connection], sync=True)

        return True, None

    @virtool.gen.coroutine
    def add(self, operation, method_name, old, new, username):
        # Construct and _id for the change entry. It is composed of the _id of the changed entry and the new version
        # number of the entry separated by a dot (eg. a7sydi.3)
        try:
            document_id = old["_id"]
        except TypeError:
            document_id = new["_id"]

        try:
            document_version = new["_version"]
        except TypeError:
            document_version = "removed"

        history_document = {
            "_id":  str(document_id) + "." + (str(document_version)),
            "_version": 0,
            "operation": operation,
            "method_name": method_name,
            "timestamp": virtool.utils.timestamp(),
            "entry_id": document_id,
            "entry_version": document_version,
            "username": username,
            "annotation": None,
            "index": "unbuilt",
            "index_version": "unbuilt"
        }

        if operation in ["update", "remove", "insert"]:
            if operation == "update":
                history_document["changes"] = list(dictdiffer.diff(old, new))
            elif operation == "remove":
                history_document["changes"] = old
            else:
                history_document["changes"] = new
        else:
            raise ValueError("Passed operation is not one of 'update', 'remove', 'insert'")

        if method_name == "set_default_isolate":
            history_document["annotation"] = yield self.get_default_isolate(new)

        if method_name == "upsert_isolate":
            history_document["annotation"] = yield self.get_upserted_isolate(old, history_document["changes"])

        if method_name == "remove_sequence":
            subject_isolate = yield self.get_isolate_of_removed_sequence(old, history_document["changes"])
            assert subject_isolate is not None
            history_document["annotation"] = subject_isolate

        if method_name == "add_sequence":
            subject_isolate = yield self.get_isolate_of_added_sequence(new, history_document["changes"])
            history_document["annotation"] = subject_isolate

        if method_name == "update_sequence":
            subject_isolate = yield self.get_info_for_updated_sequence(new, history_document["changes"])
            history_document["annotation"] = subject_isolate

        yield self.insert(history_document)

    @virtool.gen.synchronous
    def get_default_isolate(self, document):
        for isolate in document["isolates"]:
            if isolate["default"]:
                return strip_isolate(isolate)

        raise ValueError("Could not find default isolate in virus document")

    @virtool.gen.synchronous
    def get_upserted_isolate(self, document, changes):
        for change in changes:
            if change[0] == "change" and change[1][0] == "isolates":
                return document["isolates"][change[1][1]]

        return None

    @virtool.gen.synchronous
    def get_info_for_updated_sequence(self, document, changes):
        for change in changes:
            if change[0] == "change" and change[1][2] == "sequences":
                isolate = document["isolates"][change[1][1]]
                sequence_index = change[1][3]
                sequence = {key: isolate["sequences"][sequence_index][key] for key in ["_id", "definition"]}

                isolate = strip_isolate(isolate)
                isolate.update(sequence)

                return isolate

        raise ValueError("Could not find isolate of updated sequence")

    @virtool.gen.synchronous
    def get_isolate_of_added_sequence(self, document, changes):
        for change in changes:
            if change[0] == "add" and change[1][0] == "isolates":
                isolate_index = change[1][1]
                return strip_isolate(document["isolates"][isolate_index])

        raise ValueError("Could not find isolate of added sequence")

    @virtool.gen.synchronous
    def get_isolate_of_removed_sequence(self, document, changes):
        for change in changes:
            if change[0] == "remove":
                isolate_index = change[1][1]
                return strip_isolate(document["isolates"][isolate_index])

        raise ValueError("Could not find isolate of removed sequence")

    @virtool.gen.exposed_method(["modify_virus"])
    def revert(self, transaction):
        data = transaction.data

        document, patched, history_to_delete = yield self.get_versioned_document(
            data["entry_id"],
            data["entry_version"]
        )

        isolate_ids = yield virtool.viruses.extract_isolate_ids(document or patched)

        # Remove the old sequences from the collection.
        yield self.sequences_collection.remove({"isolate_id": {"$in": isolate_ids}})

        if patched != "remove":
            # Add the reverted sequences to the collection.
            for isolate in patched["isolates"]:
                for sequence in isolate["sequences"]:
                    yield self.sequences_collection.insert(sequence)

            for isolate in patched["isolates"]:
                isolate["sequence_count"] = len(isolate.pop("sequences"))

            if document:
                yield self.dispatcher.collections["viruses"].update(
                    document["_id"],
                    {"$set": patched},
                    increment_version=False
                )
            else:
                yield self.dispatcher.collections["viruses"].insert(patched)
        else:
            yield self.dispatcher.collections["viruses"].remove(document["_id"])

        yield self.remove(history_to_delete)

        return True, history_to_delete

    @virtool.gen.coroutine
    def get_versioned_document(self, virus_id, virus_version):
        current = yield self.dispatcher.collections["viruses"].join(virus_id)

        versioned = yield self.patch_virus_to_version(current or {"_id": virus_id}, virus_version)

        return current, versioned[1], versioned[2]

    @virtool.gen.coroutine
    def patch_virus_to_version(self, joined_virus, version):
        virus_history = yield self.find({"entry_id": joined_virus["_id"]}).to_list(None)

        current = joined_virus or dict()

        # A list of history_ids reverted to produce the patched entry.
        reverted_history_ids = list()

        # Sort the changes be descending entry version.
        virus_history = sorted(virus_history, key=lambda x: x["timestamp"], reverse=True)

        patched = dict(current)

        for history_document in virus_history:
            if history_document["entry_version"] == "removed" or history_document["entry_version"] >= version:
                reverted_history_ids.append(history_document["_id"])

                if history_document["method_name"] == "add":
                    patched = "remove"

                elif history_document["method_name"] == "remove":
                    patched = history_document["changes"]
                    break

                else:
                    diff = dictdiffer.swap(history_document["changes"])
                    patched = yield patch(diff, patched)
            else:
                break

        return current, patched, reverted_history_ids

    @virtool.gen.coroutine
    def reversion_update(self, _id, reverted):
        yield self.update(_id, reverted, reversion=True)

    @virtool.gen.coroutine
    def set_index_as_unbuilt(self, data):
        yield self.update({"index": data["index_id"]}, {
            "$set": {
                "index": "unbuilt",
                "index_version": "unbuilt"
            }
        })

@virtool.gen.synchronous
def patch(diff, subject):
    return dictdiffer.patch(diff, subject)

def strip_isolate(isolate):
    return {key: isolate[key] for key in ["source_type", "source_name", "isolate_id"]}
<commit_msg>Remove double variable assignment in history.py<commit_after>import dictdiffer
import motor

import virtool.utils
import virtool.gen
import virtool.database
import virtool.viruses

class Collection(virtool.database.Collection):

    def __init__(self, dispatcher):
        super(Collection, self).__init__("history", dispatcher)

        self.sync_projector.update({key: True for key in [
            "operation",
            "method_name",
            "changes",
            "timestamp",
            "entry_id",
            "entry_version",
            "username",
            "annotation",
            "index",
            "index_version"
        ]})

        self.sequences_collection = motor.MotorClient()[self.settings.get("db_name")]["sequences"]

    @virtool.gen.coroutine
    def sync_processor(self, documents):
        documents = virtool.database.coerce_list(documents)

        for document in documents:
            if document["operation"] in ["insert", "remove"]:
                document["virus"] = document["changes"]["name"]
            else:
                _, patched, _ = yield self.get_versioned_document(document["entry_id"], document["entry_version"])
                document["virus"] = patched["name"]

        return documents

    @virtool.gen.exposed_method([])
    def sync(self, transaction):
        """
        Syncs documents between the server and client. This exposed method will be requested by the client soon after
        its connection is authorized. The client supplies a dictionary of document ids and their version
        numbers. This manifest is used to calculate a list of updates and removals required to bring the client's local
        collection in sync with the one on the server. The calculation is performed by :meth:`.Collection.prepare_sync`.

        The passed ``transaction`` is then updated with the number of update and remove operation that will be
        dispatched to the client. This allows the client to display the progress of the sync operation as it receives
        updates and removals.

        :param transaction: the transaction generated from the request.
        :type transaction: :class:`.Transaction`

        :return: a boolean indicating success and the total number of operations performed.
        :rtype: tuple

        """
        manifest = transaction.data

        if manifest:
            database_ids = yield self.db.distinct("_id")
            database_ids = set(database_ids)

            manifest_ids = set(manifest.keys())

            to_update = list(database_ids - manifest_ids)

            updates = list()

            if len(to_update) > 0:
                updates = yield self.find({"_id": {
                    "$in": to_update
                }}, self.sync_projector).to_list(None)

                updates = yield self.sync_processor(updates)

            removes = list(manifest_ids - database_ids)
        else:
            updates = yield self.find({}, self.sync_projector).to_list(None)
            updates = yield self.sync_processor(updates)
            removes = list()

        expected_operation_count = len(updates) + len(removes)

        transaction.update(expected_operation_count)

        for i in range(0, len(updates), 10):
            yield self.dispatch("update", updates[i: i + 10], connections=[transaction.connection], sync=True)

        # All remaining documents should be deleted by the client since they no longer exist on the server.
        for i in range(0, len(removes), 10):
            yield self.dispatch("remove", removes[i: i + 10], connections=[transaction.connection], sync=True)

        return True, None

    @virtool.gen.coroutine
    def add(self, operation, method_name, old, new, username):
        # Construct and _id for the change entry. It is composed of the _id of the changed entry and the new version
        # number of the entry separated by a dot (eg. a7sydi.3)
        try:
            document_id = old["_id"]
        except TypeError:
            document_id = new["_id"]

        try:
            document_version = new["_version"]
        except TypeError:
            document_version = "removed"

        history_document = {
            "_id":  str(document_id) + "." + (str(document_version)),
            "_version": 0,
            "operation": operation,
            "method_name": method_name,
            "timestamp": virtool.utils.timestamp(),
            "entry_id": document_id,
            "entry_version": document_version,
            "username": username,
            "annotation": None,
            "index": "unbuilt",
            "index_version": "unbuilt"
        }

        if operation in ["update", "remove", "insert"]:
            if operation == "update":
                history_document["changes"] = list(dictdiffer.diff(old, new))
            elif operation == "remove":
                history_document["changes"] = old
            else:
                history_document["changes"] = new
        else:
            raise ValueError("Passed operation is not one of 'update', 'remove', 'insert'")

        if method_name == "set_default_isolate":
            history_document["annotation"] = yield self.get_default_isolate(new)

        if method_name == "upsert_isolate":
            history_document["annotation"] = yield self.get_upserted_isolate(old, history_document["changes"])

        if method_name == "remove_sequence":
            subject_isolate = yield self.get_isolate_of_removed_sequence(old, history_document["changes"])
            assert subject_isolate is not None
            history_document["annotation"] = subject_isolate

        if method_name == "add_sequence":
            subject_isolate = yield self.get_isolate_of_added_sequence(new, history_document["changes"])
            history_document["annotation"] = subject_isolate

        if method_name == "update_sequence":
            subject_isolate = yield self.get_info_for_updated_sequence(new, history_document["changes"])
            history_document["annotation"] = subject_isolate

        yield self.insert(history_document)

    @virtool.gen.synchronous
    def get_default_isolate(self, document):
        for isolate in document["isolates"]:
            if isolate["default"]:
                return strip_isolate(isolate)

        raise ValueError("Could not find default isolate in virus document")

    @virtool.gen.synchronous
    def get_upserted_isolate(self, document, changes):
        for change in changes:
            if change[0] == "change" and change[1][0] == "isolates":
                return document["isolates"][change[1][1]]

        return None

    @virtool.gen.synchronous
    def get_info_for_updated_sequence(self, document, changes):
        for change in changes:
            if change[0] == "change" and change[1][2] == "sequences":
                isolate = document["isolates"][change[1][1]]
                sequence_index = change[1][3]
                sequence = {key: isolate["sequences"][sequence_index][key] for key in ["_id", "definition"]}

                isolate = strip_isolate(isolate)
                isolate.update(sequence)

                return isolate

        raise ValueError("Could not find isolate of updated sequence")

    @virtool.gen.synchronous
    def get_isolate_of_added_sequence(self, document, changes):
        for change in changes:
            if change[0] == "add" and change[1][0] == "isolates":
                isolate_index = change[1][1]
                return strip_isolate(document["isolates"][isolate_index])

        raise ValueError("Could not find isolate of added sequence")

    @virtool.gen.synchronous
    def get_isolate_of_removed_sequence(self, document, changes):
        for change in changes:
            if change[0] == "remove":
                isolate_index = change[1][1]
                return strip_isolate(document["isolates"][isolate_index])

        raise ValueError("Could not find isolate of removed sequence")

    @virtool.gen.exposed_method(["modify_virus"])
    def revert(self, transaction):
        data = transaction.data

        document, patched, history_to_delete = yield self.get_versioned_document(
            data["entry_id"],
            data["entry_version"]
        )

        isolate_ids = yield virtool.viruses.extract_isolate_ids(document or patched)

        # Remove the old sequences from the collection.
        yield self.sequences_collection.remove({"isolate_id": {"$in": isolate_ids}})

        if patched != "remove":
            # Add the reverted sequences to the collection.
            for isolate in patched["isolates"]:
                for sequence in isolate["sequences"]:
                    yield self.sequences_collection.insert(sequence)

            for isolate in patched["isolates"]:
                isolate["sequence_count"] = len(isolate.pop("sequences"))

            if document:
                yield self.dispatcher.collections["viruses"].update(
                    document["_id"],
                    {"$set": patched},
                    increment_version=False
                )
            else:
                yield self.dispatcher.collections["viruses"].insert(patched)
        else:
            yield self.dispatcher.collections["viruses"].remove(document["_id"])

        yield self.remove(history_to_delete)

        return True, history_to_delete

    @virtool.gen.coroutine
    def get_versioned_document(self, virus_id, virus_version):
        current = yield self.dispatcher.collections["viruses"].join(virus_id)

        versioned = yield self.patch_virus_to_version(current or {"_id": virus_id}, virus_version)

        return current, versioned[1], versioned[2]

    @virtool.gen.coroutine
    def patch_virus_to_version(self, joined_virus, version):
        virus_history = yield self.find({"entry_id": joined_virus["_id"]}).to_list(None)

        current = joined_virus or dict()

        # A list of history_ids reverted to produce the patched entry.
        reverted_history_ids = list()

        # Sort the changes be descending entry version.
        virus_history = sorted(virus_history, key=lambda x: x["timestamp"], reverse=True)

        patched = dict(current)

        for history_document in virus_history:
            if history_document["entry_version"] == "removed" or history_document["entry_version"] >= version:
                reverted_history_ids.append(history_document["_id"])

                if history_document["method_name"] == "add":
                    patched = "remove"

                elif history_document["method_name"] == "remove":
                    patched = history_document["changes"]
                    break

                else:
                    diff = dictdiffer.swap(history_document["changes"])
                    patched = yield patch(diff, patched)
            else:
                break

        return current, patched, reverted_history_ids

    @virtool.gen.coroutine
    def reversion_update(self, _id, reverted):
        yield self.update(_id, reverted, reversion=True)

    @virtool.gen.coroutine
    def set_index_as_unbuilt(self, data):
        yield self.update({"index": data["index_id"]}, {
            "$set": {
                "index": "unbuilt",
                "index_version": "unbuilt"
            }
        })

@virtool.gen.synchronous
def patch(diff, subject):
    return dictdiffer.patch(diff, subject)

def strip_isolate(isolate):
    return {key: isolate[key] for key in ["source_type", "source_name", "isolate_id"]}
<|endoftext|>


<commit_before>class VirtualMachine:
    def __init__(self, ram_size=512, executing=True):
        self.data = {i: None for i in range(ram_size)}
        self.stack = []
        self.executing = executing
        self.pc = 0
        self.devices_start = 256

    def push(self, value):
        """Push something onto the stack."""
        self.stack += [value]

    def pop(self):
        """Pop something from the stack. Crash if empty."""
        return self.stack.pop()

    def read_memory(self, index):
        """Read from memory, crashing if index is out of bounds."""
        if isinstance(self.data[index], DeviceProxy):
            return self.data[index].read(index)
        else:
            return self.data[index]

    def write_memory(self, index, value):
        """Write to memory.  Crash if index is out of bounds."""
        if isinstance(self.data[index], DeviceProxy):
            self.data[index].write(index, value)
        else:
            self.data[index] = value

    def register_device(self, device, needed_addresses):
        """Given an instantiated device and the number of required addresses, registers it in memory"""
        # If not enough addresses, just error out
        if self.devices_start+needed_addresses > len(self.data):
            raise Exception('Not enough addresses to allocate')
        proxyed_device = DeviceProxy(device, self.devices_start)
        for i in range(self.devices_start, self.devices_start+needed_addresses):
            self.data[i] = proxyed_device
        self.devices_start += needed_addresses

    def run(self, bytecodes):
        self.bytecodes = bytecodes
        while self.executing:
            increment = self.bytecodes[self.pc].autoincrement
            self.bytecodes[self.pc].execute(self)
            if increment:
                self.pc += 1

<commit_msg>Add a Device Proxy to translate between the addresses in the vm and the device<commit_after>class VirtualMachine:
    def __init__(self, ram_size=512, executing=True):
        self.data = {i: None for i in range(ram_size)}
        self.stack = []
        self.executing = executing
        self.pc = 0
        self.devices_start = 256

    def push(self, value):
        """Push something onto the stack."""
        self.stack += [value]

    def pop(self):
        """Pop something from the stack. Crash if empty."""
        return self.stack.pop()

    def read_memory(self, index):
        """Read from memory, crashing if index is out of bounds."""
        if isinstance(self.data[index], DeviceProxy):
            return self.data[index].read(index)
        else:
            return self.data[index]

    def write_memory(self, index, value):
        """Write to memory.  Crash if index is out of bounds."""
        if isinstance(self.data[index], DeviceProxy):
            self.data[index].write(index, value)
        else:
            self.data[index] = value

    def register_device(self, device, needed_addresses):
        """Given an instantiated device and the number of required addresses, registers it in memory"""
        # If not enough addresses, just error out
        if self.devices_start+needed_addresses > len(self.data):
            raise Exception('Not enough addresses to allocate')
        proxyed_device = DeviceProxy(device, self.devices_start)
        for i in range(self.devices_start, self.devices_start+needed_addresses):
            self.data[i] = proxyed_device
        self.devices_start += needed_addresses

    def run(self, bytecodes):
        self.bytecodes = bytecodes
        while self.executing:
            increment = self.bytecodes[self.pc].autoincrement
            self.bytecodes[self.pc].execute(self)
            if increment:
                self.pc += 1

class DeviceProxy:
    """Manages address translation between devices"""
    def __init__(self, device, pos):
        self.device = device
        self.pos = pos

    def read(self, index):
        return self.device.read(self.pos-index)

    def write(self, index, value):
        self.device.write(self.pos-index, value)
<|endoftext|>


<commit_before><commit_msg>add the last chunk of text corpus to the output<commit_after><|endoftext|>


<commit_before># Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""A workflow for tiling images
"""

import io
import argparse
import logging
import math
import itertools
from concurrent.futures import ThreadPoolExecutor
from abc import ABC, abstractmethod

import imageio
import requests

import apache_beam as beam
from apache_beam.io.fileio import MatchFiles
from apache_beam.io.gcp import gcsio
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.transforms.core import ParDo

from google.cloud import storage

class DoFnWithGCSClient(ABC, beam.DoFn):
    def setup(self):
        self.client = storage.Client()
        adapter = requests.adapters.HTTPAdapter(pool_connections=128, pool_maxsize=128, max_retries=3, pool_block=True)
        self.client._http.mount("https://", adapter)
        self.client._http._auth_request.session.mount("https://", adapter)
    def start_bundle(self):
        self.executor = ThreadPoolExecutor(128)
    @abstractmethod
    def process(self, el):
        pass
    def finish_bundle(self):
        self.executor.shutdown()

class UploadGCS(DoFnWithGCSClient):
    def process(self, el):
        path = el[0]
        data = el[1]
        bucket_name = path.replace("gs://", "").split("/")[0]
        bucket = self.client.bucket(bucket_name)
        target_key = path.replace(f"gs://{bucket_name}/","")
        blob = bucket.blob(target_key)
        try:
            self.executor.submit(blob.upload_from_string, data)
        except Exception as e:
            print(e)
        return ["Ok"]

class ProcessTiles(DoFnWithGCSClient):
    def process(self, el):
        im = el["image"]
        output = el["output"]
        file_name = el["name"]
        tile_size = 256
        num_tiles = 0
        tier = math.ceil(max([math.log(shape/tile_size, 2) for shape in im.shape]))
        max_step = 2 ** tier
        steps = [int(max_step/2**i) for i in range(tier + 1)]
        tiers = [im[::step, ::step] for step in steps]
        def next_i():
            i = 0
            while i < tile_size:
                yield i // tile_size
                i += 1
        tile_group = next_i()
        for tier_number, tier in enumerate(tiers):
            for item in itertools.product([i for i in range(math.ceil(tier.shape[0]/tile_size))], [i for i in range(math.ceil(tier.shape[1]/tile_size))]):
                num_tiles += 1
                yield{
                    "tier": tier,
                    "tier_number": tier_number,
                    "item": item,
                    "tile_group": next(tile_group),
                    "output": output,
                    "name": file_name
                }
        # Save ImageProperties.xml
        img_width = im.shape[1]
        img_height = im.shape[0]
        data = f'<IMAGE_PROPERTIES WIDTH="{img_width}" HEIGHT="{img_height}" NUMTILES="{num_tiles}" NUMIMAGES="1" VERSION="1.8" TILESIZE="{tile_size}"/>'
        path = f"{output}{file_name}/ImageProperties.xml"
        bucket_name = path.replace("gs://", "").split("/")[0]
        bucket = self.client.bucket(bucket_name)
        target_key = path.replace(f"gs://{bucket_name}/","")
        blob = bucket.blob(target_key)
        try:
            self.executor.submit(blob.upload_from_string, data)
        except Exception as e:
            print(e)

def by_extension(el, extensions):
  ext = el.path.split('.')[-1]
  return ext in extensions

def img_read(path, output):
    kwargs = {}
    ext = path.split('.')[-1]
    if ext == 'png':
        kwargs['pilmode'] = 'RGB' # discard alpha channel
    gcs = gcsio.GcsIO()
    return {"image": imageio.imread(gcs.open(path), **kwargs),
            "name": path.split("/")[-1],
            "output": output
           }

def save_tile(el):
    # gcs = gcsio.GcsIO()
    tier = el["tier"]
    tier_number = el["tier_number"]
    item = el["item"]
    tile_group = el["tile_group"]
    output = el["output"] + el["name"]
    path = f"{output}/TileGroup{tile_group}/{tier_number}-{item[1]}-{item[0]}.jpg"

    f = io.BytesIO()
    # gcs.open(path, 'w', mime_type="image/jpeg")
    try:
        imageio.imwrite(
            #f"image/TileGroup-{next(tile_group)}-{tier_number}-{item[0]}-{item[1]}.jpg",
            f,
            tier[item[0]*256:min(tier.shape[0],item[0]*256+256), item[1]*256:min(tier.shape[1],item[1]*256+256), :3],
            format="JPG", quality=80)
        return [(path, f.getvalue())]
    except ValueError as e:
        print(e)

def main(argv=None, save_main_session=True):
  """Main entry point"""

  parser = argparse.ArgumentParser()
  parser.add_argument(
      '--input',
      dest='input',
      default='gs://YOUR_INPUT_BUCKET/AND_INPUT_PREFIX/',
      help='Input folder to process.')
  parser.add_argument(
      '--output',
      dest='output',
      default='gs://YOUR_OUTPUT_BUCKET/AND_OUTPUT_PREFIX',
      help='Output file to write results to.')
  parser.add_argument(
      '--extensions',
      dest='extensions',
      default='jpg',
      help='File extensions to be processed.')
  known_args, pipeline_args = parser.parse_known_args(argv)

  pipeline_options = PipelineOptions(pipeline_args)
  pipeline_options.view_as(SetupOptions).save_main_session = save_main_session

  valid_extensions = known_args.extensions.split(',')

  with beam.Pipeline(options=pipeline_options) as p:
    # Read the text file[pattern] into a PCollection.
    files = p | MatchFiles(known_args.input)
    (
        files
        | 'Filter by file extension' >> beam.Filter(by_extension, valid_extensions)
        | 'Read images to arrays' >> beam.Map(lambda x: img_read(x.path, known_args.output))
        | 'Tile images' >> beam.ParDo(ProcessTiles())
        | 'Save tile' >> beam.ParDo(save_tile)
        | 'Upload to GCS' >> beam.ParDo(UploadGCS())
    )

if __name__ == '__main__':
  logging.getLogger().setLevel(logging.INFO)
  main()
<commit_msg>Removed extension from tile directory name. Subdirectories are now preserved when saving tiles. Tiles directory is removed before new tiles are generated.<commit_after># Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""A workflow for tiling images
"""

import os
import io
import argparse
import logging
import math
import itertools
from concurrent.futures import ThreadPoolExecutor
from abc import ABC, abstractmethod

import imageio
import requests

import apache_beam as beam
from apache_beam.io.fileio import MatchFiles
from apache_beam.io.gcp import gcsio
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import SetupOptions
from apache_beam.transforms.core import ParDo

from google.cloud import storage

class DoFnWithGCSClient(ABC, beam.DoFn):
    def setup(self):
        self.client = storage.Client()
        adapter = requests.adapters.HTTPAdapter(pool_connections=128, pool_maxsize=128, max_retries=3, pool_block=True)
        self.client._http.mount("https://", adapter)
        self.client._http._auth_request.session.mount("https://", adapter)
    def start_bundle(self):
        self.executor = ThreadPoolExecutor(128)
    @abstractmethod
    def process(self, el):
        pass
    def finish_bundle(self):
        self.executor.shutdown()

class UploadGCS(DoFnWithGCSClient):
    def process(self, el):
        path = el[0]
        data = el[1]
        bucket_name = path.replace("gs://", "").split("/")[0]
        bucket = self.client.bucket(bucket_name)
        target_key = path.replace(f"gs://{bucket_name}/","")
        blob = bucket.blob(target_key)
        try:
            self.executor.submit(blob.upload_from_string, data)
        except Exception as e:
            print(e)
        return ["Ok"]

class ProcessTiles(DoFnWithGCSClient):
    def process(self, el):
        im = el["image"]
        save_to = el["save_to"]
        # Remove existing tiles if they exist
        bucket_name = save_to.replace("gs://", "").split("/")[0]
        bucket = self.client.bucket(bucket_name)
        target_key = save_to.replace(f"gs://{bucket_name}/","")
        blob = bucket.blob(target_key)
        try:
            self.executor.submit(blob.delete)
        except Exception as e:
            print(e)

        # Create new tiles
        tile_size = 256
        num_tiles = 0
        tier = math.ceil(max([math.log(shape/tile_size, 2) for shape in im.shape]))
        max_step = 2 ** tier
        steps = [int(max_step/2**i) for i in range(tier + 1)]
        tiers = [im[::step, ::step] for step in steps]
        def next_i():
            i = 0
            while i < tile_size:
                yield i // tile_size
                i += 1
        tile_group = next_i()
        for tier_number, tier in enumerate(tiers):
            for item in itertools.product([i for i in range(math.ceil(tier.shape[0]/tile_size))], [i for i in range(math.ceil(tier.shape[1]/tile_size))]):
                num_tiles += 1
                yield{
                    "tier": tier,
                    "tier_number": tier_number,
                    "item": item,
                    "tile_group": next(tile_group),
                    "save_to": save_to
                }
        # Save ImageProperties.xml
        img_width = im.shape[1]
        img_height = im.shape[0]
        data = f'<IMAGE_PROPERTIES WIDTH="{img_width}" HEIGHT="{img_height}" NUMTILES="{num_tiles}" NUMIMAGES="1" VERSION="1.8" TILESIZE="{tile_size}"/>'
        target_key = save_to.replace(f"gs://{bucket_name}/","") + "/ImageProperties.xml"
        blob = bucket.blob(target_key)
        try:
            self.executor.submit(blob.upload_from_string, data)
        except Exception as e:
            print(e)

def by_extension(el, extensions):
  ext = el.path.split('.')[-1]
  return ext.lower() in extensions

def img_read(path, output, input_dir):
    kwargs = {}
    directory, filename = os.path.split(path)
    name, ext = os.path.splitext(filename)
    subpath = directory[len(input_dir):]
    if subpath:
       # remove leading "/" before joining with output
       subpath = subpath[1:]
    save_to = output + subpath + "/" + name
    if ext.lower() == '.png':
        kwargs['pilmode'] = 'RGB' # discard alpha channel
    gcs = gcsio.GcsIO()
    return {"image": imageio.imread(gcs.open(path), **kwargs),
            "save_to": save_to
           }

def save_tile(el):
    # gcs = gcsio.GcsIO()
    tier = el["tier"]
    tier_number = el["tier_number"]
    item = el["item"]
    tile_group = el["tile_group"]
    save_to = el["save_to"]
    path = f"{save_to}/TileGroup{tile_group}/{tier_number}-{item[1]}-{item[0]}.jpg"

    f = io.BytesIO()
    # gcs.open(path, 'w', mime_type="image/jpeg")
    try:
        imageio.imwrite(
            #f"image/TileGroup-{next(tile_group)}-{tier_number}-{item[0]}-{item[1]}.jpg",
            f,
            tier[item[0]*256:min(tier.shape[0],item[0]*256+256), item[1]*256:min(tier.shape[1],item[1]*256+256), :3],
            format="JPG", quality=80)
        return [(path, f.getvalue())]
    except ValueError as e:
        print(e)

def main(argv=None, save_main_session=True):
  """Main entry point"""

  parser = argparse.ArgumentParser()
  parser.add_argument(
      '--input',
      dest='input',
      default='gs://YOUR_INPUT_BUCKET/AND_INPUT_PREFIX/',
      help='Input folder to process.')
  parser.add_argument(
      '--output',
      dest='output',
      default='gs://YOUR_OUTPUT_BUCKET/AND_OUTPUT_PREFIX',
      help='Output file to write results to.')
  parser.add_argument(
      '--extensions',
      dest='extensions',
      default='jpg',
      help='File extensions to be processed.')
  known_args, pipeline_args = parser.parse_known_args(argv)

  pipeline_options = PipelineOptions(pipeline_args)
  pipeline_options.view_as(SetupOptions).save_main_session = save_main_session

  valid_extensions = known_args.extensions.split(',')

  input_dir, file_pattern = os.path.split(known_args.input)

  with beam.Pipeline(options=pipeline_options) as p:
    # Read the text file[pattern] into a PCollection.
    files = p | MatchFiles(known_args.input)
    (
        files
        | 'Filter by file extension' >> beam.Filter(by_extension, valid_extensions)
        | 'Read images to arrays' >> beam.Map(lambda x: img_read(x.path, known_args.output, input_dir))
        | 'Tile images' >> beam.ParDo(ProcessTiles())
        | 'Save tile' >> beam.ParDo(save_tile)
        | 'Upload to GCS' >> beam.ParDo(UploadGCS())
    )

if __name__ == '__main__':
  logging.getLogger().setLevel(logging.INFO)
  main()
<|endoftext|>


<commit_before>#!/usr/bin/python

import glob
import re
import os
from html.parser import HTMLParser

linkedcss = []

htmlclasses = []
htmlids = []
cssclasses = []
cssids = []

class MyHTMLParser(HTMLParser):
    def handle_starttag(self, tag, attrs):

        if (tag == 'link'):

            for attr in attrs:

                if (attr[0] == 'href'):

                    extension = os.path.splitext(attr[1])[1]
                    if (extension == '.css'):

                        cssfile = os.path.abspath((userdir + attr[1]))
                        linkedcss.append(cssfile)

                        csshandle = open(cssfile, "r")
                        csscontent = csshandle.readlines()
                        for line in csscontent:
                            words = line.split()
                            for word in words:
                                if word.startswith("."):
                                    print (word)
                                    cssclasses.append(word)

                                if word.startswith("#"):
                                    print (word)
                                    cssids.append(word)

userdir = input("Enter a directory to search\n")
userexts = input("Enter a comma separated list of file extensions to search in e.g php,html\n")

parser = MyHTMLParser(strict=False)

exts = userexts.split(",")

for ext in exts:
    print ("current file extension is ", ext)

    files = glob.glob(userdir + "*." + ext)
    print("number of files searched", len(files))
    for file in files:

        print("current file name is ", file)
        handle = open(file, "r")
        content = handle.readlines()
        linenum = 0
        classfound = False
        idfound = False

        for line in content:
            parser.feed(line)
            classes = re.findall(r'class=\"(.+?)\"', line)
            if classes:
                classfound = True
                for htmlclass in classes:
                    print(" class found -> .", htmlclass, " on line ", linenum, sep='')
                    htmlclasses.append(htmlclass)

            ids = re.findall(r'id=\"(.+?)\"', line)
            if ids:
                idfound = True
                for htmlid in ids:
                    print(" id found -> #", htmlid, " on line ", linenum,sep='')
                    htmlids.append(htmlid)

            linenum = linenum + 1

        if not classfound:
            print("no classes found in file")

        if not idfound:
            print("no ids found in file")

print(linkedcss)
print("html class list is ", htmlclasses)
print("html id list is ", htmlids)
print("css class list is ", cssclasses)
print("css id list is ", cssids)
#need multi-demensional array to hold filename, line number of occurrence ,class name, id name etc
<commit_msg>Creating nested dicts of file names and ids:linenumbers<commit_after>#!/usr/bin/python

import glob
import re
import os
from html.parser import HTMLParser

stopparsing = False

linkedcss = []

htmlclasses = []
htmlids = {}
cssclasses = []
cssids = []

linkedfiles = {}

class MyHTMLParser(HTMLParser):

    def handle_starttag(self, tag, attrs):

        if tag == 'link':

            for attr in attrs:

                if (attr[0] == 'href'):

                    extension = os.path.splitext(attr[1])[1]
                    if (extension == '.css'):

                        cssfile = os.path.abspath((userdir + attr[1]))
                        linkedcss.append(cssfile)

                        csshandle = open(cssfile, "r")
                        csscontent = csshandle.readlines()
                        for line in csscontent:

                            words = line.split()
                            for word in words:
                                if word.startswith("."):
                                    cssclasses.append(word)

                                if word.startswith("#"):
                                    cssids.append(word)

    def handle_endtag(self, tag):
        if tag == 'head':
            #print("end tag is",tag)
            stopparsing = True

userdir = input("Enter a directory to search\n")
userexts = input("Enter a comma separated list of file extensions to search in e.g php,html\n")

parser = MyHTMLParser(strict=False)

exts = userexts.split(",")

for ext in exts:
    print ("Searching file extension... ", ext)

    files = glob.glob(userdir + "*." + ext)
    print("number of files searched:", len(files))
    for file in files:

        fileids = {}

        handle = open(file, "r")
        content = handle.readlines()
        linenum = 0
        classfound = False
        idfound = False

        for line in content:

            if not stopparsing:
                parser.feed(line)
            classes = re.findall(r'class=\"(.+?)\"', line)
            if classes:
                classfound = True
                for htmlclass in classes:
                    htmlclasses.append(htmlclass)

            ids = re.findall(r'id=\"(.+?)\"', line)
            if ids:
                idfound = True
                for htmlid in ids:
                    fileids[htmlid] = linenum

            linenum = linenum + 1

        #if not classfound:
            #print("no classes found in file")

        if idfound:
            htmlids[file] = fileids
        #else:
            #print("no ids found in file")

#print(linkedcss)
#print("html class list is ", htmlclasses)
#print("html id list is ", htmlids)
print("number of files with ids in", len(htmlids))
for fileid,ids in htmlids.items():
    print("    The file is",fileid)
    for k,v in ids.items():
        print("        ID is #",k," on line ",v,sep='')
#print("css class list is ", cssclasses)
#print("css id list is ", cssids)
#need multi-demensional array to hold html filename, linked css file, line numbers of occurrences ,class name, id name etc
<|endoftext|>


<commit_before># Exemplo de Fila em Python

import random

fila = []

def insereValor(valor, fila):
	fila.append(valor)

def buscaSentinela(chave, fila):
	fila.append(chave)
	i = 0
	while fila[i] != chave:
		i += 1
	if i == len(fila)-1:
		fila.pop()
		return -1
	fila.pop()
	return i

def deletaValor(fila):
	fila.pop(0)

def mostraLista(fila):
	print fila

for x in xrange(0, 10):
	insereValor( random.randint(10,99) , fila)

print 'Valor 25 na posicao: ' + str(buscaSentinela(25, fila))

mostraLista(fila)
deletaValor(fila)
mostraLista(fila)
deletaValor(fila)
mostraLista(fila)
deletaValor(fila)
mostraLista(fila)
insereValor( random.randint(10,99) , fila)
mostraLista(fila)
insereValor( random.randint(10,99) , fila)
mostraLista(fila)
insereValor( random.randint(10,99) , fila)
mostraLista(fila)<commit_msg>Atualize Fila em Python<commit_after>
#!/usr/bin/env python
# -*- coding: utf-8 -*-

import random

class Queue:

    def __init__(self):
        self.__queue = []

    def enqueue(self, value):
        self.__queue.append(value)

    def dequeue(self):
        return self.__queue.pop(0)

    def show(self):
        print("Queue: {}".format(self.__queue))

def main():
    queue = Queue()

    for _ in xrange(0, 10):
        queue.enqueue(random.randint(10,99))

    queue.show()

    queue.dequeue()
    queue.dequeue()

    queue.show()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># Copyright (c) 2016-2021 Renata Hodovan, Akos Kiss.
#
# Licensed under the BSD 3-Clause License
# <LICENSE.rst or https://opensource.org/licenses/BSD-3-Clause>.
# This file may not be copied, modified, or distributed except
# according to those terms.

import asyncio
import logging
import ssl
import threading

try:
    from asyncio import all_tasks as asyncio_all_tasks  # from py39, asyncio.Task.all_tasks is deprecated
except ImportError:
    asyncio_all_tasks = asyncio.Task.all_tasks  # pylint: disable=no-member

from tornado.ioloop import IOLoop
from tornado.testing import bind_unused_port
from tornado.web import Application, RequestHandler

from .fuzzer_decorator import FuzzerDecorator

logger = logging.getLogger(__name__)

class TornadoDecorator(FuzzerDecorator):
    """
    Decorator for fuzzers to transport generated content through http. It is
    useful for transporting fuzz tests to browser SUTs.

    The decorator starts a Tornado server at the start of the fuzz job and
    returns an http url as test input. If the SUT accesses the domain root
    through a GET request, then the decorated fuzzer is invoked and the
    response is the generated test. Accessing other paths can return static
    or dynamically rendered content.

    When the certfile and possibly also the keyfile optional parameters are
    defined, the traffic will be served through SSL.

    **Optional parameters of the fuzzer decorator:**

      - ``template_path``: Directory containing .html template files. These are
        served from the path ``/`` without the .html extension.
      - ``static_path``: Directory from which static files will be served.
        These are served from the path ``/static/``.
      - ``url``: Url template with ``{port}`` and ``{index}`` placeholders, that
        will be filled in with appropriate values. This is the url that will be
        served for the SUT as the test case.
        (Default: ``http(s)://localhost:{port}?index={index}``)
      - ``refresh``: Integer number denoting the time interval (in seconds)
        for the document at the root path (i.e., the test case) to refresh
        itself. Setting it to 0 means no refresh. (Default: 0)
      - ``certfile``: Path to a PEM file containing the certificate needed
        to establish the certificate’s authenticity (Default: None).
      - ``keyfile``: Path to a file containing the private key (Default: None).

    **Example configuration snippet:**

        .. code-block:: ini

            [sut.foo]
            # assuming that foo expects a http url as input, which it tries to access
            # afterwards

            [fuzz.foo-with-bar-over-http]
            sut=foo
            #fuzzer=...
            fuzzer.decorate(0)=fuzzinator.fuzzer.TornadoDecorator
            batch=5

            [fuzz.foo-with-bar-over-http.fuzzer.decorate(0)]
            template_path=/home/lili/fuzzer/templates/
            static_path=/home/lili/fuzzer/static/
            # assuming that there is a main.html in the template_path directory
            url=http://localhost:{port}/main?index={index}
            refresh=3
    """

    def __init__(self, *, template_path=None, static_path=None, url=None, refresh=None, certfile=None, keyfile=None, **kwargs):
        self.template_path = template_path
        self.static_path = static_path
        self.url = url or ('https' if certfile else 'http') + '://localhost:{port}?index={index}'
        self.refresh = int(refresh) if refresh else 0
        if certfile:
            self.ssl_ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
            self.ssl_ctx.check_hostname = False
            self.ssl_ctx.verify_mode = ssl.CERT_NONE
            self.ssl_ctx.load_cert_chain(certfile, keyfile=keyfile)
        else:
            self.ssl_ctx = None

        # Disable all the output of the tornado server to avoid messing up with Fuzzinator's messages.
        hn = logging.NullHandler()
        hn.setLevel(logging.DEBUG)
        logging.getLogger('tornado.access').addHandler(hn)
        logging.getLogger('tornado.access').propagate = False

    def init(self, cls, obj, **kwargs):
        super(cls, obj).__init__(**kwargs)
        obj.index = 0
        obj.test = None
        obj._port = None
        obj._thread = None
        obj._asyncio_loop = None
        obj._ioloop = None

    def _url(self, obj):
        return self.url.format(port=obj._port, index=obj.index)

    def _service(self, obj):
        return '{scheme}://localhost:{port}'.format(scheme='https' if self.ssl_ctx else 'http', port=obj._port)

    def call(self, cls, obj, *, index):
        if index != 0 and obj.test is None:
            return None

        return self._url(obj)

    def enter(self, cls, obj):
        decorator = self

        class MainHandler(RequestHandler):

            def data_received(self, chunk):
                pass

            def get(self):
                try:
                    obj.test = super(cls, obj).__call__(index=obj.index)

                    if obj.test is not None:
                        obj.index += 1
                        if decorator.refresh > 0:
                            self.set_header('Refresh', '{timeout}; url={url}'
                                            .format(timeout=decorator.refresh,
                                                    url=decorator._url(obj)))
                        test = obj.test
                        if not isinstance(test, (str, bytes, dict)):
                            test = str(test)
                        self.write(test)
                except Exception as e:
                    logger.warning('Unhandled exception in TornadoDecorator.', exc_info=e)

        class TemplateHandler(RequestHandler):

            def get(self, page):
                try:
                    self.render(page + '.html')
                except FileNotFoundError:
                    logger.debug('%s not found', page)
                    self.send_error(404)
                except Exception as e:
                    logger.debug('Exception while rendering %s', page, exc_info=e)
                    self.send_error(500)

        def start_tornado():
            # Create a new asyncio event loop, set it as current, and wrap it in Tornado's IOLoop.
            obj._asyncio_loop = asyncio.new_event_loop()  # save asyncio event loop so that we can cancel its tasks when shutting down
            asyncio.set_event_loop(obj._asyncio_loop)
            obj._ioloop = IOLoop.current()  # save the Tornado-wrapped event loop so that we can stop it when shutting down

            # Set up the web service (application).
            handlers = [(r'/', MainHandler)]
            if self.template_path:
                handlers += [(r'/(.+)', TemplateHandler)]

            app = Application(handlers,
                              template_path=self.template_path,
                              static_path=self.static_path,
                              debug=True)
            app.listen(obj._port, ssl_options=self.ssl_ctx)

            # Run the event loop and the application within.
            logger.debug('Starting Tornado server at %s', self._service(obj))
            obj._ioloop.start()  # block here until even loop is stopped
            obj._ioloop.close(all_fds=True)  # release port after event loop is stopped
            logger.debug('Stopped Tornado server at %s', self._service(obj))

        # Call decorated fuzzer's __enter__.
        super(cls, obj).__enter__()

        # Start Tornado in a separate thread.
        _, obj._port = bind_unused_port()  # get random available port before starting the thread, because we cannot be sure when the thread will actually start and __call__ may need it sooner
        obj._thread = threading.Thread(target=start_tornado)  # save the thread so that we can join it when shutting down
        obj._thread.start()

        return obj

    def exit(self, cls, obj, *exc):
        def stop_tornado():
            # (Ask to) cancel all pending tasks of the underlying asyncio event loop. Cancellation actually happens in a next iteration of the loop.
            for task in asyncio_all_tasks(obj._asyncio_loop):  # this is to avoid harmless(?) "ERROR:asyncio:Task was destroyed but it is pending!" messages
                task.cancel()
            # Ask to stop the event loop (after the cancellations happen).
            obj._ioloop.add_callback(obj._ioloop.stop)

        # Call decorated fuzzer's __exit__.
        suppress = super(cls, obj).__exit__(*exc)

        # Stop the thread of Tornado and wait until it terminates.
        logger.debug('Stopping Tornado server at %s', self._service(obj))
        obj._ioloop.add_callback(stop_tornado)
        obj._thread.join()

        return suppress
<commit_msg>Disable debug on the app of TornadoDecorator (#43)<commit_after># Copyright (c) 2016-2021 Renata Hodovan, Akos Kiss.
#
# Licensed under the BSD 3-Clause License
# <LICENSE.rst or https://opensource.org/licenses/BSD-3-Clause>.
# This file may not be copied, modified, or distributed except
# according to those terms.

import asyncio
import logging
import ssl
import threading

try:
    from asyncio import all_tasks as asyncio_all_tasks  # from py39, asyncio.Task.all_tasks is deprecated
except ImportError:
    asyncio_all_tasks = asyncio.Task.all_tasks  # pylint: disable=no-member

from tornado.ioloop import IOLoop
from tornado.testing import bind_unused_port
from tornado.web import Application, RequestHandler

from .fuzzer_decorator import FuzzerDecorator

logger = logging.getLogger(__name__)

class TornadoDecorator(FuzzerDecorator):
    """
    Decorator for fuzzers to transport generated content through http. It is
    useful for transporting fuzz tests to browser SUTs.

    The decorator starts a Tornado server at the start of the fuzz job and
    returns an http url as test input. If the SUT accesses the domain root
    through a GET request, then the decorated fuzzer is invoked and the
    response is the generated test. Accessing other paths can return static
    or dynamically rendered content.

    When the certfile and possibly also the keyfile optional parameters are
    defined, the traffic will be served through SSL.

    **Optional parameters of the fuzzer decorator:**

      - ``template_path``: Directory containing .html template files. These are
        served from the path ``/`` without the .html extension.
      - ``static_path``: Directory from which static files will be served.
        These are served from the path ``/static/``.
      - ``url``: Url template with ``{port}`` and ``{index}`` placeholders, that
        will be filled in with appropriate values. This is the url that will be
        served for the SUT as the test case.
        (Default: ``http(s)://localhost:{port}?index={index}``)
      - ``refresh``: Integer number denoting the time interval (in seconds)
        for the document at the root path (i.e., the test case) to refresh
        itself. Setting it to 0 means no refresh. (Default: 0)
      - ``certfile``: Path to a PEM file containing the certificate needed
        to establish the certificate’s authenticity (Default: None).
      - ``keyfile``: Path to a file containing the private key (Default: None).

    **Example configuration snippet:**

        .. code-block:: ini

            [sut.foo]
            # assuming that foo expects a http url as input, which it tries to access
            # afterwards

            [fuzz.foo-with-bar-over-http]
            sut=foo
            #fuzzer=...
            fuzzer.decorate(0)=fuzzinator.fuzzer.TornadoDecorator
            batch=5

            [fuzz.foo-with-bar-over-http.fuzzer.decorate(0)]
            template_path=/home/lili/fuzzer/templates/
            static_path=/home/lili/fuzzer/static/
            # assuming that there is a main.html in the template_path directory
            url=http://localhost:{port}/main?index={index}
            refresh=3
    """

    def __init__(self, *, template_path=None, static_path=None, url=None, refresh=None, certfile=None, keyfile=None, **kwargs):
        self.template_path = template_path
        self.static_path = static_path
        self.url = url or ('https' if certfile else 'http') + '://localhost:{port}?index={index}'
        self.refresh = int(refresh) if refresh else 0
        if certfile:
            self.ssl_ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
            self.ssl_ctx.check_hostname = False
            self.ssl_ctx.verify_mode = ssl.CERT_NONE
            self.ssl_ctx.load_cert_chain(certfile, keyfile=keyfile)
        else:
            self.ssl_ctx = None

        # Disable all the output of the tornado server to avoid messing up with Fuzzinator's messages.
        hn = logging.NullHandler()
        hn.setLevel(logging.DEBUG)
        logging.getLogger('tornado.access').addHandler(hn)
        logging.getLogger('tornado.access').propagate = False

    def init(self, cls, obj, **kwargs):
        super(cls, obj).__init__(**kwargs)
        obj.index = 0
        obj.test = None
        obj._port = None
        obj._thread = None
        obj._asyncio_loop = None
        obj._ioloop = None

    def _url(self, obj):
        return self.url.format(port=obj._port, index=obj.index)

    def _service(self, obj):
        return '{scheme}://localhost:{port}'.format(scheme='https' if self.ssl_ctx else 'http', port=obj._port)

    def call(self, cls, obj, *, index):
        if index != 0 and obj.test is None:
            return None

        return self._url(obj)

    def enter(self, cls, obj):
        decorator = self

        class MainHandler(RequestHandler):

            def data_received(self, chunk):
                pass

            def get(self):
                try:
                    obj.test = super(cls, obj).__call__(index=obj.index)

                    if obj.test is not None:
                        obj.index += 1
                        if decorator.refresh > 0:
                            self.set_header('Refresh', '{timeout}; url={url}'
                                            .format(timeout=decorator.refresh,
                                                    url=decorator._url(obj)))
                        test = obj.test
                        if not isinstance(test, (str, bytes, dict)):
                            test = str(test)
                        self.write(test)
                except Exception as e:
                    logger.warning('Unhandled exception in TornadoDecorator.', exc_info=e)

        class TemplateHandler(RequestHandler):

            def get(self, page):
                try:
                    self.render(page + '.html')
                except FileNotFoundError:
                    logger.debug('%s not found', page)
                    self.send_error(404)
                except Exception as e:
                    logger.debug('Exception while rendering %s', page, exc_info=e)
                    self.send_error(500)

        def start_tornado():
            # Create a new asyncio event loop, set it as current, and wrap it in Tornado's IOLoop.
            obj._asyncio_loop = asyncio.new_event_loop()  # save asyncio event loop so that we can cancel its tasks when shutting down
            asyncio.set_event_loop(obj._asyncio_loop)
            obj._ioloop = IOLoop.current()  # save the Tornado-wrapped event loop so that we can stop it when shutting down

            # Set up the web service (application).
            handlers = [(r'/', MainHandler)]
            if self.template_path:
                handlers += [(r'/(.+)', TemplateHandler)]

            app = Application(handlers,
                              template_path=self.template_path,
                              static_path=self.static_path,
                              debug=False)
            app.listen(obj._port, ssl_options=self.ssl_ctx)

            # Run the event loop and the application within.
            logger.debug('Starting Tornado server at %s', self._service(obj))
            obj._ioloop.start()  # block here until even loop is stopped
            obj._ioloop.close(all_fds=True)  # release port after event loop is stopped
            logger.debug('Stopped Tornado server at %s', self._service(obj))

        # Call decorated fuzzer's __enter__.
        super(cls, obj).__enter__()

        # Start Tornado in a separate thread.
        _, obj._port = bind_unused_port()  # get random available port before starting the thread, because we cannot be sure when the thread will actually start and __call__ may need it sooner
        obj._thread = threading.Thread(target=start_tornado)  # save the thread so that we can join it when shutting down
        obj._thread.start()

        return obj

    def exit(self, cls, obj, *exc):
        def stop_tornado():
            # (Ask to) cancel all pending tasks of the underlying asyncio event loop. Cancellation actually happens in a next iteration of the loop.
            for task in asyncio_all_tasks(obj._asyncio_loop):  # this is to avoid harmless(?) "ERROR:asyncio:Task was destroyed but it is pending!" messages
                task.cancel()
            # Ask to stop the event loop (after the cancellations happen).
            obj._ioloop.add_callback(obj._ioloop.stop)

        # Call decorated fuzzer's __exit__.
        suppress = super(cls, obj).__exit__(*exc)

        # Stop the thread of Tornado and wait until it terminates.
        logger.debug('Stopping Tornado server at %s', self._service(obj))
        obj._ioloop.add_callback(stop_tornado)
        obj._thread.join()

        return suppress
<|endoftext|>


<commit_before>import logging

import braintree
from django.conf import settings
from django.core import serializers
from django.core.mail import EmailMessage
from django.db import transaction
from django.utils import timezone
import requests

from gchatautorespond.lib import report_ga_event_async
from .models import License, CurrentLicense

logger = logging.getLogger(__name__)

class UnsuccessfulResultError(Exception):
    def __init__(self, message, result):
        super(UnsuccessfulResultError, self).__init__(message)
        self.result = result

def raise_unless_successful(bt_result):
    if not bt_result.is_success:
        raise UnsuccessfulResultError('', bt_result)

    logger.info("%s", bt_result)

def report_bt_event(email, result, object, action):
    value = 1 if result.is_success else -1
    report_ga_event_async(email, category=object, action=action, label='braintree', value=value)

def _transition_autoresponses(action, user):
    for autorespond in user.autoresponse_set.all():
        requests.post("http://127.0.0.1:%s/%s/%s" % (settings.WORKER_PORT, action, autorespond.id))

def notify_of_trial_expiry(user):
    paragraphs = [
        'Hi there!',
        'Your trial at https://gchat.simon.codes has just expired and your autoresponses have been disabled.',
        'To enable them again, buy a subscription at https://gchat.simon.codes/payment/detail.',
        "It's only %s per month -- less than a cup of coffee!" % settings.PRICE_REPR,
    ]
    email = EmailMessage(
        subject='trial expired, autoresponses disabled',
        to=[user.email],
        body='\n\n'.join(paragraphs),
        reply_to=['noreply@gchat.simon.codes'],
    )
    email.send(fail_silently=False)
    logger.info("sent trial expiry email to %r", user.email)

def save_init_license(user):
    with transaction.atomic():
        license = License.objects.create(user=user, trial_start=timezone.now())
        CurrentLicense.objects.create(user=user, license=license)

def get_default_payment_method(customer):
    return [m for m in customer.payment_methods if m.default][0]

def transition_license(license, new_fields):
    prev_was_active = license.is_active

    license.id = None  # django will create a new row
    for (field, val) in new_fields.items():
        setattr(license, field, val)
    license.save()
    new_is_active = license.is_active

    user = license.user
    user.currentlicense.license = license
    user.currentlicense.save()

    logger.info("current license for %s now %r", user.id, serializers.serialize("json", [license]))

    if prev_was_active and not new_is_active:
        _transition_autoresponses('stop', user)
    elif not prev_was_active and new_is_active:
        _transition_autoresponses('restart', user)

    return license

@transaction.atomic
def sync_license(license, bt_subscription):
    has_changed = False

    logger.info("comparing license %r to bt sub %r", serializers.serialize("json", [license]), bt_subscription)
    if license.bt_subscription_id != bt_subscription.id:
        logger.info("subscription id has changed")
        has_changed = True
    if license.bt_status != bt_subscription.status:
        logger.info("subscription status has changed")
        has_changed = True
        if bt_subscription.status == braintree.Subscription.Status.Canceled:
            bt_subscription.id = ''

    if has_changed:
        transition_license(
            license,
            {
                'bt_subscription_id': bt_subscription.id,
                'bt_status': bt_subscription.status,
            }
        )
<commit_msg>change trial expiry copy<commit_after>import logging

import braintree
from django.conf import settings
from django.core import serializers
from django.core.mail import EmailMessage
from django.db import transaction
from django.utils import timezone
import requests

from gchatautorespond.lib import report_ga_event_async
from .models import License, CurrentLicense

logger = logging.getLogger(__name__)

class UnsuccessfulResultError(Exception):
    def __init__(self, message, result):
        super(UnsuccessfulResultError, self).__init__(message)
        self.result = result

def raise_unless_successful(bt_result):
    if not bt_result.is_success:
        raise UnsuccessfulResultError('', bt_result)

    logger.info("%s", bt_result)

def report_bt_event(email, result, object, action):
    value = 1 if result.is_success else -1
    report_ga_event_async(email, category=object, action=action, label='braintree', value=value)

def _transition_autoresponses(action, user):
    for autorespond in user.autoresponse_set.all():
        requests.post("http://127.0.0.1:%s/%s/%s" % (settings.WORKER_PORT, action, autorespond.id))

def notify_of_trial_expiry(user):
    paragraphs = [
        'Hi there!',
        'Your trial at https://gchat.simon.codes has just expired and your autoresponses have been disabled.',
        'To enable them again, buy a subscription at https://gchat.simon.codes/payment/detail.',
        "It's only %s per month and can be cancelled at any time." % settings.PRICE_REPR,
    ]
    email = EmailMessage(
        subject='trial expired, autoresponses disabled',
        to=[user.email],
        body='\n\n'.join(paragraphs),
        reply_to=['noreply@gchat.simon.codes'],
    )
    email.send(fail_silently=False)
    logger.info("sent trial expiry email to %r", user.email)

def save_init_license(user):
    with transaction.atomic():
        license = License.objects.create(user=user, trial_start=timezone.now())
        CurrentLicense.objects.create(user=user, license=license)

def get_default_payment_method(customer):
    return [m for m in customer.payment_methods if m.default][0]

def transition_license(license, new_fields):
    prev_was_active = license.is_active

    license.id = None  # django will create a new row
    for (field, val) in new_fields.items():
        setattr(license, field, val)
    license.save()
    new_is_active = license.is_active

    user = license.user
    user.currentlicense.license = license
    user.currentlicense.save()

    logger.info("current license for %s now %r", user.id, serializers.serialize("json", [license]))

    if prev_was_active and not new_is_active:
        _transition_autoresponses('stop', user)
    elif not prev_was_active and new_is_active:
        _transition_autoresponses('restart', user)

    return license

@transaction.atomic
def sync_license(license, bt_subscription):
    has_changed = False

    logger.info("comparing license %r to bt sub %r", serializers.serialize("json", [license]), bt_subscription)
    if license.bt_subscription_id != bt_subscription.id:
        logger.info("subscription id has changed")
        has_changed = True
    if license.bt_status != bt_subscription.status:
        logger.info("subscription status has changed")
        has_changed = True
        if bt_subscription.status == braintree.Subscription.Status.Canceled:
            bt_subscription.id = ''

    if has_changed:
        transition_license(
            license,
            {
                'bt_subscription_id': bt_subscription.id,
                'bt_status': bt_subscription.status,
            }
        )
<|endoftext|>


<commit_before>from django.conf import settings
from django.conf.urls import patterns, include, url
from django.conf.urls.static import static
from django.views.generic import TemplateView

from django.contrib import admin

import symposion.views

WIKI_SLUG = r"(([\w-]{2,})(/[\w-]{2,})*)"

urlpatterns = patterns("",
    url(r"^admin/", include(admin.site.urls)),

    url(r"^$", TemplateView.as_view(template_name="homepage.html"), name="home"),
    url(r"^account/signup/$", symposion.views.SignupView.as_view(), name="account_signup"),
    url(r"^account/login/$", symposion.views.LoginView.as_view(), name="account_login"),
    url(r"^account/", include("account.urls")),
    url(r"^dashboard/", symposion.views.dashboard, name="dashboard"),
    url(r"^speaker/", include("symposion.speakers.urls")),
    url(r"^proposals/", include("symposion.proposals.urls")),
    url(r"^sponsors/", include("symposion.sponsorship.urls")),
    url(r"^teams/", include("symposion.teams.urls")),
    url(r"^reviews/", include("symposion.reviews.urls")),
    url(r"^schedule/", include("symposion.schedule.urls")),
    url(r"^markitup/", include("markitup.urls")),
)

urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
<commit_msg>Use list in urls instead of patterns.<commit_after>from django.conf import settings
from django.conf.urls import include, url
from django.conf.urls.static import static
from django.views.generic import TemplateView

from django.contrib import admin

import symposion.views

WIKI_SLUG = r"(([\w-]{2,})(/[\w-]{2,})*)"

urlpatterns = [
    url(r"^admin/", include(admin.site.urls)),

    url(r"^$", TemplateView.as_view(template_name="homepage.html"), name="home"),
    url(r"^account/signup/$", symposion.views.SignupView.as_view(), name="account_signup"),
    url(r"^account/login/$", symposion.views.LoginView.as_view(), name="account_login"),
    url(r"^account/", include("account.urls")),
    url(r"^dashboard/", symposion.views.dashboard, name="dashboard"),
    url(r"^speaker/", include("symposion.speakers.urls")),
    url(r"^proposals/", include("symposion.proposals.urls")),
    url(r"^sponsors/", include("symposion.sponsorship.urls")),
    url(r"^teams/", include("symposion.teams.urls")),
    url(r"^reviews/", include("symposion.reviews.urls")),
    url(r"^schedule/", include("symposion.schedule.urls")),
    url(r"^markitup/", include("markitup.urls")),
]

urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
<|endoftext|>


<commit_before>"""
Options for managing Confab.
"""

from fabric.api import task
from fabric.utils import _AttributeDict

from difflib import unified_diff
from magic import Magic
from re import match

def _should_render(mime_type):
    """
    Return whether a template file of with a particular mime type
    should be rendered.

    Some files may need to be excluded from template rendering;
    such files will be copied verbatim.
    """
    return next((True for pattern in ['text/', 'application/xml'] if match(pattern, mime_type)),
                False)

def _is_empty(mime_type):
    """
    Return whether a template file is an empty file.
    """
    return mime_type == 'inode/x-empty'

def _is_not_temporary(file_name):
    """
    Return whether a file name does not represent a temporary file.

    When listing configuration files, we usually want temporary
    files to be ignored.
    """
    return not file_name.endswith('~')

def _get_mime_type(file_name):
    """
    Return the mime type of a file.

    The mime_type will be used to determine if a configuration file is text.
    """
    return Magic(mime=True).from_file(file_name)

def _diff(a, b, fromfile=None, tofile=None):
    """
    Return a diff using '---', '+++', and '@@' control lines.

    By default, uses unified_diff.
    """
    return unified_diff(a, b, fromfile=fromfile, tofile=tofile)

def _as_dict(module):
    """
    Returns publicly names values in module's __dict__.
    """
    try:
        return {k: v for k, v in module.__dict__.iteritems() if not k[0:1] == '_'}
    except AttributeError:
        return None

# Options that control how confab runs.
#
# These are in opposition to options likely to changed
# between different runs of confab, such as directories,
# environments, roles, hosts, etc.
options = _AttributeDict({
    # Should yes be assumed for interactive prompts?
    'assume_yes': False,

    # How to compute a file's mime_type?
    'get_mime_type': _get_mime_type,

    # How to determine if a template should be rendered?
    'should_render': _should_render,

    # How to determine if a template is an empty file?
    'is_empty': _is_empty,

    # How do filter available templates within the jinja environment?
    'filter_func': _is_not_temporary,

    # How to determine diffs?
    'diff': _diff,

    # How to get dictionary configuration from module data?
    'module_as_dict': _as_dict,

    # What is the name of the template directory?
    'get_templates_dir': lambda: 'templates',

    # What is the name of the data directory?
    'get_data_dir': lambda: 'data',

    # What is the name of the generated directory?
    'get_generated_dir': lambda: 'generated',

    # What is the name of the remotes directory?
    'get_remotes_dir': lambda: 'remotes',
})

class Options(object):
    """
    Context manager to temporarily set options.
    """

    def __init__(self, **kwargs):
        self.kwargs = kwargs
        self.previous = {}

    def __enter__(self):
        for (k, v) in self.kwargs.iteritems():
            self.previous[k] = options.get(k)
            options[k] = v
        return self

    def __exit__(self, exc_type, value, traceback):
        for k in self.kwargs.keys():
            options[k] = self.previous[k]

@task
def assume_yes():
    """
    Set the option to assume_yes in other tasks.
    """
    options.assume_yes = True
<commit_msg>Ignore internal template file names to allow includes more nicely.<commit_after>"""
Options for managing Confab.
"""
from os.path import basename
from fabric.api import task
from fabric.utils import _AttributeDict

from difflib import unified_diff
from magic import Magic
from re import match

def _should_render(mime_type):
    """
    Return whether a template file of with a particular mime type
    should be rendered.

    Some files may need to be excluded from template rendering;
    such files will be copied verbatim.
    """
    return next((True for pattern in ['text/', 'application/xml'] if match(pattern, mime_type)),
                False)

def _is_empty(mime_type):
    """
    Return whether a template file is an empty file.
    """
    return mime_type == 'inode/x-empty'

def _is_not_temporary(file_name):
    """
    Return whether a file name does not represent a temporary file.

    When listing configuration files, we usually want temporary
    files to be ignored.
    """
    return not file_name.endswith('~')

def _is_not_internal(file_name):
    """
    Return whether a file name does not represent internal usage.

    When listing configuration files, we want to omit internal
    files, especially if they are used as Jinja includes
    """
    return not basename(file_name).startswith('_')

def _filter_func(file_name):
    """
    Return the default filter func, which excludes temporary and internal files.
    """
    return _is_not_temporary(file_name) and _is_not_internal(file_name)

def _get_mime_type(file_name):
    """
    Return the mime type of a file.

    The mime_type will be used to determine if a configuration file is text.
    """
    return Magic(mime=True).from_file(file_name)

def _diff(a, b, fromfile=None, tofile=None):
    """
    Return a diff using '---', '+++', and '@@' control lines.

    By default, uses unified_diff.
    """
    return unified_diff(a, b, fromfile=fromfile, tofile=tofile)

def _as_dict(module):
    """
    Returns publicly names values in module's __dict__.
    """
    try:
        return {k: v for k, v in module.__dict__.iteritems() if not k[0:1] == '_'}
    except AttributeError:
        return None

# Options that control how confab runs.
#
# These are in opposition to options likely to changed
# between different runs of confab, such as directories,
# environments, roles, hosts, etc.
options = _AttributeDict({
    # Should yes be assumed for interactive prompts?
    'assume_yes': False,

    # How to compute a file's mime_type?
    'get_mime_type': _get_mime_type,

    # How to determine if a template should be rendered?
    'should_render': _should_render,

    # How to determine if a template is an empty file?
    'is_empty': _is_empty,

    # How do filter available templates within the jinja environment?
    'filter_func': _filter_func,

    # How to determine diffs?
    'diff': _diff,

    # How to get dictionary configuration from module data?
    'module_as_dict': _as_dict,

    # What is the name of the template directory?
    'get_templates_dir': lambda: 'templates',

    # What is the name of the data directory?
    'get_data_dir': lambda: 'data',

    # What is the name of the generated directory?
    'get_generated_dir': lambda: 'generated',

    # What is the name of the remotes directory?
    'get_remotes_dir': lambda: 'remotes',
})

class Options(object):
    """
    Context manager to temporarily set options.
    """

    def __init__(self, **kwargs):
        self.kwargs = kwargs
        self.previous = {}

    def __enter__(self):
        for (k, v) in self.kwargs.iteritems():
            self.previous[k] = options.get(k)
            options[k] = v
        return self

    def __exit__(self, exc_type, value, traceback):
        for k in self.kwargs.keys():
            options[k] = self.previous[k]

@task
def assume_yes():
    """
    Set the option to assume_yes in other tasks.
    """
    options.assume_yes = True
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import argparse
import os
import numpy as np

import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
plt.style.use('bmh')

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('data', type=str, help='dir contains outputs of DDPG, NAF and ICNN')
    parser.add_argument('--xmax', type=float)
    parser.add_argument('--ymin', type=float)
    parser.add_argument('--ymax', type=float)
    args = parser.parse_args()

    names = ['DDPG', 'NAF', 'ICNN']
    colors = ['red', 'blue', 'green']

    fig, ax = plt.subplots(1, 1, figsize=(6, 5))
    plt.xlabel('Timestep')
    plt.ylabel('Reward')

    lines = []
    for name, color in zip(names, colors):
        dirpath = os.path.join(args.data, name)
        if os.path.isdir(dirpath):
            X, Ymin, Ymax, Ymean = get_data(dirpath)
            line, = plt.plot(X, Ymean, label=name, color=color)
            lines.append(line)
            plt.fill_between(X, Ymin, Ymax, alpha=0.1, color=color)
    plt.ylim([args.ymin, args.ymax])
    plt.legend(handles=lines, loc=2)
    fname = os.path.join(args.data, 'result.pdf')
    plt.savefig(fname)
    print('Created {}'.format(fname))

def get_data(dirpath):
    minX, maxX = None, None
    X, Y = [], []
    for d in os.listdir(dirpath):
        logName = os.path.join(dirpath, d, 'log.txt')
        if not os.path.exists(logName):
            logName = os.path.join(dirpath, d, 'test.log')
            if not os.path.exists(logName):
                print("Log file not found for: {}".format(os.path.join(dirpath, d)))
                continue

        data = np.loadtxt(logName)
        x = data[:,0].ravel()
        y = data[:,1].ravel()

        minX = np.min(x) if not minX else min(np.min(x), minX)
        maxX = np.max(x) if not maxX else max(np.max(x), maxX)
        X.append(x)
        Y.append(y)

    xs = np.linspace(minX, maxX, 3000)

    interpY = []
    for x,y in zip(X,Y):
        interpY.append(np.interp(xs, x, y))

    Y = np.asarray(interpY)
    Ysdom = 1.96 * np.std(Y, axis=0) / np.sqrt(Y.shape[0])
    Ymean = np.mean(Y, axis=0)

    return xs, Ymean - Ysdom, Ymean + Ysdom, Ymean

if __name__ == '__main__':
    main()
<commit_msg>RL.plot: Default ymin=0 and sample more points.<commit_after>#!/usr/bin/env python3

import argparse
import os
import numpy as np

import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
plt.style.use('bmh')

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('data', type=str, help='dir contains outputs of DDPG, NAF and ICNN')
    parser.add_argument('--xmax', type=float)
    parser.add_argument('--ymin', type=float, default=0.0)
    parser.add_argument('--ymax', type=float)
    args = parser.parse_args()

    names = ['DDPG', 'NAF', 'ICNN']
    colors = ['red', 'blue', 'green']

    fig, ax = plt.subplots(1, 1, figsize=(6, 5))
    plt.xlabel('Timestep')
    plt.ylabel('Reward')

    lines = []
    for name, color in zip(names, colors):
        dirpath = os.path.join(args.data, name)
        if os.path.isdir(dirpath):
            X, Ymin, Ymax, Ymean = get_data(dirpath)
            line, = plt.plot(X, Ymean, label=name, color=color)
            lines.append(line)
            plt.fill_between(X, Ymin, Ymax, alpha=0.1, color=color)
    plt.ylim([args.ymin, args.ymax])
    plt.legend(handles=lines, loc=2)
    fname = os.path.join(args.data, 'result.pdf')
    plt.savefig(fname)
    print('Created {}'.format(fname))

def get_data(dirpath):
    minX, maxX = None, None
    X, Y = [], []
    for d in os.listdir(dirpath):
        logName = os.path.join(dirpath, d, 'log.txt')
        if not os.path.exists(logName):
            logName = os.path.join(dirpath, d, 'test.log')
            if not os.path.exists(logName):
                print("Log file not found for: {}".format(os.path.join(dirpath, d)))
                continue

        data = np.loadtxt(logName)
        x = data[:,0].ravel()
        y = data[:,1].ravel()

        minX = np.min(x) if not minX else min(np.min(x), minX)
        maxX = np.max(x) if not maxX else max(np.max(x), maxX)
        X.append(x)
        Y.append(y)

    xs = np.linspace(minX, maxX, 10000)

    interpY = []
    for x,y in zip(X,Y):
        interpY.append(np.interp(xs, x, y))

    Y = np.asarray(interpY)
    Ysdom = 1.96 * np.std(Y, axis=0) / np.sqrt(Y.shape[0])
    Ymean = np.mean(Y, axis=0)

    return xs, Ymean - Ysdom, Ymean + Ysdom, Ymean

if __name__ == '__main__':
    main()
<|endoftext|>