<commit_before>#!/usr/bin/env python
#
# -----------------------------------------------------------------------------
# Copyright (c) 2017 The Regents of the University of California
#
# This file is part of kevlar (http://github.com/standage/kevlar) and is
# licensed under the MIT license: see LICENSE.
# -----------------------------------------------------------------------------

from __future__ import print_function
from collections import defaultdict, namedtuple
import argparse
import itertools
import sys
try:
    import networkx
except:
    pass
import screed
import khmer
import kevlar

def subparser(subparsers):
    subparser = subparsers.add_parser('assemble')
    subparser.add_argument('-d', '--debug', action='store_true',
                           help='print debugging output')
    subparser.add_argument('-o', '--out', metavar='FILE',
                           help='output file; default is terminal (stdout)')
    subparser.add_argument('--gml', metavar='FILE',
                           help='write graph to .gml file')
    subparser.add_argument('-x', '--max-abund', type=int, metavar='X',
                           default=500, help='discard interesting k-mers that '
                           'occur more than X times')
    subparser.add_argument('augfastq', help='annotated reads in augmented '
                           'Fastq format')

def merge_pair(pair):
    """
    Assemble a pair of overlapping reads.

    Given a pair of compatible overlapping reads, collapse and merge them into
    a single sequence.
    """
    headseq = pair.head.sequence
    if pair.sameorient is False:
        headseq = kevlar.revcom(pair.head.sequence)
    headindex = len(pair.tail.sequence) - pair.offset
    headsuffix = headseq[headindex:]
    tailprefix = pair.tail.sequence[pair.offset:pair.offset+pair.overlap]
    assert tailprefix == headseq[:headindex], \
        'error: attempted to assemble incompatible reads'

    if headseq in pair.tail.sequence:
        return pair.tail.sequence
    return pair.tail.sequence + headsuffix

def merge_and_reannotate(pair, newname):
    """
    Assemble a pair of overlapping reads and resolve their interesting k-mers.

    When a pair of compatible reads is merged, the offset of the interesting
    k-mers must be computed for one of the reads.
    """
    contig = merge_pair(pair)
    newrecord = screed.Record(name=newname, sequence=contig,
                              ikmers=pair.tail.ikmers)
    ksize = len(pair.tail.ikmers[0].sequence)
    if pair.sameorient:
        minoffset2keep = len(pair.tail.sequence) - pair.offset - ksize
        keepers = [ik for ik in pair.head.ikmers if ik.offset > minoffset2keep]
        for k in keepers:
            ikmer = kevlar.KmerOfInterest(k.sequence, k.offset + pair.offset,
                                          k.abund)
            newrecord.ikmers.append(ikmer)
    else:
        maxoffset2keep = pair.offset - ksize
        keepers = [ik for ik in pair.head.ikmers if ik.offset < maxoffset2keep]
        for k in keepers:
            ikmer = kevlar.KmerOfInterest(
                kevlar.revcom(k.sequence),
                len(pair.head.sequence) - k.offset - ksize + pair.offset,
                k.abund,
            )
            newrecord.ikmers.append(ikmer)

    return newrecord

def load_reads(instream, logstream=None):
    """
    Load reads into lookup tables for convenient access.

    The first table is a dictionary of reads indexed by read name, and the
    second table is a dictionary of read sets indexed by an interesting k-mer.
    """
    reads = dict()
    kmers = defaultdict(set)
    for n, record in enumerate(kevlar.parse_augmented_fastq(instream), 1):
        if logstream and n % 10000 == 0:
            print('[kevlar::assemble]    loaded {:d} reads'.format(n),
                  file=logstream)
        reads[record.name] = record
        for kmer in record.ikmers:
            kmerseq = kevlar.revcommin(kmer.sequence)
            kmers[kmerseq].add(record.name)
    return reads, kmers

def graph_init(reads, kmers, maxabund=500, logstream=None):
    graph = networkx.Graph()
    nkmers = len(kmers)
    for n, minkmer in enumerate(kmers, 1):
        if n % 100 == 0:
            msg = 'processed {:d}/{:d} shared novel k-mers'.format(n, nkmers)
            print('[kevlar::assemble]    ', msg, sep='', file=logstream)
        readnames = kmers[minkmer]
        if maxabund and len(readnames) > maxabund:
            msg = '            skipping k-mer with abundance {:d}'.format(
                len(readnames)
            )
            print(msg, file=logstream)
            continue
        if len(readnames) < 2:
            message = '[kevlar::assemble] WARNING: k-mer {}'.format(minkmer)
            message += ' (rev. comp. {})'.format(kevlar.revcom(minkmer))
            message += ' only has abundance {:d}'.format(len(readnames))
            out = logstream if logstream is not None else sys.stderr
            print(message, file=out)
            continue
        readset = [reads[rn] for rn in readnames]
        for read1, read2 in itertools.combinations(readset, 2):
            pair = kevlar.overlap.calc_offset(read1, read2, minkmer, logstream)
            if pair is kevlar.overlap.INCOMPATIBLE_PAIR:
                # Shared k-mer but bad overlap
                continue
            tailname, headname = pair.tail.name, pair.head.name
            if tailname in graph and headname in graph[tailname]:
                assert graph[tailname][headname]['offset'] == pair.offset
                if graph[tailname][headname]['tail'] == tailname:
                    assert graph[tailname][headname]['overlap'] == pair.overlap
            else:
                graph.add_edge(tailname, headname, offset=pair.offset,
                               overlap=pair.overlap, ikmer=minkmer,
                               orient=pair.sameorient, tail=tailname)
    return graph

def assemble_with_greed(reads, kmers, graph):
    pass

def main(args):
    debugout = None
    if args.debug:
        debugout = args.logfile

    reads, kmers = load_reads(kevlar.open(args.augfastq, 'r'), debugout)
    inputreads = list(reads.keys())
    graph = graph_init(reads, kmers, args.max_abund, debugout)
    if args.gml:
        networkx.write_gml(graph, args.gml)
        message = '[kevlar::assemble] graph written to {}'.format(args.gml)
        print(message, file=args.logfile)

    ccs = list(networkx.connected_components(graph))
    assert len(ccs) == 1

    count = 0
    while len(graph.edges()) > 0:
        count += 1
        # Sort the edges using 3 criteria. The first is the primary criterion,
        # the other two ensure deterministic behavior.
        #     - overlap (largest first)
        #     - lexicographically smaller read name
        #     - lexicographically larger read name
        edges = sorted(
            graph.edges(),
            reverse=True,
            key=lambda e: (
                graph[e[0]][e[1]]['overlap'],
                max(e),
                min(e),
            )
        )
        read1, read2 = edges[0]  # biggest overlap (greedy algorithm)
        if read2 == graph[read1][read2]['tail']:
            read1, read2 = read2, read1
        pair = kevlar.overlap.OverlappingReadPair(
            tail=reads[read1],
            head=reads[read2],
            offset=graph[read1][read2]['offset'],
            overlap=graph[read1][read2]['overlap'],
            sameorient=graph[read1][read2]['orient'],
        )
        newname = 'contig{:d}'.format(count)
        newrecord = merge_and_reannotate(pair, newname)
        if debugout:
            print('### DEBUG', read1, read2, pair.offset, pair.overlap,
                  pair.sameorient, file=debugout)
            kevlar.print_augmented_fastq(newrecord, debugout)
        for kmer in newrecord.ikmers:
            kmerseq = kevlar.revcommin(kmer.sequence)
            for readname in kmers[kmerseq]:
                already_merged = readname not in graph
                current_contig = readname in [read1, read2, newname]
                if already_merged or current_contig:
                    continue
                otherrecord = reads[readname]
                newpair = kevlar.overlap.calc_offset(
                    newrecord, otherrecord, kmerseq, debugout
                )
                if newpair == kevlar.overlap.INCOMPATIBLE_PAIR:
                    continue
                tn, hn = newpair.tail.name, newpair.head.name
                if tn in graph and hn in graph[tn]:
                    assert graph[tn][hn]['overlap'] == newpair.overlap
                    if graph[tn][hn]['tail'] == newpair.tail:
                        assert graph[tn][hn]['offset'] == newpair.offset
                else:
                    graph.add_edge(tn, hn, offset=newpair.offset,
                                   overlap=newpair.overlap, ikmer=kmerseq,
                                   orient=newpair.sameorient, tail=tn)
            kmers[kmerseq].add(newrecord.name)
        reads[newrecord.name] = newrecord
        graph.add_node(newrecord.name)
        graph.remove_node(read1)
        graph.remove_node(read2)

    contigcount = 0
    unassembledcount = 0
    outstream = kevlar.open(args.out, 'w')
    for seqname in graph.nodes():
        if seqname in inputreads:
            unassembledcount += 1
            continue
        contigcount += 1
        contigrecord = reads[seqname]
        kevlar.print_augmented_fastq(contigrecord, outstream)

    assembledcount = len(inputreads) - unassembledcount
    message = '[kevlar::assemble] assembled'
    message += ' {:d}/{:d} reads'.format(assembledcount, len(inputreads))
    message += ' into {:d} contig(s)'.format(contigcount)
    print(message, file=args.logfile)
<commit_msg>Refactor assembly code<commit_after>#!/usr/bin/env python
#
# -----------------------------------------------------------------------------
# Copyright (c) 2017 The Regents of the University of California
#
# This file is part of kevlar (http://github.com/standage/kevlar) and is
# licensed under the MIT license: see LICENSE.
# -----------------------------------------------------------------------------

from __future__ import print_function
from collections import defaultdict, namedtuple
import argparse
import itertools
import sys
try:
    import networkx
except:
    pass
import screed
import khmer
import kevlar

def subparser(subparsers):
    subparser = subparsers.add_parser('assemble')
    subparser.add_argument('-d', '--debug', action='store_true',
                           help='print debugging output')
    subparser.add_argument('-o', '--out', metavar='FILE',
                           help='output file; default is terminal (stdout)')
    subparser.add_argument('--gml', metavar='FILE',
                           help='write graph to .gml file')
    subparser.add_argument('-x', '--max-abund', type=int, metavar='X',
                           default=500, help='discard interesting k-mers that '
                           'occur more than X times')
    subparser.add_argument('augfastq', help='annotated reads in augmented '
                           'Fastq format')

def merge_pair(pair):
    """
    Assemble a pair of overlapping reads.

    Given a pair of compatible overlapping reads, collapse and merge them into
    a single sequence.
    """
    headseq = pair.head.sequence
    if pair.sameorient is False:
        headseq = kevlar.revcom(pair.head.sequence)
    headindex = len(pair.tail.sequence) - pair.offset
    headsuffix = headseq[headindex:]
    tailprefix = pair.tail.sequence[pair.offset:pair.offset+pair.overlap]
    assert tailprefix == headseq[:headindex], \
        'error: attempted to assemble incompatible reads'

    if headseq in pair.tail.sequence:
        return pair.tail.sequence
    return pair.tail.sequence + headsuffix

def merge_and_reannotate(pair, newname):
    """
    Assemble a pair of overlapping reads and resolve their interesting k-mers.

    When a pair of compatible reads is merged, the offset of the interesting
    k-mers must be computed for one of the reads.
    """
    contig = merge_pair(pair)
    newrecord = screed.Record(name=newname, sequence=contig,
                              ikmers=pair.tail.ikmers)
    ksize = len(pair.tail.ikmers[0].sequence)
    if pair.sameorient:
        minoffset2keep = len(pair.tail.sequence) - pair.offset - ksize
        keepers = [ik for ik in pair.head.ikmers if ik.offset > minoffset2keep]
        for k in keepers:
            ikmer = kevlar.KmerOfInterest(k.sequence, k.offset + pair.offset,
                                          k.abund)
            newrecord.ikmers.append(ikmer)
    else:
        maxoffset2keep = pair.offset - ksize
        keepers = [ik for ik in pair.head.ikmers if ik.offset < maxoffset2keep]
        for k in keepers:
            ikmer = kevlar.KmerOfInterest(
                kevlar.revcom(k.sequence),
                len(pair.head.sequence) - k.offset - ksize + pair.offset,
                k.abund,
            )
            newrecord.ikmers.append(ikmer)

    return newrecord

def load_reads(instream, logstream=None):
    """
    Load reads into lookup tables for convenient access.

    The first table is a dictionary of reads indexed by read name, and the
    second table is a dictionary of read sets indexed by an interesting k-mer.
    """
    reads = dict()
    kmers = defaultdict(set)
    for n, record in enumerate(kevlar.parse_augmented_fastq(instream), 1):
        if logstream and n % 10000 == 0:  # pragma: no cover
            print('[kevlar::assemble]    loaded {:d} reads'.format(n),
                  file=logstream)
        reads[record.name] = record
        for kmer in record.ikmers:
            kmerseq = kevlar.revcommin(kmer.sequence)
            kmers[kmerseq].add(record.name)
    return reads, kmers

def graph_init_abund_check_pass(numreads, minkmer, minabund=5, maxabund=500,
                                logstream=None):
    """Check whether the k-mer falls within the expected range of abundance."""
    if maxabund and numreads > maxabund:
        msg = '            skipping k-mer with abundance {:d}'.format(numreads)
        print(msg, file=logstream)
        return False
    if minabund and numreads < minabund:
        message = '[kevlar::assemble] WARNING: k-mer {}'.format(minkmer)
        message += ' (rev. comp. {})'.format(kevlar.revcom(minkmer))
        message += ' only has abundance {:d}'.format(len(readnames))
        out = logstream if logstream is not None else sys.stderr
        print(message, file=out)
        return False
    return True

def graph_add_edge(graph, pair, minkmer):
    """
    Add edge between two nodes in the "shared interesting k-mer" read graph.

    If the edge already exists, make sure that the existing edge matches the
    edge that would have been added.
    """
    tailname, headname = pair.tail.name, pair.head.name
    if tailname in graph and headname in graph[tailname]:
        assert graph[tailname][headname]['offset'] == pair.offset
        if graph[tailname][headname]['tail'] == tailname:
            assert graph[tailname][headname]['overlap'] == pair.overlap
        graph[tailname][headname]['ikmers'].add(minkmer)
    else:
        graph.add_edge(tailname, headname, offset=pair.offset,
                       overlap=pair.overlap, ikmers=set([minkmer]),
                       orient=pair.sameorient, tail=tailname)

def graph_init(reads, kmers, minabund=5, maxabund=500, logstream=None):
    """
    Initialize the "shared interesting k-mer" read graph.

    Iterate through each interesting k-mer, consider every pair of reads
    containing that k-mer, and determine whether there should be a edge between
    the pair of reads in the graph.
    """
    graph = networkx.Graph()
    nkmers = len(kmers)
    for n, minkmer in enumerate(kmers, 1):
        if n % 100 == 0:  # pragma: no cover
            msg = 'processed {:d}/{:d} shared novel k-mers'.format(n, nkmers)
            print('[kevlar::assemble]    ', msg, sep='', file=logstream)

        readnames = kmers[minkmer]
        if not graph_init_abund_check_pass(len(readnames), minkmer, minabund,
                                           maxabund, logstream):
            continue

        readset = [reads[rn] for rn in readnames]
        for read1, read2 in itertools.combinations(readset, 2):
            pair = kevlar.overlap.calc_offset(read1, read2, minkmer, logstream)
            if pair is kevlar.overlap.INCOMPATIBLE_PAIR:
                # Shared k-mer but bad overlap
                continue
            graph_add_edge(graph, pair, minkmer)
    return graph

def fetch_largest_overlapping_pair(graph, reads):
    """
    Grab the edge with the largest overlap in the graph.

    Sort the edges using 3 criteria. The first is the primary criterion, the
    other two ensure deterministic behavior.
        - overlap (largest first)
        - lexicographically smaller read name
        - lexicographically larger read name
    """
    edges = sorted(
        graph.edges(),
        reverse=True,
        key=lambda e: (
            graph[e[0]][e[1]]['overlap'],
            max(e),
            min(e),
        )
    )
    read1, read2 = edges[0]  # biggest overlap (greedy algorithm)
    if read2 == graph[read1][read2]['tail']:
        read1, read2 = read2, read1
    return kevlar.overlap.OverlappingReadPair(
        tail=reads[read1],
        head=reads[read2],
        offset=graph[read1][read2]['offset'],
        overlap=graph[read1][read2]['overlap'],
        sameorient=graph[read1][read2]['orient'],
    )

def assemble_with_greed(reads, kmers, graph, debugout=None):
    """Find shortest common superstring using a greedy assembly algorithm."""
    count = 0
    while len(graph.edges()) > 0:
        count += 1

        pair = fetch_largest_overlapping_pair(graph, reads)
        newname = 'contig{:d}'.format(count)
        newrecord = merge_and_reannotate(pair, newname)
        if debugout:
            print('### DEBUG', pair.tail.name, pair.head.name, pair.offset,
                  pair.overlap, pair.sameorient, file=debugout)
            kevlar.print_augmented_fastq(newrecord, debugout)
        for kmer in newrecord.ikmers:
            kmerseq = kevlar.revcommin(kmer.sequence)
            for readname in kmers[kmerseq]:
                already_merged = readname not in graph
                current_contig = readname in [
                    pair.tail.name, pair.head.name, newname
                ]
                if already_merged or current_contig:
                    continue
                otherrecord = reads[readname]
                newpair = kevlar.overlap.calc_offset(
                    newrecord, otherrecord, kmerseq, debugout
                )
                if newpair == kevlar.overlap.INCOMPATIBLE_PAIR:
                    continue
                tn, hn = newpair.tail.name, newpair.head.name
                if tn in graph and hn in graph[tn]:
                    assert graph[tn][hn]['overlap'] == newpair.overlap
                    if graph[tn][hn]['tail'] == newpair.tail:
                        assert graph[tn][hn]['offset'] == newpair.offset
                else:
                    graph.add_edge(tn, hn, offset=newpair.offset,
                                   overlap=newpair.overlap, ikmer=kmerseq,
                                   orient=newpair.sameorient, tail=tn)
            kmers[kmerseq].add(newrecord.name)
        reads[newrecord.name] = newrecord
        graph.add_node(newrecord.name)
        graph.remove_node(pair.tail.name)
        graph.remove_node(pair.head.name)

def main(args):
    debugout = None
    if args.debug:
        debugout = args.logfile

    reads, kmers = load_reads(kevlar.open(args.augfastq, 'r'), debugout)
    inputreads = list(reads)
    graph = graph_init(reads, kmers, 2, args.max_abund, debugout)
    if args.gml:
        networkx.write_gml(graph, args.gml)
        message = '[kevlar::assemble] graph written to {}'.format(args.gml)
        print(message, file=args.logfile)

    ccs = list(networkx.connected_components(graph))
    assert len(ccs) == 1

    assemble_with_greed(reads, kmers, graph, debugout)

    contigcount = 0
    unassembledcount = 0
    outstream = kevlar.open(args.out, 'w')
    for seqname in graph.nodes():
        if seqname in inputreads:
            unassembledcount += 1
            continue
        contigcount += 1
        contigrecord = reads[seqname]
        kevlar.print_augmented_fastq(contigrecord, outstream)

    assembledcount = len(inputreads) - unassembledcount
    message = '[kevlar::assemble] assembled'
    message += ' {:d}/{:d} reads'.format(assembledcount, len(inputreads))
    message += ' into {:d} contig(s)'.format(contigcount)
    print(message, file=args.logfile)
<|endoftext|>


<commit_before>"""
FinalProject.py
Author: Liam S
Credit:
"""
from ggame import App, Sprite, ImageAsset, Frame
from ggame import SoundAsset, Sound, TextAsset, Color
import math
from time import time

class Background(Sprite):

    asset = ImageAsset("images/grass_with_blue_sky_by_devrez-d35pm8j.png")
    width = 1032
    height = 774

    def __init__(self, position):
        super().__init__(Background.asset, position)
        self.scale = 1.4

class Truck(Sprite):

    asset = ImageAsset("images/Truck-side-edit.png")
    width = 1389
    height = 664

    def __init__(self, position):
        super().__init__(Truck.asset, position)
        self.scale = 0.14
        self.rectangularCollisionModel()

    def step(self):
        self.x -= 2
        if self.x <= -180:
            self.x = 1999

class Ship(Sprite):

    asset = ImageAsset("images/il2m3-bp-fl-am-3view-mongolian.png")

    def __init__(self, position):
        super().__init__(Ship.asset, position)
        self.vx = 1
        self.vy = 1
        self.vAddedx = 0
        self.vAddedy = 0
        self.vAddedr = 0
        self.vertThrust = 0
        self.RotThrust = 0
        self.thrustframe = 1
        self.scale = 0.25
        """
        PlaneGame.listenKeyEvent("keydown", "left arrow", self.thrustLeft)
        PlaneGame.listenKeyEvent("keyup", "right arrow", self.thrustRightoff)
        PlaneGame.listenKeyEvent("keyup", "left arrow", self.thrustLeftoff)
        PlaneGame.listenKeyEvent("keydown", "right arrow", self.thrustRight)
        """
        PlaneGame.listenKeyEvent("keydown", "up arrow", self.thrustUp)
        PlaneGame.listenKeyEvent("keyup", "down arrow", self.thrustDownoff)
        PlaneGame.listenKeyEvent("keyup", "up arrow", self.thrustUpoff)
        PlaneGame.listenKeyEvent("keydown", "down arrow", self.thrustDown)

        self.fxcenter = self.fycenter = 0.5
    def step(self):
        """
        if self.sideThrust == 1:
            self.vAddedx += 0.05
        if self.sideThrust == -1:
            self.vAddedx -= 0.05
        if self.sideThrust == 0:
            self.vAddedx += 0
        """
        if self.vertThrust == 1:
            self.vAddedy = 1.3
        if self.vertThrust == -1:
            self.vAddedy = -1.3
        if self.vertThrust == 0:
            self.vAddedy = 0
        if self.RotThrust == 1:
            self.vAddedr = -0.3
        if self.RotThrust == -1:
            self.vAddedr = 0.3
        if self.RotThrust == 0:
            self.vAddedr = 0
        if self.x >= 2000:
            self.x = 10
        if self.x <= 0:
            self.x = 1999
        if self.y <=10:
            self.y = 11
        self.x += 5
        self.y += self.vAddedy
        self.rotation = self.vAddedr
        if self.y >= 765:
            self.visible = False

    def thrustUp(self, event):
        self.vertThrust = -1
        self.RotThrust = -1

    def thrustDown(self, event):
        self.vertThrust = 1
        self.RotThrust = 1

    def thrustDownoff(self, event):
        self.vertThrust = 0
        self.RotThrust = 0

    def thrustUpoff(self, event):
        self.vertThrust = 0
        self.RotThrust = 0

class Bomb(Sprite):

    asset = ImageAsset("images/bomb.png")
    width = 2000
    height = 1000

    def __init__(self, position, ship):
        super().__init__(Bomb.asset, position)
        self.scale = 0.04
        self.ship = ship
        self.visible = False
        self.dropped = False
        self.counter = 0
        PlaneGame.listenKeyEvent("keydown", "space", self.dropBomb)
        self.circularCollisionModel()

    def dropBomb(self, event):
        self.dropped = True

    def nodropBomb(self, event):
        self.dropped = False

    def step(self):
        if self.dropped == False:
            self.x = self.ship.x
            self.y = self.ship.y+15
        if self.dropped == True:
            self.visible = True
            if self.visible == True:
                self.counter += 0.2
                self.x += 5
                self.y += self.counter
        if self.y >= 765:
            self.visible = False
            self.dropped = False
            self.counter = 0
            ExplosionBig(self.position)
        if self.x >= 2000:
            self.x = 10
        if self.x <= 0:
            self.x = 1999

class ExplosionBig(Sprite):

    asset = ImageAsset("images/explosion2.png", Frame(0,0,4800/25,195), 25)

    def __init__(self, position):
        super().__init__(ExplosionBig.asset, position)
        self.image = 0
        self.center = (0.5, 0.5)
        self.scale = 0.7

    def step(self):
        self.setImage(self.image//2)  # slow it down
        self.image = self.image + 1
        if self.image == 50:
            self.destroy()

class PlaneGame(App):
    def __init__(self, width, height):
        super().__init__(width, height)
        Background((0, 0))
        Background((1300, 0))
        Background((2000, 0))
        s = Ship((400,200))
        Bomb((400,600),s)
        Truck((200,715))

    def step(self):
        for ship in self.getSpritesbyClass(Ship):
            ship.step()
        for ship in self.getSpritesbyClass(Bomb):
            ship.step()
        for ship in self.getSpritesbyClass(ExplosionBig):
            ship.step()
        for ship in self.getSpritesbyClass(Truck):
            ship.step()

app = PlaneGame(0,0)
app.run()

<commit_msg>Updated from Brython Server: 12/13/2016 1:39:48 PM<commit_after>"""
FinalProject.py
Author: Liam S
Credit:
"""
from ggame import App, Sprite, ImageAsset, Frame
from ggame import SoundAsset, Sound, TextAsset, Color
import math
from time import time

class Background(Sprite):

    asset = ImageAsset("images/grass_with_blue_sky_by_devrez-d35pm8j.png")
    width = 1032
    height = 774

    def __init__(self, position):
        super().__init__(Background.asset, position)
        self.scale = 1.4

class Truck(Sprite):

    asset = ImageAsset("images/Truck-side-edit.png")
    width = 1389
    height = 664

    def __init__(self, position):
        super().__init__(Truck.asset, position)
        self.scale = 0.14
        self.circularCollisionModel()

    def step(self):
        self.x -= 2
        if self.x <= -180:
            self.x = 1999
        if self.collidingWithSprites(Bomb):
            ExplosionBig(self.position)

class Ship(Sprite):

    asset = ImageAsset("images/il2m3-bp-fl-am-3view-mongolian.png")

    def __init__(self, position):
        super().__init__(Ship.asset, position)
        self.vx = 1
        self.vy = 1
        self.vAddedx = 0
        self.vAddedy = 0
        self.vAddedr = 0
        self.vertThrust = 0
        self.RotThrust = 0
        self.thrustframe = 1
        self.scale = 0.25
        """
        PlaneGame.listenKeyEvent("keydown", "left arrow", self.thrustLeft)
        PlaneGame.listenKeyEvent("keyup", "right arrow", self.thrustRightoff)
        PlaneGame.listenKeyEvent("keyup", "left arrow", self.thrustLeftoff)
        PlaneGame.listenKeyEvent("keydown", "right arrow", self.thrustRight)
        """
        PlaneGame.listenKeyEvent("keydown", "up arrow", self.thrustUp)
        PlaneGame.listenKeyEvent("keyup", "down arrow", self.thrustDownoff)
        PlaneGame.listenKeyEvent("keyup", "up arrow", self.thrustUpoff)
        PlaneGame.listenKeyEvent("keydown", "down arrow", self.thrustDown)

        self.fxcenter = self.fycenter = 0.5
    def step(self):
        """
        if self.sideThrust == 1:
            self.vAddedx += 0.05
        if self.sideThrust == -1:
            self.vAddedx -= 0.05
        if self.sideThrust == 0:
            self.vAddedx += 0
        """
        if self.vertThrust == 1:
            self.vAddedy = 1.3
        if self.vertThrust == -1:
            self.vAddedy = -1.3
        if self.vertThrust == 0:
            self.vAddedy = 0
        if self.RotThrust == 1:
            self.vAddedr = -0.3
        if self.RotThrust == -1:
            self.vAddedr = 0.3
        if self.RotThrust == 0:
            self.vAddedr = 0
        if self.x >= 2000:
            self.x = 10
        if self.x <= 0:
            self.x = 1999
        if self.y <=10:
            self.y = 11
        self.x += 5
        self.y += self.vAddedy
        self.rotation = self.vAddedr
        if self.y >= 765:
            self.visible = False

    def thrustUp(self, event):
        self.vertThrust = -1
        self.RotThrust = -1

    def thrustDown(self, event):
        self.vertThrust = 1
        self.RotThrust = 1

    def thrustDownoff(self, event):
        self.vertThrust = 0
        self.RotThrust = 0

    def thrustUpoff(self, event):
        self.vertThrust = 0
        self.RotThrust = 0

class Bomb(Sprite):

    asset = ImageAsset("images/bomb.png")
    width = 2000
    height = 1000

    def __init__(self, position, ship):
        super().__init__(Bomb.asset, position)
        self.scale = 0.04
        self.ship = ship
        self.visible = False
        self.dropped = False
        self.counter = 0
        PlaneGame.listenKeyEvent("keydown", "space", self.dropBomb)
        self.circularCollisionModel()

    def dropBomb(self, event):
        self.dropped = True

    def nodropBomb(self, event):
        self.dropped = False

    def step(self):
        if self.dropped == False:
            self.x = self.ship.x
            self.y = self.ship.y+15
        if self.dropped == True:
            self.visible = True
            if self.visible == True:
                self.counter += 0.2
                self.x += 5
                self.y += self.counter
        if self.y >= 765 or self.collidingWithSprites(Truck):
            self.visible = False
            self.dropped = False
            self.counter = 0
            ExplosionBig(self.position)
        if self.x >= 2000:
            self.x = 10
        if self.x <= 0:
            self.x = 1999

class ExplosionBig(Sprite):

    asset = ImageAsset("images/explosion2.png", Frame(0,0,4800/25,195), 25)

    def __init__(self, position):
        super().__init__(ExplosionBig.asset, position)
        self.image = 0
        self.center = (0.5, 0.5)
        self.scale = 0.7

    def step(self):
        self.setImage(self.image//2)  # slow it down
        self.image = self.image + 1
        if self.image == 50:
            self.destroy()

class PlaneGame(App):
    def __init__(self, width, height):
        super().__init__(width, height)
        Background((0, 0))
        Background((1300, 0))
        Background((2000, 0))
        s = Ship((400,200))
        Bomb((400,600),s)
        Truck((200,715))

    def step(self):
        for ship in self.getSpritesbyClass(Ship):
            ship.step()
        for ship in self.getSpritesbyClass(Bomb):
            ship.step()
        for ship in self.getSpritesbyClass(ExplosionBig):
            ship.step()
        for ship in self.getSpritesbyClass(Truck):
            ship.step()

app = PlaneGame(0,0)
app.run()

<|endoftext|>


<commit_before>from django.conf import settings
from .models import Message
from rapidsms.models import Backend, Connection
from rapidsms.apps.base import AppBase
from rapidsms.messages.incoming import IncomingMessage
from rapidsms.messages.outgoing import OutgoingMessage
from rapidsms.log.mixin import LoggerMixin
from threading import Lock, Thread

from urllib import quote_plus
from urllib2 import urlopen
import time
import re

# we keep a list of all the messages currently being processed here
outgoing_pk_queue = []

# our lock for that queue
outgoing_queue_lock = Lock()

# we make sure only one thread is modifying db messages a time
outgoing_db_lock = Lock()

# our worker threads
outgoing_worker_threads = []

# whether we are currently sending mass messages, where we will queue up
# many messages at once, used as an optimization
sending_mass_messages = False

def start_sending_mass_messages():
    global sending_mass_messages
    sending_mass_messages = True

def stop_sending_mass_messages():
    global sending_mass_messages
    sending_mass_messages = False

class HttpRouterThread(Thread, LoggerMixin):
    """
    This thread is just a worker thread for messages.  The run() method pops off a message to work on
    and continues appropriately.
    """

    def __init__(self, **kwargs):
        Thread.__init__(self, **kwargs)
        self._isbusy = False

    def is_busy(self):
        return self._isbusy

    def run(self):
        global outgoing_pk_queue
        global outgoing_queue_lock
        global outgoing_db_lock
        global sending_mass_messages

        while self.is_alive():
            if not sending_mass_messages:
                outgoing_queue_lock.acquire()

                try:
                    # this gets any outgoing messages which are either pending or queued, exlcuding those
                    # which are already being processed
                    to_process = Message.objects.filter(direction='O',
                                                        status__in=['P','Q']).order_by('status').exclude(pk__in=outgoing_pk_queue)

                    if to_process.count():
                        self._isbusy = True
                        outgoing_message = to_process[0]
                        outgoing_pk_queue.append(outgoing_message.pk)
                        outgoing_queue_lock.release()

                        # process the outgoing phases for this message
                        send_msg = get_router().process_outgoing_phases(outgoing_message)

                        # if it wasn't cancelled, send it off
                        if send_msg:
                            self.send_message(outgoing_message)

                        outgoing_queue_lock.acquire()
                        outgoing_pk_queue.remove(outgoing_message.pk)
                        outgoing_queue_lock.release()
                except:
                    import traceback
                    traceback.print_exc()

                finally:
                    try:
                        outgoing_queue_lock.release()
                    except:
                        # it's ok, it wasn't locked
                        pass
            self._isbusy = False
            time.sleep(0.5)

    def send_message(self, msg, **kwargs):
        """
        Sends the message off.  We first try to directly contact our sms router to deliver it,
        if we fail, then we just add it to our outgoing queue.
        """
        global outgoing_db_lock
        params = {
            'backend': msg.connection.backend,
            'recipient': msg.connection.identity,
            'text': msg.text,
            'id': msg.pk
        }

        # add any other backend-specific parameters from kwargs
        params.update(kwargs)
        for k, v in params.items():
            try:
                params[k] = quote_plus(str(v))
            except UnicodeEncodeError:
                params[k] = quote_plus(str(v.encode('UTF-8')))

        # and actually hand the message off to our router URL
        try:
            # FIXME: clean this up
            response = urlopen(settings.ROUTER_URL % params)
            outgoing_db_lock.acquire()
            # kannel likes to send 202 responses, really any
            # 2xx value means things went okay
            if int(response.getcode()/100) == 2:
                self.info("SMS[%d] SENT" % msg.id)
                msg.status = 'S'
                msg.save()
            else:
                self.error("SMS[%d] Message not sent, got status: %s .. queued for later delivery." % (msg.id, response.getcode()))
                msg.status = 'Q'
                msg.save()
            outgoing_db_lock.release()
        except Exception as e:
            self.error("SMS[%d] Message not sent: %s .. queued for later delivery." % (msg.id, str(e)))
            outgoing_db_lock.acquire()
            msg.status = 'Q'
            msg.save()
            outgoing_db_lock.release()

class HttpRouter(object, LoggerMixin):
    """
    This is a simplified version of the normal SMS router in that it has no threading.  Instead
    it is expected that the handle_incoming and handle_outcoming calls are made in the HTTP
    thread.
    """

    incoming_phases = ("filter", "parse", "handle", "default", "cleanup")
    outgoing_phases = ("outgoing",)

    def __init__(self):
        # the apps we'll run through
        self.apps = []

        # we need to be started
        self.started = False

    @classmethod
    def normalize_number(cls, number):
        """
        Normalizes the passed in number, they should be only digits, some backends prepend + and
        maybe crazy users put in dashes or parentheses in the console.
        """
        return re.sub('[^0-9a-z]', '', number.lower())

    def add_message(self, backend, contact, text, direction, status):
        """
        Adds this message to the db.  This is both for logging, and we also keep state
        tied to it.
        """
        global outgoing_db_lock

        # lookup / create this backend
        # TODO: is this too flexible?  Perhaps we should do this upon initialization and refuse
        # any backends not found in our settings.  But I hate dropping messages on the floor.
        backend, created = Backend.objects.get_or_create(name=backend)

        contact = HttpRouter.normalize_number(contact)

        # create our connection
        connection, created = Connection.objects.get_or_create(backend=backend, identity=contact)

        # finally, create our db message
        outgoing_db_lock.acquire()
        message = Message.objects.create(connection=connection,
                                         text=text,
                                         direction=direction,
                                         status=status)
        outgoing_db_lock.release()

        return message

    def mark_delivered(self, message_id):
        """
        Marks a message as delivered by the backend.
        """
        global outgoing_db_lock

        message = Message.objects.get(pk=message_id)
        outgoing_db_lock.acquire()
        message.status = 'D'
        message.save()
        outgoing_db_lock.release()

    def handle_incoming(self, backend, sender, text):
        """
        Handles an incoming message.
        """
        global outgoing_db_lock
        # create our db message for logging
        db_message = self.add_message(backend, sender, text, 'I', 'R')

        # and our rapidsms transient message for processing
        msg = IncomingMessage(db_message.connection, text, db_message.date)

        # add an extra property to IncomingMessage, so httprouter-aware
        # apps can make use of it during the handling phase
        msg.db_message = db_message

        self.info("SMS[%d] IN (%s) : %s" % (db_message.id, msg.connection, msg.text))
        try:
            for phase in self.incoming_phases:
                self.debug("In %s phase" % phase)
                if phase == "default":
                    if msg.handled:
                        self.debug("Skipping phase")
                        break

                for app in self.apps:
                    self.debug("In %s app" % app)
                    handled = False

                    try:
                        func = getattr(app, phase)
                        handled = func(msg)

                    except Exception, err:
                        import traceback
                        traceback.print_exc(err)
                        app.exception()

                    # during the _filter_ phase, an app can return True
                    # to abort ALL further processing of this message
                    if phase == "filter":
                        if handled is True:
                            self.warning("Message filtered")
                            raise(StopIteration)

                    # during the _handle_ phase, apps can return True
                    # to "short-circuit" this phase, preventing any
                    # further apps from receiving the message
                    elif phase == "handle":
                        if handled is True:
                            self.debug("Short-circuited")
                            # mark the message handled to avoid the
                            # default phase firing unnecessarily
                            msg.handled = True
                            break

                    elif phase == "default":
                        # allow default phase of apps to short circuit
                        # for prioritized contextual responses.
                        if handled is True:
                            self.debug("Short-circuited default")
                            break

        except StopIteration:
            pass

        outgoing_db_lock.acquire()
        db_message.status = 'H'
        db_message.save()
        outgoing_db_lock.release()

        db_responses = []

        # now send the message responses
        while msg.responses:
            response = msg.responses.pop(0)
            self.handle_outgoing(response, db_message)

        # we are no longer interested in this message... but some crazy
        # synchronous backends might be, so mark it as processed.
        msg.processed = True

        return db_message

    def add_outgoing(self, connection, text, source=None, status='Q'):
        """
        Adds a message to our outgoing queue, this is a non-blocking action
        """
        global outgoing_db_lock
        outgoing_db_lock.acquire()
        db_message = Message.objects.create(connection=connection,
                                            text=text,
                                            direction='O',
                                            status=status,
                                            in_response_to=source)
        outgoing_db_lock.release()

        self.info("SMS[%d] OUT (%s) : %s" % (db_message.id, str(connection), text))

        global outgoing_worker_threads

        # if we have no ROUTER_URL configured, then immediately process our outgoing phases
        # and leave the message in the queue
        if not getattr(settings, 'ROUTER_URL', None):
            if self.process_outgoing_phases(db_message):
                outgoing_db_lock.acquire()
                db_message.status = 'Q'
                db_message.save()
                outgoing_db_lock.release()

        # otherwise, fire up any threads we need to send the message out
        else:
            # check for available worker threads in the pool, add one if necessary
            num_workers = getattr(settings, 'ROUTER_WORKERS', 5)
            all_busy = True
            for worker in outgoing_worker_threads:
                if not worker.is_busy():
                    all_busy = False
                    break

            if all_busy and len(outgoing_worker_threads) < num_workers:
                worker = HttpRouterThread()
                worker.daemon = True # they don't need to quit gracefully
                worker.start()
                outgoing_worker_threads.append(worker)

        return db_message

    def handle_outgoing(self, msg, source=None):
        """
        Sends the passed in RapidSMS message off.  Optionally ties the outgoing message to the incoming
        message which triggered it.
        """
        # add it to our outgoing queue
        db_message = self.add_outgoing(msg.connection, msg.text, source, status='P')

        return db_message

    def process_outgoing_phases(self, outgoing):
        """
        Passes the passed in message through the outgoing phase for all our configured SMS apps.

        Apps have the opportunity to cancel messages in this phase by returning False when
        called with the message.  In that case this method will also return False
        """
        # create a RapidSMS outgoing message
        msg = OutgoingMessage(outgoing.connection, outgoing.text.replace('%','%%'))
        msg.db_message = outgoing

        send_msg = True
        for phase in self.outgoing_phases:
            self.debug("Out %s phase" % phase)

            # call outgoing phases in the opposite order of the incoming
            # phases, so the first app called with an  incoming message
            # is the last app called with an outgoing message
            for app in reversed(self.apps):
                self.debug("Out %s app" % app)

                try:
                    func = getattr(app, phase)
                    keep_sending = func(msg)

                    # we have to do things this way because by default apps return
                    # None from outgoing()
                    if keep_sending is False:
                        send_msg = False
                except Exception, err:
                    app.exception()

                # during any outgoing phase, an app can return True to
                # abort ALL further processing of this message
                if not send_msg:
                    outgoing.status = 'C'

                    outgoing_db_lock.acquire()
                    outgoing.save()
                    outgoing_db_lock.release()

                    self.warning("Message cancelled")
                    send_msg = False
                    break

        return send_msg

    def add_app(self, module_name):
        """
        Find the app named *module_name*, instantiate it, and add it to
        the list of apps to be notified of incoming messages. Return the
        app instance.
        """
        try:
            cls = AppBase.find(module_name)
        except:
            cls = None

        if cls is None: return None

        app = cls(self)
        self.apps.append(app)
        return app

    def start(self):
        """
        Initializes our router.
        TODO: this happens in the HTTP thread on the first call, that could be bad.
        """
        # add all our apps
        for app_name in settings.SMS_APPS:
            self.add_app(app_name)

        # start all our apps
        for app in self.apps:
            app.start()

        # the list of messages which need to be sent, we load this from the DB
        # upon first starting up
        self.outgoing = [message for message in Message.objects.filter(status='Q')]

        # mark ourselves as started
        self.started = True

# we'll get started when we first get used
http_router = HttpRouter()
http_router_lock = Lock()

def get_router():
    """
    Takes care of performing lazy initialization of the www router.
    """
    global http_router
    global http_router_lock

    if not http_router.started:
        http_router_lock.acquire()
        try:
            if not http_router.started:
                http_router.start()
        finally:
            http_router_lock.release()

    return http_router
<commit_msg>better error messaging when an SMS app can't be found<commit_after>from django.conf import settings
from .models import Message
from rapidsms.models import Backend, Connection
from rapidsms.apps.base import AppBase
from rapidsms.messages.incoming import IncomingMessage
from rapidsms.messages.outgoing import OutgoingMessage
from rapidsms.log.mixin import LoggerMixin
from threading import Lock, Thread

from urllib import quote_plus
from urllib2 import urlopen
import time
import re

# we keep a list of all the messages currently being processed here
outgoing_pk_queue = []

# our lock for that queue
outgoing_queue_lock = Lock()

# we make sure only one thread is modifying db messages a time
outgoing_db_lock = Lock()

# our worker threads
outgoing_worker_threads = []

# whether we are currently sending mass messages, where we will queue up
# many messages at once, used as an optimization
sending_mass_messages = False

def start_sending_mass_messages():
    global sending_mass_messages
    sending_mass_messages = True

def stop_sending_mass_messages():
    global sending_mass_messages
    sending_mass_messages = False

class HttpRouterThread(Thread, LoggerMixin):
    """
    This thread is just a worker thread for messages.  The run() method pops off a message to work on
    and continues appropriately.
    """

    def __init__(self, **kwargs):
        Thread.__init__(self, **kwargs)
        self._isbusy = False

    def is_busy(self):
        return self._isbusy

    def run(self):
        global outgoing_pk_queue
        global outgoing_queue_lock
        global outgoing_db_lock
        global sending_mass_messages

        while self.is_alive():
            if not sending_mass_messages:
                outgoing_queue_lock.acquire()

                try:
                    # this gets any outgoing messages which are either pending or queued, exlcuding those
                    # which are already being processed
                    to_process = Message.objects.filter(direction='O',
                                                        status__in=['P','Q']).order_by('status').exclude(pk__in=outgoing_pk_queue)

                    if to_process.count():
                        self._isbusy = True
                        outgoing_message = to_process[0]
                        outgoing_pk_queue.append(outgoing_message.pk)
                        outgoing_queue_lock.release()

                        # process the outgoing phases for this message
                        send_msg = get_router().process_outgoing_phases(outgoing_message)

                        # if it wasn't cancelled, send it off
                        if send_msg:
                            self.send_message(outgoing_message)

                        outgoing_queue_lock.acquire()
                        outgoing_pk_queue.remove(outgoing_message.pk)
                        outgoing_queue_lock.release()
                except:
                    import traceback
                    traceback.print_exc()

                finally:
                    try:
                        outgoing_queue_lock.release()
                    except:
                        # it's ok, it wasn't locked
                        pass
            self._isbusy = False
            time.sleep(0.5)

    def send_message(self, msg, **kwargs):
        """
        Sends the message off.  We first try to directly contact our sms router to deliver it,
        if we fail, then we just add it to our outgoing queue.
        """
        global outgoing_db_lock
        params = {
            'backend': msg.connection.backend,
            'recipient': msg.connection.identity,
            'text': msg.text,
            'id': msg.pk
        }

        # add any other backend-specific parameters from kwargs
        params.update(kwargs)
        for k, v in params.items():
            try:
                params[k] = quote_plus(str(v))
            except UnicodeEncodeError:
                params[k] = quote_plus(str(v.encode('UTF-8')))

        # and actually hand the message off to our router URL
        try:
            # FIXME: clean this up
            response = urlopen(settings.ROUTER_URL % params)
            outgoing_db_lock.acquire()
            # kannel likes to send 202 responses, really any
            # 2xx value means things went okay
            if int(response.getcode()/100) == 2:
                self.info("SMS[%d] SENT" % msg.id)
                msg.status = 'S'
                msg.save()
            else:
                self.error("SMS[%d] Message not sent, got status: %s .. queued for later delivery." % (msg.id, response.getcode()))
                msg.status = 'Q'
                msg.save()
            outgoing_db_lock.release()
        except Exception as e:
            self.error("SMS[%d] Message not sent: %s .. queued for later delivery." % (msg.id, str(e)))
            outgoing_db_lock.acquire()
            msg.status = 'Q'
            msg.save()
            outgoing_db_lock.release()

class HttpRouter(object, LoggerMixin):
    """
    This is a simplified version of the normal SMS router in that it has no threading.  Instead
    it is expected that the handle_incoming and handle_outcoming calls are made in the HTTP
    thread.
    """

    incoming_phases = ("filter", "parse", "handle", "default", "cleanup")
    outgoing_phases = ("outgoing",)

    def __init__(self):
        # the apps we'll run through
        self.apps = []

        # we need to be started
        self.started = False

    @classmethod
    def normalize_number(cls, number):
        """
        Normalizes the passed in number, they should be only digits, some backends prepend + and
        maybe crazy users put in dashes or parentheses in the console.
        """
        return re.sub('[^0-9a-z]', '', number.lower())

    def add_message(self, backend, contact, text, direction, status):
        """
        Adds this message to the db.  This is both for logging, and we also keep state
        tied to it.
        """
        global outgoing_db_lock

        # lookup / create this backend
        # TODO: is this too flexible?  Perhaps we should do this upon initialization and refuse
        # any backends not found in our settings.  But I hate dropping messages on the floor.
        backend, created = Backend.objects.get_or_create(name=backend)

        contact = HttpRouter.normalize_number(contact)

        # create our connection
        connection, created = Connection.objects.get_or_create(backend=backend, identity=contact)

        # finally, create our db message
        outgoing_db_lock.acquire()
        message = Message.objects.create(connection=connection,
                                         text=text,
                                         direction=direction,
                                         status=status)
        outgoing_db_lock.release()

        return message

    def mark_delivered(self, message_id):
        """
        Marks a message as delivered by the backend.
        """
        global outgoing_db_lock

        message = Message.objects.get(pk=message_id)
        outgoing_db_lock.acquire()
        message.status = 'D'
        message.save()
        outgoing_db_lock.release()

    def handle_incoming(self, backend, sender, text):
        """
        Handles an incoming message.
        """
        global outgoing_db_lock
        # create our db message for logging
        db_message = self.add_message(backend, sender, text, 'I', 'R')

        # and our rapidsms transient message for processing
        msg = IncomingMessage(db_message.connection, text, db_message.date)

        # add an extra property to IncomingMessage, so httprouter-aware
        # apps can make use of it during the handling phase
        msg.db_message = db_message

        self.info("SMS[%d] IN (%s) : %s" % (db_message.id, msg.connection, msg.text))
        try:
            for phase in self.incoming_phases:
                self.debug("In %s phase" % phase)
                if phase == "default":
                    if msg.handled:
                        self.debug("Skipping phase")
                        break

                for app in self.apps:
                    self.debug("In %s app" % app)
                    handled = False

                    try:
                        func = getattr(app, phase)
                        handled = func(msg)

                    except Exception, err:
                        import traceback
                        traceback.print_exc(err)
                        app.exception()

                    # during the _filter_ phase, an app can return True
                    # to abort ALL further processing of this message
                    if phase == "filter":
                        if handled is True:
                            self.warning("Message filtered")
                            raise(StopIteration)

                    # during the _handle_ phase, apps can return True
                    # to "short-circuit" this phase, preventing any
                    # further apps from receiving the message
                    elif phase == "handle":
                        if handled is True:
                            self.debug("Short-circuited")
                            # mark the message handled to avoid the
                            # default phase firing unnecessarily
                            msg.handled = True
                            break

                    elif phase == "default":
                        # allow default phase of apps to short circuit
                        # for prioritized contextual responses.
                        if handled is True:
                            self.debug("Short-circuited default")
                            break

        except StopIteration:
            pass

        outgoing_db_lock.acquire()
        db_message.status = 'H'
        db_message.save()
        outgoing_db_lock.release()

        db_responses = []

        # now send the message responses
        while msg.responses:
            response = msg.responses.pop(0)
            self.handle_outgoing(response, db_message)

        # we are no longer interested in this message... but some crazy
        # synchronous backends might be, so mark it as processed.
        msg.processed = True

        return db_message

    def add_outgoing(self, connection, text, source=None, status='Q'):
        """
        Adds a message to our outgoing queue, this is a non-blocking action
        """
        global outgoing_db_lock
        outgoing_db_lock.acquire()
        db_message = Message.objects.create(connection=connection,
                                            text=text,
                                            direction='O',
                                            status=status,
                                            in_response_to=source)
        outgoing_db_lock.release()

        self.info("SMS[%d] OUT (%s) : %s" % (db_message.id, str(connection), text))

        global outgoing_worker_threads

        # if we have no ROUTER_URL configured, then immediately process our outgoing phases
        # and leave the message in the queue
        if not getattr(settings, 'ROUTER_URL', None):
            if self.process_outgoing_phases(db_message):
                outgoing_db_lock.acquire()
                db_message.status = 'Q'
                db_message.save()
                outgoing_db_lock.release()

        # otherwise, fire up any threads we need to send the message out
        else:
            # check for available worker threads in the pool, add one if necessary
            num_workers = getattr(settings, 'ROUTER_WORKERS', 5)
            all_busy = True
            for worker in outgoing_worker_threads:
                if not worker.is_busy():
                    all_busy = False
                    break

            if all_busy and len(outgoing_worker_threads) < num_workers:
                worker = HttpRouterThread()
                worker.daemon = True # they don't need to quit gracefully
                worker.start()
                outgoing_worker_threads.append(worker)

        return db_message

    def handle_outgoing(self, msg, source=None):
        """
        Sends the passed in RapidSMS message off.  Optionally ties the outgoing message to the incoming
        message which triggered it.
        """
        # add it to our outgoing queue
        db_message = self.add_outgoing(msg.connection, msg.text, source, status='P')

        return db_message

    def process_outgoing_phases(self, outgoing):
        """
        Passes the passed in message through the outgoing phase for all our configured SMS apps.

        Apps have the opportunity to cancel messages in this phase by returning False when
        called with the message.  In that case this method will also return False
        """
        # create a RapidSMS outgoing message
        msg = OutgoingMessage(outgoing.connection, outgoing.text.replace('%','%%'))
        msg.db_message = outgoing

        send_msg = True
        for phase in self.outgoing_phases:
            self.debug("Out %s phase" % phase)

            # call outgoing phases in the opposite order of the incoming
            # phases, so the first app called with an  incoming message
            # is the last app called with an outgoing message
            for app in reversed(self.apps):
                self.debug("Out %s app" % app)

                try:
                    func = getattr(app, phase)
                    keep_sending = func(msg)

                    # we have to do things this way because by default apps return
                    # None from outgoing()
                    if keep_sending is False:
                        send_msg = False
                except Exception, err:
                    app.exception()

                # during any outgoing phase, an app can return True to
                # abort ALL further processing of this message
                if not send_msg:
                    outgoing.status = 'C'

                    outgoing_db_lock.acquire()
                    outgoing.save()
                    outgoing_db_lock.release()

                    self.warning("Message cancelled")
                    send_msg = False
                    break

        return send_msg

    def add_app(self, module_name):
        """
        Find the app named *module_name*, instantiate it, and add it to
        the list of apps to be notified of incoming messages. Return the
        app instance.
        """
        try:
            cls = AppBase.find(module_name)
        except:
            cls = None

        if cls is None:
            self.error("Unable to find SMS application with module: '%s'" % module_name)
            return None

        app = cls(self)
        self.apps.append(app)
        return app

    def start(self):
        """
        Initializes our router.
        TODO: this happens in the HTTP thread on the first call, that could be bad.
        """
        # add all our apps
        for app_name in settings.SMS_APPS:
            self.add_app(app_name)

        # start all our apps
        for app in self.apps:
            app.start()

        # the list of messages which need to be sent, we load this from the DB
        # upon first starting up
        self.outgoing = [message for message in Message.objects.filter(status='Q')]

        # mark ourselves as started
        self.started = True

# we'll get started when we first get used
http_router = HttpRouter()
http_router_lock = Lock()

def get_router():
    """
    Takes care of performing lazy initialization of the www router.
    """
    global http_router
    global http_router_lock

    if not http_router.started:
        http_router_lock.acquire()
        try:
            if not http_router.started:
                http_router.start()
        finally:
            http_router_lock.release()

    return http_router
<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2020 The Meta-Dataset Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python2, python3
r"""Script for training models on the benchmark.

Launching command for batch baseline:
# pylint: disable=line-too-long
python -m meta_dataset.train \
  --train_checkpoint_dir=/tmp/bench --summary_dir=/tmp/bench \
  --records_root_dir=<records_root> \
  --alsologtostderr \
  --gin_config=meta_dataset/learn/gin/default/<exp_name>.gin
  --gin_bindings="Trainer.experiment_name='<exp_name>'"
# pylint: enable=line-too-long

where:
  <exp_name> is e.g. 'debug_proto_mini_imagenet'

To override elements from the config, you can use arguments of the form:
  For gin: --gin_bindings='foo = 1000000'
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import json
import os

from absl import app
from absl import logging
import gin
import gin.tf
from meta_dataset import data
from meta_dataset import learners  # pylint: disable=unused-import
from meta_dataset import trainer
# pylint: disable=unused-import
from meta_dataset.data import config
# pylint: enable=unused-import
import tensorflow.compat.v1 as tf
tf.disable_eager_execution()

DEFAULT_SAVING_DIRECTORY = '/tmp/metadataset'

tf.flags.DEFINE_string('train_checkpoint_dir',
                       os.path.join(DEFAULT_SAVING_DIRECTORY, 'checkpoints'),
                       'The directory to save checkpoints.')
tf.flags.DEFINE_string('summary_dir',
                       os.path.join(DEFAULT_SAVING_DIRECTORY, 'summaries'),
                       'The directory for writing summaries.')
tf.flags.DEFINE_bool(
    'reload_checkpoint_gin_config', False,
    'Whether to reload an operative Gin configuration along with a checkpoint.')
tf.flags.DEFINE_string('records_root_dir', '',
                       'Root directory containing a subdirectory per dataset.')
tf.flags.DEFINE_bool(
    'is_training', True, 'Whether we are in the training phase. '
    'Used to control whether to perform training or evaluation.')

tf.flags.DEFINE_multi_string('gin_config', None,
                             'List of paths to the config files.')
tf.flags.DEFINE_multi_string('gin_bindings', None,
                             'List of Gin parameter bindings.')

tf.flags.DEFINE_string(
    'eval_imbalance_dataset', '', 'A dataset on which to perform evaluation '
    'for assessing how class imbalance affects performance in binary episodes. '
    'By default it is empty and no imbalance analysis is performed.')

# TODO(crisnv): eval_split is supposed to substitute eval_finegrainedness and
# eval_finegrainedness_split in the future
tf.flags.DEFINE_enum(
    'eval_split', None,
    [trainer.TRAIN_SPLIT, trainer.VALID_SPLIT, trainer.TEST_SPLIT],
    'Override the evaluation split. If None (default), regular logic is used, '
    'that is, if "is_training" is True, "trainer.VALID_SPLIT" is used, '
    'otherwise "trainer.TEST_SPLIT" is used. The "is_training" case also uses '
    'the value of eval_finegrainedness_split if eval_finegrainedness is True.')

tf.flags.DEFINE_bool(
    'eval_finegrainedness', False, 'Whether to perform only 2-way ImageNet '
    'evaluation for assessing performance as a function of how finegrained '
    'each task is. This differs from usual ImageNet eval in the sampling '
    'procedure used to get episodes, and therefore requires its own setting.')

tf.flags.DEFINE_enum(
    'eval_finegrainedness_split', trainer.TRAIN_SPLIT,
    [trainer.TRAIN_SPLIT, trainer.VALID_SPLIT, trainer.TEST_SPLIT], 'The '
    'split whose results we want to use for the fine-grainedness analysis.'
    'Contrary to most analyses which are performed on the test split only, the '
    'fine-grainedness analysis may also be performed on the train or valid '
    'sub-graphs of ImageNet too, since the test sub-graph evidently does not '
    'exhibit enough variation in the fine-grainedness of its different tasks '
    'to allow for a meaningful analysis.')
# The following flag specifies substrings of variable names that should not be
# reloaded. `num_left_in_epoch' is a variable that influences the behavior of
# the EpochTrackers. Since the state of those trackers is not reloaded, neither
# should this variable. `fc_finetune' is a substring of the names of the
# variables in the episode-specific linear layer of the finetune baseline (used
# at meta-validation and meta-test times). Since this layer gets re-initialized
# to random weights in each new episode, there is no need to ever restore these
# weights. `linear_classifier' plays that role but for the MAML model: similarly
# in each new episode it is re-initialized (e.g. set to zeros or to the
# prototypes in the case of proto-MAML), so there is no need to restore these
# weights. `adam_opt' captures the variables of the within-episode optimizer of
# the finetune baseline when it is configured to perform that finetuning with
# adam. `fc' captures the variable names of the fully-connected layer for the
# all-way classification problem that the baselines solve at training time.
# There are certain situations where we need to omit reloading these weights to
# avoid getting an error. Consider for example the experiments where we train
# a baseline model, starting from weights that were previously trained on
# ImageNet. If this training now takes place on all datasets, the size of the
# all-way classification layer is now different (equal to the number of
# meta-training classes of all datasets not just of ImageNet). Thus when
# training baselines from pre-trained weights, we only reload the backbone and
# not the `fc' all-way classification layer (similarly for inference-only
# experiments for the same reason).
tf.flags.DEFINE_multi_enum(
    'omit_from_saving_and_reloading', [
        'num_left_in_epoch', 'fc_finetune', 'linear_classifier', 'adam_opt',
        'weight_copy'
    ], [
        'num_left_in_epoch', 'fc_finetune', 'linear_classifier', 'adam_opt',
        'weight_copy', 'fc'
    ],
    'A comma-separated list of substrings such that all variables containing '
    'them should not be saved and reloaded.')

FLAGS = tf.flags.FLAGS

def parse_cmdline_gin_configurations():
  """Parse Gin configurations from all command-line sources."""
  with gin.unlock_config():
    gin.parse_config_files_and_bindings(
        FLAGS.gin_config, FLAGS.gin_bindings, finalize_config=True)

def operative_config_path(operative_config_dir,
                          operative_config_filename='operative_config.gin'):
  return os.path.join(operative_config_dir, operative_config_filename)

def load_operative_gin_configurations(operative_config_dir):
  """Load operative Gin configurations from the given directory."""
  gin_log_file = operative_config_path(operative_config_dir)
  with gin.unlock_config():
    gin.parse_config_file(gin_log_file)
  gin.finalize()
  logging.info('Operative Gin configurations loaded from %s.', gin_log_file)

def record_operative_gin_configurations(operative_config_dir):
  """Record operative Gin configurations in the given directory."""
  gin_log_file = operative_config_path(operative_config_dir)
  # If it exists already, rename it instead of overwriting it.
  # This just saves the previous one, not all the ones before.
  if tf.io.gfile.exists(gin_log_file):
    tf.io.gfile.rename(gin_log_file, gin_log_file + '.old', overwrite=True)
  with tf.io.gfile.GFile(gin_log_file, 'w') as f:
    f.write(gin.operative_config_str())

def main(unused_argv):

  # Parse Gin configurations passed to this script.
  parse_cmdline_gin_configurations()

  if FLAGS.reload_checkpoint_gin_config:
    # Try to reload a previously recorded Gin configuration from an operative
    # Gin configuration file in one of the provided directories.
    # TODO(eringrant): Allow querying of a value to be bound without binding it
    # to avoid the redundant call to `parse_cmdline_gin_configurations` below.
    try:
      checkpoint_to_restore = gin.query_parameter(
          'Trainer.checkpoint_to_restore')
    except ValueError:
      checkpoint_to_restore = None

    # Load the operative Gin configurations from the checkpoint directory.
    if checkpoint_to_restore:
      restore_checkpoint_dir = os.path.dirname(checkpoint_to_restore)
      load_operative_gin_configurations(restore_checkpoint_dir)

      # Reload the command-line Gin configuration to allow overriding of the Gin
      # configuration loaded from the checkpoint directory.
      parse_cmdline_gin_configurations()

  # Wrap object instantiations to print out full Gin configuration on failure.
  try:
    (train_datasets, eval_datasets, restrict_classes,
     restrict_num_per_class) = trainer.get_datasets_and_restrictions()

    # Get a trainer or evaluator.
    trainer_instance = trainer.Trainer(
        is_training=FLAGS.is_training,
        train_dataset_list=train_datasets,
        eval_dataset_list=eval_datasets,
        restrict_classes=restrict_classes,
        restrict_num_per_class=restrict_num_per_class,
        checkpoint_dir=FLAGS.train_checkpoint_dir,
        summary_dir=FLAGS.summary_dir,
        records_root_dir=FLAGS.records_root_dir,
        eval_finegrainedness=FLAGS.eval_finegrainedness,
        eval_finegrainedness_split=FLAGS.eval_finegrainedness_split,
        eval_imbalance_dataset=FLAGS.eval_imbalance_dataset,
        omit_from_saving_and_reloading=FLAGS.omit_from_saving_and_reloading,
        eval_split=FLAGS.eval_split,
    )
  except ValueError as e:
    logging.info('Full Gin configurations:\n%s', gin.config_str())
    raise e

  # All configurable objects/functions should have been instantiated/called.
  # TODO(evcu): Tie saving of Gin configuration at training and evaluation time.
  logging.info('Operative Gin configurations:\n%s', gin.operative_config_str())
  if FLAGS.is_training and FLAGS.train_checkpoint_dir:
    record_operative_gin_configurations(FLAGS.train_checkpoint_dir)
  elif not FLAGS.is_training and FLAGS.summary_dir:
    record_operative_gin_configurations(FLAGS.summary_dir)

  datasets = train_datasets if FLAGS.is_training else eval_datasets
  logging.info('Starting %s for dataset(s) %s...',
               'training' if FLAGS.is_training else 'evaluation', datasets)
  if FLAGS.is_training:
    trainer_instance.train()
  elif set(datasets).intersection(trainer.DATASETS_WITH_EXAMPLE_SPLITS):
    if not data.POOL_SUPPORTED:
      raise NotImplementedError('Example-level splits or pools not supported.')
  else:
    if len(datasets) != 1:
      raise ValueError('Requested datasets {} for evaluation, but evaluation '
                       'should be performed on individual datasets '
                       'only.'.format(datasets))

    if FLAGS.eval_finegrainedness:
      eval_split = FLAGS.eval_finegrainedness_split
    elif FLAGS.eval_split:
      eval_split = FLAGS.eval_split
    else:
      eval_split = trainer.TEST_SPLIT

    _, _, acc_summary, ci_acc_summary = trainer_instance.evaluate(eval_split)
    if trainer_instance.summary_writer:
      trainer_instance.summary_writer.add_summary(acc_summary)
      trainer_instance.summary_writer.add_summary(ci_acc_summary)

  # Flushes the event file to disk and closes the file.
  if trainer_instance.summary_writer:
    trainer_instance.summary_writer.close()

program = main

if __name__ == '__main__':
  logging.set_verbosity(logging.INFO)
  app.run(program)
<commit_msg>Internal.<commit_after># coding=utf-8
# Copyright 2020 The Meta-Dataset Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python2, python3
r"""Script for training models on the benchmark.

Launching command for batch baseline:
# pylint: disable=line-too-long
python -m meta_dataset.train \
  --train_checkpoint_dir=/tmp/bench --summary_dir=/tmp/bench \
  --records_root_dir=<records_root> \
  --alsologtostderr \
  --gin_config=meta_dataset/learn/gin/default/<exp_name>.gin
  --gin_bindings="Trainer.experiment_name='<exp_name>'"
# pylint: enable=line-too-long

where:
  <exp_name> is e.g. 'debug_proto_mini_imagenet'

To override elements from the config, you can use arguments of the form:
  For gin: --gin_bindings='foo = 1000000'
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import json
import os

from absl import app
from absl import logging
import gin.tf
from meta_dataset import data
from meta_dataset import learners  # pylint: disable=unused-import
from meta_dataset import trainer
# pylint: disable=unused-import
from meta_dataset.data import config
# pylint: enable=unused-import
import tensorflow.compat.v1 as tf
tf.disable_eager_execution()

DEFAULT_SAVING_DIRECTORY = '/tmp/metadataset'

tf.flags.DEFINE_string('train_checkpoint_dir',
                       os.path.join(DEFAULT_SAVING_DIRECTORY, 'checkpoints'),
                       'The directory to save checkpoints.')
tf.flags.DEFINE_string('summary_dir',
                       os.path.join(DEFAULT_SAVING_DIRECTORY, 'summaries'),
                       'The directory for writing summaries.')
tf.flags.DEFINE_bool(
    'reload_checkpoint_gin_config', False,
    'Whether to reload an operative Gin configuration along with a checkpoint.')
tf.flags.DEFINE_string('records_root_dir', '',
                       'Root directory containing a subdirectory per dataset.')
tf.flags.DEFINE_bool(
    'is_training', True, 'Whether we are in the training phase. '
    'Used to control whether to perform training or evaluation.')

tf.flags.DEFINE_multi_string('gin_config', None,
                             'List of paths to the config files.')
tf.flags.DEFINE_multi_string('gin_bindings', None,
                             'List of Gin parameter bindings.')

tf.flags.DEFINE_string(
    'eval_imbalance_dataset', '', 'A dataset on which to perform evaluation '
    'for assessing how class imbalance affects performance in binary episodes. '
    'By default it is empty and no imbalance analysis is performed.')

# TODO(crisnv): eval_split is supposed to substitute eval_finegrainedness and
# eval_finegrainedness_split in the future
tf.flags.DEFINE_enum(
    'eval_split', None,
    [trainer.TRAIN_SPLIT, trainer.VALID_SPLIT, trainer.TEST_SPLIT],
    'Override the evaluation split. If None (default), regular logic is used, '
    'that is, if "is_training" is True, "trainer.VALID_SPLIT" is used, '
    'otherwise "trainer.TEST_SPLIT" is used. The "is_training" case also uses '
    'the value of eval_finegrainedness_split if eval_finegrainedness is True.')

tf.flags.DEFINE_bool(
    'eval_finegrainedness', False, 'Whether to perform only 2-way ImageNet '
    'evaluation for assessing performance as a function of how finegrained '
    'each task is. This differs from usual ImageNet eval in the sampling '
    'procedure used to get episodes, and therefore requires its own setting.')

tf.flags.DEFINE_enum(
    'eval_finegrainedness_split', trainer.TRAIN_SPLIT,
    [trainer.TRAIN_SPLIT, trainer.VALID_SPLIT, trainer.TEST_SPLIT], 'The '
    'split whose results we want to use for the fine-grainedness analysis.'
    'Contrary to most analyses which are performed on the test split only, the '
    'fine-grainedness analysis may also be performed on the train or valid '
    'sub-graphs of ImageNet too, since the test sub-graph evidently does not '
    'exhibit enough variation in the fine-grainedness of its different tasks '
    'to allow for a meaningful analysis.')
# The following flag specifies substrings of variable names that should not be
# reloaded. `num_left_in_epoch' is a variable that influences the behavior of
# the EpochTrackers. Since the state of those trackers is not reloaded, neither
# should this variable. `fc_finetune' is a substring of the names of the
# variables in the episode-specific linear layer of the finetune baseline (used
# at meta-validation and meta-test times). Since this layer gets re-initialized
# to random weights in each new episode, there is no need to ever restore these
# weights. `linear_classifier' plays that role but for the MAML model: similarly
# in each new episode it is re-initialized (e.g. set to zeros or to the
# prototypes in the case of proto-MAML), so there is no need to restore these
# weights. `adam_opt' captures the variables of the within-episode optimizer of
# the finetune baseline when it is configured to perform that finetuning with
# adam. `fc' captures the variable names of the fully-connected layer for the
# all-way classification problem that the baselines solve at training time.
# There are certain situations where we need to omit reloading these weights to
# avoid getting an error. Consider for example the experiments where we train
# a baseline model, starting from weights that were previously trained on
# ImageNet. If this training now takes place on all datasets, the size of the
# all-way classification layer is now different (equal to the number of
# meta-training classes of all datasets not just of ImageNet). Thus when
# training baselines from pre-trained weights, we only reload the backbone and
# not the `fc' all-way classification layer (similarly for inference-only
# experiments for the same reason).
tf.flags.DEFINE_multi_enum(
    'omit_from_saving_and_reloading', [
        'num_left_in_epoch', 'fc_finetune', 'linear_classifier', 'adam_opt',
        'weight_copy'
    ], [
        'num_left_in_epoch', 'fc_finetune', 'linear_classifier', 'adam_opt',
        'weight_copy', 'fc'
    ],
    'A comma-separated list of substrings such that all variables containing '
    'them should not be saved and reloaded.')

FLAGS = tf.flags.FLAGS

def parse_cmdline_gin_configurations():
  """Parse Gin configurations from all command-line sources."""
  with gin.unlock_config():
    gin.parse_config_files_and_bindings(
        FLAGS.gin_config, FLAGS.gin_bindings, finalize_config=True)

def operative_config_path(operative_config_dir,
                          operative_config_filename='operative_config.gin'):
  return os.path.join(operative_config_dir, operative_config_filename)

def load_operative_gin_configurations(operative_config_dir):
  """Load operative Gin configurations from the given directory."""
  gin_log_file = operative_config_path(operative_config_dir)
  with gin.unlock_config():
    gin.parse_config_file(gin_log_file)
  gin.finalize()
  logging.info('Operative Gin configurations loaded from %s.', gin_log_file)

def record_operative_gin_configurations(operative_config_dir):
  """Record operative Gin configurations in the given directory."""
  gin_log_file = operative_config_path(operative_config_dir)
  # If it exists already, rename it instead of overwriting it.
  # This just saves the previous one, not all the ones before.
  if tf.io.gfile.exists(gin_log_file):
    tf.io.gfile.rename(gin_log_file, gin_log_file + '.old', overwrite=True)
  with tf.io.gfile.GFile(gin_log_file, 'w') as f:
    f.write(gin.operative_config_str())

def main(unused_argv):

  # Parse Gin configurations passed to this script.
  parse_cmdline_gin_configurations()

  if FLAGS.reload_checkpoint_gin_config:
    # Try to reload a previously recorded Gin configuration from an operative
    # Gin configuration file in one of the provided directories.
    # TODO(eringrant): Allow querying of a value to be bound without binding it
    # to avoid the redundant call to `parse_cmdline_gin_configurations` below.
    try:
      checkpoint_to_restore = gin.query_parameter(
          'Trainer.checkpoint_to_restore')
    except ValueError:
      checkpoint_to_restore = None

    # Load the operative Gin configurations from the checkpoint directory.
    if checkpoint_to_restore:
      restore_checkpoint_dir = os.path.dirname(checkpoint_to_restore)
      load_operative_gin_configurations(restore_checkpoint_dir)

      # Reload the command-line Gin configuration to allow overriding of the Gin
      # configuration loaded from the checkpoint directory.
      parse_cmdline_gin_configurations()

  # Wrap object instantiations to print out full Gin configuration on failure.
  try:
    (train_datasets, eval_datasets, restrict_classes,
     restrict_num_per_class) = trainer.get_datasets_and_restrictions()

    # Get a trainer or evaluator.
    trainer_instance = trainer.Trainer(
        is_training=FLAGS.is_training,
        train_dataset_list=train_datasets,
        eval_dataset_list=eval_datasets,
        restrict_classes=restrict_classes,
        restrict_num_per_class=restrict_num_per_class,
        checkpoint_dir=FLAGS.train_checkpoint_dir,
        summary_dir=FLAGS.summary_dir,
        records_root_dir=FLAGS.records_root_dir,
        eval_finegrainedness=FLAGS.eval_finegrainedness,
        eval_finegrainedness_split=FLAGS.eval_finegrainedness_split,
        eval_imbalance_dataset=FLAGS.eval_imbalance_dataset,
        omit_from_saving_and_reloading=FLAGS.omit_from_saving_and_reloading,
        eval_split=FLAGS.eval_split,
    )
  except ValueError as e:
    logging.info('Full Gin configurations:\n%s', gin.config_str())
    raise e

  # All configurable objects/functions should have been instantiated/called.
  # TODO(evcu): Tie saving of Gin configuration at training and evaluation time.
  logging.info('Operative Gin configurations:\n%s', gin.operative_config_str())
  if FLAGS.is_training and FLAGS.train_checkpoint_dir:
    record_operative_gin_configurations(FLAGS.train_checkpoint_dir)
  elif not FLAGS.is_training and FLAGS.summary_dir:
    record_operative_gin_configurations(FLAGS.summary_dir)

  datasets = train_datasets if FLAGS.is_training else eval_datasets
  logging.info('Starting %s for dataset(s) %s...',
               'training' if FLAGS.is_training else 'evaluation', datasets)
  if FLAGS.is_training:
    trainer_instance.train()
  elif set(datasets).intersection(trainer.DATASETS_WITH_EXAMPLE_SPLITS):
    if not data.POOL_SUPPORTED:
      raise NotImplementedError('Example-level splits or pools not supported.')
  else:
    if len(datasets) != 1:
      raise ValueError('Requested datasets {} for evaluation, but evaluation '
                       'should be performed on individual datasets '
                       'only.'.format(datasets))

    if FLAGS.eval_finegrainedness:
      eval_split = FLAGS.eval_finegrainedness_split
    elif FLAGS.eval_split:
      eval_split = FLAGS.eval_split
    else:
      eval_split = trainer.TEST_SPLIT

    _, _, acc_summary, ci_acc_summary = trainer_instance.evaluate(eval_split)
    if trainer_instance.summary_writer:
      trainer_instance.summary_writer.add_summary(acc_summary)
      trainer_instance.summary_writer.add_summary(ci_acc_summary)

  # Flushes the event file to disk and closes the file.
  if trainer_instance.summary_writer:
    trainer_instance.summary_writer.close()

program = main

if __name__ == '__main__':
  logging.set_verbosity(logging.INFO)
  app.run(program)
<|endoftext|>