<commit_before>#!/usr/bin/env python3

from collections import Counter, defaultdict
import re

from xdfile.utils import progress, open_output, get_args, args_parser, COLUMN_SEPARATOR
from xdfile.utils import br_with_n
from xdfile import html, utils, catalog, pubyear
from xdfile import metadatabase as metadb
from xdfile.html import GridCalendar, mktag, year_widget
from xdfile.xdfile import num_cells
import xdfile

class PublicationStats:
    def __init__(self, pubid):
        self.pubid = pubid
        self.copyrights = Counter()  # [copyright_text] -> number of xd
        self.editors = Counter()  # [editor_name] -> number of xd
        self.formats = Counter()  # ["15x15 RS"] -> number of xd
        self.mindate = ""
        self.maxdate = ""
        self.num_xd = 0

        self.puzzles_meta = []

    def add(self, puzrow):
        self.copyrights[puzrow.Copyright.strip()] += 1
        self.editors[puzrow.Editor.strip()] += 1
        self.formats[puzrow.Size] += 1
        datestr = puzrow.Date
        if datestr:
            if not self.mindate:
                self.mindate = datestr
            else:
                self.mindate = min(self.mindate, datestr)
            if not self.maxdate:
                self.maxdate = datestr
            else:
                self.maxdate = max(self.maxdate, datestr)
        self.num_xd += 1

        self.puzzles_meta.append(puzrow)

    def meta(self):
        return 'pubid num dates formats copyrights editors'.split()

    def row(self):
        return [
                self.pubid,
                html.mkhref(str(self.num_xd), self.pubid),
                "%s — %s" % (self.mindate, self.maxdate),
                html_select_options(self.formats),
                html_select_options(self.copyrights),
                html_select_options(self.editors),
               ]

def tally_to_cell(d):
    freq_sorted = sorted([(v, k) for k, v in list(d.items())], reverse=True)

    if not freq_sorted:
        return ""
    elif len(freq_sorted) == 1:
        return "
".join("%s [x%s]" % (k, v) for v, k in freq_sorted)
    else:
        return "<select><option>" + "</option><option>".join("%s [x%s]" % (k, v) for v, k in freq_sorted) + "</select>"

def publication_header():
    return "PubId NumCollected DatesCollected Formats Copyrights Editors".split()

def main():
    parser = args_parser("generate publishers index html pages and index")

    args = get_args(parser=parser)

    outf = open_output()
    all_pubs = {}  # [(pubid,year)] -> PublicationStats
    pubyear_rows = {}
    similar = metadb.xd_similar()
    puzzles = metadb.xd_puzzles()
    outf.write_html('pub/index.html', pubyear.pubyear_html(), title='The xd crossword puzzle corpus')

    utils.log("collating puzzles")
    for puzrow in puzzles.values():
            pubid = utils.parse_pubid(puzrow.xdid)
            year = xdfile.year_from_date(puzrow.Date)
            k = (pubid, year or 9999)
            if k not in all_pubs:
                all_pubs[k] = PublicationStats(pubid)

            pubyear_rows[pubid] = []

            all_pubs[k].add(puzrow)

    pubyear_header = [ 'xdid', 'Date', 'Size', 'Title', 'Author', 'Editor', 'Copyright', 'Grid_1A_1D', 'ReusedCluePct', 'SimilarGrids' ]
    utils.log('generating index pages')

    # dict to generate pub page with calendars
    pub_grids = defaultdict(dict)

    for pair, pub in sorted(list(all_pubs.items())):
        c_grids = {}
        pubid, year = pair
        progress(pubid)

        reused_clues = 0
        reused_answers = 0
        total_clues = 0
        total_similar = []

        rows = []

        # Assign class based on xdid and similars
        def get_cell_classes(r):
            """ Return cell classes based on parameters """
            # TODO: Implement check that authors same
            classes = []
            rsim = similar.get(r.xdid)
            if rsim and float(rsim.similar_grid_pct) > 0:
                matches = [x.split('=') for x in rsim.matches.split()]
                # Get max for matches for class definition
                max_pct = int(max([ x[1] for x in matches ]))
                # < 40%: 'theme', 40-90%: 'partial', 90-99%: 'full', 100%: 'exact'
                if max_pct > 0 and max_pct < 40:
                    classes.append('pctfilled')
                if max_pct >= 40 and max_pct < 90:
                    classes.append('partial')
                if max_pct >= 90 and max_pct <= 99:
                    classes.append('full')
                if max_pct >= 100:
                    classes.append('exact')
                # Highlight only grids sized > 400 cells
                if num_cells(r.Size) >= 400:
                    classes.append('biggrid')
                # Check for pub similarity
                pubid, y, m, d = utils.split_xdid(r.xdid)
                ymd = '%s%s%s' % (y, m, d)
                if pubid not in [ x[0] for x in matches ]:
                    for m in [ x[0] for x in matches ]:
                        p, y1, m1, d1 = utils.split_xdid(m)
                        ymd1 = '%s%s%s' % (y1, m1, d1)
                        if ymd1 < ymd and p != pubid:
                            classes.append('stolen')

            return ' '.join(classes)

        for r in pub.puzzles_meta:
            similar_text = ""
            reused_clue_pct = "n/a"

            rsim = similar.get(r.xdid)
            if rsim:
                similar_pct = float(rsim.similar_grid_pct)
                if similar_pct > 0:
                    matches = [x.split('=') for x in rsim.matches.split()]
                    for xdid, pct in matches:
                        similar_text += '(%s%%) %s [%s]
' % (pct, puzzles[xdid].Author, xdid)
                    total_similar.append(similar_pct)
                else:
                    similar_text = "0"

                nclues = int(rsim.total_clues)

                reused_clues += int(rsim.reused_clues)
                reused_answers += int(rsim.reused_answers)
                total_clues += nclues

                if nclues:
                    reused_clue_pct = int(100*(float(rsim.reused_clues) / float(nclues)))
                else:
                    reused_clue_pct = ''

            row_dict = {} # Map row and style
            if similar_text and similar_text != "0":
                # http://stackoverflow.com/questions/1418838/html-making-a-link-lead-to-the-anchor-centered-in-the-middle-of-the-page
                pubidtext = '' % r.xdid
                pubidtext += ''
                pubidtext += html.mkhref(r.xdid, '/pub/' + r.xdid)
                c_grids[r.Date] = {
                        'link' : '/pub/%s%s/index.html#' % (pubid, year) + r.xdid,
                        'class': get_cell_classes(r),
                        'title': br_with_n(similar_text),
                        }
                print(c_grids[r.Date])
                row_dict['tag_params'] = {
                    'onclick': 'location.href=\'/pub/%s\'' % r.xdid,
                    'class': 'puzzlehl'
                    }
            else:
                pubidtext = r.xdid
                row_dict['class'] = 'puzzle'

            row = [
                pubidtext,
                r.Date,
                r.Size,
                r.Title,
                r.Author,
                r.Editor,
                r.Copyright,
                r.A1_D1,
                reused_clue_pct,
                similar_text
              ]

            outf.write_row('pub/%s%s.tsv' % (pubid, year), " ".join(pubyear_header), row)
            row_dict['row'] = row
            rows.append(row_dict)

        pub_grids[pubid][year] = c_grids

        # Generate calendar
        onepubyear_html = GridCalendar(c_grids).formatyear(year, 6) + "
"

        # Generate html table sorted by 2nd element of row (date)
        onepubyear_html += html.html_table(sorted(rows , key=lambda x: x['row'][1]), pubyear_header, "puzzle", "puzzles")
        outf.write_html("pub/%s%s/index.html" % (pubid, year), onepubyear_html, title="%s %s" % (pubid, year))

        cluepct = ""
        wordpct = ""
        if total_clues:
            cluepct = "%d%%" % int(100.0*(total_clues-reused_clues)/total_clues)
            wordpct = "%.2f%%" % int(100.0*(total_clues-reused_answers)/total_clues)

        pubyear_rows[pubid].append([
            pubid,
            str(year),
            len(rows),
            "%.2f/%d" % (sum(total_similar)/100.0, len(total_similar)),
            wordpct,
            cluepct
            ])

    # Generate /pub/[publisher][year] page
    for pubid in pub_grids.keys():
        body = []
        for y in sorted(pub_grids[pubid], reverse=True):
            body.append(GridCalendar(pub_grids[pubid][y]).formatyear(y, 12, vertical=True) + mktag('br','break'))
        outf.write_html("pub/%s/index.html" % pubid, ''.join(body), title="%s" % pubid)

    progress()

if __name__ == "__main__":
    main()
<commit_msg>Code cleanup<commit_after>#!/usr/bin/env python3

from collections import Counter, defaultdict
import re

from xdfile.utils import progress, open_output, get_args, args_parser, COLUMN_SEPARATOR
from xdfile.utils import br_with_n
from xdfile import html, utils, catalog, pubyear
from xdfile import metadatabase as metadb
from xdfile.html import GridCalendar, mktag, year_widget
from xdfile.xdfile import num_cells
import xdfile

class PublicationStats:
    def __init__(self, pubid):
        self.pubid = pubid
        self.copyrights = Counter()  # [copyright_text] -> number of xd
        self.editors = Counter()  # [editor_name] -> number of xd
        self.formats = Counter()  # ["15x15 RS"] -> number of xd
        self.mindate = ""
        self.maxdate = ""
        self.num_xd = 0

        self.puzzles_meta = []

    def add(self, puzrow):
        self.copyrights[puzrow.Copyright.strip()] += 1
        self.editors[puzrow.Editor.strip()] += 1
        self.formats[puzrow.Size] += 1
        datestr = puzrow.Date
        if datestr:
            if not self.mindate:
                self.mindate = datestr
            else:
                self.mindate = min(self.mindate, datestr)
            if not self.maxdate:
                self.maxdate = datestr
            else:
                self.maxdate = max(self.maxdate, datestr)
        self.num_xd += 1

        self.puzzles_meta.append(puzrow)

    def meta(self):
        return 'pubid num dates formats copyrights editors'.split()

    def row(self):
        return [
                self.pubid,
                html.mkhref(str(self.num_xd), self.pubid),
                "%s — %s" % (self.mindate, self.maxdate),
                html_select_options(self.formats),
                html_select_options(self.copyrights),
                html_select_options(self.editors),
               ]

def tally_to_cell(d):
    freq_sorted = sorted([(v, k) for k, v in list(d.items())], reverse=True)

    if not freq_sorted:
        return ""
    elif len(freq_sorted) == 1:
        return "
".join("%s [x%s]" % (k, v) for v, k in freq_sorted)
    else:
        return "<select><option>" + "</option><option>".join("%s [x%s]" % (k, v) for v, k in freq_sorted) + "</select>"

def publication_header():
    return "PubId NumCollected DatesCollected Formats Copyrights Editors".split()

def main():
    parser = args_parser("generate publishers index html pages and index")

    args = get_args(parser=parser)

    outf = open_output()
    all_pubs = {}  # [(pubid,year)] -> PublicationStats
    pubyear_rows = {}
    similar = metadb.xd_similar()
    puzzles = metadb.xd_puzzles()
    outf.write_html('pub/index.html', pubyear.pubyear_html(), title='The xd crossword puzzle corpus')

    utils.log("collating puzzles")
    for puzrow in puzzles.values():
            pubid = utils.parse_pubid(puzrow.xdid)
            year = xdfile.year_from_date(puzrow.Date)
            k = (pubid, year or 9999)
            if k not in all_pubs:
                all_pubs[k] = PublicationStats(pubid)

            pubyear_rows[pubid] = []

            all_pubs[k].add(puzrow)

    pubyear_header = [ 'xdid', 'Date', 'Size', 'Title', 'Author', 'Editor', 'Copyright', 'Grid_1A_1D', 'ReusedCluePct', 'SimilarGrids' ]
    utils.log('generating index pages')

    # dict to generate pub page with calendars
    pub_grids = defaultdict(dict)

    for pair, pub in sorted(list(all_pubs.items())):
        c_grids = {}
        pubid, year = pair
        progress(pubid)

        reused_clues = 0
        reused_answers = 0
        total_clues = 0
        total_similar = []

        rows = []

        # Assign class based on xdid and similars
        def get_cell_classes(r):
            """ Return cell classes based on parameters """
            # TODO: Implement check that authors same
            classes = []
            rsim = similar.get(r.xdid)
            if rsim and float(rsim.similar_grid_pct) > 0:
                matches = [x.split('=') for x in rsim.matches.split()]
                # Get max for matches for class definition
                max_pct = int(max([ x[1] for x in matches ]))
                # < 40%: 'theme', 40-90%: 'partial', 90-99%: 'full', 100%: 'exact'
                if max_pct > 0 and max_pct < 40:
                    classes.append('pctfilled')
                if max_pct >= 40 and max_pct < 90:
                    classes.append('partial')
                if max_pct >= 90 and max_pct <= 99:
                    classes.append('full')
                if max_pct >= 100:
                    classes.append('exact')
                # Highlight only grids sized > 400 cells
                if num_cells(r.Size) >= 400:
                    classes.append('biggrid')
                # Check for pub similarity
                pubid, y, m, d = utils.split_xdid(r.xdid)
                ymd = '%s%s%s' % (y, m, d)
                if pubid not in [ x[0] for x in matches ]:
                    for m in [ x[0] for x in matches ]:
                        p, y1, m1, d1 = utils.split_xdid(m)
                        ymd1 = '%s%s%s' % (y1, m1, d1)
                        if ymd1 < ymd and p != pubid:
                            classes.append('stolen')

            return ' '.join(classes)

        for r in pub.puzzles_meta:
            similar_text = ""
            reused_clue_pct = "n/a"

            rsim = similar.get(r.xdid)
            if rsim:
                similar_pct = float(rsim.similar_grid_pct)
                if similar_pct > 0:
                    matches = [x.split('=') for x in rsim.matches.split()]
                    for xdid, pct in matches:
                        similar_text += '(%s%%) %s [%s]
' % (pct, puzzles[xdid].Author, xdid)
                    total_similar.append(similar_pct)
                else:
                    similar_text = "0"

                nclues = int(rsim.total_clues)

                reused_clues += int(rsim.reused_clues)
                reused_answers += int(rsim.reused_answers)
                total_clues += nclues

                if nclues:
                    reused_clue_pct = int(100*(float(rsim.reused_clues) / float(nclues)))
                else:
                    reused_clue_pct = ''

            row_dict = {} # Map row and style
            if similar_text and similar_text != "0":
                # http://stackoverflow.com/questions/1418838/html-making-a-link-lead-to-the-anchor-centered-in-the-middle-of-the-page
                pubidtext = '' % r.xdid
                pubidtext += ''
                pubidtext += html.mkhref(r.xdid, '/pub/' + r.xdid)
                c_grids[r.Date] = {
                        'link' : '/pub/%s%s/index.html#' % (pubid, year) + r.xdid,
                        'class': get_cell_classes(r),
                        'title': br_with_n(similar_text),
                        }
                row_dict['tag_params'] = {
                    'onclick': 'location.href=\'/pub/%s\'' % r.xdid,
                    'class': 'puzzlehl'
                    }
            else:
                pubidtext = r.xdid
                row_dict['class'] = 'puzzle'

            row = [
                pubidtext,
                r.Date,
                r.Size,
                r.Title,
                r.Author,
                r.Editor,
                r.Copyright,
                r.A1_D1,
                reused_clue_pct,
                similar_text
              ]

            outf.write_row('pub/%s%s.tsv' % (pubid, year), " ".join(pubyear_header), row)
            row_dict['row'] = row
            rows.append(row_dict)

        pub_grids[pubid][year] = c_grids

        # Generate calendar
        onepubyear_html = GridCalendar(c_grids).formatyear(year, 6) + "
"

        # Generate html table sorted by 2nd element of row (date)
        onepubyear_html += html.html_table(sorted(rows , key=lambda x: x['row'][1]), pubyear_header, "puzzle", "puzzles")
        outf.write_html("pub/%s%s/index.html" % (pubid, year), onepubyear_html, title="%s %s" % (pubid, year))

        cluepct = ""
        wordpct = ""
        if total_clues:
            cluepct = "%d%%" % int(100.0*(total_clues-reused_clues)/total_clues)
            wordpct = "%.2f%%" % int(100.0*(total_clues-reused_answers)/total_clues)

        pubyear_rows[pubid].append([
            pubid,
            str(year),
            len(rows),
            "%.2f/%d" % (sum(total_similar)/100.0, len(total_similar)),
            wordpct,
            cluepct
            ])

    # Generate /pub/[publisher][year] page
    for pubid in pub_grids.keys():
        body = []
        for y in sorted(pub_grids[pubid], reverse=True):
            body.append(GridCalendar(pub_grids[pubid][y]).formatyear(y, 12, vertical=True) + mktag('br','break'))
        outf.write_html("pub/%s/index.html" % pubid, ''.join(body), title="%s" % pubid)

    progress()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from .exceptions import *
from .bindings import libzfs_core
from .nvlist import nvlist_in, nvlist_out

_ffi = libzfs_core._ffi
_lib = libzfs_core._lib

_lib.libzfs_core_init()

def lzc_create(name, is_zvol, props):
	ret = 0
	with nvlist_in(props) as nvlist:
		ret = _lib.lzc_create(name, is_zvol, nvlist)
	if ret != 0:
		raise {
			errno.EEXIST: FilesystemExists(name),
			errno.ENOENT: ParentNotFound(name),
		}.get(ret, ZFSError(ret, "Failed to create filesystem", name))

def lzc_snapshot(snaps, props, errlist):
	ret = 0
	with nvlist_in(snaps) as snaps_nvlist, nvlist_in(props) as props_nvlist:
		with nvlist_out(errlist) as errlist_nvlist:
			ret = _lib.lzc_snapshot(snaps_nvlist, props_nvlist, errlist_nvlist)
	if ret != 0:
		raise {
			errno.EEXIST: SnapshotExists(None),
			errno.ENOENT: FilesystemNotFound(None),
		}.get(ret, ZFSError(ret, "Failed to create snapshot", name))

def lzc_promote(name):
	ret = _lib.lzc_promote(name)
	if ret != 0:
		raise {
			errno.EEXIST: SnapshotExists(None),
			errno.ENOENT: FilesystemNotFound(None),
		}.get(ret, ZFSError(ret, "Failed to create snapshot", name))

def lzc_rollback(name):
	snapnamep = ffi.new('char[]', 256)
	ret = _lib.lzc_rollback(name, snapnamep, 256)
	return (ret, ffi.string(snapnamep))

def lzc_set_props(name, props, received):
	ret = 0
	with nvlist_in(props) as nvlist:
		ret = _lib.lzc_set_props(name, nvlist, received)
	return ret

def lzc_destroy_snaps(snaps, defer, errlist):
	ret = 0
	with nvlist_in(snaps) as snaps_nvlist:
		with nvlist_out(errlist) as errlist_nvlist:
			ret = _lib.lzc_destroy_snaps(snaps_nvlist, defer, errlist_nvlist)
	return ret

def lzc_bookmark(bookmarks, errlist):
	ret = 0
	with nvlist_in(bookmarks) as nvlist:
		with nvlist_out(errlist) as errlist_nvlist:
			ret = _lib.lzc_bookmark(nvlist, errlist_nvlist)
	return ret

def lzc_get_bookmarks(fsname, props, bmarks):
	ret = 0
	with nvlist_in(props) as nvlist:
		with nvlist_out(bmarks) as bmarks_nvlist:
			ret = _lib.lzc_get_bookmarks(fsname, nvlist, bmarks_nvlist)
	return ret

def lzc_destroy_bookmarks(bookmarks, errlist):
	ret = 0
	with nvlist_in(bookmarks) as nvlist:
		with nvlist_out(errlist) as errlist_nvlist:
			ret = _lib.lzc_destroy_bookmarks(nvlist, errlist_nvlist)
	return ret

def lzc_snaprange_space(firstsnap, lastsnap):
	valp = ffi.new('uint64_t *')
	ret = _lib.lzc_snaprange_space(firstsnap, lastsnap, valp)
	return (ret, int(valp[0]))

def lzc_hold(holds, fd, errlist):
	ret = 0
	with nvlist_in(holds) as nvlist:
		with nvlist_out(errlist) as errlist_nvlist:
			ret = _lib.lzc_hold(nvlist, fd, errlist_nvlist)
	return ret

def lzc_release(holds, errlist):
	ret = 0
	with nvlist_in(holds) as nvlist:
		with nvlist_out(errlist) as errlist_nvlist:
			ret = _lib.lzc_release(nvlist, errlist_nvlist)
	return ret

def lzc_get_holds(snapname, holds):
	ret = 0
	with nvlist_out(holds) as nvlist:
		ret = _lib.lzc_get_holds(snapname, nvlist)
	return ret

def lzc_send(snapname, fromsnap, fd, flags):
	ret = _lib.lzc_send(snapname, fromsnap, fd, flags)
	return ret

def lzc_send_ext(snapname, fromsnap, fd, props):
	ret = 0
	with nvlist_in(props) as nvlist:
		ret = _lib.lzc_send_ext(snapname, fromsnap, fd, props)
	return ret

def lzc_send_space(snapname, fromsnap):
	valp = ffi.new('uint64_t *')
	ret = _lib.lzc_send_space(snapname, fromsnap, valp)
	return (ret, int(valp[0]))

def lzc_send_progress(snapname, fd):
	valp = ffi.new('uint64_t *')
	ret = _lib.lzc_send_progress(snapname, fd, valp)
	return (ret, int(valp[0]))

def lzc_receive(snapname, props, origin, force, fd):
	ret = 0
	with nvlist_in(props) as nvlist:
		ret = _lib.lzc_receive(snapname, nvlist, origin, force, fd)
	return ret

def lzc_exists(name):
	ret = _lib.lzc_exists(name)
	return bool(ret)

<commit_msg>libzfs_core: catch up with changes, fix syntax errors<commit_after>from .exceptions import *
from .bindings import libzfs_core
from .nvlist import nvlist_in, nvlist_out

_ffi = libzfs_core.ffi
_lib = libzfs_core.lib

# TODO: a better way to init and uninit the library
_lib.libzfs_core_init()

def lzc_create(name, is_zvol, props):
	ret = 0
	with nvlist_in(props) as nvlist:
		ret = _lib.lzc_create(name, is_zvol, nvlist)
	if ret != 0:
		raise {
			errno.EEXIST: FilesystemExists(name),
			errno.ENOENT: ParentNotFound(name),
		}.get(ret, ZFSError(ret, "Failed to create filesystem", name))

def lzc_snapshot(snaps, props, errlist):
	ret = 0
	with nvlist_in(snaps) as snaps_nvlist, nvlist_in(props) as props_nvlist:
		with nvlist_out(errlist) as errlist_nvlist:
			ret = _lib.lzc_snapshot(snaps_nvlist, props_nvlist, errlist_nvlist)
	if ret != 0:
		raise {
			errno.EEXIST: SnapshotExists(None),
			errno.ENOENT: FilesystemNotFound(None),
		}.get(ret, ZFSError(ret, "Failed to create snapshot", name))

def lzc_promote(name):
	ret = _lib.lzc_promote(name)
	if ret != 0:
		raise {
			errno.EEXIST: SnapshotExists(None),
			errno.ENOENT: FilesystemNotFound(None),
		}.get(ret, ZFSError(ret, "Failed to create snapshot", name))

def lzc_rollback(name):
	snapnamep = _ffi.new('char[]', 256)
	ret = _lib.lzc_rollback(name, snapnamep, 256)
	return (ret, _ffi.string(snapnamep))

def lzc_set_props(name, props, received):
	ret = 0
	with nvlist_in(props) as nvlist:
		ret = _lib.lzc_set_props(name, nvlist, received)
	return ret

def lzc_destroy_snaps(snaps, defer, errlist):
	ret = 0
	with nvlist_in(snaps) as snaps_nvlist:
		with nvlist_out(errlist) as errlist_nvlist:
			ret = _lib.lzc_destroy_snaps(snaps_nvlist, defer, errlist_nvlist)
	return ret

def lzc_bookmark(bookmarks, errlist):
	ret = 0
	with nvlist_in(bookmarks) as nvlist:
		with nvlist_out(errlist) as errlist_nvlist:
			ret = _lib.lzc_bookmark(nvlist, errlist_nvlist)
	return ret

def lzc_get_bookmarks(fsname, props, bmarks):
	ret = 0
	with nvlist_in(props) as nvlist:
		with nvlist_out(bmarks) as bmarks_nvlist:
			ret = _lib.lzc_get_bookmarks(fsname, nvlist, bmarks_nvlist)
	return ret

def lzc_destroy_bookmarks(bookmarks, errlist):
	ret = 0
	with nvlist_in(bookmarks) as nvlist:
		with nvlist_out(errlist) as errlist_nvlist:
			ret = _lib.lzc_destroy_bookmarks(nvlist, errlist_nvlist)
	return ret

def lzc_snaprange_space(firstsnap, lastsnap):
	valp = _ffi.new('uint64_t *')
	ret = _lib.lzc_snaprange_space(firstsnap, lastsnap, valp)
	return (ret, int(valp[0]))

def lzc_hold(holds, fd, errlist):
	ret = 0
	with nvlist_in(holds) as nvlist:
		with nvlist_out(errlist) as errlist_nvlist:
			ret = _lib.lzc_hold(nvlist, fd, errlist_nvlist)
	return ret

def lzc_release(holds, errlist):
	ret = 0
	with nvlist_in(holds) as nvlist:
		with nvlist_out(errlist) as errlist_nvlist:
			ret = _lib.lzc_release(nvlist, errlist_nvlist)
	return ret

def lzc_get_holds(snapname, holds):
	ret = 0
	with nvlist_out(holds) as nvlist:
		ret = _lib.lzc_get_holds(snapname, nvlist)
	return ret

def lzc_send(snapname, fromsnap, fd, flags):
	ret = _lib.lzc_send(snapname, fromsnap, fd, flags)
	return ret

def lzc_send_ext(snapname, fromsnap, fd, props):
	ret = 0
	with nvlist_in(props) as nvlist:
		ret = _lib.lzc_send_ext(snapname, fromsnap, fd, props)
	return ret

def lzc_send_space(snapname, fromsnap):
	valp = _ffi.new('uint64_t *')
	ret = _lib.lzc_send_space(snapname, fromsnap, valp)
	return (ret, int(valp[0]))

def lzc_send_progress(snapname, fd):
	valp = _ffi.new('uint64_t *')
	ret = _lib.lzc_send_progress(snapname, fd, valp)
	return (ret, int(valp[0]))

def lzc_receive(snapname, props, origin, force, fd):
	ret = 0
	with nvlist_in(props) as nvlist:
		ret = _lib.lzc_receive(snapname, nvlist, origin, force, fd)
	return ret

def lzc_exists(name):
	ret = _lib.lzc_exists(name)
	return bool(ret)

<|endoftext|>


<commit_before>from __future__ import absolute_import

import logging
import os
import sys
import time
import traceback

from coverage import coverage
from coverage.report import Reporter
from nose.plugins.base import Plugin
from nose_bleed.diff import DiffParser
from operator import or_
from subprocess import Popen, PIPE, STDOUT
from sqlalchemy import create_engine, Table, MetaData, Integer, String, \
  Column
from sqlalchemy.sql import select

def is_py_script(filename):
    "Returns True if a file is a python executable."
    if filename.endswith(".py") and os.path.exists(filename):
        return True
    elif not os.access(filename, os.X_OK):
        return False
    else:
        try:
            first_line = open(filename, "r").next().strip()
            return "#!" in first_line and "python" in first_line
        except StopIteration:
            return False

metadata = MetaData()
Coverage = Table('coverage', metadata,
    Column('id', Integer, primary_key=True),
    Column('filename', String),
    Column('lineno', Integer),
    Column('test', String, index=True)
)

class TestCoverageDB(object):
    def __init__(self, dsn, logger):
        self.logger = logger
        self.engine = create_engine(dsn)
        self.conn = self._connect_db()

        self._coverage = {}

    def _connect_db(self):
        self.logger.info('Connecting to coverage database..')
        s = time.time()
        conn = self.engine.connect()
        self.logger.info('Connection established to coverage database in %.2fs', time.time() - s)
        return conn

    def upgrade(self):
        metadata.create_all(self.conn)

    def begin(self):
        return self.conn.begin()

    def has_seen_test(self, test):
        statement = select([Coverage.c.id], Coverage.c.test == test, limit=1)
        result = bool(self.conn.execute(statement).fetchall())
        return result

    def has_test_coverage(self, filename):
        if filename not in self._coverage:
            self._coverage[filename] = file_cover = {}
            statement = select([Coverage.c.lineno, Coverage.c.test], Coverage.c.filename == filename)
            for lineno, test in self.conn.execute(statement).fetchall():
                file_cover.setdefault(lineno, set()).add(test)
            self.commit()
        return bool(self._coverage[filename])

    def get_test_coverage(self, filename, linenos):
        if filename not in self._coverage:
            self._coverage[filename] = file_cover = {}
            statement = select([Coverage.c.lineno, Coverage.c.test], Coverage.c.filename == filename, Coverage.c.lineno.in_(linenos))
            for lineno, test in self.conn.execute(statement).fetchall():
                file_cover.setdefault(lineno, set()).add(test)
        return reduce(or_, (self._coverage[filename].get(l, set()) for l in linenos))

    def clear_test_coverage(self, test):
        # clean up existing tests
        statement = select([Coverage.c.lineno, Coverage.c.test], Coverage.c.test==test)
        for filename, lineno in self.conn.execute(statement).fetchall():
            if filename not in self._coverage:
                continue
            if lineno not in self._coverage[filename]:
                continue
            self._coverage[filename][lineno].discard(test)

        self.conn.execute(Coverage.delete().where(Coverage.c.test == test))

    def set_test_coverage(self, test, filename, linenos):
        if filename not in self._coverage:
            self._coverage[filename] = file_cover = {}
        else:
            file_cover = self._coverage[filename]

        # add new data
        for lineno in linenos:
            file_cover.setdefault(lineno, set()).add(test)
            self.conn.execute(Coverage.insert().values(filename=filename, lineno=lineno, test=test))

class TestCoveragePlugin(Plugin):
    """
    Monitors tests to discover which tests cover which
    lines of code.

    We find the diff with the parent revision for diff-tests with::

        git diff origin/master

    If you run with the --discover flag, it will attempt to discovery
    any tests that are required to run to test the changes in your current
    branch, against those of origin/master.

    """
    score = 0

    def _get_name_from_test(self, test):
        test_method_name = test._testMethodName

        # We need to determine the *actual* test path (as thats what nose gives us in wantMethod)
        # for example, maybe a test was imported in foo.bar.tests, but originated as foo.bar.something.MyTest
        # in this case, we'd need to identify that its *actually* foo.bar.something.MyTest to record the
        # proper coverage
        test_ = getattr(sys.modules[test.__module__], test.__class__.__name__)

        test_name = '%s:%s.%s' % (test_.__module__, test_.__name__,
                                                     test_method_name)

        return test_name

    def options(self, parser, env):
        parser.add_option('--coverage-parent',
                          dest="coverage_parent",
                          default="origin/master")

        parser.add_option("--record-test-coverage",
                          dest="record_test_coverage", action="store_true",
                          default=False)

        parser.add_option("--no-record-test-coverage",
                          dest="record_test_coverage", action="store_false",
                          default=False)

        parser.add_option("--skip-missing-coverage",
                          dest="skip_missing_coverage", action="store_true",
                          default=None)

        parser.add_option("--no-skip-missing-coverage",
                          dest="skip_missing_coverage", action="store_false",
                          default=None)

        parser.add_option("--no-coverage-discovery",
                          dest="discover", action="store_false",
                          default=True)

        parser.add_option("--coverage-dsn",
                          dest="coverage_dsn",
                          default='sqlite:///coverage.db')

    def configure(self, options, config):
        self.enabled = (options.record_test_coverage or options.discover)
        self.skip_missing = options.skip_missing_coverage
        self.record = options.record_test_coverage
        self.discover = options.discover
        self.logger = logging.getLogger(__name__)
        self.dsn = options.coverage_dsn
        self.parent = options.coverage_parent
        self.pending_funcs = {}

    def begin(self):
        # XXX: this is pretty hacky
        self.db = TestCoverageDB(self.dsn, self.logger)
        if self.record:
            self.db.upgrade()

        if not self.discover:
            return

        s = time.time()
        self.logger.info("Parsing diff from parent %s", self.parent)
        # pull in our diff
        # git diff `git merge-base HEAD master`
        proc = Popen(['git', 'diff', self.parent], stdout=PIPE, stderr=STDOUT)
        diff = proc.stdout.read()

        pending_funcs = self.pending_funcs

        parser = DiffParser(diff)
        files = list(parser.parse())
        self.logger.info("Parsed diff in %.2fs", time.time() - s)

        self.logger.info("Finding coverage for %d file(s)", len(files))
        s = time.time()
        for file in files:
            if file['is_header']:
                continue

            if file['old_filename'] == '/dev/null':
                continue

            filename = file['old_filename']
            if not filename.startswith('a/'):
                continue # ??

            filename = filename[2:]

            # Ignore non python files
            if not is_py_script(filename):
                continue

            # TODO: for this to be useful we need to eliminate tests
            if not self.db.has_test_coverage(filename):
                if self.skip_missing:
                    self.logger.warning('%s has no test coverage recorded', filename)
                    continue
                raise AssertionError("Missing test coverage for %s" % filename)

            for chunk in file['chunks']:
                linenos = [l['old_lineno'] for l in chunk]
                for test in self.db.get_test_coverage(filename, linenos):
                    pending_funcs[test] = None

        self.logger.info("Determined available coverage in %.2fs with %d test(s)", time.time() - s, len(pending_funcs))

    def wantMethod(self, method):
        if not self.discover:
            return

        # only works with unittest compatible functions currently
        test_name = '%s:%s.%s' % (method.im_class.__module__, method.im_class.__name__, method.__name__)

        # test has coverage for diff
        if test_name in self.pending_funcs:
            return True

        # test has no coverage recorded, defer to other plugins
        elif not self.db.has_seen_test(test_name):
            self.pending_funcs[test_name] = None
            self.logger.info("Allowing test due to missing coverage report: %s", test_name)
            return None

        return False

    def startTest(self, test):
        if not self.record:
            return

        self.coverage = coverage(include='disqus/*')
        self.coverage.start()

    def stopTest(self, test):
        if not self.record:
            return

        cov = self.coverage
        cov.stop()

        test_ = test.test
        test_name = self._get_name_from_test(test_)

        # this must have been imported under a different name
        if self.discover and test_name not in self.pending_funcs:
            self.logger.warning("Unable to determine origin for test: %s", test_name)
            return

        # initialize reporter
        rep = Reporter(cov)

        # process all files
        rep.find_code_units(None, cov.config)

        # The rest of this is all run within a single transaction
        trans = self.db.begin()

        self.db.clear_test_coverage(test_name)

        for cu in rep.code_units:
            if sys.modules[test_.__module__].__file__ == cu.filename:
                continue
            filename = cu.name + '.py'
            try:
                # TODO: this CANT work in all cases, must be a better way
                analysis = rep.coverage._analyze(cu)
                self.db.set_test_coverage(test_name, filename, analysis.statements)
            except KeyboardInterrupt:                       # pragma: no cover
                raise
            except:
                traceback.print_exc()

        trans.commit()

<commit_msg>Change pending_funcs back to a set (cant remember why I wanted to swap it to a dict :)<commit_after>from __future__ import absolute_import

import logging
import os
import sys
import time
import traceback

from coverage import coverage
from coverage.report import Reporter
from nose.plugins.base import Plugin
from nose_bleed.diff import DiffParser
from operator import or_
from subprocess import Popen, PIPE, STDOUT
from sqlalchemy import create_engine, Table, MetaData, Integer, String, \
  Column
from sqlalchemy.sql import select

def is_py_script(filename):
    "Returns True if a file is a python executable."
    if filename.endswith(".py") and os.path.exists(filename):
        return True
    elif not os.access(filename, os.X_OK):
        return False
    else:
        try:
            first_line = open(filename, "r").next().strip()
            return "#!" in first_line and "python" in first_line
        except StopIteration:
            return False

metadata = MetaData()
Coverage = Table('coverage', metadata,
    Column('id', Integer, primary_key=True),
    Column('filename', String),
    Column('lineno', Integer),
    Column('test', String, index=True)
)

class TestCoverageDB(object):
    def __init__(self, dsn, logger):
        self.logger = logger
        self.engine = create_engine(dsn)
        self.conn = self._connect_db()

        self._coverage = {}

    def _connect_db(self):
        self.logger.info('Connecting to coverage database..')
        s = time.time()
        conn = self.engine.connect()
        self.logger.info('Connection established to coverage database in %.2fs', time.time() - s)
        return conn

    def upgrade(self):
        metadata.create_all(self.conn)

    def begin(self):
        return self.conn.begin()

    def has_seen_test(self, test):
        statement = select([Coverage.c.id], Coverage.c.test == test, limit=1)
        result = bool(self.conn.execute(statement).fetchall())
        return result

    def has_test_coverage(self, filename):
        if filename not in self._coverage:
            self._coverage[filename] = file_cover = {}
            statement = select([Coverage.c.lineno, Coverage.c.test], Coverage.c.filename == filename)
            for lineno, test in self.conn.execute(statement).fetchall():
                file_cover.setdefault(lineno, set()).add(test)
            self.commit()
        return bool(self._coverage[filename])

    def get_test_coverage(self, filename, linenos):
        if filename not in self._coverage:
            self._coverage[filename] = file_cover = {}
            statement = select([Coverage.c.lineno, Coverage.c.test], Coverage.c.filename == filename, Coverage.c.lineno.in_(linenos))
            for lineno, test in self.conn.execute(statement).fetchall():
                file_cover.setdefault(lineno, set()).add(test)
        return reduce(or_, (self._coverage[filename].get(l, set()) for l in linenos))

    def clear_test_coverage(self, test):
        # clean up existing tests
        statement = select([Coverage.c.lineno, Coverage.c.test], Coverage.c.test==test)
        for filename, lineno in self.conn.execute(statement).fetchall():
            if filename not in self._coverage:
                continue
            if lineno not in self._coverage[filename]:
                continue
            self._coverage[filename][lineno].discard(test)

        self.conn.execute(Coverage.delete().where(Coverage.c.test == test))

    def set_test_coverage(self, test, filename, linenos):
        if filename not in self._coverage:
            self._coverage[filename] = file_cover = {}
        else:
            file_cover = self._coverage[filename]

        # add new data
        for lineno in linenos:
            file_cover.setdefault(lineno, set()).add(test)
            self.conn.execute(Coverage.insert().values(filename=filename, lineno=lineno, test=test))

class TestCoveragePlugin(Plugin):
    """
    Monitors tests to discover which tests cover which
    lines of code.

    We find the diff with the parent revision for diff-tests with::

        git diff origin/master

    If you run with the --discover flag, it will attempt to discovery
    any tests that are required to run to test the changes in your current
    branch, against those of origin/master.

    """
    score = 0

    def _get_name_from_test(self, test):
        test_method_name = test._testMethodName

        # We need to determine the *actual* test path (as thats what nose gives us in wantMethod)
        # for example, maybe a test was imported in foo.bar.tests, but originated as foo.bar.something.MyTest
        # in this case, we'd need to identify that its *actually* foo.bar.something.MyTest to record the
        # proper coverage
        test_ = getattr(sys.modules[test.__module__], test.__class__.__name__)

        test_name = '%s:%s.%s' % (test_.__module__, test_.__name__,
                                                     test_method_name)

        return test_name

    def options(self, parser, env):
        parser.add_option('--coverage-parent',
                          dest="coverage_parent",
                          default="origin/master")

        parser.add_option("--record-test-coverage",
                          dest="record_test_coverage", action="store_true",
                          default=False)

        parser.add_option("--no-record-test-coverage",
                          dest="record_test_coverage", action="store_false",
                          default=False)

        parser.add_option("--skip-missing-coverage",
                          dest="skip_missing_coverage", action="store_true",
                          default=None)

        parser.add_option("--no-skip-missing-coverage",
                          dest="skip_missing_coverage", action="store_false",
                          default=None)

        parser.add_option("--no-coverage-discovery",
                          dest="discover", action="store_false",
                          default=True)

        parser.add_option("--coverage-dsn",
                          dest="coverage_dsn",
                          default='sqlite:///coverage.db')

    def configure(self, options, config):
        self.enabled = (options.record_test_coverage or options.discover)
        self.skip_missing = options.skip_missing_coverage
        self.record = options.record_test_coverage
        self.discover = options.discover
        self.logger = logging.getLogger(__name__)
        self.dsn = options.coverage_dsn
        self.parent = options.coverage_parent
        self.pending_funcs = set()

    def begin(self):
        # XXX: this is pretty hacky
        self.db = TestCoverageDB(self.dsn, self.logger)
        if self.record:
            self.db.upgrade()

        if not self.discover:
            return

        s = time.time()
        self.logger.info("Parsing diff from parent %s", self.parent)
        # pull in our diff
        # git diff `git merge-base HEAD master`
        proc = Popen(['git', 'diff', self.parent], stdout=PIPE, stderr=STDOUT)
        diff = proc.stdout.read()

        pending_funcs = self.pending_funcs

        parser = DiffParser(diff)
        files = list(parser.parse())
        self.logger.info("Parsed diff in %.2fs", time.time() - s)

        self.logger.info("Finding coverage for %d file(s)", len(files))
        s = time.time()
        for file in files:
            if file['is_header']:
                continue

            if file['old_filename'] == '/dev/null':
                continue

            filename = file['old_filename']
            if not filename.startswith('a/'):
                continue # ??

            filename = filename[2:]

            # Ignore non python files
            if not is_py_script(filename):
                continue

            # TODO: for this to be useful we need to eliminate tests
            if not self.db.has_test_coverage(filename):
                if self.skip_missing:
                    self.logger.warning('%s has no test coverage recorded', filename)
                    continue
                raise AssertionError("Missing test coverage for %s" % filename)

            for chunk in file['chunks']:
                linenos = [l['old_lineno'] for l in chunk]
                for test in self.db.get_test_coverage(filename, linenos):
                    pending_funcs.add(test)

        self.logger.info("Determined available coverage in %.2fs with %d test(s)", time.time() - s, len(pending_funcs))

    def wantMethod(self, method):
        if not self.discover:
            return

        # only works with unittest compatible functions currently
        test_name = '%s:%s.%s' % (method.im_class.__module__, method.im_class.__name__, method.__name__)

        # test has coverage for diff
        if test_name in self.pending_funcs:
            return True

        # test has no coverage recorded, defer to other plugins
        elif not self.db.has_seen_test(test_name):
            self.pending_funcs.add(test_name)
            self.logger.info("Allowing test due to missing coverage report: %s", test_name)
            return None

        return False

    def startTest(self, test):
        if not self.record:
            return

        self.coverage = coverage(include='disqus/*')
        self.coverage.start()

    def stopTest(self, test):
        if not self.record:
            return

        cov = self.coverage
        cov.stop()

        test_ = test.test
        test_name = self._get_name_from_test(test_)

        # this must have been imported under a different name
        if self.discover and test_name not in self.pending_funcs:
            self.logger.warning("Unable to determine origin for test: %s", test_name)
            return

        # initialize reporter
        rep = Reporter(cov)

        # process all files
        rep.find_code_units(None, cov.config)

        # The rest of this is all run within a single transaction
        trans = self.db.begin()

        self.db.clear_test_coverage(test_name)

        for cu in rep.code_units:
            if sys.modules[test_.__module__].__file__ == cu.filename:
                continue
            filename = cu.name + '.py'
            try:
                # TODO: this CANT work in all cases, must be a better way
                analysis = rep.coverage._analyze(cu)
                self.db.set_test_coverage(test_name, filename, analysis.statements)
            except KeyboardInterrupt:                       # pragma: no cover
                raise
            except:
                traceback.print_exc()

        trans.commit()

<|endoftext|>


<commit_before>import logging
import os
import pandas as pd
import zipfile
import re
from collections import OrderedDict
import numpy as np
import argparse
import datetime
import json

from dataactcore.models.domainModels import DUNS
from dataactcore.interfaces.db import GlobalDB
from dataactcore.logging import configure_logging
from dataactvalidator.health_check import create_app
from dataactcore.config import CONFIG_BROKER
from dataactcore.utils.duns import get_client, REMOTE_SAM_EXEC_COMP_DIR
from dataactvalidator.scripts.loader_utils import insert_dataframe

logger = logging.getLogger(__name__)

def parse_exec_comp_file(filename, sess, root_dir, sftp=None, ssh_key=None, metrics=None):
    """ Parses the executive compensation file to update corresponding DUNS records

        Arguments:
            filename: name of file to import
            sess: database connection
            root_dir: working directory
            sftp: connection to remote server
            ssh_key: ssh_key for reconnecting
            metrics: dictionary representing metrics of the script
    """
    if not metrics:
        metrics = {
            'files_processed': [],
            'records_received': 0,
            'records_processed': 0,
            'updated_duns': []
        }

    file_path = os.path.join(root_dir, filename)
    if sftp:
        if sftp.sock.closed:
            # Reconnect if channel is closed
            ssh_client = get_client(ssh_key=ssh_key)
            sftp = ssh_client.open_sftp()
        with open(os.path.join(root_dir, filename), 'wb') as file:
            sftp.getfo(''.join([REMOTE_SAM_EXEC_COMP_DIR, '/', filename]), file)

    logger.info('starting file ' + file_path)
    metrics['files_processed'].append(filename)

    csv_file = os.path.splitext(filename)[0]+'.dat'
    zfile = zipfile.ZipFile(file_path)

    # can't use skipfooter, pandas' c engine doesn't work with skipfooter and the python engine doesn't work with dtype
    nrows = 0
    with zfile.open(csv_file) as zip_file:
        nrows = len(zip_file.readlines()) - 2  # subtract the header and footer
    column_header_mapping = {
        'awardee_or_recipient_uniqu': 0,
        'sam_extract': 4,
        'exec_comp_str': 89
    }
    column_header_mapping_ordered = OrderedDict(sorted(column_header_mapping.items(), key=lambda c: c[1]))
    with zfile.open(csv_file) as zip_file:
        csv_data = pd.read_csv(zip_file, dtype=str, header=None, skiprows=1, nrows=nrows, sep='|',
                               usecols=column_header_mapping_ordered.values(),
                               names=column_header_mapping_ordered.keys())
    total_data = csv_data.copy()
    metrics['records_received'] += len(total_data.index)
    total_data = total_data[total_data.awardee_or_recipient_uniqu.notnull() &
                            total_data.sam_extract.isin(['2', '3', 'A', 'E'])]
    metrics['records_processed'] += len(total_data.index)
    del total_data['sam_extract']
    # Note: we're splitting these up cause it vastly saves memory parsing only the records that are populated
    blank_exec = total_data[total_data.exec_comp_str.isnull()]
    pop_exec = total_data[total_data.exec_comp_str.notnull()]

    # parse out executive compensation from row 90 for populated records
    lambda_func = (lambda ecs: pd.Series(list(parse_exec_comp(ecs).values())))
    parsed_data = pop_exec['exec_comp_str'].apply(lambda_func)
    parsed_data.columns = list(parse_exec_comp().keys())
    del pop_exec['exec_comp_str']
    pop_exec = pop_exec.join(parsed_data)

    # leave blanks
    del blank_exec['exec_comp_str']
    blank_exec = blank_exec.assign(**parse_exec_comp())

    # setup the final dataframe
    total_data = pd.concat([pop_exec, blank_exec])
    total_data.replace('', np.nan, inplace=True)
    last_exec_comp_mod_date_str = re.findall('[0-9]{8}', filename)
    if not last_exec_comp_mod_date_str:
        raise Exception('Last Executive Compensation Mod Date not found in filename.')
    last_exec_comp_mod_date = datetime.datetime.strptime(last_exec_comp_mod_date_str[0], '%Y%m%d').date()
    total_data = total_data.assign(last_exec_comp_mod_date=last_exec_comp_mod_date)

    updated_duns = update_exec_comp_duns(sess, total_data)
    metrics['updated_duns'].extend(updated_duns)

    if sftp:
        os.remove(os.path.join(root_dir, filename))

def update_exec_comp_duns(sess, exec_comp_data):
    """ Takes in a dataframe of exec comp data and updates associated DUNS

        Arguments:
            sess: database connection
            exec_comp_data: pandas dataframe representing exec comp data

        Returns:
            list of DUNS updated
    """

    logger.info('Making temp_exec_comp_update table')
    create_table_sql = """
        CREATE TABLE IF NOT EXISTS temp_exec_comp_update (
            awardee_or_recipient_uniqu TEXT,
            high_comp_officer1_amount TEXT,
            high_comp_officer1_full_na TEXT,
            high_comp_officer2_amount TEXT,
            high_comp_officer2_full_na TEXT,
            high_comp_officer3_amount TEXT,
            high_comp_officer3_full_na TEXT,
            high_comp_officer4_amount TEXT,
            high_comp_officer4_full_na TEXT,
            high_comp_officer5_amount TEXT,
            high_comp_officer5_full_na TEXT,
            last_exec_comp_mod_date DATE
        );
    """
    sess.execute(create_table_sql)
    # Truncating in case we didn't clear out this table after a failure in the script
    sess.execute('TRUNCATE TABLE temp_exec_comp_update;')
    insert_dataframe(exec_comp_data, 'temp_exec_comp_update', sess.connection())

    # Note: this can work just by getting the row count from the following SQL
    #       but this can run multiple times on possibly the same DUNS over several days,
    #       so it'll be more accurate to keep track of which DUNS get updated
    logger.info('Getting list of DUNS that will be updated for metrics')
    update_sql = """
        SELECT duns.awardee_or_recipient_uniqu
        FROM duns
        JOIN temp_exec_comp_update AS tecu ON duns.awardee_or_recipient_uniqu=tecu.awardee_or_recipient_uniqu;
    """
    duns_list = [row['awardee_or_recipient_uniqu'] for row in sess.execute(update_sql).fetchall()]

    logger.info('Updating DUNS based on temp_exec_comp_update')
    update_sql = """
        UPDATE duns
        SET
            high_comp_officer1_amount = tecu.high_comp_officer1_amount,
            high_comp_officer1_full_na = tecu.high_comp_officer1_full_na,
            high_comp_officer2_amount = tecu.high_comp_officer2_amount,
            high_comp_officer2_full_na = tecu.high_comp_officer2_full_na,
            high_comp_officer3_amount = tecu.high_comp_officer3_amount,
            high_comp_officer3_full_na = tecu.high_comp_officer3_full_na,
            high_comp_officer4_amount = tecu.high_comp_officer4_amount,
            high_comp_officer4_full_na = tecu.high_comp_officer4_full_na,
            high_comp_officer5_amount = tecu.high_comp_officer5_amount,
            high_comp_officer5_full_na = tecu.high_comp_officer5_full_na,
            last_exec_comp_mod_date = tecu.last_exec_comp_mod_date
        FROM temp_exec_comp_update AS tecu
        WHERE duns.awardee_or_recipient_uniqu=tecu.awardee_or_recipient_uniqu;
    """
    sess.execute(update_sql)

    logger.info('Dropping temp_exec_comp_update')
    sess.execute('DROP TABLE temp_exec_comp_update;')

    sess.commit()
    return duns_list

def parse_exec_comp(exec_comp_str=None):
    """ Parses the executive compensation string into a dictionary for the ExecutiveCompensation data model

        Arguments:
            exec_comp_str: the incoming compensation string

        Returns:
            dictionary for the ExecutiveCompensation data model
    """
    exec_comp_data = OrderedDict()
    for index in range(1, 6):
        exec_comp_data['high_comp_officer{}_full_na'.format(index)] = np.nan
        exec_comp_data['high_comp_officer{}_amount'.format(index)] = np.nan

    if isinstance(exec_comp_str, str) and not exec_comp_str.isdigit():
        high_comp_officers = exec_comp_str.split('~')

        # records have inconsistent values for Null
        # 'see note above' is excluded as it may contain relevant info
        unaccepted_titles = ['x', 'n/a', 'na', 'null', 'none', '. . .', 'no one', 'no  one']

        for index, high_comp_officer in enumerate(high_comp_officers):
            index += 1
            exec_comp_split = high_comp_officer.split('^')
            if len(exec_comp_split) != 3:
                continue
            exec_name = exec_comp_split[0]
            exec_title = exec_comp_split[1]
            exec_comp = exec_comp_split[2]
            if exec_title.lower() not in unaccepted_titles and exec_name.lower() not in unaccepted_titles:
                exec_comp_data['high_comp_officer{}_full_na'.format(index)] = exec_name
                exec_comp_data['high_comp_officer{}_amount'.format(index)] = exec_comp

    return exec_comp_data

def get_parser():
    """ Generates list of command-line arguments

        Returns:
            argument parser to be used for commandline
    """
    parser = argparse.ArgumentParser(description='Get data from SAM and update execution_compensation table')
    scope = parser.add_mutually_exclusive_group(required=True)
    scope.add_argument('--historic', '-i', action='store_true', help='reload from the first monthly file on')
    scope.add_argument('--update', '-u', action='store_true', help='load only the latest daily file')
    environ = parser.add_mutually_exclusive_group(required=True)
    environ.add_argument('--local', '-l', type=str, default=None, help='local directory to work from')
    environ.add_argument('--ssh_key', '-k', type=str, default=None, help='private key used to access the API remotely')
    return parser

if __name__ == '__main__':
    now = datetime.datetime.now()

    configure_logging()
    parser = get_parser()
    args = parser.parse_args()

    historic = args.historic
    update = args.update
    local = args.local
    ssh_key = args.ssh_key

    metrics = {
        'script_name': 'load_exec_duns.py',
        'start_time': str(now),
        'files_processed': [],
        'records_received': 0,
        'records_processed': 0,
        'updated_duns': [],
        'records_updated': 0
    }

    with create_app().app_context():
        sess = GlobalDB.db().session
        sftp = None

        if ssh_key:
            root_dir = CONFIG_BROKER['d_file_storage_path']
            client = get_client(ssh_key=ssh_key)
            sftp = client.open_sftp()
            # dirlist on remote host
            dirlist = sftp.listdir(REMOTE_SAM_EXEC_COMP_DIR)
        elif local:
            root_dir = local
            dirlist = os.listdir(local)

        # generate chronological list of daily files
        sorted_monthly_file_names = sorted([monthly_file for monthly_file in dirlist if re.match('.*MONTHLY_\d+',
                                                                                monthly_file)])
        sorted_daily_file_names = sorted([daily_file for daily_file in dirlist if re.match('.*DAILY_\d+', daily_file)])

        if historic:
            parse_exec_comp_file(sorted_monthly_file_names[0], sess, root_dir, sftp=sftp, ssh_key=ssh_key,
                                 metrics=metrics)
            for daily_file in sorted_daily_file_names:
                parse_exec_comp_file(daily_file, sess, root_dir, sftp=sftp, ssh_key=ssh_key, metrics=metrics)
        elif update:
            # Insert item into sorted file list with date of last exec comp load date
            last_update = sess.query(DUNS.last_exec_comp_mod_date). \
                order_by(DUNS.last_exec_comp_mod_date.desc()). \
                filter(DUNS.last_exec_comp_mod_date.isnot(None)). \
                first()
            if not last_update:
                raise Exception('No last executive compenstation mod date found in database. '
                                'Please run historic loader first.')
            else:
                last_update = last_update[0].strftime("%Y%m%d")
            earliest_daily_file = re.sub("_DAILY_[0-9]{8}\.ZIP", "_DAILY_" +
                                         last_update + ".ZIP", sorted_daily_file_names[0])
            if earliest_daily_file:
                sorted_full_list = sorted(sorted_daily_file_names + [earliest_daily_file])
                daily_files_after = sorted_full_list[sorted_full_list.index(earliest_daily_file) + 1:]
            else:
                daily_files_after = sorted_daily_file_names

            if daily_files_after:
                for daily_file in daily_files_after:
                    parse_exec_comp_file(daily_file, sess, root_dir, sftp=sftp, ssh_key=ssh_key, metrics=metrics)
            else:
                logger.info("No daily file found.")

    metrics['records_updated'] = len(set(metrics['updated_duns']))
    del metrics['updated_duns']

    metrics['duration'] = str(datetime.datetime.now() - now)
    with open('load_exec_comp_metrics.json', 'w+') as metrics_file:
        json.dump(metrics, metrics_file)
<commit_msg>flake8<commit_after>import logging
import os
import pandas as pd
import zipfile
import re
from collections import OrderedDict
import numpy as np
import argparse
import datetime
import json

from dataactcore.models.domainModels import DUNS
from dataactcore.interfaces.db import GlobalDB
from dataactcore.logging import configure_logging
from dataactvalidator.health_check import create_app
from dataactcore.config import CONFIG_BROKER
from dataactcore.utils.duns import get_client, REMOTE_SAM_EXEC_COMP_DIR
from dataactvalidator.scripts.loader_utils import insert_dataframe

logger = logging.getLogger(__name__)

def parse_exec_comp_file(filename, sess, root_dir, sftp=None, ssh_key=None, metrics=None):
    """ Parses the executive compensation file to update corresponding DUNS records

        Arguments:
            filename: name of file to import
            sess: database connection
            root_dir: working directory
            sftp: connection to remote server
            ssh_key: ssh_key for reconnecting
            metrics: dictionary representing metrics of the script
    """
    if not metrics:
        metrics = {
            'files_processed': [],
            'records_received': 0,
            'records_processed': 0,
            'updated_duns': []
        }

    file_path = os.path.join(root_dir, filename)
    if sftp:
        if sftp.sock.closed:
            # Reconnect if channel is closed
            ssh_client = get_client(ssh_key=ssh_key)
            sftp = ssh_client.open_sftp()
        with open(os.path.join(root_dir, filename), 'wb') as file:
            sftp.getfo(''.join([REMOTE_SAM_EXEC_COMP_DIR, '/', filename]), file)

    logger.info('starting file ' + file_path)
    metrics['files_processed'].append(filename)

    csv_file = os.path.splitext(filename)[0]+'.dat'
    zfile = zipfile.ZipFile(file_path)

    # can't use skipfooter, pandas' c engine doesn't work with skipfooter and the python engine doesn't work with dtype
    nrows = 0
    with zfile.open(csv_file) as zip_file:
        nrows = len(zip_file.readlines()) - 2  # subtract the header and footer
    column_header_mapping = {
        'awardee_or_recipient_uniqu': 0,
        'sam_extract': 4,
        'exec_comp_str': 89
    }
    column_header_mapping_ordered = OrderedDict(sorted(column_header_mapping.items(), key=lambda c: c[1]))
    with zfile.open(csv_file) as zip_file:
        csv_data = pd.read_csv(zip_file, dtype=str, header=None, skiprows=1, nrows=nrows, sep='|',
                               usecols=column_header_mapping_ordered.values(),
                               names=column_header_mapping_ordered.keys())
    total_data = csv_data.copy()
    metrics['records_received'] += len(total_data.index)
    total_data = total_data[total_data.awardee_or_recipient_uniqu.notnull() &
                            total_data.sam_extract.isin(['2', '3', 'A', 'E'])]
    metrics['records_processed'] += len(total_data.index)
    del total_data['sam_extract']
    # Note: we're splitting these up cause it vastly saves memory parsing only the records that are populated
    blank_exec = total_data[total_data.exec_comp_str.isnull()]
    pop_exec = total_data[total_data.exec_comp_str.notnull()]

    # parse out executive compensation from row 90 for populated records
    lambda_func = (lambda ecs: pd.Series(list(parse_exec_comp(ecs).values())))
    parsed_data = pop_exec['exec_comp_str'].apply(lambda_func)
    parsed_data.columns = list(parse_exec_comp().keys())
    del pop_exec['exec_comp_str']
    pop_exec = pop_exec.join(parsed_data)

    # leave blanks
    del blank_exec['exec_comp_str']
    blank_exec = blank_exec.assign(**parse_exec_comp())

    # setup the final dataframe
    total_data = pd.concat([pop_exec, blank_exec])
    total_data.replace('', np.nan, inplace=True)
    last_exec_comp_mod_date_str = re.findall('[0-9]{8}', filename)
    if not last_exec_comp_mod_date_str:
        raise Exception('Last Executive Compensation Mod Date not found in filename.')
    last_exec_comp_mod_date = datetime.datetime.strptime(last_exec_comp_mod_date_str[0], '%Y%m%d').date()
    total_data = total_data.assign(last_exec_comp_mod_date=last_exec_comp_mod_date)

    updated_duns = update_exec_comp_duns(sess, total_data)
    metrics['updated_duns'].extend(updated_duns)

    if sftp:
        os.remove(os.path.join(root_dir, filename))

def update_exec_comp_duns(sess, exec_comp_data):
    """ Takes in a dataframe of exec comp data and updates associated DUNS

        Arguments:
            sess: database connection
            exec_comp_data: pandas dataframe representing exec comp data

        Returns:
            list of DUNS updated
    """

    logger.info('Making temp_exec_comp_update table')
    create_table_sql = """
        CREATE TABLE IF NOT EXISTS temp_exec_comp_update (
            awardee_or_recipient_uniqu TEXT,
            high_comp_officer1_amount TEXT,
            high_comp_officer1_full_na TEXT,
            high_comp_officer2_amount TEXT,
            high_comp_officer2_full_na TEXT,
            high_comp_officer3_amount TEXT,
            high_comp_officer3_full_na TEXT,
            high_comp_officer4_amount TEXT,
            high_comp_officer4_full_na TEXT,
            high_comp_officer5_amount TEXT,
            high_comp_officer5_full_na TEXT,
            last_exec_comp_mod_date DATE
        );
    """
    sess.execute(create_table_sql)
    # Truncating in case we didn't clear out this table after a failure in the script
    sess.execute('TRUNCATE TABLE temp_exec_comp_update;')
    insert_dataframe(exec_comp_data, 'temp_exec_comp_update', sess.connection())

    # Note: this can work just by getting the row count from the following SQL
    #       but this can run multiple times on possibly the same DUNS over several days,
    #       so it'll be more accurate to keep track of which DUNS get updated
    logger.info('Getting list of DUNS that will be updated for metrics')
    update_sql = """
        SELECT duns.awardee_or_recipient_uniqu
        FROM duns
        JOIN temp_exec_comp_update AS tecu ON duns.awardee_or_recipient_uniqu=tecu.awardee_or_recipient_uniqu;
    """
    duns_list = [row['awardee_or_recipient_uniqu'] for row in sess.execute(update_sql).fetchall()]

    logger.info('Updating DUNS based on temp_exec_comp_update')
    update_sql = """
        UPDATE duns
        SET
            high_comp_officer1_amount = tecu.high_comp_officer1_amount,
            high_comp_officer1_full_na = tecu.high_comp_officer1_full_na,
            high_comp_officer2_amount = tecu.high_comp_officer2_amount,
            high_comp_officer2_full_na = tecu.high_comp_officer2_full_na,
            high_comp_officer3_amount = tecu.high_comp_officer3_amount,
            high_comp_officer3_full_na = tecu.high_comp_officer3_full_na,
            high_comp_officer4_amount = tecu.high_comp_officer4_amount,
            high_comp_officer4_full_na = tecu.high_comp_officer4_full_na,
            high_comp_officer5_amount = tecu.high_comp_officer5_amount,
            high_comp_officer5_full_na = tecu.high_comp_officer5_full_na,
            last_exec_comp_mod_date = tecu.last_exec_comp_mod_date
        FROM temp_exec_comp_update AS tecu
        WHERE duns.awardee_or_recipient_uniqu=tecu.awardee_or_recipient_uniqu;
    """
    sess.execute(update_sql)

    logger.info('Dropping temp_exec_comp_update')
    sess.execute('DROP TABLE temp_exec_comp_update;')

    sess.commit()
    return duns_list

def parse_exec_comp(exec_comp_str=None):
    """ Parses the executive compensation string into a dictionary for the ExecutiveCompensation data model

        Arguments:
            exec_comp_str: the incoming compensation string

        Returns:
            dictionary for the ExecutiveCompensation data model
    """
    exec_comp_data = OrderedDict()
    for index in range(1, 6):
        exec_comp_data['high_comp_officer{}_full_na'.format(index)] = np.nan
        exec_comp_data['high_comp_officer{}_amount'.format(index)] = np.nan

    if isinstance(exec_comp_str, str) and not exec_comp_str.isdigit():
        high_comp_officers = exec_comp_str.split('~')

        # records have inconsistent values for Null
        # 'see note above' is excluded as it may contain relevant info
        unaccepted_titles = ['x', 'n/a', 'na', 'null', 'none', '. . .', 'no one', 'no  one']

        for index, high_comp_officer in enumerate(high_comp_officers):
            index += 1
            exec_comp_split = high_comp_officer.split('^')
            if len(exec_comp_split) != 3:
                continue
            exec_name = exec_comp_split[0]
            exec_title = exec_comp_split[1]
            exec_comp = exec_comp_split[2]
            if exec_title.lower() not in unaccepted_titles and exec_name.lower() not in unaccepted_titles:
                exec_comp_data['high_comp_officer{}_full_na'.format(index)] = exec_name
                exec_comp_data['high_comp_officer{}_amount'.format(index)] = exec_comp

    return exec_comp_data

def get_parser():
    """ Generates list of command-line arguments

        Returns:
            argument parser to be used for commandline
    """
    parser = argparse.ArgumentParser(description='Get data from SAM and update execution_compensation table')
    scope = parser.add_mutually_exclusive_group(required=True)
    scope.add_argument('--historic', '-i', action='store_true', help='reload from the first monthly file on')
    scope.add_argument('--update', '-u', action='store_true', help='load only the latest daily file')
    environ = parser.add_mutually_exclusive_group(required=True)
    environ.add_argument('--local', '-l', type=str, default=None, help='local directory to work from')
    environ.add_argument('--ssh_key', '-k', type=str, default=None, help='private key used to access the API remotely')
    return parser

if __name__ == '__main__':
    now = datetime.datetime.now()

    configure_logging()
    parser = get_parser()
    args = parser.parse_args()

    historic = args.historic
    update = args.update
    local = args.local
    ssh_key = args.ssh_key

    metrics = {
        'script_name': 'load_exec_duns.py',
        'start_time': str(now),
        'files_processed': [],
        'records_received': 0,
        'records_processed': 0,
        'updated_duns': [],
        'records_updated': 0
    }

    with create_app().app_context():
        sess = GlobalDB.db().session
        sftp = None

        if ssh_key:
            root_dir = CONFIG_BROKER['d_file_storage_path']
            client = get_client(ssh_key=ssh_key)
            sftp = client.open_sftp()
            # dirlist on remote host
            dirlist = sftp.listdir(REMOTE_SAM_EXEC_COMP_DIR)
        elif local:
            root_dir = local
            dirlist = os.listdir(local)

        # generate chronological list of daily files
        sorted_monthly_file_names = sorted([monthly_file for monthly_file in dirlist if re.match('.*MONTHLY_\d+',
                                                                                monthly_file)])
        sorted_daily_file_names = sorted([daily_file for daily_file in dirlist if re.match('.*DAILY_\d+', daily_file)])

        if historic:
            parse_exec_comp_file(sorted_monthly_file_names[0], sess, root_dir, sftp=sftp, ssh_key=ssh_key,
                                 metrics=metrics)
            for daily_file in sorted_daily_file_names:
                parse_exec_comp_file(daily_file, sess, root_dir, sftp=sftp, ssh_key=ssh_key, metrics=metrics)
        elif update:
            # Insert item into sorted file list with date of last exec comp load date
            last_update = sess.query(DUNS.last_exec_comp_mod_date). \
                order_by(DUNS.last_exec_comp_mod_date.desc()). \
                filter(DUNS.last_exec_comp_mod_date.isnot(None)). \
                first()
            if not last_update:
                raise Exception('No last executive compenstation mod date found in database. '
                                'Please run historic loader first.')
            else:
                last_update = last_update[0].strftime("%Y%m%d")
            earliest_daily_file = re.sub("_DAILY_[0-9]{8}\.ZIP", "_DAILY_" +
                                         last_update + ".ZIP", sorted_daily_file_names[0])
            if earliest_daily_file:
                sorted_full_list = sorted(sorted_daily_file_names + [earliest_daily_file])
                daily_files_after = sorted_full_list[sorted_full_list.index(earliest_daily_file) + 1:]
            else:
                daily_files_after = sorted_daily_file_names

            if daily_files_after:
                for daily_file in daily_files_after:
                    parse_exec_comp_file(daily_file, sess, root_dir, sftp=sftp, ssh_key=ssh_key, metrics=metrics)
            else:
                logger.info("No daily file found.")

    metrics['records_updated'] = len(set(metrics['updated_duns']))
    del metrics['updated_duns']

    metrics['duration'] = str(datetime.datetime.now() - now)
    with open('load_exec_comp_metrics.json', 'w+') as metrics_file:
        json.dump(metrics, metrics_file)
<|endoftext|>


<commit_before>"""
spaceshooter.py
Author: Daniel Wilson
Credit:

Assignment:
Write and submit a program that implements the spacewar game:
https://github.com/HHS-IntroProgramming/Spacewar
"""
from ggame import App, RectangleAsset, ImageAsset, Sprite, LineStyle, Color, Frame
import math
SCREEN_WIDTH = 1640
SCREEN_HEIGHT = 700

class SpaceShip(Sprite):
    """
    Animated space ship
    """
    asset = ImageAsset("images/four_spaceship_by_albertov_with_thrust.png",
        Frame(227,0,292-227,125), 4, 'vertical')

    def __init__(self, position):
        super().__init__(SpaceShip.asset, position)
        self.vx = 0
        self.vy = 0
        self.vr = 0.0
        self.thrust = 0
        self.thrustframe = 1
        SpaceGame.listenKeyEvent("keydown", "w", self.thrustOn)
        SpaceGame.listenKeyEvent("keyup", "w", self.thrustOff)
        SpaceGame.listenKeyEvent("keydown", "s", self.UPON)
        SpaceGame.listenKeyEvent("keyup", "s", self.stop)
        SpaceGame.listenKeyEvent("keydown", "w", self.DOWNON)
        SpaceGame.listenKeyEvent("keyup", "w", self.stop)
        SpaceGame.listenKeyEvent("keydown", "a", self.Left)
        SpaceGame.listenKeyEvent("keyup", "a", self.stop)
        SpaceGame.listenKeyEvent("keydown", "d", self.Right)
        SpaceGame.listenKeyEvent("keyup", "d", self.stop)
        self.fxcenter = self.fycenter = 0.5

    def step(self):
        self.x += self.vx
        self.y += self.vy
        self.rotation += self.vr
        if self.thrust == 1:
            self.setImage(self.thrustframe)
            self.thrustframe += 1
            if self.thrustframe == 4:
                self.thrustframe = 1
        else:
            self.setImage(0)

    def thrustOn(self, event):
        self.thrust = 1

    def thrustOff(self, event):
        self.thrust = 0

    def UPON(self, event):
        self.vy=1
    def DOWNON(self, event):
        self.vy=-1
    def Right(self, event):
        self.vx = 1
    def Left(self, event):
        self.vx = -1
    def stop(self, event):
        self.vy=0
        self.vx=0

class SpaceGame(App):
    """
    Tutorial4 space game example.
    """
    def __init__(self, width, height):
        super().__init__(width, height)
        black = Color(0, 1)
        noline = LineStyle(0, black)
        bg_asset = ImageAsset("images/starfield.jpg")
        bg1 = Sprite(bg_asset, (0,0))
        bg2 = Sprite(bg_asset, (512,0))
        bg3 = Sprite(bg_asset, (0, 512))
        bg4 = Sprite(bg_asset, (512, 512))
        SpaceShip((100,100))
        SpaceShip((500,100))

    def step(self):
        step.x += self.vx
        step.y += self.vy
        step.rotation += self.vr

    def step(self):
        for ship in self.getSpritesbyClass(SpaceShip):
            ship.step()

def registerKeys(self, keys):
        commands = ["left", "right", "forward", "fire"]
        self.keymap = dict(zip(keys, commands))
        [self.app.listenKeyEvent("keydown", k, self.controldown) for k in keys]
        [self.app.listenKeyEvent("keyup", k, self.controlup) for k in keys]

myapp = SpaceGame(SCREEN_WIDTH, SCREEN_HEIGHT)
myapp.run()<commit_msg>Updated from Brython Server: 3/22/2016 2:27:31 PM<commit_after>"""
spaceshooter.py
Author: Daniel Wilson
Credit:

Assignment:
Write and submit a program that implements the spacewar game:
https://github.com/HHS-IntroProgramming/Spacewar
"""
from ggame import App, RectangleAsset, ImageAsset, Sprite, LineStyle, Color, Frame
import math
SCREEN_WIDTH = 1840
SCREEN_HEIGHT = 700

class SpaceShip(Sprite):
    """
    Animated space ship
    """
    asset = ImageAsset("images/four_spaceship_by_albertov_with_thrust.png",
        Frame(227,0,292-227,125), 4, 'vertical')

    def __init__(self, position):
        super().__init__(SpaceShip.asset, position)
        self.vx = 0
        self.vy = 0
        self.vr = 0.0
        self.thrust = 0
        self.thrustframe = 1
        SpaceGame.listenKeyEvent("keydown", "w", self.thrustOn)
        SpaceGame.listenKeyEvent("keyup", "w", self.thrustOff)
        SpaceGame.listenKeyEvent("keydown", "s", self.UPON)
        SpaceGame.listenKeyEvent("keyup", "s", self.stop)
        SpaceGame.listenKeyEvent("keydown", "w", self.DOWNON)
        SpaceGame.listenKeyEvent("keyup", "w", self.stop)
        SpaceGame.listenKeyEvent("keydown", "a", self.Left)
        SpaceGame.listenKeyEvent("keyup", "a", self.stop)
        SpaceGame.listenKeyEvent("keydown", "d", self.Right)
        SpaceGame.listenKeyEvent("keyup", "d", self.stop)
        self.fxcenter = self.fycenter = 0.5

    def step(self):
        self.x += self.vx
        self.y += self.vy
        self.rotation += self.vr
        if self.thrust == 1:
            self.setImage(self.thrustframe)
            self.thrustframe += 1
            if self.thrustframe == 4:
                self.thrustframe = 1
        else:
            self.setImage(0)

    def thrustOn(self, event):
        self.thrust = 1

    def thrustOff(self, event):
        self.thrust = 0

    def UPON(self, event):
        self.vy=1
    def DOWNON(self, event):
        self.vy=-1
    def Right(self, event):
        self.vx = 1
    def Left(self, event):
        self.vx = -1
    def stop(self, event):
        self.vy=0
        self.vx=0

class SpaceGame(App):
    """
    Tutorial4 space game example.
    """
    def __init__(self, width, height):
        super().__init__(width, height)
        black = Color(0, 1)
        noline = LineStyle(0, black)
        bg_asset = ImageAsset("images/starfield.jpg")
        bg1 = Sprite(bg_asset, (0,0))
        bg2 = Sprite(bg_asset, (512,0))
        bg3 = Sprite(bg_asset, (0, 512))
        bg4 = Sprite(bg_asset, (512, 512))
        SpaceShip((100,100))
        SpaceShip((500,100))

    def step(self):
        step.x += self.vx
        step.y += self.vy
        step.rotation += self.vr

    def step(self):
        for ship in self.getSpritesbyClass(SpaceShip):
            ship.step()

def registerKeys(self, keys):
        commands = ["left", "right", "forward", "fire"]
        self.keymap = dict(zip(keys, commands))
        [self.app.listenKeyEvent("keydown", k, self.controldown) for k in keys]
        [self.app.listenKeyEvent("keyup", k, self.controlup) for k in keys]

myapp = SpaceGame(SCREEN_WIDTH, SCREEN_HEIGHT)
myapp.run()<|endoftext|>


<commit_before>"""

hellofriend.py
Author: Avery Wallis
Credit: None

"""

a = input('Please tell me your name:')
b = input('Please tell me your age:')
b = float(b)
d = (25-b)
e = "Hello, "
f = ". I am "
g = " years older than you!"
h = " years younger than you!"
i = "as old as you are!"

if d > 0:
    print(e + a + f + str(d) + g)

if d < 0:
    d = abs(d)
    print(e + a + f + str(d) + h)

if d == 0:
    print(e + a + f + i)
<commit_msg>Updated from Brython Server: 1/22/2016 2:48:43 PM<commit_after>"""

hellofriend.py
Author: Avery Wallis
Credit: None

"""

a = input('Please tell me your name:')
b = input('Please tell me your age:')
b = int(b)
d = (25-b)
e = "Hello, "
f = ". I am "
g = " years older than you!"
h = " years younger than you!"
i = "as old as you are!"

if d > 0:
    print(e + a + f + str(d) + g)

if d < 0:
    d = abs(d)
    print(e + a + f + str(d) + h)

if d == 0:
    print(e + a + f + i)
<|endoftext|>