<commit_before>from optparse import OptionParser
from pyspark import SparkContext, StorageLevel
from dictionaries import D
from geoname_extractor import processDoc
import ProbabilisticER
import json
import codecs

"""
RUN AS:
spark-submit --master local[*]    --executor-memory=8g     --driver-memory=8g \
--py-files lib/python-lib.zip main.py  /tmp/geonames/input/input.jl /tmp/geonames/geo-out \
/tmp/geonames/output/prior_dict.json 3 /tmp/geonames/output/state_dict.json /tmp/geonames/output/all_city_dict.json \
/tmp/geonames/output/city_faerie.json /tmp/geonames/output/state_faerie.json /tmp/geonames/output/all_dict_faerie.json \
 /tmp/geonames/output/tagging_dict.json /tmp/geonames/output/config.json
"""

us_states_names_readable = [
    "Alabama", "Alaska", "Arizona", "Arkansas", "California", "Colorado", "Connecticut", "Delaware", "Florida",
    "Georgia", "Hawaii", "Idaho", "Illinois", "Indiana", "Iowa", "Kansas", "Kentucky", "Louisiana", "Maine", "Maryland",
    "Massachusetts", "Michigan", "Minnesota", "Mississippi", "Missouri", "Montana", "Nebraska", "Nevada",
    "New Hampshire", "New Jersey", "New Mexico", "New York", "North Carolina", "North Dakota", "Ohio", "Oklahoma",
    "Oregon", "Pennsylvania", "Rhode Island", "South Carolina", "South Dakota", "Tennessee", "Texas", "Utah", "Vermont",
    "Virginia", "Washington", "West Virginia", "Wisconsin", "Wyoming"
]

us_states_codes = [
    "AL", "AK", "AZ", "AR", "CA", "CO", "CT", "DE", "FL", "GA", "HI", "ID", "IL", "IN", "IA", "KS", "KY", "LA", "ME",
    "MD", "MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV", "NH", "NJ", "NM", "NY", "NC", "ND", "OH", "OK", "OR", "PA",
    "RI", "SC", "SD", "TN", "TX", "UT", "VT", "VA", "WA", "WV", "WI", "WY"
]

# Given a path in json, return value if path, full path denoted by a separator,like '$'or '.',
#  (example address.name) exists, otherwise return ''
def get_value_json(path, doc, separator='.'):
    paths = path.strip().split(separator)
    for field in paths:
        if field in doc:
            doc = doc[field]
        else:
            return ''

    if type(doc) == dict or type(doc) == list:
        return json.dumps(doc)
    else:
        return doc

def get_only_city_name(x):
        values = x.split(" ")
        if len(values) > 1 and len(values[len(values) - 1]) == 2 and values[len(values) - 1].upper() != 'DC':
            return " ".join(values[0:len(values) - 1])
        return x

def get_state_from_city(x):
    values = x.split(" ")
    if len(values) > 1 and len(values[len(values) - 1]) == 2 and values[len(values) - 1].upper() != 'DC':
        short_state = values[len(values) - 1].upper()
        try:
            idx = us_states_codes.index(short_state)
            return us_states_names_readable[idx]
        except ValueError:
            return ''
    return ''

def create_input_geonames(line):
    result = []

    # line = json.loads(line)

    if line:
        json_x = line

        json_l = []
        if isinstance(json_x, dict):
            json_l.append(json_x)
        elif isinstance(json_x, list):
            json_l = json_x

        for x in json_l:
            # print x
            out = dict()
            out['country'] = get_value_json('addressCountry', x)

            city = get_value_json('addressLocality', x)
            out['locality'] = get_only_city_name(city)

            region = get_value_json('addressRegion', x)
            # out['region'] = region
            state = get_state_from_city(city)

            if region.strip() == '':
                if state.strip() != '':
                    out['region'] = state
                else:
                    out['region'] = ''
            else:
                out['region'] = region

            if out['country'].strip() == '' and out['region'].strip() == '' and out['locality'].strip() == '':
                print line['uri']

            out['uri'] = line['uri']

        result.append(out)
    return result

def getAddressName(x):
    name = ''
    city = get_value_json('addressLocality', x)
    state = get_value_json('addressRegion', x)
    country = get_value_json('addressCountry', x)

    if city != '':
        name += city + ','

    if state != '':
        name += state + ','

    if country != '':
        name += country
    if name != '':
        if name[len(name)-1] == ',':
            name = name[:len(name)-1]
    return name

def create_address_object(geo, x, d):
    address = {}
    try:
        extractor_output = getAddressName(x)
        key = ''
        top_match = geo['matches'][0]
        address['addressLocality'] = top_match['value']['city']
        address['addressRegion'] = top_match['value']['state']
        address['addressCountry'] = top_match['value']['country']
        address['uri'] = x['uri']
        address['createdBy'] = 'dig-geonames'
        address['a'] = "http://schema.org/PostalAddress"
        address['@context'] = "https://raw.githubusercontent.com/usc-isi-i2/dig-alignment/" \
                              "development/versions/3.0/karma/karma-context.json"
        key += address['addressLocality'] + ":" + address['addressRegion'] + ":" + address['addressCountry']
        city_dict = d.value.all_city_dict
        geo_uri = top_match['uri']
        address['extractorOutput'] = extractor_output
        geoname_city = city_dict[geo_uri]
        if 'longitude' in geoname_city and 'latitude' in geoname_city:
            address['geo'] = dict()
            address['geo']['longitude'] = geoname_city['longitude']
            address['geo']['latitude'] = geoname_city['latitude']
            address['geo']['a'] = 'http://schema.org/GeoCoordinates'
            key += ":" + geoname_city['longitude'] + ":" + geoname_city['latitude']
        else:
            print geo_uri

        address['key'] = key

    except Exception, e:
        print e

    return address

def create_pa_with_name(x):
    out = dict()
    out['name'] = getAddressName(x)
    out['uri'] = x['uri']
    out['a'] = "http://schema.org/PostalAddress"
    out['@context'] = "https://raw.githubusercontent.com/usc-isi-i2/dig-alignment/development/" \
                      "versions/3.0/karma/karma-context.json"
    return out

def merge_postal_addresses(x, d):
    old = x[0]
    new = x[1]

    if old:
        if new:
            if len(new['matches']) >= 1:
                score = float(new['matches'][0]['score'])
                if score >= 0.4502:
                    address = create_address_object(new, old, d)
                    old = address
                else:
                    old = create_pa_with_name(old)
            else:
                old = create_pa_with_name(old)
        else:
            old = create_pa_with_name(old)

    return old

def merge_offers_with_geonames(x, d):
    try:
        offer = x[0]
        geo = x[1]
        if offer:
            if geo:
                # get score of the top match
                score = float(geo['matches'][0]['score'])
                if score >= 0.4502:
                    address = create_address_object(geo, d)
                    offer_address = get_value_json('availableAtOrFrom.address', offer)
                    if offer_address != '':
                        offer['availableAtOrFrom']['address'].append(address)

                        addresses = list()

                        for a in offer['availableAtOrFrom']['address']:
                            if 'createdBy' in a:
                                addresses.append(a)

                        offer['availableAtOrFrom']['address'] = addresses

                    else:
                        a_list = list()
                        a_list.append(address)
                        offer['availableAtOrFrom']['address'] = a_list
    except:
        print offer
        print geo

    return offer

def filter_broken_addresses(x):
    if x:
        if get_value_json('addressLocality', x) != '' or get_value_json('name', x) != '':
            return True

    return False

def reformatDocs(jobj, all_city_dict):
    # print(jobj)
    candidates = []
    for uri in jobj['entities'].keys():
        geoname = all_city_dict[uri]
        city = geoname['name']
        state = geoname['state']
        country = geoname['country']
        candidates.append({'id': uri,
                           'value': {'city': city if type(city) is list else [city],
                                     'state': state if type(state) is list else [state],
                                     'country': country if type(country) is list else [country]}})
    return {'document': jobj['document'], 'entities': candidates, 'processtime': jobj['processtime']}

if __name__ == "__main__":
    sc = SparkContext(appName="DIG-EntityResolution")

    parser = OptionParser()

    parser.add_option("-r", "--rungeonames", dest="rungeonames", default=False,
                      action="store_true")
    parser.add_option("-l", "--loadgeonames", dest="loadgeonames", default=False,
                      action="store_true")

    (c_options, args) = parser.parse_args()

    rungeonames = c_options.rungeonames
    loadgeonames = c_options.loadgeonames

    input_path = args[0]
    output_path = args[1]
    prior_dict_file = args[2]
    topk = int(args[3])
    state_dict_path = args[4]
    all_city_path = args[5]
    city_faerie = args[6]
    state_faerie = args[7]
    all_faerie = args[8]
    tagging_dict_file = args[9]
    ERconfig = args[10]

    dictc = D(sc, state_dict_path, all_city_path, city_faerie,
              state_faerie, all_faerie, prior_dict_file, tagging_dict_file)
    d = sc.broadcast(dictc)
    EV = ProbabilisticER.initializeRecordLinkage(json.load(codecs.open(ERconfig)))
    EV_b = sc.broadcast(EV)

    input_reduced = sc.sequenceFile(input_path).mapValues(lambda x: json.loads(x))
    input_address = input_reduced.filter(lambda x: x[1]['a'] == 'http://schema.org/PostalAddress')

    # initialise as None to avoid running into errors
    resolved_geonames = None
    if rungeonames:
        input_rdd = input_address.flatMapValues(create_input_geonames)
        # resolved_geonames = input_rdd.mapValues(lambda x: processDoc(x, d)).\
        #     mapValues(lambda x: ProbabilisticER.scoreCandidates(EV_b.value, x, d.value.priorDicts,
        #                                                         d.value.taggingDicts, topk, 'raw'))
        resolved_geonames = input_rdd.mapValues(lambda x:processDoc(x, d)).\
            mapValues(lambda x: reformatDocs(x, d.value.all_city_dict)).\
            mapValues(lambda x: ProbabilisticER.scoreCandidates
            (EV_b.value, x, d.value.priorDicts, d.value.taggingDicts, topk, 'raw'))

        resolved_geonames.persist(StorageLevel.MEMORY_AND_DISK)
        resolved_geonames.setName('RESOLVED_GEONAMES')
        resolved_geonames.mapValues(lambda x: json.dumps(x)).saveAsSequenceFile(output_path)
    elif loadgeonames:
        resolved_geonames = sc.sequenceFile(output_path).mapValues(lambda x: json.loads(x))

    if resolved_geonames:
        results = input_address.join(resolved_geonames).mapValues(lambda x: merge_postal_addresses(x, d))
        results_filtered = results.filter(lambda x: filter_broken_addresses(x[1]))
        results_filtered.mapValues(lambda x: json.dumps(x)).saveAsSequenceFile(output_path + "-resolved")
    else:
        print "resolved_geonames is none"

<commit_msg>use Majid's reformatdocs<commit_after>from optparse import OptionParser
from pyspark import SparkContext, StorageLevel
from dictionaries import D
from geoname_extractor import processDoc
import ProbabilisticER
import json
import codecs

"""
RUN AS:
spark-submit --master local[*]    --executor-memory=8g     --driver-memory=8g \
--py-files lib/python-lib.zip main.py  /tmp/geonames/input/input.jl /tmp/geonames/geo-out \
/tmp/geonames/output/prior_dict.json 3 /tmp/geonames/output/state_dict.json /tmp/geonames/output/all_city_dict.json \
/tmp/geonames/output/city_faerie.json /tmp/geonames/output/state_faerie.json /tmp/geonames/output/all_dict_faerie.json \
 /tmp/geonames/output/tagging_dict.json /tmp/geonames/output/config.json
"""

us_states_names_readable = [
    "Alabama", "Alaska", "Arizona", "Arkansas", "California", "Colorado", "Connecticut", "Delaware", "Florida",
    "Georgia", "Hawaii", "Idaho", "Illinois", "Indiana", "Iowa", "Kansas", "Kentucky", "Louisiana", "Maine", "Maryland",
    "Massachusetts", "Michigan", "Minnesota", "Mississippi", "Missouri", "Montana", "Nebraska", "Nevada",
    "New Hampshire", "New Jersey", "New Mexico", "New York", "North Carolina", "North Dakota", "Ohio", "Oklahoma",
    "Oregon", "Pennsylvania", "Rhode Island", "South Carolina", "South Dakota", "Tennessee", "Texas", "Utah", "Vermont",
    "Virginia", "Washington", "West Virginia", "Wisconsin", "Wyoming"
]

us_states_codes = [
    "AL", "AK", "AZ", "AR", "CA", "CO", "CT", "DE", "FL", "GA", "HI", "ID", "IL", "IN", "IA", "KS", "KY", "LA", "ME",
    "MD", "MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV", "NH", "NJ", "NM", "NY", "NC", "ND", "OH", "OK", "OR", "PA",
    "RI", "SC", "SD", "TN", "TX", "UT", "VT", "VA", "WA", "WV", "WI", "WY"
]

# Given a path in json, return value if path, full path denoted by a separator,like '$'or '.',
#  (example address.name) exists, otherwise return ''
def get_value_json(path, doc, separator='.'):
    paths = path.strip().split(separator)
    for field in paths:
        if field in doc:
            doc = doc[field]
        else:
            return ''

    if type(doc) == dict or type(doc) == list:
        return json.dumps(doc)
    else:
        return doc

def get_only_city_name(x):
        values = x.split(" ")
        if len(values) > 1 and len(values[len(values) - 1]) == 2 and (values[len(values) - 1].upper() != 'DC' or values[len(values) - 1].upper() != 'UK' or values[len(values) - 1].upper() != 'US'):
            return " ".join(values[0:len(values) - 1])
        return x

def get_state_from_city(x):
    values = x.split(" ")
    if len(values) > 1 and len(values[len(values) - 1]) == 2 and (values[len(values) - 1].upper() != 'DC' or values[len(values) - 1].upper() != 'UK' or values[len(values) - 1].upper() != 'US'):
        short_state = values[len(values) - 1].upper()
        try:
            idx = us_states_codes.index(short_state)
            return us_states_names_readable[idx]
        except ValueError:
            return ''
    return ''

def create_input_geonames(line):
    result = []

    # line = json.loads(line)

    if line:
        json_x = line

        json_l = []
        if isinstance(json_x, dict):
            json_l.append(json_x)
        elif isinstance(json_x, list):
            json_l = json_x

        for x in json_l:
            # print x
            out = dict()
            out['country'] = get_value_json('addressCountry', x)

            city = get_value_json('addressLocality', x)
            out['locality'] = get_only_city_name(city)

            region = get_value_json('addressRegion', x)
            # out['region'] = region
            state = get_state_from_city(city)

            if region.strip() == '':
                if state.strip() != '':
                    out['region'] = state
                else:
                    out['region'] = ''
            else:
                out['region'] = region

            if out['country'].strip() == '' and out['region'].strip() == '' and out['locality'].strip() == '':
                print line['uri']

            if out['locality'].strip() == '':
                n = ''
                if out['region'].strip() != '':
                    n = out['region']
                if out['country'].strip() != '':
                    if n == '':
                        n = out['country']
                    else:
                        n += ',' + out['country']

                out['locality'] = n

            out['uri'] = line['uri']

        result.append(out)
    return result

def getAddressName(x):
    name = ''
    city = get_value_json('addressLocality', x)
    state = get_value_json('addressRegion', x)
    country = get_value_json('addressCountry', x)

    if city != '':
        name += city + ','

    if state != '':
        name += state + ','

    if country != '':
        name += country
    if name != '':
        if name[len(name)-1] == ',':
            name = name[:len(name)-1]
    return name

def create_address_object(geo, x, d):
    address = {}
    try:
        extractor_output = getAddressName(x)
        key = ''
        top_match = geo['matches'][0]
        address['addressLocality'] = top_match['value']['city']
        address['addressRegion'] = top_match['value']['state']
        address['addressCountry'] = top_match['value']['country']
        address['uri'] = x['uri']
        address['createdBy'] = 'dig-geonames'
        address['a'] = "http://schema.org/PostalAddress"
        address['@context'] = "https://raw.githubusercontent.com/usc-isi-i2/dig-alignment/" \
                              "development/versions/3.0/karma/karma-context.json"
        key += address['addressLocality'] + ":" + address['addressRegion'] + ":" + address['addressCountry']
        city_dict = d.value.all_city_dict
        geo_uri = top_match['uri']
        address['extractorOutput'] = extractor_output
        geoname_city = city_dict[geo_uri]
        if 'longitude' in geoname_city and 'latitude' in geoname_city:
            address['geo'] = dict()
            address['geo']['longitude'] = geoname_city['longitude']
            address['geo']['latitude'] = geoname_city['latitude']
            address['geo']['a'] = 'http://schema.org/GeoCoordinates'
            key += ":" + geoname_city['longitude'] + ":" + geoname_city['latitude']
        else:
            print geo_uri

        address['key'] = key

    except Exception, e:
        print e

    return address

def create_pa_with_name(x):
    out = dict()
    out['name'] = getAddressName(x)
    out['uri'] = x['uri']
    out['a'] = "http://schema.org/PostalAddress"
    out['@context'] = "https://raw.githubusercontent.com/usc-isi-i2/dig-alignment/development/" \
                      "versions/3.0/karma/karma-context.json"
    return out

def merge_postal_addresses(x, d):
    old = x[0]
    new = x[1]

    if old:
        if new:
            if len(new['matches']) >= 1:
                address = create_address_object(new, old, d)
                old = address
                return old

    old = create_pa_with_name(old)
    return old

def merge_offers_with_geonames(x, d):
    try:
        offer = x[0]
        geo = x[1]
        if offer:
            if geo:
                # get score of the top match
                score = float(geo['matches'][0]['score'])
                if score >= 0.4502:
                    address = create_address_object(geo, d)
                    offer_address = get_value_json('availableAtOrFrom.address', offer)
                    if offer_address != '':
                        offer['availableAtOrFrom']['address'].append(address)

                        addresses = list()

                        for a in offer['availableAtOrFrom']['address']:
                            if 'createdBy' in a:
                                addresses.append(a)

                        offer['availableAtOrFrom']['address'] = addresses

                    else:
                        a_list = list()
                        a_list.append(address)
                        offer['availableAtOrFrom']['address'] = a_list
    except:
        print offer
        print geo

    return offer

def filter_broken_addresses(x):
    if x:
        if get_value_json('addressLocality', x) != '' or get_value_json('name', x) != '':
            return True

    return False

def reformatDocs(jobj, all_city_dict):
    # print(jobj)
    candidates = []
    for uri in jobj['entities'].keys():
        geoname = all_city_dict[uri]
        city = geoname['name']
        state = geoname['state']
        country = geoname['country']
        candidates.append({'id': uri,
                           'value': {'city': city if type(city) is list else [city],
                                     'state': state if type(state) is list else [state],
                                     'country': country if type(country) is list else [country]}})
    return {'document': jobj['document'], 'entities': candidates, 'processtime': jobj['processtime']}

if __name__ == "__main__":
    sc = SparkContext(appName="DIG-EntityResolution")

    parser = OptionParser()

    parser.add_option("-r", "--rungeonames", dest="rungeonames", default=False,
                      action="store_true")
    parser.add_option("-l", "--loadgeonames", dest="loadgeonames", default=False,
                      action="store_true")

    (c_options, args) = parser.parse_args()

    rungeonames = c_options.rungeonames
    loadgeonames = c_options.loadgeonames

    input_path = args[0]
    output_path = args[1]
    prior_dict_file = args[2]
    topk = int(args[3])
    state_dict_path = args[4]
    all_city_path = args[5]
    city_faerie = args[6]
    state_faerie = args[7]
    all_faerie = args[8]
    tagging_dict_file = args[9]
    ERconfig = args[10]

    dictc = D(sc, state_dict_path, all_city_path, city_faerie,
              state_faerie, all_faerie, prior_dict_file, tagging_dict_file)
    d = sc.broadcast(dictc)
    EV = ProbabilisticER.initializeRecordLinkage(json.load(codecs.open(ERconfig)))
    EV_b = sc.broadcast(EV)

    input_reduced = sc.sequenceFile(input_path).mapValues(lambda x: json.loads(x))
    input_address = input_reduced.filter(lambda x: x[1]['a'] == 'http://schema.org/PostalAddress')

    # initialise as None to avoid running into errors
    resolved_geonames = None
    if rungeonames:
        input_rdd = input_address.flatMapValues(create_input_geonames)
        # resolved_geonames = input_rdd.mapValues(lambda x: processDoc(x, d)).\
        #     mapValues(lambda x: ProbabilisticER.scoreCandidates(EV_b.value, x, d.value.priorDicts,
        #                                                         d.value.taggingDicts, topk, 'raw'))
        resolved_geonames = input_rdd.mapValues(lambda x:processDoc(x, d)).\
            mapValues(lambda x: reformatDocs(x, d.value.all_city_dict)).\
            mapValues(lambda x: ProbabilisticER.scoreCandidates
            (EV_b.value, x, d.value.priorDicts, d.value.taggingDicts, topk, 'raw'))

        print resolved_geonames.first()
        resolved_geonames.persist(StorageLevel.MEMORY_AND_DISK)
        resolved_geonames.setName('RESOLVED_GEONAMES')
        resolved_geonames.mapValues(lambda x: json.dumps(x)).saveAsSequenceFile(output_path)
    elif loadgeonames:
        resolved_geonames = sc.sequenceFile(output_path).mapValues(lambda x: json.loads(x))

    if resolved_geonames:
        results = input_address.join(resolved_geonames).mapValues(lambda x: merge_postal_addresses(x, d))
        print results.first()
        # results_filtered = results.filter(lambda x: filter_broken_addresses(x[1]))
        results.mapValues(lambda x: json.dumps(x)).saveAsSequenceFile(output_path + "-resolved")
    else:
        print "resolved_geonames is none"

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2017 - 2020 Avram Lubkin, All Rights Reserved

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

"""
**Enlighten Progress Bar**

Provides progress bars and counters which play nice in a TTY console
"""

from enlighten.counter import Counter, StatusBar, SubCounter
from enlighten._manager import Manager, get_manager
from enlighten._util import EnlightenWarning, Justify

__version__ = '1.6.0'
__all__ = ('Counter', 'EnlightenWarning', 'Justify', 'Manager',
           'StatusBar', 'SubCounter', 'get_manager')
<commit_msg>Bump version to 1.6.1<commit_after># -*- coding: utf-8 -*-
# Copyright 2017 - 2020 Avram Lubkin, All Rights Reserved

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

"""
**Enlighten Progress Bar**

Provides progress bars and counters which play nice in a TTY console
"""

from enlighten.counter import Counter, StatusBar, SubCounter
from enlighten._manager import Manager, get_manager
from enlighten._util import EnlightenWarning, Justify

__version__ = '1.6.1'
__all__ = ('Counter', 'EnlightenWarning', 'Justify', 'Manager',
           'StatusBar', 'SubCounter', 'get_manager')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

try:
    import configparser
except ImportError:
    import ConfigParser as configparser

from twisted.internet import reactor
from twisted.internet.protocol import Protocol, ReconnectingClientFactory
import cmd, serial, sys, threading, readline, time, logging, os, argparse, re, codecs, signal

### set some default options
import platform
defaulthostname = platform.node()

defaultport     = "/dev/ttyUSB0"
defaultbaud     = 115200
defaultdir      = os.environ['HOME'] + os.path.sep + '.pyterm'
defaultfile     = "pyterm-" + defaulthostname + ".conf"
defaultrunname  = "default-run"

class SerCmd(cmd.Cmd):

    def __init__(self, port=None, baudrate=None, confdir=None, conffile=None, host=None, run_name=None):
        cmd.Cmd.__init__(self)
        self.port = port
        self.baudrate = baudrate
        self.configdir = confdir
        self.configfile = conffile
        self.host = host
        self.run_name = run_name

        if not self.host:
            self.host = defaulthostname

	if not self.run_name:
            self.run_name = defaultrunname

        if not os.path.exists(self.configdir):
            os.makedirs(self.configdir)

        self.aliases = dict()
        self.filters = []
        self.ignores = []
        self.json_regs = dict()
        self.init_cmd = []
        self.load_config()

        try:
            readline.read_history_file()
        except IOError:
            pass

        ### create Logging object
        my_millis = "{:.4f}".format(time.time())
        date_str = '{}.{}'.format(time.strftime('%Y%m%d-%H:%M:%S'), my_millis[-4:])
        self.startup = date_str
        # create formatter
        fmt_str = '%(asctime)s - %(levelname)s # %(message)s'
        formatter = logging.Formatter(fmt_str)

        directory = self.configdir + os.path.sep + self.host
        if not os.path.exists(directory):
            os.makedirs(directory)
        logging.basicConfig(filename=directory + os.path.sep + self.run_name + '.log', level=logging.DEBUG, format=fmt_str)
        ch = logging.StreamHandler()
        ch.setLevel(logging.DEBUG)

        # create logger
        self.logger = logging.getLogger('')
        self.logger.setLevel(logging.DEBUG)

        # add formatter to ch
        ch.setFormatter(formatter)
        # add ch to logger
        self.logger.addHandler(ch)

    def preloop(self):
        if not self.port:
            sys.stderr.write("No port specified, using default (%s)!\n" % (defaultport))
            self.port = defaultport
        self.ser = serial.Serial(port=self.port, baudrate=self.baudrate, dsrdtr=0, rtscts=0)
        self.ser.setDTR(0)
        self.ser.setRTS(0)

        time.sleep(1)
        for cmd in self.init_cmd:
            self.log.debug("WRITE ----->>>>>> '" + cmd + "'\n")
            self.ser.write(cmd + "\n")

        # start serial->console thread
        receiver_thread = threading.Thread(target=self.reader)
        receiver_thread.setDaemon(1)
        receiver_thread.start()

    def precmd(self, line):
        if (line.startswith("/")):
                return "PYTERM_" + line[1:]
        return line

    def default(self, line):
        for tok in line.split(';'):
            tok = self.get_alias(tok)
            self.ser.write((tok.strip() + "\n").encode("utf-8"))

    def do_help(self, line):
        self.ser.write("help\n".encode("utf-8"))

    def complete_date(self, text, line, begidx, endidm):
        date = time.strftime("%Y-%m-%d %H:%M:%S")
        return ["{}".format(date)]

    def do_PYTERM_reset(self, line):
        self.ser.setDTR(1)
        self.ser.setDTR(0)

    def do_PYTERM_exit(self, line, unused=None):
        print("Exiting Pyterm")
        readline.write_history_file()
        if reactor.running:
            reactor.callFromThread(reactor.stop)
        return True

    def do_PYTERM_save(self, line):
        if not self.config.has_section("general"):
            self.config.add_section("general")
        self.config.set("general", "port", self.port)
        if len(self.aliases):
            if not self.config.has_section("aliases"):
                self.config.add_section("aliases")
            for alias in self.aliases:
                self.config.set("aliases", alias, self.aliases[alias])
        if len(self.json_regs):
            if not self.config.has_section("json_regs"):
                self.config.add_section("json_regs")
            for j in self.json_regs:
                self.config.set("json_regs", j, self.json_regs[j].pattern)
        if len(self.filters):
            if not self.config.has_section("filters"):
                self.config.add_section("filters")
            i = 0
            for r in self.filters:
                self.config.set("filters", "filter%i" % i, r.pattern)
                i += 1
        if len(self.ignores):
            if not self.config.has_section("ignores"):
                self.config.add_section("ignores")
            i = 0
            for r in self.ignores:
                self.config.set("ignores", "ignore%i" % i, r.pattern)
                i += 1

        with open(self.configdir + os.path.sep + self.configfile, 'wb') as config_fd:
            self.config.write(config_fd)
            print("Config saved")

    def do_PYTERM_show_config(self, line):
        for key in self.__dict__:
            print(str(key) + ": " + str(self.__dict__[key]))

    def do_PYTERM_alias(self, line):
        if line.endswith("list"):
            for alias in self.aliases:
                print("{} = {}".format(alias, self.aliases[alias]))
            return
        if not line.count("="):
            sys.stderr.write("Usage: alias <ALIAS> = <CMD>\n")
            return
        self.aliases[line.split('=')[0].strip()] = line.split('=')[1].strip()

    def do_PYTERM_rmalias(self, line):
        if not self.aliases.pop(line, None):
            sys.stderr.write("Alias not found")

    def get_alias(self, tok):
        for alias in self.aliases:
            if tok.split()[0] == alias:
                return self.aliases[alias] + tok[len(alias):]
        return tok

    def do_PYTERM_ignore(self, line):
        self.ignores.append(re.compile(line.strip()))

    def do_PYTERM_unignore(self, line):
        for r in self.ignores:
            if (r.pattern == line.strip()):
                print("Remove ignore for %s" % r.pattern)
                self.ignores.remove(r)
                return
        sys.stderr.write("Ignore for %s not found\n" % line.strip())

    def do_PYTERM_filter(self, line):
        self.filters.append(re.compile(line.strip()))

    def do_PYTERM_unfilter(self, line):
        for r in self.filters:
            if (r.pattern == line.strip()):
                print("Remove filter for %s" % r.pattern)
                self.filters.remove(r)
                return
        sys.stderr.write("Filter for %s not found\n" % line.strip())

    def do_PYTERM_json(self, line):
        self.json_regs[line.split(' ')[0].strip()] = re.compile(line.partition(' ')[2].strip())

    def do_PYTERM_unjson(self, line):
        if not self.aliases.pop(line, None):
            sys.stderr.write("JSON regex with ID %s not found" % line)

    def load_config(self):
        self.config = configparser.SafeConfigParser()
        self.config.read([self.configdir + os.path.sep + self.configfile])

        for sec in self.config.sections():
            if sec == "filters":
                for opt in self.config.options(sec):
                    self.filters.append(re.compile(self.config.get(sec, opt)))
            if sec == "ignores":
                for opt in self.config.options(sec):
                    self.ignores.append(re.compile(self.config.get(sec, opt)))
            if sec == "json_regs":
                for opt in self.config.options(sec):
                    print("add json regex for %s" % self.config.get(sec, opt))
                    self.json_regs[opt] = re.compile(self.config.get(sec, opt))
            if sec == "aliases":
                for opt in self.config.options(sec):
                    self.aliases[opt] = self.config.get(sec, opt)
            if sec == "init_cmd":
                for opt in self.config.options(sec):
                    self.init_cmd.append(self.config.get(sec, opt))
            else:
                for opt in self.config.options(sec):
                    if opt not in self.__dict__:
                        self.__dict__[opt] = self.config.get(sec, opt)

    def reader(self):
        sr = codecs.getreader("UTF-8")(self.ser, errors='replace')
        output = ""
        while (1):
            c = sr.read(1)
            if c == '\n' or c == '\r':
                ignored = False
                if (len(self.ignores)):
                    for i in self.ignores:
                        if i.search(output):
                            ignored = True
                            break
                if (len(self.filters)):
                    for r in self.filters:
                        if r.search(output):
                            if not ignored:
                                self.logger.info(output)
                else:
                    if not ignored:
                        self.logger.info(output)

                if (len(self.json_regs)) and self.factory and self.factory.myproto:
                    for j in self.json_regs:
                        m = self.json_regs[j].search(output)
                        if m:
                            try:
                                json_obj = '{"jid":%d, ' % int(j)
                            except ValueError:
                                sys.stderr.write("Invalid JID: %s\n" % j)
                                break
                            json_obj += '"raw":"%s", ' % output
                            json_obj += '"date":%s, ' % int(time.time()*1000)
                            for g in m.groupdict():
                                try:
                                    json_obj += '"%s":%d, ' % (g, int(m.groupdict()[g]))
                                except ValueError:
                                    json_obj += '"%s":"%s", ' % (g, m.groupdict()[g])

                            # eliminate the superfluous last ", "
                            json_obj = json_obj[:-2]

                            json_obj += "}"
                            self.factory.myproto.sendMessage(json_obj)

                output = ""
            else:
                output += c
            #sys.stdout.write(c)
            #sys.stdout.flush()

class PytermProt(Protocol):
    def dataReceived(self, data):
        sys.stdout.write(data)

    def sendMessage(self, msg):
        self.transport.writeSomeData("%d#%s\n" % (len(msg), msg))

class PytermClientFactory(ReconnectingClientFactory):

    def __init__(self):
        self.myproto = None

    def buildProtocol(self, addr):
        print('Connected.')
        self.resetDelay()
        self.myproto = PytermProt()
        return self.myproto

    def clientConnectionLost(self, connector, reason):
        if reactor.running:
            print('Lost connection.  Reason:', reason)
        ReconnectingClientFactory.clientConnectionLost(self, connector, reason)

    def clientConnectionFailed(self, connector, reason):
        print('Connection failed. Reason:', reason)
        ReconnectingClientFactory.clientConnectionFailed(self, connector,
                                                         reason)

def __stop_reactor(signum, stackframe):
    sys.stderr.write("Ctrl-C is disabled, type '/exit' instead\n")

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description="Pyterm - The Python terminal program")
    parser.add_argument("-p", "--port",
            help="Specifies the serial port to use, default is %s" % defaultport,
            default=defaultport)
    parser.add_argument("-b", "--baudrate",
            help="Specifies baudrate for the serial port, default is %s" % defaultbaud,
            default=defaultbaud)
    parser.add_argument('-d', '--directory',
            help="Specify the Pyterm directory, default is %s" % defaultdir,
            default=defaultdir)
    parser.add_argument("-c", "--config",
            help="Specify the config filename, default is %s" % defaultfile,
            default=defaultfile)
    parser.add_argument("-s", "--server",
            help="Connect via TCP to this server to send output as JSON")
    parser.add_argument("-P", "--tcp_port", type=int,
            help="Port at the JSON server")
    parser.add_argument("-H", "--host",
            help="Hostname of this maschine")
    parser.add_argument("-rn", "--run-name",
            help="Run name, used for logfile")
    args = parser.parse_args()

    myshell = SerCmd(args.port, args.baudrate, args.directory, args.config, args.host, args.run_name)
    myshell.prompt = ''

    if args.server and args.tcp_port:
        myfactory = PytermClientFactory()
        reactor.connectTCP(args.server, args.tcp_port, myfactory)
        myshell.factory = myfactory
        reactor.callInThread(myshell.cmdloop, "Welcome to pyterm!\nType '/exit' to exit.")
        signal.signal(signal.SIGINT, __stop_reactor)
        reactor.run()
    else:
        myshell.cmdloop("Welcome to pyterm!\nType 'exit' to exit.")
<commit_msg>pyterm: enable saving and setting of  init commands at runtime<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

try:
    import configparser
except ImportError:
    import ConfigParser as configparser

from twisted.internet import reactor
from twisted.internet.protocol import Protocol, ReconnectingClientFactory
import cmd, serial, sys, threading, readline, time, logging, os, argparse, re, codecs, signal

### set some default options
import platform
defaulthostname = platform.node()

defaultport     = "/dev/ttyUSB0"
defaultbaud     = 115200
defaultdir      = os.environ['HOME'] + os.path.sep + '.pyterm'
defaultfile     = "pyterm-" + defaulthostname + ".conf"
defaultrunname  = "default-run"

class SerCmd(cmd.Cmd):

    def __init__(self, port=None, baudrate=None, confdir=None, conffile=None, host=None, run_name=None):
        cmd.Cmd.__init__(self)
        self.port = port
        self.baudrate = baudrate
        self.configdir = confdir
        self.configfile = conffile
        self.host = host
        self.run_name = run_name

        if not self.host:
            self.host = defaulthostname

	if not self.run_name:
            self.run_name = defaultrunname

        if not os.path.exists(self.configdir):
            os.makedirs(self.configdir)

        self.aliases = dict()
        self.filters = []
        self.ignores = []
        self.json_regs = dict()
        self.init_cmd = []
        self.load_config()

        try:
            readline.read_history_file()
        except IOError:
            pass

        ### create Logging object
        my_millis = "{:.4f}".format(time.time())
        date_str = '{}.{}'.format(time.strftime('%Y%m%d-%H:%M:%S'), my_millis[-4:])
        self.startup = date_str
        # create formatter
        fmt_str = '%(asctime)s - %(levelname)s # %(message)s'
        formatter = logging.Formatter(fmt_str)

        directory = self.configdir + os.path.sep + self.host
        if not os.path.exists(directory):
            os.makedirs(directory)
        logging.basicConfig(filename=directory + os.path.sep + self.run_name + '.log', level=logging.DEBUG, format=fmt_str)
        ch = logging.StreamHandler()
        ch.setLevel(logging.DEBUG)

        # create logger
        self.logger = logging.getLogger('')
        self.logger.setLevel(logging.DEBUG)

        # add formatter to ch
        ch.setFormatter(formatter)
        # add ch to logger
        self.logger.addHandler(ch)

    def preloop(self):
        if not self.port:
            sys.stderr.write("No port specified, using default (%s)!\n" % (defaultport))
            self.port = defaultport
        self.ser = serial.Serial(port=self.port, baudrate=self.baudrate, dsrdtr=0, rtscts=0)
        self.ser.setDTR(0)
        self.ser.setRTS(0)

        time.sleep(1)
        for cmd in self.init_cmd:
            self.logger.debug("WRITE ----->>>>>> '" + cmd + "'\n")
            self.ser.write(cmd + "\n")

        # start serial->console thread
        receiver_thread = threading.Thread(target=self.reader)
        receiver_thread.setDaemon(1)
        receiver_thread.start()

    def precmd(self, line):
        if (line.startswith("/")):
                return "PYTERM_" + line[1:]
        return line

    def default(self, line):
        for tok in line.split(';'):
            tok = self.get_alias(tok)
            self.ser.write((tok.strip() + "\n").encode("utf-8"))

    def do_help(self, line):
        self.ser.write("help\n".encode("utf-8"))

    def complete_date(self, text, line, begidx, endidm):
        date = time.strftime("%Y-%m-%d %H:%M:%S")
        return ["{}".format(date)]

    def do_PYTERM_reset(self, line):
        self.ser.setDTR(1)
        self.ser.setDTR(0)

    def do_PYTERM_exit(self, line, unused=None):
        print("Exiting Pyterm")
        readline.write_history_file()
        if reactor.running:
            reactor.callFromThread(reactor.stop)
        return True

    def do_PYTERM_save(self, line):
        if not self.config.has_section("general"):
            self.config.add_section("general")
        self.config.set("general", "port", self.port)
        if len(self.aliases):
            if not self.config.has_section("aliases"):
                self.config.add_section("aliases")
            for alias in self.aliases:
                self.config.set("aliases", alias, self.aliases[alias])
        if len(self.json_regs):
            if not self.config.has_section("json_regs"):
                self.config.add_section("json_regs")
            for j in self.json_regs:
                self.config.set("json_regs", j, self.json_regs[j].pattern)
        if len(self.filters):
            if not self.config.has_section("filters"):
                self.config.add_section("filters")
            i = 0
            for r in self.filters:
                self.config.set("filters", "filter%i" % i, r.pattern)
                i += 1
        if len(self.ignores):
            if not self.config.has_section("ignores"):
                self.config.add_section("ignores")
            i = 0
            for r in self.ignores:
                self.config.set("ignores", "ignore%i" % i, r.pattern)
                i += 1
        if len(self.init_cmd):
            if not self.config.has_section("init_cmd"):
                self.config.add_section("init_cmd")
            i = 0
            for ic in self.init_cmd:
                self.config.set("init_cmd", "init_cmd%i" % i, ic)
                i += 1

        with open(self.configdir + os.path.sep + self.configfile, 'wb') as config_fd:
            self.config.write(config_fd)
            print("Config saved")

    def do_PYTERM_show_config(self, line):
        for key in self.__dict__:
            print(str(key) + ": " + str(self.__dict__[key]))

    def do_PYTERM_alias(self, line):
        if line.endswith("list"):
            for alias in self.aliases:
                print("{} = {}".format(alias, self.aliases[alias]))
            return
        if not line.count("="):
            sys.stderr.write("Usage: alias <ALIAS> = <CMD>\n")
            return
        self.aliases[line.split('=')[0].strip()] = line.split('=')[1].strip()

    def do_PYTERM_rmalias(self, line):
        if not self.aliases.pop(line, None):
            sys.stderr.write("Alias not found")

    def get_alias(self, tok):
        for alias in self.aliases:
            if tok.split()[0] == alias:
                return self.aliases[alias] + tok[len(alias):]
        return tok

    def do_PYTERM_ignore(self, line):
        self.ignores.append(re.compile(line.strip()))

    def do_PYTERM_unignore(self, line):
        for r in self.ignores:
            if (r.pattern == line.strip()):
                print("Remove ignore for %s" % r.pattern)
                self.ignores.remove(r)
                return
        sys.stderr.write("Ignore for %s not found\n" % line.strip())

    def do_PYTERM_filter(self, line):
        self.filters.append(re.compile(line.strip()))

    def do_PYTERM_unfilter(self, line):
        for r in self.filters:
            if (r.pattern == line.strip()):
                print("Remove filter for %s" % r.pattern)
                self.filters.remove(r)
                return
        sys.stderr.write("Filter for %s not found\n" % line.strip())

    def do_PYTERM_json(self, line):
        self.json_regs[line.split(' ')[0].strip()] = re.compile(line.partition(' ')[2].strip())

    def do_PYTERM_unjson(self, line):
        if not self.aliases.pop(line, None):
            sys.stderr.write("JSON regex with ID %s not found" % line)

    def do_PYTERM_init(self, line):
        self.init_cmd.append(line.strip())

    def load_config(self):
        self.config = configparser.SafeConfigParser()
        self.config.read([self.configdir + os.path.sep + self.configfile])

        for sec in self.config.sections():
            if sec == "filters":
                for opt in self.config.options(sec):
                    self.filters.append(re.compile(self.config.get(sec, opt)))
            if sec == "ignores":
                for opt in self.config.options(sec):
                    self.ignores.append(re.compile(self.config.get(sec, opt)))
            if sec == "json_regs":
                for opt in self.config.options(sec):
                    print("add json regex for %s" % self.config.get(sec, opt))
                    self.json_regs[opt] = re.compile(self.config.get(sec, opt))
            if sec == "aliases":
                for opt in self.config.options(sec):
                    self.aliases[opt] = self.config.get(sec, opt)
            if sec == "init_cmd":
                for opt in self.config.options(sec):
                    self.init_cmd.append(self.config.get(sec, opt))
            else:
                for opt in self.config.options(sec):
                    if opt not in self.__dict__:
                        self.__dict__[opt] = self.config.get(sec, opt)

    def reader(self):
        sr = codecs.getreader("UTF-8")(self.ser, errors='replace')
        output = ""
        while (1):
            c = sr.read(1)
            if c == '\n' or c == '\r':
                ignored = False
                if (len(self.ignores)):
                    for i in self.ignores:
                        if i.search(output):
                            ignored = True
                            break
                if (len(self.filters)):
                    for r in self.filters:
                        if r.search(output):
                            if not ignored:
                                self.logger.info(output)
                else:
                    if not ignored:
                        self.logger.info(output)

                if (len(self.json_regs)) and self.factory and self.factory.myproto:
                    for j in self.json_regs:
                        m = self.json_regs[j].search(output)
                        if m:
                            try:
                                json_obj = '{"jid":%d, ' % int(j)
                            except ValueError:
                                sys.stderr.write("Invalid JID: %s\n" % j)
                                break
                            json_obj += '"raw":"%s", ' % output
                            json_obj += '"date":%s, ' % int(time.time()*1000)
                            for g in m.groupdict():
                                try:
                                    json_obj += '"%s":%d, ' % (g, int(m.groupdict()[g]))
                                except ValueError:
                                    json_obj += '"%s":"%s", ' % (g, m.groupdict()[g])

                            # eliminate the superfluous last ", "
                            json_obj = json_obj[:-2]

                            json_obj += "}"
                            self.factory.myproto.sendMessage(json_obj)

                output = ""
            else:
                output += c
            #sys.stdout.write(c)
            #sys.stdout.flush()

class PytermProt(Protocol):
    def dataReceived(self, data):
        sys.stdout.write(data)

    def sendMessage(self, msg):
        self.transport.writeSomeData("%d#%s\n" % (len(msg), msg))

class PytermClientFactory(ReconnectingClientFactory):

    def __init__(self):
        self.myproto = None

    def buildProtocol(self, addr):
        print('Connected.')
        self.resetDelay()
        self.myproto = PytermProt()
        return self.myproto

    def clientConnectionLost(self, connector, reason):
        if reactor.running:
            print('Lost connection.  Reason:', reason)
        ReconnectingClientFactory.clientConnectionLost(self, connector, reason)

    def clientConnectionFailed(self, connector, reason):
        print('Connection failed. Reason:', reason)
        ReconnectingClientFactory.clientConnectionFailed(self, connector,
                                                         reason)

def __stop_reactor(signum, stackframe):
    sys.stderr.write("Ctrl-C is disabled, type '/exit' instead\n")

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description="Pyterm - The Python terminal program")
    parser.add_argument("-p", "--port",
            help="Specifies the serial port to use, default is %s" % defaultport,
            default=defaultport)
    parser.add_argument("-b", "--baudrate",
            help="Specifies baudrate for the serial port, default is %s" % defaultbaud,
            default=defaultbaud)
    parser.add_argument('-d', '--directory',
            help="Specify the Pyterm directory, default is %s" % defaultdir,
            default=defaultdir)
    parser.add_argument("-c", "--config",
            help="Specify the config filename, default is %s" % defaultfile,
            default=defaultfile)
    parser.add_argument("-s", "--server",
            help="Connect via TCP to this server to send output as JSON")
    parser.add_argument("-P", "--tcp_port", type=int,
            help="Port at the JSON server")
    parser.add_argument("-H", "--host",
            help="Hostname of this maschine")
    parser.add_argument("-rn", "--run-name",
            help="Run name, used for logfile")
    args = parser.parse_args()

    myshell = SerCmd(args.port, args.baudrate, args.directory, args.config, args.host, args.run_name)
    myshell.prompt = ''

    if args.server and args.tcp_port:
        myfactory = PytermClientFactory()
        reactor.connectTCP(args.server, args.tcp_port, myfactory)
        myshell.factory = myfactory
        reactor.callInThread(myshell.cmdloop, "Welcome to pyterm!\nType '/exit' to exit.")
        signal.signal(signal.SIGINT, __stop_reactor)
        reactor.run()
    else:
        myshell.cmdloop("Welcome to pyterm!\nType 'exit' to exit.")
<|endoftext|>


<commit_before>#!/usr/bin/python
# File:        profileprogress.py
# Description: profileprogress
# Created:     2017-07-05

from collections import OrderedDict
from itertools import islice

from .bytes import (BytesModel, Section, S_DOUBLE,
                    SECTION_UNK_1, SECTION_UNK_2, SECTION_UNK_3)
from .common import format_duration, format_duration_dhms, format_distance
from .constants import Completion, Mode, TIMED_MODES

FTYPE_PROFILEPROGRESS = 'ProfileProgress'

def format_score(mode, score, comp):
    comp_str = Completion.to_name(comp)
    mode_str = Mode.to_name(mode)
    if mode in TIMED_MODES:
        type_str = "time"
        if score < 0:
            score_str = "None"
        else:
            score_str = format_duration(score)
    else:
        type_str = "score"
        if score < 0:
            score_str = "None"
        else:
            score_str = str(score)
    return f"{mode_str} {type_str}: {score_str} ({comp_str})"

class LevelProgress(BytesModel):

    level_path = None
    completion = ()
    scores = ()

    def parse(self, dbytes):
        self.level_path = dbytes.read_string()
        self.recoverable = True
        self.add_unknown(value=dbytes.read_string())
        self.add_unknown(1)

        self.require_equal(SECTION_UNK_1, 4)
        num_levels = dbytes.read_fixed_number(4)
        self.completion = completion = []
        for i in range(num_levels):
            completion.append(dbytes.read_fixed_number(4))

        self.require_equal(SECTION_UNK_1, 4)
        num_levels = dbytes.read_fixed_number(4)
        self.scores = scores = []
        for i in range(num_levels):
            scores.append(dbytes.read_fixed_number(4, signed=True))
        self.add_unknown(8)

    def _print_data(self, p):
        p(f"Level path: {self.level_path!r}")
        for mode, (score, comp) in enumerate(zip(self.scores, self.completion)):
            if comp != Completion.UNPLAYED:
                p(format_score(mode, score, comp))

class Stat(object):

    def __init__(self, type, ident, unit='num', nbytes=None):
        if type == 'd':
            def parse(dbytes):
                return dbytes.read_struct(S_DOUBLE)[0]
        elif type == 'u8':
            def parse(dbytes):
                return dbytes.read_fixed_number(8)
        elif type == 'unk':
            def parse(dbytes):
                return dbytes.read_n(nbytes)
        else:
            raise ValueError(f"invalid type: {type!r}")
        self.parse_value = parse
        if unit == 'sec':
            self.format = lambda v: format_duration_dhms(v * 1000)
        elif unit == 'num':
            self.format = str
        elif unit == 'unknown':
            self.format = str
        elif unit == 'meters':
            self.format = format_distance
        elif unit == 'ev':
            self.format = lambda v: f"{v:,d} eV"
        else:
            raise ValueError(f"invalid unit: {unit!r}")
        self.ident = ident
        self.type = type
        self.unit = unit

    def parse_value(self, dbytes):
        raise NotImplementedError

class AttrOrderedDict(OrderedDict):

    def __getattr__(self, name):
        try:
            return self[name]
        except KeyError:
            raise AttributeError(name)

STATS = AttrOrderedDict((s.ident, s) for s in (
    Stat('u8', 'deaths'),
    Stat('u8', 'laser_deaths'),
    Stat('u8', 'reset_deaths'),
    Stat('u8', 'impact_deaths'),
    Stat('u8', 'overheat_deaths'),
    Stat('u8', 'killgrid_deaths'),
    Stat('d', 'gibs_time', 'sec'),
    Stat('d', 'unk_0', 'meters'),
    Stat('d', 'forward', 'meters'),
    Stat('d', 'reverse', 'meters'),
    Stat('d', 'air_fly', 'meters'),
    Stat('d', 'air_nofly', 'meters'),
    Stat('d', 'wallride', 'meters'),
    Stat('d', 'ceilingride', 'meters'),
    Stat('d', 'grinding', 'meters'),
    Stat('d', 'boost', 'sec'),
    Stat('d', 'grip', 'sec'),
    Stat('u8', 'splits'),
    Stat('u8', 'impacts'),
    Stat('u8', 'checkpoints'),
    Stat('u8', 'jumps'),
    Stat('u8', 'wings'),
    Stat('u8', 'unk_1', 'unknown'),
    Stat('u8', 'horns'),
    Stat('u8', 'tricks'),
    Stat('u8', 'ev', 'ev'),
    Stat('u8', 'lamps'),
    Stat('u8', 'pumpkins'),
    Stat('u8', 'eggs'),
    Stat('u8', 'unk_2', 'unknown'),
    Stat('u8', 'unk_3', 'unknown'),
    Stat('u8', 'unk_4', 'unknown'),
    Stat('u8', 'cooldowns'),
    Stat('d', 'total', 'sec'),
    Stat('d', 'editor_working', 'sec'),
    Stat('d', 'editor_playing', 'sec'),
))

class PlayerStats(BytesModel):

    version = None
    stats = {}

    def parse(self, dbytes, version=None):
        def read_double():
            return dbytes.read_struct(S_DOUBLE)[0]
        self.version = version
        self.recoverable = True

        self.stats = stats = {}
        for k, stat in STATS.items():
            stats[k] = stat.parse_value(dbytes)

        self.require_equal(SECTION_UNK_1, 4)
        num = dbytes.read_fixed_number(4)
        self.modes_offline = offline_times = []
        for i in range(num):
            offline_times.append(read_double())

        self.require_equal(SECTION_UNK_1, 4)
        num = dbytes.read_fixed_number(4)
        modes_unknown = self.add_unknown(value=[])
        for i in range(num):
            modes_unknown.append(dbytes.read_fixed_number(8))

        self.require_equal(SECTION_UNK_1, 4)
        num = dbytes.read_fixed_number(4)
        self.modes_online = online_times = []
        for i in range(num):
            online_times.append(read_double())

        self.add_unknown(8)
        self.require_equal(SECTION_UNK_1, 4)
        num = dbytes.read_fixed_number(4)
        self.trackmogrify_mods = mods = []
        for i in range(num):
            mods.append(dbytes.read_string())

    def _print_data(self, p):
        p(f"Player stats version: {self.version}")
        s = STATS
        values = self.stats

        def ps(stat, name):
            value = values.get(stat.ident, None)
            if value is None:
                val_str = "None"
            else:
                val_str = stat.format(value)
            p.tree_next_child()
            p(f"{name}: {val_str}")

        ps(s.total, "All play time")
        with p.tree_children(5):
            total_in_modes = sum(self.modes_offline) + sum(self.modes_online)
            p.tree_next_child()
            p(f"Total time in modes: {format_duration_dhms(total_in_modes * 1000)}")
            total_offline = sum(self.modes_offline)
            total_online = sum(self.modes_online)
            p.tree_next_child()
            p(f"Total time in offline modes: {format_duration_dhms(total_offline * 1000)}")
            with p.tree_children(sum(1 for t in self.modes_offline if t >= 0.001)):
                for mode, time in enumerate(self.modes_offline):
                    if time >= 0.001:
                        p.tree_next_child()
                        p(f"Time playing {Mode.to_name(mode)} offline: {format_duration_dhms(time * 1000)}")
            p.tree_next_child()
            p(f"Total time in online modes: {format_duration_dhms(total_online * 1000)}")
            with p.tree_children(sum(1 for t in self.modes_online if t >= 0.001)):
                for mode, time in enumerate(self.modes_online):
                    if time >= 0.001:
                        p.tree_next_child()
                        p(f"Time playing {Mode.to_name(mode)} online: {format_duration_dhms(time * 1000)}")
            ps(s.editor_working, "Time working in editor")
            ps(s.editor_playing, "Time playing in editor")

        total_distance = sum(values.get(stat.ident) for stat in
                             (s.forward, s.reverse, s.air_fly, s.air_nofly))
        ps(s.deaths, "Total deaths")
        with p.tree_children(6):
            ps(s.impact_deaths, "Impact deaths")
            ps(s.killgrid_deaths, "Killgrid deaths")
            ps(s.laser_deaths, "Laser deaths")
            ps(s.overheat_deaths, "Overheat deaths")
            ps(s.reset_deaths, "Reset deaths")
            ps(s.gibs_time, "Gibs time")
        ps(s.impacts, "Impact count")
        ps(s.splits, "Split count")
        ps(s.checkpoints, "Checkpoints hit")
        ps(s.cooldowns, "Cooldowns hit")
        p(f"Distance traveled: {format_distance(total_distance)}")
        with p.tree_children(7):
            ps(s.forward, "Distance driven forward")
            ps(s.reverse, "Distance driven reverse")
            ps(s.air_fly, "Distance airborn flying")
            ps(s.air_nofly, "Distance airborn not flying")
            ps(s.wallride, "Distance wall riding")
            ps(s.ceilingride, "Distance ceiling riding")
            ps(s.grinding, "Distance grinding")
        ps(s.lamps, "Lamps broken")
        ps(s.pumpkins, "Pumpkins smashed")
        ps(s.eggs, "Eggs cracked")
        ps(s.boost, "Boost held down time")
        ps(s.grip, "Grip held down time")
        ps(s.jumps, "Jump count")
        ps(s.wings, "Wings count")
        ps(s.horns, "Horn count")
        if total_in_modes:
            avg_speed = total_distance / total_in_modes * 3.6
        else:
            avg_speed = "None"
        p(f"Average speed: {avg_speed}")

        mods = self.trackmogrify_mods
        if mods:
            p(f"Found trackmogrify mods: {len(mods)}")
            with p.tree_children((len(mods) + 4) // 5):
                for i in range(0, len(mods), 5):
                    mods_str = ', '.join(repr(m) for m in mods[i:i+5])
                    p.tree_next_child()
                    p(f"Found: {mods_str}")

class ProfileProgress(BytesModel):

    level_s2 = None
    num_levels = None
    stats_s2 = None

    def parse(self, dbytes):
        ts = self.require_type(FTYPE_PROFILEPROGRESS)
        self.report_end_pos(ts.data_end)
        s3 = self.require_section(SECTION_UNK_3)
        dbytes.pos = s3.data_end
        while dbytes.pos < ts.data_end:
            section = Section(dbytes)
            if section.ident == SECTION_UNK_2:
                with dbytes.limit(section.data_end):
                    if section.value_id == 0x6A:
                        self.level_s2 = section
                        self.add_unknown(4)
                        self.num_levels = dbytes.read_fixed_number(4)
                        self.add_unknown(4)
                    elif section.value_id == 0x8E:
                        self.stats_s2 = section
            dbytes.pos = section.data_start + section.size

    def iter_levels(self):
        s2 = self.level_s2
        if s2 is None:
            return
        data_end = s2.data_end
        dbytes = self.dbytes
        dbytes.pos = s2.data_start + 20
        with dbytes.limit(data_end):
            num = self.num_levels
            if num:
                for res in LevelProgress.iter_maybe_partial(dbytes, max_pos=data_end):
                    yield res
                    if not res[1]:
                        break
                    num -= 1
                    if num <= 0:
                        break
        self.off_mapname_start = dbytes.pos

    def iter_official_levels(self):
        dbytes = self.dbytes
        dbytes.pos = self.off_mapname_start
        self.require_equal(SECTION_UNK_1, 4)
        num_maps = dbytes.read_fixed_number(4)
        def gen():
            for i in range(num_maps):
                yield dbytes.read_string()
            self.found_tricks_start = dbytes.pos + 36
        return gen(), num_maps

    def iter_tricks(self):
        dbytes = self.dbytes
        dbytes.pos = self.found_tricks_start
        self.require_equal(SECTION_UNK_1, 4)
        num_tricks = dbytes.read_fixed_number(4)
        def gen():
            for i in range(num_tricks):
                yield dbytes.read_string()
            self.somelevel_list_start = dbytes.pos + 18
        return gen(), num_tricks

    def iter_somelevels(self):
        dbytes = self.dbytes
        dbytes.pos = self.somelevel_list_start
        self.require_equal(SECTION_UNK_1, 4)
        num_somelevels = dbytes.read_fixed_number(4)
        def gen():
            for i in range(num_somelevels):
                yield dbytes.read_string()
        return gen(), num_somelevels

    def read_stats(self):
        s2 = self.stats_s2
        if s2 is None:
            return None
        dbytes = self.dbytes
        dbytes.pos = s2.data_start + 16
        with dbytes.limit(s2.data_end):
            return PlayerStats.maybe_partial(dbytes, version=s2.version)

    def _print_data(self, p):
        if self.level_s2:
            p(f"Level progress version: {self.level_s2.version}")
            p(f"Level count: {self.num_levels}")
            with p.tree_children(self.num_levels):
                for level, sane, exc in self.iter_levels():
                    p.tree_next_child()
                    p.print_data_of(level)
            gen, length = self.iter_official_levels()
            if length:
                p(f"Unlocked levels: {length}")
            with p.tree_children((length + 4) // 5):
                # need to ensure that gen is exhausted
                gen = iter(list(gen))
                for _ in range(0, length, 5):
                    p.tree_next_child()
                    l_str = ', '.join(repr(n) for n in islice(gen, 5))
                    p(f"Levels: {l_str}")
            gen, length = self.iter_tricks()
            if length:
                p(f"Found tricks: {length}")
            with p.tree_children((length + 4) // 5):
                # need to ensure that gen is exhausted
                gen = iter(list(gen))
                for _ in range(0, length, 5):
                    p.tree_next_child()
                    t_str = ', '.join(repr(t) for t in islice(gen, 5))
                    p(f"Tricks: {t_str}")
            gen, length = self.iter_somelevels()
            if length:
                p(f"Some levels: {length}")
            with p.tree_children(length):
                for somelevel in gen:
                    p.tree_next_child()
                    p(f"Level: {somelevel!r}")
            stats, sane, exc = self.read_stats()
            if stats:
                p.print_data_of(stats)
        else:
            p("No level progress")

# vim:set sw=4 ts=8 sts=4 et sr ft=python fdm=marker tw=0:
<commit_msg>use islice<commit_after>#!/usr/bin/python
# File:        profileprogress.py
# Description: profileprogress
# Created:     2017-07-05

from collections import OrderedDict
from itertools import islice

from .bytes import (BytesModel, Section, S_DOUBLE,
                    SECTION_UNK_1, SECTION_UNK_2, SECTION_UNK_3)
from .common import format_duration, format_duration_dhms, format_distance
from .constants import Completion, Mode, TIMED_MODES

FTYPE_PROFILEPROGRESS = 'ProfileProgress'

def format_score(mode, score, comp):
    comp_str = Completion.to_name(comp)
    mode_str = Mode.to_name(mode)
    if mode in TIMED_MODES:
        type_str = "time"
        if score < 0:
            score_str = "None"
        else:
            score_str = format_duration(score)
    else:
        type_str = "score"
        if score < 0:
            score_str = "None"
        else:
            score_str = str(score)
    return f"{mode_str} {type_str}: {score_str} ({comp_str})"

class LevelProgress(BytesModel):

    level_path = None
    completion = ()
    scores = ()

    def parse(self, dbytes):
        self.level_path = dbytes.read_string()
        self.recoverable = True
        self.add_unknown(value=dbytes.read_string())
        self.add_unknown(1)

        self.require_equal(SECTION_UNK_1, 4)
        num_levels = dbytes.read_fixed_number(4)
        self.completion = completion = []
        for i in range(num_levels):
            completion.append(dbytes.read_fixed_number(4))

        self.require_equal(SECTION_UNK_1, 4)
        num_levels = dbytes.read_fixed_number(4)
        self.scores = scores = []
        for i in range(num_levels):
            scores.append(dbytes.read_fixed_number(4, signed=True))
        self.add_unknown(8)

    def _print_data(self, p):
        p(f"Level path: {self.level_path!r}")
        for mode, (score, comp) in enumerate(zip(self.scores, self.completion)):
            if comp != Completion.UNPLAYED:
                p(format_score(mode, score, comp))

class Stat(object):

    def __init__(self, type, ident, unit='num', nbytes=None):
        if type == 'd':
            def parse(dbytes):
                return dbytes.read_struct(S_DOUBLE)[0]
        elif type == 'u8':
            def parse(dbytes):
                return dbytes.read_fixed_number(8)
        elif type == 'unk':
            def parse(dbytes):
                return dbytes.read_n(nbytes)
        else:
            raise ValueError(f"invalid type: {type!r}")
        self.parse_value = parse
        if unit == 'sec':
            self.format = lambda v: format_duration_dhms(v * 1000)
        elif unit == 'num':
            self.format = str
        elif unit == 'unknown':
            self.format = str
        elif unit == 'meters':
            self.format = format_distance
        elif unit == 'ev':
            self.format = lambda v: f"{v:,d} eV"
        else:
            raise ValueError(f"invalid unit: {unit!r}")
        self.ident = ident
        self.type = type
        self.unit = unit

    def parse_value(self, dbytes):
        raise NotImplementedError

class AttrOrderedDict(OrderedDict):

    def __getattr__(self, name):
        try:
            return self[name]
        except KeyError:
            raise AttributeError(name)

STATS = AttrOrderedDict((s.ident, s) for s in (
    Stat('u8', 'deaths'),
    Stat('u8', 'laser_deaths'),
    Stat('u8', 'reset_deaths'),
    Stat('u8', 'impact_deaths'),
    Stat('u8', 'overheat_deaths'),
    Stat('u8', 'killgrid_deaths'),
    Stat('d', 'gibs_time', 'sec'),
    Stat('d', 'unk_0', 'meters'),
    Stat('d', 'forward', 'meters'),
    Stat('d', 'reverse', 'meters'),
    Stat('d', 'air_fly', 'meters'),
    Stat('d', 'air_nofly', 'meters'),
    Stat('d', 'wallride', 'meters'),
    Stat('d', 'ceilingride', 'meters'),
    Stat('d', 'grinding', 'meters'),
    Stat('d', 'boost', 'sec'),
    Stat('d', 'grip', 'sec'),
    Stat('u8', 'splits'),
    Stat('u8', 'impacts'),
    Stat('u8', 'checkpoints'),
    Stat('u8', 'jumps'),
    Stat('u8', 'wings'),
    Stat('u8', 'unk_1', 'unknown'),
    Stat('u8', 'horns'),
    Stat('u8', 'tricks'),
    Stat('u8', 'ev', 'ev'),
    Stat('u8', 'lamps'),
    Stat('u8', 'pumpkins'),
    Stat('u8', 'eggs'),
    Stat('u8', 'unk_2', 'unknown'),
    Stat('u8', 'unk_3', 'unknown'),
    Stat('u8', 'unk_4', 'unknown'),
    Stat('u8', 'cooldowns'),
    Stat('d', 'total', 'sec'),
    Stat('d', 'editor_working', 'sec'),
    Stat('d', 'editor_playing', 'sec'),
))

class PlayerStats(BytesModel):

    version = None
    stats = {}

    def parse(self, dbytes, version=None):
        def read_double():
            return dbytes.read_struct(S_DOUBLE)[0]
        self.version = version
        self.recoverable = True

        self.stats = stats = {}
        for k, stat in STATS.items():
            stats[k] = stat.parse_value(dbytes)

        self.require_equal(SECTION_UNK_1, 4)
        num = dbytes.read_fixed_number(4)
        self.modes_offline = offline_times = []
        for i in range(num):
            offline_times.append(read_double())

        self.require_equal(SECTION_UNK_1, 4)
        num = dbytes.read_fixed_number(4)
        modes_unknown = self.add_unknown(value=[])
        for i in range(num):
            modes_unknown.append(dbytes.read_fixed_number(8))

        self.require_equal(SECTION_UNK_1, 4)
        num = dbytes.read_fixed_number(4)
        self.modes_online = online_times = []
        for i in range(num):
            online_times.append(read_double())

        self.add_unknown(8)
        self.require_equal(SECTION_UNK_1, 4)
        num = dbytes.read_fixed_number(4)
        self.trackmogrify_mods = mods = []
        for i in range(num):
            mods.append(dbytes.read_string())

    def _print_data(self, p):
        p(f"Player stats version: {self.version}")
        s = STATS
        values = self.stats

        def ps(stat, name):
            value = values.get(stat.ident, None)
            if value is None:
                val_str = "None"
            else:
                val_str = stat.format(value)
            p.tree_next_child()
            p(f"{name}: {val_str}")

        ps(s.total, "All play time")
        with p.tree_children(5):
            total_in_modes = sum(self.modes_offline) + sum(self.modes_online)
            p.tree_next_child()
            p(f"Total time in modes: {format_duration_dhms(total_in_modes * 1000)}")
            total_offline = sum(self.modes_offline)
            total_online = sum(self.modes_online)
            p.tree_next_child()
            p(f"Total time in offline modes: {format_duration_dhms(total_offline * 1000)}")
            with p.tree_children(sum(1 for t in self.modes_offline if t >= 0.001)):
                for mode, time in enumerate(self.modes_offline):
                    if time >= 0.001:
                        p.tree_next_child()
                        p(f"Time playing {Mode.to_name(mode)} offline: {format_duration_dhms(time * 1000)}")
            p.tree_next_child()
            p(f"Total time in online modes: {format_duration_dhms(total_online * 1000)}")
            with p.tree_children(sum(1 for t in self.modes_online if t >= 0.001)):
                for mode, time in enumerate(self.modes_online):
                    if time >= 0.001:
                        p.tree_next_child()
                        p(f"Time playing {Mode.to_name(mode)} online: {format_duration_dhms(time * 1000)}")
            ps(s.editor_working, "Time working in editor")
            ps(s.editor_playing, "Time playing in editor")

        total_distance = sum(values.get(stat.ident) for stat in
                             (s.forward, s.reverse, s.air_fly, s.air_nofly))
        ps(s.deaths, "Total deaths")
        with p.tree_children(6):
            ps(s.impact_deaths, "Impact deaths")
            ps(s.killgrid_deaths, "Killgrid deaths")
            ps(s.laser_deaths, "Laser deaths")
            ps(s.overheat_deaths, "Overheat deaths")
            ps(s.reset_deaths, "Reset deaths")
            ps(s.gibs_time, "Gibs time")
        ps(s.impacts, "Impact count")
        ps(s.splits, "Split count")
        ps(s.checkpoints, "Checkpoints hit")
        ps(s.cooldowns, "Cooldowns hit")
        p(f"Distance traveled: {format_distance(total_distance)}")
        with p.tree_children(7):
            ps(s.forward, "Distance driven forward")
            ps(s.reverse, "Distance driven reverse")
            ps(s.air_fly, "Distance airborn flying")
            ps(s.air_nofly, "Distance airborn not flying")
            ps(s.wallride, "Distance wall riding")
            ps(s.ceilingride, "Distance ceiling riding")
            ps(s.grinding, "Distance grinding")
        ps(s.lamps, "Lamps broken")
        ps(s.pumpkins, "Pumpkins smashed")
        ps(s.eggs, "Eggs cracked")
        ps(s.boost, "Boost held down time")
        ps(s.grip, "Grip held down time")
        ps(s.jumps, "Jump count")
        ps(s.wings, "Wings count")
        ps(s.horns, "Horn count")
        if total_in_modes:
            avg_speed = total_distance / total_in_modes * 3.6
        else:
            avg_speed = "None"
        p(f"Average speed: {avg_speed}")

        mods = self.trackmogrify_mods
        if mods:
            p(f"Found trackmogrify mods: {len(mods)}")
            with p.tree_children((len(mods) + 4) // 5):
                for i in range(0, len(mods), 5):
                    mods_str = ', '.join(repr(m) for m in islice(mods, i, i + 5))
                    p.tree_next_child()
                    p(f"Found: {mods_str}")

class ProfileProgress(BytesModel):

    level_s2 = None
    num_levels = None
    stats_s2 = None

    def parse(self, dbytes):
        ts = self.require_type(FTYPE_PROFILEPROGRESS)
        self.report_end_pos(ts.data_end)
        s3 = self.require_section(SECTION_UNK_3)
        dbytes.pos = s3.data_end
        while dbytes.pos < ts.data_end:
            section = Section(dbytes)
            if section.ident == SECTION_UNK_2:
                with dbytes.limit(section.data_end):
                    if section.value_id == 0x6A:
                        self.level_s2 = section
                        self.add_unknown(4)
                        self.num_levels = dbytes.read_fixed_number(4)
                        self.add_unknown(4)
                    elif section.value_id == 0x8E:
                        self.stats_s2 = section
            dbytes.pos = section.data_start + section.size

    def iter_levels(self):
        s2 = self.level_s2
        if s2 is None:
            return
        data_end = s2.data_end
        dbytes = self.dbytes
        dbytes.pos = s2.data_start + 20
        with dbytes.limit(data_end):
            num = self.num_levels
            if num:
                for res in LevelProgress.iter_maybe_partial(dbytes, max_pos=data_end):
                    yield res
                    if not res[1]:
                        break
                    num -= 1
                    if num <= 0:
                        break
        self.off_mapname_start = dbytes.pos

    def iter_official_levels(self):
        dbytes = self.dbytes
        dbytes.pos = self.off_mapname_start
        self.require_equal(SECTION_UNK_1, 4)
        num_maps = dbytes.read_fixed_number(4)
        def gen():
            for i in range(num_maps):
                yield dbytes.read_string()
            self.found_tricks_start = dbytes.pos + 36
        return gen(), num_maps

    def iter_tricks(self):
        dbytes = self.dbytes
        dbytes.pos = self.found_tricks_start
        self.require_equal(SECTION_UNK_1, 4)
        num_tricks = dbytes.read_fixed_number(4)
        def gen():
            for i in range(num_tricks):
                yield dbytes.read_string()
            self.somelevel_list_start = dbytes.pos + 18
        return gen(), num_tricks

    def iter_somelevels(self):
        dbytes = self.dbytes
        dbytes.pos = self.somelevel_list_start
        self.require_equal(SECTION_UNK_1, 4)
        num_somelevels = dbytes.read_fixed_number(4)
        def gen():
            for i in range(num_somelevels):
                yield dbytes.read_string()
        return gen(), num_somelevels

    def read_stats(self):
        s2 = self.stats_s2
        if s2 is None:
            return None
        dbytes = self.dbytes
        dbytes.pos = s2.data_start + 16
        with dbytes.limit(s2.data_end):
            return PlayerStats.maybe_partial(dbytes, version=s2.version)

    def _print_data(self, p):
        if self.level_s2:
            p(f"Level progress version: {self.level_s2.version}")
            p(f"Level count: {self.num_levels}")
            with p.tree_children(self.num_levels):
                for level, sane, exc in self.iter_levels():
                    p.tree_next_child()
                    p.print_data_of(level)
            gen, length = self.iter_official_levels()
            if length:
                p(f"Unlocked levels: {length}")
            with p.tree_children((length + 4) // 5):
                # need to ensure that gen is exhausted
                gen = iter(list(gen))
                for _ in range(0, length, 5):
                    p.tree_next_child()
                    l_str = ', '.join(repr(n) for n in islice(gen, 5))
                    p(f"Levels: {l_str}")
            gen, length = self.iter_tricks()
            if length:
                p(f"Found tricks: {length}")
            with p.tree_children((length + 4) // 5):
                # need to ensure that gen is exhausted
                gen = iter(list(gen))
                for _ in range(0, length, 5):
                    p.tree_next_child()
                    t_str = ', '.join(repr(t) for t in islice(gen, 5))
                    p(f"Tricks: {t_str}")
            gen, length = self.iter_somelevels()
            if length:
                p(f"Some levels: {length}")
            with p.tree_children(length):
                for somelevel in gen:
                    p.tree_next_child()
                    p(f"Level: {somelevel!r}")
            stats, sane, exc = self.read_stats()
            if stats:
                p.print_data_of(stats)
        else:
            p("No level progress")

# vim:set sw=4 ts=8 sts=4 et sr ft=python fdm=marker tw=0:
<|endoftext|>


<commit_before>import os

from .JAVA import Executor as JavaExecutor
from executors.utils import test_executor
from judgeenv import env

class Executor(JavaExecutor):
    JAVA = 'java8'
    JAVAC = 'javac8'

def initialize():
    if 'java8' not in env['runtime']:
        return False
    if not os.path.isfile(env['runtime']['java8']):
        return False
    return test_executor('JAVA8', Executor, '''\
public class self_test {
    public static void main(String[] args) {
        System.out.println("Hello, World!");
    }
}
''', problem='self_test')
<commit_msg>It would help to actually check for Java 8.<commit_after>import os

from .JAVA import Executor as JavaExecutor
from executors.utils import test_executor
from judgeenv import env

class Executor(JavaExecutor):
    JAVA = 'java8'
    JAVAC = 'javac8'

def initialize():
    if 'java8' not in env['runtime']:
        return False
    if not os.path.isfile(env['runtime']['java8']):
        return False
    return test_executor('JAVA8', Executor, '''\
public class self_test {
    public static void run(Runnable target) {
        target.run();
    }

    public static void main(String[] args) {
        run(() -> System.out.println("Hello, World!"));
    }
}
''', problem='self_test')
<|endoftext|>


<commit_before>from djangosanetesting.cases import SeleniumTestCase
from django.core.management import call_command
from helpers import silence_print, unsilence_print
class QiSeleniumTestCase(SeleniumTestCase):
    # selenium_fixtures = []

    def setUp(self, *args, **kwargs):
        self.verificationErrors = []

    def tearDown(self, *args, **kwargs):
        self.assertEqual([], self.verificationErrors)

class QiConservativeSeleniumTestCase(QiSeleniumTestCase):

    def tearDown(self, *args, **kwargs):
        super(QiConservativeSeleniumTestCase,self).tearDown(*args, **kwargs)
        _p = silence_print()
        call_command('flush', interactive=False)
        unsilence_print(_p)

<commit_msg>js_refresh to handle hashes<commit_after>from djangosanetesting.cases import SeleniumTestCase
from django.core.management import call_command
from helpers import silence_print, unsilence_print
class QiSeleniumTestCase(SeleniumTestCase):
    # selenium_fixtures = []

    def setUp(self, *args, **kwargs):
        self.verificationErrors = []

    def tearDown(self, *args, **kwargs):
        self.assertEqual([], self.verificationErrors)

    def js_refresh(self):
        sel = self.selenium
        sel.get_eval("window.location.href")
        sel.get_eval("window.location.href=window.location.href;")
        time.sleep(4)

class QiConservativeSeleniumTestCase(QiSeleniumTestCase):

    def tearDown(self, *args, **kwargs):
        super(QiConservativeSeleniumTestCase,self).tearDown(*args, **kwargs)
        _p = silence_print()
        call_command('flush', interactive=False)
        unsilence_print(_p)

<|endoftext|>