<commit_before>'''
Main wrapper script for Genomel Haplotype caller workflow
'''
import os
import time
import argparse
import logging
import sys
import uuid
import tempfile
import utils.s3
import utils.pipeline
import datetime
import socket
import json

import postgres.status
import postgres.utils
import postgres.mixins

from sqlalchemy.exc import NoSuchTableError

def is_nat(x):
    '''
    Checks that a value is a natural number.
    '''
    if int(x) > 0:
        return int(x)
    raise argparse.ArgumentTypeError('%s must be positive, non-zero' % x)

def get_args():
    '''
    Loads the parser
    '''
    # Main parser
    parser = argparse.ArgumentParser(description="Genomel Haplotype caller Pipeline")
    # Args
    required = parser.add_argument_group("Required input parameters")
    # Metadata from input table
    required.add_argument("--input_id", default=None, help="INPUT_ID, internal production id.")
    required.add_argument("--project", default=None, help="PROJECT, PDC project name.")
    required.add_argument('--input_table', required=True, help='Table from where input_id was retrieved')
    required.add_argument("--md5", default=None, help="MD5, md5 of input file.")
    required.add_argument("--s3_url", default=None, help="S3_URL, s3 url of the input.")
    required.add_argument("--s3_profile", required=True, help="S3 profile name for project tenant.")
    required.add_argument("--s3_endpoint", required=True, help="S3 endpoint url for project tenant.")
    # Parameters for pipeline
    required.add_argument("--basedir", default="/mnt/SCRATCH/", help="Base directory for computations.")
    required.add_argument("--refdir", required=True, help="Path to reference directory.")
    required.add_argument("--cwl", required=True, help="Path to Genomel Recalibration CWL workflow yaml.")
    required.add_argument("--s3dir", default="s3://", help="S3 dir for uploading output files.")
    required.add_argument('--output_id', required=True, help='UUID for the output')

    required.add_argument('--java_heap', required=True, help='Java heap memory limit.')
    required.add_argument('--thread_count', type=is_nat, default=8, help='Threads count.')

    return parser.parse_args()

def run_pipeline(args, statusclass, metricsclass):
    '''
    Executes the CWL pipeline and record status/metrics tables
    '''
    if not os.path.isdir(args.basedir):
        raise Exception("Could not find path to base directory: %s" %args.basedir)

    # Get output uuid
    output_id = args.output_id
    # Get hostname
    hostname = socket.gethostname()
    # Get datetime start
    datetime_start = str(datetime.datetime.now())

    # Create directory structure
    jobdir = tempfile.mkdtemp(prefix="hc_%s_" % str(output_id), dir=args.basedir)
    workdir = tempfile.mkdtemp(prefix="workdir_", dir=jobdir)
    inputdir = tempfile.mkdtemp(prefix="input_", dir=jobdir)
    resultdir = tempfile.mkdtemp(prefix="result_", dir=jobdir)
    refdir = args.refdir

    # Setup logger
    log_file = os.path.join(resultdir, "%s.genomel.hc.cwl.log" % str(output_id))
    logger = utils.pipeline.setup_logging(logging.INFO, str(output_id), log_file)

    # Logging inputs
    logger.info("hostname: %s" % (hostname))
    logger.info("input_id: %s" % (args.input_id))
    logger.info("project: %s" % (args.project))
    logger.info("md5: %s" % (args.md5))
    logger.info("datetime_start: %s" % (datetime_start))

    # Setup start point
    cwl_start = time.time()

    # Getting refs
    logger.info("getting resources")
    reference_data        = utils.pipeline.load_reference_json('etc/reference.json')
    reference_fasta_path  = os.path.join(refdir, reference_data["reference_fasta"])
    reference_dbsnp       = os.path.join(refdir, reference_data["reference_dbsnp"])
    reference_intervals   = os.path.join(refdir, reference_data["reference_intervals"])
    postgres_config       = os.path.join(refdir, reference_data["pg_config"])

    # Logging pipeline info
    cwl_version    = reference_data["cwl_version"]
    docker_version = reference_data["docker_version"]
    logger.info("cwl_version: %s" % (cwl_version))
    logger.info("docker_version: %s" % (docker_version))

    # Download input bam and index
    input_url = args.s3_url
    input_bam = os.path.join(inputdir, os.path.basename(input_url))
    input_index_url = input_url.replace('.bam','.bai')
    project_s3_profile = args.s3_profile
    project_s3_endpoint_url = args.s3_endpoint
    download_exit_code = utils.s3.aws_s3_get(logger, input_url, inputdir,
                                             project_s3_profile, project_s3_endpoint_url, recursive=False)
    download_exit_code = utils.s3.aws_s3_get(logger, input_index_url, inputdir,
                                             project_s3_profile, project_s3_endpoint_url, recursive=False)

    download_end_time = time.time()
    download_time = download_end_time - cwl_start
    if not (download_exit_code != 0 or str(utils.pipeline.get_md5(input_bam)) != args.md5):
        logger.info("Download input %s successfully in %s, and md5 matches. Input bam is %s." % (args.input_id, download_time, input_bam))
    else:
        cwl_elapsed = download_time
        datetime_end = str(datetime.datetime.now())
        engine = postgres.utils.get_db_engine(postgres_config)
        postgres.utils.set_download_error(download_exit_code, logger, engine,
                                          output_id, [args.input_id], output_id,
                                          datetime_start, datetime_end,
                                          hostname, cwl_version, docker_version,
                                          download_time, cwl_elapsed, statusclass, metricsclass)
        # Exit
        sys.exit(download_exit_code)

    # Define output
    output_gvcf = os.path.join(workdir, str(output_id) + '.g.vcf.gz')

    # Create input json
    input_json_file = os.path.join(resultdir, '{0}.genomel.hc.inputs.json'.format(str(output_id)))
    input_json_data = {
      "java_opts": args.java_heap,
      "input_bam_path": {"class": "File", "path": input_bam},
      "reference_fasta_path": {"class": "File", "path": reference_fasta_path},
      "reference_snp_path": {"class": "File", "path": reference_dbsnp},
      "interval_file_path": {"class": "File", "path": reference_intervals},
      "output_gvcf_name": os.path.basename(output_gvcf),
      "thread_count": str(args.thread_count)
    }
    with open(input_json_file, 'wt') as o:
        json.dump(input_json_data, o, indent=4)
    logger.info("Preparing input json: %s" % (input_json_file))

    # Run CWL
    os.chdir(workdir)
    logger.info('Running CWL workflow')
    cmd = ['/usr/bin/time', '-v',
           '/home/ubuntu/.virtualenvs/p2/bin/cwltool',
           "--debug",
           "--tmpdir-prefix", inputdir,
           "--tmp-outdir-prefix", workdir,
           args.cwl,
           input_json_file]
    cwl_exit = utils.pipeline.run_command(cmd, logger)
    cwl_failure = False
    if cwl_exit:
        cwl_failure = True

    # Get md5 and file size
    md5 = utils.pipeline.get_md5(output_gvcf)
    file_size = utils.pipeline.get_file_size(output_gvcf)

    # Upload output
    upload_start = time.time()
    output_tbi = output_gvcf + '.tbi'
    upload_dir_location = os.path.join(args.s3dir, str(output_id))
    upload_gvcf_location = os.path.join(upload_dir_location, os.path.basename(output_gvcf))
    upload_tbi_location = os.path.join(upload_dir_location, os.path.basename(output_tbi))
    logger.info("Uploading workflow output to %s" % (upload_gvcf_location))
    upload_exit = utils.s3.aws_s3_put(logger, upload_gvcf_location, output_gvcf, args.s3_profile, args.s3_endpoint, recursive=False)
    upload_exit = utils.s3.aws_s3_put(logger, upload_tbi_location, output_tbi, args.s3_profile, args.s3_endpoint, recursive=False)

    # Establish connection with database
    engine = postgres.utils.get_db_engine(postgres_config)

    # Calculate times
    cwl_end = time.time()
    upload_time = cwl_end - upload_start
    cwl_elapsed = cwl_end - cwl_start
    datetime_end = str(datetime.datetime.now())
    logger.info("datetime_end: %s" % (datetime_end))

    # Get status info
    logger.info("Get status/metrics info")
    status, loc = postgres.status.get_status(upload_exit, cwl_exit, upload_gvcf_location, upload_dir_location, logger)

    # Get metrics info
    time_metrics = utils.pipeline.get_time_metrics(log_file)

    # Set status table
    logger.info("Updating status")
    postgres.utils.add_pipeline_status(engine, output_id, [args.input_id], args.input_table, output_id,
                                       status, loc, datetime_start, datetime_end,
                                       md5, file_size, hostname, cwl_version, docker_version, statusclass)
    # Set metrics table
    logger.info("Updating metrics")
    postgres.utils.add_pipeline_metrics(engine, output_id, [args.input_id], args.input_table, download_time,
                                        upload_time, args.thread_count, cwl_elapsed,
                                        time_metrics['system_time'],
                                        time_metrics['user_time'],
                                        time_metrics['wall_clock'],
                                        time_metrics['percent_of_cpu'],
                                        time_metrics['maximum_resident_set_size'],
                                        status, metricsclass)

    # Remove job directories, upload final log file
    logger.info("Uploading main log file")
    utils.s3.aws_s3_put(logger, upload_dir_location + '/' + os.path.basename(log_file), log_file, args.s3_profile, args.s3_endpoint, recursive=False)
    utils.pipeline.remove_dir(jobdir)

if __name__ == '__main__':

    # Get args
    args = get_args()
    project = args.project.lower()

    # Setup postgres classes for tables
    class CallerStatus(postgres.mixins.StatusTypeMixin, postgres.utils.Base):
        __tablename__ = project + '_hc_cwl_status'
    class CallerMetrics(postgres.mixins.MetricsTypeMixin, postgres.utils.Base):
        __tablename__ = project + '_hc_cwl_metrics'

    # Run pipeline
    run_pipeline(args, CallerStatus, CallerMetrics)
<commit_msg>feat(slurm): update project in hc<commit_after>'''
Main wrapper script for Genomel Haplotype caller workflow
'''
import os
import time
import argparse
import logging
import sys
import uuid
import tempfile
import utils.s3
import utils.pipeline
import datetime
import socket
import json

import postgres.status
import postgres.utils
import postgres.mixins

from sqlalchemy.exc import NoSuchTableError

def is_nat(x):
    '''
    Checks that a value is a natural number.
    '''
    if int(x) > 0:
        return int(x)
    raise argparse.ArgumentTypeError('%s must be positive, non-zero' % x)

def get_args():
    '''
    Loads the parser
    '''
    # Main parser
    parser = argparse.ArgumentParser(description="Genomel Haplotype caller Pipeline")
    # Args
    required = parser.add_argument_group("Required input parameters")
    # Metadata from input table
    required.add_argument("--input_id", default=None, help="INPUT_ID, internal production id.")
    required.add_argument("--project", default=None, help="PROJECT, PDC project name.")
    required.add_argument('--input_table', required=True, help='Table from where input_id was retrieved')
    required.add_argument("--md5", default=None, help="MD5, md5 of input file.")
    required.add_argument("--s3_url", default=None, help="S3_URL, s3 url of the input.")
    required.add_argument("--s3_profile", required=True, help="S3 profile name for project tenant.")
    required.add_argument("--s3_endpoint", required=True, help="S3 endpoint url for project tenant.")
    # Parameters for pipeline
    required.add_argument("--basedir", default="/mnt/SCRATCH/", help="Base directory for computations.")
    required.add_argument("--refdir", required=True, help="Path to reference directory.")
    required.add_argument("--cwl", required=True, help="Path to Genomel Recalibration CWL workflow yaml.")
    required.add_argument("--s3dir", default="s3://", help="S3 dir for uploading output files.")
    required.add_argument('--output_id', required=True, help='UUID for the output')

    required.add_argument('--java_heap', required=True, help='Java heap memory limit.')
    required.add_argument('--thread_count', type=is_nat, default=8, help='Threads count.')

    return parser.parse_args()

def run_pipeline(args, statusclass, metricsclass):
    '''
    Executes the CWL pipeline and record status/metrics tables
    '''
    if not os.path.isdir(args.basedir):
        raise Exception("Could not find path to base directory: %s" %args.basedir)

    # Get output uuid
    output_id = args.output_id
    # Get hostname
    hostname = socket.gethostname()
    # Get datetime start
    datetime_start = str(datetime.datetime.now())

    # Create directory structure
    jobdir = tempfile.mkdtemp(prefix="hc_%s_" % str(output_id), dir=args.basedir)
    workdir = tempfile.mkdtemp(prefix="workdir_", dir=jobdir)
    inputdir = tempfile.mkdtemp(prefix="input_", dir=jobdir)
    resultdir = tempfile.mkdtemp(prefix="result_", dir=jobdir)
    refdir = args.refdir

    # Setup logger
    log_file = os.path.join(resultdir, "%s.genomel.hc.cwl.log" % str(output_id))
    logger = utils.pipeline.setup_logging(logging.INFO, str(output_id), log_file)

    # Logging inputs
    logger.info("hostname: %s" % (hostname))
    logger.info("input_id: %s" % (args.input_id))
    logger.info("project: %s" % (args.project))
    logger.info("md5: %s" % (args.md5))
    logger.info("datetime_start: %s" % (datetime_start))

    # Setup start point
    cwl_start = time.time()

    # Getting refs
    logger.info("getting resources")
    reference_data        = utils.pipeline.load_reference_json('etc/reference.json')
    reference_fasta_path  = os.path.join(refdir, reference_data["reference_fasta"])
    reference_dbsnp       = os.path.join(refdir, reference_data["reference_dbsnp"])
    reference_intervals   = os.path.join(refdir, reference_data["reference_intervals"])
    postgres_config       = os.path.join(refdir, reference_data["pg_config"])

    # Logging pipeline info
    cwl_version    = reference_data["cwl_version"]
    docker_version = reference_data["docker_version"]
    logger.info("cwl_version: %s" % (cwl_version))
    logger.info("docker_version: %s" % (docker_version))

    # Download input bam and index
    input_url = args.s3_url
    input_bam = os.path.join(inputdir, os.path.basename(input_url))
    input_index_url = input_url.replace('.bam','.bai')
    project_s3_profile = args.s3_profile
    project_s3_endpoint_url = args.s3_endpoint
    download_exit_code = utils.s3.aws_s3_get(logger, input_url, inputdir,
                                             project_s3_profile, project_s3_endpoint_url, recursive=False)
    download_exit_code = utils.s3.aws_s3_get(logger, input_index_url, inputdir,
                                             project_s3_profile, project_s3_endpoint_url, recursive=False)

    download_end_time = time.time()
    download_time = download_end_time - cwl_start
    if not (download_exit_code != 0 or str(utils.pipeline.get_md5(input_bam)) != args.md5):
        logger.info("Download input %s successfully in %s, and md5 matches. Input bam is %s." % (args.input_id, download_time, input_bam))
    else:
        cwl_elapsed = download_time
        datetime_end = str(datetime.datetime.now())
        engine = postgres.utils.get_db_engine(postgres_config)
        postgres.utils.set_download_error(download_exit_code, logger, engine,
                                          output_id, [args.input_id], output_id,
                                          datetime_start, datetime_end,
                                          hostname, cwl_version, docker_version,
                                          download_time, cwl_elapsed, statusclass, metricsclass)
        # Exit
        sys.exit(download_exit_code)

    # Define output
    output_gvcf = os.path.join(workdir, str(output_id) + '.g.vcf.gz')

    # Create input json
    input_json_file = os.path.join(resultdir, '{0}.genomel.hc.inputs.json'.format(str(output_id)))
    input_json_data = {
      "java_opts": args.java_heap,
      "input_bam_path": {"class": "File", "path": input_bam},
      "reference_fasta_path": {"class": "File", "path": reference_fasta_path},
      "reference_snp_path": {"class": "File", "path": reference_dbsnp},
      "interval_file_path": {"class": "File", "path": reference_intervals},
      "output_gvcf_name": os.path.basename(output_gvcf),
      "thread_count": str(args.thread_count)
    }
    with open(input_json_file, 'wt') as o:
        json.dump(input_json_data, o, indent=4)
    logger.info("Preparing input json: %s" % (input_json_file))

    # Run CWL
    os.chdir(workdir)
    logger.info('Running CWL workflow')
    cmd = ['/usr/bin/time', '-v',
           '/home/ubuntu/.virtualenvs/p2/bin/cwltool',
           "--debug",
           "--tmpdir-prefix", inputdir,
           "--tmp-outdir-prefix", workdir,
           args.cwl,
           input_json_file]
    cwl_exit = utils.pipeline.run_command(cmd, logger)
    cwl_failure = False
    if cwl_exit:
        cwl_failure = True

    # Get md5 and file size
    md5 = utils.pipeline.get_md5(output_gvcf)
    file_size = utils.pipeline.get_file_size(output_gvcf)

    # Upload output
    upload_start = time.time()
    output_tbi = output_gvcf + '.tbi'
    upload_dir_location = os.path.join(args.s3dir, str(output_id))
    upload_gvcf_location = os.path.join(upload_dir_location, os.path.basename(output_gvcf))
    upload_tbi_location = os.path.join(upload_dir_location, os.path.basename(output_tbi))
    logger.info("Uploading workflow output to %s" % (upload_gvcf_location))
    upload_exit = utils.s3.aws_s3_put(logger, upload_gvcf_location, output_gvcf, args.s3_profile, args.s3_endpoint, recursive=False)
    upload_exit = utils.s3.aws_s3_put(logger, upload_tbi_location, output_tbi, args.s3_profile, args.s3_endpoint, recursive=False)

    # Establish connection with database
    engine = postgres.utils.get_db_engine(postgres_config)

    # Calculate times
    cwl_end = time.time()
    upload_time = cwl_end - upload_start
    cwl_elapsed = cwl_end - cwl_start
    datetime_end = str(datetime.datetime.now())
    logger.info("datetime_end: %s" % (datetime_end))

    # Get status info
    logger.info("Get status/metrics info")
    status, loc = postgres.status.get_status(upload_exit, cwl_exit, upload_gvcf_location, upload_dir_location, logger)

    # Get metrics info
    time_metrics = utils.pipeline.get_time_metrics(log_file)

    # Set status table
    logger.info("Updating status")
    postgres.utils.add_pipeline_status(engine, args.project, output_id, [args.input_id], args.input_table, output_id,
                                       status, loc, datetime_start, datetime_end,
                                       md5, file_size, hostname, cwl_version, docker_version, statusclass)
    # Set metrics table
    logger.info("Updating metrics")
    postgres.utils.add_pipeline_metrics(engine, output_id, [args.input_id], args.input_table, download_time,
                                        upload_time, args.thread_count, cwl_elapsed,
                                        time_metrics['system_time'],
                                        time_metrics['user_time'],
                                        time_metrics['wall_clock'],
                                        time_metrics['percent_of_cpu'],
                                        time_metrics['maximum_resident_set_size'],
                                        status, metricsclass)

    # Remove job directories, upload final log file
    logger.info("Uploading main log file")
    utils.s3.aws_s3_put(logger, upload_dir_location + '/' + os.path.basename(log_file), log_file, args.s3_profile, args.s3_endpoint, recursive=False)
    utils.pipeline.remove_dir(jobdir)

if __name__ == '__main__':

    # Get args
    args = get_args()
    project = args.project.lower()
    program = project.split('-')[0]

    # Setup postgres classes for tables
    class CallerStatus(postgres.mixins.StatusTypeMixin, postgres.utils.Base):
        __tablename__ = project + '_hc_cwl_status'
    class CallerMetrics(postgres.mixins.MetricsTypeMixin, postgres.utils.Base):
        __tablename__ = project + '_hc_cwl_metrics'

    # Run pipeline
    run_pipeline(args, CallerStatus, CallerMetrics)
<|endoftext|>


<commit_before>import json
import sqlite3
import time
import datetime
import requests
from time import sleep
from collections import defaultdict, deque
from math import exp
import numpy
from scipy.optimize import fsolve

# Helper function for dealing with time stamps
time_format = "%a %b %d %H:%M:%S %Z %Y"
def formatTime(s, time_format) :
    return datetime.datetime(*time.strptime(s, time_format)[:6])

# Rate of change estimator for irregularly sampled data
# we will want to solve for x
#
# Cho and Garcia Molina, 2003, Estimating Frequency of Change
# http://dl.acm.org/citation.cfm?id=857170
def irregularCGM(intervals, Z) :
    def iCGM(x) :
        L = 0
        K = 0
        for i, z in enumerate(Z) :
            if z == 1 :
                L = L + intervals[i]/(exp(x*intervals[i])-1)
            else :
                K = K + intervals[i]
        return L - K
    return iCGM

# Set up DB
con = sqlite3.connect("plow.db")
cur = con.cursor()

cur.execute("""
CREATE TABLE IF NOT EXISTS route_points
(object_id INTEGER,
 posting_time DATETIME,
 X DOUBLE,
 Y DOUBLE,
 UNIQUE(object_id, posting_time) ON CONFLICT REPLACE)""")

cur.execute("""
CREATE TABLE IF NOT EXISTS assets
(object_id INTEGER,
 asset_name TEXT,
 asset_type TEXT,
 PRIMARY KEY(object_id) ON CONFLICT REPLACE)""")

con.commit()
con.close()

# The feed for City Of Chicago's Plow Data
gps_data_url = "https://gisapps.cityofchicago.org/snowplows/services/trackingservice/getPositions"

# We'll use these variables to keep track of whether we observe a new
# plow position. 30 observations should be sufficient for a reasonable
# estimate.
previous_posting_time = datetime.datetime(1,1,1)
last_posting_time = datetime.datetime(1,1,1)
update_history = defaultdict( lambda: deque([], 30))
intervals = deque([],30)

# We want adjust our sampling intervals depending upon our estimated
# rate of updates for the plows
sampling_frequency = 10

# Even if we are unable to get any data, we need to keep track of the
# time we spent on that attempt
fault_sleep = 60
faults = 0

while True:
    # Try to handle anything besides a well-formed json response
    try:
        payload = {"TrackingInput":{"envelope":{"minX":0,"minY":0,"maxX":0,"maxY":0},"trackingDuration":15}}
        response = requests.post(gps_data_url, data=json.dumps(payload))
    except Exception as e :
        print e
        sleep(fault_sleep)
        faults += 1
        continue
    if "Sorry, servers are currently down" in response:
        print "Sorry, servers are currently down"
        sleep(fault_sleep)
        faults += 1
        continue
    if len(response.json()['TrackingResponse']['locationList']) == 0:
        print "No traces present"
        sleep(fault_sleep)
        faults += 1
        continue

    # This is inside the loop as an act of perhaps irrational
    # defensive programming, as the script stopped updating the db for
    # no apparent reasons and without throwing an error.
    con = sqlite3.connect("plow.db")
    cur = con.cursor()

    try:
        read_data = response.json()
    except:
        print "Can't parse JSON feed"
        read_data = {'TrackingResponse': {'locationList': {}}}

    updates = 0
    for route_point in read_data['TrackingResponse']['locationList'] :

        try:
            (object_id,
             asset_name,
             asset_type,
             posting_time,
             x,
             y) = (route_point['objectid'],
                   route_point['assetName'],
                   route_point['assetType'],
                   route_point['postingTimeFormatted'],
                   route_point['XCoord'],
                   route_point['YCoord'])
        except TypeError:
            continue

        posting_time = formatTime(posting_time, time_format)

        # Insert Data into DB
        cur.execute("""insert into route_points (object_id, posting_time, X, Y)
                       values (?, ?, ?, ?)""",
                    (object_id, posting_time, x, y))
        cur.execute("""insert into assets (object_id, asset_name, asset_type)
                       values (?, ?, ?)""",
                    (object_id, asset_name, asset_type))

        # Update whether or not we observed a new position for every
        # plow
        if posting_time > last_posting_time :
            last_posting_time = posting_time

        if posting_time > previous_posting_time :
            update_history[object_id].append(1)
            updates += 1
        else :
            update_history[object_id].append(0)

    con.commit()
    con.close()

    # Add the sampling interval
    previous_posting_time = last_posting_time
    if faults :
        intervals.append(sampling_frequency
                         + faults*fault_sleep)
        faults = 0
    else :
        intervals.append(sampling_frequency)

    # Estimate the update rate
    r = []
    for object_id in update_history :
        z = sum(update_history[object_id])
        if z > 0 :
            icgm = irregularCGM(intervals, update_history[object_id])
            r.append(fsolve(icgm, .01))

    # Assuming that updates are drawn from a Poisson distribution,
    # then with some probability, we will observe LESS than 2 events
    # in this period. This does not mean the probability that we will
    # observe 1 update, as it is very likely that we will observe no
    # update.
    #
    # We also do not allow the interval to get too small so we don't
    # slam the city's servers.
    #
    # .95 : .355362
    # .90 : .531812
    # .80 : .824388
    #
    # http://www.wolframalpha.com/input/?i=exp(-bx)%2Bbxexp(-bx)%3D.8

    sampling_frequency = max(.824388/max(r), 10)
    print "Estimated Average Update Interval: " + str(int(1/max(r))) + " seconds"
    print "Sampling Interval:                 " + str(int(sampling_frequency)) + " seconds"
    print "Updates:                           " + str(updates)
    print ""
    sleep(sampling_frequency)

<commit_msg>added fault check for empty json feed<commit_after>import json
import sqlite3
import time
import datetime
import requests
from time import sleep
from collections import defaultdict, deque
from math import exp
import numpy
from scipy.optimize import fsolve

# Helper function for dealing with time stamps
time_format = "%a %b %d %H:%M:%S %Z %Y"
def formatTime(s, time_format) :
    return datetime.datetime(*time.strptime(s, time_format)[:6])

# Rate of change estimator for irregularly sampled data
# we will want to solve for x
#
# Cho and Garcia Molina, 2003, Estimating Frequency of Change
# http://dl.acm.org/citation.cfm?id=857170
def irregularCGM(intervals, Z) :
    def iCGM(x) :
        L = 0
        K = 0
        for i, z in enumerate(Z) :
            if z == 1 :
                L = L + intervals[i]/(exp(x*intervals[i])-1)
            else :
                K = K + intervals[i]
        return L - K
    return iCGM

# Set up DB
con = sqlite3.connect("plow.db")
cur = con.cursor()

cur.execute("""
CREATE TABLE IF NOT EXISTS route_points
(object_id INTEGER,
 posting_time DATETIME,
 X DOUBLE,
 Y DOUBLE,
 UNIQUE(object_id, posting_time) ON CONFLICT REPLACE)""")

cur.execute("""
CREATE TABLE IF NOT EXISTS assets
(object_id INTEGER,
 asset_name TEXT,
 asset_type TEXT,
 PRIMARY KEY(object_id) ON CONFLICT REPLACE)""")

con.commit()
con.close()

# The feed for City Of Chicago's Plow Data
gps_data_url = "https://gisapps.cityofchicago.org/snowplows/services/trackingservice/getPositions"

# We'll use these variables to keep track of whether we observe a new
# plow position. 30 observations should be sufficient for a reasonable
# estimate.
previous_posting_time = datetime.datetime(1,1,1)
last_posting_time = datetime.datetime(1,1,1)
update_history = defaultdict( lambda: deque([], 30))
intervals = deque([],30)

# We want adjust our sampling intervals depending upon our estimated
# rate of updates for the plows
sampling_frequency = 10

# Even if we are unable to get any data, we need to keep track of the
# time we spent on that attempt
fault_sleep = 60
faults = 0

while True:
    # Try to handle anything besides a well-formed json response
    try:
        payload = {"TrackingInput":{"envelope":{"minX":0,"minY":0,"maxX":0,"maxY":0},"trackingDuration":15}}
        response = requests.post(gps_data_url, data=json.dumps(payload))
    except Exception as e :
        print e
        sleep(fault_sleep)
        faults += 1
        continue
    if "Sorry, servers are currently down" in response:
        print "Sorry, servers are currently down"
        sleep(fault_sleep)
        faults += 1
        continue
    if len(response.json()) <= 1:
        print "No traces present"
        sleep(fault_sleep)
        faults += 1
        continue
    if len(response.json()['TrackingResponse']['locationList']) == 0:
        print "No traces present"
        sleep(fault_sleep)
        faults += 1
        continue

    # This is inside the loop as an act of perhaps irrational
    # defensive programming, as the script stopped updating the db for
    # no apparent reasons and without throwing an error.
    con = sqlite3.connect("plow.db")
    cur = con.cursor()

    try:
        read_data = response.json()
    except:
        print "Can't parse JSON feed"
        read_data = {'TrackingResponse': {'locationList': {}}}

    updates = 0
    for route_point in read_data['TrackingResponse']['locationList'] :

        try:
            (object_id,
             asset_name,
             asset_type,
             posting_time,
             x,
             y) = (route_point['objectid'],
                   route_point['assetName'],
                   route_point['assetType'],
                   route_point['postingTimeFormatted'],
                   route_point['XCoord'],
                   route_point['YCoord'])
        except TypeError:
            continue

        posting_time = formatTime(posting_time, time_format)

        # Insert Data into DB
        cur.execute("""insert into route_points (object_id, posting_time, X, Y)
                       values (?, ?, ?, ?)""",
                    (object_id, posting_time, x, y))
        cur.execute("""insert into assets (object_id, asset_name, asset_type)
                       values (?, ?, ?)""",
                    (object_id, asset_name, asset_type))

        # Update whether or not we observed a new position for every
        # plow
        if posting_time > last_posting_time :
            last_posting_time = posting_time

        if posting_time > previous_posting_time :
            update_history[object_id].append(1)
            updates += 1
        else :
            update_history[object_id].append(0)

    con.commit()
    con.close()

    # Add the sampling interval
    previous_posting_time = last_posting_time
    if faults :
        intervals.append(sampling_frequency
                         + faults*fault_sleep)
        faults = 0
    else :
        intervals.append(sampling_frequency)

    # Estimate the update rate
    r = []
    for object_id in update_history :
        z = sum(update_history[object_id])
        if z > 0 :
            icgm = irregularCGM(intervals, update_history[object_id])
            r.append(fsolve(icgm, .01))

    # Assuming that updates are drawn from a Poisson distribution,
    # then with some probability, we will observe LESS than 2 events
    # in this period. This does not mean the probability that we will
    # observe 1 update, as it is very likely that we will observe no
    # update.
    #
    # We also do not allow the interval to get too small so we don't
    # slam the city's servers.
    #
    # .95 : .355362
    # .90 : .531812
    # .80 : .824388
    #
    # http://www.wolframalpha.com/input/?i=exp(-bx)%2Bbxexp(-bx)%3D.8

    sampling_frequency = max(.824388/max(r), 10)
    print "Estimated Average Update Interval: " + str(int(1/max(r))) + " seconds"
    print "Sampling Interval:                 " + str(int(sampling_frequency)) + " seconds"
    print "Updates:                           " + str(updates)
    print ""
    sleep(sampling_frequency)

<|endoftext|>


<commit_before>from django.contrib.auth.models import User
from django.core.exceptions import ValidationError
from django.core.urlresolvers import reverse
from django.core.validators import RegexValidator
from django.db import models
from django.db.models import Max, Sum
from django.utils import timezone
import pytz
from operator import itemgetter, attrgetter

from judge.judgeapi import judge_submission, abort_submission
from judge.model_choices import ACE_THEMES

def make_timezones():
    data = {}
    for tz in pytz.all_timezones:
        if '/' in tz:
            area, loc = tz.split('/', 1)
        else:
            area, loc = 'Other', tz
        if area in data:
            data[area].append((tz, loc))
        else:
            data[area] = [(tz, loc)]
    data = data.items()
    data.sort(key=itemgetter(0))
    return data

TIMEZONE = make_timezones()
del make_timezones

class Language(models.Model):
    key = models.CharField(max_length=6, verbose_name='Short identifier', unique=True)
    name = models.CharField(max_length=20, verbose_name='Name as shown to user')
    ace = models.CharField(max_length=20, verbose_name='ACE mode name')

    def __unicode__(self):
        return self.name

    @classmethod
    def get_python2(cls):
        # We really need a default language, and this app is in Python 2
        return Language.objects.get_or_create(key='PY2', name='Python 2')[0]

    class Meta:
        ordering = ['key']

class GraderType(models.Model):
    key = models.CharField(max_length=20)
    name = models.CharField(max_length=50)

    def __unicode__(self):
        return self.name

class Profile(models.Model):
    user = models.OneToOneField(User, verbose_name='User associated')
    name = models.CharField(max_length=50, verbose_name='Display name', null=True, blank=True)
    about = models.TextField(verbose_name='Self-description', null=True, blank=True)
    timezone = models.CharField(max_length=50, verbose_name='Timezone', default='America/Toronto', choices=TIMEZONE)
    language = models.ForeignKey(Language, verbose_name='Default language')
    points = models.FloatField(default=0, db_index=True)
    ace_theme = models.CharField(max_length=30, choices=ACE_THEMES, default='github')

    def calculate_points(self):
        self.points = sum(map(itemgetter('points'),
                              Submission.objects.filter(user=self, points__isnull=False).values('problem_id').distinct()
                              .annotate(points=Max('points'))))
        self.save()
        return self.points

    @property
    def display_name(self):
        if self.name:
            return self.name
        return self.user.username

    @property
    def long_display_name(self):
        if self.name:
            return u'%s (%s)' % (self.user.username, self.name)
        return self.user.username

    @property
    def display_rank(self):
        return 'admin' if self.is_admin else ('setter' if self.is_problem_setter else 'user')

    @property
    def is_admin(self):
        return self.user.is_superuser or self.user.groups.filter(name='Admin').exists()

    @property
    def is_problem_setter(self):
        return self.user.is_superuser or self.user.groups.filter(name='ProblemSetter').exists()

    @property
    def problems(self):
        return Submission.objects.filter(user=self, points__gt=0).values('problem').distinct().count()

    def __unicode__(self):
        # return u'Profile of %s in %s speaking %s' % (self.long_display_name(), self.timezone, self.language)
        return self.long_display_name

class ProblemType(models.Model):
    name = models.CharField(max_length=20, verbose_name='Problem category ID', unique=True)
    full_name = models.CharField(max_length=100, verbose_name='Problem category name')

    def __unicode__(self):
        return self.full_name

class ProblemGroup(models.Model):
    name = models.CharField(max_length=20, verbose_name='Problem group ID', unique=True)
    full_name = models.CharField(max_length=100, verbose_name='Problem group name')

    def __unicode__(self):
        return self.full_name

def validate_grader_param(value):
    if not all('=' in i for i in value.split(';')):
        raise ValidationError('not all params are key-value pairs')

class Problem(models.Model):
    code = models.CharField(max_length=20, verbose_name='Problem code', unique=True,
                            validators=[RegexValidator('^[a-z0-9]+$', 'Problem code must be ^[a-z0-9]+$')])
    name = models.CharField(max_length=100, verbose_name='Problem name', db_index=True)
    description = models.TextField(verbose_name='Problem body')
    user = models.ForeignKey(Profile, verbose_name='Creator')
    types = models.ManyToManyField(ProblemType, verbose_name='Problem types')
    groups = models.ManyToManyField(ProblemGroup, verbose_name='Problem groups')
    time_limit = models.IntegerField(verbose_name='Time limit')
    memory_limit = models.IntegerField(verbose_name='Memory limit')
    short_circuit = models.BooleanField(default=False)
    grader = models.ForeignKey(GraderType)
    grader_param = models.CharField(verbose_name='Grader parameters', max_length=100, blank=True,
                                    validators=[validate_grader_param])
    points = models.FloatField(verbose_name='Points')
    partial = models.BooleanField(verbose_name='Allows partial points')
    allowed_languages = models.ManyToManyField(Language, verbose_name='Allowed languages')

    def types_list(self):
        return map(attrgetter('full_name'), self.types.all())

    def languages_list(self):
        return map(attrgetter('name'), self.allowed_languages.all())

    def number_of_users(self):
        return Submission.objects.filter(problem=self, result='AC').values('user').distinct().count()

    def __unicode__(self):
        return self.name

    def get_absolute_url(self):
        return reverse('judge.views.problem', args=(self.code,))

SUBMISSION_RESULT = (
    ('AC', 'Accepted'),
    ('WA', 'Wrong Answer'),
    ('TLE', 'Time Limit Exceeded'),
    ('MLE', 'Memory Limit Exceeded'),
    ('IR', 'Invalid Return'),
    ('RTE', 'Runtime Error'),
    ('CE', 'Compile Error'),
    ('IE', 'Internal Error'),
)

class Submission(models.Model):
    STATUS = (
        ('QU', 'Queued'),
        ('C', 'Compiled'),
        ('G', 'Grading'),
        ('D', 'Completed'),
        ('IE', 'Internal Error'),
        ('CE', 'Compile Error'),
    )
    RESULT = SUBMISSION_RESULT
    USER_DISPLAY_CODES = {
        'AC': 'Accepted',
        'WA': 'Wrong Answer',
        'TLE': 'Time Limit Exceeded',
        'MLE': 'Memory Limit Exceeded',
        'IR': 'Invalid Return',
        'RTE': 'Runtime Error (invalid syscall)',
        'CE': 'Compile Error',
        'IE': 'Internal Error (judging server error)',
        'QU': 'Queued',
        'C': 'Compiled',
        'G': 'Grading',
        'D': 'Completed'
    }

    user = models.ForeignKey(Profile)
    problem = models.ForeignKey(Problem)
    date = models.DateTimeField(verbose_name='Submission time', auto_now_add=True)
    time = models.FloatField(verbose_name='Execution time', null=True, db_index=True)
    memory = models.FloatField(verbose_name='Memory usage', null=True)
    points = models.FloatField(verbose_name='Points granted', null=True, db_index=True)
    language = models.ForeignKey(Language, verbose_name='Submission language')
    source = models.TextField(verbose_name='Source code')
    status = models.CharField(max_length=2, choices=STATUS, default='QU', db_index=True)
    result = models.CharField(max_length=3, choices=SUBMISSION_RESULT, default=None, null=True,
                              blank=True, db_index=True)
    error = models.TextField(verbose_name='Compile Errors', null=True, blank=True)

    @property
    def long_status(self):
        return Submission.USER_DISPLAY_CODES.get(self.result if self.result else self.status, '')

    def judge(self):
        return judge_submission(self)

    def abort(self):
        return abort_submission(self)

    def is_graded(self):
        return self.status not in ["QU", "C", "G"]

    @property
    def testcase_granted_points(self):
        return SubmissionTestCase.objects.filter(submission=self).values('submission')\
                                 .annotate(score=Sum('points'))[0]['score']

    @property
    def testcase_total_points(self):
        return SubmissionTestCase.objects.filter(submission=self).values('submission')\
                                 .annotate(score=Sum('total'))[0]['score']

    def __unicode__(self):
        return u'Submission %d of %s by %s' % (self.id, self.problem, self.user)

    class Meta:
        permissions = (
            ('rejudge_submission', 'Rejudge the submission'),
            ('rejudge_submission_lot', 'Rejudge a lot of submissions'),
            ('spam_submission', 'Submit without limit')
        )

class SubmissionTestCase(models.Model):
    RESULT = SUBMISSION_RESULT

    submission = models.ForeignKey(Submission, verbose_name='Associated submission')
    case = models.IntegerField(verbose_name='Test case ID')
    status = models.CharField(max_length=3, verbose_name='Status flag', choices=SUBMISSION_RESULT)
    time = models.FloatField(verbose_name='Execution time', null=True)
    memory = models.FloatField(verbose_name='Memory usage', null=True)
    points = models.FloatField(verbose_name='Points granted', null=True)
    total = models.FloatField(verbose_name='Points possible', null=True)

class Comment(models.Model):
    author = models.ForeignKey(Profile, verbose_name='Commenter')
    time = models.DateTimeField(verbose_name='Posted time', auto_now_add=True)
    page = models.CharField(max_length=30, verbose_name='Associated Page')
    parent = models.ForeignKey('self', related_name='replies', null=True, blank=True)
    score = models.IntegerField(verbose_name='Votes', default=0)
    title = models.CharField(max_length=200, verbose_name='Title of comment')
    body = models.TextField(verbose_name='Body of comment', blank=True)

    def __unicode__(self):
        return self.title

class CommentVote(models.Model):
    class Meta:
        unique_together = ['voter', 'comment']

    voter = models.ForeignKey(Profile)
    comment = models.ForeignKey(Comment)
    score = models.IntegerField()

class MiscConfig(models.Model):
    key = models.CharField(max_length=30, db_index=True)
    value = models.TextField()

class Judge(models.Model):
    name = models.CharField(max_length=50, help_text='Server name, hostname-style', unique=True)
    created = models.DateTimeField(auto_now_add=True)
    auth_key = models.CharField(max_length=100, help_text='A key to authenticated this judge',
                                verbose_name='Authentication key')
    online = models.BooleanField(default=False)
    last_connect = models.DateTimeField(verbose_name='Last connection time', null=True)
    ping = models.FloatField(verbose_name='Response time', null=True)
    load = models.FloatField(verbose_name='System load', null=True,
                             help_text='Load for the last minute, divided by processors to be fair.')
    problems = models.ManyToManyField(Problem)
    runtimes = models.ManyToManyField(Language)

    @property
    def uptime(self):
        return timezone.now() - self.last_connect if self.online else 'N/A'

    @property
    def ping_ms(self):
        return self.ping * 1000

    @property
    def runtime_list(self):
        return map(attrgetter('name'), self.runtimes.all())
<commit_msg>If not graded, return 0/0 for points<commit_after>from django.contrib.auth.models import User
from django.core.exceptions import ValidationError
from django.core.urlresolvers import reverse
from django.core.validators import RegexValidator
from django.db import models
from django.db.models import Max, Sum
from django.utils import timezone
import pytz
from operator import itemgetter, attrgetter

from judge.judgeapi import judge_submission, abort_submission
from judge.model_choices import ACE_THEMES

def make_timezones():
    data = {}
    for tz in pytz.all_timezones:
        if '/' in tz:
            area, loc = tz.split('/', 1)
        else:
            area, loc = 'Other', tz
        if area in data:
            data[area].append((tz, loc))
        else:
            data[area] = [(tz, loc)]
    data = data.items()
    data.sort(key=itemgetter(0))
    return data

TIMEZONE = make_timezones()
del make_timezones

class Language(models.Model):
    key = models.CharField(max_length=6, verbose_name='Short identifier', unique=True)
    name = models.CharField(max_length=20, verbose_name='Name as shown to user')
    ace = models.CharField(max_length=20, verbose_name='ACE mode name')

    def __unicode__(self):
        return self.name

    @classmethod
    def get_python2(cls):
        # We really need a default language, and this app is in Python 2
        return Language.objects.get_or_create(key='PY2', name='Python 2')[0]

    class Meta:
        ordering = ['key']

class GraderType(models.Model):
    key = models.CharField(max_length=20)
    name = models.CharField(max_length=50)

    def __unicode__(self):
        return self.name

class Profile(models.Model):
    user = models.OneToOneField(User, verbose_name='User associated')
    name = models.CharField(max_length=50, verbose_name='Display name', null=True, blank=True)
    about = models.TextField(verbose_name='Self-description', null=True, blank=True)
    timezone = models.CharField(max_length=50, verbose_name='Timezone', default='America/Toronto', choices=TIMEZONE)
    language = models.ForeignKey(Language, verbose_name='Default language')
    points = models.FloatField(default=0, db_index=True)
    ace_theme = models.CharField(max_length=30, choices=ACE_THEMES, default='github')

    def calculate_points(self):
        self.points = sum(map(itemgetter('points'),
                              Submission.objects.filter(user=self, points__isnull=False).values('problem_id').distinct()
                              .annotate(points=Max('points'))))
        self.save()
        return self.points

    @property
    def display_name(self):
        if self.name:
            return self.name
        return self.user.username

    @property
    def long_display_name(self):
        if self.name:
            return u'%s (%s)' % (self.user.username, self.name)
        return self.user.username

    @property
    def display_rank(self):
        return 'admin' if self.is_admin else ('setter' if self.is_problem_setter else 'user')

    @property
    def is_admin(self):
        return self.user.is_superuser or self.user.groups.filter(name='Admin').exists()

    @property
    def is_problem_setter(self):
        return self.user.is_superuser or self.user.groups.filter(name='ProblemSetter').exists()

    @property
    def problems(self):
        return Submission.objects.filter(user=self, points__gt=0).values('problem').distinct().count()

    def __unicode__(self):
        # return u'Profile of %s in %s speaking %s' % (self.long_display_name(), self.timezone, self.language)
        return self.long_display_name

class ProblemType(models.Model):
    name = models.CharField(max_length=20, verbose_name='Problem category ID', unique=True)
    full_name = models.CharField(max_length=100, verbose_name='Problem category name')

    def __unicode__(self):
        return self.full_name

class ProblemGroup(models.Model):
    name = models.CharField(max_length=20, verbose_name='Problem group ID', unique=True)
    full_name = models.CharField(max_length=100, verbose_name='Problem group name')

    def __unicode__(self):
        return self.full_name

def validate_grader_param(value):
    if not all('=' in i for i in value.split(';')):
        raise ValidationError('not all params are key-value pairs')

class Problem(models.Model):
    code = models.CharField(max_length=20, verbose_name='Problem code', unique=True,
                            validators=[RegexValidator('^[a-z0-9]+$', 'Problem code must be ^[a-z0-9]+$')])
    name = models.CharField(max_length=100, verbose_name='Problem name', db_index=True)
    description = models.TextField(verbose_name='Problem body')
    user = models.ForeignKey(Profile, verbose_name='Creator')
    types = models.ManyToManyField(ProblemType, verbose_name='Problem types')
    groups = models.ManyToManyField(ProblemGroup, verbose_name='Problem groups')
    time_limit = models.IntegerField(verbose_name='Time limit')
    memory_limit = models.IntegerField(verbose_name='Memory limit')
    short_circuit = models.BooleanField(default=False)
    grader = models.ForeignKey(GraderType)
    grader_param = models.CharField(verbose_name='Grader parameters', max_length=100, blank=True,
                                    validators=[validate_grader_param])
    points = models.FloatField(verbose_name='Points')
    partial = models.BooleanField(verbose_name='Allows partial points')
    allowed_languages = models.ManyToManyField(Language, verbose_name='Allowed languages')

    def types_list(self):
        return map(attrgetter('full_name'), self.types.all())

    def languages_list(self):
        return map(attrgetter('name'), self.allowed_languages.all())

    def number_of_users(self):
        return Submission.objects.filter(problem=self, result='AC').values('user').distinct().count()

    def __unicode__(self):
        return self.name

    def get_absolute_url(self):
        return reverse('judge.views.problem', args=(self.code,))

SUBMISSION_RESULT = (
    ('AC', 'Accepted'),
    ('WA', 'Wrong Answer'),
    ('TLE', 'Time Limit Exceeded'),
    ('MLE', 'Memory Limit Exceeded'),
    ('IR', 'Invalid Return'),
    ('RTE', 'Runtime Error'),
    ('CE', 'Compile Error'),
    ('IE', 'Internal Error'),
)

class Submission(models.Model):
    STATUS = (
        ('QU', 'Queued'),
        ('C', 'Compiled'),
        ('G', 'Grading'),
        ('D', 'Completed'),
        ('IE', 'Internal Error'),
        ('CE', 'Compile Error'),
    )
    RESULT = SUBMISSION_RESULT
    USER_DISPLAY_CODES = {
        'AC': 'Accepted',
        'WA': 'Wrong Answer',
        'TLE': 'Time Limit Exceeded',
        'MLE': 'Memory Limit Exceeded',
        'IR': 'Invalid Return',
        'RTE': 'Runtime Error (invalid syscall)',
        'CE': 'Compile Error',
        'IE': 'Internal Error (judging server error)',
        'QU': 'Queued',
        'C': 'Compiled',
        'G': 'Grading',
        'D': 'Completed'
    }

    user = models.ForeignKey(Profile)
    problem = models.ForeignKey(Problem)
    date = models.DateTimeField(verbose_name='Submission time', auto_now_add=True)
    time = models.FloatField(verbose_name='Execution time', null=True, db_index=True)
    memory = models.FloatField(verbose_name='Memory usage', null=True)
    points = models.FloatField(verbose_name='Points granted', null=True, db_index=True)
    language = models.ForeignKey(Language, verbose_name='Submission language')
    source = models.TextField(verbose_name='Source code')
    status = models.CharField(max_length=2, choices=STATUS, default='QU', db_index=True)
    result = models.CharField(max_length=3, choices=SUBMISSION_RESULT, default=None, null=True,
                              blank=True, db_index=True)
    error = models.TextField(verbose_name='Compile Errors', null=True, blank=True)

    @property
    def long_status(self):
        return Submission.USER_DISPLAY_CODES.get(self.result if self.result else self.status, '')

    def judge(self):
        return judge_submission(self)

    def abort(self):
        return abort_submission(self)

    def is_graded(self):
        return self.status not in ["QU", "C", "G"]

    @property
    def testcase_granted_points(self):
        score = SubmissionTestCase.objects.filter(submission=self).values('submission')\
                                  .annotate(score=Sum('points'))[0]['score']
        return score[0]['score'] if score else 0

    @property
    def testcase_total_points(self):
        score = SubmissionTestCase.objects.filter(submission=self).values('submission')\
                                  .annotate(score=Sum('total'))
        return score[0]['score'] if score else 0

    def __unicode__(self):
        return u'Submission %d of %s by %s' % (self.id, self.problem, self.user)

    class Meta:
        permissions = (
            ('rejudge_submission', 'Rejudge the submission'),
            ('rejudge_submission_lot', 'Rejudge a lot of submissions'),
            ('spam_submission', 'Submit without limit')
        )

class SubmissionTestCase(models.Model):
    RESULT = SUBMISSION_RESULT

    submission = models.ForeignKey(Submission, verbose_name='Associated submission')
    case = models.IntegerField(verbose_name='Test case ID')
    status = models.CharField(max_length=3, verbose_name='Status flag', choices=SUBMISSION_RESULT)
    time = models.FloatField(verbose_name='Execution time', null=True)
    memory = models.FloatField(verbose_name='Memory usage', null=True)
    points = models.FloatField(verbose_name='Points granted', null=True)
    total = models.FloatField(verbose_name='Points possible', null=True)

class Comment(models.Model):
    author = models.ForeignKey(Profile, verbose_name='Commenter')
    time = models.DateTimeField(verbose_name='Posted time', auto_now_add=True)
    page = models.CharField(max_length=30, verbose_name='Associated Page')
    parent = models.ForeignKey('self', related_name='replies', null=True, blank=True)
    score = models.IntegerField(verbose_name='Votes', default=0)
    title = models.CharField(max_length=200, verbose_name='Title of comment')
    body = models.TextField(verbose_name='Body of comment', blank=True)

    def __unicode__(self):
        return self.title

class CommentVote(models.Model):
    class Meta:
        unique_together = ['voter', 'comment']

    voter = models.ForeignKey(Profile)
    comment = models.ForeignKey(Comment)
    score = models.IntegerField()

class MiscConfig(models.Model):
    key = models.CharField(max_length=30, db_index=True)
    value = models.TextField()

class Judge(models.Model):
    name = models.CharField(max_length=50, help_text='Server name, hostname-style', unique=True)
    created = models.DateTimeField(auto_now_add=True)
    auth_key = models.CharField(max_length=100, help_text='A key to authenticated this judge',
                                verbose_name='Authentication key')
    online = models.BooleanField(default=False)
    last_connect = models.DateTimeField(verbose_name='Last connection time', null=True)
    ping = models.FloatField(verbose_name='Response time', null=True)
    load = models.FloatField(verbose_name='System load', null=True,
                             help_text='Load for the last minute, divided by processors to be fair.')
    problems = models.ManyToManyField(Problem)
    runtimes = models.ManyToManyField(Language)

    @property
    def uptime(self):
        return timezone.now() - self.last_connect if self.online else 'N/A'

    @property
    def ping_ms(self):
        return self.ping * 1000

    @property
    def runtime_list(self):
        return map(attrgetter('name'), self.runtimes.all())
<|endoftext|>


<commit_before>#!/usr/bin/env python

import xlrd as xl
import numpy as np
import matplotlib.pyplot as mpl

class Run(object):
	"""Stores variables for each run for easy access.
	   Allows fake namespace/struct like thing."""

	def __init__(self, numRows):
		self.size = numRows
		self.time = [None]*self.size
		self.position = [None]*self.size
		self.velocity = [None]*self.size
		self.fit= [None]*2
		self.fitDirect = []

	def findVelocity(self):
		"""Given Run object with positions and times, return correpsonding
			velocity"""
		#Most of the algorithm is completely stolen from the Lab Manual.
		for i in range(self.size):
			if i == 0:
				self.velocity[i] = (self.position[i+1] - self.position[i])/ \
				(self.time[i+1] - self.time[i])
			elif i == (self.size-1):
				self.velocity[i] = (self.position[i] - self.position[i-1])/ \
				(self.time[i] - self.time[i-1])
			else:
				self.velocity[i] = (self.position[i+1] - self.position[i-1])/ \
				(self.time[i+1] - self.time[i-1])

	def findAcceleration(self):
		"""Calculate the acceleration and y intercept from the time
		and velocity variables, add return the linfit"""
		self.fit = np.polyfit(self.time, self.velocity, 1)

	def plotVelocity(self):
		"""Plots velocity vs time"""
		fit_fun = np.poly1d(self.fit)
		mpl.plot(self.time,self.velocity,'bs')
		mpl.show()

	def fitAcceleration(self):
		"""Calculates the acceleration directly from the position and time data
		and numpy's linfit option"""
		self.fitDirect = np.polyfit(self.time, self.position, 2)

def exportData(fileName, sheetName):
	"""Returns Run object, given the name of the excel sheet and the name of
	   the sheet"""
	book = xl.open_workbook(fileName)

	runSheet = book.sheet_by_name(sheetName)
	sheetSize = runSheet.nrows - 1
	run = Run(sheetSize)

	#Extract the position values for run1.
	for i in range(run.size):
		run.position[i] = runSheet.cell(i,0).value

	#Extract the time value for run1
	for i in range(run.size):
		run.time[i] = runSheet.cell(i,1).value

	return run

def main():
	"""Performs data analysis for Moore's Lab 2."""
	excelFile = "ExcelTest.xls"
	sheetName = u"Sheet1"
	Run1 = exportData(excelFile,sheetName)
	Run1.findVelocity()
	Run1.findAcceleration()
	Run1.plotVelocity()
	Run1.fitAcceleration()
	acceleration = Run1.fitDirect[0]*2.
	print "Acceleration = %d" % acceleration

if __name__ == "__main__":
	main()
<commit_msg>Fixed plot waiting issue.<commit_after>#!/usr/bin/env python

import xlrd as xl
import numpy as np
import matplotlib.pyplot as mpl

class Run(object):
	"""Stores variables for each run for easy access.
	   Allows fake namespace/struct like thing."""

	def __init__(self, numRows):
		self.size = numRows
		self.time = [None]*self.size
		self.position = [None]*self.size
		self.velocity = [None]*self.size
		self.fit= [None]*2
		self.fitDirect = []

	def findVelocity(self):
		"""Given Run object with positions and times, return correpsonding
			velocity"""
		#Most of the algorithm is completely stolen from the Lab Manual.
		for i in range(self.size):
			if i == 0:
				self.velocity[i] = (self.position[i+1] - self.position[i])/ \
				(self.time[i+1] - self.time[i])
			elif i == (self.size-1):
				self.velocity[i] = (self.position[i] - self.position[i-1])/ \
				(self.time[i] - self.time[i-1])
			else:
				self.velocity[i] = (self.position[i+1] - self.position[i-1])/ \
				(self.time[i+1] - self.time[i-1])

	def findAcceleration(self):
		"""Calculate the acceleration and y intercept from the time
		and velocity variables, add return the linfit"""
		self.fit = np.polyfit(self.time, self.velocity, 1)

	def plotVelocity(self):
		"""Plots velocity vs time"""
		fit_fun = np.poly1d(self.fit)
		mpl.plot(self.time,self.velocity,'bs')
		mpl.show()

	def fitAcceleration(self):
		"""Calculates the acceleration directly from the position and time data
		and numpy's linfit option"""
		self.fitDirect = np.polyfit(self.time, self.position, 2)

def exportData(fileName, sheetName):
	"""Returns Run object, given the name of the excel sheet and the name of
	   the sheet"""
	book = xl.open_workbook(fileName)

	runSheet = book.sheet_by_name(sheetName)
	sheetSize = runSheet.nrows - 1
	run = Run(sheetSize)

	#Extract the position values for run1.
	for i in range(run.size):
		run.position[i] = runSheet.cell(i,0).value

	#Extract the time value for run1
	for i in range(run.size):
		run.time[i] = runSheet.cell(i,1).value

	return run

def main():
	"""Performs data analysis for Moore's Lab 2."""
	excelFile = "ExcelTest.xls"
	sheetName = u"Sheet1"
	Run1 = exportData(excelFile,sheetName)
	Run1.findVelocity()
	#Run1.findAcceleration()
	Run1.fitAcceleration()
	acceleration = Run1.fitDirect[0]*2.
	print "Acceleration = %d" % acceleration
	Run1.plotVelocity()

if __name__ == "__main__":
	main()
<|endoftext|>


<commit_before>"""
core.py

Created by Kang Zhang on 2009-07-09
"""
import os
import sys
import warnings

from .py27compat import configparser

from keyring import logger
from keyring import backend
from keyring.util import platform_ as platform

def set_keyring(keyring):
    """Set current keyring backend.
    """
    global _keyring_backend
    if not isinstance(keyring, backend.KeyringBackend):
        raise TypeError("The keyring must be a subclass of KeyringBackend")
    _keyring_backend = keyring

def get_keyring():
    """Get current keyring backend.
    """
    return _keyring_backend

def get_password(service_name, username):
    """Get password from the specified service.
    """
    return _keyring_backend.get_password(service_name, username)

def set_password(service_name, username, password):
    """Set password for the user in the specified service.
    """
    _keyring_backend.set_password(service_name, username, password)

def delete_password(service_name, username):
    """Delete the password for the user in the specified service.
    """
    _keyring_backend.delete_password(service_name, username)

def init_backend():
    """Load a keyring from a config file or for the default platform.

    First try to load the keyring in the config file, if it has not
    been declared, assign a default keyring according to the platform.
    """
    # select a backend according to the config file
    keyring = load_config()

    # if the user doesn't specify a keyring, we apply a default one
    if keyring is None:

        keyrings = backend.get_all_keyring()
        # rank by priority
        keyrings.sort(key = lambda x: -x.priority)
        # get the most recommended one
        keyring = keyrings[0]

    set_keyring(keyring)

def load_keyring(keyring_path, keyring_name):
    """
    Load the specified keyring by name (a fully-qualified name to the
    keyring, such as 'keyring.backends.file.PlaintextKeyring')

    `keyring_path` is an additional, optional search path and may be None.
    **deprecated** In the future, keyring_path must be None.
    """
    module_name, sep, class_name = keyring_name.rpartition('.')
    if keyring_path is not None and keyring_path not in sys.path:
        warnings.warn("keyring_path is deprecated and should always be None",
            DeprecationWarning)
        sys.path.insert(0, keyring_path)
    __import__(module_name)
    module = sys.modules[module_name]
    return getattr(module, class_name)()

def load_config():
    """Load a keyring using the config file.

    The config file can be in the current working directory, or in the user's
    home directory.
    """
    keyring = None

    filename = 'keyringrc.cfg'

    local_path = os.path.join(os.getcwd(), filename)
    config_path = os.path.join(platform.data_root(), filename)

    # search from current working directory and the data root
    keyring_cfg_candidates = [local_path, config_path]

    # initialize the keyring_config with the first detected config file
    keyring_cfg = None
    for path in keyring_cfg_candidates:
        keyring_cfg = path
        if os.path.exists(path):
            break

    if os.path.exists(keyring_cfg):
        config = configparser.RawConfigParser()
        config.read(keyring_cfg)
        _load_keyring_path(config)

        # load the keyring class name, and then load this keyring
        try:
            if config.has_section("backend"):
                keyring_name = config.get("backend", "default-keyring").strip()
            else:
                raise configparser.NoOptionError('backend', 'default-keyring')

            keyring = load_keyring(None, keyring_name)
        except (configparser.NoOptionError, ImportError):
            logger.warning("Keyring config file contains incorrect values.\n" +
                           "Config file: %s" % keyring_cfg)

    return keyring

def _load_keyring_path(config):
    "load the keyring-path option (if present)"
    try:
        path = config.get("backend", "keyring-path").strip()
        sys.path.insert(0, path)
    except (configparser.NoOptionError, configparser.NoSectionError):
        pass

# init the _keyring_backend
init_backend()
<commit_msg>Extracted _get_best_keyring function.<commit_after>"""
core.py

Created by Kang Zhang on 2009-07-09
"""
import os
import sys
import warnings

from .py27compat import configparser

from keyring import logger
from keyring import backend
from keyring.util import platform_ as platform

def set_keyring(keyring):
    """Set current keyring backend.
    """
    global _keyring_backend
    if not isinstance(keyring, backend.KeyringBackend):
        raise TypeError("The keyring must be a subclass of KeyringBackend")
    _keyring_backend = keyring

def get_keyring():
    """Get current keyring backend.
    """
    return _keyring_backend

def get_password(service_name, username):
    """Get password from the specified service.
    """
    return _keyring_backend.get_password(service_name, username)

def set_password(service_name, username, password):
    """Set password for the user in the specified service.
    """
    _keyring_backend.set_password(service_name, username, password)

def delete_password(service_name, username):
    """Delete the password for the user in the specified service.
    """
    _keyring_backend.delete_password(service_name, username)

def init_backend():
    """
    Load a keyring specified in the config file or infer the best available.
    """
    set_keyring(load_config() or _get_best_keyring())

def _get_best_keyring():
    """
    Return the best keyring backend for the given environment based on
    priority.
    """
    keyrings = backend.get_all_keyring()
    # rank by priority
    keyrings.sort(key = lambda x: -x.priority)
    # get the most recommended one
    return keyrings[0]

def load_keyring(keyring_path, keyring_name):
    """
    Load the specified keyring by name (a fully-qualified name to the
    keyring, such as 'keyring.backends.file.PlaintextKeyring')

    `keyring_path` is an additional, optional search path and may be None.
    **deprecated** In the future, keyring_path must be None.
    """
    module_name, sep, class_name = keyring_name.rpartition('.')
    if keyring_path is not None and keyring_path not in sys.path:
        warnings.warn("keyring_path is deprecated and should always be None",
            DeprecationWarning)
        sys.path.insert(0, keyring_path)
    __import__(module_name)
    module = sys.modules[module_name]
    return getattr(module, class_name)()

def load_config():
    """Load a keyring using the config file.

    The config file can be in the current working directory, or in the user's
    home directory.
    """
    keyring = None

    filename = 'keyringrc.cfg'

    local_path = os.path.join(os.getcwd(), filename)
    config_path = os.path.join(platform.data_root(), filename)

    # search from current working directory and the data root
    keyring_cfg_candidates = [local_path, config_path]

    # initialize the keyring_config with the first detected config file
    keyring_cfg = None
    for path in keyring_cfg_candidates:
        keyring_cfg = path
        if os.path.exists(path):
            break

    if os.path.exists(keyring_cfg):
        config = configparser.RawConfigParser()
        config.read(keyring_cfg)
        _load_keyring_path(config)

        # load the keyring class name, and then load this keyring
        try:
            if config.has_section("backend"):
                keyring_name = config.get("backend", "default-keyring").strip()
            else:
                raise configparser.NoOptionError('backend', 'default-keyring')

            keyring = load_keyring(None, keyring_name)
        except (configparser.NoOptionError, ImportError):
            logger.warning("Keyring config file contains incorrect values.\n" +
                           "Config file: %s" % keyring_cfg)

    return keyring

def _load_keyring_path(config):
    "load the keyring-path option (if present)"
    try:
        path = config.get("backend", "keyring-path").strip()
        sys.path.insert(0, path)
    except (configparser.NoOptionError, configparser.NoSectionError):
        pass

# init the _keyring_backend
init_backend()
<|endoftext|>


<commit_before>#
# Kiwi: a Framework and Enhanced Widgets for Python
#
# Copyright (C) 2003-2005 Async Open Source
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
# USA
#
# Author(s): Christian Reis <kiko@async.com.br>
#            Lorenzo Gil Sanchez <lgs@sicem.biz>
#            Johan Dahlin <jdahlin@async.com.br>
#

import gettext
import os
import sys

__all__ = ['Application', 'Library', 'environ', 'require_gazpacho',
           'is_gazpacho_required']

class Environment:
    """Environment control

    When you want to access a resource on the filesystem, such as
    an image or a glade file you use this object.

    External libraries or applications are free to add extra directories"""

    def __init__(self, root='.'):
        self._resources = {}
        self._root = root

        self._add_resource_variable("glade", "KIWI_GLADE_PATH")
        self._add_resource_variable("image", "KIWI_IMAGE_PATH")

    def _add_resource_variable(self, resource, variable):
        """Add resources from an environment variable"""
        env = os.environ.get(variable, '')
        for path in env.split(':'):
            if not path:
                continue
            self.add_resource(resource, env)

    def add_resource(self, resource, path):
        if not os.path.isdir(path):
            raise EnvironmentError("path %s must be a directory" % path)

        path = os.path.join(self._root, path)
        resources = self._resources
        if not resource in resources:
            resources[resource] = [path]
        else:
            resources[resource].append(path)

    def add_resources(self, **kwargs):
        for resource, path in kwargs.items():
            self.add_resource(resource, path)

    def find_resource(self, resource, name):
        """Locate a specific resource of called name of type resource"""

        if not resource in self._resources:
            raise EnvironmentError("No resource called: %s" % resource)

        for path in self._resources[resource]:
            filename = os.path.join(self._root, path, name)
            if os.path.exists(filename):
                return filename

        raise EnvironmentError("Could not find %s resource: %s" % (
            resource, name))

class Library(Environment):
    """A Library is a local environment object, it's a subclass of the
    Environment class.
    It's used by libraries and applications (through the Application class)

    It provides a way to manage local resources, which should only be seen
    in the current context.
    """
    def __init__(self, name, root='..'):
        self.name = name
        # Figure out the absolute path to the caller
        caller = sys._getframe(1).f_locals['__file__']
        caller_abs = os.path.split(caller)[0]
        root = os.path.abspath(os.path.join(caller_abs, root))

        Environment.__init__(self, root=root)

        # Load installed
        try:
            module = __import__(name + '.__installed__',
                                globals(), locals(), name)
        except ImportError, e:
            uninstalled = True
        else:
            uninstalled = False
            if not hasattr(module, 'resources'):
                raise ValueError("module %s.__installed must define a "
                                 "resources attribute" % name)
            if not hasattr(module, 'global_resources'):
                raise ValueError("module %s.__installed must define a "
                                 "global_resources attribute" % name)
            self.add_resources(**module.resources)
            self.add_global_resources(**module.global_resources)

        self.uninstalled = uninstalled

    def enable_translation(self, domain=None, locale='locale',
                           charset='utf-8'):
        if not domain:
            domain = self.name

        self._resources.get(locale)
        gettext.bindtextdomain(domain, locale)
        gettext.bind_textdomain_codeset(domain, charset)

    def add_global_resource(self, resource, path):
        """Convenience method to add a global resource.
        This is the same as calling kiwi.environ.environ.add_resource
        """
        global environ
        environ.add_resource(resource, os.path.join(self._root, path))

    def add_global_resources(self, **kwargs):
        for resource, path in kwargs.items():
            self.add_global_resource(resource, path)

class Application(Library):
    """An Application extends a library. The additions are:
    - a run() method, used to run the application
    - path, a reference to the callable object to run, defaults to 'main'
    """
    def __init__(self, name, root='..', path='main'):
        Library.__init__(self, name, root)
        self._path = path

    def _get_main(self):
        path = self._path
        main_module = os.path.join(self.root, path)

        try:
            module = __import__(main_module, globals(), locals(), path)
        except Exception, e:
            raise SystemExit("ERROR: Failed to import required module %s\n\n"
                             "Exception raised during import:\n %s: %s\n" %
                             (main_module, e.__class__.__name__, e))

        main = getattr(module, path, None)
        if not main or not callable(main):
            raise SystemExit("ERROR: Could not find item '%s' in module %s" %
                             path, main_module)
        return main

    def enable_translation(self, domain=None, locale='locale',
                           charset='utf-8'):
        if not domain:
            domain = self.name

        Library.enable_translation(self, domain, locale, charset)
        gettext.textdomain(domain)

    def run(self):
        main = self._get_main()

        try:
            sys.exit(main(sys.argv))
        except KeyboardInterrupt:
            raise SystemExit

_require_gazpacho_loader = False

def require_gazpacho():
    global _require_gazpacho_loader
    _require_gazpacho_loader = True

def is_gazpacho_required():
    global _require_gazpacho_loader
    return _require_gazpacho_loader

# Global instance, shared between apps and libraries
environ = Environment()
<commit_msg>Translation bugfix<commit_after>#
# Kiwi: a Framework and Enhanced Widgets for Python
#
# Copyright (C) 2003-2005 Async Open Source
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
# USA
#
# Author(s): Christian Reis <kiko@async.com.br>
#            Lorenzo Gil Sanchez <lgs@sicem.biz>
#            Johan Dahlin <jdahlin@async.com.br>
#

import gettext
import os
import sys

__all__ = ['Application', 'Library', 'environ', 'require_gazpacho',
           'is_gazpacho_required']

class Environment:
    """Environment control

    When you want to access a resource on the filesystem, such as
    an image or a glade file you use this object.

    External libraries or applications are free to add extra directories"""

    def __init__(self, root='.'):
        self._resources = {}
        self._root = root

        self._add_resource_variable("glade", "KIWI_GLADE_PATH")
        self._add_resource_variable("image", "KIWI_IMAGE_PATH")

    def _add_resource_variable(self, resource, variable):
        """Add resources from an environment variable"""
        env = os.environ.get(variable, '')
        for path in env.split(':'):
            if not path:
                continue
            self.add_resource(resource, env)

    def add_resource(self, resource, path):
        if not os.path.isdir(path):
            raise EnvironmentError("path %s must be a directory" % path)

        path = os.path.join(self._root, path)
        resources = self._resources
        if not resource in resources:
            resources[resource] = [path]
        else:
            resources[resource].append(path)

    def add_resources(self, **kwargs):
        for resource, path in kwargs.items():
            self.add_resource(resource, path)

    def find_resource(self, resource, name):
        """Locate a specific resource of called name of type resource"""

        if not resource in self._resources:
            raise EnvironmentError("No resource called: %s" % resource)

        for path in self._resources[resource]:
            filename = os.path.join(self._root, path, name)
            if os.path.exists(filename):
                return filename

        raise EnvironmentError("Could not find %s resource: %s" % (
            resource, name))

class Library(Environment):
    """A Library is a local environment object, it's a subclass of the
    Environment class.
    It's used by libraries and applications (through the Application class)

    It provides a way to manage local resources, which should only be seen
    in the current context.
    """
    def __init__(self, name, root='..'):
        self.name = name
        # Figure out the absolute path to the caller
        caller = sys._getframe(1).f_locals['__file__']
        caller_abs = os.path.split(caller)[0]
        root = os.path.abspath(os.path.join(caller_abs, root))

        Environment.__init__(self, root=root)

        # Load installed
        try:
            module = __import__(name + '.__installed__',
                                globals(), locals(), name)
        except ImportError, e:
            uninstalled = True
        else:
            uninstalled = False
            if not hasattr(module, 'resources'):
                raise ValueError("module %s.__installed must define a "
                                 "resources attribute" % name)
            if not hasattr(module, 'global_resources'):
                raise ValueError("module %s.__installed must define a "
                                 "global_resources attribute" % name)
            self.add_resources(**module.resources)
            self.add_global_resources(**module.global_resources)

        self.uninstalled = uninstalled

    def enable_translation(self, domain=None, locale='locale',
                           charset='utf-8'):
        if not domain:
            domain = self.name

        # XXX: locale should not be a list
        localedir = self._resources.get(locale)[0]
        gettext.bindtextdomain(domain, localedir)
        gettext.bind_textdomain_codeset(domain, charset)

    def add_global_resource(self, resource, path):
        """Convenience method to add a global resource.
        This is the same as calling kiwi.environ.environ.add_resource
        """
        global environ
        environ.add_resource(resource, os.path.join(self._root, path))

    def add_global_resources(self, **kwargs):
        for resource, path in kwargs.items():
            self.add_global_resource(resource, path)

class Application(Library):
    """An Application extends a library. The additions are:
    - a run() method, used to run the application
    - path, a reference to the callable object to run, defaults to 'main'
    """
    def __init__(self, name, root='..', path='main'):
        Library.__init__(self, name, root)
        self._path = path

    def _get_main(self):
        path = self._path
        main_module = os.path.join(self.root, path)

        try:
            module = __import__(main_module, globals(), locals(), path)
        except Exception, e:
            raise SystemExit("ERROR: Failed to import required module %s\n\n"
                             "Exception raised during import:\n %s: %s\n" %
                             (main_module, e.__class__.__name__, e))

        main = getattr(module, path, None)
        if not main or not callable(main):
            raise SystemExit("ERROR: Could not find item '%s' in module %s" %
                             path, main_module)
        return main

    def enable_translation(self, domain=None, locale='locale',
                           charset='utf-8'):
        if not domain:
            domain = self.name

        Library.enable_translation(self, domain, locale, charset)
        gettext.textdomain(domain)

    def run(self):
        main = self._get_main()

        try:
            sys.exit(main(sys.argv))
        except KeyboardInterrupt:
            raise SystemExit

_require_gazpacho_loader = False

def require_gazpacho():
    global _require_gazpacho_loader
    _require_gazpacho_loader = True

def is_gazpacho_required():
    global _require_gazpacho_loader
    return _require_gazpacho_loader

# Global instance, shared between apps and libraries
environ = Environment()
<|endoftext|>