<commit_before># -*- coding: utf-8 -*-
# Copyright (C) 2016-2017 Cordell Bloor
# Published under the MIT License

"""Shared utilities between checkguard, guard2once and once2guard."""

from __future__ import print_function
import sys
import os
import re
from fnmatch import fnmatch
from functools import partial
from itertools import islice

py2 = sys.version_info < (3,)

def ends_with_blank_line(contents):
    """
    Returns true if the given string ends with a line that is either empty or
    only composed of whitespace.
    """
    return re.search('\n\s*\n\Z', contents) is not None

def next_nonempty_line(contents, start):
    """
    Returns the index of the first character of the next line containing
    non-whitespace characters, or returns len(contents) if there is no such
    line.
    """
    line_end = len(contents) - 1
    for i, c in enumerate(islice(contents, start, None), start):
        if c == '\n':
           line_end = i
        elif c not in ' \t':
           break
    return line_end + 1

def next_line(contents, start):
    """
    Returns the index of the first character of the next line, or returns
    len(contents) if there is no next line.
    """
    for i, c in enumerate(islice(contents, start, None), start):
        if c == '\n':
           return i + 1
    return len(contents)

def guess_guard(contents):
    """
    Returns the guard, as well as the start and end indexes of the include
    guard from the start of the file, or throws ValueError if not found.
    Comments are not supported.
    """
    regex = re.compile(r"^[ \t]*\#[ \t]*ifndef[ \t]+([\w]+)"
        + r"[ \t]*\n[ \t]*\#[ \t]*define[ \t]+\1([ \t]+1)?[ \t]*$",
        re.MULTILINE)
    guard_found = False
    for match in regex.finditer(contents):
        # if it matches our regex, it's probably a guard, but check
        # that there's some content within to be a little more sure
        nextline_start = next_nonempty_line(contents, match.end())
        nextline_end = next_line(contents, nextline_start)
        try:
            index_guard_end(contents, nextline_start, nextline_end)
        except ValueError as e:
            guard_found = True
            break
    if not guard_found:
        raise ValueError('guard start not found')
    return (match.group(1),) + match.span()

def index_pragma_once(contents):
    """
    Returns the start and end indexes of the pragma once directive from
    the start of the file, or throws ValueError if not found. Comments
    are not supported.
    """
    regex = re.compile(r"^[ \t]*\#[ \t]*pragma[ \t]+once[ \t]*$",
        re.MULTILINE)
    match = regex.search(contents)
    if not match:
        raise ValueError('pragma once not found')
    return match.span()

def index_guard_start(contents, guard_symbol):
    """
    Returns the start and end indexes of the include guard from the start of the
    file, or throws ValueError if not found. Comments are not supported.
    """
    regex = re.compile(r"^[ \t]*\#[ \t]*ifndef[ \t]+" + guard_symbol
        + r"[ \t]*\n[ \t]*\#[ \t]*define[ \t]+" + guard_symbol
        + r"([ \t]+1)?[ \t]*$",
        re.MULTILINE)
    match = regex.search(contents)
    if not match:
        raise ValueError('guard start not found')
    return match.span()

def index_guard_end(contents, pos=0, endpos=None):
    """
    Returns the start and end indexes of the last endif line from the
    file, or throws ValueError if not found. Comments are not supported.
    """
    if endpos is None:
        endpos = len(contents)
    regex = re.compile(r"^[ \t]*\#[ \t]*endif([/ \t]+.*|[ \t]*)$",
        re.MULTILINE)
    match = None
    for match in regex.finditer(contents, pos, endpos):
        pass
    if not match:
        raise ValueError('guard end not found')
    return match.span()

class FileMetadata:
    """
    Information about a file's encoding and line endings.
    """
    def __init__(self):
        self.bom = False
        self.crlf = False

bom = '\xef\xbb\xbf' if py2 else '\ufeff'

def get_file_contents(filename):
    with open(filename, 'r') as f:
        contents = f.read()
        metadata = FileMetadata()
        if contents.startswith(bom):
            contents = contents[len(bom):]
            metadata.bom = True
        return (contents, metadata)

def write_file_contents(filename, contents, metadata):
    with open(filename, 'w') as f:
        if metadata.bom:
            contents = bom + contents
        f.write(contents)

def print_error(error):
    print(error, file=sys.stderr)

def is_header_file(filename):
    '''Returns true if the given file is identified as a C/C++ header file.'''
    return filename.endswith(('.h', '.hpp', '.H', '.hh'))

def is_excluded(filepath, exclusions):
    '''Returns true if the given file matches any of the exclusion patterns.'''
    return any(map(partial(fnmatch, filepath), exclusions))

def apply_to_headers(func, directory, exclusions):
    for root, dirs, files in os.walk(directory, onerror=print_error):
        for filename in files:
            filepath = os.path.join(root, filename)
            if is_header_file(filename) and not is_excluded(filepath, exclusions):
                func(filepath, filename)
<commit_msg>Fix reading UTF-8 files with Python 3 on Windows<commit_after># -*- coding: utf-8 -*-
# Copyright (C) 2016-2017 Cordell Bloor
# Published under the MIT License

"""Shared utilities between checkguard, guard2once and once2guard."""

from __future__ import print_function
import sys
import os
import re
from fnmatch import fnmatch
from functools import partial
from itertools import islice

py2 = sys.version_info < (3,)

def ends_with_blank_line(contents):
    """
    Returns true if the given string ends with a line that is either empty or
    only composed of whitespace.
    """
    return re.search('\n\s*\n\Z', contents) is not None

def next_nonempty_line(contents, start):
    """
    Returns the index of the first character of the next line containing
    non-whitespace characters, or returns len(contents) if there is no such
    line.
    """
    line_end = len(contents) - 1
    for i, c in enumerate(islice(contents, start, None), start):
        if c == '\n':
           line_end = i
        elif c not in ' \t':
           break
    return line_end + 1

def next_line(contents, start):
    """
    Returns the index of the first character of the next line, or returns
    len(contents) if there is no next line.
    """
    for i, c in enumerate(islice(contents, start, None), start):
        if c == '\n':
           return i + 1
    return len(contents)

def guess_guard(contents):
    """
    Returns the guard, as well as the start and end indexes of the include
    guard from the start of the file, or throws ValueError if not found.
    Comments are not supported.
    """
    regex = re.compile(r"^[ \t]*\#[ \t]*ifndef[ \t]+([\w]+)"
        + r"[ \t]*\n[ \t]*\#[ \t]*define[ \t]+\1([ \t]+1)?[ \t]*$",
        re.MULTILINE)
    guard_found = False
    for match in regex.finditer(contents):
        # if it matches our regex, it's probably a guard, but check
        # that there's some content within to be a little more sure
        nextline_start = next_nonempty_line(contents, match.end())
        nextline_end = next_line(contents, nextline_start)
        try:
            index_guard_end(contents, nextline_start, nextline_end)
        except ValueError as e:
            guard_found = True
            break
    if not guard_found:
        raise ValueError('guard start not found')
    return (match.group(1),) + match.span()

def index_pragma_once(contents):
    """
    Returns the start and end indexes of the pragma once directive from
    the start of the file, or throws ValueError if not found. Comments
    are not supported.
    """
    regex = re.compile(r"^[ \t]*\#[ \t]*pragma[ \t]+once[ \t]*$",
        re.MULTILINE)
    match = regex.search(contents)
    if not match:
        raise ValueError('pragma once not found')
    return match.span()

def index_guard_start(contents, guard_symbol):
    """
    Returns the start and end indexes of the include guard from the start of the
    file, or throws ValueError if not found. Comments are not supported.
    """
    regex = re.compile(r"^[ \t]*\#[ \t]*ifndef[ \t]+" + guard_symbol
        + r"[ \t]*\n[ \t]*\#[ \t]*define[ \t]+" + guard_symbol
        + r"([ \t]+1)?[ \t]*$",
        re.MULTILINE)
    match = regex.search(contents)
    if not match:
        raise ValueError('guard start not found')
    return match.span()

def index_guard_end(contents, pos=0, endpos=None):
    """
    Returns the start and end indexes of the last endif line from the
    file, or throws ValueError if not found. Comments are not supported.
    """
    if endpos is None:
        endpos = len(contents)
    regex = re.compile(r"^[ \t]*\#[ \t]*endif([/ \t]+.*|[ \t]*)$",
        re.MULTILINE)
    match = None
    for match in regex.finditer(contents, pos, endpos):
        pass
    if not match:
        raise ValueError('guard end not found')
    return match.span()

class FileMetadata:
    """
    Information about a file's encoding and line endings.
    """
    def __init__(self):
        self.bom = False
        self.crlf = False

bom = '\xef\xbb\xbf' if py2 else '\ufeff'
open_kwargs = {} if py2 else {'encoding':'utf8'}

def get_file_contents(filename):
    with open(filename, 'r', **open_kwargs) as f:
        contents = f.read()
        metadata = FileMetadata()
        if contents.startswith(bom):
            contents = contents[len(bom):]
            metadata.bom = True
        return (contents, metadata)

def write_file_contents(filename, contents, metadata):
    with open(filename, 'w', **open_kwargs) as f:
        if metadata.bom:
            contents = bom + contents
        f.write(contents)

def print_error(error):
    print(error, file=sys.stderr)

def is_header_file(filename):
    '''Returns true if the given file is identified as a C/C++ header file.'''
    return filename.endswith(('.h', '.hpp', '.H', '.hh'))

def is_excluded(filepath, exclusions):
    '''Returns true if the given file matches any of the exclusion patterns.'''
    return any(map(partial(fnmatch, filepath), exclusions))

def apply_to_headers(func, directory, exclusions):
    for root, dirs, files in os.walk(directory, onerror=print_error):
        for filename in files:
            filepath = os.path.join(root, filename)
            if is_header_file(filename) and not is_excluded(filepath, exclusions):
                func(filepath, filename)
<|endoftext|>


<commit_before>from flask import Flask, render_template, request, redirect, abort, flash, session
from flask_script import Manager
import settings
from database import db, Gateways,Address,InboundMapping,OutboundRoutes
import os
import subprocess

app = Flask(__name__, static_folder="./static", static_url_path="/static")
app.debug = settings.DEBUG
#app.add_template_filter()

@app.route('/')
def index():
    if not session.get('logged_in'):
        return render_template('index.html',version=settings.VERSION)
    else:
        action = request.args.get('action')
        return render_template('dashboard.html',show_add_onload=action)

@app.route('/login', methods=['POST'])
def login():
    if request.form['password'] == settings.PASSWORD and request.form['username'] == settings.USERNAME:
        session['logged_in'] = True
        session['username'] = request.form['username']
    else:
        flash('wrong password!')
    return index()

@app.route('/logout')
def logout():
    session['logged_in'] = False
    return index()

@app.route('/carriers')
def displayCarriers():
    if session.get('logged_in'):
       res = db.query(Gateways).filter(Gateways.type==settings.FLT_CARRIER).all()
       return render_template('carriers.html',rows=res)
    else:
       return index()

@app.route('/carriers', methods=['POST'])
def addUpdateCarriers():
    gwid = request.form['gwid']
    name = request.form['name']
    ip_addr = request.form['ip_addr']
    strip = request.form['strip']
    prefix = request.form['prefix']
    # Adding
    if len(gwid) <= 0:
        print(name)
        Gateway = Gateways(name,ip_addr,strip,prefix,settings.FLT_CARRIER)
        Addr=Address(name,ip_addr,32,settings.FLT_PBX)
        db.add(Gateway)
        db.add(Addr)
        db.commit()
        return displayCarriers()
    # Updating
    else:
        db.query(Gateways).filter(Gateways.gwid==gwid).update({'description':"name:"+name,'address':ip_addr,'strip':strip,'pri_prefix':prefix})
        #TODO: You will end up with multiple Address records -will fix
        Addr=Address(name,ip_addr,32,settings.FLT_PBX)
        db.add(Addr)
        db.commit()
        return displayCarriers()

@app.route('/carrierdelete',methods=['POST'])
def deleteCarriers():
    gwid = request.form['gwid']
    name = request.form['name']
    d = db.query(Gateways).filter(Gateways.gwid==gwid)
    d.delete(synchronize_session=False)
    a = db.query(Address).filter(Address.tag=='name:'+name)
    a.delete(synchronize_session=False)
    return displayCarriers()

@app.route('/pbx')
def displayPBX():
    if session.get('logged_in'):
       res = db.query(Gateways).filter(Gateways.type==settings.FLT_PBX).all()
       return render_template('pbxs.html',rows=res)
    else:
       return index()

@app.route('/pbx', methods=['POST'])
def addUpdatePBX():
    gwid = request.form['gwid']
    name = request.form['name']
    ip_addr = request.form['ip_addr']
    strip = request.form['strip']
    prefix = request.form['prefix']
    # Adding
    if len(gwid) <= 0:
        print(name)
        Gateway = Gateways(name,ip_addr,strip,prefix,settings.FLT_PBX)
        Addr=Address(name,ip_addr,32,settings.FLT_PBX)
        db.add(Gateway)
        db.add(Addr)
        db.commit()
        return displayPBX()
    # Updating
    else:
        db.query(Gateways).filter(Gateways.gwid==gwid).update({'description':"name:"+name,'address':ip_addr,'strip':strip,'pri_prefix':prefix})
        #TODO: You will end up with multiple Address records -will fix
        Addr=Address(name,ip_addr,32,settings.FLT_PBX)
        db.add(Addr)
        db.commit()
        return displayPBX()

@app.route('/pbxdelete',methods=['POST'])
def deletePBX():
    gwid = request.form['gwid']
    name = request.form['name']
    d = db.query(Gateways).filter(Gateways.gwid==gwid)
    d.delete(synchronize_session=False)
    a = db.query(Address).filter(Address.tag=='name:'+name)
    a.delete(synchronize_session=False)
    return displayPBX()

@app.route('/inboundmapping')
def displayInboundMapping():
    if session.get('logged_in'):
        res = db.execute('select * from dr_rules r,dr_gateways g where r.gwlist = g.gwid and r.groupid > 8000')
	#res = db.query(InboundMapping).filter(InboundMapping.groupid>8000).all()
        gateways = db.query(Gateways).filter(Gateways.type==settings.FLT_PBX).all()
        return render_template('inboundmapping.html',rows=res,gwlist=gateways)
    else:
       return index()

@app.route('/inboundmapping', methods=['POST'])
def addInboundMapping():
    if request.form['ruleid']:
       ruleid = request.form['ruleid']
    else:
       ruleid = None
    gwid = request.form['gwid']
    prefix = request.form['prefix']
    # Adding
    if not ruleid:
       groupid=9000
       IMap  = InboundMapping(groupid,prefix,gwid)
       db.add(IMap)
       db.commit()
    # Updating
    else:
       db.query(InboundMapping).filter(InboundMapping.ruleid==ruleid).update({'prefix':prefix,'gwlist':gwid})
       db.commit()
    return  displayInboundMapping()

@app.route('/inboundmappingdelete',methods=['POST'])
def deleteInboundMapping():
    ruleid = request.form['ruleid']
    d = db.query(InboundMapping).filter(InboundMapping.ruleid==ruleid)
    d.delete(synchronize_session=False)
    return  displayInboundMapping()

@app.route('/outboundroutes')
def displayOutboundRoutes():
    if session.get('logged_in'):
       gwlist = db.query(OutboundRoutes.gwlist).filter(OutboundRoutes.groupid==8000).scalar()
       return render_template('outboundroutes.html',outboundroutes=gwlist)
    else:
       return index()

@app.route('/outboundroutes', methods=['POST'])
def addUpateOutboundRoutes():
    if request.form['gwlist']:
       gwlist = request.form['gwlist']
    else:
       gwlist = None
    # Adding
    if not gwlist:
       groupid=8000
       prefix = "."
       OMap  = OutboundRoutes(groupid=groupid,prefix='.',gwlist=gwlist,timerec='',routeid='')
       db.add(OMap)
       db.commit()
    # Updating
    else:
       db.query(OutboundRoutes).filter(OutboundRoutes.groupid==8000).update({'gwlist':gwlist})
       db.commit()
    return  displayOutboundRoutes()

@app.route('/reloadkam')
def reloadkam():
       try:
           subprocess.call(['kamcmd' ,'permissions.addressReload'])
           subprocess.call(['kamcmd','drouting.reload'])
           session['last_page'] = request.headers['Referer']
           return render_template('dashboard.html',reloadstatus='successful')
       except:
           return render_template('dashboard.html',reloadstatus='failed')

manager = Manager(app)

def init_app(flask_app):
    #Setup the Flask session manager with a random secret key
    app.secret_key = os.urandom(12)

    #Add jinga2 filter
    app.jinja_env.filters["attrFilter"]=attrFilter

    #db.init_app(flask_app)
    #db.app = app

    manager.run()

def attrFilter(list,field):
    if ":" in list:
        d = dict(item.split(":") for item in list.split(","))
        try:
            return d[field]
        except:
            return
    else:
        return list

if __name__ == "__main__":
    init_app(app)
<commit_msg>added execute permissions<commit_after>from flask import Flask, render_template, request, redirect, abort, flash, session
from flask_script import Manager
import settings
from database import db, Gateways,Address,InboundMapping,OutboundRoutes
import os
import subprocess

app = Flask(__name__, static_folder="./static", static_url_path="/static")
app.debug = settings.DEBUG
#app.add_template_filter()

@app.route('/')
def index():
    if not session.get('logged_in'):
        return render_template('index.html',version=settings.VERSION)
    else:
        action = request.args.get('action')
        return render_template('dashboard.html',show_add_onload=action)

@app.route('/login', methods=['POST'])
def login():
    if request.form['password'] == settings.PASSWORD and request.form['username'] == settings.USERNAME:
        session['logged_in'] = True
        session['username'] = request.form['username']
    else:
        flash('wrong password!')
    return index()

@app.route('/logout')
def logout():
    session['logged_in'] = False
    return index()

@app.route('/carriers')
def displayCarriers():
    if session.get('logged_in'):
       res = db.query(Gateways).filter(Gateways.type==settings.FLT_CARRIER).all()
       return render_template('carriers.html',rows=res)
    else:
       return index()

@app.route('/carriers', methods=['POST'])
def addUpdateCarriers():
    gwid = request.form['gwid']
    name = request.form['name']
    ip_addr = request.form['ip_addr']
    strip = request.form['strip']
    prefix = request.form['prefix']
    # Adding
    if len(gwid) <= 0:
        print(name)
        Gateway = Gateways(name,ip_addr,strip,prefix,settings.FLT_CARRIER)
        Addr=Address(name,ip_addr,32,settings.FLT_PBX)
        db.add(Gateway)
        db.add(Addr)
        db.commit()
        return displayCarriers()
    # Updating
    else:
        db.query(Gateways).filter(Gateways.gwid==gwid).update({'description':"name:"+name,'address':ip_addr,'strip':strip,'pri_prefix':prefix})
        #TODO: You will end up with multiple Address records -will fix
        Addr=Address(name,ip_addr,32,settings.FLT_PBX)
        db.add(Addr)
        db.commit()
        return displayCarriers()

@app.route('/carrierdelete',methods=['POST'])
def deleteCarriers():
    gwid = request.form['gwid']
    name = request.form['name']
    d = db.query(Gateways).filter(Gateways.gwid==gwid)
    d.delete(synchronize_session=False)
    a = db.query(Address).filter(Address.tag=='name:'+name)
    a.delete(synchronize_session=False)
    return displayCarriers()

@app.route('/pbx')
def displayPBX():
    if session.get('logged_in'):
       res = db.query(Gateways).filter(Gateways.type==settings.FLT_PBX).all()
       return render_template('pbxs.html',rows=res)
    else:
       return index()

@app.route('/pbx', methods=['POST'])
def addUpdatePBX():
    gwid = request.form['gwid']
    name = request.form['name']
    ip_addr = request.form['ip_addr']
    strip = request.form['strip']
    prefix = request.form['prefix']
    # Adding
    if len(gwid) <= 0:
        print(name)
        Gateway = Gateways(name,ip_addr,strip,prefix,settings.FLT_PBX)
        Addr=Address(name,ip_addr,32,settings.FLT_PBX)
        db.add(Gateway)
        db.add(Addr)
        db.commit()
        return displayPBX()
    # Updating
    else:
        db.query(Gateways).filter(Gateways.gwid==gwid).update({'description':"name:"+name,'address':ip_addr,'strip':strip,'pri_prefix':prefix})
        #TODO: You will end up with multiple Address records -will fix
        Addr=Address(name,ip_addr,32,settings.FLT_PBX)
        db.add(Addr)
        db.commit()
        return displayPBX()

@app.route('/pbxdelete',methods=['POST'])
def deletePBX():
    gwid = request.form['gwid']
    name = request.form['name']
    d = db.query(Gateways).filter(Gateways.gwid==gwid)
    d.delete(synchronize_session=False)
    a = db.query(Address).filter(Address.tag=='name:'+name)
    a.delete(synchronize_session=False)
    return displayPBX()

@app.route('/inboundmapping')
def displayInboundMapping():
    if session.get('logged_in'):
        res = db.execute('select * from dr_rules r,dr_gateways g where r.gwlist = g.gwid and r.groupid > 8000')
	#res = db.query(InboundMapping).filter(InboundMapping.groupid>8000).all()
        gateways = db.query(Gateways).filter(Gateways.type==settings.FLT_PBX).all()
        return render_template('inboundmapping.html',rows=res,gwlist=gateways)
    else:
       return index()

@app.route('/inboundmapping', methods=['POST'])
def addInboundMapping():
    if request.form['ruleid']:
       ruleid = request.form['ruleid']
    else:
       ruleid = None
    gwid = request.form['gwid']
    prefix = request.form['prefix']
    # Adding
    if not ruleid:
       groupid=9000
       IMap  = InboundMapping(groupid,prefix,gwid)
       db.add(IMap)
       db.commit()
    # Updating
    else:
       db.query(InboundMapping).filter(InboundMapping.ruleid==ruleid).update({'prefix':prefix,'gwlist':gwid})
       db.commit()
    return  displayInboundMapping()

@app.route('/inboundmappingdelete',methods=['POST'])
def deleteInboundMapping():
    ruleid = request.form['ruleid']
    d = db.query(InboundMapping).filter(InboundMapping.ruleid==ruleid)
    d.delete(synchronize_session=False)
    return  displayInboundMapping()

@app.route('/outboundroutes')
def displayOutboundRoutes():
    if session.get('logged_in'):
       gwlist = db.query(OutboundRoutes.gwlist).filter(OutboundRoutes.groupid==8000).scalar()
       return render_template('outboundroutes.html',outboundroutes=gwlist)
    else:
       return index()

@app.route('/outboundroutes', methods=['POST'])
def addUpateOutboundRoutes():
    if request.form['gwlist']:
       gwlist = request.form['gwlist']
    else:
       gwlist = None
    # Adding
    if not gwlist:
       groupid=8000
       prefix = "."
       OMap  = OutboundRoutes(groupid=groupid,prefix='.',gwlist=gwlist,timerec='',routeid='')
       db.add(OMap)
       db.commit()
    # Updating
    else:
       db.query(OutboundRoutes).filter(OutboundRoutes.groupid==8000).update({'gwlist':gwlist})
       db.commit()
    return  displayOutboundRoutes()

@app.route('/reloadkam')
def reloadkam():
       try:
           subprocess.call(['kamcmd' ,'permissions.addressReload'])
           subprocess.call(['kamcmd','drouting.reload'])
           session['last_page'] = request.headers['Referer']
           return render_template('dashboard.html',reloadstatus='successful')
       except:
           return render_template('dashboard.html',reloadstatus='failed')

manager = Manager(app)

def init_app(flask_app):
    #Setup the Flask session manager with a random secret key
    app.secret_key = os.urandom(12)

    #Add jinga2 filter
    app.jinja_env.filters["attrFilter"]=attrFilter

    #db.init_app(flask_app)
    #db.app = app

    manager.run()

def attrFilter(list,field):
    if ":" in list:
        d = dict(item.split(":") for item in list.split(","))
        try:
            return d[field]
        except:
            return
    else:
        return list

if __name__ == "__main__":
    init_app(app)
<|endoftext|>


<commit_before>from tsadm.tests import TSAdmTestBase

class TSAdmTest(TSAdmTestBase):
    def test_SiteView(self):
        resp = self.client.get(self.getURL('site:home', kwargs={'name': 'fakesite'}))
        self.assertEqual(resp.status_code, 200)
<commit_msg>create a fakesite on TSAdmTest setUp<commit_after>from tsadm.tests import TSAdmTestBase
from ..models import TSAdmSiteDB

class TSAdmTest(TSAdmTestBase):
    def setUp(self):
        super(TSAdmTest, self).setUp()
        site = TSAdmSiteDB(name='fakesite')
        site.save()

    def test_SiteView(self):
        resp = self.client.get(self.getURL('site:home', kwargs={'name': 'fakesite'}))
        self.assertEqual(resp.status_code, 200)
<|endoftext|>


<commit_before>from ggame import App, RectangleAsset, ImageAsset, SoundAsset, Sprite, Sound
from ggame import LineStyle, Color

SCREEN_WIDTH = 1900
SCREEN_HEIGHT = 900

yellow = Color(0xfff400, 1.0)
red = Color(0xff0000, 1.0)
green = Color(0x00ff00, 1.0)
blue = Color(0x0000ff, 1.0)
white = Color(0xffffff, 1.0)
black = Color(0, 1)
noline = LineStyle(0, black)
bg_asset = RectangleAsset(SCREEN_WIDTH, SCREEN_HEIGHT, noline, yellow)
bg = Sprite(bg_asset, (0,0))

bunny_asset = ImageAsset("images/bunny.png")
bunny = Sprite(bunny_asset, (0,0))
bunny.scale = 1.5
bunny.dir = 2.5
bunny.go = True

beachball_asset = ImageAsset("images/beach-ball-575425_640.png")
beachball = Sprite(beachball_asset, (20,300))
beachball.scale = .2
beachball.dir = 2.5
beachball.go = True

sun_asset = ImageAsset("images/sun.png")
sun = Sprite(sun_asset, (20,500))
sun.scale = 1.5
sun.dir = 2.5
sun.go = True

marble_asset = ImageAsset("images/orb-150545_640.png")
marble = Sprite(marble_asset, (0,700))
marble.scale = .2
marble.dir = 2.5
marble.go = True

def reverse(b):
    b.dir *= -1

def reverse(c):
    c.dir *= -1

def reverse(s):
    s.dir *= -1

def reverse(m):
    m.dir *= -1

def step():
    if bunny.go:
        bunny.y += bunny.dir
        if bunny.y + bunny.width > SCREEN_HEIGHT or bunny.y < 0:
            bunny.y -= bunny.dir
            bunny.dir *= -1
    if beachball.go:
        beachball.x += beachball.dir
        if beachball.x + beachball.width > SCREEN_WIDTH or beachball.x < 0:
            beachball.x -= beachball.dir
            beachball.dir *= -1
    if sun.go:
        sun.x += sun.dir
        if sun.x + sun.width > SCREEN_WIDTH or sun.x < 20:
            sun.x -= sun.dir
            sun.dir *= -1
    if marble.go:
        marble.x += marble.dir
        if marble.x + marble.width > SCREEN_WIDTH or marble.x < 0:
            marble.x -= marble.dir
            marble.dir *= -1

if marble.x <= bunny.width and marble.y <= bunny.height:
    bunny.go = not bunny.go

def spaceKey(event):
    bunny.go = not bunny.go

def rKey(event):
    reverse(bunny)

def zKey(event):
    bunny.scale = 1

def xKey(event):
    bunny.scale = 1.5

def mKey(event):
    if bunny.dir > 2.5 or bunny.dir < -2.5:
        bunny.dir /= 2

def nKey(event):
    if bunny.dir < 5 and bunny.dir > -5:
        bunny.dir *= 2

def cKey(event):
    bunny.dir = 2.5

def vKey(event):
    bunny.dir = -2.5

def bKey(event):
    bunny.dir /= 2

def tKey(event):
    beachball.go = not beachball.go

def eKey(event):
    reverse(beachball)

def wKey(event):
    if beachball.scale >= .025:
        beachball.scale /= 2

def qKey(event):
    if beachball.scale <= .2:
        beachball.scale *= 2

def yKey(event):
     if beachball.dir > 2.5 or beachball.dir < -2.5:
        beachball.dir /= 2

def uKey(event):
    if beachball.dir < 5 and beachball.dir > -5:
        beachball.dir *= 2

def aKey(event):
    sun.go = not sun.go

def dKey(event):
    reverse(sun)

def gKey(event):
    if sun.scale > 1 and sun.scale < 2.5:
        sun.scale -= .1

def hKey(event):
    if sun.scale > .5 and sun.scale < 2:
        sun.scale += .1

def fKey(event):
    if sun.dir >= 1.5 and sun.dir <= 5:
        sun.dir -= .25

def sKey(event):
    if sun.dir >= 1 and sun.dir <= 5:
        sun.dir += .25

def lKey(event):
    marble.go = not marble.go

def iKey(event):
    reverse(marble)

def jKey(event):
    marble.scale = .01

def kKey(event):
    marble.scale = .325

def oKey(event):
    if marble.dir >= .5 or marble.dir <= -.5:
        marble.dir *= -.5

def pKey(event):
    if marble.dir <= 4.5:
        marble.dir /= -.75

myapp = App(SCREEN_WIDTH, SCREEN_HEIGHT)
myapp.listenKeyEvent('keydown', 'space', spaceKey)
myapp.listenKeyEvent('keydown', 'r', rKey)
myapp.listenKeyEvent('keydown', 'z', zKey)
myapp.listenKeyEvent('keydown', 'x', xKey)
myapp.listenKeyEvent('keydown', 'm', mKey)
myapp.listenKeyEvent('keydown', 'n', nKey)
myapp.listenKeyEvent('keydown', 'c', cKey)
myapp.listenKeyEvent('keydown', 'v', vKey)
myapp.listenKeyEvent('keydown', 'b', bKey)
myapp.listenKeyEvent('keydown', 't', tKey)
myapp.listenKeyEvent('keydown', 'e', eKey)
myapp.listenKeyEvent('keydown', 'w', wKey)
myapp.listenKeyEvent('keydown', 'q', qKey)
myapp.listenKeyEvent('keydown', 'y', yKey)
myapp.listenKeyEvent('keydown', 'u', uKey)
myapp.listenKeyEvent('keydown', 'a', aKey)
myapp.listenKeyEvent('keydown', 'd', dKey)
myapp.listenKeyEvent('keydown', 'g', gKey)
myapp.listenKeyEvent('keydown', 'h', hKey)
myapp.listenKeyEvent('keydown', 'f', fKey)
myapp.listenKeyEvent('keydown', 's', sKey)
myapp.listenKeyEvent('keydown', 'l', lKey)
myapp.listenKeyEvent('keydown', 'i', iKey)
myapp.listenKeyEvent('keydown', 'j', jKey)
myapp.listenKeyEvent('keydown', 'k', kKey)
myapp.listenKeyEvent('keydown', 'o', oKey)
myapp.listenKeyEvent('keydown', 'p', pKey)
myapp.run(step)
<commit_msg>Updated from Brython Server: 1/12/2017 12:10:37 PM<commit_after>from ggame import App, RectangleAsset, ImageAsset, SoundAsset, Sprite, Sound
from ggame import LineStyle, Color

SCREEN_WIDTH = 1900
SCREEN_HEIGHT = 900

yellow = Color(0xfff400, 1.0)
red = Color(0xff0000, 1.0)
green = Color(0x00ff00, 1.0)
blue = Color(0x0000ff, 1.0)
white = Color(0xffffff, 1.0)
black = Color(0, 1)
noline = LineStyle(0, black)
bg_asset = RectangleAsset(SCREEN_WIDTH, SCREEN_HEIGHT, noline, yellow)
bg = Sprite(bg_asset, (0,0))

bunny_asset = ImageAsset("images/bunny.png")
bunny = Sprite(bunny_asset, (0,0))
bunny.scale = 1.5
bunny.dir = 2.5
bunny.go = True

beachball_asset = ImageAsset("images/beach-ball-575425_640.png")
beachball = Sprite(beachball_asset, (20,300))
beachball.scale = .2
beachball.dir = 2.5
beachball.go = True

sun_asset = ImageAsset("images/sun.png")
sun = Sprite(sun_asset, (20,500))
sun.scale = 1.5
sun.dir = 2.5
sun.go = True

marble_asset = ImageAsset("images/orb-150545_640.png")
marble = Sprite(marble_asset, (0,700))
marble.scale = .2
marble.dir = 2.5
marble.go = True

def reverse(b):
    b.dir *= -1

def reverse(c):
    c.dir *= -1

def reverse(s):
    s.dir *= -1

def reverse(m):
    m.dir *= -1

def step():
    if bunny.go:
        bunny.y += bunny.dir
        if bunny.y + bunny.width > SCREEN_HEIGHT or bunny.y < 0:
            bunny.y -= bunny.dir
            bunny.dir *= -1
    if beachball.go:
        beachball.x += beachball.dir
        if beachball.x + beachball.width > SCREEN_WIDTH or beachball.x < 0:
            beachball.x -= beachball.dir
            beachball.dir *= -1
    if sun.go:
        sun.x += sun.dir
        if sun.x + sun.width > SCREEN_WIDTH or sun.x < 20:
            sun.x -= sun.dir
            sun.dir *= -1
    if marble.go:
        marble.x += marble.dir
        if marble.x + marble.width > SCREEN_WIDTH or marble.x < 0:
            marble.x -= marble.dir
            marble.dir *= -1

def spaceKey(event):
    bunny.go = not bunny.go

def rKey(event):
    reverse(bunny)

def zKey(event):
    bunny.scale = 1

def xKey(event):
    bunny.scale = 1.5

def mKey(event):
    if bunny.dir > 2.5 or bunny.dir < -2.5:
        bunny.dir /= 2

def nKey(event):
    if bunny.dir < 5 and bunny.dir > -5:
        bunny.dir *= 2

def cKey(event):
    bunny.dir = 2.5

def vKey(event):
    bunny.dir = -2.5

def bKey(event):
    bunny.dir /= 2

def tKey(event):
    beachball.go = not beachball.go

def eKey(event):
    reverse(beachball)

def wKey(event):
    if beachball.scale >= .025:
        beachball.scale /= 2

def qKey(event):
    if beachball.scale <= .2:
        beachball.scale *= 2

def yKey(event):
     if beachball.dir > 2.5 or beachball.dir < -2.5:
        beachball.dir /= 2

def uKey(event):
    if beachball.dir < 5 and beachball.dir > -5:
        beachball.dir *= 2

def aKey(event):
    sun.go = not sun.go

def dKey(event):
    reverse(sun)

def gKey(event):
    if sun.scale > 1 and sun.scale < 2.5:
        sun.scale -= .1

def hKey(event):
    if sun.scale > .5 and sun.scale < 2:
        sun.scale += .1

def fKey(event):
    if sun.dir >= 1.5 and sun.dir <= 5:
        sun.dir -= .25

def sKey(event):
    if sun.dir >= 1 and sun.dir <= 5:
        sun.dir += .25

def lKey(event):
    marble.go = not marble.go

def iKey(event):
    reverse(marble)

def jKey(event):
    marble.scale = .01

def kKey(event):
    marble.scale = .325

def oKey(event):
    if marble.dir >= .5 or marble.dir <= -.5:
        marble.dir *= -.5

def pKey(event):
    if marble.dir <= 4.5:
        marble.dir /= -.75

myapp = App(SCREEN_WIDTH, SCREEN_HEIGHT)
myapp.listenKeyEvent('keydown', 'space', spaceKey)
myapp.listenKeyEvent('keydown', 'r', rKey)
myapp.listenKeyEvent('keydown', 'z', zKey)
myapp.listenKeyEvent('keydown', 'x', xKey)
myapp.listenKeyEvent('keydown', 'm', mKey)
myapp.listenKeyEvent('keydown', 'n', nKey)
myapp.listenKeyEvent('keydown', 'c', cKey)
myapp.listenKeyEvent('keydown', 'v', vKey)
myapp.listenKeyEvent('keydown', 'b', bKey)
myapp.listenKeyEvent('keydown', 't', tKey)
myapp.listenKeyEvent('keydown', 'e', eKey)
myapp.listenKeyEvent('keydown', 'w', wKey)
myapp.listenKeyEvent('keydown', 'q', qKey)
myapp.listenKeyEvent('keydown', 'y', yKey)
myapp.listenKeyEvent('keydown', 'u', uKey)
myapp.listenKeyEvent('keydown', 'a', aKey)
myapp.listenKeyEvent('keydown', 'd', dKey)
myapp.listenKeyEvent('keydown', 'g', gKey)
myapp.listenKeyEvent('keydown', 'h', hKey)
myapp.listenKeyEvent('keydown', 'f', fKey)
myapp.listenKeyEvent('keydown', 's', sKey)
myapp.listenKeyEvent('keydown', 'l', lKey)
myapp.listenKeyEvent('keydown', 'i', iKey)
myapp.listenKeyEvent('keydown', 'j', jKey)
myapp.listenKeyEvent('keydown', 'k', kKey)
myapp.listenKeyEvent('keydown', 'o', oKey)
myapp.listenKeyEvent('keydown', 'p', pKey)
myapp.run(step)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os, sys, subprocess

p0=os.path.dirname(sys.argv[0])
p2=os.path.abspath(os.path.join(p0, "..", "lithium"))
sys.path.append(p2)
lithiumpy = os.path.join(p2, "lithium.py")
jsunhappypy = os.path.join(p0, "jsunhappy.py")

import jsunhappy

timeout = int(sys.argv[1])
knownPath = os.path.expanduser(sys.argv[2])
runThis = sys.argv[3:]
jsfunfuzzPath = runThis[-1]

def showtail(filename):
    cmd = "tail -n 20 %s" % filename
    print cmd
    print ""
    os.system(cmd)
    print ""
    print ""

def many_timed_runs():
    iteration = 0
    while True:
        iteration += 1
        logPrefix = "w%d" % iteration

        level = jsunhappy.level(runThis, timeout, knownPath, logPrefix)

        if level > jsunhappy.JS_TIMED_OUT:
            showtail(logPrefix + "-out")
            showtail(logPrefix + "-err")

            # splice jsfunfuzz.js with `grep FRC wN-out`
            filenameToReduce = logPrefix + "-reduced.js"
            [before, after] = fuzzSplice(open(jsfunfuzzPath))
            newfileLines = before + linesWith(open(logPrefix + "-out"), "FRC") + after
            writeLinesToFile(newfileLines, logPrefix + "-orig.js")
            writeLinesToFile(newfileLines, filenameToReduce)

            # Run Lithium as a subprocess: reduce to the smallest file that has at least the same unhappiness level
            lithArgs = [jsunhappypy, str(level), str(timeout), knownPath] + runThis[:-1] + [filenameToReduce]
            print "multi_timed_run is running Lithium..."
            print repr([lithiumpy] + lithArgs)
            subprocess.call([lithiumpy] + lithArgs, stdout=open(logPrefix + "-lith1", "w"))
            if level > jsunhappy.JS_DID_NOT_FINISH:
                subprocess.call([lithiumpy, "-c"] + lithArgs, stdout=open(logPrefix + "-lith2", "w"))
            print "Done running Lithium"

        else:
            os.remove(logPrefix + "-out")
            os.remove(logPrefix + "-err")
            if (os.path.exists(logPrefix + "-crash")):
                os.remove(logPrefix + "-crash")
            if (os.path.exists(logPrefix + "-core")):
                os.remove(logPrefix + "-core")

def fuzzSplice(file):
    '''Returns the lines of a file, minus the ones between the two lines containing SPLICE'''
    before = []
    after = []
    for line in file:
        before.append(line)
        if line.find("SPLICE") != -1:
            break
    for line in file:
        if line.find("SPLICE") != -1:
            after.append(line)
            break
    for line in file:
        after.append(line)
    file.close()
    return [before, after]

def fuzzDice(file):
    '''Returns the lines of the file, except for the one line containing DICE'''
    before = []
    after = []
    for line in file:
        if line.find("SPLICE") != -1:
            break
        before.append(line)
    for line in file:
        after.append(line)
    file.close()
    return [before, after]

def linesWith(file, searchFor):
    '''Returns the lines from a file that contain a given string'''
    matchingLines = []
    for line in file:
        if line.find(searchFor) != -1:
            matchingLines.append(line)
    file.close()
    return matchingLines

def writeLinesToFile(lines, filename):
      f = open(filename, "w")
      f.writelines(lines)
      f.close()

jsunhappy.initWithKnownPath(knownPath)
many_timed_runs()
<commit_msg>Delete valgrind log files after NORMAL multi_timed_run runs.<commit_after>#!/usr/bin/env python

import os, sys, subprocess

p0=os.path.dirname(sys.argv[0])
p2=os.path.abspath(os.path.join(p0, "..", "lithium"))
sys.path.append(p2)
lithiumpy = os.path.join(p2, "lithium.py")
jsunhappypy = os.path.join(p0, "jsunhappy.py")

import jsunhappy

timeout = int(sys.argv[1])
knownPath = os.path.expanduser(sys.argv[2])
runThis = sys.argv[3:]
jsfunfuzzPath = runThis[-1]

def showtail(filename):
    cmd = "tail -n 20 %s" % filename
    print cmd
    print ""
    os.system(cmd)
    print ""
    print ""

def many_timed_runs():
    iteration = 0
    while True:
        iteration += 1
        logPrefix = "w%d" % iteration

        level = jsunhappy.level(runThis, timeout, knownPath, logPrefix)

        if level > jsunhappy.JS_TIMED_OUT:
            showtail(logPrefix + "-out")
            showtail(logPrefix + "-err")

            # splice jsfunfuzz.js with `grep FRC wN-out`
            filenameToReduce = logPrefix + "-reduced.js"
            [before, after] = fuzzSplice(open(jsfunfuzzPath))
            newfileLines = before + linesWith(open(logPrefix + "-out"), "FRC") + after
            writeLinesToFile(newfileLines, logPrefix + "-orig.js")
            writeLinesToFile(newfileLines, filenameToReduce)

            # Run Lithium as a subprocess: reduce to the smallest file that has at least the same unhappiness level
            lithArgs = [jsunhappypy, str(level), str(timeout), knownPath] + runThis[:-1] + [filenameToReduce]
            print "multi_timed_run is running Lithium..."
            print repr([lithiumpy] + lithArgs)
            subprocess.call([lithiumpy] + lithArgs, stdout=open(logPrefix + "-lith1", "w"))
            if level > jsunhappy.JS_DID_NOT_FINISH:
                subprocess.call([lithiumpy, "-c"] + lithArgs, stdout=open(logPrefix + "-lith2", "w"))
            print "Done running Lithium"

        else:
            os.remove(logPrefix + "-out")
            os.remove(logPrefix + "-err")
            if (os.path.exists(logPrefix + "-crash")):
                os.remove(logPrefix + "-crash")
            if (os.path.exists(logPrefix + "-vg.xml")):
                os.remove(logPrefix + "-vg.xml")
            if (os.path.exists(logPrefix + "-core")):
                os.remove(logPrefix + "-core")

def fuzzSplice(file):
    '''Returns the lines of a file, minus the ones between the two lines containing SPLICE'''
    before = []
    after = []
    for line in file:
        before.append(line)
        if line.find("SPLICE") != -1:
            break
    for line in file:
        if line.find("SPLICE") != -1:
            after.append(line)
            break
    for line in file:
        after.append(line)
    file.close()
    return [before, after]

def fuzzDice(file):
    '''Returns the lines of the file, except for the one line containing DICE'''
    before = []
    after = []
    for line in file:
        if line.find("SPLICE") != -1:
            break
        before.append(line)
    for line in file:
        after.append(line)
    file.close()
    return [before, after]

def linesWith(file, searchFor):
    '''Returns the lines from a file that contain a given string'''
    matchingLines = []
    for line in file:
        if line.find(searchFor) != -1:
            matchingLines.append(line)
    file.close()
    return matchingLines

def writeLinesToFile(lines, filename):
      f = open(filename, "w")
      f.writelines(lines)
      f.close()

jsunhappy.initWithKnownPath(knownPath)
many_timed_runs()
<|endoftext|>


<commit_before>"""
cachelib
--------

"Cache rules everything around me"
    -- Method Man

Pure Python implementations of various cache algorithms.

License: MIT
"""

import collections
import threading

from collections import deque, OrderedDict

# Linked list indexes
LINK_PREV = 0
LINK_NEXT = 1
LINK_KEY = 2
LINK_VALUE = 3
LINK_ACCESS_COUNT = 4
LINK_EXPIRE_TIME = 5

def make_circular_queue(slots):
    root = []
    root[:] = [root, root] + [None] * slots
    return root

class Cache(collections.Mapping):
    def __init__(self, maxsize, get_missing=None, on_evict=None):
        self.maxsize = maxsize
        # Function which returns items missing from the cache.
        if get_missing:
            self.get_missing = get_missing
        # Callback for item eviction.
        self.on_evict = on_evict

        self.size = 0
        self.hits = self.misses = 0

        # Make linked list updates atomic.
        self._lock = threading.RLock()

    def get_missing(self, key):
        raise KeyError(key)

    def __contains__(self, key):
        return self._cache.__contains__(key)

    def __delitem__(self, key):
        raise NotImplementedError

    def __iter__(self):
        return iter(self._cache)

    def __len__(self):
        return self._cache.__len__()

    def discard(self, key):
        try:
            self.__delitem__(key)
        except KeyError:
            pass

    def flush(self):
        raise NotImplementedError

    def _hit(self):
        self.hits += 1

    def _miss(self):
        self.misses += 1

    @classmethod
    def cache(cls, *args, **kwargs):
        """
        Function decorator.
        """
        typed = kwargs.pop('typed', False)
        from functools import wraps, _make_key

        def decorator(f):
            kwargs['get_missing'] = f
            instance = cls(*args, **kwargs)

            @wraps(f)
            def wrapped(*args, **kwargs):
                key = _make_key(args, kwargs, typed)
                return instance[key]

            wrapped.__cache__ = instance

            return wrapped

        return decorator

class LRUCache(Cache):
    """
    Implementation of a cache with a last-recently-used replacement policy.

    LRU is susceptible to cache pollution caused by iterating over a
    sufficiently large number of unique items (scan vulnerability).
    """
    def __init__(self, *args, **kwargs):

        super(LRUCache, self).__init__(*args, **kwargs)

        # Mapping of keys to links.
        self._cache = {}

        # Cache consists of a doubly-linked list implementing a circular
        # queue with the following layout:
        # <- (prev) (next) ->
        # ItemN (MRU) <-> root (empty) <-> Item0 (LRU), Item1 <-> ...
        # The order of items in the queue is the eviction order.
        self._root = make_circular_queue(slots=3)

    def __getitem__(self, key):
        root = self._root
        with self._lock:
            try:
                link = self._cache[key]
            except KeyError:
                # Cache miss.
                assert self._miss() or True

                value = self.get_missing(key)

                if self.size < self.maxsize:
                    # Cache can hold the new item. Create a new link and put it
                    # in the MRU (end of queue).
                    mru = root[LINK_PREV]
                    link = [mru, root, key, value]
                    self._cache[key] = mru[LINK_NEXT] = root[LINK_PREV] = link
                    self.size += 1
                else:
                    # Replace the current root with link.
                    root[LINK_KEY] = key
                    root[LINK_VALUE] = value
                    link = self._cache[key] = root

                    # Evict LRU.
                    lru = root[LINK_NEXT]
                    del self._cache[lru[LINK_KEY]]

                    # Clear LRU and make it the new root.
                    lru[LINK_KEY] = lru[LINK_VALUE] = None
                    self._root = lru

            else:
                # Cache hit.
                assert self._hit() or True

                prev, next, _, value, _ = link

                # Remove the link from its current position.
                prev[LINK_NEXT] = next
                next[LINK_PREV] = prev

                # Move the link to the MRU (end of queue).
                mru = root[LINK_PREV]
                root[LINK_PREV] = mru[LINK_NEXT] = link
                link[LINK_PREV] = mru
                link[LINK_NEXT] = root

        return value

    def __delitem__(self, key):
        with self._lock:
            link = self._cache.pop(key)

            # Remove the link from its current position.
            prev = link[LINK_PREV]
            next = link[LINK_NEXT]
            link[:] = []

            prev[LINK_NEXT] = next
            next[LINK_PREV] = prev

            self.size -= 1

class MQCache(Cache):
    """
    Implementation of a cache using the Multi-Queue algorithm.

    Zhou, Yuanyuan, James Philbin, and Kai Li. "The Multi-Queue Replacement
    Algorithm for Second Level Buffer Caches." USENIX Annual Technical
    Conference, General Track. 2001.
    """

    def __init__(self, *args, m=2, q_out_factor=4, **kwargs):
        """
        By default, `MQCache` uses two queues, making it similar in principle
        to 2Q.

        The number of queues is determined by the adjustable parameter `m`.

        :param m: Number of LRU queues. At least 2 and usually less than 10.
        :param q_out_size: Number of items to keep in the eviction history.
        """

        super(MQCache, self).__init__(*args, **kwargs)

        # m = 1 doesn't give any benefits (it does exactly the same as LRU,
        # but with unnecessary overhead). m < 1 obviously doesn't make sense
        # at all.
        if m < 2:
            raise ValueError('m must be at least 2')

        self.m = m

        # Fast lookup. (Key -> Link)
        self._cache = {}

        # Eviction history.
        self._q_out = OrderedDict()
        self._q_out_size = self.maxsize * q_out_factor

        # LRU queue stack.
        self._queues = [make_circular_queue(slots=4) for _ in range(m)]

        self.life_time = 10 # 10 accesses
        self.current_time = 0

    def __getitem__(self, key):
        q_out = self._q_out
        with self._lock:
            try:
                link = self._cache[key]
            except KeyError:
                # Cache miss.

                if self.size < self.maxsize:
                    link = [None] * 6
                    self.size += 1
                else:
                    link = self.evict()
                    prev, next = link[LINK_PREV], link[LINK_NEXT]

                    # Remove the link from its current location.
                    prev[LINK_NEXT] = next
                    next[LINK_PREV] = prev

                link[LINK_VALUE] = value = self.get_missing(key)
                link[LINK_KEY] = key

                access_count = q_out.pop(key, 0)

            else:
                # Cache hit.
                prev, next, _, value, access_count, _ = link

                # Remove the link from its current location.
                prev[LINK_NEXT] = next
                next[LINK_PREV] = prev

            access_count += 1

            queue_num = self.queue_num(access_count)
            root = self._queues[queue_num]
            old_tail = root[LINK_PREV]

            # Insert link at tail of queue (MRU).
            old_tail[LINK_NEXT] = root[LINK_PREV] = link
            link[LINK_PREV] = old_tail
            link[LINK_NEXT] = root

            link[LINK_ACCESS_COUNT] = access_count
            link[LINK_EXPIRE_TIME] = self.current_time + self.life_time

            # Store fast reference.
            self._cache[key] = link

        self.adjust()

        return value

    def evict(self):
        # Gotcha: it is the responsibility of the caller to remove or
        #         replace (potentially circular) references in the
        #         victim link's LINK_PREV and LINK_NEXT slots.

        with self._lock:
            # Get the first non-empty queue.
            q_out = self._q_out
            for k, root in enumerate(self._queues):
                # Get victim from head (LRU).
                link = root[LINK_NEXT]
                if link is not root:
                    key = link[LINK_KEY]
                    access_count = link[LINK_ACCESS_COUNT]
                    prev, next = link[LINK_PREV], link[LINK_NEXT]

                    # Remove the link.
                    prev[LINK_NEXT] = next[LINK_PREV]
                    next[LINK_PREV] = prev[LINK_NEXT]

                    # Remove victim reference from internal dict.
                    del self._cache[key]

                    # Pop eviction history if it's full.
                    if len(q_out) >= self._q_out_size:
                        q_out.popitem(last=False) # FIFO

                    # Remember key and access count.
                    q_out[key] = access_count

                    # Invoke callback.
                    on_evict = self.on_evict
                    if on_evict is not None:
                        on_evict(key, link[LINK_VALUE])

                    return link

        raise KeyError('cache is empty')

    def adjust(self):
        # Intuition: This causes the head of each queue (except the first) to
        # 'age', possibly moving them to the tail end of a lower queue in the
        # stack. Without this, items with a high access count would linger in
        # the cache even after a drop in access frequency (a type of cache
        # pollution).

        with self._lock:

            self.current_time += 1

            for k in range(1, self.m):
                root = self._queues[k]
                link = root[LINK_NEXT]

                if link is root:
                    # Queue is empty.
                    continue

                if link[LINK_EXPIRE_TIME] < self.current_time:
                    # Demote item to tail of previous queue in stack.
                    root = self._queues[k-1]
                    old_tail = root[LINK_PREV]
                    old_tail[LINK_NEXT] = root[LINK_PREV] = link
                    link[LINK_PREV] = old_tail
                    link[LINK_NEXT] = root

                    link[LINK_EXPIRE_TIME] = (
                        self.current_time + self.life_time)

    def queue_num(self, access_count):
        import math
        return min(int(math.log(access_count, 2)), self.m - 1)

<commit_msg>MQ: _q_out_size needs to be an int.<commit_after>"""
cachelib
--------

"Cache rules everything around me"
    -- Method Man

Pure Python implementations of various cache algorithms.

License: MIT
"""

import collections
import threading

from collections import deque, OrderedDict

# Linked list indexes
LINK_PREV = 0
LINK_NEXT = 1
LINK_KEY = 2
LINK_VALUE = 3
LINK_ACCESS_COUNT = 4
LINK_EXPIRE_TIME = 5

def make_circular_queue(slots):
    root = []
    root[:] = [root, root] + [None] * slots
    return root

class Cache(collections.Mapping):
    def __init__(self, maxsize, get_missing=None, on_evict=None):
        self.maxsize = maxsize
        # Function which returns items missing from the cache.
        if get_missing:
            self.get_missing = get_missing
        # Callback for item eviction.
        self.on_evict = on_evict

        self.size = 0
        self.hits = self.misses = 0

        # Make linked list updates atomic.
        self._lock = threading.RLock()

    def get_missing(self, key):
        raise KeyError(key)

    def __contains__(self, key):
        return self._cache.__contains__(key)

    def __delitem__(self, key):
        raise NotImplementedError

    def __iter__(self):
        return iter(self._cache)

    def __len__(self):
        return self._cache.__len__()

    def discard(self, key):
        try:
            self.__delitem__(key)
        except KeyError:
            pass

    def flush(self):
        raise NotImplementedError

    def _hit(self):
        self.hits += 1

    def _miss(self):
        self.misses += 1

    @classmethod
    def cache(cls, *args, **kwargs):
        """
        Function decorator.
        """
        typed = kwargs.pop('typed', False)
        from functools import wraps, _make_key

        def decorator(f):
            kwargs['get_missing'] = f
            instance = cls(*args, **kwargs)

            @wraps(f)
            def wrapped(*args, **kwargs):
                key = _make_key(args, kwargs, typed)
                return instance[key]

            wrapped.__cache__ = instance

            return wrapped

        return decorator

class LRUCache(Cache):
    """
    Implementation of a cache with a last-recently-used replacement policy.

    LRU is susceptible to cache pollution caused by iterating over a
    sufficiently large number of unique items (scan vulnerability).
    """
    def __init__(self, *args, **kwargs):

        super(LRUCache, self).__init__(*args, **kwargs)

        # Mapping of keys to links.
        self._cache = {}

        # Cache consists of a doubly-linked list implementing a circular
        # queue with the following layout:
        # <- (prev) (next) ->
        # ItemN (MRU) <-> root (empty) <-> Item0 (LRU), Item1 <-> ...
        # The order of items in the queue is the eviction order.
        self._root = make_circular_queue(slots=3)

    def __getitem__(self, key):
        root = self._root
        with self._lock:
            try:
                link = self._cache[key]
            except KeyError:
                # Cache miss.
                assert self._miss() or True

                value = self.get_missing(key)

                if self.size < self.maxsize:
                    # Cache can hold the new item. Create a new link and put it
                    # in the MRU (end of queue).
                    mru = root[LINK_PREV]
                    link = [mru, root, key, value]
                    self._cache[key] = mru[LINK_NEXT] = root[LINK_PREV] = link
                    self.size += 1
                else:
                    # Replace the current root with link.
                    root[LINK_KEY] = key
                    root[LINK_VALUE] = value
                    link = self._cache[key] = root

                    # Evict LRU.
                    lru = root[LINK_NEXT]
                    del self._cache[lru[LINK_KEY]]

                    # Clear LRU and make it the new root.
                    lru[LINK_KEY] = lru[LINK_VALUE] = None
                    self._root = lru

            else:
                # Cache hit.
                assert self._hit() or True

                prev, next, _, value, _ = link

                # Remove the link from its current position.
                prev[LINK_NEXT] = next
                next[LINK_PREV] = prev

                # Move the link to the MRU (end of queue).
                mru = root[LINK_PREV]
                root[LINK_PREV] = mru[LINK_NEXT] = link
                link[LINK_PREV] = mru
                link[LINK_NEXT] = root

        return value

    def __delitem__(self, key):
        with self._lock:
            link = self._cache.pop(key)

            # Remove the link from its current position.
            prev = link[LINK_PREV]
            next = link[LINK_NEXT]
            link[:] = []

            prev[LINK_NEXT] = next
            next[LINK_PREV] = prev

            self.size -= 1

class MQCache(Cache):
    """
    Implementation of a cache using the Multi-Queue algorithm.

    Zhou, Yuanyuan, James Philbin, and Kai Li. "The Multi-Queue Replacement
    Algorithm for Second Level Buffer Caches." USENIX Annual Technical
    Conference, General Track. 2001.
    """

    def __init__(self, *args, m=2, q_out_factor=4, **kwargs):
        """
        By default, `MQCache` uses two queues, making it similar in principle
        to 2Q.

        The number of queues is determined by the adjustable parameter `m`.

        :param m: Number of LRU queues. At least 2 and usually less than 10.
        :param q_out_size: Number of items to keep in the eviction history.
        """

        super(MQCache, self).__init__(*args, **kwargs)

        # m = 1 doesn't give any benefits (it does exactly the same as LRU,
        # but with unnecessary overhead). m < 1 obviously doesn't make sense
        # at all.
        if m < 2:
            raise ValueError('m must be at least 2')

        self.m = m

        # Fast lookup. (Key -> Link)
        self._cache = {}

        # Eviction history.
        self._q_out = OrderedDict()
        self._q_out_size = int(self.maxsize * q_out_factor)

        # LRU queue stack.
        self._queues = [make_circular_queue(slots=4) for _ in range(m)]

        self.life_time = 10 # 10 accesses
        self.current_time = 0

    def __getitem__(self, key):
        q_out = self._q_out
        with self._lock:
            try:
                link = self._cache[key]
            except KeyError:
                # Cache miss.

                if self.size < self.maxsize:
                    link = [None] * 6
                    self.size += 1
                else:
                    link = self.evict()
                    prev, next = link[LINK_PREV], link[LINK_NEXT]

                    # Remove the link from its current location.
                    prev[LINK_NEXT] = next
                    next[LINK_PREV] = prev

                link[LINK_VALUE] = value = self.get_missing(key)
                link[LINK_KEY] = key

                access_count = q_out.pop(key, 0)

            else:
                # Cache hit.
                prev, next, _, value, access_count, _ = link

                # Remove the link from its current location.
                prev[LINK_NEXT] = next
                next[LINK_PREV] = prev

            access_count += 1

            queue_num = self.queue_num(access_count)
            root = self._queues[queue_num]
            old_tail = root[LINK_PREV]

            # Insert link at tail of queue (MRU).
            old_tail[LINK_NEXT] = root[LINK_PREV] = link
            link[LINK_PREV] = old_tail
            link[LINK_NEXT] = root

            link[LINK_ACCESS_COUNT] = access_count
            link[LINK_EXPIRE_TIME] = self.current_time + self.life_time

            # Store fast reference.
            self._cache[key] = link

        self.adjust()

        return value

    def evict(self):
        # Gotcha: it is the responsibility of the caller to remove or
        #         replace (potentially circular) references in the
        #         victim link's LINK_PREV and LINK_NEXT slots.

        with self._lock:
            # Get the first non-empty queue.
            q_out = self._q_out
            for k, root in enumerate(self._queues):
                # Get victim from head (LRU).
                link = root[LINK_NEXT]
                if link is not root:
                    key = link[LINK_KEY]
                    access_count = link[LINK_ACCESS_COUNT]
                    prev, next = link[LINK_PREV], link[LINK_NEXT]

                    # Remove the link.
                    prev[LINK_NEXT] = next[LINK_PREV]
                    next[LINK_PREV] = prev[LINK_NEXT]

                    # Remove victim reference from internal dict.
                    del self._cache[key]

                    # Pop eviction history if it's full.
                    if len(q_out) >= self._q_out_size:
                        q_out.popitem(last=False) # FIFO

                    # Remember key and access count.
                    q_out[key] = access_count

                    # Invoke callback.
                    on_evict = self.on_evict
                    if on_evict is not None:
                        on_evict(key, link[LINK_VALUE])

                    return link

        raise KeyError('cache is empty')

    def adjust(self):
        # Intuition: This causes the head of each queue (except the first) to
        # 'age', possibly moving them to the tail end of a lower queue in the
        # stack. Without this, items with a high access count would linger in
        # the cache even after a drop in access frequency (a type of cache
        # pollution).

        with self._lock:

            self.current_time += 1

            for k in range(1, self.m):
                root = self._queues[k]
                link = root[LINK_NEXT]

                if link is root:
                    # Queue is empty.
                    continue

                if link[LINK_EXPIRE_TIME] < self.current_time:
                    # Demote item to tail of previous queue in stack.
                    root = self._queues[k-1]
                    old_tail = root[LINK_PREV]
                    old_tail[LINK_NEXT] = root[LINK_PREV] = link
                    link[LINK_PREV] = old_tail
                    link[LINK_NEXT] = root

                    link[LINK_EXPIRE_TIME] = (
                        self.current_time + self.life_time)

    def queue_num(self, access_count):
        import math
        return min(int(math.log(access_count, 2)), self.m - 1)
<|endoftext|>


<commit_before># Lint as: python3
"""Metrics for evaluating models on simulated data."""

from absl import logging
import numpy as np
import xarray as xr

def _helper_data_validator(data_inf, predictions):
  # TODO(eklein, mcoram) decide if there's a better way to handle missing data
  """Function to make sure that data and predictions cover the same time.

  Args:
    data_inf: an xr.DataArray containing the ground-truth infections
    predictions: an xr.DataArray representing predicted new_infections with
      dimensions of (location, time, sample, model). Where model is the
       Estimator used to generate the predicted new_infections.
  Returns:
    obs_plus_pred: an xr.Daraset containing the observed and predicted new
      infections at each point in time.
  Raises:
    ValueError: if predictions does not have results for as far into the future
      as data does.
  """
  # TODO(edklein) proper handling of null and/or nan values
  data_end_time = data_inf.time.max()
  pred_end_time = predictions.time.max()

  if data_end_time > pred_end_time:
    raise ValueError('Data extend farther into the future than predictions')

  if np.any(data_inf.time != predictions.time):
    logging.warning('The times in data do not match the times in predicted')

  observed_data = data_inf.sel(
      time=(data_inf.time < predictions.time.min()))
  obs_plus_pred = xr.concat(
      xr.broadcast(observed_data, predictions, exclude=['time']), dim='time')

  obs_plus_pred_clip = obs_plus_pred.sel(
      time=(obs_plus_pred.time <= data_inf.time.max()))

  return obs_plus_pred_clip

def _helper_construct_dataarray(ground_truth, predictions):
  """Helper function to construct an xr.DataArray with an 'value_type' dim.

  Args:
    ground_truth: an xr.DataArray containing the ground truth values
      for a give metric. Has dims (model, location, sample).
    predictions: an xr.DataArray containing the predicted values for a given
      metric. Has dims (model, location, sample).
  Returns:
    all_values: an xr.DataArray containing the ground truth and predicted values
      as well as their difference. Has dims (value_type, model, location,
      sample)
  """
  all_values = xr.concat(
      [ground_truth, predictions, ground_truth-predictions], dim='value_type')
  all_values.coords['value_type'] = ['ground_truth', 'predicted', 'difference']
  return all_values

def total_size_error(data_inf, predictions):
  """Calculate the error in the predicted cumulative epidemic size.

  Returns the error in the predicted vs. true cumulative epidemic sizes.
  The (signed) error is true - predicted.

  Args:
    data_inf: an xr.DataArray containing the ground-truth infections
    predictions: an xr.DataArray representing predicted new_infections with
      dimensions of (model, location, sample, time). Where model is the
       Estimator used to generate the predicted new_infections.
  Returns:
    total_size: an xr.DataArray containing the ground truth and predicted total
      sizes as well as their difference. Has dims (value_type, model, location,
      sample)
  """
  obs_plus_pred = _helper_data_validator(data_inf, predictions)

  true_total_size = data_inf.sum('time')

  # Calculate the predicted size
  pred_total_size = obs_plus_pred.sum('time')

  return _helper_construct_dataarray(true_total_size, pred_total_size)

def helper_time_percent_complete(inf, percent_complete=0.25):
  """Helper function to calculate when the infection is percent_complete.

  Args:
    inf: an xr.DataArray representing new_infections with
      dimensions of (location, sample, time).
    percent_complete: a float representing the percent of the epidemic that we
      want to see completed.
  Returns:
    true_time: an xr.DataArray containing the ground truth time of
      the percent complete. Has dims (location, sample).
  """
  # Calculate the total epidemic size
  total_size = inf.sum('time')
  target_size = percent_complete * total_size
  cumulative_infections = inf.cumsum()
  target_time = cumulative_infections.where(
      cumulative_infections <= target_size).argmax('time')
  return target_time

def time_percent_complete_error(data_inf, predictions, percent_complete=0.25):
  """Calculate the error in the predicted cumulative epidemic size.

  Returns the error in the predicted vs. true cumulative epidemic sizes.
  The (signed) error is true - predicted.

  Args:
    data_inf: an xr.DataArray containing the ground-truth infections
    predictions: an xr.DataArray representing predicted new_infections with
      dimensions of (model, location, sample, time). Where model is the
      Estimator used to generate the predicted new_infections.
    percent_complete: a float representing the percent of the epidemic that we
      want to see completed.
  Returns:
    time_percent_complete: an xr.DataArray containing the ground truth and
      predicted times as well as their difference. Has dims
      (value_type, model, location, sample)
  """
  obs_plus_pred = _helper_data_validator(data_inf, predictions)

  target_time = helper_time_percent_complete(obs_plus_pred, percent_complete)

  true_time = helper_time_percent_complete(data_inf, percent_complete)

  return _helper_construct_dataarray(true_time, target_time)

def peak_size_error(data_inf, predictions):
  """Calculate the error in the predicted maximum number of infections.

  Returns the error in the predicted vs. true peak epidemic sizes.
  Peak epidemic size is defined as the maximum number of new infections
  in a single time step.
  The (signed) error is true - predicted.

  Args:
    data_inf: an xr.DataArray containing the ground-truth infections
    predictions: an xr.DataArray representing predicted new_infections with
      dimensions of (model, location, sample, time). Where model is the
       Estimator used to generate the predicted new_infections.
  Returns:
    peak_size: an xr.DataArray containing the ground truth and predicted peak
      sizes as well as their difference. Has dims (value_type, model, location,
      sample)
  """
  true_peak_size = data_inf.max('time')

  # Sometimes we'll have already observed the infection peak
  # Need to account for this, so we'll look at the max between
  # observed and predicted
  obs_plus_pred = _helper_data_validator(data_inf, predictions)
  pred_peak_size = obs_plus_pred.max('time')

  return _helper_construct_dataarray(true_peak_size, pred_peak_size)

def peak_time_error(data_inf, predictions):
  """Calculate the error in the predicted maximum number of infections.

  Returns the error in the time of the predicted vs. true peak epidemic.
  Peak epidemic time is defined as the time coordinate with the maximum number
  of new infections in a single time step.
  The (signed) error is true - predicted.

  Args:
    data_inf: an xr.DataArray containing the ground-truth infections
    predictions: an xr.DataArray representing predicted new_infections with
      dimensions of (location, time, sample, model). Where model is the
       Estimator used to generate the predicted new_infections.
  Returns:
    peak_time: an xr.DataArray containing the ground truth and predicted peak
      times as well as their difference. Has dims (value_type, model, location,
      sample)
  """
  true_peak_time = data_inf.argmax('time', skipna=True)

  obs_plus_pred = _helper_data_validator(data_inf, predictions)
  pred_peak_time = obs_plus_pred.argmax('time', skipna=True)

  return _helper_construct_dataarray(true_peak_time, pred_peak_time)
<commit_msg>Google export<commit_after># Lint as: python3
"""Metrics for evaluating models on simulated data."""

from absl import logging
import numpy as np
import xarray as xr

def _helper_data_validator(data_inf, predictions):
  # TODO(eklein, mcoram) decide if there's a better way to handle missing data
  """Function to make sure that data and predictions cover the same time.

  Args:
    data_inf: an xr.DataArray containing the ground-truth infections
    predictions: an xr.DataArray representing predicted new_infections with
      dimensions of (location, time, sample, model). Where model is the
       Estimator used to generate the predicted new_infections.
  Returns:
    obs_plus_pred: an xr.Daraset containing the observed and predicted new
      infections at each point in time.
  Raises:
    ValueError: if predictions does not have results for as far into the future
      as data does.
  """
  # TODO(edklein) proper handling of null and/or nan values
  data_end_time = data_inf.time.max()
  pred_end_time = predictions.time.max()

  if data_end_time > pred_end_time:
    raise ValueError('Data extend farther into the future than predictions')

  if np.any(data_inf.time != predictions.time):
    logging.warning('The times in data do not match the times in predicted')

  observed_data = data_inf.sel(
      time=(data_inf.time < predictions.time.min()))
  obs_plus_pred = xr.concat(
      xr.broadcast(observed_data, predictions, exclude=['time']), dim='time')

  obs_plus_pred_clip = obs_plus_pred.sel(
      time=(obs_plus_pred.time <= data_inf.time.max()))

  return obs_plus_pred_clip

def _helper_construct_dataarray(ground_truth, predictions):
  """Helper function to construct an xr.DataArray with an 'value_type' dim.

  Args:
    ground_truth: an xr.DataArray containing the ground truth values
      for a give metric. Has dims (model, location, sample).
    predictions: an xr.DataArray containing the predicted values for a given
      metric. Has dims (model, location, sample).
  Returns:
    all_values: an xr.DataArray containing the ground truth and predicted values
      as well as their difference. Has dims (value_type, model, location,
      sample)
  """
  all_values = xr.concat(
      [ground_truth, predictions, ground_truth-predictions], dim='value_type')
  all_values.coords['value_type'] = ['ground_truth', 'predicted', 'difference']
  return all_values

def total_size_error(data_inf, predictions):
  """Calculate the error in the predicted cumulative epidemic size.

  Returns the error in the predicted vs. true cumulative epidemic sizes.
  The (signed) error is true - predicted.

  Args:
    data_inf: an xr.DataArray containing the ground-truth infections
    predictions: an xr.DataArray representing predicted new_infections with
      dimensions of (model, location, sample, time). Where model is the
       Estimator used to generate the predicted new_infections.
  Returns:
    total_size: an xr.DataArray containing the ground truth and predicted total
      sizes as well as their difference. Has dims (value_type, model, location,
      sample)
  """
  obs_plus_pred = _helper_data_validator(data_inf, predictions)

  true_total_size = data_inf.sum('time')

  # Calculate the predicted size
  pred_total_size = obs_plus_pred.sum('time')

  return _helper_construct_dataarray(true_total_size, pred_total_size)

def helper_time_percent_complete(inf, percent_complete=0.25):
  """Helper function to calculate when the infection is percent_complete.

  Args:
    inf: an xr.DataArray representing new_infections with
      dimensions of (location, sample, time).
    percent_complete: a float representing the percent of the epidemic that we
      want to see completed.
  Returns:
    true_time: an xr.DataArray containing the ground truth time of
      the percent complete. Has dims (location, sample).
  """
  # Calculate the total epidemic size
  total_size = inf.sum('time')
  target_size = percent_complete * total_size
  cumulative_infections = inf.cumsum('time')
  target_time = cumulative_infections.where(
      cumulative_infections <= target_size).argmax('time')
  return target_time

def time_percent_complete_error(data_inf, predictions, percent_complete=0.25):
  """Calculate the error in the predicted cumulative epidemic size.

  Returns the error in the predicted vs. true cumulative epidemic sizes.
  The (signed) error is true - predicted.

  Args:
    data_inf: an xr.DataArray containing the ground-truth infections
    predictions: an xr.DataArray representing predicted new_infections with
      dimensions of (model, location, sample, time). Where model is the
      Estimator used to generate the predicted new_infections.
    percent_complete: a float representing the percent of the epidemic that we
      want to see completed.
  Returns:
    time_percent_complete: an xr.DataArray containing the ground truth and
      predicted times as well as their difference. Has dims
      (value_type, model, location, sample)
  """
  obs_plus_pred = _helper_data_validator(data_inf, predictions)

  target_time = helper_time_percent_complete(obs_plus_pred, percent_complete)

  true_time = helper_time_percent_complete(data_inf, percent_complete)

  return _helper_construct_dataarray(true_time, target_time)

def peak_size_error(data_inf, predictions):
  """Calculate the error in the predicted maximum number of infections.

  Returns the error in the predicted vs. true peak epidemic sizes.
  Peak epidemic size is defined as the maximum number of new infections
  in a single time step.
  The (signed) error is true - predicted.

  Args:
    data_inf: an xr.DataArray containing the ground-truth infections
    predictions: an xr.DataArray representing predicted new_infections with
      dimensions of (model, location, sample, time). Where model is the
       Estimator used to generate the predicted new_infections.
  Returns:
    peak_size: an xr.DataArray containing the ground truth and predicted peak
      sizes as well as their difference. Has dims (value_type, model, location,
      sample)
  """
  true_peak_size = data_inf.max('time')

  # Sometimes we'll have already observed the infection peak
  # Need to account for this, so we'll look at the max between
  # observed and predicted
  obs_plus_pred = _helper_data_validator(data_inf, predictions)
  pred_peak_size = obs_plus_pred.max('time')

  return _helper_construct_dataarray(true_peak_size, pred_peak_size)

def peak_time_error(data_inf, predictions):
  """Calculate the error in the predicted maximum number of infections.

  Returns the error in the time of the predicted vs. true peak epidemic.
  Peak epidemic time is defined as the time coordinate with the maximum number
  of new infections in a single time step.
  The (signed) error is true - predicted.

  Args:
    data_inf: an xr.DataArray containing the ground-truth infections
    predictions: an xr.DataArray representing predicted new_infections with
      dimensions of (location, time, sample, model). Where model is the
       Estimator used to generate the predicted new_infections.
  Returns:
    peak_time: an xr.DataArray containing the ground truth and predicted peak
      times as well as their difference. Has dims (value_type, model, location,
      sample)
  """
  true_peak_time = data_inf.argmax('time', skipna=True)

  obs_plus_pred = _helper_data_validator(data_inf, predictions)
  pred_peak_time = obs_plus_pred.argmax('time', skipna=True)

  return _helper_construct_dataarray(true_peak_time, pred_peak_time)
<|endoftext|>


<commit_before>#ckwg +28
# Copyright 2012-2013 by Kitware, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#  * Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
#
#  * Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
#  * Neither name of Kitware, Inc. nor the names of any contributors may be used
#    to endorse or promote products derived from this software without specific
#    prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS''
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

def find_tests(scope):
    prefix = 'test_'
    tests = {}

    for name, obj in scope.items():
        if name.startswith(prefix) and callable(obj):
            tests[name[len(prefix):]] = obj

    return tests

def test_error(msg):
    import sys

    sys.stderr.write("Error: %s\n" % msg)

def expect_exception(action, kind, func, *args, **kwargs):
    got_exception = False

    try:
        func(*args, **kwargs)
    except kind:
        got_exception = True
    except BaseException:
        import sys
        import traceback

        e = sys.exc_info()[1]
        bt = sys.exc_info()[2]
        bt_str = ''.join(traceback.format_tb(bt))

        test_error("Got unexpected exception: %s:\n%s" % (str(e), bt_str))

        got_exception = True
    except:
        test_error("Got non-standard exception")

        got_exception = True

    if not got_exception:
        test_error("Did not get exception when %s" % action)

def run_test(testname, tests, *args, **kwargs):
    if testname not in tests:
        import sys

        test_error("No such test '%s'" % testname)

        sys.exit(1)

    try:
        tests[testname](*args, **kwargs)
    except BaseException:
        import sys
        import traceback

        e = sys.exc_info()[1]
        bt = sys.exc_info()[2]
        bt_str = ''.join(traceback.format_tb(bt))

        test_error("Unexpected exception: %s:\n%s" % (str(e), bt_str))
<commit_msg>Improve exception output for Python tests<commit_after>#ckwg +28
# Copyright 2012-2013 by Kitware, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#  * Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
#
#  * Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
#  * Neither name of Kitware, Inc. nor the names of any contributors may be used
#    to endorse or promote products derived from this software without specific
#    prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS''
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

def find_tests(scope):
    prefix = 'test_'
    tests = {}

    for name, obj in scope.items():
        if name.startswith(prefix) and callable(obj):
            tests[name[len(prefix):]] = obj

    return tests

def test_error(msg):
    import sys

    sys.stderr.write("Error: %s\n" % msg)

def expect_exception(action, kind, func, *args, **kwargs):
    got_exception = False

    try:
        func(*args, **kwargs)
    except kind:
        import sys

        t = sys.exc_info()[0]
        e = sys.exc_info()[1]

        sys.stderr.write("Got expected exception: %s: %s\n" % (str(t.__name__), str(e)))

        got_exception = True
    except BaseException:
        import sys
        import traceback

        t = sys.exc_info()[0]
        e = sys.exc_info()[1]
        bt = sys.exc_info()[2]
        bt_str = ''.join(traceback.format_tb(bt))

        test_error("Got unexpected exception: %s: %s:\n%s" % (str(t.__name__), str(e), bt_str))

        got_exception = True
    except:
        test_error("Got non-standard exception")

        got_exception = True

    if not got_exception:
        test_error("Did not get exception when %s" % action)

def run_test(testname, tests, *args, **kwargs):
    if testname not in tests:
        import sys

        test_error("No such test '%s'" % testname)

        sys.exit(1)

    try:
        tests[testname](*args, **kwargs)
    except BaseException:
        import sys
        import traceback

        e = sys.exc_info()[1]
        bt = sys.exc_info()[2]
        bt_str = ''.join(traceback.format_tb(bt))

        test_error("Unexpected exception: %s:\n%s" % (str(e), bt_str))
<|endoftext|>


<commit_before>"""The Truncated Gaussian distribution."""
import numpy as np
from scipy.special import erf, erfinv, gamma, beta, betainc, gammainc
from distribution import Distribution
from gaussian import *
from scipy.stats import truncnorm
RECURRENCE_PDF_SAMPLES = 8000

class TruncatedGaussian(Distribution):
    """
    The class defines a Truncated-Gaussian object. It is the child of Distribution.
    :param double mean:
		Mean of the truncated Gaussian distribution.
	:param double variance:
		Variance of the truncated Gaussian distribution.
    :param double lower:
        Lower bound of the truncated Gaussian distribution.
    :param double upper:
        Upper bound of the truncated Gaussian distribution.
    """
    def __init__(self, mean, variance, lower, upper):
        if (mean is not None) and (variance is not None) and (lower is not None) and (upper is not None):
            meanParent = mean
            varianceParent = variance
            self.std = Gaussian(mean = 0.0, variance = 1.0)
            self.parent = Gaussian(mean = meanParent, variance = varianceParent)
            self.lower = lower
            self.upper = upper
            self.skewness = 0.0
            self.kurtosis = 0.0
            self.bounds = np.array([-np.inf, np.inf])
            self.beta  = (self.upper - self.parent.mean)/np.sqrt(self.parent.variance)
            self.alpha = (self.lower - meanParent)/np.sqrt(varianceParent)
            self.x_range_for_pdf = np.linspace(self.lower, self.upper, RECURRENCE_PDF_SAMPLES)

            self.parents = truncnorm(a =self.alpha , b =self.beta, loc=meanParent, scale=np.sqrt(varianceParent))
            self.mean = self.parents.mean()
            self.variance = self.parents.var()
            self.sigma = np.sqrt(self.variance)

    def getDescription(self):
        """
        A description of the truncated Gaussian.

        :param truncated Gaussian self:
            An instance of the truncated Gaussian class.
        :return:
            A string describing the truncated Gaussian.
        """
        text = "A truncated Gaussian distribution with a mean of "+str(self.mean)+" and a variance of "+str(self.variance)+", and a lower bound of "+str(self.lower)+" and an upper bound of "+str(self.upper)+"."
        return text

    def getPDF(self, points=None):
        """
        A truncated Gaussian probability distribution.

        :param truncated Gaussian self:
            An instance of the truncated Gaussian class.
	:param matrix points:
            Matrix of points over the support of the distribution; default value is 500.
        :return:
            An array of N values over the support of the distribution.
        :return:
            Probability density values along the support of the truncated Gaussian distribution.
        """
        #if points is not None:
        #    num = self.parent.getPDF(points = points)
        #    den = self.parent.getCDF(points= self.upper)-self.parent.getCDF(points=self.lower)
        #    w = num/den
        #    return w
        if points is not None:
            return self.parents.pdf(points)
        else:
            raise(ValueError, 'Please digit an input for getPDF method')

    def getCDF(self, points = None):
        """
        A truncated Gaussian cumulative density function.

	:param truncated Gaussian self:
            An instance of the Gaussian class.
        :param matrix points:
            Matrix of points for defining the cumulative density function; default value is 500.
        :return:
            An array of N values over the support of the truncated Gaussian.
        :return:
            Gaussian cumulative density values.
        """
        if points is not None:
            return self.parents.cdf(points)
        else:
            raise(ValueError, 'Please digit an input for getCDF method')

    def getiCDF(self, xx):
        """ A truncated gaussian inverse cumulative density function,

        :param truncnorm:
            An instance of Truncated-Gaussian class.
        :param array xx:
            A matrix of points at which the inverse of cumulative density function needs to be evaluated.
        :return:
            Inverse cumulative density function values of the Truncated Gaussian distributuion.
        """
        return self.parents.ppf(xx)

    def getSamples(self, m=None):
        """ Generates samples from the Truncated-Gaussian distribution.

         :param trunc-norm self:
             An instance of the Truncated-Gaussian class.
         :param integer m:
             Number of random samples. If no value is provided, a default of     5e5 is assumed.
         :return:
             A N-by-1 vector that contains the samples.
        """
        if m is not None:
           number = m
        else:
           number = 500000
        return self.parents.rvs(size=number)

<commit_msg>Delete truncated_gaussian.py<commit_after><|endoftext|>


<commit_before># Copyright (c) 2014-2016 Hewlett Packard Enterprise Development LP
#
# Redistribution and use of this software in source and binary forms,
# with or without modification, are permitted provided that the following
# conditions are met:
#
#   Redistributions of source code must retain the above copyright notice,
#   this list of conditions and the following disclaimer.
#
#   Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in the
#   documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

from requestbuilder import Arg, MutuallyExclusiveArgList

from euca2ools.commands.argtypes import delimited_list
from euca2ools.commands.cloudformation import CloudFormationRequest
from euca2ools.commands.cloudformation.argtypes import parameter_list

class UpdateStack(CloudFormationRequest):
    DESCRIPTION = 'Update a stack with a new template'
    ARGS = [Arg('StackName', metavar='STACK',
                help='name of the stack to update (required)'),
            MutuallyExclusiveArgList(
                Arg('--template-file', dest='TemplateBody',
                    metavar='FILE', type=open,
                    help='file containing a new JSON template for the stack'),
                Arg('--template-url', dest='TemplateURL', metavar='URL',
                    help='URL pointing to a new JSON template for the stack'))
            .required(),
            Arg('--capabilities', dest='Capabilities.member',
                metavar='CAP[,...]', type=delimited_list(','),
                help='capabilities needed to update the stack'),
            Arg('-p', '--parameter', dest='param_sets', route_to=None,
                metavar='KEY=VALUE', type=parameter_list, action='append',
                help='''key and value of the parameters to use with the
                stack's template, separated by an "=" character''')]

    def configure(self):
        CloudFormationRequest.configure(self)
        stack_params = sum(self.args.get('param_sets') or [], [])
        self.params['Parameters.member'] = stack_params

    def print_result(self, result):
        print result.get('StackId')
<commit_msg>Add tag support to euform-update-stack<commit_after># Copyright (c) 2014-2016 Hewlett Packard Enterprise Development LP
#
# Redistribution and use of this software in source and binary forms,
# with or without modification, are permitted provided that the following
# conditions are met:
#
#   Redistributions of source code must retain the above copyright notice,
#   this list of conditions and the following disclaimer.
#
#   Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in the
#   documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

from requestbuilder import Arg, EMPTY, MutuallyExclusiveArgList

from euca2ools.commands.argtypes import binary_tag_def, delimited_list
from euca2ools.commands.cloudformation import CloudFormationRequest
from euca2ools.commands.cloudformation.argtypes import parameter_list

class UpdateStack(CloudFormationRequest):
    DESCRIPTION = 'Update a stack with a new template'
    ARGS = [Arg('StackName', metavar='STACK',
                help='name of the stack to update (required)'),
            MutuallyExclusiveArgList(
                Arg('--template-file', dest='TemplateBody',
                    metavar='FILE', type=open,
                    help='file containing a new JSON template for the stack'),
                Arg('--template-url', dest='TemplateURL', metavar='URL',
                    help='URL pointing to a new JSON template for the stack'))
            .required(),
            Arg('--capabilities', dest='Capabilities.member',
                metavar='CAP[,...]', type=delimited_list(','),
                help='capabilities needed to update the stack'),
            Arg('-p', '--parameter', dest='param_sets', route_to=None,
                metavar='KEY=VALUE', type=parameter_list, action='append',
                help='''key and value of the parameters to use with the
                stack's template, separated by an "=" character'''),
            MutuallyExclusiveArgList(
                Arg('--tag', dest='Tags.member', metavar='KEY[=VALUE]',
                    type=binary_tag_def, action='append',
                    help='''key and optional value of a tag to add, separated
                    by an "=" character.  If no value is given the tag's value
                    is set to an empty string.'''),
                Arg('--delete-tags', dest='Tags', action='store_const',
                    const=EMPTY,
                    help='remove all tags associated with the stack'))]

    def configure(self):
        CloudFormationRequest.configure(self)
        stack_params = sum(self.args.get('param_sets') or [], [])
        self.params['Parameters.member'] = stack_params

    def print_result(self, result):
        print result.get('StackId')
<|endoftext|>