<commit_before>
from aql_utils import toSequence

class   UniqueList (object):

  __slots__ = (
    '__values_list',
    '__values_set',
  )

  #//-------------------------------------------------------//

  def     __init__( self, values = None ):

    self.__values_list = []
    self.__values_set = set()

    self.__addValues( values )

  #//-------------------------------------------------------//

  def   __addValueFront( self, value ):

    values_set = self.__values_set
    values_list = self.__values_list

    if value in values_set:
      values_list.remove( value )
    else:
      values_set.add( value )

    values_list.insert( 0, value )

  #//-------------------------------------------------------//

  def   __addValue( self, value ):

    values_set = self.__values_set
    values_list = self.__values_list

    if value not in values_set:
      values_set.add( value )
      values_list.append( value )

  #//-------------------------------------------------------//

  def   __addValues( self, values ):

    values_set_add = self.__values_set.add
    values_list_append = self.__values_list.append
    values_set_size = self.__values_set.__len__
    values_list_size = self.__values_list.__len__

    for value in toSequence( values ):
      values_set_add( value )
      if values_set_size() > values_list_size():
        values_list_append( value )

  #//-------------------------------------------------------//

  def   __addValuesFront( self, values ):

    values_set = self.__values_set
    values_list = self.__values_list

    values_set_add = values_set.add
    values_list_append = values_list.append
    values_set_size = values_set.__len__
    values_list_size = values_list.__len__
    values_list_index = values_list.index

    pos = 0

    for value in toSequence( values ):
      values_set_add( value )
      if values_set_size() == values_list_size():
        i = values_list_index( value )
        if i < pos:
          continue

        del values_list[ i ]

      values_list.insert( pos, value )

      pos += 1

  #//-------------------------------------------------------//

  def   __removeValue( self, value ):

    try:
      self.__values_set.remove( value )
      self.__values_list.remove( value )
    except (KeyError, ValueError):
      pass

  #//-------------------------------------------------------//

  def   __removeValues( self, values ):

    values_set_remove = self.__values_set.remove
    values_list_remove = self.__values_list.remove

    for value in toSequence( values ):
      try:
        values_set_remove( value )
        values_list_remove( value )
      except (KeyError, ValueError):
        pass

  #//-------------------------------------------------------//

  def   __contains__( self, other ):
    return other in self.__values_set

  #//-------------------------------------------------------//

  def   __len__( self ):
    return len(self.__values_list)

  #//-------------------------------------------------------//

  def   __iter__( self ):
    return iter(self.__values_list)

  #//-------------------------------------------------------//

  def   __reversed__( self ):
    return reversed(self.__values_list)

  #//-------------------------------------------------------//

  def   __str__( self ):
      return str(self.__values_list)

  #//-------------------------------------------------------//

  def   __eq__( self, other ):
    if isinstance( other, UniqueList ):
      return self.__values_set == other.__values_set

    return self.__values_set == set( toSequence( other ) )

  #//-------------------------------------------------------//

  def   __ne__( self, other ):
    if isinstance( other, UniqueList ):
      return self.__values_set != other.__values_set

    return self.__values_set != set( toSequence( other ) )

  #//-------------------------------------------------------//

  def   __lt__( self, other ):
    if not isinstance( other, UniqueList ):
      other = UniqueList( other )

    return self.__values_list < other.__values_list

  #//-------------------------------------------------------//

  def   __le__( self, other ):
    if isinstance( other, UniqueList ):
      other = UniqueList( other )

    return self.__values_list <= other.__values_list

  def   __gt__( self, other ):
    if isinstance( other, UniqueList ):
      other = UniqueList( other )

    return self.__values_list > other.__values_list

  #//-------------------------------------------------------//

  def   __ge__( self, other ):
    if isinstance( other, UniqueList ):
      other = UniqueList( other )

    return self.__values_list >= other.__values_list

  #//-------------------------------------------------------//

  def   __getitem__(self, index ):
    return self.__values_list[ index ]

  #//-------------------------------------------------------//

  def   __iadd__(self, values ):
    self.__addValues( values )
    return self

  #//-------------------------------------------------------//

  def   __isub__(self, values ):
    self.__removeValues( values )
    return self

  #//-------------------------------------------------------//

  def   append( self, value ):
    self.__addValue( value )

  #//-------------------------------------------------------//

  def   extend( self, values ):
    self.__addValues( values )

  #//-------------------------------------------------------//

  def   reverse( self ):
    self.__values_list.reverse()

  #//-------------------------------------------------------//

  def   append_front( self, value ):
    self.__addValueFront( value )

  #//-------------------------------------------------------//

  def   extend_front( self, values ):
    self.__addValuesFront( values )

  #//-------------------------------------------------------//

  def   remove( self, value ):
    self.__removeValue( value )

  #//-------------------------------------------------------//

  def   pop( self ):
    value = self.__values_list.pop()
    self.__values_set.remove( value )

    return value

  #//-------------------------------------------------------//

  def   pop_front( self ):
    value = self.__values_list.pop(0)
    self.__values_set.remove( value )

    return value

  #//-------------------------------------------------------//

  def   selfTest( self ):
    size = len(self)
    if size != len(self.__values_list):
      raise AssertionError("size(%s) != len(self.__values_list)(%s)" % (size, len(self.__values_list)) )

    if size != len(self.__values_set):
      raise AssertionError("size(%s) != len(self.__values_set)(%s)" % (size, len(self.__values_set)) )

    if self.__values_set != set(self.__values_list):
      raise AssertionError( "self.__values_set != self.__values_list" )

#//===========================================================================//

class   List (list):

  #//-------------------------------------------------------//

  def     __init__( self, values = None ):
    super( List, self).__init__( toSequence( values ) )

  #//-------------------------------------------------------//

  def   __iadd__( self, values ):
    return super(List,self).__iadd__( toSequence(values) )

  #//-------------------------------------------------------//

  def   __isub__( self, values ):
    for value in toSequence(values):
      while True:
        try:
          self.remove( value )
        except ValueError:
          break

    return self

  #//-------------------------------------------------------//

  def   append_front( self, value ):
    self.insert( 0, value )

  #//-------------------------------------------------------//

  def   extend( self, values ):
    super(List, self).extend( toSequence( values ) )

  #//-------------------------------------------------------//

  def   extend_front( self, values ):
    self[:0] = toSequence( values )

  #//-------------------------------------------------------//

  def   pop_front( self ):
    return self.__values_list.pop(0)

#//===========================================================================//

def   SplitListType( list_type, separators ):

  separator = separators[0]
  other_separators = separators[1:]

  class   SplitList (list_type):

    #//-------------------------------------------------------//

    def   __toSequence( self, values, separator = separator, other_separators = other_separators ):
      if not isinstance( values, str ):
        return values

      for sep in other_separators:
        values = values.replace( sep, separator )

      return filter( None, values.split( separator ))

    #//-------------------------------------------------------//

    def   __toSplitList( self, values ):
      if isinstance( values, SplitList ):
        return values

      return SplitList( values )

    #//-------------------------------------------------------//

    def     __init__( self, values = None ):
      super(SplitList,self).__init__( self.__toSequence( values ) )

    #//-------------------------------------------------------//

    def   __iadd__( self, values ):
      return super(SplitList,self).__iadd__( self.__toSequence( values ) )

    #//-------------------------------------------------------//

    def   __isub__( self, values ):
      return super(SplitList,self).__isub__( self.__toSequence( values ) )

    #//-------------------------------------------------------//

    def   extend( self, values ):
      super(SplitList,self).extend( self.__toSequence( values ) )

    #//-------------------------------------------------------//

    def   extend_front( self, values ):
      super(SplitList,self).extend_front( self.__toSequence( values ) )

    #//-------------------------------------------------------//

    def   __eq__( self, other ):
      return super(SplitList,self).__eq__( self.__toSplitList( other ) )
    def   __ne__( self, other ):
      return super(SplitList,self).__ne__( self.__toSplitList( other ) )
    def   __lt__( self, other ):
      return super(SplitList,self).__lt__( self.__toSplitList( other ) )
    def   __le__( self, other ):
      return super(SplitList,self).__le__( self.__toSplitList( other ) )
    def   __gt__( self, other ):
      return super(SplitList,self).__gt__( self.__toSplitList( other ) )
    def   __ge__( self, other ):
      return super(SplitList,self).__ge__( self.__toSplitList( other ) )

    #//-------------------------------------------------------//

    def   __str__( self ):
      return separator.join( map( str, iter(self) ) )

  #//=======================================================//

  return SplitList

#//===========================================================================//

def   ValueListType( list_type, value_type ):

  class   _ValueList (list_type):

    #//-------------------------------------------------------//

    def   __toSequence( self, values ):
      if isinstance( values, _ValueList):
        return values

      return map( value_type, toSequence(values) )

    #//-------------------------------------------------------//

    def   __toValueList( self, values ):
      if isinstance( values, _ValueList):
        return values

      return _ValueList(values)

    #//-------------------------------------------------------//

    def     __init__( self, values = None ):
      super(_ValueList,self).__init__( self.__toSequence( values ) )

    #//-------------------------------------------------------//

    def   __iadd__( self, values ):
      return super(_ValueList,self).__iadd__( self.__toSequence( values ) )

    #//-------------------------------------------------------//

    def   __isub__( self, values ):
      return super(_ValueList,self).__isub__( self.__toSequence( values ) )

    #//-------------------------------------------------------//

    def   extend( self, values ):
      super(_ValueList,self).extend( self.__toSequence( values ) )

    #//-------------------------------------------------------//

    def   extend_front( self, values ):
      super(_ValueList,self).extend_front( self.__toSequence( values ) )

    #//-------------------------------------------------------//

    def   __eq__( self, other ):
      return super(_ValueList,self).__eq__( self.__toValueList( other ) )
    def   __ne__( self, other ):
      return super(_ValueList,self).__ne__( self.__toValueList( other ) )
    def   __lt__( self, other ):
      return super(_ValueList,self).__lt__( self.__toValueList( other ) )
    def   __le__( self, other ):
      return super(_ValueList,self).__le__( self.__toValueList( other ) )
    def   __gt__( self, other ):
      return super(_ValueList,self).__gt__( self.__toValueList( other ) )
    def   __ge__( self, other ):
      return super(_ValueList,self).__ge__( self.__toValueList( other ) )

    #//-------------------------------------------------------//

    def   __contains__( self, other ):
      return super(_ValueList,self).__contains__( value_type(other) )

  #//=======================================================//

  return _ValueList
<commit_msg><commit_after>
from aql_utils import toSequence

class   UniqueList (object):

  __slots__ = (
    '__values_list',
    '__values_set',
  )

  #//-------------------------------------------------------//

  def     __init__( self, values = None ):

    self.__values_list = []
    self.__values_set = set()

    self.__addValues( values )

  #//-------------------------------------------------------//

  def   __addValueFront( self, value ):

    values_set = self.__values_set
    values_list = self.__values_list

    if value in values_set:
      values_list.remove( value )
    else:
      values_set.add( value )

    values_list.insert( 0, value )

  #//-------------------------------------------------------//

  def   __addValue( self, value ):

    values_set = self.__values_set
    values_list = self.__values_list

    if value not in values_set:
      values_set.add( value )
      values_list.append( value )

  #//-------------------------------------------------------//

  def   __addValues( self, values ):

    values_set_add = self.__values_set.add
    values_list_append = self.__values_list.append
    values_set_size = self.__values_set.__len__
    values_list_size = self.__values_list.__len__

    for value in toSequence( values ):
      values_set_add( value )
      if values_set_size() > values_list_size():
        values_list_append( value )

  #//-------------------------------------------------------//

  def   __addValuesFront( self, values ):

    values_set = self.__values_set
    values_list = self.__values_list

    values_set_add = values_set.add
    values_list_append = values_list.append
    values_set_size = values_set.__len__
    values_list_size = values_list.__len__
    values_list_index = values_list.index

    pos = 0

    for value in toSequence( values ):
      values_set_add( value )
      if values_set_size() == values_list_size():
        i = values_list_index( value )
        if i < pos:
          continue

        del values_list[ i ]

      values_list.insert( pos, value )

      pos += 1

  #//-------------------------------------------------------//

  def   __removeValue( self, value ):

    try:
      self.__values_set.remove( value )
      self.__values_list.remove( value )
    except (KeyError, ValueError):
      pass

  #//-------------------------------------------------------//

  def   __removeValues( self, values ):

    values_set_remove = self.__values_set.remove
    values_list_remove = self.__values_list.remove

    for value in toSequence( values ):
      try:
        values_set_remove( value )
        values_list_remove( value )
      except (KeyError, ValueError):
        pass

  #//-------------------------------------------------------//

  def   __contains__( self, other ):
    return other in self.__values_set

  #//-------------------------------------------------------//

  def   __len__( self ):
    return len(self.__values_list)

  #//-------------------------------------------------------//

  def   __iter__( self ):
    return iter(self.__values_list)

  #//-------------------------------------------------------//

  def   __reversed__( self ):
    return reversed(self.__values_list)

  #//-------------------------------------------------------//

  def   __str__( self ):
      return str(self.__values_list)

  #//-------------------------------------------------------//

  def   __eq__( self, other ):
    if isinstance( other, UniqueList ):
      return self.__values_set == other.__values_set

    return self.__values_set == set( toSequence( other ) )

  #//-------------------------------------------------------//

  def   __ne__( self, other ):
    if isinstance( other, UniqueList ):
      return self.__values_set != other.__values_set

    return self.__values_set != set( toSequence( other ) )

  #//-------------------------------------------------------//

  def   __lt__( self, other ):
    if not isinstance( other, UniqueList ):
      other = UniqueList( other )

    return self.__values_list < other.__values_list

  #//-------------------------------------------------------//

  def   __le__( self, other ):
    if isinstance( other, UniqueList ):
      other = UniqueList( other )

    return self.__values_list <= other.__values_list

  def   __gt__( self, other ):
    if isinstance( other, UniqueList ):
      other = UniqueList( other )

    return self.__values_list > other.__values_list

  #//-------------------------------------------------------//

  def   __ge__( self, other ):
    if isinstance( other, UniqueList ):
      other = UniqueList( other )

    return self.__values_list >= other.__values_list

  #//-------------------------------------------------------//

  def   __getitem__(self, index ):
    return self.__values_list[ index ]

  #//-------------------------------------------------------//

  def   __iadd__(self, values ):
    self.__addValues( values )
    return self

  #//-------------------------------------------------------//

  def   __isub__(self, values ):
    self.__removeValues( values )
    return self

  #//-------------------------------------------------------//

  def   append( self, value ):
    self.__addValue( value )

  #//-------------------------------------------------------//

  def   extend( self, values ):
    self.__addValues( values )

  #//-------------------------------------------------------//

  def   reverse( self ):
    self.__values_list.reverse()

  #//-------------------------------------------------------//

  def   append_front( self, value ):
    self.__addValueFront( value )

  #//-------------------------------------------------------//

  def   extend_front( self, values ):
    self.__addValuesFront( values )

  #//-------------------------------------------------------//

  def   remove( self, value ):
    self.__removeValue( value )

  #//-------------------------------------------------------//

  def   pop( self ):
    value = self.__values_list.pop()
    self.__values_set.remove( value )

    return value

  #//-------------------------------------------------------//

  def   pop_front( self ):
    value = self.__values_list.pop(0)
    self.__values_set.remove( value )

    return value

  #//-------------------------------------------------------//

  def   selfTest( self ):
    size = len(self)
    if size != len(self.__values_list):
      raise AssertionError("size(%s) != len(self.__values_list)(%s)" % (size, len(self.__values_list)) )

    if size != len(self.__values_set):
      raise AssertionError("size(%s) != len(self.__values_set)(%s)" % (size, len(self.__values_set)) )

    if self.__values_set != set(self.__values_list):
      raise AssertionError( "self.__values_set != self.__values_list" )

#//===========================================================================//

class   List (list):

  #//-------------------------------------------------------//

  def     __init__( self, values = None ):
    super( List, self).__init__( toSequence( values ) )

  #//-------------------------------------------------------//

  def   __iadd__( self, values ):
    return super(List,self).__iadd__( toSequence(values) )

  #//-------------------------------------------------------//

  def   __isub__( self, values ):
    for value in toSequence(values):
      while True:
        try:
          self.remove( value )
        except ValueError:
          break

    return self

  #//-------------------------------------------------------//

  def   append_front( self, value ):
    self.insert( 0, value )

  #//-------------------------------------------------------//

  def   extend( self, values ):
    super(List, self).extend( toSequence( values ) )

  #//-------------------------------------------------------//

  def   extend_front( self, values ):
    self[:0] = toSequence( values )

  #//-------------------------------------------------------//

  def   pop_front( self ):
    return self.__values_list.pop(0)

#//===========================================================================//

def   SplitListType( list_type, separators ):

  separator = separators[0]
  other_separators = separators[1:]

  class   SplitList (list_type):

    #//-------------------------------------------------------//

    def   __toSequence( self, values, separator = separator, other_separators = other_separators ):
      if not isinstance( values, str ):
        return values

      for sep in other_separators:
        values = values.replace( sep, separator )

      return filter( None, values.split( separator ))

    #//-------------------------------------------------------//

    def   __toSplitList( self, values ):
      if isinstance( values, SplitList ):
        return values

      return SplitList( values )

    #//-------------------------------------------------------//

    def     __init__( self, values = None ):
      super(SplitList,self).__init__( self.__toSequence( values ) )

    #//-------------------------------------------------------//

    def   __iadd__( self, values ):
      return super(SplitList,self).__iadd__( self.__toSequence( values ) )

    #//-------------------------------------------------------//

    def   __isub__( self, values ):
      return super(SplitList,self).__isub__( self.__toSequence( values ) )

    #//-------------------------------------------------------//

    def   extend( self, values ):
      super(SplitList,self).extend( self.__toSequence( values ) )

    #//-------------------------------------------------------//

    def   extend_front( self, values ):
      super(SplitList,self).extend_front( self.__toSequence( values ) )

    #//-------------------------------------------------------//

    def   __eq__( self, other ):
      return super(SplitList,self).__eq__( self.__toSplitList( other ) )
    def   __ne__( self, other ):
      return super(SplitList,self).__ne__( self.__toSplitList( other ) )
    def   __lt__( self, other ):
      return super(SplitList,self).__lt__( self.__toSplitList( other ) )
    def   __le__( self, other ):
      return super(SplitList,self).__le__( self.__toSplitList( other ) )
    def   __gt__( self, other ):
      return super(SplitList,self).__gt__( self.__toSplitList( other ) )
    def   __ge__( self, other ):
      return super(SplitList,self).__ge__( self.__toSplitList( other ) )

    #//-------------------------------------------------------//

    def   __str__( self ):
      return separator.join( map( str, iter(self) ) )

  #//=======================================================//

  return SplitList

#//===========================================================================//

def   ValueListType( list_type, value_type ):

  class   _ValueList (list_type):

    #//-------------------------------------------------------//

    def   __toSequence( self, values ):
      if isinstance( values, _ValueList):
        return values

      return map( value_type, toSequence(values) )

    #//-------------------------------------------------------//

    def   __toValueList( self, values ):
      if isinstance( values, _ValueList):
        return values

      return _ValueList(values)

    #//-------------------------------------------------------//

    def     __init__( self, values = None ):
      super(_ValueList,self).__init__( self.__toSequence( values ) )

    def   __iadd__( self, values ):
      return super(_ValueList,self).__iadd__( self.__toSequence( values ) )

    def   __isub__( self, values ):
      return super(_ValueList,self).__isub__( self.__toSequence( values ) )

    def   extend( self, values ):
      super(_ValueList,self).extend( self.__toSequence( values ) )

    def   extend_front( self, values ):
      super(_ValueList,self).extend_front( self.__toSequence( values ) )

    def   append( self, value ):
      super(_ValueList,self).append( value_type(value) )

    def   count( self, value ):
      return super(_ValueList,self).count( value_type(value) )

    def   index( self, value, i = 0, j = -1 ):
      return super(_ValueList,self).index( value_type(value), i, j )

    def   insert( self, i, value ):
      return super(_ValueList,self).insert( i, value_type(value) )

    def   remove( self, value ):
      return super(_ValueList,self).remove( value_type(value) )

    def   __setitem__( self, index, value ):
      return super(_ValueList,self).__setitem__( index, value_type(value) )

    #//-------------------------------------------------------//

    def   __eq__( self, other ):
      return super(_ValueList,self).__eq__( self.__toValueList( other ) )
    def   __ne__( self, other ):
      return super(_ValueList,self).__ne__( self.__toValueList( other ) )
    def   __lt__( self, other ):
      return super(_ValueList,self).__lt__( self.__toValueList( other ) )
    def   __le__( self, other ):
      return super(_ValueList,self).__le__( self.__toValueList( other ) )
    def   __gt__( self, other ):
      return super(_ValueList,self).__gt__( self.__toValueList( other ) )
    def   __ge__( self, other ):
      return super(_ValueList,self).__ge__( self.__toValueList( other ) )

    #//-------------------------------------------------------//

    def   __contains__( self, other ):
      return super(_ValueList,self).__contains__( value_type(other) )

  #//=======================================================//

  return _ValueList
<|endoftext|>


<commit_before>from flask import Flask, render_template, url_for
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from database_setup import connection_str

app = Flask(__name__)

@app.route('/')
@app.route('/restaurants/')
def home():
    return render_template('restaurants_list.html', restaurants=None)

def new_restaurant():
    return None

def edit_restaurant(restaurant_id):
    return None

def delete_restaurant(restaurant_id):
    return None

@app.route('/restaurants/<int:restaurant_id>/menu/')
def menu(restaurant_id):
    return render_template('menu_list.html')

def new_menu(restaurant_id):
    return None

def update_menu(restaurant_id, menu_id):
    return None

def delete_menu_item(restaurant_id, menu_id):
    return None

if __name__ == '__main__':
    app.debug = True
    app.run('0.0.0.0', port=5000)
<commit_msg>Connected to the postgresql database and added restaurants<commit_after>from flask import Flask, render_template, url_for
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from database_setup import connection_str

app = Flask(__name__)

engine = create_engine(connection_str)
DBSession = sessionmaker(bind=engine)
session = DBSession()

@app.route('/')
@app.route('/restaurants/')
def home():
    restaurants = session.query().all()
    return render_template('restaurants_list.html', restaurants=restaurants)

def new_restaurant():
    return None

def edit_restaurant(restaurant_id):
    return None

def delete_restaurant(restaurant_id):
    return None

@app.route('/restaurants/<int:restaurant_id>/menu/')
def menu(restaurant_id):
    return render_template('menu_list.html')

def new_menu(restaurant_id):
    return None

def update_menu(restaurant_id, menu_id):
    return None

def delete_menu_item(restaurant_id, menu_id):
    return None

if __name__ == '__main__':
    app.debug = True
    app.run('0.0.0.0', port=5000)
<|endoftext|>


<commit_before>#!/usr/bin/env python

__author__ = 'setten'

import timeit

n = 10000

def test_write():
    """Stupid test function"""
    l = []
    for i in range(100000):
        l.append(i)
    f = open('test', 'w')
    f.write(str(l))
    f.close()

if __name__ == '__main__':
    n = 100
    print(timeit.timeit("test_write()", setup="from __main__ import test_write", number=n))
<commit_msg>test times<commit_after>#!/usr/bin/env python

__author__ = 'setten'

import timeit
import os

def test_write():
    """file write test function"""
    l = []
    for i in range(100000):
        l.append(i)
    f = open('test', 'w')
    f.write(str(l))
    f.close()

def test_read():
    f = open('test', mode='r')
    line = f.read()
    f.close()

def test_folders():
    for n in range(0, 100, 1):
        os.mkdir('test_folder')
        os.chdir('test_folder')
        os.chdir('..')
        os.rmdir('test_folder')

if __name__ == '__main__':
    n = 100
    print 'write', timeit.timeit("test_write()", setup="from __main__ import test_write", number=n)
    print 'read', timeit.timeit("test_read()", setup="from __main__ import test_read", number=n)
    print 'folders', timeit.timeit("test_folders()", setup="from __main__ import test_folders", number=n)
<|endoftext|>


<commit_before>#!/usr/bin/python

# Profile you are referencing for authentication
# profile = "firstwatch"  # for local testing
profile = ""

# Region you want to run in.
# region = "us-east-1"  # for local testing
region = "us-west-2"

# Comma delimited array of quoted instance names to encrypt the root drive.
# names = ["CLPJDEA5005","CLPJDEA5007"]
names = ["Test"]

# Customer Master Key to use if you're not using the default AWS key.  Full ARN.
customer_master_key = ""
<commit_msg>updating config<commit_after>#!/usr/bin/python

# Profile you are referencing for authentication
# profile = "myprofile"  # for local testing
profile = ""

# Region you want to run in.
# region = "us-east-1"  # for local testing
region = "us-west-2"

# Comma delimited array of quoted instance names to encrypt the root drive.
names = ["Test1", "Test2"]

# Customer Master Key to use if you're not using the default AWS key.  Full ARN.
customer_master_key = ""
<|endoftext|>


<commit_before>#!/usr/bin/env python

# TODO
# Why are records over & underclumped? ie. Charles Linder 7475467, Nirmala Ramanujam 7570988, Edwin L. Thomas 7799416
# Investigate "WEIRDNESS" output : Jaro-Winkler exception thrown on comparison between  and ROBERT BRUCE
# Underclumping: unique records vs. total records

import sqlite3, sys, csv, datetime;
sys.path.append( './lib/' )
sys.path.append("lib")
from fwork import *;

def bmVerify(results, filepath="", outdir = ""):
        """
        Analysis function on disambiguation results, assuming that all benchmark data
        are in the large results dataset.

        Creates analysis detail csv file and prints summary information on
        over- and underclumping statistics.

        Running from the command line (make sure to set correct file paths in file)
        python bmVerify_v3.py "input filepath" "output directory" databases
        example:
        python bmVerify_v3.py /home/ysun/disambig/newcode/all/ /home/ayu/results_v2/ invpatC_NBNA.good.Jan2011 invpatC_NBYA.good.Jan2011 invpatC_YBNA.good.Jan2011

        Running interactively:
        import bmVerify_v3
        bmVerify(['final_r7', 'final_r8'], filepath="/home/ysun/disambig/newcode/all/", outdir = "/home/ayu/results_v2/")

        """
        for result in results:
<<<<<<< HEAD
                uqB = "lower"
                tblB = "invpat"
                #fileS = "/home/ron/disambig/BM/DefTruth5.csv"
                #fileS = "/var/share/patentdata/benchmarks/DefTruth5.csv"
                #fileS = "/home/alvin/src/patentprocessor/test/goldstandard/berkeleyinventors.csv"
                #fileS = "/home/alvin/src/patentprocessor/test/goldstandard/berkeleyinventors.csv"
                fileS = "/home/alvin/src/patentprocessor/test/goldstandard/ikhlaq.csv"
=======
                uqB = "FOOBAR"
                tblB = "invpat"
                #fileS = "/home/ron/disambig/BM/DefTruth5.csv"
                #fileS = "/var/share/patentdata/benchmarks/DefTruth5.csv"
                fileS = "/var/share/patentdata/disambiguation/experiments/mercury/2/benchmark.csv"
>>>>>>> 39c819113636d5908622990fdaf38898ce2912d4
                fileB = filepath + "{result}.sqlite3".format(result=result)
                output = outdir + "{result}_DT5.csv".format(result=result)

                t=datetime.datetime.now()

                print "Start time: " + str(datetime.datetime.now())
                class freqUQ:
                    def __init__(self):
                        self.list=[]
                    def step(self, value):
                        self.list.append(value)
                    def finalize(self):
                        return sorted([(self.list.count(x), x) for x in set(self.list)], reverse=True)[0][1]

                #MAKE THIS SO IT CAN ATTACH SQLITE3 FOR BENCHMARK
                dataS = uniVert([x for x in csv.reader(open(fileS, "rb"))])

		print dataS

                #1 = Variables, 2 = Type, 3 = Format (If necessary), 4 = Matching Type
                tList = ["%s %s" % (dataS[0][i], x) for i,x in enumerate(dataS[1]) if  x != ""]

                dataS2 = [dataS[0]]
                dataS2.extend(dataS[4:])

                print dataS[2]

                #Format if its necessary --> Basically for Patents..
                for i,x in enumerate(dataS[2]):
                    if x!="":
                        for j in xrange(1,len(dataS2)):
                            if dataS2[j][i].isdigit():
                                dataS2[j][i] = x % int(dataS2[j][i])

                conn = sqlite3.connect(":memory:")
                conn.create_function("jarow", 2, jarow)
                conn.create_function("errD", 2, lambda x,y: (x!=y) and 1 or None)
                conn.create_aggregate("freqUQ", 1, freqUQ)
                c = conn.cursor()

                #FIGURE OUT WHICH ONES HAVE EXACT/FUZZY
                exact = [dataS[0][i] for i,x in enumerate(dataS[3]) if x.upper()[0]=="E"]
		print exact
                fuzzy = [dataS[0][i] for i,x in enumerate(dataS[3]) if x.upper()[0]=="F"]
		print fuzzy
                uqS =   [dataS[0][i] for i,x in enumerate(dataS[3]) if x.upper()[0]=="U"][0]

                #CREATE INDEX, MERGE DATA BASED ON EXACTS
                print "Creating indices... " + str(datetime.datetime.now())
                exAnd = " AND ".join(["a.%s=b.%s" % (x, x) for x in exact])
                exCom = ", ".join(exact)

                if fileB.split(".")[-1].lower()=="csv":
                    dataB = uniVert([x for x in csv.reader(open("%s" % fileB, "rb"))])
                    quickSQL(c, data=dataB,  table="dataB", header=True, typeList=["Patent VARCHAR"])
                    c.execute("CREATE INDEX IF NOT EXISTS dB_E ON dataB (%s)" % exCom)
                    c.execute("CREATE INDEX IF NOT EXISTS dB_U ON dataB (%s)" % uqB)
                    fBnme = "dataB"
                else:
                    c.execute("ATTACH DATABASE '%s' AS db" % fileB)
                    if tblB=="":
                        fBnme = "db.%s" % fileB.split(".")[-2].split("/")[-1]
                    else:
                        fBnme = "db.%s" % tblB

                quickSQL(c, data=dataS2, table="dataS", header=True, typeList=tList)

                if fuzzy:
                    c.execute("CREATE INDEX IF NOT EXISTS dS_E ON dataS (%s);" % (exCom))
                    c.executescript("""
                        CREATE INDEX IF NOT EXISTS dS_E ON dataS (%s);

                        /* RETAIN ONLY JARO>0.9 FUZZY AND EXACT MATCHES */
                        CREATE TABLE dataM AS
                            SELECT  a.*, %s AS uqB, %s AS uqS, %s AS jaro
                              FROM  %s AS a
                        INNER JOIN  dataS AS b
                                ON  %s
                             WHERE  jaro>0.90;

                        /* DETERMINE MAXIMUM JARO FOR EACH UQ AND EXACT COMBO */
                        CREATE TABLE dataT AS
                            SELECT  uqS, %s, MAX(jaro) AS jaro, count(*) as cnt
                              FROM  dataM
                          GROUP BY  uqS, %s;

                        /* RETAIN ONLY MAXIMUM JARO */
                        CREATE TABLE dataM2 AS
                            SELECT  a.*
                              FROM  dataM AS a
                        INNER JOIN  dataT AS b
                                ON  a.uqS=b.uqS AND a.jaro=b.jaro AND %s;
                        """ % (exCom, uqB, uqS,
                               "*".join(["jarow(a.%s, b.%s)" % (x,x) for x in fuzzy]),
                               fBnme, exAnd, exCom, exCom, exAnd))
                else:
                    c.executescript("""
                        CREATE INDEX IF NOT EXISTS dS_E ON dataS (%s);
                        CREATE TABLE dataM2 AS
                            SELECT  *, %s AS uqB, %s AS uqS
                              FROM  %s AS a
                        INNER JOIN  dataS AS b
                                ON  %s;
                        """ % (exCom, uqB, uqS, fBnme, exAnd))

                c.executescript("""
                    /* EXPAND UNIQUE BASE AND INDICATE ACTIVE MATCHES */
                    CREATE TABLE dataM3 AS
                        SELECT  uqS, a.*
                          FROM (SELECT  uqS AS uqSUB, a.*
                                  FROM (SELECT  uqB, b.*
                                          FROM  (SELECT DISTINCT(uqB) FROM dataM2 WHERE uqB!="") AS a
                                    INNER JOIN  %s AS b
                                            ON  a.uqB=b.%s) AS a
                             LEFT JOIN (SELECT %s, uqB, uqS FROM dataM2) AS b
                                    ON  a.uqB=b.uqB AND %s) AS a
                    INNER JOIN (SELECT DISTINCT uqB, uqS FROM dataM2) AS b
                            ON  a.%s=b.uqB;

                    /* INDICATE INVENTORS WHO DO NOT MATCH */
                    CREATE TABLE dataM4 AS
                        SELECT  errD(a.ErrUQ, uqB) AS ErrUQ, b.*
                          FROM (SELECT uqS, freqUQ(uqB) as ErrUQ FROM dataM3 GROUP BY uqS) AS a
                    INNER JOIN  dataM3 AS b
                            ON  a.uqS=b.uqS AND b.AppYear <= '2009' /*AND a.uqS not in (83, 85, 93)*/
                      ORDER BY  uqS, %s;

                    """ % (fBnme, uqB, exCom, exAnd, uqB, exCom))

                print "Indices Done ... " + str(datetime.datetime.now())

                #EXPORT THE RESULTS
                writer = csv.writer(open(output, "wb"), lineterminator="\n")
                writer.writerows([[x[1] for x in c.execute("PRAGMA TABLE_INFO(dataM4)")]])
                writer.writerows(c.execute("SELECT * FROM dataM4").fetchall())
                print "Printing results ..." + str(datetime.datetime.now())
                rep = [list(x) for x in c.execute("SELECT ErrUQ, uqSUB FROM dataM4")]
		print rep
                orig = len([x for x in rep if x[1]!=None])
                errm = sum([int(x[0]) for x in rep if x[0]!=None])
		print errm
                u = 1.0*errm/orig
                o = 1-(float(orig)/len(rep))
                recall = 1.0 - u
                print """

                RESULTS ==================

                     Original: {original}
                  New Records: {new}
                        Total: {total}

                    Overclump: {overclump} ({o:.2%})
                   Underclump: {underclump} ({u:.2%})
                    Precision: {precision:.2%}
                       Recall: {recall:.2%}
                  File Detail: {filename}
                         Time: {time}
                """.format(original = orig, new = len(rep)-orig, total = len(rep), overclump = len(rep)-orig, o = o,
                           underclump = errm, u = u, recall = recall, precision = recall/(recall+o), filename = output, time = datetime.datetime.now()-t)
                c.close()
                conn.close()

if __name__ == "__main__":
    import sys
    if(sys.argv[1] == 'help' or sys.argv[1] == '?'):
        print bmVerify.__doc__

    else:
            bmVerify(sys.argv[3:], sys.argv[1], sys.argv[2])
<commit_msg>fixed conflicts from uqB<commit_after>#!/usr/bin/env python

# TODO
# Why are records over & underclumped? ie. Charles Linder 7475467, Nirmala Ramanujam 7570988, Edwin L. Thomas 7799416
# Investigate "WEIRDNESS" output : Jaro-Winkler exception thrown on comparison between  and ROBERT BRUCE
# Underclumping: unique records vs. total records

import sqlite3, sys, csv, datetime;
sys.path.append( './lib/' )
sys.path.append("lib")
from fwork import *;

def bmVerify(results, filepath="", outdir = ""):
        """
        Analysis function on disambiguation results, assuming that all benchmark data
        are in the large results dataset.

        Creates analysis detail csv file and prints summary information on
        over- and underclumping statistics.

        Running from the command line (make sure to set correct file paths in file)
        python bmVerify_v3.py "input filepath" "output directory" databases
        example:
        python bmVerify_v3.py /home/ysun/disambig/newcode/all/ /home/ayu/results_v2/ invpatC_NBNA.good.Jan2011 invpatC_NBYA.good.Jan2011 invpatC_YBNA.good.Jan2011

        Running interactively:
        import bmVerify_v3
        bmVerify(['final_r7', 'final_r8'], filepath="/home/ysun/disambig/newcode/all/", outdir = "/home/ayu/results_v2/")

        """
        for result in results:
                uqB = "FOOBAR"
                tblB = "invpat"
                #fileS = "/home/ron/disambig/BM/DefTruth5.csv"
                #fileS = "/var/share/patentdata/benchmarks/DefTruth5.csv"
                fileS = "/var/share/patentdata/disambiguation/experiments/mercury/2/benchmark.csv"
                fileB = filepath + "{result}.sqlite3".format(result=result)
                output = outdir + "{result}_DT5.csv".format(result=result)

                t=datetime.datetime.now()

                print "Start time: " + str(datetime.datetime.now())
                class freqUQ:
                    def __init__(self):
                        self.list=[]
                    def step(self, value):
                        self.list.append(value)
                    def finalize(self):
                        return sorted([(self.list.count(x), x) for x in set(self.list)], reverse=True)[0][1]

                #MAKE THIS SO IT CAN ATTACH SQLITE3 FOR BENCHMARK
                dataS = uniVert([x for x in csv.reader(open(fileS, "rb"))])

		print dataS

                #1 = Variables, 2 = Type, 3 = Format (If necessary), 4 = Matching Type
                tList = ["%s %s" % (dataS[0][i], x) for i,x in enumerate(dataS[1]) if  x != ""]

                dataS2 = [dataS[0]]
                dataS2.extend(dataS[4:])

                print dataS[2]

                #Format if its necessary --> Basically for Patents..
                for i,x in enumerate(dataS[2]):
                    if x!="":
                        for j in xrange(1,len(dataS2)):
                            if dataS2[j][i].isdigit():
                                dataS2[j][i] = x % int(dataS2[j][i])

                conn = sqlite3.connect(":memory:")
                conn.create_function("jarow", 2, jarow)
                conn.create_function("errD", 2, lambda x,y: (x!=y) and 1 or None)
                conn.create_aggregate("freqUQ", 1, freqUQ)
                c = conn.cursor()

                #FIGURE OUT WHICH ONES HAVE EXACT/FUZZY
                exact = [dataS[0][i] for i,x in enumerate(dataS[3]) if x.upper()[0]=="E"]
		print exact
                fuzzy = [dataS[0][i] for i,x in enumerate(dataS[3]) if x.upper()[0]=="F"]
		print fuzzy
                uqS =   [dataS[0][i] for i,x in enumerate(dataS[3]) if x.upper()[0]=="U"][0]

                #CREATE INDEX, MERGE DATA BASED ON EXACTS
                print "Creating indices... " + str(datetime.datetime.now())
                exAnd = " AND ".join(["a.%s=b.%s" % (x, x) for x in exact])
                exCom = ", ".join(exact)

                if fileB.split(".")[-1].lower()=="csv":
                    dataB = uniVert([x for x in csv.reader(open("%s" % fileB, "rb"))])
                    quickSQL(c, data=dataB,  table="dataB", header=True, typeList=["Patent VARCHAR"])
                    c.execute("CREATE INDEX IF NOT EXISTS dB_E ON dataB (%s)" % exCom)
                    c.execute("CREATE INDEX IF NOT EXISTS dB_U ON dataB (%s)" % uqB)
                    fBnme = "dataB"
                else:
                    c.execute("ATTACH DATABASE '%s' AS db" % fileB)
                    if tblB=="":
                        fBnme = "db.%s" % fileB.split(".")[-2].split("/")[-1]
                    else:
                        fBnme = "db.%s" % tblB

                quickSQL(c, data=dataS2, table="dataS", header=True, typeList=tList)

                if fuzzy:
                    c.execute("CREATE INDEX IF NOT EXISTS dS_E ON dataS (%s);" % (exCom))
                    c.executescript("""
                        CREATE INDEX IF NOT EXISTS dS_E ON dataS (%s);

                        /* RETAIN ONLY JARO>0.9 FUZZY AND EXACT MATCHES */
                        CREATE TABLE dataM AS
                            SELECT  a.*, %s AS uqB, %s AS uqS, %s AS jaro
                              FROM  %s AS a
                        INNER JOIN  dataS AS b
                                ON  %s
                             WHERE  jaro>0.90;

                        /* DETERMINE MAXIMUM JARO FOR EACH UQ AND EXACT COMBO */
                        CREATE TABLE dataT AS
                            SELECT  uqS, %s, MAX(jaro) AS jaro, count(*) as cnt
                              FROM  dataM
                          GROUP BY  uqS, %s;

                        /* RETAIN ONLY MAXIMUM JARO */
                        CREATE TABLE dataM2 AS
                            SELECT  a.*
                              FROM  dataM AS a
                        INNER JOIN  dataT AS b
                                ON  a.uqS=b.uqS AND a.jaro=b.jaro AND %s;
                        """ % (exCom, uqB, uqS,
                               "*".join(["jarow(a.%s, b.%s)" % (x,x) for x in fuzzy]),
                               fBnme, exAnd, exCom, exCom, exAnd))
                else:
                    c.executescript("""
                        CREATE INDEX IF NOT EXISTS dS_E ON dataS (%s);
                        CREATE TABLE dataM2 AS
                            SELECT  *, %s AS uqB, %s AS uqS
                              FROM  %s AS a
                        INNER JOIN  dataS AS b
                                ON  %s;
                        """ % (exCom, uqB, uqS, fBnme, exAnd))

                c.executescript("""
                    /* EXPAND UNIQUE BASE AND INDICATE ACTIVE MATCHES */
                    CREATE TABLE dataM3 AS
                        SELECT  uqS, a.*
                          FROM (SELECT  uqS AS uqSUB, a.*
                                  FROM (SELECT  uqB, b.*
                                          FROM  (SELECT DISTINCT(uqB) FROM dataM2 WHERE uqB!="") AS a
                                    INNER JOIN  %s AS b
                                            ON  a.uqB=b.%s) AS a
                             LEFT JOIN (SELECT %s, uqB, uqS FROM dataM2) AS b
                                    ON  a.uqB=b.uqB AND %s) AS a
                    INNER JOIN (SELECT DISTINCT uqB, uqS FROM dataM2) AS b
                            ON  a.%s=b.uqB;

                    /* INDICATE INVENTORS WHO DO NOT MATCH */
                    CREATE TABLE dataM4 AS
                        SELECT  errD(a.ErrUQ, uqB) AS ErrUQ, b.*
                          FROM (SELECT uqS, freqUQ(uqB) as ErrUQ FROM dataM3 GROUP BY uqS) AS a
                    INNER JOIN  dataM3 AS b
                            ON  a.uqS=b.uqS AND b.AppYear <= '2009' /*AND a.uqS not in (83, 85, 93)*/
                      ORDER BY  uqS, %s;

                    """ % (fBnme, uqB, exCom, exAnd, uqB, exCom))

                print "Indices Done ... " + str(datetime.datetime.now())

                #EXPORT THE RESULTS
                writer = csv.writer(open(output, "wb"), lineterminator="\n")
                writer.writerows([[x[1] for x in c.execute("PRAGMA TABLE_INFO(dataM4)")]])
                writer.writerows(c.execute("SELECT * FROM dataM4").fetchall())
                print "Printing results ..." + str(datetime.datetime.now())
                rep = [list(x) for x in c.execute("SELECT ErrUQ, uqSUB FROM dataM4")]
		print rep
                orig = len([x for x in rep if x[1]!=None])
                errm = sum([int(x[0]) for x in rep if x[0]!=None])
		print errm
                u = 1.0*errm/orig
                o = 1-(float(orig)/len(rep))
                recall = 1.0 - u
                print """

                RESULTS ==================

                     Original: {original}
                  New Records: {new}
                        Total: {total}

                    Overclump: {overclump} ({o:.2%})
                   Underclump: {underclump} ({u:.2%})
                    Precision: {precision:.2%}
                       Recall: {recall:.2%}
                  File Detail: {filename}
                         Time: {time}
                """.format(original = orig, new = len(rep)-orig, total = len(rep), overclump = len(rep)-orig, o = o,
                           underclump = errm, u = u, recall = recall, precision = recall/(recall+o), filename = output, time = datetime.datetime.now()-t)
                c.close()
                conn.close()

if __name__ == "__main__":
    import sys
    if(sys.argv[1] == 'help' or sys.argv[1] == '?'):
        print bmVerify.__doc__

    else:
            bmVerify(sys.argv[3:], sys.argv[1], sys.argv[2])
<|endoftext|>


<commit_before>#!/usr/bin/env python

#-----------------------------------------------------------------------------
# Copyright (c) 2013, The BiPy Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

__author__ = "Greg Caporaso"
__copyright__ = "Copyright 2013, The QIIME Project"
__credits__ = ["Greg Caporaso", "Daniel McDonald", "Doug Wendel",
               "Jai Ram Rideout"]
__license__ = "BSD"
__version__ = "0.1.0-dev"
__maintainer__ = "Greg Caporaso"
__email__ = "gregcaporaso@gmail.com"

from pyqi.core.interface import Interface, InterfaceOption, \
        InterfaceUsageExample, InterfaceResult
from pyqi.core.factory import general_factory
from pyqi.core.exception import IncompetentDeveloperError
from pyqi.core.command import Parameter
from pyqi.option_parsing import (OptionParser, OptionGroup, Option,
                                 OptionValueError, OptionError, make_option)
import os

CLTypes = set(['float','int','string','existing_filepath', float, int, str,
               None, 'new_filepath','new_dirpath','existing_dirpath'])
CLActions = set(['store','store_true','store_false', 'append'])

def new_filepath(data, path):
    if os.path.exists(path):
        raise IOError("Output path %s already exists." % path)
    f = open(path, 'w')
    f.write(data)
    f.close()

class OptparseResult(InterfaceResult):
    def _validate_result(self):
        ### anything to validate here?
        pass

class OptparseOption(InterfaceOption):
    """An augmented option that expands a Parameter into an Option"""
    def _validate_option(self):
        ### require Name, type, etc???
        pass

    def __str__(self):
        if self.ShortName is None:
            return '--%s' % self.Name
        else:
            return '-%s/--%s' % (self.ShortName, self.Name)

    def getOptparseOption(self):
        if self.Required:
            # If the option doesn't already end with [REQUIRED], add it.
            help_text = self.Help
            if not help_text.strip().endswith('[REQUIRED]'):
                help_text += ' [REQUIRED]'

            if self.ShortName is None:
                option = make_option('--' + self.Name, type=self.CLType,
                                     help=help_text)
            else:
                option = make_option('-' + self.ShortName,
                                     '--' + self.Name, type=self.CLType,
                                     help=help_text)
        else:
            if self.DefaultDescription is None:
                help_text = '%s [default: %%default]' % self.Help
            else:
                help_text = '%s [default: %s]' % (self.Help,
                                                  self.DefaultDescription)

            if self.ShortName is None:
                option = make_option('--' + self.Name, type=self.CLType,
                                     help=help_text, default=self.Default)
            else:
                option = make_option('-' + self.ShortName,
                                     '--' + self.Name, type=self.CLType,
                                     help=help_text, default=self.Default)
        return option

class OptparseUsageExample(InterfaceUsageExample):
    """Provide structure to a usage example"""
    def _validate_usage_example(self):
        if self.ShortDesc is None:
            raise UsageExampleError("Must define ShortDesc")
        if self.LongDesc is None:
            raise UsageExampleError("Must define LongDesc")
        if self.Ex is None:
            raise UsageExampleError("Must define Ex")

class OptparseInterface(Interface):
    """A command line interface"""
    DisallowPositionalArguments = True
    HelpOnNoArguments = True
    OptionalInputLine = '[] indicates optional input (order unimportant)'
    RequiredInputLine = '{} indicates required input (order unimportant)'

    def __init__(self, **kwargs):
        self.BelovedFunctionality = {}
        self.UsageExamples = []
        self.UsageExamples.extend(self._get_usage_examples())

        if len(self.UsageExamples) < 1:
            raise IncompetentDeveloperError("There are no usage examples "
                                            "associated with this command.")

        super(OptparseInterface, self).__init__(**kwargs)

    def _the_in_validator(self, in_):
        """Validate input coming from the command line"""
        if not isinstance(in_, list):
            raise IncompetentDeveloperError("Unsupported input '%r'. Input "
                                            "must be a list." % in_)

    def _input_handler(self, in_, *args, **kwargs):
        """Parses command-line input."""
        required_opts = [opt for opt in self.Options if opt.Required]
        optional_opts = [opt for opt in self.Options if not opt.Required]

        # Build the usage and version strings
        usage = self._build_usage_lines(required_opts)
        version = 'Version: %prog ' + __version__

        # Instantiate the command line parser object
        parser = OptionParser(usage=usage, version=version)

        # If no arguments were provided, print the help string (unless the
        # caller specified not to).
        if self.HelpOnNoArguments and len(in_) == 0:
            parser.print_usage()
            return parser.exit(-1)

        if required_opts:
            # Define an option group so all required options are grouped
            # together and under a common header.
            required = OptionGroup(parser, "REQUIRED options",
                                   "The following options must be provided "
                                   "under all circumstances.")
            for ro in required_opts:
                required.add_option(ro.getOptparseOption())
            parser.add_option_group(required)

        # Add the optional options.
        for oo in optional_opts:
            parser.add_option(oo.getOptparseOption())

        #####
        # THIS IS THE NATURAL BREAKING POINT FOR THIS FUNCTIONALITY
        #####

        # Parse our input.
        opts, args = parser.parse_args(in_)

        # If positional arguments are not allowed, and any were provided, raise
        # an error.
        if self.DisallowPositionalArguments and len(args) != 0:
            parser.error("Positional argument detected: %s\n" % str(args[0]) +
             " Be sure all parameters are identified by their option name.\n" +
             " (e.g.: include the '-i' in '-i INPUT_DIR')")

        # Test that all required options were provided.
        if required_opts:
            required_option_ids = [o.dest for o in required.option_list]
            for required_option_id in required_option_ids:
                if getattr(opts,required_option_id) == None:
                    parser.error('Required option --%s omitted.' %
                                 required_option_id)

        beloved_functionality = opts.__dict__
        self.BelovedFunctionality = beloved_functionality
        for k, v in self.ParameterConversionInfo.items():
            if v.InHandler is not None:
                long_name = v.LongName
                value = self.BelovedFunctionality[long_name]
                self.BelovedFunctionality[k] = v.InHandler(value)
            else:
                pass
        return self.BelovedFunctionality

    def _build_usage_lines(self, required_options):
        """ Build the usage string from components """
        line1 = 'usage: %prog [options] ' + \
                '{%s}' % ' '.join(['%s %s' % (str(rp),rp.Name.upper())
                                   for rp in required_options])

        formatted_usage_examples = []
        for usage_example in self.UsageExamples:
            short_description = usage_example.ShortDesc.strip(':').strip()
            long_description = usage_example.LongDesc.strip(':').strip()
            example = usage_example.Ex.strip()

            if short_description:
                formatted_usage_examples.append('%s: %s\n %s' %
                                                (short_description,
                                                 long_description, example))
            else:
                formatted_usage_examples.append('%s\n %s' %
                                                (long_description,example))

        formatted_usage_examples = '\n\n'.join(formatted_usage_examples)

        lines = (line1,
                 '', # Blank line
                 self.OptionalInputLine,
                 self.RequiredInputLine,
                 '', # Blank line
                 self.CmdInstance.LongDescription,
                 '', # Blank line
                 'Example usage: ',
                 'Print help message and exit',
                 ' %prog -h\n',
                 formatted_usage_examples)

        return '\n'.join(lines)

    def _output_handler(self, results):
        """Deal with things in output if we know how"""
        for k,handler in self._get_output_map().items():
            if k not in results:
                raise IncompetentDeveloperError("Did not find the expected "
                                                "output '%s' in results." % k)

            if handler.OptionName is None:
                results[k] = handler.Function(k, results[k])
            else:
                opt_value = self.BelovedFunctionality[handler.OptionName]
                results[k] = handler.Function(k, results[k], opt_value)

def optparse_factory(command_constructor, usage_examples, inputs, outputs):
    """Optparse command line interface factory

    command_constructor - a subclass of ``Command``
    usage_examples - usage examples for using ``command_constructor`` on via a
        command line interface.
    inputs  - config ``inputs`` or a list of ``OptparseOptions``
    otuputs - config ``outputs`` or a list of ``OptparseResults``
    """
    return general_factory(command_constructor, usage_examples, inputs, outputs,
                           OptparseInterface)

def optparse_main(interface_object, local_argv):
    """Construct and execute an interface object"""
    optparse_cmd = interface_object()
    result = optparse_cmd(local_argv[1:])
    return 0
<commit_msg>input handler uses self._get_inputs CLType -> InputType removed CLType/CLAction lookups<commit_after>#!/usr/bin/env python

#-----------------------------------------------------------------------------
# Copyright (c) 2013, The BiPy Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

__author__ = "Greg Caporaso"
__copyright__ = "Copyright 2013, The QIIME Project"
__credits__ = ["Greg Caporaso", "Daniel McDonald", "Doug Wendel",
               "Jai Ram Rideout"]
__license__ = "BSD"
__version__ = "0.1.0-dev"
__maintainer__ = "Greg Caporaso"
__email__ = "gregcaporaso@gmail.com"

from pyqi.core.interface import Interface, InterfaceOption, \
        InterfaceUsageExample, InterfaceResult
from pyqi.core.factory import general_factory
from pyqi.core.exception import IncompetentDeveloperError
from pyqi.core.command import Parameter
from pyqi.option_parsing import (OptionParser, OptionGroup, Option,
                                 OptionValueError, OptionError, make_option)
import os

def new_filepath(data, path):
    if os.path.exists(path):
        raise IOError("Output path %s already exists." % path)
    f = open(path, 'w')
    f.write(data)
    f.close()

class OptparseResult(InterfaceResult):
    def _validate_result(self):
        ### anything to validate here?
        pass

class OptparseOption(InterfaceOption):
    """An augmented option that expands a Parameter into an Option"""
    def _validate_option(self):
        ### require Name, type, etc???
        pass

    def __str__(self):
        if self.ShortName is None:
            return '--%s' % self.Name
        else:
            return '-%s/--%s' % (self.ShortName, self.Name)

    def getOptparseOption(self):
        if self.Required:
            # If the option doesn't already end with [REQUIRED], add it.
            help_text = self.Help
            if not help_text.strip().endswith('[REQUIRED]'):
                help_text += ' [REQUIRED]'

            if self.ShortName is None:
                option = make_option('--' + self.Name, type=self.InputType,
                                     help=help_text)
            else:
                option = make_option('-' + self.ShortName,
                                     '--' + self.Name, type=self.InputType,
                                     help=help_text)
        else:
            if self.DefaultDescription is None:
                help_text = '%s [default: %%default]' % self.Help
            else:
                help_text = '%s [default: %s]' % (self.Help,
                                                  self.DefaultDescription)

            if self.ShortName is None:
                option = make_option('--' + self.Name, type=self.InputType,
                                     help=help_text, default=self.Default)
            else:
                option = make_option('-' + self.ShortName,
                                     '--' + self.Name, type=self.InputType,
                                     help=help_text, default=self.Default)
        return option

class OptparseUsageExample(InterfaceUsageExample):
    """Provide structure to a usage example"""
    def _validate_usage_example(self):
        if self.ShortDesc is None:
            raise UsageExampleError("Must define ShortDesc")
        if self.LongDesc is None:
            raise UsageExampleError("Must define LongDesc")
        if self.Ex is None:
            raise UsageExampleError("Must define Ex")

class OptparseInterface(Interface):
    """A command line interface"""
    DisallowPositionalArguments = True
    HelpOnNoArguments = True
    OptionalInputLine = '[] indicates optional input (order unimportant)'
    RequiredInputLine = '{} indicates required input (order unimportant)'

    def __init__(self, **kwargs):
        self.BelovedFunctionality = {}
        self.UsageExamples = []
        self.UsageExamples.extend(self._get_usage_examples())

        if len(self.UsageExamples) < 1:
            raise IncompetentDeveloperError("There are no usage examples "
                                            "associated with this command.")

        super(OptparseInterface, self).__init__(**kwargs)

    def _the_in_validator(self, in_):
        """Validate input coming from the command line"""
        if not isinstance(in_, list):
            raise IncompetentDeveloperError("Unsupported input '%r'. Input "
                                            "must be a list." % in_)

    def _input_handler(self, in_, *args, **kwargs):
        """Parses command-line input."""
        required_opts = [opt for opt in self._get_inputs() if opt.Required]
        optional_opts = [opt for opt in self._get_inputs() if not opt.Required]

        # Build the usage and version strings
        usage = self._build_usage_lines(required_opts)
        version = 'Version: %prog ' + __version__

        # Instantiate the command line parser object
        parser = OptionParser(usage=usage, version=version)

        # If no arguments were provided, print the help string (unless the
        # caller specified not to).
        if self.HelpOnNoArguments and len(in_) == 0:
            parser.print_usage()
            return parser.exit(-1)

        if required_opts:
            # Define an option group so all required options are grouped
            # together and under a common header.
            required = OptionGroup(parser, "REQUIRED options",
                                   "The following options must be provided "
                                   "under all circumstances.")
            for ro in required_opts:
                required.add_option(ro.getOptparseOption())
            parser.add_option_group(required)

        # Add the optional options.
        for oo in optional_opts:
            parser.add_option(oo.getOptparseOption())

        #####
        # THIS IS THE NATURAL BREAKING POINT FOR THIS FUNCTIONALITY
        #####

        # Parse our input.
        opts, args = parser.parse_args(in_)

        # If positional arguments are not allowed, and any were provided, raise
        # an error.
        if self.DisallowPositionalArguments and len(args) != 0:
            parser.error("Positional argument detected: %s\n" % str(args[0]) +
             " Be sure all parameters are identified by their option name.\n" +
             " (e.g.: include the '-i' in '-i INPUT_DIR')")

        # Test that all required options were provided.
        if required_opts:
            required_option_ids = [o.dest for o in required.option_list]
            for required_option_id in required_option_ids:
                if getattr(opts,required_option_id) == None:
                    parser.error('Required option --%s omitted.' %
                                 required_option_id)

        beloved_functionality = opts.__dict__
        self.BelovedFunctionality = beloved_functionality
        for k, v in self.ParameterConversionInfo.items():
            if v.InHandler is not None:
                long_name = v.LongName
                value = self.BelovedFunctionality[long_name]
                self.BelovedFunctionality[k] = v.InHandler(value)
            else:
                pass
        return self.BelovedFunctionality

    def _build_usage_lines(self, required_options):
        """ Build the usage string from components """
        line1 = 'usage: %prog [options] ' + \
                '{%s}' % ' '.join(['%s %s' % (str(rp),rp.Name.upper())
                                   for rp in required_options])

        formatted_usage_examples = []
        for usage_example in self.UsageExamples:
            short_description = usage_example.ShortDesc.strip(':').strip()
            long_description = usage_example.LongDesc.strip(':').strip()
            example = usage_example.Ex.strip()

            if short_description:
                formatted_usage_examples.append('%s: %s\n %s' %
                                                (short_description,
                                                 long_description, example))
            else:
                formatted_usage_examples.append('%s\n %s' %
                                                (long_description,example))

        formatted_usage_examples = '\n\n'.join(formatted_usage_examples)

        lines = (line1,
                 '', # Blank line
                 self.OptionalInputLine,
                 self.RequiredInputLine,
                 '', # Blank line
                 self.CmdInstance.LongDescription,
                 '', # Blank line
                 'Example usage: ',
                 'Print help message and exit',
                 ' %prog -h\n',
                 formatted_usage_examples)

        return '\n'.join(lines)

    def _output_handler(self, results):
        """Deal with things in output if we know how"""
        for k,handler in self._get_output_map().items():
            if k not in results:
                raise IncompetentDeveloperError("Did not find the expected "
                                                "output '%s' in results." % k)

            if handler.OptionName is None:
                results[k] = handler.Function(k, results[k])
            else:
                opt_value = self.BelovedFunctionality[handler.OptionName]
                results[k] = handler.Function(k, results[k], opt_value)

def optparse_factory(command_constructor, usage_examples, inputs, outputs):
    """Optparse command line interface factory

    command_constructor - a subclass of ``Command``
    usage_examples - usage examples for using ``command_constructor`` on via a
        command line interface.
    inputs  - config ``inputs`` or a list of ``OptparseOptions``
    otuputs - config ``outputs`` or a list of ``OptparseResults``
    """
    return general_factory(command_constructor, usage_examples, inputs, outputs,
                           OptparseInterface)

def optparse_main(interface_object, local_argv):
    """Construct and execute an interface object"""
    optparse_cmd = interface_object()
    result = optparse_cmd(local_argv[1:])
    return 0
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
##
## Author: Adriano Monteiro Marques <adriano@umitproject.org>
## Author: Diogo Pinheiro <diogormpinheiro@gmail.com>
##
## Copyright (C) 2011 S2S Network Consultoria e Tecnologia da Informacao LTDA
##
## This program is free software: you can redistribute it and/or modify
## it under the terms of the GNU Affero General Public License as
## published by the Free Software Foundation, either version 3 of the
## License, or (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
## GNU Affero General Public License for more details.
##
## You should have received a copy of the GNU Affero General Public License
## along with this program.  If not, see <http://www.gnu.org/licenses/>.
##

from icm_tests.models import *
from django.contrib import admin

admin.site.register(WebsiteTest)
admin.site.register(ServiceTest)
admin.site.register(WebsiteTestUpdateAggregation)
admin.site.register(ServiceTestUpdateAggregation)<commit_msg>Register test aggregation models into admin.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
##
## Author: Adriano Monteiro Marques <adriano@umitproject.org>
## Author: Diogo Pinheiro <diogormpinheiro@gmail.com>
##
## Copyright (C) 2011 S2S Network Consultoria e Tecnologia da Informacao LTDA
##
## This program is free software: you can redistribute it and/or modify
## it under the terms of the GNU Affero General Public License as
## published by the Free Software Foundation, either version 3 of the
## License, or (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
## GNU Affero General Public License for more details.
##
## You should have received a copy of the GNU Affero General Public License
## along with this program.  If not, see <http://www.gnu.org/licenses/>.
##

from icm_tests.models import *
from django.contrib import admin

admin.site.register(WebsiteTest)
admin.site.register(ServiceTest)

# Register aggregation models.
admin.site.register(WebsiteTestsGlobalAggregation)
admin.site.register(WebsiteTestsCountryAggregation)
admin.site.register(WebsiteTestsRegionAggregation)
admin.site.register(WebsiteTestsCityAggregation)
admin.site.register(ServiceTestsGlobalAggregation)
admin.site.register(ServiceTestsCountryAggregation)
admin.site.register(ServiceTestsRegionAggregation)
admin.site.register(ServiceTestsCityAggregation)
<|endoftext|>


<commit_before># coding=utf-8
from __future__ import division
import time
from collections import defaultdict

from thrift.transport import TSocket
from thrift.protocol import TBinaryProtocol
from thrift.transport import TTransport
from .hbase import Hbase
import pymongo

SUBSTITUTION_TOKEN = 'SUB'
SKIP_START = '<SKIP:'

class NgramService(object):
    h_rate = None
    h_start = None
    h_client = None

    @staticmethod
    def _is_subst(ngram):
        return SUBSTITUTION_TOKEN in set(ngram)

    @classmethod
    def configure(cls, substitutions, mongo_host=('localhost', '27017'), hbase_host=('localhost', '9090')):
        cls.substitutions = sorted(substitutions)

        cls.m_client = pymongo.MongoClient(host=mongo_host[0], port=int(mongo_host[1]))
        cls.m_1grams = cls.m_client['1grams']['default']
        cls.m_ngrams = cls.m_client['ngrams']['default']

        # HBASE
        cls.h_transport = TTransport.TBufferedTransport(TSocket.TSocket(*hbase_host))
        protocol = TBinaryProtocol.TBinaryProtocolAccelerated(cls.h_transport)
        cls.h_client = Hbase.Client(protocol)
        cls.h_transport.open()
        cls.h_rate = 0
        cls.h_start = time.time()

        cls.substitution_counts = dict([(subst, cls.get_freq(subst)[subst]) for subst in cls.substitutions])

    @classmethod
    def hbase_count(cls, table, ngram):
        """
        :rtype: int
        """
        res = cls.hbase_raw(table, ngram, "ngram:value")
        if res is None:
            res = 0
        else:
            res = long(res)
        return res

    @classmethod
    def hbase_raw(cls, table, ngram, column):
        from . import DEBUG
        cls.h_rate += 1
        time_diff = time.time() - cls.h_start
        if time_diff > 30 and DEBUG:
            print "HBase req rate:", cls.h_rate/time_diff, "r/s"
            cls.h_start = time.time()
            cls.h_rate = 0
        try:
            res = cls.h_client.get(table, ngram.encode('utf-8'), column, None)
            return res[0].value
        except (ValueError, IndexError):
            return None

    @staticmethod
    def _tuple(ngram):
        """
        :type ngram: list
        :returns: tuple with replacement if necessary
        """
        return tuple([x for x in ngram if not x.startswith(SKIP_START)])

    @classmethod
    def get_freq(cls, ngram):
        """Get ngram frequency from Google Ngram corpus"""
        split_ngram = ngram.split()
        split_len = len(split_ngram)
        if NgramService._is_subst(split_ngram):
            sub_index = split_ngram.index(SUBSTITUTION_TOKEN)
            if split_len == 1:
                return cls.substitution_counts
            if 1 < split_len < 5:
                res = cls.m_ngrams.find_one({'ngram': ngram})
            else:
                raise Exception('%d-grams are not supported yet' % split_len)
            try:
                counts = res['count']
            except:
                counts = defaultdict(lambda: 0)
            res = {}
            for subst in cls.substitutions:
                cur_ngram = split_ngram[:]
                cur_ngram[sub_index] = subst
                res[cls._tuple(cur_ngram)] = long(counts.get(subst, 0))
        else:
            if split_len == 1:
                try:
                    count = cls.m_1grams.find_one({'ngram': ngram})['count']
                except:
                    count = 0
                # Stupid NLTK needs a word when n==1, and tuple if n>1
                res = {ngram: count}
            elif 2 <= split_len <= 3:
                count = cls.hbase_count('ngrams', ngram)
                res = {cls._tuple(split_ngram): count}
            else:
                raise Exception('%d-grams are not supported' % split_len)
        return res
<commit_msg>mongo and hbase configs are now none by default<commit_after># coding=utf-8
from __future__ import division
import time
from collections import defaultdict

from thrift.transport import TSocket
from thrift.protocol import TBinaryProtocol
from thrift.transport import TTransport
from .hbase import Hbase
import pymongo

SUBSTITUTION_TOKEN = 'SUB'
SKIP_START = '<SKIP:'

class NgramService(object):
    h_rate = None
    h_start = None
    h_client = None

    @staticmethod
    def _is_subst(ngram):
        return SUBSTITUTION_TOKEN in set(ngram)

    @classmethod
    def configure(cls, substitutions, mongo_host=None, hbase_host=None):
        cls.substitutions = sorted(substitutions)

        if mongo_host:
            cls.m_client = pymongo.MongoClient(host=mongo_host[0], port=int(mongo_host[1]))
            cls.m_1grams = cls.m_client['1grams']['default']
            cls.m_ngrams = cls.m_client['ngrams']['default']

        if hbase_host:
            # HBASE
            cls.h_transport = TTransport.TBufferedTransport(TSocket.TSocket(*hbase_host))
            protocol = TBinaryProtocol.TBinaryProtocolAccelerated(cls.h_transport)
            cls.h_client = Hbase.Client(protocol)
            cls.h_transport.open()
            cls.h_rate = 0
            cls.h_start = time.time()

        cls.substitution_counts = dict([(subst, cls.get_freq(subst)[subst]) for subst in cls.substitutions])

    @classmethod
    def hbase_count(cls, table, ngram):
        """
        :rtype: int
        """
        res = cls.hbase_raw(table, ngram, "ngram:value")
        if res is None:
            res = 0
        else:
            res = long(res)
        return res

    @classmethod
    def hbase_raw(cls, table, ngram, column):
        from . import DEBUG
        cls.h_rate += 1
        time_diff = time.time() - cls.h_start
        if time_diff > 30 and DEBUG:
            print "HBase req rate:", cls.h_rate/time_diff, "r/s"
            cls.h_start = time.time()
            cls.h_rate = 0
        try:
            res = cls.h_client.get(table, ngram.encode('utf-8'), column, None)
            return res[0].value
        except (ValueError, IndexError):
            return None

    @staticmethod
    def _tuple(ngram):
        """
        :type ngram: list
        :returns: tuple with replacement if necessary
        """
        return tuple([x for x in ngram if not x.startswith(SKIP_START)])

    @classmethod
    def get_freq(cls, ngram):
        """Get ngram frequency from Google Ngram corpus"""
        split_ngram = ngram.split()
        split_len = len(split_ngram)
        if NgramService._is_subst(split_ngram):
            sub_index = split_ngram.index(SUBSTITUTION_TOKEN)
            if split_len == 1:
                return cls.substitution_counts
            if 1 < split_len < 5:
                res = cls.m_ngrams.find_one({'ngram': ngram})
            else:
                raise Exception('%d-grams are not supported yet' % split_len)
            try:
                counts = res['count']
            except:
                counts = defaultdict(lambda: 0)
            res = {}
            for subst in cls.substitutions:
                cur_ngram = split_ngram[:]
                cur_ngram[sub_index] = subst
                res[cls._tuple(cur_ngram)] = long(counts.get(subst, 0))
        else:
            if split_len == 1:
                try:
                    count = cls.m_1grams.find_one({'ngram': ngram})['count']
                except:
                    count = 0
                # Stupid NLTK needs a word when n==1, and tuple if n>1
                res = {ngram: count}
            elif 2 <= split_len <= 3:
                count = cls.hbase_count('ngrams', ngram)
                res = {cls._tuple(split_ngram): count}
            else:
                raise Exception('%d-grams are not supported' % split_len)
        return res
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import subprocess
import itertools
import shutil
import sys

try:
    import blessings
    term = blessings.Terminal()
except ImportError:
    class FakeTerminal(object):
        def __getattr__(self, a):
            return ''
    term = FakeTerminal()

def usage():
    print >>sys.stderr, "Usage: " + sys.argv[0] + " <build A> <build B>"
    print >>sys.stderr, "Available builds are:",
    print >>sys.stderr, ", ".join(name for name, _ in AVAILABLE_BUILDS.items())
    sys.exit(1)

def parse_time(s):
    sdelim = "Median time: "
    edelim = " us"
    assert s.startswith(sdelim)
    assert s.endswith(edelim)
    return int(s[len(sdelim):-len(edelim)])

def build(builds):
    binaries = []
    with open(os.devnull, "w") as devnull:
        redirects = {"stdout": devnull, "stderr": devnull}
        subprocess.check_call(["make", "clean"], **redirects)

        for name, cmd in builds:
            subprocess.check_call(cmd.split(" "), **redirects)
            binaries.append("lyra2-" + name)
            shutil.move("lyra2", binaries[-1])
    return binaries

AVAILABLE_BUILDS = {
    "lyra2-clang": "make CC=clang",
    "lyra2-gcc": "make CC=gcc",
    "ref-clang": "make bench-ref CC=clang",
    "ref-gcc": "make bench-ref CC=gcc"
}

if len(sys.argv) != 3:
    usage()

nA, nB = sys.argv[1:]
bA, bB = map(AVAILABLE_BUILDS.get, sys.argv[1:])
if None in (bA, bB):
    print >>sys.stderr, "Invalid build '%s'!" % (nA if bB is not None else nB)
    usage()

bins = build([(nA, bA), (nB, bB)])
a_output = subprocess.check_output(["./" + bins[0]])[:-1] # remove trailing \n
b_output = subprocess.check_output(["./" + bins[1]])[:-1]
a_output_lines = a_output.split("\n")
b_output_lines = b_output.split("\n")
assert len(a_output_lines) == len(b_output_lines)

os.remove(bins[0])
os.remove(bins[1])

print "Done running builds, speedup will be relative to '%s'" % nB

it = itertools.izip(a_output_lines, b_output_lines)
for al, bl in it:
    assert al == bl, "different inputs"
    if not al:
        break

for al, bl in it:
    assert al == bl, "different paramenters"
    _, parameters = al.split(' ', 1)
    print parameters + ": "

    al, bl = next(it)
    assert al == bl, "different outputs"

    at, bt = map(parse_time, next(it))
    print "    %s: %d us" % (nA, at),
    speedup = 100 * (bt - at) / float(bt)
    if speedup > 0:
        print "{term.green}{:.2f}% faster{term.normal}".format(speedup, term = term)
    else:
        print "{term.red}{:.2f}% slower{term.normal}".format(-speedup, term = term)
    print "    %s: %d us" % (nB, bt)

    al, bl = next(it)
    assert not al and not bl

<commit_msg>Simplify usage() in benchmark.py.<commit_after>#!/usr/bin/env python

import os
import subprocess
import itertools
import shutil
import sys

try:
    import blessings
    term = blessings.Terminal()
except ImportError:
    class FakeTerminal(object):
        def __getattr__(self, a):
            return ''
    term = FakeTerminal()

def usage():
    print >>sys.stderr, "Usage: " + sys.argv[0] + " <build A> <build B>"
    print >>sys.stderr, "Available builds are:",
    print >>sys.stderr, ", ".join(AVAILABLE_BUILDS.keys())
    sys.exit(1)

def parse_time(s):
    sdelim = "Median time: "
    edelim = " us"
    assert s.startswith(sdelim)
    assert s.endswith(edelim)
    return int(s[len(sdelim):-len(edelim)])

def build(builds):
    binaries = []
    with open(os.devnull, "w") as devnull:
        redirects = {"stdout": devnull, "stderr": devnull}
        subprocess.check_call(["make", "clean"], **redirects)

        for name, cmd in builds:
            subprocess.check_call(cmd.split(" "), **redirects)
            binaries.append("lyra2-" + name)
            shutil.move("lyra2", binaries[-1])
    return binaries

AVAILABLE_BUILDS = {
    "lyra2-clang": "make CC=clang",
    "lyra2-gcc": "make CC=gcc",
    "ref-clang": "make bench-ref CC=clang",
    "ref-gcc": "make bench-ref CC=gcc"
}

if len(sys.argv) != 3:
    usage()

nA, nB = sys.argv[1:]
bA, bB = map(AVAILABLE_BUILDS.get, sys.argv[1:])
if None in (bA, bB):
    print >>sys.stderr, "Invalid build '%s'!" % (nA if bB is not None else nB)
    usage()

bins = build([(nA, bA), (nB, bB)])
a_output = subprocess.check_output(["./" + bins[0]])[:-1] # remove trailing \n
b_output = subprocess.check_output(["./" + bins[1]])[:-1]
a_output_lines = a_output.split("\n")
b_output_lines = b_output.split("\n")
assert len(a_output_lines) == len(b_output_lines)

os.remove(bins[0])
os.remove(bins[1])

print "Done running builds, speedup will be relative to '%s'" % nB

it = itertools.izip(a_output_lines, b_output_lines)
for al, bl in it:
    assert al == bl, "different inputs"
    if not al:
        break

for al, bl in it:
    assert al == bl, "different paramenters"
    _, parameters = al.split(' ', 1)
    print parameters + ": "

    al, bl = next(it)
    assert al == bl, "different outputs"

    at, bt = map(parse_time, next(it))
    print "    %s: %d us" % (nA, at),
    speedup = 100 * (bt - at) / float(bt)
    if speedup > 0:
        print "{term.green}{:.2f}% faster{term.normal}".format(speedup, term = term)
    else:
        print "{term.red}{:.2f}% slower{term.normal}".format(-speedup, term = term)
    print "    %s: %d us" % (nB, bt)

    al, bl = next(it)
    assert not al and not bl

<|endoftext|>