<commit_before>from __future__ import unicode_literals
# Ensure 'assert_raises' context manager support for Python 2.6
import tests.backport_assert_raises
from nose.tools import assert_raises

import boto
from boto.exception import EC2ResponseError
import sure  # noqa

from moto import mock_ec2
from tests.helpers import requires_boto_gte

@mock_ec2
def test_route_tables_defaults():
    conn = boto.connect_vpc('the_key', 'the_secret')
    vpc = conn.create_vpc("10.0.0.0/16")

    all_route_tables = conn.get_all_route_tables()
    all_route_tables.should.have.length_of(1)

    main_route_table = all_route_tables[0]
    main_route_table.vpc_id.should.equal(vpc.id)

    routes = main_route_table.routes
    routes.should.have.length_of(1)

    local_route = routes[0]
    local_route.gateway_id.should.equal('local')
    local_route.state.should.equal('active')
    local_route.destination_cidr_block.should.equal(vpc.cidr_block)

    vpc.delete()

    all_route_tables = conn.get_all_route_tables()
    all_route_tables.should.have.length_of(0)

@mock_ec2
def test_route_tables_additional():
    conn = boto.connect_vpc('the_key', 'the_secret')
    vpc = conn.create_vpc("10.0.0.0/16")
    route_table = conn.create_route_table(vpc.id)

    all_route_tables = conn.get_all_route_tables()
    all_route_tables.should.have.length_of(2)
    all_route_tables[0].vpc_id.should.equal(vpc.id)
    all_route_tables[1].vpc_id.should.equal(vpc.id)

    all_route_table_ids = [route_table.id for route_table in all_route_tables]
    all_route_table_ids.should.contain(route_table.id)

    routes = route_table.routes
    routes.should.have.length_of(1)

    local_route = routes[0]
    local_route.gateway_id.should.equal('local')
    local_route.state.should.equal('active')
    local_route.destination_cidr_block.should.equal(vpc.cidr_block)

    conn.delete_route_table(route_table.id)

    all_route_tables = conn.get_all_route_tables()
    all_route_tables.should.have.length_of(1)

    with assert_raises(EC2ResponseError) as cm:
        conn.delete_route_table("rtb-1234abcd")
    cm.exception.code.should.equal('InvalidRouteTableID.NotFound')
    cm.exception.status.should.equal(400)
    cm.exception.request_id.should_not.be.none

@mock_ec2
def test_route_tables_filters():
    conn = boto.connect_vpc('the_key', 'the_secret')

    vpc1 = conn.create_vpc("10.0.0.0/16")
    route_table1 = conn.create_route_table(vpc1.id)

    vpc2 = conn.create_vpc("10.0.0.0/16")
    route_table2 = conn.create_route_table(vpc2.id)

    all_route_tables = conn.get_all_route_tables()
    all_route_tables.should.have.length_of(4)

    # Filter by main route table
    main_route_tables = conn.get_all_route_tables(filters={'association.main':'true'})
    main_route_tables.should.have.length_of(2)
    main_route_table_ids = [route_table.id for route_table in main_route_tables]
    main_route_table_ids.should_not.contain(route_table1.id)
    main_route_table_ids.should_not.contain(route_table2.id)

    # Filter by VPC
    vpc1_route_tables = conn.get_all_route_tables(filters={'vpc-id':vpc1.id})
    vpc1_route_tables.should.have.length_of(2)
    vpc1_route_table_ids = [route_table.id for route_table in vpc1_route_tables]
    vpc1_route_table_ids.should.contain(route_table1.id)
    vpc1_route_table_ids.should_not.contain(route_table2.id)

    # Filter by VPC and main route table
    vpc2_main_route_tables = conn.get_all_route_tables(filters={'association.main':'true', 'vpc-id':vpc2.id})
    vpc2_main_route_tables.should.have.length_of(1)
    vpc2_main_route_table_ids = [route_table.id for route_table in vpc2_main_route_tables]
    vpc2_main_route_table_ids.should_not.contain(route_table1.id)
    vpc2_main_route_table_ids.should_not.contain(route_table2.id)

    # Unsupported filter
    conn.get_all_route_tables.when.called_with(filters={'not-implemented-filter': 'foobar'}).should.throw(NotImplementedError)

@mock_ec2
def test_routes_additional():
    conn = boto.connect_vpc('the_key', 'the_secret')
    vpc = conn.create_vpc("10.0.0.0/16")
    main_route_table = conn.get_all_route_tables()[0]
    local_route = main_route_table.routes[0]
    igw = conn.create_internet_gateway()
    ROUTE_CIDR = "10.0.0.4/24"

    conn.create_route(main_route_table.id, ROUTE_CIDR, gateway_id=igw.id)

    main_route_table = conn.get_all_route_tables()[0] # Refresh route table

    main_route_table.routes.should.have.length_of(2)
    new_routes = [route for route in main_route_table.routes if route.destination_cidr_block != vpc.cidr_block]
    new_routes.should.have.length_of(1)

    new_route = new_routes[0]
    new_route.gateway_id.should.equal(igw.id)
    new_route.instance_id.should.be.none
    new_route.state.should.equal('active')
    new_route.destination_cidr_block.should.equal(ROUTE_CIDR)

    conn.delete_route(main_route_table.id, ROUTE_CIDR)

    main_route_table = conn.get_all_route_tables()[0] # Refresh route table

    main_route_table.routes.should.have.length_of(1)
    new_routes = [route for route in main_route_table.routes if route.destination_cidr_block != vpc.cidr_block]
    new_routes.should.have.length_of(0)

    with assert_raises(EC2ResponseError) as cm:
        conn.delete_route(main_route_table.id, ROUTE_CIDR)
    cm.exception.code.should.equal('InvalidRoute.NotFound')
    cm.exception.status.should.equal(400)
    cm.exception.request_id.should_not.be.none

    conn.create_route.when.called_with(main_route_table.id, ROUTE_CIDR, interface_id='eni-1234abcd').should.throw(NotImplementedError)

@mock_ec2
def test_routes_replace():
    conn = boto.connect_vpc('the_key', 'the_secret')
    vpc = conn.create_vpc("10.0.0.0/16")
    main_route_table = conn.get_all_route_tables(filters={'association.main':'true','vpc-id':vpc.id})[0]
    local_route = main_route_table.routes[0]
    ROUTE_CIDR = "10.0.0.4/24"

    # Various route targets
    igw = conn.create_internet_gateway()

    reservation = conn.run_instances('ami-1234abcd')
    instance = reservation.instances[0]

    # Create initial route
    conn.create_route(main_route_table.id, ROUTE_CIDR, gateway_id=igw.id)

    # Replace...
    def get_target_route():
        route_table = conn.get_all_route_tables(main_route_table.id)[0]
        routes = [route for route in route_table.routes if route.destination_cidr_block != vpc.cidr_block]
        routes.should.have.length_of(1)
        return routes[0]

    conn.replace_route(main_route_table.id, ROUTE_CIDR, instance_id=instance.id)

    target_route = get_target_route()
    target_route.gateway_id.should.be.none
    target_route.instance_id.should.equal(instance.id)
    target_route.state.should.equal('active')
    target_route.destination_cidr_block.should.equal(ROUTE_CIDR)

    conn.replace_route(main_route_table.id, ROUTE_CIDR, gateway_id=igw.id)

    target_route = get_target_route()
    target_route.gateway_id.should.equal(igw.id)
    target_route.instance_id.should.be.none
    target_route.state.should.equal('active')
    target_route.destination_cidr_block.should.equal(ROUTE_CIDR)

    with assert_raises(EC2ResponseError) as cm:
        conn.replace_route('rtb-1234abcd', ROUTE_CIDR, gateway_id=igw.id)
    cm.exception.code.should.equal('InvalidRouteTableID.NotFound')
    cm.exception.status.should.equal(400)
    cm.exception.request_id.should_not.be.none

    conn.replace_route.when.called_with(main_route_table.id, ROUTE_CIDR, interface_id='eni-1234abcd').should.throw(NotImplementedError)

@requires_boto_gte("2.32.0")
@mock_ec2
def test_routes_vpc_peering_connection():
    conn = boto.connect_vpc('the_key', 'the_secret')
    vpc = conn.create_vpc("10.0.0.0/16")
    main_route_table = conn.get_all_route_tables(filters={'association.main':'true','vpc-id':vpc.id})[0]
    local_route = main_route_table.routes[0]
    ROUTE_CIDR = "10.0.0.4/24"

    peer_vpc = conn.create_vpc("11.0.0.0/16")
    vpc_pcx = conn.create_vpc_peering_connection(vpc.id, peer_vpc.id)

    conn.create_route(main_route_table.id, ROUTE_CIDR, vpc_peering_connection_id=vpc_pcx.id)

    # Refresh route table
    main_route_table = conn.get_all_route_tables(main_route_table.id)[0]
    new_routes = [route for route in main_route_table.routes if route.destination_cidr_block != vpc.cidr_block]
    new_routes.should.have.length_of(1)

    new_route = new_routes[0]
    new_route.gateway_id.should.be.none
    new_route.instance_id.should.be.none
    new_route.state.should.equal('blackhole')
    new_route.destination_cidr_block.should.equal(ROUTE_CIDR)

<commit_msg>Route Tables / Routes: Initial implementation. (Commented out test which requires not-yet-released Boto 2.32.2.)<commit_after>from __future__ import unicode_literals
# Ensure 'assert_raises' context manager support for Python 2.6
import tests.backport_assert_raises
from nose.tools import assert_raises

import boto
from boto.exception import EC2ResponseError
import sure  # noqa

from moto import mock_ec2
from tests.helpers import requires_boto_gte

@mock_ec2
def test_route_tables_defaults():
    conn = boto.connect_vpc('the_key', 'the_secret')
    vpc = conn.create_vpc("10.0.0.0/16")

    all_route_tables = conn.get_all_route_tables()
    all_route_tables.should.have.length_of(1)

    main_route_table = all_route_tables[0]
    main_route_table.vpc_id.should.equal(vpc.id)

    routes = main_route_table.routes
    routes.should.have.length_of(1)

    local_route = routes[0]
    local_route.gateway_id.should.equal('local')
    local_route.state.should.equal('active')
    local_route.destination_cidr_block.should.equal(vpc.cidr_block)

    vpc.delete()

    all_route_tables = conn.get_all_route_tables()
    all_route_tables.should.have.length_of(0)

@mock_ec2
def test_route_tables_additional():
    conn = boto.connect_vpc('the_key', 'the_secret')
    vpc = conn.create_vpc("10.0.0.0/16")
    route_table = conn.create_route_table(vpc.id)

    all_route_tables = conn.get_all_route_tables()
    all_route_tables.should.have.length_of(2)
    all_route_tables[0].vpc_id.should.equal(vpc.id)
    all_route_tables[1].vpc_id.should.equal(vpc.id)

    all_route_table_ids = [route_table.id for route_table in all_route_tables]
    all_route_table_ids.should.contain(route_table.id)

    routes = route_table.routes
    routes.should.have.length_of(1)

    local_route = routes[0]
    local_route.gateway_id.should.equal('local')
    local_route.state.should.equal('active')
    local_route.destination_cidr_block.should.equal(vpc.cidr_block)

    conn.delete_route_table(route_table.id)

    all_route_tables = conn.get_all_route_tables()
    all_route_tables.should.have.length_of(1)

    with assert_raises(EC2ResponseError) as cm:
        conn.delete_route_table("rtb-1234abcd")
    cm.exception.code.should.equal('InvalidRouteTableID.NotFound')
    cm.exception.status.should.equal(400)
    cm.exception.request_id.should_not.be.none

@mock_ec2
def test_route_tables_filters():
    conn = boto.connect_vpc('the_key', 'the_secret')

    vpc1 = conn.create_vpc("10.0.0.0/16")
    route_table1 = conn.create_route_table(vpc1.id)

    vpc2 = conn.create_vpc("10.0.0.0/16")
    route_table2 = conn.create_route_table(vpc2.id)

    all_route_tables = conn.get_all_route_tables()
    all_route_tables.should.have.length_of(4)

    # Filter by main route table
    main_route_tables = conn.get_all_route_tables(filters={'association.main':'true'})
    main_route_tables.should.have.length_of(2)
    main_route_table_ids = [route_table.id for route_table in main_route_tables]
    main_route_table_ids.should_not.contain(route_table1.id)
    main_route_table_ids.should_not.contain(route_table2.id)

    # Filter by VPC
    vpc1_route_tables = conn.get_all_route_tables(filters={'vpc-id':vpc1.id})
    vpc1_route_tables.should.have.length_of(2)
    vpc1_route_table_ids = [route_table.id for route_table in vpc1_route_tables]
    vpc1_route_table_ids.should.contain(route_table1.id)
    vpc1_route_table_ids.should_not.contain(route_table2.id)

    # Filter by VPC and main route table
    vpc2_main_route_tables = conn.get_all_route_tables(filters={'association.main':'true', 'vpc-id':vpc2.id})
    vpc2_main_route_tables.should.have.length_of(1)
    vpc2_main_route_table_ids = [route_table.id for route_table in vpc2_main_route_tables]
    vpc2_main_route_table_ids.should_not.contain(route_table1.id)
    vpc2_main_route_table_ids.should_not.contain(route_table2.id)

    # Unsupported filter
    conn.get_all_route_tables.when.called_with(filters={'not-implemented-filter': 'foobar'}).should.throw(NotImplementedError)

@mock_ec2
def test_routes_additional():
    conn = boto.connect_vpc('the_key', 'the_secret')
    vpc = conn.create_vpc("10.0.0.0/16")
    main_route_table = conn.get_all_route_tables()[0]
    local_route = main_route_table.routes[0]
    igw = conn.create_internet_gateway()
    ROUTE_CIDR = "10.0.0.4/24"

    conn.create_route(main_route_table.id, ROUTE_CIDR, gateway_id=igw.id)

    main_route_table = conn.get_all_route_tables()[0] # Refresh route table

    main_route_table.routes.should.have.length_of(2)
    new_routes = [route for route in main_route_table.routes if route.destination_cidr_block != vpc.cidr_block]
    new_routes.should.have.length_of(1)

    new_route = new_routes[0]
    new_route.gateway_id.should.equal(igw.id)
    new_route.instance_id.should.be.none
    new_route.state.should.equal('active')
    new_route.destination_cidr_block.should.equal(ROUTE_CIDR)

    conn.delete_route(main_route_table.id, ROUTE_CIDR)

    main_route_table = conn.get_all_route_tables()[0] # Refresh route table

    main_route_table.routes.should.have.length_of(1)
    new_routes = [route for route in main_route_table.routes if route.destination_cidr_block != vpc.cidr_block]
    new_routes.should.have.length_of(0)

    with assert_raises(EC2ResponseError) as cm:
        conn.delete_route(main_route_table.id, ROUTE_CIDR)
    cm.exception.code.should.equal('InvalidRoute.NotFound')
    cm.exception.status.should.equal(400)
    cm.exception.request_id.should_not.be.none

    conn.create_route.when.called_with(main_route_table.id, ROUTE_CIDR, interface_id='eni-1234abcd').should.throw(NotImplementedError)

@mock_ec2
def test_routes_replace():
    conn = boto.connect_vpc('the_key', 'the_secret')
    vpc = conn.create_vpc("10.0.0.0/16")
    main_route_table = conn.get_all_route_tables(filters={'association.main':'true','vpc-id':vpc.id})[0]
    local_route = main_route_table.routes[0]
    ROUTE_CIDR = "10.0.0.4/24"

    # Various route targets
    igw = conn.create_internet_gateway()

    reservation = conn.run_instances('ami-1234abcd')
    instance = reservation.instances[0]

    # Create initial route
    conn.create_route(main_route_table.id, ROUTE_CIDR, gateway_id=igw.id)

    # Replace...
    def get_target_route():
        route_table = conn.get_all_route_tables(main_route_table.id)[0]
        routes = [route for route in route_table.routes if route.destination_cidr_block != vpc.cidr_block]
        routes.should.have.length_of(1)
        return routes[0]

    conn.replace_route(main_route_table.id, ROUTE_CIDR, instance_id=instance.id)

    target_route = get_target_route()
    target_route.gateway_id.should.be.none
    target_route.instance_id.should.equal(instance.id)
    target_route.state.should.equal('active')
    target_route.destination_cidr_block.should.equal(ROUTE_CIDR)

    conn.replace_route(main_route_table.id, ROUTE_CIDR, gateway_id=igw.id)

    target_route = get_target_route()
    target_route.gateway_id.should.equal(igw.id)
    target_route.instance_id.should.be.none
    target_route.state.should.equal('active')
    target_route.destination_cidr_block.should.equal(ROUTE_CIDR)

    with assert_raises(EC2ResponseError) as cm:
        conn.replace_route('rtb-1234abcd', ROUTE_CIDR, gateway_id=igw.id)
    cm.exception.code.should.equal('InvalidRouteTableID.NotFound')
    cm.exception.status.should.equal(400)
    cm.exception.request_id.should_not.be.none

    conn.replace_route.when.called_with(main_route_table.id, ROUTE_CIDR, interface_id='eni-1234abcd').should.throw(NotImplementedError)

#@requires_boto_gte("2.32.2")
#@mock_ec2
#def test_routes_vpc_peering_connection():
#    conn = boto.connect_vpc('the_key', 'the_secret')
#    vpc = conn.create_vpc("10.0.0.0/16")
#    main_route_table = conn.get_all_route_tables(filters={'association.main':'true','vpc-id':vpc.id})[0]
#    local_route = main_route_table.routes[0]
#    ROUTE_CIDR = "10.0.0.4/24"
#
#    peer_vpc = conn.create_vpc("11.0.0.0/16")
#    vpc_pcx = conn.create_vpc_peering_connection(vpc.id, peer_vpc.id)
#
#    conn.create_route(main_route_table.id, ROUTE_CIDR, vpc_peering_connection_id=vpc_pcx.id)
#
#    # Refresh route table
#    main_route_table = conn.get_all_route_tables(main_route_table.id)[0]
#    new_routes = [route for route in main_route_table.routes if route.destination_cidr_block != vpc.cidr_block]
#    new_routes.should.have.length_of(1)
#
#    new_route = new_routes[0]
#    new_route.gateway_id.should.be.none
#    new_route.instance_id.should.be.none
#    new_route.state.should.equal('blackhole')
#    new_route.destination_cidr_block.should.equal(ROUTE_CIDR)

<|endoftext|>


<commit_before>#!/usr/bin/python

"""This is the master parsing file. It will take a folder of raw texts, metadata, and marcXML
	and construct the necessary input.txt and jsoncatalog.txt for Presidio to compile."""

#build core files for Presidio database

import re
import string
import os, sys
import subprocess
import xml.etree.ElementTree as ET
#may not need some of these, preserve memory
sys.setrecursionlimit(60000)

#class Countdown(object):
#    def __iter__(self):
#        for n in [5, 4, 3, 2, 1, 'launch']:
#            yield n

def parseFolder(folder, extension):
    files = []
    for item in os.listdir(folder):
        if (os.path.isfile(os.path.join(folder, item))):
            files.append(item)
    return files

#open the appropriate file, pull its contents
def readFolder(count, folder, files):
	file = open((os.path.join(folder, files[count])), 'r')
	data = file.read()
	file.close()
	return data

#build jsoncatalog.txt
def makeMeta(count):
	print "building jsoncatalog.txt..."
	xfiles = parseFolder("/data/MHL/MHL_download/mhl_meta_xml_files", '_meta.xml')
	subprocess.call(['touch', 'jsoncatalog.txt'])
	meta = open('jsoncatalog.txt', 'a')

	#extract XML, MARC tags, place in jsoncatalog.txt
	def buildMeta(count):
		root = ET.fromstring(readFolder(count, "/data/MHL/MHL_download/mhl_meta_xml_files", xfiles))

		#filename
		for identifier in root.findall('identifier'):
			filename = identifier.text
			#print filename #debugging

		#optional tags to extract
			"""note: I have hacked in some robust error handling, there were scope issues with
			   more elegant solutions, and it would require dynamically named variables,
			   so for now it is reliable brute force. Apologies for the mess,
			   I am aware that this is bad coding, and hope to clean it up later"""

		#date: handles inconsistent formatting
		for date in root.findall('date'):

			try:
				#if (i.e. 1822-1946), take last four digits
				if date.text[-4:].isdigit():
					year = date.text[-4:]

				#i.e. 1922-26 -> 1926
				elif date.text[-2:].isdigit() and not(date.text[-3:].isdigit()) and len(date.text) < 8:
					year = date.text[:2] + date.text[-2:]

				else:
					#take first four characters for any other inconsistency
					year = date.text[:4]
			except TypeError:
				year = ""
		#hacked error handling
		if 'year' in locals():
			pass
		else:
			year = ""

		#contributing library
		try:
			for contributor in root.findall('contributor'):
				library = contributor.text

		except TypeError:
			library = ""
		#hacked error handling
		if 'library' in locals():
			pass
		else:
			library = "Other"

		#language
		for language in root.findall('language'):
			language = language.text

			if language=="English": #MHL database inconsistency
				language = "eng"
		#hacked error handling
		if 'language' in locals():
			pass
		else:
			language = ""

		#title
		for title in root.findall('title'):
			title = title.text
		#sets title to filename if it does not exist
		if 'title' in locals():
			title = filter(lambda x: x in string.printable, title)
			title = title.replace('"', '')
			title = title.replace("'", " ")
		else:
			title = filename

		#author
		for creator in root.findall('creator'):
			author = creator.text

		if 'author' in locals():
			author= author.replace('"', '')
			author= author.replace("'", "")
		else:
			author = ""
		#the above XML elements could be potentially abstracted into a single function

		#searchstring
		for arlink in root.findall('identifier-access'):
			searchstring = arlink.text
			searchstring = "[" + author + "], " + title + \
				 			" (" + year + ") read"
			searchstring = searchstring.encode('ascii','ignore')
			searchstring = searchstring.replace('"', '')
			searchstring = searchstring.replace("'", " ")

		#hacked error handling
		if 'searchstring' in locals():
			pass
		else:
			searchstring = ""

		#subjects
		subjectArray = []
		for subject in root.iter('subject'):
			subject = subject.text
			if subject is not None:
				subject = subject.replace('"', '')
				stext = subject.replace(',', '-')
				stext = stext.replace('"', '')
				stext = filter(lambda x: x in string.ascii_letters, stext)
				subArray = subject.split()
				subjectArray.append(stext)

				#cleans up the words i.e. semicolons will not appear as subjects
				for word in subArray:
					rx = re.compile('\W+')
					cleanWord = rx.sub(' ', word).strip()
					cleanWord = cleanWord.replace('"', '')
					if cleanWord[:4].isdigit():
						pass
					elif len(cleanWord) < 3:
						pass
					#should probably make a dictionary
					elif cleanWord=="and":
						pass
					elif cleanWord=="for":
						pass
					elif cleanWord=="drug" or "Drug":
						pass
					elif cleanWord=="industry" or "Industry":
						pass
					elif cleanWord=="system" or "System":
						pass
					else:
						cleanWord = cleanWord.lower().capitalize()
						subjectArray.append(cleanWord)
			else:
				subjectArray = "Miscellaneous"

		#write json object to file
		jdict = {"library" : library,
				 "searchstring" : searchstring,
				 "filename" : filename,
				 "language" : language,
				 "year" : year,
				 "subject" : subjectArray,
				}

		json = str(jdict)
		json = json.replace("'", '"')
		#raw string is printing double backslashes - quick fix
		json = json.replace('\\"', '\"')

		print "writing %s metadata to jsoncatalog.txt..." % filename
		meta.write(json + '\n')

		#recursion
		if count < ((len(xfiles)) -1):
			count = count + 1
			buildMeta(count)
		else:
			print "\n jsoncatalog.txt done building! \n"

	buildMeta(count)

makeMeta(0)

<commit_msg>added marc Parsing to old JSON parser<commit_after>#!/usr/bin/python

"""This is the master parsing file. It will take a folder of raw texts, metadata, and marcXML
    and construct the necessary input.txt and jsoncatalog.txt for Presidio to compile."""

#build core files for Presidio database

import re
import string
import os, sys
import subprocess
import xml.etree.ElementTree as ET
from xml.dom.minidom import parseString
import inspect
from pymarc import MARCReader
#may not need some of these, preserve memory
sys.setrecursionlimit(60000)

def parseFolder(folder, extension):
    files = []
    for item in os.listdir(folder):
        if (os.path.isfile(os.path.join(folder, item))):
            files.append(item)
    return files

#open the appropriate file, pull its contents
def readMarcFolder(count, folder, files):
    xmlItem = files[count]
    marcItem = xmlItem[:9] + "_marc.xml"
    file = open((os.path.join('/data/MARC', marcItem)), 'r')
    data = file.read()
    file.close()
    return data

def readFolder(count, folder, files):
    file = open((os.path.join(folder, files[count])), 'r')
    data = file.read()
    file.close()
    return data

#build jsoncatalog.txt
def makeMeta(count):
    print "building jsoncatalog.txt..."
    xfiles = parseFolder("/home/will/MHL_download/mhl_meta_xml_files", '_meta.xml')
    subprocess.call(['touch', 'jsoncatalog.txt'])
    meta = open('jsoncatalog.txt', 'a')

    #extract XML, MARC tags, place in jsoncatalog.txt
    def buildMeta(count):
        root = ET.fromstring(readFolder(count, "/home/will/MHL_download/mhl_meta_xml_files", xfiles))

        #filename
        for identifier in root.findall('identifier'):
            filename = identifier.text
            #print filename #debugging

        #optional tags to extract
            """note: I have hacked in some robust error handling, there were scope issues with
               more elegant solutions, and it would require dynamically named variables,
               so for now it is reliable brute force. Apologies for the mess,
               I am aware that this is bad coding, and hope to clean it up later"""

        #date: handles inconsistent formatting
        for date in root.findall('date'):

            try:
                #if (i.e. 1822-1946), take last four digits
                if date.text[-4:].isdigit():
                    year = date.text[-4:]

                #i.e. 1922-26 -> 1926
                elif date.text[-2:].isdigit() and not(date.text[-3:].isdigit()) and len(date.text) < 8:
                    year = date.text[:2] + date.text[-2:]

                else:
                    #take first four characters for any other inconsistency
                    year = date.text[:4]
            except TypeError:
                year = ""
        #hacked error handling
        if 'year' in locals():
            pass
        else:
            year = ""

        #contributing library
        try:
            for contributor in root.findall('contributor'):
                library = contributor.text

        except TypeError:
            library = ""
        #hacked error handling
        if 'library' in locals():
            pass
        else:
            library = "Other"

        #language
        for language in root.findall('language'):
            language = language.text

            if language=="English": #MHL database inconsistency
                language = "eng"
        #hacked error handling
        if 'language' in locals():
            pass
        else:
            language = ""

        #title
        for title in root.findall('title'):
            title = title.text
        #sets title to filename if it does not exist
        if 'title' in locals():
            title = filter(lambda x: x in string.printable, title)
            title = title.replace('"', '')
            title = title.replace("'", " ")
        else:
            title = filename

        #author
        for creator in root.findall('creator'):
            author = creator.text

        if 'author' in locals():
            try:
                author = author.replace('"', '')
                author = author.replace("'", "")
            except AttributeError:
                author = ""
        else:
            author = ""
        #the above XML elements could be potentially abstracted into a single function

        #searchstring
        for arlink in root.findall('identifier-access'):
            searchstring = arlink.text
            searchstring = "[" + author + "], " + title + \
                        " (" + year + ") read"
            searchstring = searchstring.encode('ascii','ignore')
            searchstring = searchstring.replace('"', '')
            searchstring = searchstring.replace("'", " ")

        #hacked error handling
        if 'searchstring' in locals():
           pass
        else:
           searchstring = ""

        #location
        try:
            xmlItem = xfiles[count]
            marcItem = xmlItem[:-9] + "_marc.xml"
            print marcItem
            xml = open((os.path.join('/data/MARC', marcItem)), 'r')
            xmlRead = xml.read()
            xml.close()
            rootMarc = ET.fromstring(xmlRead)
            subjectArray = []
            for child in rootMarc:
                    tag = child.get('tag')
                    if tag == "260":
                        for sub in child:
                            print sub.text[:9]
                            city = filter(lambda x: x in string.ascii_letters, sub.text)
                            if city == "Londonetc":
                            	location = "London"
                            elif city == "NewYork":
                            	location ="New York"
                            else:
                            	location = city

                            #code = child.get('c:ode')
                           # if code == "a":
                            #    print sub
            if 'location' in locals():
                pass
            else:
                location = ""

        except IOError:
            location = ""

        #subjects
        try:
            xmlItem = xfiles[count]
            marcItem = xmlItem[:-9] + "_marc.xml"
            xml = open((os.path.join('/data/MARC', marcItem)), 'r')
            xmlRead = xml.read()
            xml.close()
            rootMarc = ET.fromstring(xmlRead)
            subjectArray = []
            for child in rootMarc:
                    tag = child.get('tag')
                    if tag == "650":
                            for sub in child:

                                try:
                                    subword = sub.text
                                    subword = subword.replace(".", "")
                                    subword = subword.replace(";", "")
                                    subjectArray.append(subword)
                                except AttributeError:
                                    subjectArray.append(sub.text)
        except IOError:

            subjectArray = []
            for subject in root.iter('subject'):
                subject = subject.text
                if subject is not None:
                    stext = subject.replace(',', '-')
                    stext = stext.replace('"', '')
                    #stext = filter(lambda x: x in string.ascii_letters, stext)
                    subArray = subject.split("--")
                    subArray = subject.split("-")
                    subjectArray.append(stext)

                    #cleans up the words i.e. semicolons will not appear as subjects
                    for word in subArray:
                        rx = re.compile('\W+')
                       #cleanWord = rx.sub('', word).strip()
                        cleanWord = rx
                        cleanWord = cleanWord.sub('"', '')
                        if cleanWord[:4].isdigit():
                            pass
                        elif len(cleanWord) < 3:
                            pass
                        #should probably make a dictionary
                        elif cleanWord=="and":
                            pass
                        elif cleanWord=="for":
                            pass
                        elif cleanWord=="drug" or "Drug":
                            pass
                        elif cleanWord=="industry" or "Industry":
                            pass
                        elif cleanWord=="system" or "System":
                            pass
                        else:
                            cleanWord = cleanWord.lower().capitalize()
                            subjectArray.append(cleanWord)
                else:
                    subjectArray = "Miscellaneous"

        #write json object to file
        jdict = {"library" : library,
                 "searchstring" : searchstring,
                 "filename" : filename,
                 "language" : language,
                 "year" : year,
                 "subject" : subjectArray,
                 "location": location,
                }

        json = str(jdict)
        json = json.replace("'", '"')
        #raw string is printing double backslashes - quick fix
        json = json.replace('\\"', '\"')

        print "writing %s metadata to jsoncatalog.txt..." % filename
        meta.write(json + '\n')

        #recursion
        if count < ((len(xfiles)) -1):
            count = count + 1
            buildMeta(count)
        else:
            print "\n jsoncatalog.txt done building! \n"

    buildMeta(count)

makeMeta(0)

<|endoftext|>


<commit_before>#!/usr/bin/env python2

# Copyright (c) 2014 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.

#
# Run Regression Test Suite
#

import os
import sys
import subprocess
import re
from tests_config import *
from sets import Set

#If imported values are not defined then set to zero (or disabled)
if not vars().has_key('ENABLE_WALLET'):
    ENABLE_WALLET=0
if not vars().has_key('ENABLE_BITCOIND'):
    ENABLE_BITCOIND=0
if not vars().has_key('ENABLE_UTILS'):
    ENABLE_UTILS=0

#Create a set to store arguments and create the passOn string
opts = Set()
passOn = ""
p = re.compile("^--")
for i in range(1,len(sys.argv)):
    if (p.match(sys.argv[i]) or sys.argv[i] == "-h"):
        passOn += " " + sys.argv[i]
    else:
        opts.add(sys.argv[i])

#Set env vars
buildDir = BUILDDIR
os.environ["BITCOIND"] = buildDir + '/src/bitcoind' + EXEEXT
os.environ["BITCOINCLI"] = buildDir + '/src/bitcoin-cli' + EXEEXT

#Disable Windows tests by default
if EXEEXT == ".exe" and "-win" not in opts:
    print "Win tests currently disabled.  Use -win option to enable"
    sys.exit(0)

#Tests
testScripts = [
    'bip68-112-113-p2p.py',
    'spv.py',
    'wallet.py',
    'listtransactions.py',
    'mempool_resurrect_test.py',
    'txn_clone.py',
    'txn_doublespend.py',
    'txn_doublespendrelay.py',
    'getchaintips.py',
    'rawtransactions.py',
    'rest.py',
    'mempool_spendcoinbase.py',
    'mempool_reorg.py',
    'httpbasics.py',
    'zapwallettxes.py',
    'proxy_test.py',
    'merkle_blocks.py',
    'signrawtransactions.py',
    'walletbackup.py',
    'p2p-versionbits-warning.py',
    'decodescript.py',
    'sendheaders.py'
]
testScriptsExt = [
    'bigblocks.py',
    'bip9-softforks.py',
    'bipdersig-p2p.py',
    'bip65-cltv-p2p.py',
    'bip65-cltv.py',
    'bip68-sequence.py',
    'getblocktemplate_longpoll.py',
    'getblocktemplate_proposals.py',
    'txn_clone.py --mineblock',
    'txn_doublespend.py --mineblock',
    'mempool_limit.py',
    'pruning.py',
    'forknotify.py',
    'invalidateblock.py',
    'keypool.py',
    'receivedby.py',
    'reindex.py',
    'rpcbind_test.py',
#   'script_test.py',
    'smartfees.py',
    'maxblocksinflight.py',
    'invalidblockrequest.py',
    'rawtransactions.py',
    'p2p-acceptblock.py',
    'p2p-compactblocks.py',
    'mempool_packages.py'
]

if(ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1):
    rpcTestDir = buildDir + '/qa/rpc-tests/'
    #Run Tests
    for i in range(len(testScripts)):
       if (len(opts) == 0 or (len(opts) == 1 and "-win" in opts ) or '-extended' in opts
           or testScripts[i] in opts or  re.sub(".py$", "", testScripts[i]) in opts ):
            print  "Running testscript " + testScripts[i] + "..."
            subprocess.call(rpcTestDir + testScripts[i] + " --srcdir " + buildDir + '/src ' + passOn,shell=True)
	    #exit if help is called so we print just one set of instructions
            p = re.compile(" -h| --help")
            if p.match(passOn):
                sys.exit(0)

    #Run Extended Tests
    for i in range(len(testScriptsExt)):
        if ('-extended' in opts or testScriptsExt[i] in opts
           or re.sub(".py$", "", testScriptsExt[i]) in opts):
            print  "Running 2nd level testscript " + testScriptsExt[i] + "..."
            subprocess.call(rpcTestDir + testScriptsExt[i] + " --srcdir " + buildDir + '/src ' + passOn,shell=True)
else:
    print "No rpc tests to run. Wallet, utils, and bitcoind must all be enabled"
<commit_msg>qa/pull-tester/rpc-tests.py: chmod 0755<commit_after>#!/usr/bin/env python2
# Copyright (c) 2014 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.

#
# Run Regression Test Suite
#

import os
import sys
import subprocess
import re
from tests_config import *
from sets import Set

#If imported values are not defined then set to zero (or disabled)
if not vars().has_key('ENABLE_WALLET'):
    ENABLE_WALLET=0
if not vars().has_key('ENABLE_BITCOIND'):
    ENABLE_BITCOIND=0
if not vars().has_key('ENABLE_UTILS'):
    ENABLE_UTILS=0

#Create a set to store arguments and create the passOn string
opts = Set()
passOn = ""
p = re.compile("^--")
for i in range(1,len(sys.argv)):
    if (p.match(sys.argv[i]) or sys.argv[i] == "-h"):
        passOn += " " + sys.argv[i]
    else:
        opts.add(sys.argv[i])

#Set env vars
buildDir = BUILDDIR
os.environ["BITCOIND"] = buildDir + '/src/bitcoind' + EXEEXT
os.environ["BITCOINCLI"] = buildDir + '/src/bitcoin-cli' + EXEEXT

#Disable Windows tests by default
if EXEEXT == ".exe" and "-win" not in opts:
    print "Win tests currently disabled.  Use -win option to enable"
    sys.exit(0)

#Tests
testScripts = [
    'bip68-112-113-p2p.py',
    'spv.py',
    'wallet.py',
    'listtransactions.py',
    'mempool_resurrect_test.py',
    'txn_clone.py',
    'txn_doublespend.py',
    'txn_doublespendrelay.py',
    'getchaintips.py',
    'rawtransactions.py',
    'rest.py',
    'mempool_spendcoinbase.py',
    'mempool_reorg.py',
    'httpbasics.py',
    'zapwallettxes.py',
    'proxy_test.py',
    'merkle_blocks.py',
    'signrawtransactions.py',
    'walletbackup.py',
    'p2p-versionbits-warning.py',
    'decodescript.py',
    'sendheaders.py'
]
testScriptsExt = [
    'bigblocks.py',
    'bip9-softforks.py',
    'bipdersig-p2p.py',
    'bip65-cltv-p2p.py',
    'bip65-cltv.py',
    'bip68-sequence.py',
    'getblocktemplate_longpoll.py',
    'getblocktemplate_proposals.py',
    'txn_clone.py --mineblock',
    'txn_doublespend.py --mineblock',
    'mempool_limit.py',
    'pruning.py',
    'forknotify.py',
    'invalidateblock.py',
    'keypool.py',
    'receivedby.py',
    'reindex.py',
    'rpcbind_test.py',
#   'script_test.py',
    'smartfees.py',
    'maxblocksinflight.py',
    'invalidblockrequest.py',
    'rawtransactions.py',
    'p2p-acceptblock.py',
    'p2p-compactblocks.py',
    'mempool_packages.py'
]

if(ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1):
    rpcTestDir = buildDir + '/qa/rpc-tests/'
    #Run Tests
    for i in range(len(testScripts)):
       if (len(opts) == 0 or (len(opts) == 1 and "-win" in opts ) or '-extended' in opts
           or testScripts[i] in opts or  re.sub(".py$", "", testScripts[i]) in opts ):
            print  "Running testscript " + testScripts[i] + "..."
            subprocess.call(rpcTestDir + testScripts[i] + " --srcdir " + buildDir + '/src ' + passOn,shell=True)
	    #exit if help is called so we print just one set of instructions
            p = re.compile(" -h| --help")
            if p.match(passOn):
                sys.exit(0)

    #Run Extended Tests
    for i in range(len(testScriptsExt)):
        if ('-extended' in opts or testScriptsExt[i] in opts
           or re.sub(".py$", "", testScriptsExt[i]) in opts):
            print  "Running 2nd level testscript " + testScriptsExt[i] + "..."
            subprocess.call(rpcTestDir + testScriptsExt[i] + " --srcdir " + buildDir + '/src ' + passOn,shell=True)
else:
    print "No rpc tests to run. Wallet, utils, and bitcoind must all be enabled"
<|endoftext|>


<commit_before>from unittest import TestCase
from equadratures.induced_distributions import InducedSampling

class TestInducedDistribution(TestCase):

    def test_generate_sample_measure(self):
        # test if the method returns a function object for induced sampling
        measure = InducedSampling(1, 1, 1, "Chebyshev", 0, 0)
        func = measure.generate_sample_measure("Chebyshev", 0, 0)
        assert callable(func)
<commit_msg>Remove previous induced sampling test as this is not relevent for now<commit_after>from unittest import TestCase
# from equadratures.induced_distributions import InducedSampling

class TestInducedDistribution(TestCase):

    def test_generate_sample_measure(self):
        # test if the method returns a function object for induced sampling
        # measure = InducedSampling(1, 1, 1, "Chebyshev", 0, 0)
        # func = measure.generate_sample_measure("Chebyshev", 0, 0)
        assert True
<|endoftext|>


<commit_before>'''
@author: David W.H. Swenson
'''

import os
import numpy as np

from nose.tools import (assert_equal, assert_not_equal, assert_items_equal,
                        assert_almost_equal, raises)
from nose.plugins.skip import Skip, SkipTest
from test_helpers import (assert_equal_array_array,
                          assert_not_equal_array_array)

from opentis.ensemble import LengthEnsemble
from opentis.trajectory import Sample
from opentis.sample import SampleSet
from opentis.pathmover import *

class testMakeListOfPairs(object):
    def setup(self):
        self.correct = [ [0, 1], [2, 3], [4, 5] ]

    @raises(TypeError)
    def test_not_iterable_type_error(self):
        result = make_list_of_pairs(1)

    def test_list_of_list_pairs(self):
        result = make_list_of_pairs([[0, 1], [2, 3], [4, 5]])
        assert_equal_array_array(result, self.correct)

    @raises(AssertionError)
    def test_list_of_list_notpairs(self):
        result = make_list_of_pairs([[0], [1], [2]])

    @raises(AssertionError)
    def test_list_not_even(self):
        result = make_list_of_pairs([0, 1, 2, 3, 4])

    def test_list_even(self):
        result = make_list_of_pairs([0, 1, 2, 3, 4, 5])
        assert_equal_array_array(result, self.correct)

    def test_empty(self):
        assert_equal(make_list_of_pairs(None), None)

class testPathMover(object):
    def setup(self):
        self.l1 = LengthEnsemble(1)
        self.l2 = LengthEnsemble(2)
        self.l3 = LengthEnsemble(3)
        self.repsAll_ensNone = PathMover(replicas='all')
        self.reps12_ensNone = PathMover(replicas=[1, 2])
        self.repsAll_ens1 = PathMover(ensembles=self.l1)
        self.repsAll_ens12 = PathMover(ensembles=[self.l1, self.l2])
        self.reps1_ens2 = PathMover(replicas=1, ensembles=[self.l2])
        self.s1 = Sample(replica=1, ensemble=self.l2)
        self.s2 = Sample(replica=2, ensemble=self.l1)
        self.s3 = Sample(replica=3, ensemble=self.l1)
        self.s4 = Sample(replica=2, ensemble=self.l3)
        self.sset = SampleSet([self.s1, self.s2, self.s3, self.s4])

    def test_legal_sample_set(self):
        assert_items_equal(self.repsAll_ensNone.legal_sample_set(self.sset),
                           [self.s1, self.s2, self.s3, self.s4])
        assert_items_equal(self.reps12_ensNone.legal_sample_set(self.sset),
                           [self.s1, self.s2, self.s4])
        assert_items_equal(self.repsAll_ens12.legal_sample_set(self.sset),
                           [self.s1, self.s2, self.s3])
        assert_items_equal(self.repsAll_ens1.legal_sample_set(self.sset),
                           [self.s2, self.s3])
        assert_items_equal(self.reps1_ens2.legal_sample_set(self.sset),
                           [self.s1])
        assert_items_equal(
            self.repsAll_ensNone.legal_sample_set(self.sset, ensembles=self.l1),
            [self.s2, self.s3]
        )
        assert_items_equal(
            self.repsAll_ensNone.legal_sample_set(self.sset, ensembles=[self.l1]),
            [self.s2, self.s3]
        )

    def test_select_sample(self):
        assert_equal(self.reps1_ens2.select_sample(self.sset), self.s1)
        try:
            assert_equal(self.repsAll_ens1.select_sample(self.sset), self.s2)
        except AssertionError:
            assert_equal(self.repsAll_ens1.select_sample(self.sset), self.s3)

class testForwardShootMover(object):
    def setup(self):
        pass

    def test_move(self):
        raise SkipTest

class testBackwardShootMover(object):
    def setup(self):
        pass

    def test_move(self):
        raise SkipTest

<commit_msg>Fixed stupid mistake.<commit_after>'''
@author: David W.H. Swenson
'''

import os
import numpy as np

from nose.tools import (assert_equal, assert_not_equal, assert_items_equal,
                        assert_almost_equal, raises)
from nose.plugins.skip import Skip, SkipTest
from test_helpers import (assert_equal_array_array,
                          assert_not_equal_array_array)

from opentis.ensemble import LengthEnsemble
from opentis.trajectory import Sample
from opentis.sample import SampleSet
from opentis.pathmover import *

class testMakeListOfPairs(object):
    def setup(self):
        self.correct = [ [0, 1], [2, 3], [4, 5] ]

    @raises(TypeError)
    def test_not_iterable_type_error(self):
        result = make_list_of_pairs(1)

    def test_list_of_list_pairs(self):
        result = make_list_of_pairs([[0, 1], [2, 3], [4, 5]])
        assert_equal_array_array(result, self.correct)

    @raises(AssertionError)
    def test_list_of_list_notpairs(self):
        result = make_list_of_pairs([[0], [1], [2]])

    @raises(AssertionError)
    def test_list_not_even(self):
        result = make_list_of_pairs([0, 1, 2, 3, 4])

    def test_list_even(self):
        result = make_list_of_pairs([0, 1, 2, 3, 4, 5])
        assert_equal_array_array(result, self.correct)

    def test_empty(self):
        assert_equal(make_list_of_pairs(None), None)

class testPathMover(object):
    def setup(self):
        self.l1 = LengthEnsemble(1)
        self.l2 = LengthEnsemble(2)
        self.l3 = LengthEnsemble(3)
        self.repsAll_ensNone = PathMover(replicas='all')
        self.reps12_ensNone = PathMover(replicas=[1, 2])
        self.repsAll_ens1 = PathMover(ensembles=self.l1)
        self.repsAll_ens12 = PathMover(ensembles=[self.l1, self.l2])
        self.reps1_ens2 = PathMover(replicas=1, ensembles=[self.l2])
        self.s1 = Sample(replica=1, ensemble=self.l2)
        self.s2 = Sample(replica=2, ensemble=self.l1)
        self.s3 = Sample(replica=3, ensemble=self.l1)
        self.s4 = Sample(replica=2, ensemble=self.l3)
        self.sset = SampleSet([self.s1, self.s2, self.s3, self.s4])

    def test_legal_sample_set(self):
        assert_items_equal(self.repsAll_ensNone.legal_sample_set(self.sset),
                           [self.s1, self.s2, self.s3, self.s4])
        assert_items_equal(self.reps12_ensNone.legal_sample_set(self.sset),
                           [self.s1, self.s2, self.s4])
        assert_items_equal(self.repsAll_ens12.legal_sample_set(self.sset),
                           [self.s1, self.s2, self.s3])
        assert_items_equal(self.repsAll_ens1.legal_sample_set(self.sset),
                           [self.s2, self.s3])
        assert_items_equal(self.reps1_ens2.legal_sample_set(self.sset),
                           [self.s1])
        assert_items_equal(
            self.repsAll_ensNone.legal_sample_set(self.sset, ensembles=self.l1),
            [self.s2, self.s3]
        )
        assert_items_equal(
            self.repsAll_ensNone.legal_sample_set(self.sset, ensembles=[self.l1]),
            [self.s2, self.s3]
        )

    def test_select_sample(self):
        assert_equal(self.reps1_ens2.select_sample(self.sset), self.s1)
        selected = self.repsAll_ens1.select_sample(self.sset)
        try:
            assert_equal(selected, self.s2)
        except AssertionError:
            assert_equal(selected, self.s3)

class testForwardShootMover(object):
    def setup(self):
        pass

    def test_move(self):
        raise SkipTest

class testBackwardShootMover(object):
    def setup(self):
        pass

    def test_move(self):
        raise SkipTest

<|endoftext|>


<commit_before>'''
@author: David W.H. Swenson
'''

import os
import numpy as np

from nose.tools import (assert_equal, assert_not_equal, assert_items_equal,
                        assert_almost_equal, raises)
from nose.plugins.skip import Skip, SkipTest

from opentis.pathmover import *

class testMakeListOfPairs(object):
    def setup(self):
        pass

    def test_not_iterable_type_error(self):
        raise SkipTest

    def test_list_of_list_pairs(object):
        raise SkipTest

    def test_list_of_list_notpairs(object):
        raise SkipTest

    def test_list_not_even(object):
        raise SkipTest

    def test_list_even(object):
        raise SkipTest

class testPathMover(object):
    def test_legal_sample_set(self):
        raise SkipTest

    def test_select_sample(self):
        raise SkipTest

class testForwardShootMover(object):
    def setup(self):
        pass

    def test_move(self):
        raise SkipTest

class testBackwardShootMover(object):
    def setup(self):
        pass

    def test_move(self):
        raise SkipTest

<commit_msg>Tests for make_list_of_pairs<commit_after>'''
@author: David W.H. Swenson
'''

import os
import numpy as np

from nose.tools import (assert_equal, assert_not_equal, assert_items_equal,
                        assert_almost_equal, raises)
from nose.plugins.skip import Skip, SkipTest
from test_helpers import assert_equal_array_array, assert_not_equal_array_array

from opentis.pathmover import *

class testMakeListOfPairs(object):
    def setup(self):
        self.correct = [ [0, 1], [2, 3], [4, 5] ]
        pass

    @raises(TypeError)
    def test_not_iterable_type_error(self):
        result = make_list_of_pairs(1)

    def test_list_of_list_pairs(self):
        result = make_list_of_pairs([[0, 1], [2, 3], [4, 5]])
        assert_equal_array_array(result, self.correct)

    @raises(AssertionError)
    def test_list_of_list_notpairs(self):
        result = make_list_of_pairs([[0], [1], [2]])

    @raises(AssertionError)
    def test_list_not_even(self):
        result = make_list_of_pairs([0, 1, 2, 3, 4])

    def test_list_even(self):
        result = make_list_of_pairs([0, 1, 2, 3, 4, 5])
        assert_equal_array_array(result, self.correct)

class testPathMover(object):
    def test_legal_sample_set(self):
        raise SkipTest

    def test_select_sample(self):
        raise SkipTest

class testForwardShootMover(object):
    def setup(self):
        pass

    def test_move(self):
        raise SkipTest

class testBackwardShootMover(object):
    def setup(self):
        pass

    def test_move(self):
        raise SkipTest

<|endoftext|>


<commit_before># Copyright 2016 Acsone
# Copyright 2020 Onestein (<https://www.onestein.eu>)
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

from odoo import fields
from odoo.exceptions import ValidationError

from odoo.addons.account.tests.invoice_test_common import InvoiceTestCommon

class TestAccountInvoiceSupplierRefUnique(InvoiceTestCommon):
    @classmethod
    def setUpClass(cls):
        super().setUpClass()

        # ENVIRONMENTS
        cls.account_account = cls.env["account.account"]
        cls.account_invoice = cls.env["account.move"].with_context(
            {"tracking_disable": True}
        )

        # INSTANCES
        cls.partner = cls.env.ref("base.res_partner_2")
        # Account for invoice
        cls.account = cls.account_account.search(
            [
                (
                    "user_type_id",
                    "=",
                    cls.env.ref("account.data_account_type_receivable").id,
                )
            ],
            limit=1,
        )
        # Invoice with unique reference 'ABC123'
        cls.invoice = cls.account_invoice.create(
            {
                "partner_id": cls.partner.id,
                "type": "in_invoice",
                "supplier_invoice_number": "ABC123",
            }
        )

    def test_check_unique_supplier_invoice_number_insensitive(self):
        # A new invoice instance with an existing supplier_invoice_number
        with self.assertRaises(ValidationError):
            self.account_invoice.create(
                {
                    "partner_id": self.partner.id,
                    "type": "in_invoice",
                    "supplier_invoice_number": "ABC123",
                }
            )
        # A new invoice instance with a new supplier_invoice_number
        self.account_invoice.create(
            {
                "partner_id": self.partner.id,
                "type": "in_invoice",
                "supplier_invoice_number": "ABC123bis",
            }
        )

    def test_onchange_supplier_invoice_number(self):
        self.invoice._onchange_supplier_invoice_number()
        self.assertEqual(
            self.invoice.ref,
            self.invoice.supplier_invoice_number,
            "_onchange_supplier_invoice_number",
        )

    def test_copy_invoice(self):
        invoice2 = self.invoice.copy()
        self.assertNotEqual(self.invoice.ref, "")
        self.assertEqual(invoice2.ref, "")

    def test_reverse_invoice(self):
        move_reversal = (
            self.env["account.move.reversal"]
            .with_context(active_model="account.move", active_ids=self.invoice.ids)
            .create(
                {
                    "date": fields.Date.today(),
                    "reason": "no reason",
                    "refund_method": "refund",
                }
            )
        )
        reversal = move_reversal.reverse_moves()
        refund = self.env["account.move"].browse(reversal["res_id"])
        self.assertNotEqual(self.invoice.ref, "")
        self.assertEqual(refund.ref, "")

    def test_reverse_moves_robustness(self):
        res = self.invoice._reverse_moves()
        self.assertTrue(res.is_purchase_document(include_receipts=True))
<commit_msg>[FIX] account_invoice_supplier_ref_unique: fix tests<commit_after># Copyright 2016 Acsone
# Copyright 2020 Onestein (<https://www.onestein.eu>)
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

from odoo import fields
from odoo.exceptions import ValidationError
from odoo.tests import tagged

from odoo.addons.account.tests.account_test_savepoint import AccountTestInvoicingCommon

@tagged("post_install", "-at_install")
class TestAccountInvoiceSupplierRefUnique(AccountTestInvoicingCommon):
    @classmethod
    def setUpClass(cls, chart_template_ref=None):
        super().setUpClass(chart_template_ref=chart_template_ref)

        # ENVIRONMENTS
        cls.account_account = cls.env["account.account"]
        cls.account_invoice = cls.env["account.move"].with_context(
            {"tracking_disable": True}
        )

        # INSTANCES
        cls.partner = cls.env.ref("base.res_partner_2")
        # Account for invoice
        cls.account = cls.account_account.search(
            [
                (
                    "user_type_id",
                    "=",
                    cls.env.ref("account.data_account_type_receivable").id,
                )
            ],
            limit=1,
        )
        # Invoice with unique reference 'ABC123'
        cls.invoice = cls.account_invoice.create(
            {
                "partner_id": cls.partner.id,
                "type": "in_invoice",
                "supplier_invoice_number": "ABC123",
            }
        )

    def test_check_unique_supplier_invoice_number_insensitive(self):
        # A new invoice instance with an existing supplier_invoice_number
        with self.assertRaises(ValidationError):
            self.account_invoice.create(
                {
                    "partner_id": self.partner.id,
                    "type": "in_invoice",
                    "supplier_invoice_number": "ABC123",
                }
            )
        # A new invoice instance with a new supplier_invoice_number
        self.account_invoice.create(
            {
                "partner_id": self.partner.id,
                "type": "in_invoice",
                "supplier_invoice_number": "ABC123bis",
            }
        )

    def test_onchange_supplier_invoice_number(self):
        self.invoice._onchange_supplier_invoice_number()
        self.assertEqual(
            self.invoice.ref,
            self.invoice.supplier_invoice_number,
            "_onchange_supplier_invoice_number",
        )

    def test_copy_invoice(self):
        invoice2 = self.invoice.copy()
        self.assertNotEqual(self.invoice.ref, "")
        self.assertEqual(invoice2.ref, "")

    def test_reverse_invoice(self):
        move_reversal = (
            self.env["account.move.reversal"]
            .with_context(active_model="account.move", active_ids=self.invoice.ids)
            .create(
                {
                    "date": fields.Date.today(),
                    "reason": "no reason",
                    "refund_method": "refund",
                }
            )
        )
        reversal = move_reversal.reverse_moves()
        refund = self.env["account.move"].browse(reversal["res_id"])
        self.assertNotEqual(self.invoice.ref, "")
        self.assertEqual(refund.ref, "")

    def test_reverse_moves_robustness(self):
        res = self.invoice._reverse_moves()
        self.assertTrue(res.is_purchase_document(include_receipts=True))
<|endoftext|>


<commit_before>"""
Register reports for an experiment
"""
import importlib
import logging
import typing as t
from benchbuild.settings import CFG

LOG = logging.getLogger(__name__)

def discover():
    """
    Import all experiments listed in *_PLUGINS_REPORTS.

    Tests:
        >>> from benchbuild.settings import CFG
        >>> from benchbuild.reports import discover
        >>> import logging as lg
        >>> import sys
        >>> l = lg.getLogger('benchbuild')
        >>> l.setLevel(lg.DEBUG)
        >>> l.handlers = [lg.StreamHandler(stream=sys.stdout)]
        >>> CFG["plugins"]["reports"] = ["benchbuild.non.existing", "benchbuild.reports.raw"]
        >>> discover()
        Could not find 'benchbuild.non.existing'
        Found report: benchbuild.reports.raw
    """
    if CFG["plugins"]["autoload"].value():
        report_plugins = CFG["plugins"]["reports"].value()
        for ep in report_plugins:
            try:
                importlib.import_module(ep)
                LOG.debug("Found report: %s", ep)
            except ImportError:
                LOG.error("Could not find '%s'", ep)

class ReportRegistry(type):
    reports = {}

    def __init__(cls, name, bases, dict):
        super(ReportRegistry, cls).__init__(name, bases, dict)
        if cls.NAME is not None:
            ReportRegistry.reports[cls.NAME] = cls

def load_experiment_ids_from_names(session, names):
    from sqlalchemy import func, column
    from sqlalchemy.sql import select, bindparam

    exps = select([column('id')]).\
        select_from(
            func.experiments(bindparam('names'))
        )
    r1 = session.execute(exps.unique_params(names=names))
    return r1.fetchall()

class Report(object, metaclass=ReportRegistry):

    SUPPORTED_EXPERIMENTS = []
    NAME = None

    def __new__(cls, *args, **kwargs):
        del args, kwargs  # Temporarily unused
        new_self = super(Report, cls).__new__(cls)
        if not cls.SUPPORTED_EXPERIMENTS:
            raise AttributeError(
                "{0} @ {1} does not define a SUPPORTED_EXPERIMENTS attribute"
                .format(cls.__name__, cls.__module__))

        if cls.NAME is None:
            raise AttributeError("{0} @ {1} does not define a NAME attribute"
                                 .format(cls.__name__, cls.__module__))
        new_self.name = cls.NAME
        return new_self

    def __init__(self, exp_name, exp_ids, out_path):
        import benchbuild.utils.schema as schema
        import uuid
        self.out_path = out_path
        self.session = schema.Session()
        if not exp_ids:
            exp_ids = load_experiment_ids_from_names(
                self.session,
                [exp for exp in self.SUPPORTED_EXPERIMENTS if exp == exp_name])
            exp_ids = [v[0] for v in exp_ids]
        else:
            exp_ids = [uuid.UUID(v) for v in exp_ids]

        self.experiment_ids = exp_ids
<commit_msg>reports: attributize Report class<commit_after>"""
Register reports for an experiment
"""
import importlib
import logging
import typing as t
import uuid

import attr

import benchbuild.utils.schema as schema
from benchbuild.settings import CFG

LOG = logging.getLogger(__name__)

def discover():
    """
    Import all experiments listed in *_PLUGINS_REPORTS.

    Tests:
        >>> from benchbuild.settings import CFG
        >>> from benchbuild.reports import discover
        >>> import logging as lg
        >>> import sys
        >>> l = lg.getLogger('benchbuild')
        >>> l.setLevel(lg.DEBUG)
        >>> l.handlers = [lg.StreamHandler(stream=sys.stdout)]
        >>> CFG["plugins"]["reports"] = ["benchbuild.non.existing", "benchbuild.reports.raw"]
        >>> discover()
        Could not find 'benchbuild.non.existing'
        Found report: benchbuild.reports.raw
    """
    if CFG["plugins"]["autoload"].value():
        report_plugins = CFG["plugins"]["reports"].value()
        for ep in report_plugins:
            try:
                importlib.import_module(ep)
                LOG.debug("Found report: %s", ep)
            except ImportError:
                LOG.error("Could not find '%s'", ep)

class ReportRegistry(type):
    reports = {}

    def __init__(cls, name, bases, dict):
        super(ReportRegistry, cls).__init__(name, bases, dict)
        if cls.NAME is not None:
            ReportRegistry.reports[cls.NAME] = cls

def load_experiment_ids_from_names(session, names):
    from sqlalchemy import func, column
    from sqlalchemy.sql import select, bindparam

    exps = select([column('id')]).\
        select_from(
            func.experiments(bindparam('names'))
        )
    r1 = session.execute(exps.unique_params(names=names))
    return r1.fetchall()

@attr.s
class Report(object, metaclass=ReportRegistry):

    SUPPORTED_EXPERIMENTS = []
    NAME = None

    def __new__(cls, *args, **kwargs):
        del args, kwargs  # Temporarily unused
        new_self = super(Report, cls).__new__(cls)
        if not cls.SUPPORTED_EXPERIMENTS:
            raise AttributeError(
                "{0} @ {1} does not define a SUPPORTED_EXPERIMENTS attribute"
                .format(cls.__name__, cls.__module__))

        if cls.NAME is None:
            raise AttributeError("{0} @ {1} does not define a NAME attribute"
                                 .format(cls.__name__, cls.__module__))
        new_self.name = cls.NAME
        return new_self

    experiment_name = attr.ib()
    exp_ids = attr.ib()
    out_path = attr.ib()

    name = attr.ib(
        default=attr.Factory(lambda self: type(self).NAME, takes_self=True))

    supported_experiments = attr.ib(
        default=attr.Factory(lambda self: type(self).NAME, takes_self=True))

    session = attr.ib(default=schema.Session())

    experiment_ids = attr.ib(default=None)

    def __attrs_post_init__(self):
        if not self.exp_ids:
            exp_ids = load_experiment_ids_from_names(self.session, [
                exp for exp in self.SUPPORTED_EXPERIMENTS
                if exp == self.experiment_name
            ])
            exp_ids = [v[0] for v in exp_ids]
        else:
            exp_ids = [uuid.UUID(v) for v in self.exp_ids]
        self.experiment_ids = exp_ids<|endoftext|>


<commit_before>from iocbuilder import AutoSubstitution, SetSimulation, Device, Architecture, ModuleBase
from iocbuilder.arginfo import *

from iocbuilder.modules.motor import basic_asyn_motor, MotorRecord
from iocbuilder.modules.asyn import AsynPort
from iocbuilder.modules.calc import Calc
from iocbuilder.modules.busy import Busy
from iocbuilder.modules.seq import Seq
from iocbuilder.modules.genSub import GenSub
from iocbuilder.modules.streamDevice import AutoProtocol

import os, sys

class PmacUtil(Device):
    Dependencies = (GenSub,Seq)
    LibFileList = ["pmacUtil"]
    DbdFileList = ["pmacUtilSupport"]
    AutoInstantiate = True

class autohome(AutoSubstitution):
    TemplateFile = 'autohome.template'
SetSimulation(autohome, None)

try:
    from iocbuilder.modules.pmacCoord import PmacCoord

    class _dls_pmac_asyn_motor(AutoSubstitution):
        WarnMacros = False
        TemplateFile = 'dls_pmac_asyn_motor.template'

    class dls_pmac_asyn_motor(basic_asyn_motor, AutoProtocol):
        ProtocolFiles = ['pmac.proto']
        Dependencies = (Busy,PmacCoord)
        TemplateFile = 'dls_pmac_asyn_motor.template'
        Arguments = basic_asyn_motor.Arguments + _dls_pmac_asyn_motor.Arguments
        ArgInfo = basic_asyn_motor.ArgInfo + _dls_pmac_asyn_motor.ArgInfo
    dls_pmac_asyn_motor.Defaults.update(_dls_pmac_asyn_motor.Defaults)

    def dls_pmac_asyn_motor_sim(**args):
        # if it's a simulation, just connect it to a basic_asyn_motor
        return basic_asyn_motor(**filter_dict(args, basic_asyn_motor.Arguments))
    SetSimulation(dls_pmac_asyn_motor, dls_pmac_asyn_motor_sim)

except ImportError:
    print "# pmacCoord not found, dls_pmac_asyn_motor will not be available"

class _dls_pmac_cs_asyn_motor(AutoSubstitution):
    WarnMacros = False
    TemplateFile = 'dls_pmac_cs_asyn_motor.template'

class dls_pmac_cs_asyn_motor(basic_asyn_motor):
    TemplateFile = 'dls_pmac_cs_asyn_motor.template'
    Arguments = basic_asyn_motor.Arguments + _dls_pmac_cs_asyn_motor.Arguments
    ArgInfo = basic_asyn_motor.ArgInfo + _dls_pmac_cs_asyn_motor.ArgInfo
dls_pmac_cs_asyn_motor.Defaults.update(basic_asyn_motor.Defaults)

def dls_pmac_cs_asyn_motor_sim(**args):
    # if it's a simulation, just connect it to a basic_asyn_motor
    return basic_asyn_motor(**filter_dict(args, basic_asyn_motor.Arguments))
SetSimulation(dls_pmac_cs_asyn_motor, dls_pmac_cs_asyn_motor_sim)

class _pmacStatusAxis(AutoSubstitution, AutoProtocol):
    ProtocolFiles = ['pmac.proto']
    TemplateFile = 'pmacStatusAxis.template'

class pmacStatus(AutoSubstitution, AutoProtocol):
    Dependencies = (PmacUtil,)
    ProtocolFiles = ['pmac.proto']
    TemplateFile = 'pmacStatus.template'

    def __init__(self, **args):
        # init the super class
        self.__super.__init__(**args)
        self.axes = []
        NAXES = int(args["NAXES"])
        assert NAXES in range(1,33), "Number of axes (%d) must be in range 1..32" % NAXES
        # for each axis
        for i in range(1, NAXES + 1):
            args["AXIS"] = i
            # make a _pmacStatusAxis instance
            self.axes.append(
                _pmacStatusAxis(
                    **filter_dict(args, _pmacStatusAxis.ArgInfo.Names())))

class CS_B22_Optics_Box(AutoSubstitution):
    TemplateFile = 'B22_optics_box.template'

SetSimulation(CS_B22_Optics_Box, None)

class gather(AutoSubstitution, Device):
    '''Setup PMAC or Geobrick gathering template'''

    Dependencies = (PmacUtil,)

    def PostIocInitialise(self):
        if Architecture() == "linux-x86":
            print 'seq(gather,"P=%(P)s,M=%(M)s")' % self.args
        else:
            print 'seq &gather,"P=%(P)s,M=%(M)s"' % self.args

    # Substitution attributes
    TemplateFile = 'gather.template'

class positionCompare(AutoSubstitution, AutoProtocol):
    '''Setup position compare on a delta tau. Needs PLC_PMAC_position_compare
    or PLC_BRICK_position_compare'''
    Dependencies = (Calc,)
    ProtocolFiles = ['pmac.proto']
    TemplateFile = 'positionCompare.template'
<commit_msg>pmacCoord: fixed builder file<commit_after>from iocbuilder import AutoSubstitution, SetSimulation, Device, Architecture, ModuleBase
from iocbuilder.arginfo import *

from iocbuilder.modules.motor import basic_asyn_motor, MotorRecord
from iocbuilder.modules.asyn import AsynPort
from iocbuilder.modules.calc import Calc
from iocbuilder.modules.busy import Busy
from iocbuilder.modules.seq import Seq
from iocbuilder.modules.genSub import GenSub
from iocbuilder.modules.streamDevice import AutoProtocol

import os, sys

class PmacUtil(Device):
    Dependencies = (GenSub,Seq)
    LibFileList = ["pmacUtil"]
    DbdFileList = ["pmacUtilSupport"]
    AutoInstantiate = True

class autohome(AutoSubstitution):
    TemplateFile = 'autohome.template'
SetSimulation(autohome, None)

def add_basic(cls):
    """Convenience function to add basic_asyn_motor attributes to a class that
    includes it via an msi include statement rather than verbatim"""
    cls.Arguments += [x for x in basic_asyn_motor.Arguments if x not in cls.Arguments]
    cls.ArgInfo += basic_asyn_motor.ArgInfo
    cls.Defaults.update(basic_asyn_motor.Defaults)
    return cls

try:
    from iocbuilder.modules.pmacCoord import PmacCoord

    @add_basic
    class dls_pmac_asyn_motor(AutoSubstitution, AutoProtocol):
        WarnMacros = False
        TemplateFile = 'dls_pmac_asyn_motor.template'
        ProtocolFiles = ['pmac.proto']
        Dependencies = (Busy,PmacCoord)

    def dls_pmac_asyn_motor_sim(**args):
        # if it's a simulation, just connect it to a basic_asyn_motor
        return basic_asyn_motor(**filter_dict(args, basic_asyn_motor.Arguments))
    SetSimulation(dls_pmac_asyn_motor, dls_pmac_asyn_motor_sim)

except ImportError:
    print "# pmacCoord not found, dls_pmac_asyn_motor will not be available"

@add_basic
class dls_pmac_cs_asyn_motor(AutoSubstitution):
    WarnMacros = False
    TemplateFile = 'dls_pmac_cs_asyn_motor.template'
    Dependencies = (Busy,)

def dls_pmac_cs_asyn_motor_sim(**args):
    # if it's a simulation, just connect it to a basic_asyn_motor
    return basic_asyn_motor(**filter_dict(args, basic_asyn_motor.Arguments))
SetSimulation(dls_pmac_cs_asyn_motor, dls_pmac_cs_asyn_motor_sim)

class _pmacStatusAxis(AutoSubstitution, AutoProtocol):
    ProtocolFiles = ['pmac.proto']
    TemplateFile = 'pmacStatusAxis.template'

class pmacStatus(AutoSubstitution, AutoProtocol):
    Dependencies = (PmacUtil,)
    ProtocolFiles = ['pmac.proto']
    TemplateFile = 'pmacStatus.template'

    def __init__(self, **args):
        # init the super class
        self.__super.__init__(**args)
        self.axes = []
        NAXES = int(args["NAXES"])
        assert NAXES in range(1,33), "Number of axes (%d) must be in range 1..32" % NAXES
        # for each axis
        for i in range(1, NAXES + 1):
            args["AXIS"] = i
            # make a _pmacStatusAxis instance
            self.axes.append(
                _pmacStatusAxis(
                    **filter_dict(args, _pmacStatusAxis.ArgInfo.Names())))

class gather(AutoSubstitution, Device):
    '''Setup PMAC or Geobrick gathering template'''

    Dependencies = (PmacUtil,)

    def PostIocInitialise(self):
        if Architecture() == "linux-x86":
            print 'seq(gather,"P=%(P)s,M=%(M)s")' % self.args
        else:
            print 'seq &gather,"P=%(P)s,M=%(M)s"' % self.args

    # Substitution attributes
    TemplateFile = 'gather.template'

class positionCompare(AutoSubstitution, AutoProtocol):
    '''Setup position compare on a delta tau. Needs PLC_PMAC_position_compare
    or PLC_BRICK_position_compare'''
    Dependencies = (Calc,)
    ProtocolFiles = ['pmac.proto']
    TemplateFile = 'positionCompare.template'
<|endoftext|>


<commit_before># Copyright 2014 Red Hat, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Provides a process model

This module defines a class representing a process,
potentially with subprocesses.
"""

import psutil

import oslo_reports.models.with_default_views as mwdv
import oslo_reports.views.text.process as text_views

PS1 = psutil.version_info[0] == 1

class ProcessModel(mwdv.ModelWithDefaultViews):
    """A Process Model

    This model holds data about a process,
    including references to any subprocesses

    :param process: a :class:`psutil.Process` object
    """

    def __init__(self, process):
        super(ProcessModel, self).__init__(
            text_view=text_views.ProcessView())

        self['pid'] = process.pid
        self['parent_pid'] = (process.ppid if PS1 else process.ppid())
        if hasattr(process, 'uids'):
            self['uids'] = {
                'real': (process.uids.real if PS1 else process.uids().real),
                'effective': (process.uids.effective if PS1
                              else process.uids().effective),
                'saved': (process.uids.saved if PS1 else process.uids().saved)
            }
        else:
            self['uids'] = {'real': None,
                            'effective': None,
                            'saved': None}

        if hasattr(process, 'gids'):
            self['gids'] = {
                'real': (process.gids.real if PS1 else process.gids().real),
                'effective': (process.gids.effective if PS1
                              else process.gids().effective),
                'saved': (process.gids.saved if PS1 else process.gids().saved)
            }
        else:
            self['gids'] = {'real': None,
                            'effective': None,
                            'saved': None}

        self['username'] = process.username if PS1 else process.username()
        self['command'] = process.cmdline if PS1 else process.cmdline()
        self['state'] = process.status if PS1 else process.status()

        children = process.get_children() if PS1 else process.children()
        self['children'] = [ProcessModel(pr) for pr in children]
<commit_msg>Remove psutil 1.x compat wrapping<commit_after># Copyright 2014 Red Hat, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Provides a process model

This module defines a class representing a process,
potentially with subprocesses.
"""

import oslo_reports.models.with_default_views as mwdv
import oslo_reports.views.text.process as text_views

class ProcessModel(mwdv.ModelWithDefaultViews):
    """A Process Model

    This model holds data about a process,
    including references to any subprocesses

    :param process: a :class:`psutil.Process` object
    """

    def __init__(self, process):
        super(ProcessModel, self).__init__(
            text_view=text_views.ProcessView())

        self['pid'] = process.pid
        self['parent_pid'] = process.ppid()
        if hasattr(process, 'uids'):
            self['uids'] = {
                'real': process.uids().real,
                'effective': process.uids().effective,
                'saved': process.uids().saved
            }
        else:
            self['uids'] = {'real': None,
                            'effective': None,
                            'saved': None}

        if hasattr(process, 'gids'):
            self['gids'] = {
                'real': process.gids().real,
                'effective': process.gids().effective,
                'saved': process.gids().saved
            }
        else:
            self['gids'] = {'real': None,
                            'effective': None,
                            'saved': None}

        self['username'] = process.username()
        self['command'] = process.cmdline()
        self['state'] = process.status()

        children = process.children()
        self['children'] = [ProcessModel(pr) for pr in children]
<|endoftext|>


<commit_before>import os
import sys
import unittest
import psycopg2

from testing import postgresql as pgsql_test
from conf import settings

def handler(postgresql):
    f = open(os.path.dirname(os.path.realpath(__file__)) + "/../sql/init.sql")
    sql_query = f.read()
    f.close()
    conn = psycopg2.connect(**postgresql.dsn())
    cursor = conn.cursor()
    cursor.execute(sql_query)
    cursor.close()
    conn.commit()
    conn.close()

PGSQLFactory = pgsql_test.PostgresqlFactory(
    cache_initialized_db=True,
    on_initialized=handler
)

postgresql = PGSQLFactory()
settings['db_url'] = postgresql.url()

sys.path.append(sys.path[0] + "/..")
from main import app  # noqa: E402

class AppTestCase(unittest.TestCase):
    test_cli = app.test_client()
<commit_msg>Implements AppTestCase.upload_match_report<commit_after>import gzip
import os
import sys
import unittest
import psycopg2

from testing import postgresql as pgsql_test
from conf import settings

def handler(postgresql):
    f = open(os.path.dirname(os.path.realpath(__file__)) + "/../sql/init.sql")
    sql_query = f.read()
    f.close()
    conn = psycopg2.connect(**postgresql.dsn())
    cursor = conn.cursor()
    cursor.execute(sql_query)
    cursor.close()
    conn.commit()
    conn.close()

PGSQLFactory = pgsql_test.PostgresqlFactory(
    cache_initialized_db=True,
    on_initialized=handler
)

postgresql = PGSQLFactory()
settings['db_url'] = postgresql.url()

sys.path.append(sys.path[0] + "/..")
from main import app  # noqa: E402

class AppTestCase(unittest.TestCase):
    test_cli = app.test_client()

    def upload_match_report(self, sample_name=None, sample=None, headers=None):
        if sample_name is None and sample is None:
            raise AssertionError("Both sample_name and sample are NOT given")

        if sample_name is not None and sample is not None:
            raise AssertionError("sample_name and sample are BOTH given. Only one of them is required")

        if headers is None:
            headers = {
                'Content-Type': "text/plain",
                'X-D0-Blind-Id-Detached-Signature': "dummy",
            }

        if sample_name is not None:
            f = gzip.open(os.path.dirname(os.path.realpath(__file__)) + "/match_samples/" + sample_name + ".gz")
            sample = f.read()
            f.close()

        return self.test_cli.post("/stats/submit", headers=headers, data=sample)
<|endoftext|>