<commit_before>from ..migrator import upgrade_step
from pyramid.traversal import find_root

@upgrade_step('file', '', '2')
def file_0_2(value, system):
    # http://redmine.encodedcc.org/issues/1295
    # http://redmine.encodedcc.org/issues/1307
    if 'status' in value:
        value['status'] = value['status'].lower()

@upgrade_step('file', '2', '3')
def file_2_3(value, system):
    #  http://redmine.encodedcc.org/issues/1572
    file_format = value.get('file_format')
    file_name = value['submitted_file_name'].rsplit('/', 1)[-1]
    file_ext = file_name[file_name.find('.'):]

    # REJECTIONS
    if file_ext in ['.gtf.bigBed', '.pdf', '.pdf.gz', '.gff.bigBed', '.spikeins']:
        value['status'] = 'deleted'

    # Find the miscatorgorized bedMethyls
    if file_ext == '.bed.bigBed' and 'MethylRrbs' in value.get('submitted_file_name'):
        value['file_format'] = 'bedMethyl'
    if file_ext == '.bed.gz' and 'MethylRrbs' in value.get('submitted_file_name'):
        value['file_format'] = 'bed_bedMethyl'

    unknownDict = {'.CEL.gz': 'CEL',
                    '.bb': 'bedMethyl',
                    '.bed': 'bed',
                    '.bed.bigBed': 'bigBed',
                    '.bed9': 'bedMethyl',
                    '.bedCluster.bigBed': 'bigBed',
                    '.bedLogR.bigBed': 'bedLogR',
                    '.bedRnaElements.bigBed': 'bedRnaElements',
                    '.bedRrbs.bigBed': 'bedMethyl',
                    '.broadPeak.gz': 'bed_broadPeak',
                    '.bigBed': 'bigBed',
                    '.csfasta.gz': 'csfasta',
                    '.csqual.gz': 'csqual',
                    '.fasta.gz': 'fasta',
                    '.gff.bigBed': 'bigBed',
                    '.gff.gz': 'gtf',
                    '.gp.bigBed': 'bigBed',
                    '.matrix.gz': 'tsv',
                    '.matrix.tgz': 'tar',
                    '.narrowPeak': 'bed_narrowPeak',
                    '.narrowPeak.gz': 'bed_narrowPeak',
                    '.pdf': 'tsv',  # These are going to be obsolete
                    '.pdf.gz': 'tsv',  # These are going to be obsolete
                    '.peaks.gz': 'tsv',
                    '.peptideMapping.bigBed': '.bigBed',
                    '.shortFrags.bigBed': 'bigBed',
                    '.sorted.bigBed': 'bigBed',
                    '.tab.gz': 'tsv',
                    '.tgz': 'tar',
                    '.txt': 'tsv',
                    '.xlsx': 'tsv',  # These need to be converted to tsv
                   }
    if file_format in ['unknown', 'customTrack']:
        value['file_format'] = unknownDict[file_ext]

    # http://redmine.encodedcc.org/issues/1429
    status = value.get('status')
    context = system['context']
    root = find_root(context)
    dataset = root.get_by_uuid(value['dataset']) #.upgrade_properties(finalize=False)
    dataset_status = dataset.get('status')

    if status == 'current':
        if dataset_status == 'released':
            value['status'] = 'released'
    if status == 'obsolete':
        if dataset_status in ['released', 'revoked']:
            value['status'] = 'revoked'
        else:
            value['status'] = 'deleted'

    # http://redmine.encodedcc.org/issues/1618
    value['award'] = dataset['award']
    value['lab'] = dataset['lab']

    output_type_dict = {
                        "Alignments": "alignments",
                        "bigBed": "sites",
                        "bigWig": "sites",
                        "Clusters": "clusters",
                        "Contigs": "contigs",
                        "FastqRd1": "reads",
                        "FastqRd2": "reads",
                        "forebrain_enhancers": "enhancers_forebrain",
                        "heart_enhancers": "enhancers_heart",
                        "GreenIdat": "idat green file",
                        "hotspot_broad_peaks": "hotspots",
                        "hotspot_narrow_peaks": "hotspots",
                        "hotspot_signal": "hotspots",
                        "Hotspots": "hotspots",
                        "Interactions": "interactions",
                        "MinusRawSignal": "raw minus signal",
                        "PlusRawSignal": "plus raw signal",
                        "macs2_dnase_peaks": "peaks",
                        "macs2_dnase_signal": "signal",
                        "MinusSignal": "minus signal",
                        "minusSignal": "minus signal",
                        "MultiMinus": "multi-read minus signal",
                        "MultiPlus": "multi-read plus signal",
                        "MultiSignal": "multi-read signal",
                        "MultiUnstranded": "multi-read signal",
                        "RawData2": "reads",
                        "RedIdat": "idat red file",
                        "peak": "peaks",
                        "PeakCalls": "peaks",
                        "Peaks": "peaks",
                        "PlusSignal": "plus signal",
                        "plusSignal": "plus signal",
                        "predicted_enhancers_heart": "enhancers_heart",
                        "RawSignal": "raw signal",
                        "RawData": "raw data",
                        "rcc": "raw data",
                        "Read": "reads",
                        "read": "reads",
                        "read1": "reads",
                        "rejected_reads": "rejected reads",
                        "RepPeaks": "peaks",
                        "RepSignal": "signal",
                        "Signal": "signal",
                        "SimpleSignal": "signal",
                        "Sites": "sites",
                        "Spikeins": "spike-ins",
                        "Spikes": "spike-ins",
                        "Splices": "splice junctions",
                        "uniqueReads": "unique signal",
                        "UniqueSignal": "unique signal",
                        "uniqueSignal": "unique signal",
                        "UniqueMinus": "unique minus signal",
                        "uniqueMinusSignal": "unique minus signal",
                        "UniquePlus": "unique plus signal",
                        "uniquePlusSignal": "unique plus signal",
                        "UniqueUnstranded": "unique signal",
                        "UnstrandedSignal": "signal",
                        "dataset_used": "enhancers",
                        "TRAINING_DATA_MOUSE_VISTA": "enhancers",
                        "method_description": "enhancers",
                        "unknown": "enhancers",
                        "Protocol": "raw data",
                        }

    if value['output_type'] in output_type_dict:
        temp = output_type_dict[value['output_type']]
        value['output_type'] = temp

    # Help the raw data problem
    if value['output_type'] == 'raw data' and value['file_format'] == "fastq":
        value['output_type'] = 'reads' <commit_msg>using download_path instead of submitted_filename<commit_after>from ..migrator import upgrade_step
from pyramid.traversal import find_root

@upgrade_step('file', '', '2')
def file_0_2(value, system):
    # http://redmine.encodedcc.org/issues/1295
    # http://redmine.encodedcc.org/issues/1307
    if 'status' in value:
        value['status'] = value['status'].lower()

@upgrade_step('file', '2', '3')
def file_2_3(value, system):
    #  http://redmine.encodedcc.org/issues/1572
    file_format = value.get('file_format')
    file_name = value['download_path'].rsplit('/', 1)[-1]
    file_ext = file_name[file_name.find('.'):]

    # REJECTIONS
    if file_ext in ['.gtf.bigBed', '.pdf', '.pdf.gz', '.gff.bigBed', '.spikeins']:
        value['status'] = 'deleted'

    # Find the miscatorgorized bedMethyls
    if file_ext == '.bed.bigBed' and 'MethylRrbs' in value.get('submitted_file_name'):
        value['file_format'] = 'bedMethyl'
    if file_ext == '.bed.gz' and 'MethylRrbs' in value.get('submitted_file_name'):
        value['file_format'] = 'bed_bedMethyl'

    unknownDict = {'.CEL.gz': 'CEL',
                    '.bb': 'bedMethyl',
                    '.bed': 'bed',
                    '.bed.bigBed': 'bigBed',
                    '.bed9': 'bedMethyl',
                    '.bedCluster.bigBed': 'bigBed',
                    '.bedLogR.bigBed': 'bedLogR',
                    '.bedRnaElements.bigBed': 'bedRnaElements',
                    '.bedRrbs.bigBed': 'bedMethyl',
                    '.broadPeak.gz': 'bed_broadPeak',
                    '.bigBed': 'bigBed',
                    '.csfasta.gz': 'csfasta',
                    '.csqual.gz': 'csqual',
                    '.fasta.gz': 'fasta',
                    '.gff.bigBed': 'bigBed',
                    '.gff.gz': 'gtf',
                    '.gp.bigBed': 'bigBed',
                    '.matrix.gz': 'tsv',
                    '.matrix.tgz': 'tar',
                    '.narrowPeak': 'bed_narrowPeak',
                    '.narrowPeak.gz': 'bed_narrowPeak',
                    '.pdf': 'tsv',  # These are going to be obsolete
                    '.pdf.gz': 'tsv',  # These are going to be obsolete
                    '.peaks.gz': 'tsv',
                    '.peptideMapping.bigBed': '.bigBed',
                    '.shortFrags.bigBed': 'bigBed',
                    '.sorted.bigBed': 'bigBed',
                    '.tab.gz': 'tsv',
                    '.tgz': 'tar',
                    '.txt': 'tsv',
                    '.xlsx': 'tsv',  # These need to be converted to tsv
                   }
    if file_format in ['unknown', 'customTrack']:
        value['file_format'] = unknownDict[file_ext]

    # http://redmine.encodedcc.org/issues/1429
    status = value.get('status')
    context = system['context']
    root = find_root(context)
    dataset = root.get_by_uuid(value['dataset']) #.upgrade_properties(finalize=False)
    dataset_status = dataset.get('status')

    if status == 'current':
        if dataset_status == 'released':
            value['status'] = 'released'
    if status == 'obsolete':
        if dataset_status in ['released', 'revoked']:
            value['status'] = 'revoked'
        else:
            value['status'] = 'deleted'

    # http://redmine.encodedcc.org/issues/1618
    value['award'] = dataset['award']
    value['lab'] = dataset['lab']

    output_type_dict = {
                        "Alignments": "alignments",
                        "bigBed": "sites",
                        "bigWig": "sites",
                        "Clusters": "clusters",
                        "Contigs": "contigs",
                        "FastqRd1": "reads",
                        "FastqRd2": "reads",
                        "forebrain_enhancers": "enhancers_forebrain",
                        "heart_enhancers": "enhancers_heart",
                        "GreenIdat": "idat green file",
                        "hotspot_broad_peaks": "hotspots",
                        "hotspot_narrow_peaks": "hotspots",
                        "hotspot_signal": "hotspots",
                        "Hotspots": "hotspots",
                        "Interactions": "interactions",
                        "MinusRawSignal": "raw minus signal",
                        "PlusRawSignal": "plus raw signal",
                        "macs2_dnase_peaks": "peaks",
                        "macs2_dnase_signal": "signal",
                        "MinusSignal": "minus signal",
                        "minusSignal": "minus signal",
                        "MultiMinus": "multi-read minus signal",
                        "MultiPlus": "multi-read plus signal",
                        "MultiSignal": "multi-read signal",
                        "MultiUnstranded": "multi-read signal",
                        "RawData2": "reads",
                        "RedIdat": "idat red file",
                        "peak": "peaks",
                        "PeakCalls": "peaks",
                        "Peaks": "peaks",
                        "PlusSignal": "plus signal",
                        "plusSignal": "plus signal",
                        "predicted_enhancers_heart": "enhancers_heart",
                        "RawSignal": "raw signal",
                        "RawData": "raw data",
                        "rcc": "raw data",
                        "Read": "reads",
                        "read": "reads",
                        "read1": "reads",
                        "rejected_reads": "rejected reads",
                        "RepPeaks": "peaks",
                        "RepSignal": "signal",
                        "Signal": "signal",
                        "SimpleSignal": "signal",
                        "Sites": "sites",
                        "Spikeins": "spike-ins",
                        "Spikes": "spike-ins",
                        "Splices": "splice junctions",
                        "uniqueReads": "unique signal",
                        "UniqueSignal": "unique signal",
                        "uniqueSignal": "unique signal",
                        "UniqueMinus": "unique minus signal",
                        "uniqueMinusSignal": "unique minus signal",
                        "UniquePlus": "unique plus signal",
                        "uniquePlusSignal": "unique plus signal",
                        "UniqueUnstranded": "unique signal",
                        "UnstrandedSignal": "signal",
                        "dataset_used": "enhancers",
                        "TRAINING_DATA_MOUSE_VISTA": "enhancers",
                        "method_description": "enhancers",
                        "unknown": "enhancers",
                        "Protocol": "raw data",
                        }

    if value['output_type'] in output_type_dict:
        temp = output_type_dict[value['output_type']]
        value['output_type'] = temp

    # Help the raw data problem
    if value['output_type'] == 'raw data' and value['file_format'] == "fastq":
        value['output_type'] = 'reads' <|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import common
import gdata
import logging
import argparse
import socket
import struct
import threading
import time

__program__ = "DREMO Client"
__version__ = '0.1a'
__author__ = "Josep Pon Farreny, Marc Piñol Pueyo"
__license__ = "MIT"
__status__ = "Development"

def main():
	global options

	setUpLogger()
	logging.info('Logging set up')

	host = options.host
	port = options.port
	logging.debug('Host to connect to: %s ' % host)
	logging.debug('Port to connect to: %s' % port)

	sinfo = common.SysInfo()
	awakener = threading.Event()
	auto_update = AutoUpdater(options.tbu, host, port, sinfo, awakener)

	mcsock = beginConnection(host, port)
	update(host, port, sinfo)

	auto_update.start()

	# crear socket
	# select
	# si socket de connexio -> tractarlo i tal
	# si socket de multicast -> lo mismo

#
#
def beginConnection(host, port):
	hello = '%c %s' %(gdata.SOH, port)

	response = communicateWithServer(host, port, hello)

	if response[0] == '200 OK':
		multicast_group, multicast_port = response[1].split()
		multicast_port = int(multicast_port)
		logging.debug('Received multicast group: %s' % multicast_group)
		logging.debug('Received multicast port: %d' % multicast_port)

	else:
		logging.info('Error: %s' % response[0])
		sys.exit(1)

	return initMulticast(multicast_group, multicast_port)

#
#
def initMulticast(group, port):
	sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
	sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # more than one sock can use this port
	sock.bind(('', port))
	mreq = struct.pack("4sl", socket.inet_aton(group), socket.INADDR_ANY)

	sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq)

	return sock

#
#
def communicateWithServer(host, port, msg):
	sock = socket.socket()
	sock.settimeout(3) # TODO: REMOVE MAGIC NUMBER
	sock.connect((host, port))
	sock.send(msg+'\n')

	logging.debug('Message \'%s\' sent' % msg)

	response = common.util.recvEnd(sock, '\n')

	logging.debug('Response from the server (response code): %s' % response)

	if response:
		if response[0] == '200 OK':
			resp = common.util.recvEnd(sock, '\n')
			response.extend(resp)

			if not resp: response = ['Unknown error']

			logging.debug('Response from the server (response): %s' % resp)
	else:
		response = ['Unknown error']

	logging.debug('Total response from the server: %s' % response)

	return response

#
#
# use it only before starting the autoupdater.
# Afterwards just awaken the autoupdater thread
# and it will send the data. This way thread
# synch and concurrent ${sinfo} edit
# avoiding is not needed
def update(host, port, sinfo):

	sinfo.update()
	sendXML(host, port, sinfo)

#
#
def sendXML(host, port, sinfo):
	XMLbuilder = common.sysinfoxml.SysInfoXMLBuilder()

	dao = sinfo.getSysInfoData()
	XMLbuilder.setXMLData(dao)

	msg = '%c %s %c' % (gdata.STX, XMLbuilder.getAsString(), gdata.ETX)

	response = communicateWithServer(host, port, msg)

	if response[0] != '200 OK':
		logging.info('Server error %s' % response[0])

#
#
def setUpLogger():
	"""setUpLogger(options) -> void

	Sets up the loggin module.

	"""
	global options

	lvl = logging.DEBUG if options.debug else logging.INFO
	#format = '[%(levelname)s] (%(asctime)s): %(message)s'
	format = '[%(levelname)s]: %(message)s'
	datefmt='%d/%m/%Y %H:%M:%S'

	logging.basicConfig(stream = options.logfile, level=lvl, format=format,
						datefmt=datefmt)

#
#
class AutoUpdater(threading.Thread):
	def __init__(self, tbu, host, port, sinfo, awakener):
		threading.Thread.__init__(self)

		self.tbu = tbu
		self.host = host
		self.port = port
		self.sinfo = sinfo
		self.awakener = awakener

	def run(self):
		logging.debug('Updater thread started')

		while True:
			update(self.host, self.port, self.sinfo)

			if self.awakener.wait(self.tbu): # sleep until tbu or someone calls awakener.set()
				self.awakener.clear() # reset the internal flag

#
#
if __name__ == '__main__':

	parser = argparse.ArgumentParser(
				usage=globals()['__doc__'],
				version=__version__,
				description='Remote resource monitoring tool client')

	parser.add_argument('-host', required=True, help='server host name/ip')

	parser.add_argument('-port', required=True, type=int, help='server port')

	parser.add_argument('-timeout', required=False, type=int, help='connection timeout')

	parser.add_argument('-tbu', required=False, type=float,
				help='time between every update sent to the server (default: 0.5)', default=0.5)

	parser.add_argument('-lf', '--logfile', type=argparse.FileType('a'),
				default=sys.stderr,
				help='logging file (default [stderr])')

	parser.add_argument('-d', '--debug', action='store_true', default=False,
				help='sets the loggin leve to DEBUG')

	options = parser.parse_args()

	main()

<commit_msg>Self updating commands done. 3rd client updating begun. 	modified:   src/cli.py<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import common
import gdata
import logging
import argparse
import socket
import struct
import threading
import time
import select
import helper

__program__ = "DREMO Client"
__version__ = '0.4'
__author__ = "Josep Pon Farreny, Marc Piñol Pueyo"
__license__ = "MIT"
__status__ = "Development"

def main():
	global options

	setUpLogger()
	logging.info('Logging set up')
	logging.debug('Host to connect to: %s ' % options.host)
	logging.debug('Port to connect to: %s' % options.port)

	sinfo = common.SysInfo()
	sinfo.update()
	awakener = threading.Event()

	mcsock, client_id = beginConnection(options.host, options.port, sinfo)

	auto_update = AutoUpdater(options.tbu, options.host, options.port, client_id, sinfo, awakener)
	auto_update.start()

	sock = createTCPSocket('0.0.0.0', options.listenport, False)
	in_socks = set([mcsock, sock])

	_acceptForever(in_socks, sock, mcsock, awakener, sinfo, client_id)

#
#
def _acceptForever(in_socks, sock, mcsock, awakener, sinfo, client_id):
	while True:
		i, o , e = select.select(in_socks, [], [])

		for s in i:
			if s is sock:
				ns, addr = s.accept()
				logging.info('New request from %s:%d' % addr)
				_processInstruction(ns, 'tcp', awakener, sinfo, client_id)

			elif s is mcsock:
				logging.info('New multicast request')
				_processInstruction(ns, 'mc', awakener, sinfo, client_id)

#
#
def _processInstruction(sock, connection_type, awakener, sinfo, client_id):
	instruction = common.recvEnd(sock, '\n').strip()
	logging.debug('Received instruction %s by %s' % (instruction, connection_type))
	instruction, sec, params = instruction.partition(' ')

	if connection_type == 'tcp':
		if not params:
			_updateFromSelf(sock, instruction, awakener, sinfo, client_id)
		else:
			_updateFromOther(sock, instruction, params)

	else:
		_updateFromSelf(sock, instruction, awakener, sinfo, client_id, connection_type)

#
#
def _updateFromSelf(sock, instruction, awakener, sinfo, client_id, connection_type=''):
	if helper.isCmdUpdate(instruction):
			sock.send(helper.getOkMessage())
			awakener.set()
			logging.debug('Requested update sent')

	elif not connection_type and helper.isCmdGet(instruction):
		sock.send(helper.getOkMessage())
		sendXML(sock, sinfo, client_id, False)
		logging.debug('Requested get sent')

	else:
		sock.send(getUnknownCmdError('Unknown command'))
		logging.debug('Request error sent')

#
#
def _updateFromOther(sock, instruction, params):
	ip, port = params.split(':')
	port = int(port)
	response = ''

	try:
		sock_to_other = createTCPSocket(ip, port)
		response_head,m,n = sendThroughSocket(sock_to_other, instruction + '\n')

		if response_head == '200':
			response = common.recvEnd(sock_to_other, dgata.ETX)
			response.join(common.recvEnd(sock_to_other, dgata.ETX))
	except:
		response = helper.getMonitorUnreachableError('Connection failed')

	sendThroughSocket(sock, response, wait_for_response = False)

#
#
def createTCPSocket(host, port, connect=True):
	sock = socket.socket()
	sock.settimeout(3) # TODO: REMOVE MAGIC NUMBER
	sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

	if connect:
		sock.connect((host, port))
	else:
		try:
			sock.bind((host, port))
			sock.listen(5)
		except:
			logging.info('Error binding socket')
			sys.exit(1)

	return sock

#
#
def beginConnection(host, port, sinfo):
	hello = '%c %s %c %s %c' %(gdata.BEL, port, gdata.ETX, buildXML(sinfo), gdata.ETX)

	ret_code, detail, response = sendThroughSocket(createTCPSocket(host, port), hello)

	if ret_code == '200':
		ret_id, multicast_group, multicast_port  = response.split()
		multicast_port = int(multicast_port)

		logging.debug('Received client ID: %s' % ret_id)
		logging.debug('Received multicast group: %s' % multicast_group)
		logging.debug('Received multicast port: %d' % multicast_port)

	else:
		if ret_code:
			logging.info('Error: %s' % ret_code)
		else:
			logging.info('Unknown error')
		sys.exit(1)

	return _initMulticast(multicast_group, multicast_port), ret_id

#
#
def _initMulticast(group, port):
	sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
	sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # more than one sock can use this port
	sock.bind(('', port))

	mreq = struct.pack("4sl", socket.inet_aton(group), socket.INADDR_ANY)

	sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq)
	logging.debug('Multicast socket connected to %s:%d' %(group, port))

	return sock

#
#
def sendThroughSocket(sock, to_send, delim='\n\n', wait_for_response=True):
	code = ''
	detail = ''
	msg = ''

	try:
		sock.send(to_send)
		logging.debug('Message sent to server')

		if wait_for_response:
			first, sep, msg = common.recvEnd(sock, delim).partition('\n')
			code, sep, detail = first.partition(' ')
			msg, sep, garbage = msg.partition('\n')

			code.strip()
			detail.strip()
			msg.strip()

			logging.debug('Response from the server (response code): %s %s' % (code, detail))
			logging.debug('Response from the server (response): %s' % msg)

	except:
		code = helper.getMonitorUnreachableError()
		msg = 'Connection timed out'

	return code.strip(), detail.strip(), msg.strip()

#
#
def buildXML(sinfo):
	XMLbuilder = common.sysinfoxml.SysInfoXMLBuilder()

	dao = sinfo.getSysInfoData()
	XMLbuilder.setXMLData(dao)

	return XMLbuilder.getAsString()

#
#
def sendXML(sock, sinfo, client_id, wait_for_response=True):
	xml = buildXML(sinfo)
	msg = '%c %s %c %s %c' % (gdata.SOH, client_id, gdata.ETX, xml, gdata.ETX)

	return sendThroughSocket(sock, msg, wait_for_response=wait_for_response)

#
#
def setUpLogger():
	"""setUpLogger(options) -> void

	Sets up the loggin module.

	"""
	global options

	lvl = logging.DEBUG if options.debug else logging.INFO
	#format = '[%(levelname)s] (%(asctime)s): %(message)s'
	format = '[%(levelname)s]: %(message)s'
	datefmt='%d/%m/%Y %H:%M:%S'

	logging.basicConfig(stream = options.logfile, level=lvl, format=format,
						datefmt=datefmt)

#
#
class AutoUpdater(threading.Thread):
	def __init__(self, tbu, host, port, client_id, sinfo, awakener):
		threading.Thread.__init__(self)

		self.tbu = tbu
		self.host = host
		self.port = port
		self.client_id = client_id
		self.sinfo = sinfo
		self.awakener = awakener

	#
	#
	def run(self):
		logging.debug('Updater thread started')

		while True:
			self.update(createTCPSocket(self.host, self.port), self.sinfo, self.client_id)

			if self.awakener.wait(self.tbu): # sleep until tbu or someone calls awakener.set()
				self.awakener.clear() # reset the internal flag

	#
	#
	def update(self, sock, sinfo, client_id):

		sinfo.update()
		sendXML(sock, sinfo, client_id)

#
#
if __name__ == '__main__':

	parser = argparse.ArgumentParser(
				usage=globals()['__doc__'],
				version=__version__,
				description='Remote resource monitoring tool client')

	parser.add_argument('-host', required=True, help='server host name/ip')

	parser.add_argument('-port', required=True, type=int, help='server port')

	parser.add_argument('-listenport', required=True, type=int, help='port to bind the client and wait for commands')

	parser.add_argument('-timeout', required=False, type=int, help='connection timeout')

	parser.add_argument('-tbu', required=False, type=float,
				help='time between every update sent to the server (default: 0.5)', default=0.5)

	parser.add_argument('-lf', '--logfile', type=argparse.FileType('a'),
				default=sys.stderr,
				help='logging file (default [stderr])')

	parser.add_argument('-d', '--debug', action='store_true', default=False,
				help='sets the loggin leve to DEBUG')

	options = parser.parse_args()

	main()<|endoftext|>


<commit_before>from __future__ import print_function

import click

@click.command()
@click.option('--string', default='World',
        help='This is the thing that is greeted.')
def cli(string):
    """ Click """
    print(string)
<commit_msg>Add subcommand test stuff<commit_after>from __future__ import print_function

import click

class Config:
    def __init__(self):
        self.verbose = False

pass_config = click.make_pass_decorator(Config, ensure=True)

@click.group()
@click.option('--verbose', is_flag=True)
@pass_config
def cli(config, verbose):
    """ Click """
    config.verbose = verbose

@cli.command()
@click.option('--string', default='World',
        help='This is the thing that is greeted.')
@click.option('--repeat', default=1,
        help='How many times to greet the thing.')
@click.argument('out', type=click.File('w'), default='-',
        required=False)
@pass_config
def say(config, string, repeat, out):
    if config.verbose:
        print('We are in verbose mode.')
    for x in xrange(repeat):
        print('Hello, %s!' % string, file=out)
<|endoftext|>


<commit_before># geodriver.py
# Angel Farguell, March 2020

import gdal, osr, pyproj, rasterio
import logging
from utils import Dict
import numpy as np
from geo.write_geogrid import write_geogrid_var
from geo.geo_utils import coord2str

class GeoDriverError(Exception):
    """
    Raised when a GeoDriver failed.
    """
    pass

class GeoDriver(object):
    """
    Represents the content of one GeoDriver file.
    """

    def __init__(self, gdal_ds):
        """
        Initializes metadata information.

        :param gdal_ds: gdal dataset object
        """
        if not isinstance(gdal_ds,gdal.Dataset):
            raise GeoDriverError('GeoDriver: need a gdal.Dataset object instead of %s.\nTry using: GeoDriver.from_file or GeoDriver.from_elements' % type(gdal_ds))
        self.ds = gdal_ds
        # raster size and spacing
        self.nx = self.ds.RasterXSize
        self.ny = self.ds.RasterYSize
        self.bands = self.ds.RasterCount
        # define projection objects
        self.init_proj()

    def close(self):
        """
        Close the current file.
        """
        self.ds = None

    def init_proj(self):
        """
        Get all necessary projection information.
        """
        # get WKT string projection
        self.wkt = self.ds.GetProjectionRef()
        # get Spatial Reference (projection)
        self.crs = osr.SpatialReference()
        self.crs.ImportFromWkt(self.wkt)
        # get Geo Transform
        self.gt = self.ds.GetGeoTransform()
        # get proj4 string
        self.proj4 = self.crs.ExportToProj4()
        # get rasterio object
        self.rasterio = rasterio.crs.CRS.from_proj4(self.proj4)
        # get pyproj element for tif file
        self.pyproj = pyproj.Proj(self.proj4)
        # projection short string
        self.projstr = self.rasterio.to_dict().get('proj','longlat')
        # WPS projections from proj4 attribute +proj
        self.projwrf = {'lcc': 'lambert',
            'stere': 'polar',
            'merc': 'mercator',
            'longlat': 'regular_ll',
            'aea': 'albers_nad83',
            'stere': 'polar_wgs84'
        }.get(self.projstr,None)
        # If not found, resample to lambert projection
        if not self.projwrf:
            self.projwrf = 'lambert'
            self.reproj = True
            # the GeoTIFF needs to be reprojected to lambert projection
            logging.error('GeoTIFF.init_proj() with reprojection is not implemented yet')
        else:
            self.reproj = False

    def get_array(self,bbox=None):
        """
        Get array information.

        :param bbox: optional, WGS84 bounding box (min_lon,max_lon,min_lat,max_lat)
        """
        if isinstance(bbox,(tuple,list,np.ndarray)):
            return self.resample_bbox(bbox)
        else:
            return self.ds.ReadAsArray()

    def get_coord(self):
        """
        Get coordinate arrays for the whole GeoTIFF file using initial projection.
        """
        x0,dx,_,y0,_,dy = self.gt
        xx = np.linspace(x0,x0+dx*self.nx,self.nx)
        yy = np.linspace(y0,y0+dy*self.ny,self.ny)
        return np.meshgrid(xx,yy)

    def resample_bbox(self,bbox):
        """
        Resample using lon-lat bounding box.

        :param bbox: optional, WGS84 bounding box (min_lon,max_lon,min_lat,max_lat)
        """
        logging.info('GeoTIFF.resample_bbox - resampling GeoDriver into bounding box: %s' % list(bbox))
        # geotransform
        x0,dx,_,y0,_,dy = self.gt
        xx = np.arange(x0,x0+dx*self.nx,dx)
        yy = np.arange(y0,y0+dy*self.ny,dy)
        # get pyproj element for WGS84
        ref_proj = pyproj.Proj(proj='longlat',ellps='WGS84',datum='WGS84',no_defs=True)
        # get bounding box in projection of the GeoTIFF
        ref_corners = ((bbox[0],bbox[2]),(bbox[0],bbox[3]),(bbox[1],bbox[3]),(bbox[1],bbox[2]))
        proj_corners = [pyproj.transform(ref_proj,self.pyproj,c[0],c[1]) for c in ref_corners]
        x_min = min([c[0] for c in proj_corners])
        x_max = max([c[0] for c in proj_corners])
        y_min = min([c[1] for c in proj_corners])
        y_max = max([c[1] for c in proj_corners])
        # find resample indexes
        i_mins = np.where(x_min <= xx)[0]
        i_maxs = np.where(xx <= x_max)[0]
        if dx > 0:
            i_min = i_mins.min()
            i_max = i_maxs.max()
        else:
            i_max = i_mins.max()
            i_min = i_maxs.min()
        j_mins = np.where(y_min <= yy)[0]
        j_maxs = np.where(yy <= y_max)[0]
        if dy > 0:
            j_min = j_mins.min()
            j_max = j_maxs.max()
        else:
            j_max = j_mins.max()
            j_min = j_maxs.min()
        # save resample indexes
        self.resample_indxs = (i_min,i_max,j_min,j_max)
        # only working on Linux
        try:
            # get virtual array and resample
            a_r = self.ds.GetVirtualMemArray()[j_min:j_max,i_min:i_max].copy()
        except:
            logging.warning('GeoTIFF.resample_bbox - reading the whole array and sampling after')
            # get array and resample
            a_r = self.ds.ReadAsArray()[j_min:j_max,i_min:i_max]
        # update other elements
        self.gt = (xx[i_min],dx,0,yy[j_min],0,dy)
        self.ny,self.nx = a_r.shape
        return a_r

    def geogrid_index(self):
        """
        Geolocation in a form suitable for geogrid index.

        :return: dictionary key:value
        """
        # spacing
        dx,dy = self.gt[1],self.gt[5]
        # known points in the center of the array (mass-staggered mesh from 1 at origin)
        known_x = (self.nx+1)/2.
        known_y = (self.ny+1)/2.
        # known points in the center of the array (unstaggered mesh from 0 at origin)
        known_x_uns = known_x-.5
        known_y_uns = known_y-.5
        # get pyproj element for reference lat-long coordinates
        ref_proj = self.pyproj.to_latlong()
        # lat/lon known points
        posX, posY = gdal.ApplyGeoTransform(self.gt,known_x_uns,known_y_uns)
        known_lon,known_lat = pyproj.transform(self.pyproj,ref_proj,posX,posY)
        # print ceter lon-lat coordinates to check with gdalinfo
        logging.info('GeoTIFF.geogrid_index - center lon/lat coordinates using pyproj.transform: %s' % coord2str(known_lon,known_lat))
        logging.info('GeoTIFF.geogrid_index - center lon/lat coordinates using gdal.Info: %s' % gdal.Info(self.ds).split('Center')[1].split(') (')[1].split(')')[0])
        # row order depends on dy
        if dy > 0:
            row_order = 'bottom_top'
        else:
            row_order = 'top_bottom'
        # write geogrid index
        return Dict({'projection' : self.projwrf,
            'dx' : dx,
            'dy' : dy,
            'truelat1' : self.crs.GetProjParm("standard_parallel_1"),
            'truelat2' : self.crs.GetProjParm("standard_parallel_2"),
            'stdlon' : self.crs.GetProjParm("longitude_of_center",self.crs.GetProjParm("central_meridian",self.crs.GetProjParm("longitude_of_origin"))),
            'known_x' : known_x,
            'known_y' : known_y,
            'known_lon' : known_lon,
            'known_lat' : known_lat,
            'row_order' : row_order
        })

    def to_geogrid(self,path,var,bbox=None):
        """
        Transform to geogrid files

        :param path: path to write the geogrid files
        :param var: variable name for WPS (available options in src/geo/var_wisdom.py)
        :param bbox: optional, WGS84 bounding box (min_lon,max_lon,min_lat,max_lat)
        """
        logging.info('GeoTIFF.to_geogrid() - getting array')
        array = self.get_array(bbox)
        # WPS is generating regular lat-lon arrays upside down
        if self.projwrf == 'regular_ll':
            array = np.flipud(array)
        logging.info('GeoTIFF.to_geogrid() - creating index')
        index = self.geogrid_index()
        logging.info('GeoTIFF.to_geogrid() - writting geogrid')
        write_geogrid_var(path,var,array,index,coord=self.get_coord())

    def to_geotiff(self,path,**kwargs):
        """
        Transform to geotiff file

        :param path: path to write the geotiff file
        :param bbox: optional, WGS84 bounding box (min_lon,max_lon,min_lat,max_lat)
        """
        bbox = kwargs.get('bbox')
        logging.info('GeoTIFF.to_geotiff() - getting array')
        array = self.get_array(bbox)
        dims = array.shape
        if len(dims) == 2:
            array = array.reshape((1,)+dims)
        logging.info('GeoTIFF.to_geotiff() - writting geotiff')
        ds = gdal.GetDriverByName('GTiff').Create(path,self.nx,self.ny,self.bands,gdal.GDT_Float32)
        ds.SetGeoTransform(self.gt)
        ds.SetProjection(self.crs.ExportToWkt())
        for b in range(self.bands):
            band = ds.GetRasterBand(b+1)
            band.WriteArray(array[b,:,:])
            if 'unit' in kwargs.keys():
                if isinstance(kwargs['unit'],(tuple,list,np.ndarray)):
                    band.SetUnitType(kwargs['unit'][b])
                else:
                    band.SetUnitType(kwargs['unit'])
            if 'desc' in kwargs.keys():
                if isinstance(kwargs['desc'],(tuple,list,np.ndarray)):
                    band.SetDescription(kwargs['desc'][b])
                else:
                    band.SetDescription(kwargs['desc'])
            if 'ndv' in kwargs.keys():
                if isinstance(kwargs['ndv'],(tuple,list,np.ndarray)):
                    band.SetNoDataValue(kwargs['ndv'][b])
                else:
                    band.SetNoDataValue(kwargs['ndv'])
        ds.FlushCache()
        ds = None

    def __str__(self):
        """
        Returns a string representation of the message as provided by gdal.

        :return: descriptive string
        """
        return str(self.ds)

    @classmethod
    def from_file(cls, path):
        """
        Construct a GeoDriver from file.

        :param path: the path to the file
        """
        logging.info('Reading file %s' % path)
        # open GeoTIFF file
        ds = gdal.Open(path)
        if ds:
            gd = cls(ds)
            gd.path = path
        else:
            gd = None
        return gd

    @classmethod
    def from_elements(cls, data, crs, gt):
        """
        Construct a GeoDriver from file.

        :param data: data array
        :oaran crs: spatial reference
        :param gt: geotransform tuple
        """
        dims = data.shape
        if len(dims) == 2:
            data = data.reshape((1,)+dims)
            bands,rows,cols = data.shape
        elif len(dims) == 3:
            bands,rows,cols = dims
        else:
            raise GeoDriverError('GeoDriver.from_elements - data provided must be a 2d or 3d array')
        ds = gdal.GetDriverByName('MEM').Create('',cols,rows,bands,gdal.GDT_Float32)
        ds.SetProjection(crs.ExportToWkt())
        ds.SetGeoTransform(gt)
        for b in range(bands):
            ds.GetRasterBand(b+1).WriteArray(data[b,:,:])
        gd = cls(ds)
        return gd
<commit_msg>including multi-band in GeoDriver.resample_bbox<commit_after># geodriver.py
# Angel Farguell, March 2020

import gdal, osr, pyproj, rasterio
import logging
from utils import Dict
import numpy as np
from geo.write_geogrid import write_geogrid_var
from geo.geo_utils import coord2str

class GeoDriverError(Exception):
    """
    Raised when a GeoDriver failed.
    """
    pass

class GeoDriver(object):
    """
    Represents the content of one GeoDriver file.
    """

    def __init__(self, gdal_ds):
        """
        Initializes metadata information.

        :param gdal_ds: gdal dataset object
        """
        if not isinstance(gdal_ds,gdal.Dataset):
            raise GeoDriverError('GeoDriver: need a gdal.Dataset object instead of %s.\nTry using: GeoDriver.from_file or GeoDriver.from_elements' % type(gdal_ds))
        self.ds = gdal_ds
        # raster size and spacing
        self.nx = self.ds.RasterXSize
        self.ny = self.ds.RasterYSize
        self.bands = self.ds.RasterCount
        # define projection objects
        self.init_proj()

    def close(self):
        """
        Close the current file.
        """
        self.ds = None

    def init_proj(self):
        """
        Get all necessary projection information.
        """
        # get WKT string projection
        self.wkt = self.ds.GetProjectionRef()
        # get Spatial Reference (projection)
        self.crs = osr.SpatialReference()
        self.crs.ImportFromWkt(self.wkt)
        # get Geo Transform
        self.gt = self.ds.GetGeoTransform()
        # get proj4 string
        self.proj4 = self.crs.ExportToProj4()
        # get rasterio object
        self.rasterio = rasterio.crs.CRS.from_proj4(self.proj4)
        # get pyproj element for tif file
        self.pyproj = pyproj.Proj(self.proj4)
        # projection short string
        self.projstr = self.rasterio.to_dict().get('proj','longlat')
        # WPS projections from proj4 attribute +proj
        self.projwrf = {'lcc': 'lambert',
            'stere': 'polar',
            'merc': 'mercator',
            'longlat': 'regular_ll',
            'aea': 'albers_nad83',
            'stere': 'polar_wgs84'
        }.get(self.projstr,None)
        # If not found, resample to lambert projection
        if not self.projwrf:
            self.projwrf = 'lambert'
            self.reproj = True
            # the GeoTIFF needs to be reprojected to lambert projection
            raise GeoDriverError('GeoTIFF.init_proj() with reprojection is not implemented yet')
        else:
            self.reproj = False

    def get_array(self,bbox=None):
        """
        Get array information.

        :param bbox: optional, WGS84 bounding box (min_lon,max_lon,min_lat,max_lat)
        """
        if isinstance(bbox,(tuple,list,np.ndarray)):
            return self.resample_bbox(bbox)
        else:
            return self.ds.ReadAsArray()

    def get_coord(self):
        """
        Get coordinate arrays for the whole GeoTIFF file using initial projection.
        """
        x0,dx,_,y0,_,dy = self.gt
        xx = np.linspace(x0,x0+dx*self.nx,self.nx)
        yy = np.linspace(y0,y0+dy*self.ny,self.ny)
        return np.meshgrid(xx,yy)

    def resample_bbox(self,bbox):
        """
        Resample using lon-lat bounding box.

        :param bbox: optional, WGS84 bounding box (min_lon,max_lon,min_lat,max_lat)
        """
        logging.info('GeoTIFF.resample_bbox - resampling GeoDriver into bounding box: %s' % list(bbox))
        # geotransform
        x0,dx,_,y0,_,dy = self.gt
        xx = np.arange(x0,x0+dx*self.nx,dx)
        yy = np.arange(y0,y0+dy*self.ny,dy)
        # get pyproj element for WGS84
        ref_proj = pyproj.Proj(proj='longlat',ellps='WGS84',datum='WGS84',no_defs=True)
        # get bounding box in projection of the GeoTIFF
        ref_corners = ((bbox[0],bbox[2]),(bbox[0],bbox[3]),(bbox[1],bbox[3]),(bbox[1],bbox[2]))
        proj_corners = [pyproj.transform(ref_proj,self.pyproj,c[0],c[1]) for c in ref_corners]
        x_min = min([c[0] for c in proj_corners])
        x_max = max([c[0] for c in proj_corners])
        y_min = min([c[1] for c in proj_corners])
        y_max = max([c[1] for c in proj_corners])
        # find resample indexes
        i_mins = np.where(x_min <= xx)[0]
        i_maxs = np.where(xx <= x_max)[0]
        if dx > 0:
            i_min = i_mins.min()
            i_max = i_maxs.max()
        else:
            i_max = i_mins.max()
            i_min = i_maxs.min()
        j_mins = np.where(y_min <= yy)[0]
        j_maxs = np.where(yy <= y_max)[0]
        if dy > 0:
            j_min = j_mins.min()
            j_max = j_maxs.max()
        else:
            j_max = j_mins.max()
            j_min = j_maxs.min()
        # save resample indexes
        self.resample_indxs = (i_min,i_max,j_min,j_max)
        # only working on Linux
        try:
            # get virtual array and resample
            if self.bands == 1:
                a_r = self.ds.GetVirtualMemArray()[j_min:j_max,i_min:i_max].copy()
            else:
                a_r = self.ds.GetVirtualMemArray()[:,j_min:j_max,i_min:i_max].copy()
        except:
            logging.warning('GeoTIFF.resample_bbox - reading the whole array and sampling after')
            # get array and resample
            if self.bands == 1:
                a_r = self.ds.ReadAsArray()[j_min:j_max,i_min:i_max]
            else:
                a_r = self.ds.ReadAsArray()[:,j_min:j_max,i_min:i_max]
        # update other elements
        self.gt = (xx[i_min],dx,0,yy[j_min],0,dy)
        if self.bands == 1:
            self.ny,self.nx = a_r.shape
        else:
            _,self.ny,self.nx = a_r.shape
        return a_r

    def geogrid_index(self):
        """
        Geolocation in a form suitable for geogrid index.

        :return: dictionary key:value
        """
        # spacing
        dx,dy = self.gt[1],self.gt[5]
        # known points in the center of the array (mass-staggered mesh from 1 at origin)
        known_x = (self.nx+1)/2.
        known_y = (self.ny+1)/2.
        # known points in the center of the array (unstaggered mesh from 0 at origin)
        known_x_uns = known_x-.5
        known_y_uns = known_y-.5
        # get pyproj element for reference lat-long coordinates
        ref_proj = self.pyproj.to_latlong()
        # lat/lon known points
        posX, posY = gdal.ApplyGeoTransform(self.gt,known_x_uns,known_y_uns)
        known_lon,known_lat = pyproj.transform(self.pyproj,ref_proj,posX,posY)
        # print ceter lon-lat coordinates to check with gdalinfo
        logging.info('GeoTIFF.geogrid_index - center lon/lat coordinates using pyproj.transform: %s' % coord2str(known_lon,known_lat))
        logging.info('GeoTIFF.geogrid_index - center lon/lat coordinates using gdal.Info: %s' % gdal.Info(self.ds).split('Center')[1].split(') (')[1].split(')')[0])
        # row order depends on dy
        if dy > 0:
            row_order = 'bottom_top'
        else:
            row_order = 'top_bottom'
        # write geogrid index
        return Dict({'projection' : self.projwrf,
            'dx' : dx,
            'dy' : dy,
            'truelat1' : self.crs.GetProjParm("standard_parallel_1"),
            'truelat2' : self.crs.GetProjParm("standard_parallel_2"),
            'stdlon' : self.crs.GetProjParm("longitude_of_center",self.crs.GetProjParm("central_meridian",self.crs.GetProjParm("longitude_of_origin"))),
            'known_x' : known_x,
            'known_y' : known_y,
            'known_lon' : known_lon,
            'known_lat' : known_lat,
            'row_order' : row_order
        })

    def to_geogrid(self,path,var,bbox=None):
        """
        Transform to geogrid files

        :param path: path to write the geogrid files
        :param var: variable name for WPS (available options in src/geo/var_wisdom.py)
        :param bbox: optional, WGS84 bounding box (min_lon,max_lon,min_lat,max_lat)
        """
        logging.info('GeoTIFF.to_geogrid() - getting array')
        array = self.get_array(bbox)
        if len(array.shape) > 2:
            raise GeoDriverError('GeoTIFF.to_geogrid() - generation of 3d geogrid arrays is not supported')
        # WPS is generating regular lat-lon arrays upside down
        if self.projwrf == 'regular_ll':
            array = np.flipud(array)
        logging.info('GeoTIFF.to_geogrid() - creating index')
        index = self.geogrid_index()
        logging.info('GeoTIFF.to_geogrid() - writting geogrid')
        write_geogrid_var(path,var,array,index,coord=self.get_coord())

    def to_geotiff(self,path,**kwargs):
        """
        Transform to geotiff file

        :param path: path to write the geotiff file
        :param bbox: optional, WGS84 bounding box (min_lon,max_lon,min_lat,max_lat)
        """
        bbox = kwargs.get('bbox')
        logging.info('GeoTIFF.to_geotiff() - getting array')
        array = self.get_array(bbox)
        dims = array.shape
        if len(dims) == 2:
            array = array.reshape((1,)+dims)
        logging.info('GeoTIFF.to_geotiff() - writting geotiff')
        ds = gdal.GetDriverByName('GTiff').Create(path,self.nx,self.ny,self.bands,gdal.GDT_Float32)
        ds.SetGeoTransform(self.gt)
        ds.SetProjection(self.crs.ExportToWkt())
        for b in range(self.bands):
            band = ds.GetRasterBand(b+1)
            band.WriteArray(array[b,:,:])
            if 'unit' in kwargs.keys():
                if isinstance(kwargs['unit'],(tuple,list,np.ndarray)):
                    band.SetUnitType(kwargs['unit'][b])
                else:
                    band.SetUnitType(kwargs['unit'])
            if 'desc' in kwargs.keys():
                if isinstance(kwargs['desc'],(tuple,list,np.ndarray)):
                    band.SetDescription(kwargs['desc'][b])
                else:
                    band.SetDescription(kwargs['desc'])
            if 'ndv' in kwargs.keys():
                if isinstance(kwargs['ndv'],(tuple,list,np.ndarray)):
                    band.SetNoDataValue(kwargs['ndv'][b])
                else:
                    band.SetNoDataValue(kwargs['ndv'])
        ds.FlushCache()
        ds = None

    def __str__(self):
        """
        Returns a string representation of the message as provided by gdal.

        :return: descriptive string
        """
        return str(self.ds)

    @classmethod
    def from_file(cls, path):
        """
        Construct a GeoDriver from file.

        :param path: the path to the file
        """
        logging.info('Reading file %s' % path)
        # open GeoTIFF file
        ds = gdal.Open(path)
        if ds:
            gd = cls(ds)
            gd.path = path
        else:
            gd = None
        return gd

    @classmethod
    def from_elements(cls, data, crs, gt):
        """
        Construct a GeoDriver from file.

        :param data: data array
        :oaran crs: spatial reference
        :param gt: geotransform tuple
        """
        dims = data.shape
        if len(dims) == 2:
            data = data.reshape((1,)+dims)
            bands,rows,cols = data.shape
        elif len(dims) == 3:
            bands,rows,cols = dims
        else:
            raise GeoDriverError('GeoDriver.from_elements - data provided must be a 2d or 3d array')
        ds = gdal.GetDriverByName('MEM').Create('',cols,rows,bands,gdal.GDT_Float32)
        ds.SetProjection(crs.ExportToWkt())
        ds.SetGeoTransform(gt)
        for b in range(bands):
            ds.GetRasterBand(b+1).WriteArray(data[b,:,:])
        gd = cls(ds)
        return gd
<|endoftext|>


<commit_before># -*- coding: utf-8
from __future__ import absolute_import

from django.apps import AppConfig

class XlsDataSyncConfig(AppConfig):
    name = 'excel_data_sync'

    def ready(self):
        try:
            from concurrency.fields import VersionField  # noqa
            import excel_data_sync.extras.concurrency  # noqa
        except ImportError:  # pragma: no-cover
            pass
<commit_msg>change no-cover to 'no cover'. Try to fix codecov<commit_after># -*- coding: utf-8
from __future__ import absolute_import

from django.apps import AppConfig

class XlsDataSyncConfig(AppConfig):
    name = 'excel_data_sync'

    def ready(self):
        try:
            from concurrency.fields import VersionField  # noqa
            import excel_data_sync.extras.concurrency  # noqa
        except ImportError:  # pragma: no cover
            pass
<|endoftext|>


<commit_before>import numpy as np

class TwoBodyEnvironment:

    def __init__(self, positions, species, cell, cutoff, atom):
        # set permanent features if needed
        if not hasattr(self, 'bond_list'):
            TwoBodyEnvironment.bond_list = self.species_to_bonds(species)
            TwoBodyEnvironment.brav_inv = np.linalg.inv(cell)
            TwoBodyEnvironment.vec1 = cell[:, 0]
            TwoBodyEnvironment.vec2 = cell[:, 1]
            TwoBodyEnvironment.vec3 = cell[:, 2]
            TwoBodyEnvironment.cutoff = cutoff

        self.positions = positions
        self.species = species

        bond_lengths, bond_pos, bonds, etyps, ctyp =\
            self.get_atoms_within_cutoff(atom)
        self.bond_lengths = bond_lengths
        self.bond_pos = bond_pos
        self.bonds = bonds
        self.etyps = etyps
        self.ctyp = ctyp

    @staticmethod
    def species_to_bonds(species_list):
        """Converts list of species to a list of bonds.

        :param species_list: all species in the simulation
        :type species_list: list of strings
        :return: all possible bonds
        :rtype: list of lists of two strings
        """

        # get unique species
        unique_species = []
        for species in species_list:
            if species not in unique_species:
                unique_species.append(species)

        # create bond list
        bond_list = []
        for m in range(len(unique_species)):
            species_1 = unique_species[m]

            for n in range(m, len(unique_species)):
                species_2 = unique_species[n]
                bond_list.append([species_1, species_2])

        return bond_list

    @staticmethod
    def is_bond(species1, species2, bond):
        """Check if two species form a specified bond.

        :param species1: first species
        :type species1: str
        :param species2: second species
        :type species2: str
        :param bond: bond to be checked
        :type bond: list<str>
        :return: True or False
        :rtype: bool
        """

        return ([species1, species2] == bond) or ([species2, species1] == bond)

    def species_to_index(self, species1, species2):
        """Given two species, get the corresponding bond index.

        :param species1: first species
        :type species1: string
        :param species2: second species
        :type species2: string
        :param bond_list: all possible bonds
        :type bond_list: list
        :return: bond index
        :rtype: integer
        """

        for bond_index, bond in enumerate(self.bond_list):
            if TwoBodyEnvironment.is_bond(species1, species2, bond):
                return bond_index

    def get_local_atom_images(self, vec):
        """Get periodic images of an atom within the cutoff radius.

        :param vec: atomic position
        :type vec: nparray of shape (3,)
        :return: vectors and distances of atoms within cutoff radius
        :rtype: list of nparrays, list of floats
        """

        # get bravais coefficients
        coeff = np.matmul(self.brav_inv, vec)

        # get bravais coefficients for atoms within one super-super-cell
        coeffs = [[], [], []]
        for n in range(3):
            coeffs[n].append(coeff[n])
            coeffs[n].append(coeff[n]-1)
            coeffs[n].append(coeff[n]+1)
            coeffs[n].append(coeff[n]-2)
            coeffs[n].append(coeff[n]+2)

        # get vectors within cutoff
        vecs = []
        dists = []
        for m in range(len(coeffs[0])):
            for n in range(len(coeffs[1])):
                for p in range(len(coeffs[2])):
                    vec_curr = coeffs[0][m]*self.vec1 +\
                               coeffs[1][n]*self.vec2 +\
                               coeffs[2][p]*self.vec3
                    dist = np.linalg.norm(vec_curr)

                    if dist < self.cutoff:
                        vecs.append(vec_curr)
                        dists.append(dist)

        return vecs, dists

    # return information about atoms inside cutoff region
    def get_atoms_within_cutoff(self, atom):

        pos_atom = self.positions[atom]  # position of central atom
        central_type = self.species[atom]  # type of central atom

        positions = []
        distances = []
        environment_types = []
        bonds = []

        # find all atoms and images in the neighborhood
        for n in range(len(self.positions)):
            diff_curr = self.positions[n] - pos_atom
            typ_curr = self.species[n]
            bond_curr = self.species_to_index(central_type, typ_curr)

            # get images within cutoff
            vecs, dists = self.get_local_atom_images(diff_curr)

            for vec, dist in zip(vecs, dists):
                # ignore self interaction
                if dist != 0:
                    positions.append(vec)
                    distances.append(dist)
                    environment_types.append(typ_curr)
                    bonds.append(bond_curr)

        return distances, positions, bonds, environment_types, central_type

# testing ground (will be moved to test suite later)
if __name__ == '__main__':
    # create simple test environment
    positions = [np.array([0, 0, 0]), np.array([0.5, 0.5, 0.5])]
    species = ['B', 'A']
    cell = np.eye(3)
    cutoff = np.linalg.norm(np.array([0.5, 0.5, 0.5])) + 0.001
    atom = 0

    test_env = TwoBodyEnvironment(positions, species, cell, cutoff, atom)

    # test species_to_bond
    assert(test_env.bond_list == [['B', 'B'], ['B', 'A'], ['A', 'A']])

    # test is_bond (static method)
    assert(TwoBodyEnvironment.is_bond('A', 'B', ['A', 'B']))
    assert(TwoBodyEnvironment.is_bond('B', 'A', ['A', 'B']))
    assert(not TwoBodyEnvironment.is_bond('C', 'A', ['A', 'B']))

    # test species_to_index
    assert(test_env.species_to_index('B', 'B') == 0)
    assert(test_env.species_to_index('B', 'A') == 1)
    assert(test_env.species_to_index('A', 'A') == 2)

    # test get_local_atom_images
    vec = np.array([0.5, 0.5, 0.5])
    vecs, dists = test_env.get_local_atom_images(vec)
    assert(len(dists) == 8)
    assert(len(vecs) == 8)

    # test get_atoms_within_cutoff
    atom = 0
    distances, positions, bonds, environment_types, central_type = \
        test_env.get_atoms_within_cutoff(atom)

    assert(len(distances) == 8)
<commit_msg>add njit two body kernel<commit_after>"""
Jon V
"""

import numpy as np
from math import exp
from numba import njit

# get two body kernel between chemical environments
@njit
def two_body_jit(bond_array_1, bond_types_1, bond_array_2,
                 bond_types_2, d1, d2, sig, ls):
    d = sig*sig/(ls*ls*ls*ls)
    e = ls*ls
    f = 1/(2*ls*ls)
    kern = 0

    x1_len = len(bond_types_1)
    x2_len = len(bond_types_2)

    for m in range(x1_len):
        r1 = bond_array_1[m, 0]
        coord1 = bond_array_1[m, d1]
        typ1 = bond_types_1[m]

        for n in range(x2_len):
            r2 = bond_array_2[n, 0]
            coord2 = bond_array_2[n, d2]
            typ2 = bond_types_2[n]

            # check that bonds match
            if typ1 == typ2:
                rr = (r1-r2)*(r1-r2)
                kern += d*exp(-f*rr)*coord1*coord2*(e-rr)

    return kern

class TwoBodyEnvironment:

    def __init__(self, positions, species, cell, cutoff, atom):
        # set permanent features if needed
        if not hasattr(self, 'bond_list'):
            TwoBodyEnvironment.bond_list = self.species_to_bonds(species)
            TwoBodyEnvironment.brav_inv = np.linalg.inv(cell)
            TwoBodyEnvironment.vec1 = cell[:, 0]
            TwoBodyEnvironment.vec2 = cell[:, 1]
            TwoBodyEnvironment.vec3 = cell[:, 2]
            TwoBodyEnvironment.cutoff = cutoff

        self.positions = positions
        self.species = species

        bond_array, bond_types, etyps, ctyp =\
            self.get_atoms_within_cutoff(atom)

        self.bond_array = bond_array
        self.bond_types = bond_types
        self.etyps = etyps
        self.ctyp = ctyp

    @staticmethod
    def species_to_bonds(species_list):
        """Converts list of species to a list of bonds.

        :param species_list: all species in the simulation
        :type species_list: list of strings
        :return: all possible bonds
        :rtype: list of lists of two strings
        """

        # get unique species
        unique_species = []
        for species in species_list:
            if species not in unique_species:
                unique_species.append(species)

        # create bond list
        bond_list = []
        for m in range(len(unique_species)):
            species_1 = unique_species[m]

            for n in range(m, len(unique_species)):
                species_2 = unique_species[n]
                bond_list.append([species_1, species_2])

        return bond_list

    @staticmethod
    def is_bond(species1, species2, bond):
        """Check if two species form a specified bond.

        :param species1: first species
        :type species1: str
        :param species2: second species
        :type species2: str
        :param bond: bond to be checked
        :type bond: list<str>
        :return: True or False
        :rtype: bool
        """

        return ([species1, species2] == bond) or ([species2, species1] == bond)

    def species_to_index(self, species1, species2):
        """Given two species, get the corresponding bond index.

        :param species1: first species
        :type species1: string
        :param species2: second species
        :type species2: string
        :param bond_list: all possible bonds
        :type bond_list: list
        :return: bond index
        :rtype: integer
        """

        for bond_index, bond in enumerate(self.bond_list):
            if TwoBodyEnvironment.is_bond(species1, species2, bond):
                return bond_index

    def get_local_atom_images(self, vec):
        """Get periodic images of an atom within the cutoff radius.

        :param vec: atomic position
        :type vec: nparray of shape (3,)
        :return: vectors and distances of atoms within cutoff radius
        :rtype: list of nparrays, list of floats
        """

        # get bravais coefficients
        coeff = np.matmul(self.brav_inv, vec)

        # get bravais coefficients for atoms within one super-super-cell
        coeffs = [[], [], []]
        for n in range(3):
            coeffs[n].append(coeff[n])
            coeffs[n].append(coeff[n]-1)
            coeffs[n].append(coeff[n]+1)
            coeffs[n].append(coeff[n]-2)
            coeffs[n].append(coeff[n]+2)

        # get vectors within cutoff
        vecs = []
        dists = []
        for m in range(len(coeffs[0])):
            for n in range(len(coeffs[1])):
                for p in range(len(coeffs[2])):
                    vec_curr = coeffs[0][m]*self.vec1 +\
                               coeffs[1][n]*self.vec2 +\
                               coeffs[2][p]*self.vec3
                    dist = np.linalg.norm(vec_curr)

                    if dist < self.cutoff:
                        vecs.append(vec_curr)
                        dists.append(dist)

        return vecs, dists

    # return information about atoms inside cutoff region
    def get_atoms_within_cutoff(self, atom):

        pos_atom = self.positions[atom]  # position of central atom
        central_type = self.species[atom]  # type of central atom

        bond_array = []
        bond_types = []
        environment_types = []

        # find all atoms and images in the neighborhood
        for n in range(len(self.positions)):
            diff_curr = self.positions[n] - pos_atom
            typ_curr = self.species[n]
            bond_curr = self.species_to_index(central_type, typ_curr)

            # get images within cutoff
            vecs, dists = self.get_local_atom_images(diff_curr)

            for vec, dist in zip(vecs, dists):
                # ignore self interaction
                if dist != 0:
                    environment_types.append(typ_curr)
                    bond_array.append([dist, vec[0]/dist, vec[1]/dist,
                                       vec[2]/dist])
                    bond_types.append(bond_curr)

        bond_array = np.array(bond_array)
        bond_types = np.array(bond_types)
        return bond_array, bond_types, environment_types, central_type

# testing ground (will be moved to test suite later)
if __name__ == '__main__':
    # create simple test environment
    positions = [np.array([0, 0, 0]), np.array([0.5, 0.5, 0.5])]
    species = ['B', 'A']
    cell = np.eye(3)
    cutoff = np.linalg.norm(np.array([0.5, 0.5, 0.5])) + 0.001
    atom = 0

    test_env = TwoBodyEnvironment(positions, species, cell, cutoff, atom)

    # test species_to_bond
    assert(test_env.bond_list == [['B', 'B'], ['B', 'A'], ['A', 'A']])

    # test is_bond (static method)
    assert(TwoBodyEnvironment.is_bond('A', 'B', ['A', 'B']))
    assert(TwoBodyEnvironment.is_bond('B', 'A', ['A', 'B']))
    assert(not TwoBodyEnvironment.is_bond('C', 'A', ['A', 'B']))

    # test species_to_index
    assert(test_env.species_to_index('B', 'B') == 0)
    assert(test_env.species_to_index('B', 'A') == 1)
    assert(test_env.species_to_index('A', 'A') == 2)

    # test get_local_atom_images
    vec = np.array([0.5, 0.5, 0.5])
    vecs, dists = test_env.get_local_atom_images(vec)
    assert(len(dists) == 8)
    assert(len(vecs) == 8)

    # test get_atoms_within_cutoff
    atom = 0
    bond_array, bonds, environment_types, central_type =\
        test_env.get_atoms_within_cutoff(atom)

    assert(bond_array.shape[0] == 8)
<|endoftext|>


<commit_before>import os
import subprocess
import sys
from itertools import cycle

from lxml import etree

sumo_root = '/usr/share/sumo'
try:
    sumo_home = os.path.join(sumo_root, 'tools')
    sys.path.append(sumo_home)  # tutorial in docs
    print(sys.path)
    from sumolib import checkBinary
except ImportError:
    sys.exit(
        "please declare environment variable 'SUMO_HOME' as the root directory of your sumo installation (it should contain folders 'bin', 'tools' and 'docs')")

class SUMOENV:
    def __init__(self, data_dir, netname, xnumber, ynumber, tlstype='actuated'):
        if xnumber < 2 or ynumber < 2:
            raise ValueError("The number of nodes must be at least 2 in both directions.")
        self.netname = netname
        self.data_dir = data_dir
        self.net_dir = os.path.join(self.data_dir, self.netname)
        if not os.path.isdir(self.net_dir):
            os.makedirs(self.net_dir)
        self.output_dir = os.path.join(self.net_dir, 'output')
        if not os.path.isdir(self.output_dir):
            os.makedirs(self.output_dir)
        self.xnumber = str(xnumber)
        self.ynumber = str(ynumber)
        self.tlstype = tlstype
        self.netfile = os.path.join(self.net_dir, self.netname + '.net.xml')
        self.tripfile = os.path.join(self.net_dir, self.netname + '.trip.xml')
        self.roufile = os.path.join(self.net_dir, self.netname + '.rou.xml')
        self.detectors = [os.path.join(self.net_dir, self.netname + '_e%d.add.xml' % (i + 1))
                          for i in range(3)]
        self.sumocfg = os.path.join(self.data_dir, self.netname, self.netname + '.sumo.cfg')
        self.sumocfg_nodet = os.path.join(self.data_dir, self.netname, self.netname + '_nodet.sumo.cfg')

    def create(self):
        self.gen_network(self.xnumber, self.ynumber, '400', '400', tlstype=self.tlstype)  # Set length to 400
        self.gen_randomtrips('10')  # Set edge prop to 10
        self.gen_detectors()
        self.sumocfg = self.gen_sumocfg()
        self.sumocfg_nodet = self.gen_sumocfg(withdetector=False)

    def iscreated(self):
        return os.path.isfile(self.sumocfg)

    def run(self, port, gui=False, withdet=True, host='localhost'):
        if gui:
            sumoBinary = checkBinary('sumo-gui')
        else:
            sumoBinary = checkBinary('sumo')
        if withdet:
            sumocfg = self.sumocfg
        else:
            sumocfg = self.sumocfg_nodet

        sumoProcess = subprocess.Popen([sumoBinary, '-c', sumocfg, '--remote-port', str(port)],
                                       stdout=sys.stdout, stderr=sys.stderr)
        # sumoProcess.wait()
        return sumoProcess

    def gen_network(self, xnumber, ynumber, xlength, ylength,
                    nettype='grid', tlstype='actuated'):
        netgenerate = checkBinary('netgenerate')
        netgenProcessor = subprocess.Popen([netgenerate, '--%s' % nettype,
                                            '--grid.x-number', xnumber, '--grid.y-number', ynumber,
                                            '--grid.x-length', xlength, '--grid.y-length', ylength,
                                            '--tls.guess', 'true', '--tls.default-type', tlstype,
                                            '--default.lanenumber', '2',
                                            '--%s.attach-length' % nettype, xlength,
                                            '-o', self.netfile], stdout=sys.stdout, stderr=sys.stderr)

    def gen_randomtrips(self, edgeprob, endtime='3600', seed='42',
                        trip_attrib="departLane=\"best\" departSpeed=\"max\" departPos=\"random\""):
        """
        Warning: Sometimes routefile might not be created
        :param edgeprob:
        :param endtime:
        :param seed:
        :param trip_attrib:
        :return:
        """
        rantrip_generator = os.path.join(sumo_home, 'randomTrips.py')
        gentripProcessor = subprocess.Popen([rantrip_generator, '-n', self.netfile,
                                             '-e', endtime, '-s', seed,
                                             '--fringe-factor', edgeprob, '--trip-attributes', trip_attrib,
                                             '-o', self.tripfile, '-r', self.roufile], stdout=sys.stdout,
                                            stderr=sys.stderr)

    def gen_detectors(self):
        e1generator = os.path.join(sumo_home, 'output', 'generateTLSE1Detectors.py')
        e2generator = os.path.join(sumo_home, 'output', 'generateTLSE2Detectors.py')
        e3generator = os.path.join(sumo_home, 'output', 'generateTLSE3Detectors.py')
        det_outputs = ['output/' + 'e_%doutput.xml' % (i + 1) for i in range(3)]
        e_generators = [e1generator, e2generator, e3generator]
        paras = zip(e_generators, cycle(['-n']), cycle([self.netfile]),
                    cycle(['-o']), self.detectors, cycle(['-r']), det_outputs)
        # detectors = list()
        for p in paras:
            p = list(p)
            d = subprocess.Popen(p, stdout=sys.stdout, stderr=sys.stderr)
            # detectors.append(d)
            # detectors = map(subprocess.Popen, paras)

    def gen_sumocfg(self, withdetector=True):
        conf_root = etree.Element("configuration", nsmap={'xsi': "http://www.w3.org/2001/XMLSchema-instance"})
        # conf_root.set('xsi:noNamespaceSchemaLocation', "http://sumo.dlr.de/xsd/sumoConfiguration.xsd")
        # Set Input file
        conf_input = etree.SubElement(conf_root, 'input')
        netfile = self.netname + '.net.xml'  # Use relative address
        roufile = self.netname + '.rou.xml'  # makes configuaration portable
        detectors = [self.netname + '_e%d.add.xml' % (i + 1) for i in range(3)]
        input_netfile = etree.SubElement(conf_input, 'net-file')
        input_netfile.set('value', netfile)
        input_roufile = etree.SubElement(conf_input, 'route-files')
        input_roufile.set('value', roufile)
        if withdetector:
            input_addfile = etree.SubElement(conf_input, 'additional-files')
            input_addfile.set('value', " ".join(detectors))
        # Set Output file
        conf_output = etree.SubElement(conf_root, 'output')
        netstatfile = 'output/' + self.netname + '_netstate.sumo.tr'
        tripinfo = 'output/' + self.netname + '_tripinfo.xml'
        vehroute = 'output/' + self.netname + '_vehroutes.xml'
        output_nets = etree.SubElement(conf_output, 'netstate-dump')
        output_nets.set('value', netstatfile)
        output_tripinfo = etree.SubElement(conf_output, 'tripinfo-output')
        output_tripinfo.set('value', tripinfo)
        output_vehroute = etree.SubElement(conf_output, 'vehroute-output')
        output_vehroute.set('value', vehroute)
        # Set Time
        conf_time = etree.SubElement(conf_root, 'time')
        time_begin = etree.SubElement(conf_time, 'begin')
        time_begin.set('value', '0')
        time_end = etree.SubElement(conf_time, 'end')
        time_end.set('value', '3600')
        time_roustep = etree.SubElement(conf_time, 'route-steps')
        time_roustep.set('value', '-1')
        # Set Time to teleport
        conf_teleport = etree.SubElement(conf_root, 'time-to-teleport')
        conf_teleport.set('value', '-1')

        # Write to sumo.cfg
        conf_tree = etree.ElementTree(conf_root)
        if withdetector:
            sumocfg_file = self.sumocfg
        else:
            sumocfg_file = self.sumocfg_nodet
        conf_tree.write(sumocfg_file, pretty_print=True, xml_declaration=True, encoding='utf-8')
        return sumocfg_file
<commit_msg>XML generator complete<commit_after>import os
import subprocess
import sys
from itertools import cycle

from lxml import etree

sumo_root = '/usr/share/sumo'
try:
    sumo_home = os.path.join(sumo_root, 'tools')
    sys.path.append(sumo_home)  # tutorial in docs
    print(sys.path)
    from sumolib import checkBinary
except ImportError:
    sys.exit(
        "please declare environment variable 'SUMO_HOME' as the root directory of your sumo installation (it should contain folders 'bin', 'tools' and 'docs')")

class SUMOENV:
    def __init__(self, data_dir, netname, xnumber, ynumber, tlstype='actuated'):
        if xnumber < 2 or ynumber < 2:
            raise ValueError("The number of nodes must be at least 2 in both directions.")
        self.netname = netname
        self.data_dir = data_dir
        self.net_dir = os.path.join(self.data_dir, self.netname)
        if not os.path.isdir(self.net_dir):
            os.makedirs(self.net_dir)
        self.output_dir = os.path.join(self.net_dir, 'output')
        if not os.path.isdir(self.output_dir):
            os.makedirs(self.output_dir)
        self.xnumber = str(xnumber)
        self.ynumber = str(ynumber)
        self.tlstype = tlstype
        self.netfile = os.path.join(self.net_dir, self.netname + '.net.xml')
        self.tripfile = os.path.join(self.net_dir, self.netname + '.trip.xml')
        self.roufile = os.path.join(self.net_dir, self.netname + '.rou.xml')
        self.detectors = [os.path.join(self.net_dir, self.netname + '_e%d.add.xml' % (i + 1))
                          for i in range(3)]
        self.sumocfg = os.path.join(self.data_dir, self.netname, self.netname + '.sumo.cfg')
        self.sumocfg_nodet = os.path.join(self.data_dir, self.netname, self.netname + '_nodet.sumo.cfg')

    def create(self):
        self.gen_network(self.xnumber, self.ynumber, '400', '400', tlstype=self.tlstype)  # Set length to 400
        self.gen_randomtrips('10')  # Set edge prop to 10
        self.gen_detectors()
        self.sumocfg = self.gen_sumocfg()
        self.sumocfg_nodet = self.gen_sumocfg(withdetector=False)

    def iscreated(self):
        return os.path.isfile(self.sumocfg)

    def run(self, port, gui=False, withdet=True, host='localhost'):
        if gui:
            sumoBinary = checkBinary('sumo-gui')
        else:
            sumoBinary = checkBinary('sumo')
        if withdet:
            sumocfg = self.sumocfg
        else:
            sumocfg = self.sumocfg_nodet

        sumoProcess = subprocess.Popen([sumoBinary, '-c', sumocfg, '--remote-port', str(port)],
                                       stdout=sys.stdout, stderr=sys.stderr)
        # sumoProcess.wait()
        return sumoProcess

    def gen_network(self, xnumber, ynumber, xlength, ylength,
                    nettype='grid', tlstype='actuated'):
        netgenerate = checkBinary('netgenerate')
        netgenProcessor = subprocess.Popen([netgenerate, '--%s' % nettype,
                                            '--grid.x-number', xnumber, '--grid.y-number', ynumber,
                                            '--grid.x-length', xlength, '--grid.y-length', ylength,
                                            '--tls.guess', 'true', '--tls.default-type', tlstype,
                                            '--default.lanenumber', '2',
                                            '--%s.attach-length' % nettype, xlength,
                                            '-o', self.netfile], stdout=sys.stdout, stderr=sys.stderr)

    def gen_randomtrips(self, edgeprob, endtime='3600', seed='42',
                        trip_attrib="departLane=\"best\" departSpeed=\"max\" departPos=\"random\""):
        """
        Warning: Sometimes routefile might not be created
        :param edgeprob:
        :param endtime:
        :param seed:
        :param trip_attrib:
        :return:
        """
        rantrip_generator = os.path.join(sumo_home, 'randomTrips.py')
        gentripProcessor = subprocess.Popen([rantrip_generator, '-n', self.netfile,
                                             '-e', endtime, '-s', seed,
                                             '--fringe-factor', edgeprob, '--trip-attributes', trip_attrib,
                                             '-o', self.tripfile, '-r', self.roufile], stdout=sys.stdout,
                                            stderr=sys.stderr)

    def gen_detectors(self):
        e1generator = os.path.join(sumo_home, 'output', 'generateTLSE1Detectors.py')
        e2generator = os.path.join(sumo_home, 'output', 'generateTLSE2Detectors.py')
        e3generator = os.path.join(sumo_home, 'output', 'generateTLSE3Detectors.py')
        det_outputs = ['output/' + 'e%d_output.xml' % (i + 1) for i in range(3)]
        e_generators = [e1generator, e2generator, e3generator]
        paras = zip(e_generators, cycle(['-n']), cycle([self.netfile]),
                    cycle(['-o']), self.detectors, cycle(['-r']), det_outputs)
        # detectors = list()
        for p in paras:
            p = list(p)
            d = subprocess.Popen(p, stdout=sys.stdout, stderr=sys.stderr)
            # detectors.append(d)
            # detectors = map(subprocess.Popen, paras)

    def gen_sumocfg(self, withdetector=True):
        conf_root = etree.Element("configuration", nsmap={'xsi': "http://www.w3.org/2001/XMLSchema-instance"})
        # conf_root.set('xsi:noNamespaceSchemaLocation', "http://sumo.dlr.de/xsd/sumoConfiguration.xsd")
        # Set Input file
        conf_input = etree.SubElement(conf_root, 'input')
        netfile = self.netname + '.net.xml'  # Use relative address
        roufile = self.netname + '.rou.xml'  # makes configuaration portable
        detectors = [self.netname + '_e%d.add.xml' % (i + 1) for i in range(3)]
        input_netfile = etree.SubElement(conf_input, 'net-file')
        input_netfile.set('value', netfile)
        input_roufile = etree.SubElement(conf_input, 'route-files')
        input_roufile.set('value', roufile)
        if withdetector:
            input_addfile = etree.SubElement(conf_input, 'additional-files')
            input_addfile.set('value', " ".join(detectors))
        # Set Output file
        conf_output = etree.SubElement(conf_root, 'output')
        netstatfile = 'output/' + self.netname + '_netstate.sumo.tr'
        tripinfo = 'output/' + self.netname + '_tripinfo.xml'
        vehroute = 'output/' + self.netname + '_vehroutes.xml'
        output_nets = etree.SubElement(conf_output, 'netstate-dump')
        output_nets.set('value', netstatfile)
        output_tripinfo = etree.SubElement(conf_output, 'tripinfo-output')
        output_tripinfo.set('value', tripinfo)
        output_vehroute = etree.SubElement(conf_output, 'vehroute-output')
        output_vehroute.set('value', vehroute)
        # Set Time
        conf_time = etree.SubElement(conf_root, 'time')
        time_begin = etree.SubElement(conf_time, 'begin')
        time_begin.set('value', '0')
        time_end = etree.SubElement(conf_time, 'end')
        time_end.set('value', '3600')
        time_roustep = etree.SubElement(conf_time, 'route-steps')
        time_roustep.set('value', '-1')
        # Set Time to teleport
        conf_teleport = etree.SubElement(conf_root, 'time-to-teleport')
        conf_teleport.set('value', '-1')

        # Write to sumo.cfg
        conf_tree = etree.ElementTree(conf_root)
        if withdetector:
            sumocfg_file = self.sumocfg
        else:
            sumocfg_file = self.sumocfg_nodet
        conf_tree.write(sumocfg_file, pretty_print=True, xml_declaration=True, encoding='utf-8')
        return sumocfg_file
<|endoftext|>


<commit_before>#!/usr/bin/env python

###
# Copyright (c) 2002, Jeremiah Fincher
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#   * Redistributions of source code must retain the above copyright notice,
#     this list of conditions, and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions, and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#   * Neither the name of the author of this software nor the name of
#     contributors to this software may be used to endorse or promote products
#     derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###

"""
Fixes stuff that Python should have but doesn't.
"""

__all__ = ['ignore', 'catch', 'reviter', 'window', 'group',
           'partition', 'any', 'all', 'rsplit']

import string
string.ascii = string.maketrans('', '')

def ignore(*args, **kwargs):
    """Simply ignore the arguments sent to it."""
    pass

def catch(f, *args, **kwargs):
    """Catches all exceptions raises by f."""
    try:
        return f(*args, **kwargs)
    except:
        return None

def reviter(L):
    """Iterates through a list in reverse."""
    for i in xrange(len(L) - 1, -1, -1):
        yield L[i]

def window(L, size):
    """Returns a sliding 'window' through the list L of size size."""
    if size < 1:
        raise ValueError, 'size <= 0 unallowed.'
    for i in xrange(len(L) - (size-1)):
        yield L[i:i+size]

import itertools
def ilen(iterator):
    """Returns the length of an iterator."""
    i = 0
    for _ in iterator:
        i += 1
    return i
itertools.ilen = ilen

def group(seq, groupSize, noneFill=True):
    """Groups a given sequence into sublists of length groupSize."""
    ret = []
    L = []
    i = groupSize
    for elt in seq:
        if i > 0:
            L.append(elt)
        else:
            ret.append(L)
            i = groupSize
            L = []
            L.append(elt)
        i -= 1
    if L:
        if noneFill:
            while len(L) < groupSize:
                L.append(None)
        ret.append(L)
    return ret

def partition(p, L):
    """Partitions a list L based on a predicate p.  Returns a (yes,no) tuple"""
    no = []
    yes = []
    for elt in L:
        if p(elt):
            yes.append(elt)
        else:
            no.append(elt)
    return (yes, no)

def any(p, seq):
    """Returns true if any element in seq satisfies predicate p."""
    if p is None:
        p = bool
    for elt in seq:
        if p(elt):
            return True
    return False

def all(p, seq):
    """Returns true if all elements in seq satisfy predicate p."""
    if p is None:
        p = bool
    for elt in seq:
        if not p(elt):
            return False
    return True

def rsplit(s, sep=None, maxsplit=-1):
    """Equivalent to str.split, except splitting from the right."""
    if sep is not None:
        sep = sep[::-1]
    L = s[::-1].split(sep, maxsplit)
    L.reverse()
    return [s[::-1] for s in L]

for name in __all__:
    __builtins__[name] = globals()[name]

# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:

<commit_msg>Changed __all__ to the empty list so people won't misuse fix.py.<commit_after>#!/usr/bin/env python

###
# Copyright (c) 2002, Jeremiah Fincher
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#   * Redistributions of source code must retain the above copyright notice,
#     this list of conditions, and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions, and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#   * Neither the name of the author of this software nor the name of
#     contributors to this software may be used to endorse or promote products
#     derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###

"""
Fixes stuff that Python should have but doesn't.
"""

__all__ = []

exported = ['ignore', 'catch', 'reviter', 'window', 'group',
           'partition', 'any', 'all', 'rsplit']

import string
string.ascii = string.maketrans('', '')

def ignore(*args, **kwargs):
    """Simply ignore the arguments sent to it."""
    pass

def catch(f, *args, **kwargs):
    """Catches all exceptions raises by f."""
    try:
        return f(*args, **kwargs)
    except:
        return None

def reviter(L):
    """Iterates through a list in reverse."""
    for i in xrange(len(L) - 1, -1, -1):
        yield L[i]

def window(L, size):
    """Returns a sliding 'window' through the list L of size size."""
    if size < 1:
        raise ValueError, 'size <= 0 unallowed.'
    for i in xrange(len(L) - (size-1)):
        yield L[i:i+size]

import itertools
def ilen(iterator):
    """Returns the length of an iterator."""
    i = 0
    for _ in iterator:
        i += 1
    return i
itertools.ilen = ilen

def group(seq, groupSize, noneFill=True):
    """Groups a given sequence into sublists of length groupSize."""
    ret = []
    L = []
    i = groupSize
    for elt in seq:
        if i > 0:
            L.append(elt)
        else:
            ret.append(L)
            i = groupSize
            L = []
            L.append(elt)
        i -= 1
    if L:
        if noneFill:
            while len(L) < groupSize:
                L.append(None)
        ret.append(L)
    return ret

def partition(p, L):
    """Partitions a list L based on a predicate p.  Returns a (yes,no) tuple"""
    no = []
    yes = []
    for elt in L:
        if p(elt):
            yes.append(elt)
        else:
            no.append(elt)
    return (yes, no)

def any(p, seq):
    """Returns true if any element in seq satisfies predicate p."""
    if p is None:
        p = bool
    for elt in seq:
        if p(elt):
            return True
    return False

def all(p, seq):
    """Returns true if all elements in seq satisfy predicate p."""
    if p is None:
        p = bool
    for elt in seq:
        if not p(elt):
            return False
    return True

def rsplit(s, sep=None, maxsplit=-1):
    """Equivalent to str.split, except splitting from the right."""
    if sep is not None:
        sep = sep[::-1]
    L = s[::-1].split(sep, maxsplit)
    L.reverse()
    return [s[::-1] for s in L]

for name in exported:
    __builtins__[name] = globals()[name]

# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:

<|endoftext|>


<commit_before>#! python

#import sys
#import SocketServer
from string import *

from xml.sax import ContentHandler
from xml.sax import parseString
from xml.sax import make_parser
from xml.sax.handler import *

spade = dict()
domain = dict()
acc = dict()
ams = dict()
df = dict()
jabber = dict()
simba = dict()

class ParseObject(dict):
        def __getattr__(self, name): return self[name]

class spadeXML:
	'''Analizador de cadenas XML especificamente configurado
	para SPADE'''
	def __init__(self):
		self.ch = spadeXMLHandler()

	def parseString(self, xmlstream, tag_attr, chars):
		self.ch.tag_attr = tag_attr
		parseString(xmlstream, self.ch)
		return self.ch.tag_name, self.ch.message

class spadeXMLHandler(ContentHandler):
	def __init__(self):
		self.chars = ""
		self.message = ""
		self.current_tag = ""
		pass

	def startDocument(self):

		self.content = ParseObject()

	def startElement(self, name, attrs):
		'''Llamado cuando empieza un nodo XML en el flujo'''
		self.chars = ""
		self.tag_name = name
		if name == "platform":
			self.current_tag = "platform"
			self.content['platform'] = ParseObject()
			#print "Current Tag: " + name
		elif name == "acc":
			self.current_tag = "acc"
			self.content['acc'] = ParseObject()
			#print "Current Tag: " + name
		elif name == "ams":
			self.current_tag = "ams"
			self.content['ams'] = ParseObject()
			#print "Current Tag: " + name
		elif name == "df":
			self.current_tag = "df"
			self.content['df'] = ParseObject()
			#print "Current Tag: " + name
		elif name == "mtp":
			self.current_tag = attrs.getValue("name")
			self.content.acc[self.current_tag] = ParseObject()

		else:
			self.chars = ""
			self.tag_name = name

	def characters(self, ch):
		'''Caracteres incluidos en un nodo'''
		self.chars = str(self.chars + ch)

	def endElement(self, name):
		'''Llamado al finalizar un nodo'''
		#print 'Final de: ' + name + ' con current_tag: ' + self.current_tag
		#print 'Caracteres : ' + self.chars
		if name == "hostname":
			if self.current_tag == "platform":
				self.content.platform['hostname'] = self.chars
			elif self.current_tag == "ams":
				self.content.ams['hostname'] = self.chars
			elif self.current_tag == "df":
				self.content.df['hostname'] = self.chars
		elif name == "password":
			if self.current_tag == "ams":
				self.content.ams['password'] = self.chars
			elif self.current_tag == "df":
				self.content.df['password'] = self.chars
		elif name == "port":
			if self.current_tag == "platform":
				self.content.platform['port'] = self.chars
			elif self.current_tag == "ams":
				self.content.ams['port'] = self.chars
			elif self.current_tag == "df":
				self.content.df['port'] = self.chars

			else:
				print self.current_tag
				self.content.acc[self.current_tag]['port'] = self.chars

		elif name == "protocol":
				self.content.acc[self.current_tag]['protocol'] = self.chars

		elif name == "address":
				self.content.acc[self.current_tag]['address'] = self.chars

		elif name == "instance":
				self.content.acc[self.current_tag]['instance'] = self.chars

class ConfigParser:

	"""def get(self, section, keyword):
		answer = ""
		try:
			d = spade[section]
			answer = d[keyword]
		except:
			print "Section %s does not have keyword %s" % (str(section), keyword)
		return answer"""

	def __init__(self):
    		# Create a parser
    		self.parser = make_parser()

    		# Tell the parser we are not interested in XML namespaces
    		self.parser.setFeature(feature_namespaces, 0)

    		# Create the handler
    		dh = spadeXMLHandler()

    		# Tell the parser to use our handler
    		self.parser.setContentHandler(dh)

	def parse(self,fn):

    		# Parse the input
    		self.parser.parse(fn)

		return dh.content

		"""
    		# Manage the results
    		spade['domain'] = domain
    		spade['acc'] = acc
    		spade['ams'] = ams
    		spade['df'] = df
    		spade['jabber'] = jabber
		spade['simba'] = simba
    		#print spade
		"""

<commit_msg>packaging<commit_after>#! python

#import sys
#import SocketServer
from string import *

from xml.sax import ContentHandler
from xml.sax import parseString
from xml.sax import make_parser
from xml.sax.handler import *

spade = dict()
domain = dict()
acc = dict()
ams = dict()
df = dict()
jabber = dict()
simba = dict()

class ParseObject(dict):
        def __getattr__(self, name): return self[name]

class spadeXML:
	'''Analizador de cadenas XML especificamente configurado
	para SPADE'''
	def __init__(self):
		self.ch = spadeXMLHandler()

	def parseString(self, xmlstream, tag_attr, chars):
		self.ch.tag_attr = tag_attr
		parseString(xmlstream, self.ch)
		return self.ch.tag_name, self.ch.message

class spadeXMLHandler(ContentHandler):
	def __init__(self):
		self.chars = ""
		self.message = ""
		self.current_tag = ""
		pass

	def startDocument(self):

		self.content = ParseObject()

	def startElement(self, name, attrs):
		'''Llamado cuando empieza un nodo XML en el flujo'''
		self.chars = ""
		self.tag_name = name
		if name == "platform":
			self.current_tag = "platform"
			self.content['platform'] = ParseObject()
			#print "Current Tag: " + name
		elif name == "acc":
			self.current_tag = "acc"
			self.content['acc'] = ParseObject()
			#print "Current Tag: " + name
		elif name == "ams":
			self.current_tag = "ams"
			self.content['ams'] = ParseObject()
			#print "Current Tag: " + name
		elif name == "df":
			self.current_tag = "df"
			self.content['df'] = ParseObject()
			#print "Current Tag: " + name
		elif name == "mtp":
			self.current_tag = attrs.getValue("name")
			self.content.acc[self.current_tag] = ParseObject()

		else:
			self.chars = ""
			self.tag_name = name

	def characters(self, ch):
		'''Caracteres incluidos en un nodo'''
		self.chars = str(self.chars + ch)

	def endElement(self, name):
		'''Llamado al finalizar un nodo'''
		#print 'Final de: ' + name + ' con current_tag: ' + self.current_tag
		#print 'Caracteres : ' + self.chars
		if name == "hostname":
			if self.current_tag == "platform":
				self.content.platform['hostname'] = self.chars
			elif self.current_tag == "ams":
				self.content.ams['hostname'] = self.chars
			elif self.current_tag == "df":
				self.content.df['hostname'] = self.chars
		elif name == "password":
			if self.current_tag == "ams":
				self.content.ams['password'] = self.chars
			elif self.current_tag == "df":
				self.content.df['password'] = self.chars
		elif name == "port":
			if self.current_tag == "platform":
				self.content.platform['port'] = self.chars
			elif self.current_tag == "ams":
				self.content.ams['port'] = self.chars
			elif self.current_tag == "df":
				self.content.df['port'] = self.chars

			else:
				print self.current_tag
				self.content.acc[self.current_tag]['port'] = self.chars

		elif name == "protocol":
				self.content.acc[self.current_tag]['protocol'] = self.chars

		elif name == "address":
				self.content.acc[self.current_tag]['address'] = self.chars

		elif name == "instance":
				self.content.acc[self.current_tag]['instance'] = self.chars

class ConfigParser:

	"""def get(self, section, keyword):
		answer = ""
		try:
			d = spade[section]
			answer = d[keyword]
		except:
			print "Section %s does not have keyword %s" % (str(section), keyword)
		return answer"""

	def __init__(self):
    		# Create a parser
    		self.parser = make_parser()

    		# Tell the parser we are not interested in XML namespaces
    		self.parser.setFeature(feature_namespaces, 0)

    		# Create the handler
    		self.dh = spadeXMLHandler()

    		# Tell the parser to use our handler
    		self.parser.setContentHandler(self.dh)

	def parse(self,fn):

    		# Parse the input
    		self.parser.parse(fn)

		return self.dh.content

		"""
    		# Manage the results
    		spade['domain'] = domain
    		spade['acc'] = acc
    		spade['ams'] = ams
    		spade['df'] = df
    		spade['jabber'] = jabber
		spade['simba'] = simba
    		#print spade
		"""

<|endoftext|>