<commit_before>#!/usr/bin/env python3

import sys
import socket
import select
import struct
import os

from ecc import ECC

class InvalidDataException(Exception):
	pass

class A2MXServer():
	def __init__(self, nodelist):
		self.nodelist = nodelist

		bind = ('', 0xA22)

		self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		self.sock.setblocking(0)
		self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
		while True:
			try:
				self.sock.bind(bind)
				break
			except OSError:
				bind = (bind[0], bind[1] + 1)
		print("I am", ECC.b58(self.nodelist.ecc.pubkey_hash()).decode('ascii'), ECC.bb(self.nodelist.ecc.pubkey_hash()))
		print("bound to", bind)
		self.sock.listen(5)
		self.nodelist.add(self)

	def fileno(self):
		return self.sock.fileno()

	def select_r(self):
		sock, addr = self.sock.accept()
		A2MXStream(self.nodelist, sock=sock)

	def select_w(self):
		assert False
	def select_e(self):
		assert False

	def finish(self):
		self.sock.close()
		self.nodelist.remove(self)

def A2MXRequest(fn):
	fn.A2MXRequest__marker__ = True
	return fn

class A2MXStream():
	def __init__(self, nodelist=None, uri=None, sock=None):
		assert nodelist != None
		assert (uri == None and sock != None) or (uri != None and sock == None)
		self.nodelist = nodelist
		self.uri = uri
		self.__connected = False
		self.remote_ecc = None
		self.__remote_auth = False
		self.__pub_sent = False
		if sock:
			self.sock = sock
			self.sock.setblocking(0)
			self.connected()
		elif uri != None:
			assert uri.startswith('ax://')
			self.uri = uri
			hostport = uri[5:].split(':')
			assert len(hostport) >= 1
			host = hostport[0]
			if len(hostport) == 1:
				port = 0xA22
			elif len(hostport) == 2:
				port = int(hostport[1])
			else:
				assert False

			self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
			self.sock.setblocking(0)
			try:
				self.sock.connect((host, port))
			except BlockingIOError as e:
				if e.errno != 115:
					raise
			self.nodelist.wadd(self)

	def fileno(self):
		return self.sock.fileno()

	def select_r(self):
		try:
			data = self.sock.recv(4096)
		except ConnectionResetError:
			data = False
		if not data:
			self.finish()
			return
		self.data += data
		while len(self.data) >= self.handler[0]:
			self.handler[1](self.handler[0])

	def select_w(self):
		if not self.__connected:
			self.nodelist.wremove(self)
			self.connected()
		else:
			assert False

	def select_e(self):
		print("select_e")
		self.finish()

	def connected(self):
		self.__connected = True
		self.data = bytearray()
		self.handler = (4, self.getlength)
		self.nodelist.add(self)
		if self.uri != None:
			self.__send_pub()

	def __send_pub(self):
		ecc = self.nodelist.ecc
		self.send(ecc.pubkey_c())
		self.__pub_sent = True

	def getlength(self, length):
		assert len(self.data) >= 4
		length = struct.unpack_from('>L', self.data[:4])[0]
		del self.data[:4]
		self.handler = (length, self.getdata)

	def getdata(self, length):
		assert len(self.data) >= length
		data = self.data[:length]
		del self.data[:length]

		if self.__pub_sent:		# if we sent our public key then all data we receive has to be encrypted
			data = self.nodelist.ecc.decrypt(bytes(data))

		if self.remote_ecc == None:	# first data we expect is the compressed remote public key
			pubkey_x, pubkey_y = self.nodelist.ecc.key_uncompress(data)
			self.remote_ecc = ECC(pubkey_x=pubkey_x, pubkey_y=pubkey_y)

			# ok we have the remote public key, from now on we send everything encrypted
			self.__raw_send = self.send
			self.send = self.encrypted_send

			if not self.__pub_sent:	# send our public key if we haven't already
				self.__send_pub()

			# send the remote public key signed by us
			self.send(self.nodelist.ecc.sign(self.remote_ecc.get_pubkey()))
		elif not self.__remote_auth:	# second data is our own public key signed by remote
			auth = self.remote_ecc.verify(bytes(data), self.nodelist.ecc.get_pubkey())
			if not auth:
				raise InvalidDataException('3')
			self.__remote_auth = True
			print("connection up with", ECC.b58(self.remote_ecc.pubkey_hash()).decode('ascii'), ECC.bb(self.remote_ecc.pubkey_hash()))

			# inform the peer about us
			ecc = self.nodelist.ecc
			data = bytearray()
			data += ecc.pubkey_c()
			data += self.remote_ecc.pubkey_c()
			data = bytes(data)
			r = self.request(b'I', ecc.pubkey_c(), self.remote_ecc.pubkey_c(), ecc.sign(data))
			self.send(r)
		else:
			for d in self.parse(data):
				fn = d[0].decode('UTF-8')
				try:
					f = getattr(self, fn)
				except AttributeError:
					pass
				else:
					if hasattr(f, 'A2MXRequest__marker__') and f.A2MXRequest__marker__ == True:
						f(*d[1:])
						continue
				print("Invalid request {}".format(fn))
		self.handler = (4, self.getlength)

	@staticmethod
	def parse(data):
		i = 0
		while i < len(data):
			requestLen = struct.unpack('>L', data[i:i+4])[0]
			i += 4
			ri = i
			args = []
			while ri < i + requestLen:
				argLen = struct.unpack('>L', data[ri:ri+4])[0]
				ri += 4
				arg = data[ri:ri+argLen]
				ri += argLen
				args.append(arg)
			i = ri
			yield args
		assert i == len(data)

	@staticmethod
	def request(*args):
		data = b''
		for arg in args:
			if isinstance(arg, str):
				arg = arg.encode('UTF-8')
			data += struct.pack('>L', len(arg))
			data += arg
		return struct.pack('>L', len(data)) + data

	def send(self, *data):
		length = 0
		for d in data:
			length += len(d)
		self.sock.send(struct.pack('>L', length), socket.MSG_MORE)
		for d in data[:-1]:
			self.sock.send(d, socket.MSG_MORE)
		self.sock.send(data[-1])

	def encrypted_send(self, *data):
		data = b''.join(data)
		data = self.nodelist.ecc.encrypt(data, self.remote_ecc.get_pubkey())
		self.__raw_send(data)

	def finish(self):
		print("finish")
		self.sock.close()
		self.nodelist.remove(self)

	@A2MXRequest
	def I(self, path, mypub, signature):
		self.nodelist.new_path(path, mypub, signature, self)

class A2MXPath():
	def __init__(self, path, pub, signature):
		print("A2MXPath", len(path), len(pub), len(signature))
		self.endnode = False
		self.nextpath = False

		self.pub = ECC()
		self.pub.pubkey_x, self.pub.pubkey_y = self.pub.key_uncompress(pub)

		if len(path) > 68:
			for d in A2MXStream.parse(path):
				if len(d) != 3:
					raise InvalidDataException('unparseable path')
				self.nextpath = A2MXPath(*d)
				print("nextpath")
		else:
			self.endnode = ECC()
			self.endnode.pubkey_x, self.endnode.pubkey_y = self.endnode.key_uncompress(path)

		data = b''.join((path, pub))
		if self.nextpath:
			verify = self.nextpath.pub.verify(signature, data)
		else:
			verify = self.endnode.verify(signature, data)
		if not verify:
			raise InvalidDataException('signature failure')
		self.path = path
		self.signature = signature

	def __str__(self):
		return "A2MXPath({})<{}>".format(ECC.b58(self.endnode.pubkey_hash()).decode('ascii') if self.endnode else str(self.nextpath), ECC.b58(self.pub.pubkey_hash()).decode('ascii'))

class A2MXNodelist():
	def __init__(self):
		self.rlist = []
		self.wlist = []
		self.networkInfo = {}
		try:
			with open('.a2mx/priv', 'rb') as f:
				privkey = f.read()
			with open('.a2mx/pub', 'rb') as f:
				pubkey = f.read()
		except FileNotFoundError:
			privkey = None
			pubkey = None
		self.ecc = ECC(privkey=privkey, pubkey=pubkey)
		if privkey == None:
			os.umask(63)	# 0700
			try:
				os.mkdir('.a2mx')
			except FileExistsError:
				pass
			with open('.a2mx/priv', 'wb') as f:
				f.write(self.ecc.get_privkey())
			with open('.a2mx/pub', 'wb') as f:
				f.write(self.ecc.get_pubkey())

	def add(self, selectable):
		self.rlist.append(selectable)
	def remove(self, selectable):
		self.rlist.remove(selectable)
	def wadd(self, selectable):
		self.wlist.append(selectable)
	def wremove(self, selectable):
		self.wlist.remove(selectable)
	def select(self):
		r, w, e = select.select(self.rlist, self.wlist, self.rlist)
		for sock in e:
			sock.select_e()
		for sock in w:
			sock.select_w()
		for sock in r:
			sock.select_r()
	def shutdown(self):
		for sock in set(self.rlist + self.wlist):
			sock.finish()

	def new_path(self, path, mypub, signature, stream):
		p = A2MXPath(path, mypub, signature)
		print("new_path", p)
		if p.pub.get_pubkey() != self.ecc.get_pubkey():
			raise InvalidDataException('path not for me?')

		nodes = []
		x = p
		while x:
			nodes.append(x.pub.get_pubkey())
			if x.endnode:
				nodes.append(x.endnode.get_pubkey())
			x = x.nextpath
		for stream in filter(lambda s: isinstance(s, A2MXStream), self.rlist):
			if stream.remote_ecc.get_pubkey() in nodes:
				continue

			print("INFORM STREAM", ECC.b58(stream.remote_ecc.pubkey_hash()).decode('ascii'))
			oldpath = stream.request(p.path, p.pub.pubkey_c(), p.signature)
			data = bytearray()
			data += oldpath
			data += stream.remote_ecc.pubkey_c()
			data = bytes(data)
			r = stream.request(b'I', oldpath, stream.remote_ecc.pubkey_c(), self.ecc.sign(data))
			stream.send(r)

		return
		if mypub != self.ecc.pubkey_c():
			# second public key is not our own, ignore request
			raise InvalidDataError('invalid public key')
		data = b''.join((path, mypub))
		if not stream.remote_ecc.verify(signature, data):
			# failed to verify signature
			raise InvalidDataError('invalid signature')

		# path:
		#   signed_hop2(signed_hop3(signed_endnode(endnode), hop3), hop2)
		# parse path...
		#self.networkInfo[endnode] = [ stream, hop2, hop3, ..., endnode ]

nodelist = A2MXNodelist()
server = A2MXServer(nodelist)

for uri in sys.argv[1:]:
	print("connect to", uri)
	c = A2MXStream(nodelist, uri=uri)

try:
	while True:
		nodelist.select()
except KeyboardInterrupt:
	nodelist.shutdown()

<commit_msg>cleanup<commit_after>#!/usr/bin/env python3

import sys
import socket
import select
import struct
import os

from ecc import ECC

class InvalidDataException(Exception):
	pass

class A2MXServer():
	def __init__(self, nodelist):
		self.nodelist = nodelist

		bind = ('', 0xA22)

		self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		self.sock.setblocking(0)
		self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
		while True:
			try:
				self.sock.bind(bind)
				break
			except OSError:
				bind = (bind[0], bind[1] + 1)
		print("I am", ECC.b58(self.nodelist.ecc.pubkey_hash()).decode('ascii'), "bound to", bind)
		self.sock.listen(5)
		self.nodelist.add(self)

	def fileno(self):
		return self.sock.fileno()

	def select_r(self):
		sock, addr = self.sock.accept()
		A2MXStream(self.nodelist, sock=sock)

	def select_w(self):
		assert False
	def select_e(self):
		assert False

	def finish(self):
		self.sock.close()
		self.nodelist.remove(self)

def A2MXRequest(fn):
	fn.A2MXRequest__marker__ = True
	return fn

class A2MXStream():
	def __init__(self, nodelist=None, uri=None, sock=None):
		assert nodelist != None
		assert (uri == None and sock != None) or (uri != None and sock == None)
		self.nodelist = nodelist
		self.uri = uri
		self.__connected = False
		self.remote_ecc = None
		self.__remote_auth = False
		self.__pub_sent = False
		if sock:
			self.sock = sock
			self.sock.setblocking(0)
			self.connected()
		elif uri != None:
			assert uri.startswith('ax://')
			self.uri = uri
			hostport = uri[5:].split(':')
			assert len(hostport) >= 1
			host = hostport[0]
			if len(hostport) == 1:
				port = 0xA22
			elif len(hostport) == 2:
				port = int(hostport[1])
			else:
				assert False

			self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
			self.sock.setblocking(0)
			try:
				self.sock.connect((host, port))
			except BlockingIOError as e:
				if e.errno != 115:
					raise
			self.nodelist.wadd(self)

	def fileno(self):
		return self.sock.fileno()

	def select_r(self):
		try:
			data = self.sock.recv(4096)
		except ConnectionResetError:
			data = False
		if not data:
			self.finish()
			return
		self.data += data
		while len(self.data) >= self.handler[0]:
			self.handler[1](self.handler[0])

	def select_w(self):
		if not self.__connected:
			self.nodelist.wremove(self)
			self.connected()
		else:
			assert False

	def select_e(self):
		print("select_e")
		self.finish()

	def connected(self):
		self.__connected = True
		self.data = bytearray()
		self.handler = (4, self.getlength)
		self.nodelist.add(self)
		if self.uri != None:
			self.__send_pub()

	def __send_pub(self):
		ecc = self.nodelist.ecc
		self.send(ecc.pubkey_c())
		self.__pub_sent = True

	def getlength(self, length):
		assert len(self.data) >= 4
		length = struct.unpack_from('>L', self.data[:4])[0]
		del self.data[:4]
		self.handler = (length, self.getdata)

	def getdata(self, length):
		assert len(self.data) >= length
		data = self.data[:length]
		del self.data[:length]

		if self.__pub_sent:		# if we sent our public key then all data we receive has to be encrypted
			data = self.nodelist.ecc.decrypt(bytes(data))

		if self.remote_ecc == None:	# first data we expect is the compressed remote public key
			pubkey_x, pubkey_y = self.nodelist.ecc.key_uncompress(data)
			self.remote_ecc = ECC(pubkey_x=pubkey_x, pubkey_y=pubkey_y)

			# ok we have the remote public key, from now on we send everything encrypted
			self.__raw_send = self.send
			self.send = self.encrypted_send

			if not self.__pub_sent:	# send our public key if we haven't already
				self.__send_pub()

			# send the remote public key signed by us
			self.send(self.nodelist.ecc.sign(self.remote_ecc.get_pubkey()))
		elif not self.__remote_auth:	# second data is our own public key signed by remote
			auth = self.remote_ecc.verify(bytes(data), self.nodelist.ecc.get_pubkey())
			if not auth:
				raise InvalidDataException('3')
			self.__remote_auth = True
			print("connection up with", ECC.b58(self.remote_ecc.pubkey_hash()).decode('ascii'))

			# inform the peer about us
			ecc = self.nodelist.ecc
			data = bytearray()
			data += ecc.pubkey_c()
			data += self.remote_ecc.pubkey_c()
			data = bytes(data)
			r = self.request(b'I', ecc.pubkey_c(), self.remote_ecc.pubkey_c(), ecc.sign(data))
			self.send(r)
		else:
			for d in self.parse(data):
				fn = d[0].decode('UTF-8')
				try:
					f = getattr(self, fn)
				except AttributeError:
					pass
				else:
					if hasattr(f, 'A2MXRequest__marker__') and f.A2MXRequest__marker__ == True:
						f(*d[1:])
						continue
				print("Invalid request {}".format(fn))
		self.handler = (4, self.getlength)

	@staticmethod
	def parse(data):
		i = 0
		while i < len(data):
			requestLen = struct.unpack('>L', data[i:i+4])[0]
			i += 4
			ri = i
			args = []
			while ri < i + requestLen:
				argLen = struct.unpack('>L', data[ri:ri+4])[0]
				ri += 4
				arg = data[ri:ri+argLen]
				ri += argLen
				args.append(arg)
			i = ri
			yield args
		assert i == len(data)

	@staticmethod
	def request(*args):
		data = b''
		for arg in args:
			if isinstance(arg, str):
				arg = arg.encode('UTF-8')
			data += struct.pack('>L', len(arg))
			data += arg
		return struct.pack('>L', len(data)) + data

	def send(self, *data):
		length = 0
		for d in data:
			length += len(d)
		self.sock.send(struct.pack('>L', length), socket.MSG_MORE)
		for d in data[:-1]:
			self.sock.send(d, socket.MSG_MORE)
		self.sock.send(data[-1])

	def encrypted_send(self, *data):
		data = b''.join(data)
		data = self.nodelist.ecc.encrypt(data, self.remote_ecc.get_pubkey())
		self.__raw_send(data)

	def finish(self):
		print("finish")
		self.sock.close()
		self.nodelist.remove(self)

	@A2MXRequest
	def I(self, path, mypub, signature):
		self.nodelist.new_path(path, mypub, signature, self)

class A2MXPath():
	def __init__(self, path, pub, signature):
		self.endnode = False
		self.nextpath = False

		self.pub = ECC()
		self.pub.pubkey_x, self.pub.pubkey_y = self.pub.key_uncompress(pub)

		if len(path) > 68:
			for d in A2MXStream.parse(path):
				if len(d) != 3:
					raise InvalidDataException('unparseable path')
				self.nextpath = A2MXPath(*d)
		else:
			self.endnode = ECC()
			self.endnode.pubkey_x, self.endnode.pubkey_y = self.endnode.key_uncompress(path)

		data = b''.join((path, pub))
		if self.nextpath:
			verify = self.nextpath.pub.verify(signature, data)
		else:
			verify = self.endnode.verify(signature, data)
		if not verify:
			raise InvalidDataException('signature failure')
		self.path = path
		self.signature = signature

	def __str__(self):
		return "({}, {})".format(ECC.b58(self.endnode.pubkey_hash()).decode('ascii') if self.endnode else str(self.nextpath), ECC.b58(self.pub.pubkey_hash()).decode('ascii'))

class A2MXNodelist():
	def __init__(self):
		self.rlist = []
		self.wlist = []
		self.networkInfo = {}
		try:
			with open('.a2mx/priv', 'rb') as f:
				privkey = f.read()
			with open('.a2mx/pub', 'rb') as f:
				pubkey = f.read()
		except FileNotFoundError:
			privkey = None
			pubkey = None
		self.ecc = ECC(privkey=privkey, pubkey=pubkey)
		if privkey == None:
			os.umask(63)	# 0700
			try:
				os.mkdir('.a2mx')
			except FileExistsError:
				pass
			with open('.a2mx/priv', 'wb') as f:
				f.write(self.ecc.get_privkey())
			with open('.a2mx/pub', 'wb') as f:
				f.write(self.ecc.get_pubkey())

	def add(self, selectable):
		self.rlist.append(selectable)
	def remove(self, selectable):
		self.rlist.remove(selectable)
	def wadd(self, selectable):
		self.wlist.append(selectable)
	def wremove(self, selectable):
		self.wlist.remove(selectable)
	def select(self):
		r, w, e = select.select(self.rlist, self.wlist, self.rlist)
		for sock in e:
			sock.select_e()
		for sock in w:
			sock.select_w()
		for sock in r:
			sock.select_r()
	def shutdown(self):
		for sock in set(self.rlist + self.wlist):
			sock.finish()

	def new_path(self, path, mypub, signature, stream):
		p = A2MXPath(path, mypub, signature)
		print("new_path", p)
		if p.pub.get_pubkey() != self.ecc.get_pubkey():
			raise InvalidDataException('path not for me?')

		nodes = []
		x = p
		while x:
			nodes.append(x.pub.get_pubkey())
			if x.endnode:
				nodes.append(x.endnode.get_pubkey())
			x = x.nextpath
		for stream in filter(lambda s: isinstance(s, A2MXStream), self.rlist):
			if stream.remote_ecc.get_pubkey() in nodes:
				continue

			print("INFORM STREAM", ECC.b58(stream.remote_ecc.pubkey_hash()).decode('ascii'))
			oldpath = stream.request(p.path, p.pub.pubkey_c(), p.signature)
			data = bytearray()
			data += oldpath
			data += stream.remote_ecc.pubkey_c()
			data = bytes(data)
			r = stream.request(b'I', oldpath, stream.remote_ecc.pubkey_c(), self.ecc.sign(data))
			stream.send(r)

nodelist = A2MXNodelist()
server = A2MXServer(nodelist)

for uri in sys.argv[1:]:
	print("connect to", uri)
	c = A2MXStream(nodelist, uri=uri)

try:
	while True:
		nodelist.select()
except KeyboardInterrupt:
	nodelist.shutdown()

<|endoftext|>


<commit_before>#!/usr/bin/env python3.2

"""
INPUT:

The input is a tab-delimited output file that is the product of parsing eukaryotic
annotation evidence.  It has the following columns:

   0. Transcript ID
   1. ORF ID
   2. ORF length
   3. BLAST description
   4. BLAST E-value
   5. HMM description
   6. HMM total score
   7. HMM total E-value
   8. GO terms (derived from HMM), comma-seprated
   9. EC numbers (dervied from HMM), comma-separated

OUTPUT:

WARNING: This script creates a lot of files in the specified output directory.
To be more specific, it will create (N*2)+1 where N is the number of molecular
fragments in the input nucleotide set.  This is true whether that number is
10 supercontigs of a large genome or 50,000 transcripts annotated without
a genome.

Pathologic format described:
http://bioinformatics.ai.sri.com/ptools/pathologic-format.pdf

Example (genetic elements list):
http://bioinformatics.ai.sri.com/ptools/sample-genetic-elements.dat

Example (per molecule):
http://bioinformatics.ai.sri.com/ptools/tpal.pf

"""
import argparse
import os
import biocodeutils
import biothings

def main():
    parser = argparse.ArgumentParser( description='Put a description of your script here')

    ## output file to be written
    parser.add_argument('-a', '--annotation_tab', type=str, required=True, help='Path to an input file to be parsed' )
    parser.add_argument('-g', '--genomic_fasta', type=str, required=True, help='Underlying nucleotide FASTA file for the annotated proteins' )
    parser.add_argument('-o', '--output_dir', type=str, required=True, help='Path to an output file to be created' )
    args = parser.parse_args()

    ntseqs = biocodeutils.fasta_dict_from_file( args.genomic_fasta )

    write_elements_file(ntseqs, args.output_dir )
    write_seq_files(ntseqs, args.output_dir)

    for line in open(args.annotation_tab):
        if line.startswith("#"):
            continue

def write_seq_files( seqs, outdir ):
    for seq in seqs:
        ## any changes to this path convention need to be also updated in write_elements_file()
        seqout_path = "{0}/{1}.fna".format(outdir, seq)
        seqout = open(seqout_path, mode='wt', encoding='utf-8')
        seqout.write( ">{0} {1}\n".format(seq, seqs[seq]['h']) )

        for i in range(0, len(seqs[seq]['s']), 60):
            seqout.write(seqs[seq]['s'][i : i + 60] + "\n")

        seqout.close()

def write_elements_file( seqs, outdir ):
    elements_file_path = "{0}/genetic-elements.dat".format(outdir)

    fout = open(elements_file_path, mode='wt', encoding='utf-8')

    for seq in seqs:
        fout.write( "ID\t{0}\n".format(seq) )

        if not seqs[seq]['h'].isspace():
            fout.write( "NAME\t{0} {1}\n".format(seq, seqs[seq]['h']) )

        fout.write( "CIRCULAR?\tN\n" )
        fout.write( "ANNOT-FILE\t{0}/{1}.pf\n".format(outdir, seq) ) ## must be full path
        fout.write( "SEQ-FILE\t{0}/{1}.fna\n".format(outdir, seq) ) ## must be full path
        fout.write("//\n")

    fout.close()

if __name__ == '__main__':
    main()

<commit_msg>minor: corrected script description<commit_after>#!/usr/bin/env python3.2

"""
INPUT:

The input is a tab-delimited output file that is the product of parsing eukaryotic
annotation evidence.  It has the following columns:

   0. Transcript ID
   1. ORF ID
   2. ORF length
   3. BLAST description
   4. BLAST E-value
   5. HMM description
   6. HMM total score
   7. HMM total E-value
   8. GO terms (derived from HMM), comma-seprated
   9. EC numbers (dervied from HMM), comma-separated

OUTPUT:

WARNING: This script creates a lot of files in the specified output directory.
To be more specific, it will create (N*2)+1 where N is the number of molecular
fragments in the input nucleotide set.  This is true whether that number is
10 supercontigs of a large genome or 50,000 transcripts annotated without
a genome.

Pathologic format described:
http://bioinformatics.ai.sri.com/ptools/pathologic-format.pdf

Example (genetic elements list):
http://bioinformatics.ai.sri.com/ptools/sample-genetic-elements.dat

Example (per molecule):
http://bioinformatics.ai.sri.com/ptools/tpal.pf

"""
import argparse
import os
import biocodeutils
import biothings

def main():
    parser = argparse.ArgumentParser( description='Transforms a tab-delimited annotation file to PathoLogic format')

    ## output file to be written
    parser.add_argument('-a', '--annotation_tab', type=str, required=True, help='Path to an input file to be parsed' )
    parser.add_argument('-g', '--genomic_fasta', type=str, required=True, help='Underlying nucleotide FASTA file for the annotated proteins' )
    parser.add_argument('-o', '--output_dir', type=str, required=True, help='Path to an output file to be created' )
    args = parser.parse_args()

    ntseqs = biocodeutils.fasta_dict_from_file( args.genomic_fasta )

    write_elements_file(ntseqs, args.output_dir )
    write_seq_files(ntseqs, args.output_dir)

    for line in open(args.annotation_tab):
        if line.startswith("#"):
            continue

def write_seq_files( seqs, outdir ):
    for seq in seqs:
        ## any changes to this path convention need to be also updated in write_elements_file()
        seqout_path = "{0}/{1}.fna".format(outdir, seq)
        seqout = open(seqout_path, mode='wt', encoding='utf-8')
        seqout.write( ">{0} {1}\n".format(seq, seqs[seq]['h']) )

        for i in range(0, len(seqs[seq]['s']), 60):
            seqout.write(seqs[seq]['s'][i : i + 60] + "\n")

        seqout.close()

def write_elements_file( seqs, outdir ):
    elements_file_path = "{0}/genetic-elements.dat".format(outdir)

    fout = open(elements_file_path, mode='wt', encoding='utf-8')

    for seq in seqs:
        fout.write( "ID\t{0}\n".format(seq) )

        if not seqs[seq]['h'].isspace():
            fout.write( "NAME\t{0} {1}\n".format(seq, seqs[seq]['h']) )

        fout.write( "CIRCULAR?\tN\n" )
        fout.write( "ANNOT-FILE\t{0}/{1}.pf\n".format(outdir, seq) ) ## must be full path
        fout.write( "SEQ-FILE\t{0}/{1}.fna\n".format(outdir, seq) ) ## must be full path
        fout.write("//\n")

    fout.close()

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Python 2 compatibility
from __future__ import unicode_literals

# Standard library imports
import warnings
import sys
import re

# Import the faster C implementation of ElementTree whenever available
try:
    from xml.etree import cElementTree as Etree
except ImportError:
    from xml.etree import ElementTree as Etree

# Check python version to set the object that can detect non unicode strings
if sys.version_info >= (3, 0):
    # noinspection PyCompatibility
    from html.parser import HTMLParser
    basestring = (bytes, str)
else:
    # noinspection PyUnresolvedReferences,PyCompatibility
    from HTMLParser import HTMLParser

__all__ = ["HTMLement", "fromstring", "fromstringlist", "parse", "HTMLParseError"]
__copyright__ = "Copyright (C) 2016 William Forde"
__author__ = "William Forde"
__license__ = "Public "
__version__ = "0.1"
__credit__ = "Rafael Marmelo"

def fromstring(text, tag=None, attrs=None, encoding=None):
    """
    Parse's "HTML" document from a string into an element_tree.

    :param text: The "HTML" document to parse.
    :type text: str, bytes

    :param str tag: (optional) see :class:`HTMLement` for details.
    :param dict attrs: (optional) see :class:`HTMLement` for details.
    :param str encoding: (optional) The encoding used for/in *text*

    :return: The root element of the element tree.
    :rtype: xml.etree.ElementTree.Element

    :raises HTMLParseError: If parsing of HTML document fails.
    :raises UnicodeDecodeError: If decoding of *text* fails.
    """
    parser = HTMLement(tag, attrs, encoding)
    parser.feed(text)
    return parser.close()

def fromstringlist(sequence, tag=None, attrs=None, encoding=None):
    """
    Parses an HTML document from a sequence of "HTML sections" into an element tree.

    :param sequence: A sequence of "HTML sections" to parse.
    :type sequence: list[str,bytes]

    :param str tag: (optional) see :class:`HTMLement` for details.
    :param dict attrs: (optional) see :class:`HTMLement` for details.
    :param str encoding: (optional) The encoding used for each section within *sequence*

    :return: The root element of the element tree.
    :rtype: xml.etree.ElementTree.Element

    :raises HTMLParseError: If parsing of "HTML document" fails.
    :raises UnicodeDecodeError: If decoding of a section within *sequence* fails.
    """
    parser = HTMLement(tag, attrs, encoding)
    for text in sequence:
        parser.feed(text)
    return parser.close()

def parse(source, tag=None, attrs=None, encoding=None):
    """
    Load an external HTML document into element tree.

    :param source: A filename or file like object containing HTML data.
    :type source: str, io.TextIOBase

    :param str tag: (optional) see :class:`HTMLement` for details.
    :param dict attrs: (optional) see :class:`HTMLement` for details.
    :param str encoding: (optional) The encoding used for *source*.

    :return: The root element of the element tree.
    :rtype: xml.etree.ElementTree.Element

    :raises HTMLParseError: If parsing of "HTML document" fails.
    :raises UnicodeDecodeError: If decoding of *source* fails.
    """
    # Assume that source is a file pointer if no read methods is found
    if hasattr(source, "read"):
        source = open(source, "rb")
        close_source = True
    else:
        close_source = False

    try:
        parser = HTMLement(tag, attrs, encoding)
        while True:
            # Read in 64k at a time
            data = source.read(65536)
            if not data:
                break

            # Feed the parser
            parser.feed(data)

        # Return the root element
        return parser.close()

    finally:
        if close_source:
            source.close()

# Required for raiseing HTMLParseError in python3, emulates python2
class HTMLParseError(Exception):
    """Exception raised for all parse errors."""
    def __init__(self, msg, position=(None, None)):
        self.msg = msg
        self.lineno = position[0]
        self.offset = position[1]

    def __str__(self):
        result = self.msg
        if self.lineno is not None:
            result += ", at line %d" % self.lineno
        if self.offset is not None:
            result += ", column %d" % self.offset
        return result

class HTMLement(object):
    """
    Python HTMLParser extension with ElementTree Parser support.

    This HTML Parser extends :class:`html.parser.HTMLParser`, returning an :class:`xml.etree.ElementTree.Element`
    instance. The returned root element natively supports the ElementTree API.
    (e.g. you may use its limited support for `XPath expressions`__)

    When a "tag" and "tag attributes" are given the parser will search for a required section. Only when the required
    section is found, does the parser start parsing the "HTML document". The element that matches the search criteria
    will then become the new "root element".

    :param tag: (optional) Name of "tag / element" which is used to filter down "the tree" to a required section.
    :type tag: str

    :param attrs: (optional) The attributes of the element, that will be used, when searching for the required section.
    :type attrs: dict

    :param encoding: (optional) Encoding used, when decoding the source data before feeding it to the parser.
    :type encoding: str

    :raises HTMLParseError: If parsing of "HTML document" fails.

    .. _Xpath: https://docs.python.org/3.6/library/xml.etree.elementtree.html#xpath-support
    __ XPath_
    """
    def __init__(self, tag="", attrs=None, encoding=""):
        self._parser = ParseHTML(tag, attrs)
        self.encoding = encoding
        self.finished = False

    def feed(self, data):
        """
        Feeds data to the parser.

        If *data*, is of "type `bytes` and where no encoding was specified, then the encoding
        will be extracted from *data* using "meta tags" if available.
        Otherwise encoding will default to "ISO-8859-1"

        :param data: HTML data
        :type data: str, bytes

        :raises UnicodeDecodeError: If decoding of *data* fails.
        """
        # Skip feeding data into parser if we already have what we want
        if self.finished is True:
            return None

        # Make sure that we have unicode before continuing
        if isinstance(data, bytes):
            if self.encoding:
                data = data.decode(self.encoding)
            else:
                data = self._make_unicode(data)

        # Parse the html document
        try:
            self._parser.feed(data)
        except EOFError:
            self.finished = True
            self._parser.reset()

    def close(self):
        """
        Close the "tree builder" and return the "root element" of the "element tree".

        :return: The "root element" of the "element tree".
        :rtype: xml.etree.ElementTree.Element

        :raises RuntimeError: If no element, matching search criteria, was found when a filter is given.
        """
        return self._parser.close()

    def _make_unicode(self, data):
        """
        Convert *data* from type `bytes` to type `str`.

        :param data: The html document.
        :type data: bytes

        :return: HTML data decoded into str(unicode).
        :rtype: str
        """
        # Atemp to find the encoding from the html source
        end_head_tag = data.find(b"</head>")
        if end_head_tag:
            # Search for the charset attribute within the meta tags
            charset_refind = b'<meta.+?charset=[\'"]*(.+?)["\'].*?>'
            charset = re.search(charset_refind, data[:end_head_tag], re.IGNORECASE)
            if charset:
                self.encoding = encoding = str(charset.group(1))
                return data.decode(encoding)

        # Decode the string into unicode using default encoding
        warn_msg = "Unable to determine encoding, defaulting to iso-8859-1"
        warnings.warn(warn_msg, UnicodeWarning, stacklevel=2)
        self.encoding = "iso-8859-1"
        return data.decode("iso-8859-1")

class ParseHTML(HTMLParser):
    def __init__(self, tag="", attrs=None):
        # Initiate HTMLParser
        HTMLParser.__init__(self)
        self.convert_charrefs = True
        self._root = None  # root element
        self._data = []  # data collector
        self._factory = Etree.Element

        # Split attributes into wanted and unwanted attributes
        self._unw_attrs = [attrs.pop(key) for key, value in attrs.items() if value is False] if attrs else []
        self.attrs = attrs if attrs else {}
        self.enabled = not tag
        self.tag = tag

        # Some tags in html do not require closing tags so thoes tags will need to be auto closed (Void elements)
        # Refer to: https://www.w3.org/TR/html/syntax.html#void-elements
        self._voids = frozenset(("area", "base", "br", "col", "hr", "img", "input", "link", "meta", "param",
                                 # Only in HTML5
                                 "embed", "keygen", "source", "track",
                                 # Not supported in HTML5
                                 "basefont", "frame", "isindex",
                                 # SVG self closing tags
                                 "rect", "circle", "ellipse", "line", "polyline", "polygon",
                                 "path", "stop", "use", "image", "animatetransform"))

        # Create temporary root element to protect from badly written sites that either
        # have no html starting tag or multiple top level elements
        elem = self._factory("html")
        self._elem = [elem]
        self._last = elem
        self._tail = 0

    def handle_starttag(self, tag, attrs):
        self._handle_starttag(tag, attrs, self_closing=tag in self._voids)

    def handle_startendtag(self, tag, attrs):
        self._handle_starttag(tag, attrs, self_closing=True)

    def _handle_starttag(self, tag, attrs, self_closing=False):
        enabled = self.enabled
        # Add tag element to tree if we have no filter or that the filter matches
        if enabled or self._search(tag, attrs):
            # Convert attrs to dictionary
            attrs = dict(attrs) if attrs else {}
            self._flush()

            # Create the new element
            elem = self._factory(tag, attrs)
            self._elem[-1].append(elem)
            self._last = elem

            # Only append the element to the list of elements if it's not a self closing element
            if self_closing:
                self._tail = 1
            else:
                self._elem.append(elem)
                self._tail = 0

            # Set this element as the root element when the filter search matches
            if not enabled:
                self._root = elem
                self.enabled = True

    def handle_endtag(self, tag):
        # Only process end tags when we have no filter or that the filter has been matched
        if self.enabled and tag not in self._voids:
            _elem = self._elem
            _root = self._root
            # Check that the closing tag is what's actualy expected
            if _elem[-1].tag == tag:
                self._flush()
                self._tail = 1
                self._last = elem = _elem.pop()
                if elem is _root:
                    raise EOFError

            # If the previous element is what we actually have then the expected element was not
            # properly closed so we must close that before closing what we have now
            elif len(_elem) >= 2 and _elem[-2].tag == tag:
                self._flush()
                self._tail = 1
                for _ in range(2):
                    self._last = elem = _elem.pop()
                    if elem is _root:
                        raise EOFError
            else:
                # Unable to match the tag to an element, ignoring it
                return None

    def handle_data(self, data):
        data = data.strip()
        if data and self.enabled:
            self._data.append(data)

    def handle_comment(self, data):
        data = data.strip()
        if data and self.enabled:
            elem = Etree.Comment(data)
            self._elem[-1].append(elem)

    def close(self):
        self._flush()
        if self.enabled is False:
            msg = "Unable to find requested section with tag of '{}' and attributes of {}"
            raise RuntimeError(msg.format(self.tag, self.attrs))
        elif self._root is not None:
            return self._root
        else:
            # Search the root element to find a proper html root element if one exists
            tmp_root = self._elem[0]
            proper_root = tmp_root.find("html")
            if proper_root is None:
                # Not proper root was found
                return tmp_root
            else:
                # Proper root found
                return proper_root

    def error(self, message):
        raise HTMLParseError(message, self.getpos())

    def _flush(self):
        if self._data:
            if self._last is not None:
                text = "".join(self._data)
                if self._tail:
                    self._last.tail = text
                else:
                    self._last.text = text
            self._data = []

    def _search(self, tag, attrs):
        # Only search when the tag matches
        if tag == self.tag:
            # If we have required attrs to match then search all attrs for wanted attrs
            # And also check that we do not have any attrs that are unwanted
            if self.attrs or self._unw_attrs:
                if attrs:
                    wanted_attrs = self.attrs.copy()
                    unwanted_attrs = self._unw_attrs
                    for key, value in attrs:
                        # Check for unwanted attrs
                        if key in unwanted_attrs:
                            return False

                        # Check for wanted attrs
                        elif key in wanted_attrs:
                            c_value = wanted_attrs[key]
                            if c_value == value or c_value is True:
                                # Remove this attribute from the wanted dict of attributes
                                # to indicate that this attribute has been found
                                del wanted_attrs[key]

                    # If wanted_attrs is now empty then all attributes must have been found
                    if not wanted_attrs:
                        return True
            else:
                # We only need to match tag
                return True

        # Unable to find required section
        return False
<commit_msg>fix bug when searching attribute with a value of false fails<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Python 2 compatibility
from __future__ import unicode_literals

# Standard library imports
import warnings
import sys
import re

# Import the faster C implementation of ElementTree whenever available
try:
    from xml.etree import cElementTree as Etree
except ImportError:
    from xml.etree import ElementTree as Etree

# Check python version to set the object that can detect non unicode strings
if sys.version_info >= (3, 0):
    # noinspection PyCompatibility
    from html.parser import HTMLParser
    basestring = (bytes, str)
else:
    # noinspection PyUnresolvedReferences,PyCompatibility
    from HTMLParser import HTMLParser

__all__ = ["HTMLement", "fromstring", "fromstringlist", "parse", "HTMLParseError"]
__copyright__ = "Copyright (C) 2016 William Forde"
__author__ = "William Forde"
__license__ = "Public "
__version__ = "0.1"
__credit__ = "Rafael Marmelo"

def fromstring(text, tag=None, attrs=None, encoding=None):
    """
    Parse's "HTML" document from a string into an element_tree.

    :param text: The "HTML" document to parse.
    :type text: str, bytes

    :param str tag: (optional) see :class:`HTMLement` for details.
    :param dict attrs: (optional) see :class:`HTMLement` for details.
    :param str encoding: (optional) The encoding used for/in *text*

    :return: The root element of the element tree.
    :rtype: xml.etree.ElementTree.Element

    :raises HTMLParseError: If parsing of HTML document fails.
    :raises UnicodeDecodeError: If decoding of *text* fails.
    """
    parser = HTMLement(tag, attrs, encoding)
    parser.feed(text)
    return parser.close()

def fromstringlist(sequence, tag=None, attrs=None, encoding=None):
    """
    Parses an HTML document from a sequence of "HTML sections" into an element tree.

    :param sequence: A sequence of "HTML sections" to parse.
    :type sequence: list[str,bytes]

    :param str tag: (optional) see :class:`HTMLement` for details.
    :param dict attrs: (optional) see :class:`HTMLement` for details.
    :param str encoding: (optional) The encoding used for each section within *sequence*

    :return: The root element of the element tree.
    :rtype: xml.etree.ElementTree.Element

    :raises HTMLParseError: If parsing of "HTML document" fails.
    :raises UnicodeDecodeError: If decoding of a section within *sequence* fails.
    """
    parser = HTMLement(tag, attrs, encoding)
    for text in sequence:
        parser.feed(text)
    return parser.close()

def parse(source, tag=None, attrs=None, encoding=None):
    """
    Load an external HTML document into element tree.

    :param source: A filename or file like object containing HTML data.
    :type source: str, io.TextIOBase

    :param str tag: (optional) see :class:`HTMLement` for details.
    :param dict attrs: (optional) see :class:`HTMLement` for details.
    :param str encoding: (optional) The encoding used for *source*.

    :return: The root element of the element tree.
    :rtype: xml.etree.ElementTree.Element

    :raises HTMLParseError: If parsing of "HTML document" fails.
    :raises UnicodeDecodeError: If decoding of *source* fails.
    """
    # Assume that source is a file pointer if no read methods is found
    if hasattr(source, "read"):
        source = open(source, "rb")
        close_source = True
    else:
        close_source = False

    try:
        parser = HTMLement(tag, attrs, encoding)
        while True:
            # Read in 64k at a time
            data = source.read(65536)
            if not data:
                break

            # Feed the parser
            parser.feed(data)

        # Return the root element
        return parser.close()

    finally:
        if close_source:
            source.close()

# Required for raiseing HTMLParseError in python3, emulates python2
class HTMLParseError(Exception):
    """Exception raised for all parse errors."""
    def __init__(self, msg, position=(None, None)):
        self.msg = msg
        self.lineno = position[0]
        self.offset = position[1]

    def __str__(self):
        result = self.msg
        if self.lineno is not None:
            result += ", at line %d" % self.lineno
        if self.offset is not None:
            result += ", column %d" % self.offset
        return result

class HTMLement(object):
    """
    Python HTMLParser extension with ElementTree Parser support.

    This HTML Parser extends :class:`html.parser.HTMLParser`, returning an :class:`xml.etree.ElementTree.Element`
    instance. The returned root element natively supports the ElementTree API.
    (e.g. you may use its limited support for `XPath expressions`__)

    When a "tag" and "tag attributes" are given the parser will search for a required section. Only when the required
    section is found, does the parser start parsing the "HTML document". The element that matches the search criteria
    will then become the new "root element".

    :param tag: (optional) Name of "tag / element" which is used to filter down "the tree" to a required section.
    :type tag: str

    :param attrs: (optional) The attributes of the element, that will be used, when searching for the required section.
    :type attrs: dict

    :param encoding: (optional) Encoding used, when decoding the source data before feeding it to the parser.
    :type encoding: str

    :raises HTMLParseError: If parsing of "HTML document" fails.

    .. _Xpath: https://docs.python.org/3.6/library/xml.etree.elementtree.html#xpath-support
    __ XPath_
    """
    def __init__(self, tag="", attrs=None, encoding=""):
        self._parser = ParseHTML(tag, attrs)
        self.encoding = encoding
        self.finished = False

    def feed(self, data):
        """
        Feeds data to the parser.

        If *data*, is of "type `bytes` and where no encoding was specified, then the encoding
        will be extracted from *data* using "meta tags" if available.
        Otherwise encoding will default to "ISO-8859-1"

        :param data: HTML data
        :type data: str, bytes

        :raises UnicodeDecodeError: If decoding of *data* fails.
        """
        # Skip feeding data into parser if we already have what we want
        if self.finished is True:
            return None

        # Make sure that we have unicode before continuing
        if isinstance(data, bytes):
            if self.encoding:
                data = data.decode(self.encoding)
            else:
                data = self._make_unicode(data)

        # Parse the html document
        try:
            self._parser.feed(data)
        except EOFError:
            self.finished = True
            self._parser.reset()

    def close(self):
        """
        Close the "tree builder" and return the "root element" of the "element tree".

        :return: The "root element" of the "element tree".
        :rtype: xml.etree.ElementTree.Element

        :raises RuntimeError: If no element, matching search criteria, was found when a filter is given.
        """
        return self._parser.close()

    def _make_unicode(self, data):
        """
        Convert *data* from type `bytes` to type `str`.

        :param data: The html document.
        :type data: bytes

        :return: HTML data decoded into str(unicode).
        :rtype: str
        """
        # Atemp to find the encoding from the html source
        end_head_tag = data.find(b"</head>")
        if end_head_tag:
            # Search for the charset attribute within the meta tags
            charset_refind = b'<meta.+?charset=[\'"]*(.+?)["\'].*?>'
            charset = re.search(charset_refind, data[:end_head_tag], re.IGNORECASE)
            if charset:
                self.encoding = encoding = str(charset.group(1))
                return data.decode(encoding)

        # Decode the string into unicode using default encoding
        warn_msg = "Unable to determine encoding, defaulting to iso-8859-1"
        warnings.warn(warn_msg, UnicodeWarning, stacklevel=2)
        self.encoding = "iso-8859-1"
        return data.decode("iso-8859-1")

class ParseHTML(HTMLParser):
    def __init__(self, tag="", attrs=None):
        # Initiate HTMLParser
        HTMLParser.__init__(self)
        self.convert_charrefs = True
        self._root = None  # root element
        self._data = []  # data collector
        self._factory = Etree.Element
        self.enabled = not tag
        self._unw_attrs = []
        self.tag = tag

        # Split attributes into wanted and unwanted attributes
        if attrs:
            self.attrs = attrs
            for key, value in attrs.copy().items():
                if value is False:
                    self._unw_attrs.append(key)
                    del attrs[key]
        else:
            self.attrs = {}

        # Some tags in html do not require closing tags so thoes tags will need to be auto closed (Void elements)
        # Refer to: https://www.w3.org/TR/html/syntax.html#void-elements
        self._voids = frozenset(("area", "base", "br", "col", "hr", "img", "input", "link", "meta", "param",
                                 # Only in HTML5
                                 "embed", "keygen", "source", "track",
                                 # Not supported in HTML5
                                 "basefont", "frame", "isindex",
                                 # SVG self closing tags
                                 "rect", "circle", "ellipse", "line", "polyline", "polygon",
                                 "path", "stop", "use", "image", "animatetransform"))

        # Create temporary root element to protect from badly written sites that either
        # have no html starting tag or multiple top level elements
        elem = self._factory("html")
        self._elem = [elem]
        self._last = elem
        self._tail = 0

    def handle_starttag(self, tag, attrs):
        self._handle_starttag(tag, attrs, self_closing=tag in self._voids)

    def handle_startendtag(self, tag, attrs):
        self._handle_starttag(tag, attrs, self_closing=True)

    def _handle_starttag(self, tag, attrs, self_closing=False):
        enabled = self.enabled
        # Add tag element to tree if we have no filter or that the filter matches
        if enabled or self._search(tag, attrs):
            # Convert attrs to dictionary
            attrs = dict(attrs) if attrs else {}
            self._flush()

            # Create the new element
            elem = self._factory(tag, attrs)
            self._elem[-1].append(elem)
            self._last = elem

            # Only append the element to the list of elements if it's not a self closing element
            if self_closing:
                self._tail = 1
            else:
                self._elem.append(elem)
                self._tail = 0

            # Set this element as the root element when the filter search matches
            if not enabled:
                self._root = elem
                self.enabled = True

    def handle_endtag(self, tag):
        # Only process end tags when we have no filter or that the filter has been matched
        if self.enabled and tag not in self._voids:
            _elem = self._elem
            _root = self._root
            # Check that the closing tag is what's actualy expected
            if _elem[-1].tag == tag:
                self._flush()
                self._tail = 1
                self._last = elem = _elem.pop()
                if elem is _root:
                    raise EOFError

            # If the previous element is what we actually have then the expected element was not
            # properly closed so we must close that before closing what we have now
            elif len(_elem) >= 2 and _elem[-2].tag == tag:
                self._flush()
                self._tail = 1
                for _ in range(2):
                    self._last = elem = _elem.pop()
                    if elem is _root:
                        raise EOFError
            else:
                # Unable to match the tag to an element, ignoring it
                return None

    def handle_data(self, data):
        data = data.strip()
        if data and self.enabled:
            self._data.append(data)

    def handle_comment(self, data):
        data = data.strip()
        if data and self.enabled:
            elem = Etree.Comment(data)
            self._elem[-1].append(elem)

    def close(self):
        self._flush()
        if self.enabled is False:
            msg = "Unable to find requested section with tag of '{}' and attributes of {}"
            raise RuntimeError(msg.format(self.tag, self.attrs))
        elif self._root is not None:
            return self._root
        else:
            # Search the root element to find a proper html root element if one exists
            tmp_root = self._elem[0]
            proper_root = tmp_root.find("html")
            if proper_root is None:
                # Not proper root was found
                return tmp_root
            else:
                # Proper root found
                return proper_root

    def error(self, message):
        raise HTMLParseError(message, self.getpos())

    def _flush(self):
        if self._data:
            if self._last is not None:
                text = "".join(self._data)
                if self._tail:
                    self._last.tail = text
                else:
                    self._last.text = text
            self._data = []

    def _search(self, tag, attrs):
        # Only search when the tag matches
        if tag == self.tag:
            # If we have required attrs to match then search all attrs for wanted attrs
            # And also check that we do not have any attrs that are unwanted
            if self.attrs or self._unw_attrs:
                if attrs:
                    wanted_attrs = self.attrs.copy()
                    unwanted_attrs = self._unw_attrs
                    for key, value in attrs:
                        # Check for unwanted attrs
                        if key in unwanted_attrs:
                            return False

                        # Check for wanted attrs
                        elif key in wanted_attrs:
                            c_value = wanted_attrs[key]
                            if c_value == value or c_value is True:
                                # Remove this attribute from the wanted dict of attributes
                                # to indicate that this attribute has been found
                                del wanted_attrs[key]

                    # If wanted_attrs is now empty then all attributes must have been found
                    if not wanted_attrs:
                        return True
            else:
                # We only need to match tag
                return True

        # Unable to find required section
        return False
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Attempt to comunicate with Nitroshare.

 [x] listen nitroshare broadcasts and show who is available
   [x] listen for UDP packets on port 40816

"""

# --- communicating.. networking.. ---

import socket

IP4 = socket.AF_INET
UDP = socket.SOCK_DGRAM

class UDPSocketStream(object):
  """ Convert network socket endpoint to a readable stream object """
  def __init__(self, host='0.0.0.0', port=40816):
    # reading from socket blocks keyboard input, so CtrlC/CtrlBreak
    # may not work until read operation completes
    sock = socket.socket(IP4, UDP)
    # on socket level, allow multiple programs listen in the same address
    #sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    sock.bind((host, port))
    self.sock = sock

  def read(self, size):
      return self.sock.recvfrom(size)[0]

  def close(self):
    try:
      self.sock.shutdown(socket.SHUT_RDWR)
    except:
      pass
    self.sock.close()

# --- /networking ---

def nitrolisten():
  import json
  s = UDPSocketStream()

  while True:
    data, remote = s.sock.recvfrom(1024)
    #print("Got packet!")
    #print data
    # msg = {u'uuid': u'{eb8d3a1e-c50f-459d-9be7-6a70b91ca6bd}', u'operating_system': u'linux', u'name': u'XONiTE', u'port': u'40818'}
    msg = json.loads(data)
    print("Got response from %s (%s) on %s:%s " % (msg[u'name'], msg[u'operating_system'].capitalize(), remote[0], msg[u'port']))

if __name__ == '__main__':
  nitrolisten()
<commit_msg>net/udp/nitroshare.py: Remove hardcoded port from UDPSocketStream, making it 0 by default.<commit_after>#!/usr/bin/env python
"""
NitroShare server discovery.

https://github.com/nitroshare/nitroshare-desktop/wiki/Broadcast-Protocol

 [x] listen nitroshare broadcasts and show who is available
   [x] listen for UDP packets on port 40816

"""

# --- communicating.. networking.. ---

import socket

IP4 = socket.AF_INET
UDP = socket.SOCK_DGRAM

class UDPSocketStream(object):
  """ Convert network socket endpoint to a readable stream object """
  host = None
  port = None

  def __init__(self, host='0.0.0.0', port=0):
    """
    By default, listen on all interfaces on random port.
    """
    # reading from socket blocks keyboard input, so CtrlC/CtrlBreak
    # may not work until read operation completes
    sock = socket.socket(IP4, UDP)
    # on socket level, allow multiple programs listen in the same address
    #sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    sock.bind((host, port))
    self.sock = sock
    self.host, self.port = sock.getsockname()

  def read(self, size):
      return self.sock.recvfrom(size)[0]

  def close(self):
    try:
      self.sock.shutdown(socket.SHUT_RDWR)
    except:
      pass
    self.sock.close()

# --- /networking ---

def nitrolisten():
  import json
  s = UDPSocketStream(port=40816)
  print("Listening for NitroShare on %s:%s" % (s.host, s.port))

  while True:
    data, remote = s.sock.recvfrom(1024)
    #print("Got packet!")
    #print data
    # msg = {u'uuid': u'{eb8d3a1e-c50f-459d-9be7-6a70b91ca6bd}', u'operating_system': u'linux', u'name': u'XONiTE', u'port': u'40818'}
    msg = json.loads(data)
    print("Got response from %s (%s) %s:%s " % (msg[u'name'], msg[u'operating_system'].capitalize(), remote[0], msg[u'port']))

if __name__ == '__main__':
  nitrolisten()
<|endoftext|>


<commit_before>import cv2
from ev3control.rpc import Robot
from brain.controllers import move_to_brick_simple, move_to_brick_blind_and_grip, rotation_search_brick
from brain.core import State
from brain.core import main_loop

print("Creating robot...")

with Robot(cv2.VideoCapture(1)) as robot:
    robot.map = [(100, 100)]
    print("These are the robot motor positions before planning:", robot.left_track.position, robot.right_track.position)
    # Define the state graph, we can do this better, currently each method
    # returns the next state name
    states = [
        State(
            name="SEARCH",
            act=rotation_search_brick,
            default_args={}),
        State(
            name="MOVE_TO_BRICK",
            act=move_to_brick_simple,
            default_args={"atol" : 30}
        ),
        State(
            name="MOVE_TO_BRICK_BLIND_AND_GRIP",
            act=move_to_brick_blind_and_grip,
            default_args={}
        ),
        State(
            name="FINAL_STATE",
            act=lambda robot, frame: robot.reset()
        )
    ]
    print(states[0])
    state_dict = {}
    for state in states:
        state_dict[state.name] = state

    start_state = states[0]

    main_loop(robot, start_state, state_dict, delay=0.03)
<commit_msg>Added an additional parameter for error tolerance<commit_after>import cv2
from ev3control.rpc import Robot
from brain.controllers import move_to_brick_simple, move_to_brick_blind_and_grip, rotation_search_brick
from brain.core import State
from brain.core import main_loop

print("Creating robot...")

with Robot(cv2.VideoCapture(1)) as robot:
    robot.map = [(100, 100)]
    print("These are the robot motor positions before planning:", robot.left_track.position, robot.right_track.position)
    # Define the state graph, we can do this better, currently each method
    # returns the next state name
    states = [
        State(
            name="SEARCH",
            act=rotation_search_brick,
            default_args={}),
        State(
            name="MOVE_TO_BRICK",
            act=move_to_brick_simple,
            default_args={"atol": 30,
                          "atol_move_blind" : 30
                          }
        ),
        State(
            name="MOVE_TO_BRICK_BLIND_AND_GRIP",
            act=move_to_brick_blind_and_grip,
            default_args={}
        ),
        State(
            name="FINAL_STATE",
            act=lambda robot, frame: robot.reset()
        )
    ]
    print(states[0])
    state_dict = {}
    for state in states:
        state_dict[state.name] = state

    start_state = states[0]

    main_loop(robot, start_state, state_dict, delay=0.03)
<|endoftext|>


<commit_before>#!/usr/bin/env python

from ctypes import *
from ctypes.util import find_library
import sys
import os

# For unix the prefix 'lib' is not considered.
if find_library('svm'):
	libsvm = CDLL(find_library('svm'))
elif find_library('libsvm'):
	libsvm = CDLL(find_library('libsvm'))
else:
	for i, binary in enumerate((
	  "libsvm-win32.dll",     # 1) 32-bit Windows
	  "libsvm-mac32.so",      # 2) 32-bit Mac OS X
	  "libsvm-mac64.so",      # 3) 64-bit Mac OS X
	  "libsvm-ubuntu64.so")): # 4) 64-bit Linux Ubuntu
		try:
			libsvm = CDLL(os.path.join(os.path.dirname(__file__), binary)); break
		except OSError:
			if i == 4-1: # <= Adjust according to available binaries.
				raise ImportError, "can't import libsvm (%sbit-%s)" % (
					sizeof(c_voidp) * 8,
					sys.platform)

# Construct constants
SVM_TYPE = ['C_SVC', 'NU_SVC', 'ONE_CLASS', 'EPSILON_SVR', 'NU_SVR' ]
KERNEL_TYPE = ['LINEAR', 'POLY', 'RBF', 'SIGMOID', 'PRECOMPUTED']
for i, s in enumerate(SVM_TYPE): exec("%s = %d" % (s , i))
for i, s in enumerate(KERNEL_TYPE): exec("%s = %d" % (s , i))

PRINT_STRING_FUN = CFUNCTYPE(None, c_char_p)
def print_null(s):
	return

def genFields(names, types):
	return list(zip(names, types))

def fillprototype(f, restype, argtypes):
	f.restype = restype
	f.argtypes = argtypes

class svm_node(Structure):
	_names = ["index", "value"]
	_types = [c_int, c_double]
	_fields_ = genFields(_names, _types)

def gen_svm_nodearray(xi, feature_max=None, issparse=None):
	if isinstance(xi, dict):
		index_range = xi.keys()
	elif isinstance(xi, (list, tuple)):
		index_range = range(len(xi))
	else:
		raise TypeError('xi should be a dictionary, list or tuple')

	if feature_max:
		assert(isinstance(feature_max, int))
		index_range = filter(lambda j: j <= feature_max, index_range)
	if issparse:
		index_range = filter(lambda j:xi[j] != 0, index_range)

	index_range = sorted(index_range)
	ret = (svm_node * (len(index_range)+1))()
	ret[-1].index = -1
	for idx, j in enumerate(index_range):
		ret[idx].index = j
		ret[idx].value = xi[j]
	max_idx = 0
	if index_range:
		max_idx = index_range[-1]
	return ret, max_idx

class svm_problem(Structure):
	_names = ["l", "y", "x"]
	_types = [c_int, POINTER(c_double), POINTER(POINTER(svm_node))]
	_fields_ = genFields(_names, _types)

	def __init__(self, y, x):
		if len(y) != len(x):
			raise ValueError("len(y) != len(x)")
		self.l = l = len(y)

		max_idx = 0
		x_space = self.x_space = []
		for i, xi in enumerate(x):
			tmp_xi, tmp_idx = gen_svm_nodearray(xi)
			x_space += [tmp_xi]
			max_idx = max(max_idx, tmp_idx)
		self.n = max_idx

		self.y = (c_double * l)()
		for i, yi in enumerate(y): self.y[i] = yi

		self.x = (POINTER(svm_node) * l)()
		for i, xi in enumerate(self.x_space): self.x[i] = xi

class svm_parameter(Structure):
	_names = ["svm_type", "kernel_type", "degree", "gamma", "coef0",
			"cache_size", "eps", "C", "nr_weight", "weight_label", "weight",
			"nu", "p", "shrinking", "probability"]
	_types = [c_int, c_int, c_int, c_double, c_double,
			c_double, c_double, c_double, c_int, POINTER(c_int), POINTER(c_double),
			c_double, c_double, c_int, c_int]
	_fields_ = genFields(_names, _types)

	def __init__(self, options = None):
		if options == None:
			options = ''
		self.parse_options(options)

	def show(self):
		attrs = svm_parameter._names + self.__dict__.keys()
		values = map(lambda attr: getattr(self, attr), attrs)
		for attr, val in zip(attrs, values):
			print(' %s: %s' % (attr, val))

	def set_to_default_values(self):
		self.svm_type = C_SVC;
		self.kernel_type = RBF
		self.degree = 3
		self.gamma = 0
		self.coef0 = 0
		self.nu = 0.5
		self.cache_size = 100
		self.C = 1
		self.eps = 0.001
		self.p = 0.1
		self.shrinking = 1
		self.probability = 0
		self.nr_weight = 0
		self.weight_label = (c_int*0)()
		self.weight = (c_double*0)()
		self.cross_validation = False
		self.nr_fold = 0
		self.print_func = None

	def parse_options(self, options):
		argv = options.split()
		self.set_to_default_values()
		self.print_func = cast(None, PRINT_STRING_FUN)
		weight_label = []
		weight = []

		i = 0
		while i < len(argv):
			if argv[i] == "-s":
				i = i + 1
				self.svm_type = int(argv[i])
			elif argv[i] == "-t":
				i = i + 1
				self.kernel_type = int(argv[i])
			elif argv[i] == "-d":
				i = i + 1
				self.degree = int(argv[i])
			elif argv[i] == "-g":
				i = i + 1
				self.gamma = float(argv[i])
			elif argv[i] == "-r":
				i = i + 1
				self.coef0 = float(argv[i])
			elif argv[i] == "-n":
				i = i + 1
				self.nu = float(argv[i])
			elif argv[i] == "-m":
				i = i + 1
				self.cache_size = float(argv[i])
			elif argv[i] == "-c":
				i = i + 1
				self.C = float(argv[i])
			elif argv[i] == "-e":
				i = i + 1
				self.eps = float(argv[i])
			elif argv[i] == "-p":
				i = i + 1
				self.p = float(argv[i])
			elif argv[i] == "-h":
				i = i + 1
				self.shrinking = int(argv[i])
			elif argv[i] == "-b":
				i = i + 1
				self.probability = int(argv[i])
			elif argv[i] == "-q":
				self.print_func = PRINT_STRING_FUN(print_null)
			elif argv[i] == "-v":
				i = i + 1
				self.cross_validation = 1
				self.nr_fold = int(argv[i])
				if self.nr_fold < 2:
					raise ValueError("n-fold cross validation: n must >= 2")
			elif argv[i].startswith("-w"):
				i = i + 1
				self.nr_weight += 1
				nr_weight = self.nr_weight
				weight_label += [int(argv[i-1][2:])]
				weight += [float(argv[i])]
			else:
				raise ValueError("Wrong options")
			i += 1

		libsvm.svm_set_print_string_function(self.print_func)
		self.weight_label = (c_int*self.nr_weight)()
		self.weight = (c_double*self.nr_weight)()
		for i in range(self.nr_weight):
			self.weight[i] = weight[i]
			self.weight_label[i] = weight_label[i]

class svm_model(Structure):
	_names = ['param', 'nr_class', 'l', 'SV', 'sv_coef', 'rho',
			'probA', 'probB', 'label', 'nSV', 'free_sv']
	_types = [svm_parameter, c_int, c_int, POINTER(POINTER(svm_node)),
			POINTER(POINTER(c_double)), POINTER(c_double),
			POINTER(c_double), POINTER(c_double), POINTER(c_int),
			POINTER(c_int), c_int]
	_fields_ = genFields(_names, _types)

	def __init__(self):
		self.__createfrom__ = 'python'

	def __del__(self):
		# free memory created by C to avoid memory leak
		if hasattr(self, '__createfrom__') and self.__createfrom__ == 'C':
			try: libsvm.svm_free_and_destroy_model(pointer(self))
			except:
				pass

	def get_svm_type(self):
		return libsvm.svm_get_svm_type(self)

	def get_nr_class(self):
		return libsvm.svm_get_nr_class(self)

	def get_svr_probability(self):
		return libsvm.svm_get_svr_probability(self)

	def get_labels(self):
		nr_class = self.get_nr_class()
		labels = (c_int * nr_class)()
		libsvm.svm_get_labels(self, labels)
		return labels[:nr_class]

	def is_probability_model(self):
		return (libsvm.svm_check_probability_model(self) == 1)

	def get_sv_coef(self):
		return [tuple(self.sv_coef[j][i] for j in xrange(self.nr_class - 1))
				for i in xrange(self.l)]

	def get_SV(self):
		result = []
		for sparse_sv in self.SV[:self.l]:
			row = dict()

			i = 0
			while True:
				row[sparse_sv[i].index] = sparse_sv[i].value
				if sparse_sv[i].index == -1:
					break
				i += 1

			result.append(row)
		return result

def toPyModel(model_ptr):
	"""
	toPyModel(model_ptr) -> svm_model

	Convert a ctypes POINTER(svm_model) to a Python svm_model
	"""
	if bool(model_ptr) == False:
		raise ValueError("Null pointer")
	m = model_ptr.contents
	m.__createfrom__ = 'C'
	return m

fillprototype(libsvm.svm_train, POINTER(svm_model), [POINTER(svm_problem), POINTER(svm_parameter)])
fillprototype(libsvm.svm_cross_validation, None, [POINTER(svm_problem), POINTER(svm_parameter), c_int, POINTER(c_double)])

fillprototype(libsvm.svm_save_model, c_int, [c_char_p, POINTER(svm_model)])
fillprototype(libsvm.svm_load_model, POINTER(svm_model), [c_char_p])

fillprototype(libsvm.svm_get_svm_type, c_int, [POINTER(svm_model)])
fillprototype(libsvm.svm_get_nr_class, c_int, [POINTER(svm_model)])
fillprototype(libsvm.svm_get_labels, None, [POINTER(svm_model), POINTER(c_int)])
fillprototype(libsvm.svm_get_svr_probability, c_double, [POINTER(svm_model)])

fillprototype(libsvm.svm_predict_values, c_double, [POINTER(svm_model), POINTER(svm_node), POINTER(c_double)])
fillprototype(libsvm.svm_predict, c_double, [POINTER(svm_model), POINTER(svm_node)])
fillprototype(libsvm.svm_predict_probability, c_double, [POINTER(svm_model), POINTER(svm_node), POINTER(c_double)])

fillprototype(libsvm.svm_free_model_content, None, [POINTER(svm_model)])
fillprototype(libsvm.svm_free_and_destroy_model, None, [POINTER(POINTER(svm_model))])
fillprototype(libsvm.svm_destroy_param, None, [POINTER(svm_parameter)])

fillprototype(libsvm.svm_check_parameter, c_char_p, [POINTER(svm_problem), POINTER(svm_parameter)])
fillprototype(libsvm.svm_check_probability_model, c_int, [POINTER(svm_model)])
fillprototype(libsvm.svm_set_print_string_function, None, [PRINT_STRING_FUN])
<commit_msg>svmlib additional comments<commit_after>#!/usr/bin/env python

from ctypes import *
from ctypes.util import find_library
import sys
import os

# For unix the prefix 'lib' is not considered.
if find_library('svm'):
	libsvm = CDLL(find_library('svm'))
elif find_library('libsvm'):
	libsvm = CDLL(find_library('libsvm'))
else:
	for i, binary in enumerate((
	  # If you have OS X 32-bit, you need a 32-bit Python and libsvm-mac32.so.
	  # If you have OS X 32-bit with 64-bit Python,
	  # it will try to load	libsvm-mac64.so which fails since OS X is 32-bit.
	  # It won't load libsvm-mac32.so since Python is 64-bit.
	  "libsvm-win32.dll",     # 1) 32-bit Windows
	  "libsvm-mac32.so",      # 2) 32-bit Mac OS X
	  "libsvm-mac64.so",      # 3) 64-bit Mac OS X
	  "libsvm-ubuntu64.so")): # 4) 64-bit Linux Ubuntu
		try:
			libsvm = CDLL(os.path.join(os.path.dirname(__file__), binary)); break
		except OSError, e:
			if i == 4-1: # <= Adjust according to available binaries.
				raise ImportError, "can't import libsvm (%sbit-%s)" % (
					sizeof(c_voidp) * 8,
					sys.platform)

# Construct constants
SVM_TYPE = ['C_SVC', 'NU_SVC', 'ONE_CLASS', 'EPSILON_SVR', 'NU_SVR' ]
KERNEL_TYPE = ['LINEAR', 'POLY', 'RBF', 'SIGMOID', 'PRECOMPUTED']
for i, s in enumerate(SVM_TYPE): exec("%s = %d" % (s , i))
for i, s in enumerate(KERNEL_TYPE): exec("%s = %d" % (s , i))

PRINT_STRING_FUN = CFUNCTYPE(None, c_char_p)
def print_null(s):
	return

def genFields(names, types):
	return list(zip(names, types))

def fillprototype(f, restype, argtypes):
	f.restype = restype
	f.argtypes = argtypes

class svm_node(Structure):
	_names = ["index", "value"]
	_types = [c_int, c_double]
	_fields_ = genFields(_names, _types)

def gen_svm_nodearray(xi, feature_max=None, issparse=None):
	if isinstance(xi, dict):
		index_range = xi.keys()
	elif isinstance(xi, (list, tuple)):
		index_range = range(len(xi))
	else:
		raise TypeError('xi should be a dictionary, list or tuple')

	if feature_max:
		assert(isinstance(feature_max, int))
		index_range = filter(lambda j: j <= feature_max, index_range)
	if issparse:
		index_range = filter(lambda j:xi[j] != 0, index_range)

	index_range = sorted(index_range)
	ret = (svm_node * (len(index_range)+1))()
	ret[-1].index = -1
	for idx, j in enumerate(index_range):
		ret[idx].index = j
		ret[idx].value = xi[j]
	max_idx = 0
	if index_range:
		max_idx = index_range[-1]
	return ret, max_idx

class svm_problem(Structure):
	_names = ["l", "y", "x"]
	_types = [c_int, POINTER(c_double), POINTER(POINTER(svm_node))]
	_fields_ = genFields(_names, _types)

	def __init__(self, y, x):
		if len(y) != len(x):
			raise ValueError("len(y) != len(x)")
		self.l = l = len(y)

		max_idx = 0
		x_space = self.x_space = []
		for i, xi in enumerate(x):
			tmp_xi, tmp_idx = gen_svm_nodearray(xi)
			x_space += [tmp_xi]
			max_idx = max(max_idx, tmp_idx)
		self.n = max_idx

		self.y = (c_double * l)()
		for i, yi in enumerate(y): self.y[i] = yi

		self.x = (POINTER(svm_node) * l)()
		for i, xi in enumerate(self.x_space): self.x[i] = xi

class svm_parameter(Structure):
	_names = ["svm_type", "kernel_type", "degree", "gamma", "coef0",
			"cache_size", "eps", "C", "nr_weight", "weight_label", "weight",
			"nu", "p", "shrinking", "probability"]
	_types = [c_int, c_int, c_int, c_double, c_double,
			c_double, c_double, c_double, c_int, POINTER(c_int), POINTER(c_double),
			c_double, c_double, c_int, c_int]
	_fields_ = genFields(_names, _types)

	def __init__(self, options = None):
		if options == None:
			options = ''
		self.parse_options(options)

	def show(self):
		attrs = svm_parameter._names + self.__dict__.keys()
		values = map(lambda attr: getattr(self, attr), attrs)
		for attr, val in zip(attrs, values):
			print(' %s: %s' % (attr, val))

	def set_to_default_values(self):
		self.svm_type = C_SVC;
		self.kernel_type = RBF
		self.degree = 3
		self.gamma = 0
		self.coef0 = 0
		self.nu = 0.5
		self.cache_size = 100
		self.C = 1
		self.eps = 0.001
		self.p = 0.1
		self.shrinking = 1
		self.probability = 0
		self.nr_weight = 0
		self.weight_label = (c_int*0)()
		self.weight = (c_double*0)()
		self.cross_validation = False
		self.nr_fold = 0
		self.print_func = None

	def parse_options(self, options):
		argv = options.split()
		self.set_to_default_values()
		self.print_func = cast(None, PRINT_STRING_FUN)
		weight_label = []
		weight = []

		i = 0
		while i < len(argv):
			if argv[i] == "-s":
				i = i + 1
				self.svm_type = int(argv[i])
			elif argv[i] == "-t":
				i = i + 1
				self.kernel_type = int(argv[i])
			elif argv[i] == "-d":
				i = i + 1
				self.degree = int(argv[i])
			elif argv[i] == "-g":
				i = i + 1
				self.gamma = float(argv[i])
			elif argv[i] == "-r":
				i = i + 1
				self.coef0 = float(argv[i])
			elif argv[i] == "-n":
				i = i + 1
				self.nu = float(argv[i])
			elif argv[i] == "-m":
				i = i + 1
				self.cache_size = float(argv[i])
			elif argv[i] == "-c":
				i = i + 1
				self.C = float(argv[i])
			elif argv[i] == "-e":
				i = i + 1
				self.eps = float(argv[i])
			elif argv[i] == "-p":
				i = i + 1
				self.p = float(argv[i])
			elif argv[i] == "-h":
				i = i + 1
				self.shrinking = int(argv[i])
			elif argv[i] == "-b":
				i = i + 1
				self.probability = int(argv[i])
			elif argv[i] == "-q":
				self.print_func = PRINT_STRING_FUN(print_null)
			elif argv[i] == "-v":
				i = i + 1
				self.cross_validation = 1
				self.nr_fold = int(argv[i])
				if self.nr_fold < 2:
					raise ValueError("n-fold cross validation: n must >= 2")
			elif argv[i].startswith("-w"):
				i = i + 1
				self.nr_weight += 1
				nr_weight = self.nr_weight
				weight_label += [int(argv[i-1][2:])]
				weight += [float(argv[i])]
			else:
				raise ValueError("Wrong options")
			i += 1

		libsvm.svm_set_print_string_function(self.print_func)
		self.weight_label = (c_int*self.nr_weight)()
		self.weight = (c_double*self.nr_weight)()
		for i in range(self.nr_weight):
			self.weight[i] = weight[i]
			self.weight_label[i] = weight_label[i]

class svm_model(Structure):
	_names = ['param', 'nr_class', 'l', 'SV', 'sv_coef', 'rho',
			'probA', 'probB', 'label', 'nSV', 'free_sv']
	_types = [svm_parameter, c_int, c_int, POINTER(POINTER(svm_node)),
			POINTER(POINTER(c_double)), POINTER(c_double),
			POINTER(c_double), POINTER(c_double), POINTER(c_int),
			POINTER(c_int), c_int]
	_fields_ = genFields(_names, _types)

	def __init__(self):
		self.__createfrom__ = 'python'

	def __del__(self):
		# free memory created by C to avoid memory leak
		if hasattr(self, '__createfrom__') and self.__createfrom__ == 'C':
			try: libsvm.svm_free_and_destroy_model(pointer(self))
			except:
				pass

	def get_svm_type(self):
		return libsvm.svm_get_svm_type(self)

	def get_nr_class(self):
		return libsvm.svm_get_nr_class(self)

	def get_svr_probability(self):
		return libsvm.svm_get_svr_probability(self)

	def get_labels(self):
		nr_class = self.get_nr_class()
		labels = (c_int * nr_class)()
		libsvm.svm_get_labels(self, labels)
		return labels[:nr_class]

	def is_probability_model(self):
		return (libsvm.svm_check_probability_model(self) == 1)

	def get_sv_coef(self):
		return [tuple(self.sv_coef[j][i] for j in xrange(self.nr_class - 1))
				for i in xrange(self.l)]

	def get_SV(self):
		result = []
		for sparse_sv in self.SV[:self.l]:
			row = dict()

			i = 0
			while True:
				row[sparse_sv[i].index] = sparse_sv[i].value
				if sparse_sv[i].index == -1:
					break
				i += 1

			result.append(row)
		return result

def toPyModel(model_ptr):
	"""
	toPyModel(model_ptr) -> svm_model

	Convert a ctypes POINTER(svm_model) to a Python svm_model
	"""
	if bool(model_ptr) == False:
		raise ValueError("Null pointer")
	m = model_ptr.contents
	m.__createfrom__ = 'C'
	return m

fillprototype(libsvm.svm_train, POINTER(svm_model), [POINTER(svm_problem), POINTER(svm_parameter)])
fillprototype(libsvm.svm_cross_validation, None, [POINTER(svm_problem), POINTER(svm_parameter), c_int, POINTER(c_double)])

fillprototype(libsvm.svm_save_model, c_int, [c_char_p, POINTER(svm_model)])
fillprototype(libsvm.svm_load_model, POINTER(svm_model), [c_char_p])

fillprototype(libsvm.svm_get_svm_type, c_int, [POINTER(svm_model)])
fillprototype(libsvm.svm_get_nr_class, c_int, [POINTER(svm_model)])
fillprototype(libsvm.svm_get_labels, None, [POINTER(svm_model), POINTER(c_int)])
fillprototype(libsvm.svm_get_svr_probability, c_double, [POINTER(svm_model)])

fillprototype(libsvm.svm_predict_values, c_double, [POINTER(svm_model), POINTER(svm_node), POINTER(c_double)])
fillprototype(libsvm.svm_predict, c_double, [POINTER(svm_model), POINTER(svm_node)])
fillprototype(libsvm.svm_predict_probability, c_double, [POINTER(svm_model), POINTER(svm_node), POINTER(c_double)])

fillprototype(libsvm.svm_free_model_content, None, [POINTER(svm_model)])
fillprototype(libsvm.svm_free_and_destroy_model, None, [POINTER(POINTER(svm_model))])
fillprototype(libsvm.svm_destroy_param, None, [POINTER(svm_parameter)])

fillprototype(libsvm.svm_check_parameter, c_char_p, [POINTER(svm_problem), POINTER(svm_parameter)])
fillprototype(libsvm.svm_check_probability_model, c_int, [POINTER(svm_model)])
fillprototype(libsvm.svm_set_print_string_function, None, [PRINT_STRING_FUN])
<|endoftext|>


<commit_before>#!/usr/bin/env python3

from distutils.sysconfig import get_python_lib
import glob
import os
import shutil
import subprocess
import tarfile
import tempfile
import zipfile

import util

script_tag = "[OT PyEnv Builder] "
script_tab = "                   "

# The project_root_dir depends on the location of this file, so it cannot be
# moved without updating this line
project_root_dir = \
    os.path.dirname(                                  # going up 1 level
        os.path.dirname(os.path.realpath(__file__)))  # folder dir of this

ENV_WORKPATH = os.path.join(project_root_dir, "env-build")
ENV_DISTPATH = os.path.join(project_root_dir, "env-dist")

IS_WIN = util.get_os() == 'win'

def remove_directory(dir_to_remove):
    """ :param dir_to_remove: Directory to remove. """
    if os.path.exists(dir_to_remove):
        print(script_tab + "Removing directory %s" % dir_to_remove)
        shutil.rmtree(dir_to_remove)
    else:
        print(script_tab + "Directory %s was not found." % dir_to_remove)

def alter_activate_script():
    """
    """
    print(script_tab + "Altering VIRUTALENV activate script")

    if IS_WIN:
        virtualenv_scripts_dir = 'Scripts'
    else:
        virtualenv_scripts_dir = 'bin'

    path_to_activate_scripts = glob.glob(
        os.path.join(ENV_WORKPATH, virtualenv_scripts_dir, 'activate*')
    )

    print(script_tab + "found these scripts:", path_to_activate_scripts)

    get_ext = lambda x: os.path.splitext(x)[-1]

    unix_activate_scripts = list(
        filter(lambda x: get_ext(x) == '', path_to_activate_scripts)
    )
    windows_activate_scripts = list(
        filter(lambda x: get_ext(x) == '.bat', path_to_activate_scripts)
    )

    print(script_tab + "Detected the following unix activate scripts: ",
            unix_activate_scripts)
    print(script_tab + "Detected the following windows activate scripts: ",
            windows_activate_scripts)

    def alter(path, alter_val):
        new_activate = path + '.bak'
        new_activate_file = open(new_activate, 'w')
        with open(path) as old_activate:
            for i, line in enumerate(old_activate):
                line = line.replace(ENV_WORKPATH, alter_val)
                new_activate_file.write(line)
        new_activate_file.close()
        os.remove(path)
        os.rename(new_activate, path)

    list(map(lambda x: alter(x, '$1'), unix_activate_scripts))
    list(map(lambda x: alter(x, '%1'), windows_activate_scripts))

def build_environment():
    """Runs virtual env in a subprocess to create the environment and makes
    environment relocatable
    """
    if util.get_os() == 'win':
        # Note: Sys python must be python3 for this to work
        subprocess.call(
            ["virtualenv", ENV_WORKPATH, "--always-copy"]
        )
    else:
        subprocess.call(
            ["virtualenv", ENV_WORKPATH, "-p", "python3", "--always-copy"]
        )

    subprocess.call(["virtualenv", ENV_WORKPATH, "--relocatable"])

def copy_site_packages():
    """Copy python site to virtualenv
    """
    import logging
    python_std_lib_path = os.path.normpath(
        os.path.join(logging.__file__, '..', '..')
    )
    if IS_WIN:
        sys_python = shutil.which('python')
        sys_python = os.path.normpath(
            os.path.join(os.path.realpath(sys_python), '..')
        )
    else:
        sys_python = shutil.which('python3')
        sys_python = os.path.normpath(
            os.path.join(os.path.realpath(sys_python), '..', '..')
        )

    def make_dirs(path):
        dir_path = os.path.splitext(path)[0]
        if not os.path.exists(dir_path):
            os.makedirs(dir_path)

    def copy(src, dest):
        for item in os.listdir(src):
            if item in ('site-packages', 'Headers', '__pycache__'):
                continue
            s = os.path.join(src, item)
            d = os.path.join(dest, item)

            if os.path.realpath(s) == os.path.realpath(src):
                continue

            if os.path.isdir(s):
                # shutil.copytree(s, d)
                copy(s, d)
            else:
                if os.path.exists(d):
                    continue
                make_dirs(d)
                shutil.copy(s, d)
    copy(python_std_lib_path, os.path.join(ENV_WORKPATH, 'lib'))
    copy(sys_python, os.path.join(ENV_WORKPATH))

def compress_virtual_env(path):
    """Zips environment located in ENV_WORKPATH into path
    """
    old_cwd = os.getcwd()
    os.chdir(ENV_WORKPATH)

    # Tar virtualenv in order to keep permissions, flags, etc
    tar_name = os.path.join(tempfile.gettempdir(), 'venv.tar')
    tar_output = tarfile.open(tar_name, 'w')
    tar_output.add('.')
    tar_output.close()

    # Zip tar file
    os.chdir(os.path.dirname(tar_name))
    zip_name = os.path.join(path, 'venv.zip')
    zip_output = zipfile.ZipFile(zip_name, 'w', zipfile.ZIP_DEFLATED)
    zip_output.write(os.path.basename(tar_name))
    zip_output.close()

    # Restore working dir
    os.chdir(old_cwd)

def copy_virtual_env_to_release_dir():
    """
    Copies artifact "ENV_DISTPATH/venv.zip" to "releases/venv-{build-tag}.zip"
    """
    tag = util.get_build_tag(util.get_os())
    shutil.copy(
        os.path.join(ENV_DISTPATH, 'venv.zip'),
        os.path.join(project_root_dir, 'releases', 'venv-{}.zip'.format(tag))
    )

def copy_virtual_env_to_app():
    """
    Copies artifact "ENV_DISTPATH/venv.zip" to "releases/venv-{build-tag}.zip"
    """
    shutil.copy(
        os.path.join(ENV_DISTPATH, 'venv.zip'),
        os.path.join(
            project_root_dir, 'app', 'backend-env', util.get_os(), 'venv.zip')
    )

def create_env():
    """Creates standalone, distributable python virtual environment
    and zips it
    """
    print(script_tag + "Creating virtual environment in: {}".format(ENV_WORKPATH))
    remove_directory(ENV_WORKPATH)
    (not os.path.exists(ENV_WORKPATH) and os.mkdir(ENV_WORKPATH))

    print(script_tag + "Running virtualenv cmd in subprocess.")
    build_environment()

    print(script_tag + "Copying python site-packages to virtualenv")
    copy_site_packages()

    print(
        script_tag +\
        "Altering virtualenv activate script "
        "to take env path as an argument"
    )
    alter_activate_script()

    print(script_tag + "Compress virtualenv environment")
    (not os.path.exists(ENV_DISTPATH) and os.mkdir(ENV_DISTPATH))
    compress_virtual_env(ENV_DISTPATH)

    print(script_tag + "Copying virtualenv to release dir")
    copy_virtual_env_to_release_dir()

    # print(script_tag + "Copying virtualenv to app")
    # copy_virtual_env_to_app()

if __name__ == '__main__':
    create_env()
<commit_msg>remove commented code<commit_after>#!/usr/bin/env python3

from distutils.sysconfig import get_python_lib
import glob
import os
import shutil
import subprocess
import tarfile
import tempfile
import zipfile

import util

script_tag = "[OT PyEnv Builder] "
script_tab = "                   "

# The project_root_dir depends on the location of this file, so it cannot be
# moved without updating this line
project_root_dir = \
    os.path.dirname(                                  # going up 1 level
        os.path.dirname(os.path.realpath(__file__)))  # folder dir of this

ENV_WORKPATH = os.path.join(project_root_dir, "env-build")
ENV_DISTPATH = os.path.join(project_root_dir, "env-dist")

IS_WIN = util.get_os() == 'win'

def remove_directory(dir_to_remove):
    """ :param dir_to_remove: Directory to remove. """
    if os.path.exists(dir_to_remove):
        print(script_tab + "Removing directory %s" % dir_to_remove)
        shutil.rmtree(dir_to_remove)
    else:
        print(script_tab + "Directory %s was not found." % dir_to_remove)

def alter_activate_script():
    """
    """
    print(script_tab + "Altering VIRUTALENV activate script")

    if IS_WIN:
        virtualenv_scripts_dir = 'Scripts'
    else:
        virtualenv_scripts_dir = 'bin'

    path_to_activate_scripts = glob.glob(
        os.path.join(ENV_WORKPATH, virtualenv_scripts_dir, 'activate*')
    )

    print(script_tab + "found these scripts:", path_to_activate_scripts)

    get_ext = lambda x: os.path.splitext(x)[-1]

    unix_activate_scripts = list(
        filter(lambda x: get_ext(x) == '', path_to_activate_scripts)
    )
    windows_activate_scripts = list(
        filter(lambda x: get_ext(x) == '.bat', path_to_activate_scripts)
    )

    print(script_tab + "Detected the following unix activate scripts: ",
            unix_activate_scripts)
    print(script_tab + "Detected the following windows activate scripts: ",
            windows_activate_scripts)

    def alter(path, alter_val):
        new_activate = path + '.bak'
        new_activate_file = open(new_activate, 'w')
        with open(path) as old_activate:
            for i, line in enumerate(old_activate):
                line = line.replace(ENV_WORKPATH, alter_val)
                new_activate_file.write(line)
        new_activate_file.close()
        os.remove(path)
        os.rename(new_activate, path)

    list(map(lambda x: alter(x, '$1'), unix_activate_scripts))
    list(map(lambda x: alter(x, '%1'), windows_activate_scripts))

def build_environment():
    """Runs virtual env in a subprocess to create the environment and makes
    environment relocatable
    """
    if util.get_os() == 'win':
        # Note: Sys python must be python3 for this to work
        subprocess.call(
            ["virtualenv", ENV_WORKPATH, "--always-copy"]
        )
    else:
        subprocess.call(
            ["virtualenv", ENV_WORKPATH, "-p", "python3", "--always-copy"]
        )

    subprocess.call(["virtualenv", ENV_WORKPATH, "--relocatable"])

def copy_site_packages():
    """Copy python site to virtualenv
    """
    import logging
    python_std_lib_path = os.path.normpath(
        os.path.join(logging.__file__, '..', '..')
    )
    if IS_WIN:
        sys_python = shutil.which('python')
        sys_python = os.path.normpath(
            os.path.join(os.path.realpath(sys_python), '..')
        )
    else:
        sys_python = shutil.which('python3')
        sys_python = os.path.normpath(
            os.path.join(os.path.realpath(sys_python), '..', '..')
        )

    def make_dirs(path):
        dir_path = os.path.splitext(path)[0]
        if not os.path.exists(dir_path):
            os.makedirs(dir_path)

    def copy(src, dest):
        for item in os.listdir(src):
            if item in ('site-packages', 'Headers', '__pycache__'):
                continue
            s = os.path.join(src, item)
            d = os.path.join(dest, item)

            if os.path.realpath(s) == os.path.realpath(src):
                continue

            if os.path.isdir(s):
                # shutil.copytree(s, d)
                copy(s, d)
            else:
                if os.path.exists(d):
                    continue
                make_dirs(d)
                shutil.copy(s, d)
    copy(python_std_lib_path, os.path.join(ENV_WORKPATH, 'lib'))
    copy(sys_python, os.path.join(ENV_WORKPATH))

def compress_virtual_env(path):
    """Zips environment located in ENV_WORKPATH into path
    """
    old_cwd = os.getcwd()
    os.chdir(ENV_WORKPATH)

    # Tar virtualenv in order to keep permissions, flags, etc
    tar_name = os.path.join(tempfile.gettempdir(), 'venv.tar')
    tar_output = tarfile.open(tar_name, 'w')
    tar_output.add('.')
    tar_output.close()

    # Zip tar file
    os.chdir(os.path.dirname(tar_name))
    zip_name = os.path.join(path, 'venv.zip')
    zip_output = zipfile.ZipFile(zip_name, 'w', zipfile.ZIP_DEFLATED)
    zip_output.write(os.path.basename(tar_name))
    zip_output.close()

    # Restore working dir
    os.chdir(old_cwd)

def copy_virtual_env_to_release_dir():
    """
    Copies artifact "ENV_DISTPATH/venv.zip" to "releases/venv-{build-tag}.zip"
    """
    tag = util.get_build_tag(util.get_os())
    shutil.copy(
        os.path.join(ENV_DISTPATH, 'venv.zip'),
        os.path.join(project_root_dir, 'releases', 'venv-{}.zip'.format(tag))
    )

def create_env():
    """Creates standalone, distributable python virtual environment
    and zips it
    """
    print(script_tag + "Creating virtual environment in: {}".format(ENV_WORKPATH))
    remove_directory(ENV_WORKPATH)
    (not os.path.exists(ENV_WORKPATH) and os.mkdir(ENV_WORKPATH))

    print(script_tag + "Running virtualenv cmd in subprocess.")
    build_environment()

    print(script_tag + "Copying python site-packages to virtualenv")
    copy_site_packages()

    print(
        script_tag +\
        "Altering virtualenv activate script "
        "to take env path as an argument"
    )
    alter_activate_script()

    print(script_tag + "Compress virtualenv environment")
    (not os.path.exists(ENV_DISTPATH) and os.mkdir(ENV_DISTPATH))
    compress_virtual_env(ENV_DISTPATH)

    print(script_tag + "Copying virtualenv to release dir")
    copy_virtual_env_to_release_dir()

if __name__ == '__main__':
    create_env()
<|endoftext|>


<commit_before><commit_msg>Can't monitor queue size from input plugin<commit_after><|endoftext|>


<commit_before>"""Script used to define the paystack Subaccount class."""

from paystackapi.base import PayStackBase

class SubAccount(PayStackBase):
    """docstring for SubAccount."""

    @classmethod
    def create(cls, **kwargs):
        """
        Function defined to create subaccount.

        Args:
            business_name: name of business for subaccount
            settlement_bank: name of Bank (accepted banks)
            account_number: NUBAN Bank Account number
            percentage_charge: default percentage charged on subaccount?
            **kwargs

        Returns:
            Json data from paystack API.
        """
        return cls().requests.post('subaccount', data=kwargs,)

    @classmethod
	def list(cls, **kwargs):
		"""
		List subaccounts

		Args:
			perPage: records you want to retrieve per page (Integer)
			page: what page you want to retrieve (Integer)

		Returns:
		JSON data from paystack's API.
		"""

		return cls().requests.get('subaccount', data=kwargs)

    @classmethod
    def fetch(cls, id_or_slug):
        """
        Get a single subaccount by id or slug.

        Args:
            id_or_slug: id or subaccount_code

        Returns:
            Json data from paystack API.
        """
        return cls().requests.get(f"subaccount/{id_or_slug}")

    @classmethod
    def update(cls, id_or_slug, **kwargs):
        """
        Update a single subaccount by id or slug.

        Args:
            id_or_slug: id or subaccount_code
            business_name: name of business for subaccount
            settlement_bank: name of Bank (accepted banks)
            account_number: NUBAN Bank Account number
            percentage_charge: default percentage charged on subaccount?
            **kwargs

        Returns:
            Json data from paystack API.
        """
        return cls().requests.get(f"subaccount/{id_or_slug}")
<commit_msg>fix inconsistent use of tabs and spaces<commit_after>"""Script used to define the paystack Subaccount class."""

from paystackapi.base import PayStackBase

class SubAccount(PayStackBase):
    """docstring for SubAccount."""

    @classmethod
    def create(cls, **kwargs):
        """
        Function defined to create subaccount.

        Args:
            business_name: name of business for subaccount
            settlement_bank: name of Bank (accepted banks)
            account_number: NUBAN Bank Account number
            percentage_charge: default percentage charged on subaccount?
            **kwargs

        Returns:
            Json data from paystack API.
        """
        return cls().requests.post('subaccount', data=kwargs,)

    @classmethod
    def list(cls, **kwargs):
        """
        List subaccounts

        Args:
            perPage: records you want to retrieve per page (Integer)
            page: what page you want to retrieve (Integer)

        Returns:
        JSON data from paystack's API.
        """

        return cls().requests.get('subaccount', data=kwargs)

    @classmethod
    def fetch(cls, id_or_slug):
        """
        Get a single subaccount by id or slug.

        Args:
            id_or_slug: id or subaccount_code

        Returns:
            Json data from paystack API.
        """
        return cls().requests.get(f"subaccount/{id_or_slug}")

    @classmethod
    def update(cls, id_or_slug, **kwargs):
        """
        Update a single subaccount by id or slug.

        Args:
            id_or_slug: id or subaccount_code
            business_name: name of business for subaccount
            settlement_bank: name of Bank (accepted banks)
            account_number: NUBAN Bank Account number
            percentage_charge: default percentage charged on subaccount?
            **kwargs

        Returns:
            Json data from paystack API.
        """
        return cls().requests.get(f"subaccount/{id_or_slug}")
<|endoftext|>