<commit_before>#! /usr/bin/python3

import sys #Arguments gathering, exit script, ...
from Bio import SeqIO #Biopython, use to read fasta file sequences
import time
from memory_profiler import memory_usage #Use to get the memory usages. /!\ You may need to install memory_profiler.
import gc #garbage collector. Use to call it only at good moments

'''
usage: ./estimate_mergeScore_2_seq.py <file.fasta>
This script is used to test algorithms to give an estimate percent score
between 2 sequences.
It wil compare each pair of a set of sequences (given by a fasta file)
The main goal is to find a suitable algorithm for a GPU
usage. Therefore, a fairly precise algorithm is required (the reverse
complement is not an obligation as we can re-calculate it),
using as little memory as possible (the computation time is less important).
The ideal algorithm would not care if the beginning of the first read and the
ending of the second one are not merged with the other read if a lot of the
reads are merged, example :
 -read1: CCCGATTCA
 -merge:  ||||| ||
 -read2:  CCGATCCAAT
 -score: high percent
'''

def main(seqs):
	# Correct the seqs
	nbSeqs = len(seqs)
	#print("\n==== Sequences ====")
	for s in range(nbSeqs):
		seqs[s] = seqs[s].upper() #To have only upper cases
		#print("[-- SEQ "+str(s)+" --]\n"+seqs[s])

	#Run algorithms on each couple
	gc.disable() # Disable automatic garbage collector
	#print("\n==== Algorithms run ====")
	print("algo\tseqID1\tseqID2\tscore (%)\tmem (MB)\ttime (s)")

	for i in range(nbSeqs):
		for j in range(nbSeqs):
			coupleDescript = "\t"+str(i)+"\t"+str(j)+"\t"
			#needleman based
			print("needle"+coupleDescript+analyzeFct(needle,seqs[i],seqs[j]))

			#dot_cut
			print("dot_cut"+coupleDescript+analyzeFct(dot_cut,seqs[i],seqs[j]))

'''
Tool function to run an cmp function and get (and output) its informations
as time, value, etc...
input: function, seq1, seq2
output: string "score(%)\tmem(?)\ttime(s)"
'''
def analyzeFct(fct, seq1, seq2):
	# Execute and get delta time
	t0 = time.time()
	score = fct(seq1, seq2)
	t1 = time.time()

	# Get memory usage during the function exection
	gc.collect() #Run garbage collector
	mem = memory_usage((fct,(seq1,seq2)),interval=0.1)

	return str(round(score,2))+"\t"+str(round(max(mem),2))+"\t"+str(round(t1-t0,2))

'''
Algorithm 'Needleman&Wunsch'
Based on an needleman principe: each case is the best of
 - score[i-1][j-1] + (seq1[i]==seq2[j]?1:-1) #Match / mismatch
 - score[i-1][j]-1 #Indel
 - score[i][j-1] #Indel
The first nuc of seq2 is either a match of score 1 or a mismatch of score -1: the gap before this on the seq1 doesn't count.
After the end seq1, indels on seq2 doesn't count.
The best score is the best of the last score line and the last column
'''
def needle(seq1, seq2):
	# Get sequences length
	len1 = len(seq1)
	len2 = len(seq2)

	# Prepare scoring matrix[len1][len2]
	matrix = [[0 for j in range(len2)] for i in range(len1)]

	# Fill first column: first nuc of seq2 against all seq1 => gap is allowed, only match/mismatch of score 1/-1
	for i in range(len1): matrix[i][0] = 1 if seq1[i]==seq2[0] else -1

	# Fill first line: it is a gap so the -1 increase
	for j in range(1,len2): matrix[0][j] = -j

	#Fill the remaining of the matrix
	#Keep the best possibilities with the end of seq2 (seq2 is inside seq1)
	bestIN=0
	for i in range(1,len1):
		for j in range(1,len2):
			# Match / mismatch ?
			match = matrix[i-1][j-1] + (1 if seq1[i]==seq2[j] else -1)

			# indel ?
			i1 = matrix[i-1][j]-1
			i2 = matrix[i][j-1]-1

			# keep best
			matrix[i][j] = max(match, i1, i2)

		# Best seq2 inside seq 1 ?
		if(matrix[i][len2-1] > bestIN): bestIN = matrix[i][len2-1]

	# Get the list of best concurrents
	best_co = matrix[len1-1][:]
	for i in range(len1):
		best_co.append(matrix[i][-1])
	#Add the se2 is in seq1 possibility
	best_co.append(bestIN)

	# Get score
	return 100*max(best_co)/min(len1,len2)

''''
Algorithm 'Needle3Arrays
Same algorithm but using a score matrix of size (seq1.size(),2)
as we only need the previous line to calculate the current one.
'''

'''
Algorithm 'dot_cut'.
The principe is to increase the score when the merge is contiguous (diagonal
comparaison) but an difference make a score=0 :
	IF seq1[i] == seq2[j]: score_actual[j] = score_previous[j-1]+1
	ELSE: score_actual[j] = 0
	...do stuff...
	score_previous = score_actual

Then the final percent score is the sum of higher scores before a difference
compared to a perfect match.
This way you have only to memorize 2 'comparaison' column score and the best
scores.
When a part of seq2 has been 'scored', only check the remaining (on the right)
of seq2 to avoid to count a same parts multiple times and to do "come-backs".
'''

def dot_cut(seq1, seq2):
	#Get sequences length
	len1 = len(seq1)
	len2 = len(seq2)

	# Init the scores arrays
	previous = [0]*len2
	actual = [0]*len2
	previous_best = 0
	best_scores =  list()
	begin_len2 = 0 #From which nuc of len2 the cmp start

	# Do cmp
	for i in range(len1):
		#Cmp seq2 to actual nuc of seq1
		for j in range(begin_len2, len2):
			if seq1[i] == seq2[j]:
				if j>0: actual[j] = previous[j-1]+1
				else: actual[j] = 1
			else: actual[j]=0

		# Get the best. Do we have a drop ?
		best = max(actual)

		if best < previous_best:
			#Best found ! (the previous_best ;) )
			best_scores.append(previous_best)

			#Now, start on the next len2 nuc from the last nuc of the
			#previous_best for further check
			begin_len2 = previous.index(previous_best)+1

			#Clear the buffers
			best = 0
			actual = [0]*len2

		previous_best = best

		#Move the scores buffers
		previous = actual
		actual = [0]*len2

	# Add the last best score
	best_scores.append(previous_best)

	# Results
	return 100*sum(best_scores)/min(len1,len2)

'''
Python will run the following part at start.
It gets the sequences then run the main function.
'''

# Run the main function
if __name__ == "__main__":
	# Get the argument (the fasta file)
	if len(sys.argv) != 2:
		print("Expect only one argument : the fasta file of sequences")
		sys.exit()

	#Read fasta file
	seqs = []
	for record in  SeqIO.parse(open(sys.argv[1]), 'fasta'):
		seqs.append(record.seq)

	# Run the main function
	main(seqs)

<commit_msg>Ajout de l'algo need2a qui est une amélioration mémoire du needle<commit_after>#! /usr/bin/python3

import sys #Arguments gathering, exit script, ...
from Bio import SeqIO #Biopython, use to read fasta file sequences
import time
from memory_profiler import memory_usage #Use to get the memory usages. /!\ You may need to install memory_profiler.
import gc #garbage collector. Use to call it only at good moments

'''
usage: ./estimate_mergeScore_2_seq.py <file.fasta>
This script is used to test algorithms to give an estimate percent score
between 2 sequences.
It wil compare each pair of a set of sequences (given by a fasta file)
The main goal is to find a suitable algorithm for a GPU
usage. Therefore, a fairly precise algorithm is required (the reverse
complement is not an obligation as we can re-calculate it),
using as little memory as possible (the computation time is less important).
The ideal algorithm would not care if the beginning of the first read and the
ending of the second one are not merged with the other read if a lot of the
reads are merged, example :
 -read1: CCCGATTCA
 -merge:  ||||| ||
 -read2:  CCGATCCAAT
 -score: high percent
'''

def main(seqs):
	# Correct the seqs
	nbSeqs = len(seqs)
	#print("\n==== Sequences ====")
	for s in range(nbSeqs):
		seqs[s] = seqs[s].upper() #To have only upper cases
		#print("[-- SEQ "+str(s)+" --]\n"+seqs[s])

	#Run algorithms on each couple
	gc.disable() # Disable automatic garbage collector
	#print("\n==== Algorithms run ====")
	print("algo\tseqID1\tseqID2\tscore (%)\tmem (MB)\ttime (s)")

	for i in range(nbSeqs):
		for j in range(nbSeqs):
			coupleDescript = "\t"+str(i)+"\t"+str(j)+"\t"
			#needleman based
			print("needle"+coupleDescript+analyzeFct(needle,seqs[i],seqs[j]))

			#needleman 2 arrays based
			print("need2a"+coupleDescript+analyzeFct(need2arrays,seqs[i],seqs[j]))

			#dot_cut
			print("dot_cut"+coupleDescript+analyzeFct(dot_cut,seqs[i],seqs[j]))

'''
Tool function to run an cmp function and get (and output) its informations
as time, value, etc...
input: function, seq1, seq2
output: string "score(%)\tmem(?)\ttime(s)"
'''
def analyzeFct(fct, seq1, seq2):
	# Execute and get delta time
	t0 = time.time()
	score = fct(seq1, seq2)
	t1 = time.time()

	# Get memory usage during the function exection
	gc.collect() #Run garbage collector
	mem = memory_usage(proc=(fct,(seq1,seq2)),interval=0.1)

	return str(round(score,2))+"\t"+str(round(max(mem),2))+"\t"+str(round(t1-t0,2))

'''
Algorithm 'Needleman&Wunsch'
Based on an needleman principe: each case is the best of
 - score[i-1][j-1] + (seq1[i]==seq2[j]?1:-1) #Match / mismatch
 - score[i-1][j]-1 #Indel
 - score[i][j-1] #Indel
The first nuc of seq2 is either a match of score 1 or a mismatch of score -1: the gap before this on the seq1 doesn't count.
After the end seq1, indels on seq2 doesn't count.
The best score is the best of the last score line and the last column
'''
def needle(seq1, seq2):
	# Get sequences length
	len1 = len(seq1)
	len2 = len(seq2)

	# Prepare scoring matrix[len1][len2]
	matrix = [[0 for j in range(len2)] for i in range(len1)]

	# Fill first column: first nuc of seq2 against all seq1 => gap is allowed, only match/mismatch of score 1/-1
	for i in range(len1): matrix[i][0] = 1 if seq1[i]==seq2[0] else -1

	# Fill first line: it is a gap so the -1 increase
	for j in range(1,len2): matrix[0][j] = -j

	#Fill the remaining of the matrix
	#Keep the best possibilities with the end of seq2 (seq2 is inside seq1)
	bestIN=0
	for i in range(1,len1):
		for j in range(1,len2):
			# Match / mismatch ?
			match = matrix[i-1][j-1] + (1 if seq1[i]==seq2[j] else -1)

			# indel ?
			i1 = matrix[i-1][j]-1
			i2 = matrix[i][j-1]-1

			# keep best
			matrix[i][j] = max(match, i1, i2)

		# Best seq2 inside seq 1 ?
		if(matrix[i][len2-1] > bestIN): bestIN = matrix[i][len2-1]

	# Get the list of best concurrents
	best_co = matrix[len1-1][:]
	for i in range(len1):
		best_co.append(matrix[i][-1])
	#Add the se2 is in seq1 possibility
	best_co.append(bestIN)

	# Get score
	return 100*max(best_co)/min(len1,len2)

'''
Algorithm 'Need2Arrays
Same algorithm (needleman&wunsch) but using a score matrix of size (seq1.size(),2)
as we only need the previous line to calculate the current one.
'''
def need2arrays(seq1, seq2):
	#Get sequences length
	len1 = len(seq1)
	len2 = len(seq2)

	# Prepare scoring arrays
	previous = [0]*len2
	current = [0]*len2

	# bestIN is the best case of seq2 being INSIDE seq1

	# Calculate the first line (seq2 against seq1[0])
	# Note: we can't 'allow' a gap in seq2 'first nuc'
	for j in range(0,len2): previous[j] = -j
	bestIN = previous[len2-1]

	# Complete the needle
	for i in range(1, len1):
		# Test first nuc, a gap is allowed So it's just a (mis)match test
		current[0] = (1 if seq1[i]==seq2[0] else -1)

		# Finish the line
		for j in range(1,len2):
			scores=[0,0,0]
			# Match/mismatch ?
			scores[0] = previous[j-1] + (1 if seq1[i]==seq2[j] else -1)
			#Indels ?
			scores[1] = previous[j]-1
			scores[2] = current[j-1]-1
			#keep best
			current[j]=max(scores)

		# Best seq2 inside possibility ?
		if(current[j] > bestIN): bestIN=current[j]

		# move buffers
		previous = current
		current = [0]*len2

	# Get the best possibilities score which is inside the last line + the bestIN (pseudo last column)
	previous.append(bestIN) #Previous is last because we moved buffers. We add bestIN to it
	return 100*max(previous)/min(len1,len2) #Percent and normalize by theorical max best score (full match)

'''
Algorithm 'dot_cut'.
The principe is to increase the score when the merge is contiguous (diagonal
comparaison) but an difference make a score=0 :
	IF seq1[i] == seq2[j]: score_actual[j] = score_previous[j-1]+1
	ELSE: score_actual[j] = 0
	...do stuff...
	score_previous = score_actual

Then the final percent score is the sum of higher scores before a difference
compared to a perfect match.
This way you have only to memorize 2 'comparaison' column score and the best
scores.
When a part of seq2 has been 'scored', only check the remaining (on the right)
of seq2 to avoid to count a same parts multiple times and to do "come-backs".
'''

def dot_cut(seq1, seq2):
	#Get sequences length
	len1 = len(seq1)
	len2 = len(seq2)

	# Init the scores arrays
	previous = [0]*len2
	actual = [0]*len2
	previous_best = 0
	best_scores =  list()
	begin_len2 = 0 #From which nuc of len2 the cmp start

	# Do cmp
	for i in range(len1):
		#Cmp seq2 to actual nuc of seq1
		for j in range(begin_len2, len2):
			if seq1[i] == seq2[j]:
				if j>0: actual[j] = previous[j-1]+1
				else: actual[j] = 1
			else: actual[j]=0

		# Get the best. Do we have a drop ?
		best = max(actual)

		if best < previous_best:
			#Best found ! (the previous_best ;) )
			best_scores.append(previous_best)

			#Now, start on the next len2 nuc from the last nuc of the
			#previous_best for further check
			begin_len2 = previous.index(previous_best)+1

			#Clear the buffers
			best = 0
			actual = [0]*len2

		previous_best = best

		#Move the scores buffers
		previous = actual
		actual = [0]*len2

	# Add the last best score
	best_scores.append(previous_best)

	# Results
	return 100*sum(best_scores)/min(len1,len2)

'''
Python will run the following part at start.
It gets the sequences then run the main function.
'''

# Run the main function
if __name__ == "__main__":
	# Get the argument (the fasta file)
	if len(sys.argv) != 2:
		print("Expect only one argument : the fasta file of sequences")
		sys.exit()

	#Read fasta file
	seqs = []
	for record in  SeqIO.parse(open(sys.argv[1]), 'fasta'):
		seqs.append(record.seq)

	# Run the main function
	main(seqs)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

class Conflict(Exception):
    pass

class GenericError(Exception):
    pass

class NotFound(Exception):
    pass

class BadRequest(Exception):
    pass

class AuthenticationFailed(Exception):
    pass
<commit_msg>Improved exception hierarchy.<commit_after># -*- coding: utf-8 -*-

class Error(Exception):
    pass

class UnexpectedError(Error):
    pass

class FeedReaderExited(Error):
    pass

class ApiError(Error):
    pass

class GenericError(ApiError):
    pass

class Conflict(ApiError):
    pass

class NotFound(ApiError):
    pass

class BadRequest(ApiError):
    pass

class AuthenticationFailed(ApiError):
    pass
<|endoftext|>


<commit_before>import time
from operator import attrgetter
from pyethereum.db import EphemDB
from pyethereum.utils import privtoaddr, sha3
import rlp
from rlp.utils import decode_hex, encode_hex
from pyethereum import blocks
from pyethereum import processblock
from pyethereum import peermanager
from pyethereum.transactions import Transaction
from pyethereum.miner import Miner
from pyethereum.synchronizer import Synchronizer
from pyethereum.peer import MAX_GET_CHAIN_SEND_HASHES
from pyethereum.peer import MAX_GET_CHAIN_REQUEST_BLOCKS
from pyethereum.slogging import get_logger
from pyethereum.chain import Chain
from devp2p.service import WiredService
import eth_protocol
log = get_logger('eth.chainservice')

rlp_hash_hex = lambda data: encode_hex(sha3(rlp.encode(data)))

NUM_BLOCKS_PER_REQUEST = 256  # MAX_GET_CHAIN_REQUEST_BLOCKS

class ChainService(WiredService):

    """
    Manages the chain and requests to it.
    """
    # required by BaseService
    name = 'chain'
    default_config = dict(chain=dict(coinbase=privtoaddr(sha3('cow'))))

    # required by WiredService
    wire_protocol = eth_protocol.ETHProtocol  # create for each peer

    # initialized after configure:
    chain = None
    genesis = None
    miner = None
    synchronizer = None
    config = None

    def __init__(self, app):
        self.config = app.config
        self.db = app.services.db
        assert self.db is not None
        super(ChainService, self).__init__(app)
        log.info('initializing chain')
        self.chain = Chain(self.db, new_head_cb=self._on_new_head)
        self.new_miner()
        self.synchronizer = Synchronizer(self)

    def _on_new_head(self, block):
        self.new_miner()  # reset mining
        # if we are not syncing, forward all blocks
        if not self.synchronizer.synchronization_tasks:
            log.debug("broadcasting new head", block=block)
            # signals.broadcast_new_block.send(sender=None, block=block)

    def loop_body(self):
        ts = time.time()
        pct_cpu = self.config['misc']['mining']
        if pct_cpu > 0:
            self.mine()
            delay = (time.time() - ts) * (100. / pct_cpu - 1)
            assert delay >= 0
            time.sleep(min(delay, 1.))
        else:
            time.sleep(.01)

    def new_miner(self):
        "new miner is initialized if HEAD is updated"
        if not self.config:
            return  # not configured yet

        # prepare uncles
        uncles = set(self.chain.get_uncles(self.chain.head))
        blk = self.chain.head
        for i in range(8):
            for u in blk.uncles:  # assuming uncle headers
                u = sha3(rlp.encode(u))
                if u in self:
                    uncles.discard(self.chain.get(u))
            if blk.has_parent():
                blk = blk.get_parent()

        coinbase = self.config['chain']['coinbase']
        uncles = list(uncles)  # FIXME VERY MUCH !!!
        miner = Miner(self.chain.head, uncles, coinbase)
        if self.miner:
            for tx in self.miner.get_transactions():
                miner.add_transaction(tx)
        self.miner = miner

    def mine(self):
        with self.lock:
            block = self.miner.mine()
            if block:
                # create new block
                if not self.chain.add_block(block):
                    log.debug("newly mined block is invalid!?", block_hash=block)
                    self.new_miner()

    def receive_chain(self, transient_blocks, peer=None):
        with self.lock:
            _db = EphemDB()
            # assuming to receive chain order w/ oldest block first
            transient_blocks.sort(key=attrgetter('number'))
            assert transient_blocks[0].number <= transient_blocks[-1].number

            # notify syncer
            self.synchronizer.received_blocks(peer, transient_blocks)

            for t_block in transient_blocks:  # oldest to newest
                log.debug('Checking PoW', block_hash=t_block.hash)
                if not t_block.header.check_pow(_db):
                    log.debug('Invalid PoW', block_hash=t_block.hash)
                    continue
                log.debug('Deserializing', block_hash=t_block.hash)
                try:
                    block = blocks.Block(t_block.header, t_block.transaction_list, t_block.uncles,
                                         db=self.chain.db)
                except processblock.InvalidTransaction as e:
                    # FIXME there might be another exception in
                    # blocks.deserializeChild when replaying transactions
                    # if this fails, we need to rewind state
                    log.debug('invalid transaction', block_hash=t_block, error=e)
                    # stop current syncing of this chain and skip the child blocks
                    self.synchronizer.stop_synchronization(peer)
                    return
                except blocks.UnknownParentException:
                    if t_block.prevhash == blocks.GENESIS_PREVHASH:
                        log.debug('Rec Incompatible Genesis', block_hash=t_block)
                        if peer:
                            peer.send_Disconnect(reason='Wrong genesis block')
                    else:  # should be a single newly mined block
                        assert t_block.prevhash not in self
                        assert t_block.prevhash != self.chain.genesis.hash
                        log.debug('unknown parent', block_hash=t_block,
                                  parent_hash=encode_hex(t_block.prevhash), remote_id=peer)
                        if len(transient_blocks) != 1:
                            # strange situation here.
                            # we receive more than 1 block, so it's not a single newly mined one
                            # sync/network/... failed to add the needed parent at some point
                            # well, this happens whenever we can't validate a block!
                            # we should disconnect!
                            log.warn(
                                'blocks received, but unknown parent.', num=len(transient_blocks))
                        if peer:
                            # request chain for newest known hash
                            self.synchronizer.synchronize_unknown_block(
                                peer, transient_blocks[-1].hash)
                    break
                if block.hash in self.chain:
                    log.debug('known', block_hash=block)
                else:
                    assert block.has_parent()
                    # assume single block is newly mined block
                    success = self.chain.add_block(block)
                    if success:
                        log.debug('added', block_hash=block)

    def add_transaction(self, transaction):
        _log = log.bind(tx_hash=transaction)
        _log.debug("add transaction")
        with self.lock:
            res = self.miner.add_transaction(transaction)
            if res:
                _log.debug("broadcasting valid")
                #signals.send_local_transactions.send(sender=None, transactions=[transaction])
            return res

    def get_transactions(self):
        log.debug("get_transactions called")
        return self.miner.get_transactions()

    # wire protocol receivers ###########

    def on_peer_handshake(self, proto):
        log.debug('on_peer_handshake', proto=proto)
        assert isinstance(proto, self.wire_protocol)
        proto.receive_status_callbacks.append(self.receive_status)
        # proto.receive_transactions_callbacks.append(self.receive_transactions)
        # proto.receive_getblockhashes_callbacks.append(self.receive_getblockhashes)
        # proto.receive_blockhashes_callbacks.append(self.receive_blockhashes)
        # proto.receive_getblocks_callbacks.append(self.receive_getblocks)
        # proto.receive_blocks_callbacks.append(self.receive_blocks)
        # proto.receive_newblock_callbacks.append(self.receive_newblock)

        # send status
        head = self.chain.head
        proto.send_status(total_difficulty=head.chain_difficulty(), chain_head_hash=head.hash,
                          genesis_hash=self.chain.genesis.hash)

    def receive_status(self, proto, eth_version, network_id, total_difficulty, chain_head_hash, genesis_hash):
        log.debug('status received', proto=proto, eth_version=eth_version)

    def on_peer_disconnect(self, proto):
        assert isinstance(proto, self.wire_protocol)

    def handle_get_block_hashes(self, proto, block_hash, count, peer, **kwargs):
        _log = log.bind(block_hash=encode_hex(block_hash))
        _log.debug("handle_get_block_hashes", count=count)
        max_hashes = min(count, MAX_GET_CHAIN_SEND_HASHES)
        found = []
        if block_hash not in self.chain:
            log.debug("unknown block")
            peer.send_BlockHashes([])
        last = self.chain.get(block_hash)
        while len(found) < max_hashes:
            if last.has_parent():
                last = last.get_parent()
                found.append(last.hash)
            else:
                break
        _log.debug("sending: found block_hashes", count=len(found))
        peer.send_BlockHashes(found)

    def handle_get_blocks(self, proto, block_hashes, peer, **kwargs):
        log.debug("handle_get_blocks", count=len(block_hashes))
        found = []
        for bh in block_hashes[:MAX_GET_CHAIN_REQUEST_BLOCKS]:
            if bh in self.chain:
                found.append(self.chain.get(bh))
            else:
                log.debug("unknown block requested", block_hash=encode_hex(bh))
        log.debug("found", count=len(found))
        peer.send_Blocks(found)

    def peer_status_received(self, proto, genesis_hash, peer, **kwargs):
        log.debug("received status", remote_id=peer, genesis_hash=encode_hex(genesis_hash))
        # check genesis
        if genesis_hash != self.chain.genesis.hash:
            return peer.send_Disconnect(reason='wrong genesis block')

        # request chain
        self.synchronizer.synchronize_status(
            peer, peer.status_head_hash, peer.status_total_difficulty)
        # send transactions
        log.debug("sending transactions", remote_id=peer)
        transactions = self.get_transactions()
        transactions = [rlp.decode(x.serialize()) for x in transactions]
        peer.send_Transactions(transactions)

    def peer_handshake(self, proto, peer, **kwargs):
        # reply with status if not yet sent
        if peer.has_ethereum_capabilities() and not peer.status_sent:
            log.debug("handshake, sending status", remote_id=peer)
            peer.send_Status(self.chainhead.hash, self.chain.head.chain_difficulty(),
                             self.chain.genesis.hash)
        else:
            log.debug("handshake, but peer has no 'eth' capablities", remote_id=peer)

    def remote_transactions_received_handler(self, proto, transactions, peer, **kwargs):
        "receives rlp.decoded serialized"
        txl = [Transaction.deserialize(rlp.encode(tx)) for tx in transactions]
        log.debug('remote_transactions_received', count=len(txl), remote_id=peer)
        for tx in txl:
            peermanager.txfilter.add(tx, peer)  # FIXME
            self.add_transaction(tx)

    def local_transaction_received_handler(self, proto, transaction, **kwargs):
        "receives transaction object"
        log.debug('local_transaction_received', tx_hash=transaction)
        self.add_transaction(transaction)

    def new_block_received_handler(self, proto, block, peer, **kwargs):
        log.debug("recv new remote block", block_hash=block, remote_id=peer)
        self.receive_chain([block], peer)

    def remote_blocks_received_handler(self, proto, transient_blocks, peer, **kwargs):
        log.debug("recv remote blocks", count=len(transient_blocks), remote_id=peer,
                  highest_number=max(x.number for x in transient_blocks))
        if transient_blocks:
            self.receive_chain(transient_blocks, peer)

    def remote_block_hashes_received_handler(self, proto, block_hashes, peer, **kwargs):
        if block_hashes:
            log.debug("recv remote block_hashes", count=len(block_hashes), remote_id=peer,
                      first=encode_hex(block_hashes[0]), last=encode_hex(block_hashes[-1]))
        else:
            log.debug("recv 0 remore block hashes, signifying genesis block")
        self.synchronizer.received_block_hashes(peer, block_hashes)
<commit_msg>rename on_wire_protocol_start/stop<commit_after>import time
from operator import attrgetter
from pyethereum.db import EphemDB
from pyethereum.utils import privtoaddr, sha3
import rlp
from rlp.utils import decode_hex, encode_hex
from pyethereum import blocks
from pyethereum import processblock
from pyethereum import peermanager
from pyethereum.transactions import Transaction
from pyethereum.miner import Miner
from pyethereum.synchronizer import Synchronizer
from pyethereum.peer import MAX_GET_CHAIN_SEND_HASHES
from pyethereum.peer import MAX_GET_CHAIN_REQUEST_BLOCKS
from pyethereum.slogging import get_logger
from pyethereum.chain import Chain
from devp2p.service import WiredService
import eth_protocol
log = get_logger('eth.chainservice')

rlp_hash_hex = lambda data: encode_hex(sha3(rlp.encode(data)))

NUM_BLOCKS_PER_REQUEST = 256  # MAX_GET_CHAIN_REQUEST_BLOCKS

class ChainService(WiredService):

    """
    Manages the chain and requests to it.
    """
    # required by BaseService
    name = 'chain'
    default_config = dict(chain=dict(coinbase=privtoaddr(sha3('cow'))))

    # required by WiredService
    wire_protocol = eth_protocol.ETHProtocol  # create for each peer

    # initialized after configure:
    chain = None
    genesis = None
    miner = None
    synchronizer = None
    config = None

    def __init__(self, app):
        self.config = app.config
        self.db = app.services.db
        assert self.db is not None
        super(ChainService, self).__init__(app)
        log.info('initializing chain')
        self.chain = Chain(self.db, new_head_cb=self._on_new_head)
        self.new_miner()
        self.synchronizer = Synchronizer(self)

    def _on_new_head(self, block):
        self.new_miner()  # reset mining
        # if we are not syncing, forward all blocks
        if not self.synchronizer.synchronization_tasks:
            log.debug("broadcasting new head", block=block)
            # signals.broadcast_new_block.send(sender=None, block=block)

    def loop_body(self):
        ts = time.time()
        pct_cpu = self.config['misc']['mining']
        if pct_cpu > 0:
            self.mine()
            delay = (time.time() - ts) * (100. / pct_cpu - 1)
            assert delay >= 0
            time.sleep(min(delay, 1.))
        else:
            time.sleep(.01)

    def new_miner(self):
        "new miner is initialized if HEAD is updated"
        if not self.config:
            return  # not configured yet

        # prepare uncles
        uncles = set(self.chain.get_uncles(self.chain.head))
        blk = self.chain.head
        for i in range(8):
            for u in blk.uncles:  # assuming uncle headers
                u = sha3(rlp.encode(u))
                if u in self:
                    uncles.discard(self.chain.get(u))
            if blk.has_parent():
                blk = blk.get_parent()

        coinbase = self.config['chain']['coinbase']
        uncles = list(uncles)  # FIXME VERY MUCH !!!
        miner = Miner(self.chain.head, uncles, coinbase)
        if self.miner:
            for tx in self.miner.get_transactions():
                miner.add_transaction(tx)
        self.miner = miner

    def mine(self):
        with self.lock:
            block = self.miner.mine()
            if block:
                # create new block
                if not self.chain.add_block(block):
                    log.debug("newly mined block is invalid!?", block_hash=block)
                    self.new_miner()

    def receive_chain(self, transient_blocks, peer=None):
        with self.lock:
            _db = EphemDB()
            # assuming to receive chain order w/ oldest block first
            transient_blocks.sort(key=attrgetter('number'))
            assert transient_blocks[0].number <= transient_blocks[-1].number

            # notify syncer
            self.synchronizer.received_blocks(peer, transient_blocks)

            for t_block in transient_blocks:  # oldest to newest
                log.debug('Checking PoW', block_hash=t_block.hash)
                if not t_block.header.check_pow(_db):
                    log.debug('Invalid PoW', block_hash=t_block.hash)
                    continue
                log.debug('Deserializing', block_hash=t_block.hash)
                try:
                    block = blocks.Block(t_block.header, t_block.transaction_list, t_block.uncles,
                                         db=self.chain.db)
                except processblock.InvalidTransaction as e:
                    # FIXME there might be another exception in
                    # blocks.deserializeChild when replaying transactions
                    # if this fails, we need to rewind state
                    log.debug('invalid transaction', block_hash=t_block, error=e)
                    # stop current syncing of this chain and skip the child blocks
                    self.synchronizer.stop_synchronization(peer)
                    return
                except blocks.UnknownParentException:
                    if t_block.prevhash == blocks.GENESIS_PREVHASH:
                        log.debug('Rec Incompatible Genesis', block_hash=t_block)
                        if peer:
                            peer.send_Disconnect(reason='Wrong genesis block')
                    else:  # should be a single newly mined block
                        assert t_block.prevhash not in self
                        assert t_block.prevhash != self.chain.genesis.hash
                        log.debug('unknown parent', block_hash=t_block,
                                  parent_hash=encode_hex(t_block.prevhash), remote_id=peer)
                        if len(transient_blocks) != 1:
                            # strange situation here.
                            # we receive more than 1 block, so it's not a single newly mined one
                            # sync/network/... failed to add the needed parent at some point
                            # well, this happens whenever we can't validate a block!
                            # we should disconnect!
                            log.warn(
                                'blocks received, but unknown parent.', num=len(transient_blocks))
                        if peer:
                            # request chain for newest known hash
                            self.synchronizer.synchronize_unknown_block(
                                peer, transient_blocks[-1].hash)
                    break
                if block.hash in self.chain:
                    log.debug('known', block_hash=block)
                else:
                    assert block.has_parent()
                    # assume single block is newly mined block
                    success = self.chain.add_block(block)
                    if success:
                        log.debug('added', block_hash=block)

    def add_transaction(self, transaction):
        _log = log.bind(tx_hash=transaction)
        _log.debug("add transaction")
        with self.lock:
            res = self.miner.add_transaction(transaction)
            if res:
                _log.debug("broadcasting valid")
                #signals.send_local_transactions.send(sender=None, transactions=[transaction])
            return res

    def get_transactions(self):
        log.debug("get_transactions called")
        return self.miner.get_transactions()

    # wire protocol receivers ###########

    def on_wire_protocol_start(self, proto):
        log.debug('on_peer_handshake', proto=proto)
        assert isinstance(proto, self.wire_protocol)
        proto.receive_status_callbacks.append(self.receive_status)
        # proto.receive_transactions_callbacks.append(self.receive_transactions)
        # proto.receive_getblockhashes_callbacks.append(self.receive_getblockhashes)
        # proto.receive_blockhashes_callbacks.append(self.receive_blockhashes)
        # proto.receive_getblocks_callbacks.append(self.receive_getblocks)
        # proto.receive_blocks_callbacks.append(self.receive_blocks)
        # proto.receive_newblock_callbacks.append(self.receive_newblock)

        # send status
        head = self.chain.head
        proto.send_status(total_difficulty=head.chain_difficulty(), chain_head_hash=head.hash,
                          genesis_hash=self.chain.genesis.hash)

    def receive_status(self, proto, eth_version, network_id, total_difficulty, chain_head_hash, genesis_hash):
        log.debug('status received', proto=proto, eth_version=eth_version)

    def on_peer_disconnect(self, proto):
        assert isinstance(proto, self.wire_protocol)

    def handle_get_block_hashes(self, proto, block_hash, count, peer, **kwargs):
        _log = log.bind(block_hash=encode_hex(block_hash))
        _log.debug("handle_get_block_hashes", count=count)
        max_hashes = min(count, MAX_GET_CHAIN_SEND_HASHES)
        found = []
        if block_hash not in self.chain:
            log.debug("unknown block")
            peer.send_BlockHashes([])
        last = self.chain.get(block_hash)
        while len(found) < max_hashes:
            if last.has_parent():
                last = last.get_parent()
                found.append(last.hash)
            else:
                break
        _log.debug("sending: found block_hashes", count=len(found))
        peer.send_BlockHashes(found)

    def handle_get_blocks(self, proto, block_hashes, peer, **kwargs):
        log.debug("handle_get_blocks", count=len(block_hashes))
        found = []
        for bh in block_hashes[:MAX_GET_CHAIN_REQUEST_BLOCKS]:
            if bh in self.chain:
                found.append(self.chain.get(bh))
            else:
                log.debug("unknown block requested", block_hash=encode_hex(bh))
        log.debug("found", count=len(found))
        peer.send_Blocks(found)

    def peer_status_received(self, proto, genesis_hash, peer, **kwargs):
        log.debug("received status", remote_id=peer, genesis_hash=encode_hex(genesis_hash))
        # check genesis
        if genesis_hash != self.chain.genesis.hash:
            return peer.send_Disconnect(reason='wrong genesis block')

        # request chain
        self.synchronizer.synchronize_status(
            peer, peer.status_head_hash, peer.status_total_difficulty)
        # send transactions
        log.debug("sending transactions", remote_id=peer)
        transactions = self.get_transactions()
        transactions = [rlp.decode(x.serialize()) for x in transactions]
        peer.send_Transactions(transactions)

    def peer_handshake(self, proto, peer, **kwargs):
        # reply with status if not yet sent
        if peer.has_ethereum_capabilities() and not peer.status_sent:
            log.debug("handshake, sending status", remote_id=peer)
            peer.send_Status(self.chainhead.hash, self.chain.head.chain_difficulty(),
                             self.chain.genesis.hash)
        else:
            log.debug("handshake, but peer has no 'eth' capablities", remote_id=peer)

    def remote_transactions_received_handler(self, proto, transactions, peer, **kwargs):
        "receives rlp.decoded serialized"
        txl = [Transaction.deserialize(rlp.encode(tx)) for tx in transactions]
        log.debug('remote_transactions_received', count=len(txl), remote_id=peer)
        for tx in txl:
            peermanager.txfilter.add(tx, peer)  # FIXME
            self.add_transaction(tx)

    def local_transaction_received_handler(self, proto, transaction, **kwargs):
        "receives transaction object"
        log.debug('local_transaction_received', tx_hash=transaction)
        self.add_transaction(transaction)

    def new_block_received_handler(self, proto, block, peer, **kwargs):
        log.debug("recv new remote block", block_hash=block, remote_id=peer)
        self.receive_chain([block], peer)

    def remote_blocks_received_handler(self, proto, transient_blocks, peer, **kwargs):
        log.debug("recv remote blocks", count=len(transient_blocks), remote_id=peer,
                  highest_number=max(x.number for x in transient_blocks))
        if transient_blocks:
            self.receive_chain(transient_blocks, peer)

    def remote_block_hashes_received_handler(self, proto, block_hashes, peer, **kwargs):
        if block_hashes:
            log.debug("recv remote block_hashes", count=len(block_hashes), remote_id=peer,
                      first=encode_hex(block_hashes[0]), last=encode_hex(block_hashes[-1]))
        else:
            log.debug("recv 0 remore block hashes, signifying genesis block")
        self.synchronizer.received_block_hashes(peer, block_hashes)
<|endoftext|>


<commit_before>"""Defines the version of pyexperiment
"""

__version__ = '0.2.8'
<commit_msg>Bump version...<commit_after>"""Defines the version of pyexperiment
"""

__version__ = '0.2.9'
<|endoftext|>


<commit_before>from __future__ import division, print_function, absolute_import

import numpy as np

def _path_to_inlet(bore_connectivity, bore_index):
    """
    Returns the path from a borehole to the bore field inlet.

    Parameters
    ----------
    bore_connectivity : list
        Index of fluid inlet into each borehole. -1 corresponds to a borehole
        connected to the bore field inlet.
    bore_index : int
        Index of borehole to evaluate path.

    Returns
    -------
    path : list
        List of boreholes leading to the bore field inlet, starting from
        borehole bore_index

    """
    # Initialize path
    path = [bore_index]
    # Index of borehole feeding into borehole (bore_index)
    index_in = bore_connectivity[bore_index]
    # Stop when bore field inlet is reached (index_in == -1)
    while not index_in == -1:
        # Add index of upstream borehole to path
        path.append(index_in)
        # Get index of next upstream borehole
        index_in = bore_connectivity[index_in]

    return path

def _verify_bore_connectivity(bore_connectivity, nBoreholes):
    """
    Verifies that borehole connectivity is valid.

    This function raises an error if the supplied borehole connectivity is
    invalid.

    Parameters
    ----------
    bore_connectivity : list
        Index of fluid inlet into each borehole. -1 corresponds to a borehole
        connected to the bore field inlet.
    nBoreholes : int
        Number of boreholes in the bore field.

    """
    if not len(bore_connectivity) == nBoreholes:
        raise ValueError(
            'The length of the borehole connectivity list does not correspond '
            'to the number of boreholes in the bore field.')
    if max(bore_connectivity) >= nBoreholes:
        raise ValueError(
            'The borehole connectivity list contains borehole indices that '
            'are not part of the network.')
    # Cycle through each borehole and verify that connections lead to -1
    # (-1 is the bore field inlet) and that no two boreholes have the same
    # index of fluid inlet (except for -1).
    for i in range(nBoreholes):
        n = 0 # Initialize step counter
        # Index of borehole feeding into borehole i
        index_in = bore_connectivity[i]
        if index_in != -1 and bore_connectivity.count(index_in) > 1:
            raise ValueError(
                'Two boreholes cannot have the same inlet, except fort the '
                'network inlet (index of -1).')
        # Stop when bore field inlet is reached (index_in == -1)
        while not index_in == -1:
            index_in = bore_connectivity[index_in]
            n += 1 # Increment step counter
            # Raise error if n exceeds the number of boreholes
            if n > nBoreholes:
                raise ValueError(
                    'The borehole connectivity list is invalid.')
    return
<commit_msg>Initial Network class<commit_after>from __future__ import division, print_function, absolute_import

import numpy as np

class Network(object):
    """
    Network class.

    #TODO.

    Attributes
    ----------
    boreholes : list of Borehole objects
        List of boreholes included in the bore field.
    pipes : list of pipe objects
        #TODO.
    bore_connectivity : list
        Index of fluid inlet into each borehole. -1 corresponds to a borehole
        connected to the bore field inlet.

    """
    def __init__(self, boreholes, pipes, bore_connectivity):
        self.b = boreholes
        self.nBoreholes = len(boreholes)
        self.p = pipes
        self.c = bore_connectivity

        # Verify that borehole connectivity is valid
        _verify_bore_connectivity(bore_connectivity, self.nBoreholes)
        iInlets, nInlets, iOutlets, nOutlets, iCircuit = _find_inlets_outlets(
                bore_connectivity, self.nBoreholes)

        # Number of inlets and outlets in network
        self.nInlets = nInlets
        self.nOutlets = nOutlets
        # Indices of inlets and outlets in network
        self.iInlets = iInlets
        self.iOutlets = iOutlets
        # Indices of circuit of each borehole in network
        self.iCircuit = iCircuit

    def _initialize_stored_coefficients(self):
        nMethods = 8    # Number of class methods
        self._stored_coefficients = [() for i in range(nMethods)]
        self._stored_m_flow_cp = [np.empty(self.nInlets)
                                  for i in range(nMethods)]
        self._stored_nSegments = [np.nan for i in range(nMethods)]
        self._m_flow_cp_model_variables = np.empty(self.nInlets)
        self._nSegments_model_variables = np.nan

        return

    def _set_stored_coefficients(self, m_flow, cp, nSegments, coefficients,
                                 method_id):
        self._stored_coefficients[method_id] = coefficients
        self._stored_m_flow_cp[method_id] = m_flow*cp
        self._stored_nSegments[method_id] = nSegments

        return

    def _get_stored_coefficients(self, method_id):
        coefficients = self._stored_coefficients[method_id]

        return coefficients

    def _check_coefficients(self, m_flow, cp, nSegments, method_id, tol=1e-6):
        stored_m_flow_cp = self._stored_m_flow_cp[method_id]
        stored_nSegments = self._stored_nSegments[method_id]
        if (np.allclose(m_flow*cp, stored_m_flow_cp, rtol=tol)
                and nSegments == stored_nSegments):
            check = True
        else:
            check = False

        return check

    def _format_inputs(self, m_flow, cp, nSegments):
        """
        Format mass flow rate and heat capacity inputs.
        """
        # Format mass flow rate inputs
        # Mass flow rate in each fluid circuit
        m_flow_in = np.atleast_1d(m_flow)
        if len(m_flow_in) == 1:
            m_flow_in = np.tile(m_flow/self.nInlets, self.nInlets)
        elif not len(m_flow_in) == self.nInlets:
            raise ValueError(
                'Incorrect length of mass flow vector.')
        self._m_flow_in = m_flow_in

        # Format heat capacity inputs
        # Heat capacity in each fluid circuit
        cp_in = np.atleast_1d(cp)
        if len(cp_in) == 1:
            cp_in = np.tile(cp, self.nInlets)
        elif not len(cp_in) == self.nInlets:
            raise ValueError(
                'Incorrect length of heat capacity vector.')
        elif not np.all(cp_in == cp_in[0]):
            raise ValueError(
                'The heat capacity should be the same in all circuits.')
        self._cp_in = cp_in

        # Mass flow rate in boreholes
        m_flow_borehole = np.array([m_flow_in[i] for i in self.iCircuit])
        self._m_flow_borehole = m_flow_borehole
        # Heat capacity in boreholes
        cp_borehole = np.array([cp_in[i] for i in self.iCircuit])
        self._cp_borehole = cp_borehole

        # Format number of segments for each borehole
        nSeg = np.atleast_1d(nSegments)
        if len(nSeg) == 1:
            nSeg = [nSegments for i in range(self.nBoreholes)]
        elif not len(nSeg) == self.nBoreholes:
            raise ValueError(
                'Incorrect length of number of segments list.')
        self.nSegments = nSeg

def _find_inlets_outlets(bore_connectivity, nBoreholes):
    """
    Finds the numbers of boreholes connected to the inlet and outlet of the
    network and the indices of the boreholes.

    This function raises an error if the supplied borehole connectivity is
    invalid.

    Parameters
    ----------
    bore_connectivity : list
        Index of fluid inlet into each borehole. -1 corresponds to a borehole
        connected to the bore field inlet.
    nBoreholes : int
        Number of boreholes in the bore field.

    """
    # Number and indices of inlets
    nInlets = bore_connectivity.count(-1)
    iInlets = [i for i in range(nBoreholes) if bore_connectivity[i]==-1]
    # Number and indices of outlets
    iOutlets = [i for i in range(nBoreholes) if i not in bore_connectivity]
    nOutlets = len(iOutlets)
    iCircuit = [iInlets.index(_path_to_inlet(bore_connectivity, i)[-1])
                for i in range(nBoreholes)]
    if not nInlets == nOutlets:
        raise ValueError(
            'The network should have as many inlets as outlets.')

    return iInlets, nInlets, iOutlets, nOutlets, iCircuit

def _path_to_inlet(bore_connectivity, bore_index):
    """
    Returns the path from a borehole to the bore field inlet.

    Parameters
    ----------
    bore_connectivity : list
        Index of fluid inlet into each borehole. -1 corresponds to a borehole
        connected to the bore field inlet.
    bore_index : int
        Index of borehole to evaluate path.

    Returns
    -------
    path : list
        List of boreholes leading to the bore field inlet, starting from
        borehole bore_index

    """
    # Initialize path
    path = [bore_index]
    # Index of borehole feeding into borehole (bore_index)
    index_in = bore_connectivity[bore_index]
    # Stop when bore field inlet is reached (index_in == -1)
    while not index_in == -1:
        # Add index of upstream borehole to path
        path.append(index_in)
        # Get index of next upstream borehole
        index_in = bore_connectivity[index_in]

    return path

def _verify_bore_connectivity(bore_connectivity, nBoreholes):
    """
    Verifies that borehole connectivity is valid.

    This function raises an error if the supplied borehole connectivity is
    invalid.

    Parameters
    ----------
    bore_connectivity : list
        Index of fluid inlet into each borehole. -1 corresponds to a borehole
        connected to the bore field inlet.
    nBoreholes : int
        Number of boreholes in the bore field.

    """
    if not len(bore_connectivity) == nBoreholes:
        raise ValueError(
            'The length of the borehole connectivity list does not correspond '
            'to the number of boreholes in the bore field.')
    if max(bore_connectivity) >= nBoreholes:
        raise ValueError(
            'The borehole connectivity list contains borehole indices that '
            'are not part of the network.')
    # Cycle through each borehole and verify that connections lead to -1
    # (-1 is the bore field inlet) and that no two boreholes have the same
    # index of fluid inlet (except for -1).
    for i in range(nBoreholes):
        n = 0 # Initialize step counter
        # Index of borehole feeding into borehole i
        index_in = bore_connectivity[i]
        if index_in != -1 and bore_connectivity.count(index_in) > 1:
            raise ValueError(
                'Two boreholes cannot have the same inlet, except fort the '
                'network inlet (index of -1).')
        # Stop when bore field inlet is reached (index_in == -1)
        while not index_in == -1:
            index_in = bore_connectivity[index_in]
            n += 1 # Increment step counter
            # Raise error if n exceeds the number of boreholes
            if n > nBoreholes:
                raise ValueError(
                    'The borehole connectivity list is invalid.')
    return
<|endoftext|>


<commit_before>from .module import Module

class State(Module):
    possible_events = {'changed', 'falling', 'rising'}

    def __init__(self, id, alias, device):
        Module.__init__(self, 'State', id, alias, device)
        self._value = False

    @property
    def state(self):
        return self._value == True

    @state.setter
    def state(self, new_val):
        self._value == new_val
        self._push_value('io_state', new_val)

    def _update(self, new_state):
        Module._update(self, new_state)
        if 'io_state' in new_state:
            new_state = new_state['io_state']
            if new_state != self._value:
                self._pub_event('changed', self._value, new_state)

                evt = 'pressed' if new_state == True else 'released'
                self._pub_event(evt, self._value, new_state)

                self._value = new_state

    def control(self):
        def switch(state):
            self.state = state

        return interact(switch, state=self._value)
<commit_msg>fix state module event error<commit_after>from .module import Module

class State(Module):
    possible_events = {'changed', 'pressed', 'released'}

    def __init__(self, id, alias, device):
        Module.__init__(self, 'State', id, alias, device)
        self._value = False

    @property
    def state(self):
        return self._value == True

    @state.setter
    def state(self, new_val):
        self._value == new_val
        self._push_value('io_state', new_val)

    def _update(self, new_state):
        Module._update(self, new_state)
        if 'io_state' in new_state:
            new_state = new_state['io_state']
            if new_state != self._value:
                self._pub_event('changed', self._value, new_state)

                evt = 'pressed' if new_state == True else 'released'
                self._pub_event(evt, self._value, new_state)

                self._value = new_state

    def control(self):
        def switch(state):
            self.state = state

        return interact(switch, state=self._value)
<|endoftext|>


<commit_before>import subprocess
import re

from .parser import BlockParser
from .parser import colon_field
from .parser import once_per_block
from .parser import ignore_rule
from .parser import rule
from .parser import yesnobool
from .parser import oknokbool
from .parser import parse_bytes
from .parser import parse_time

class MegaCLIBase(object):
    def __init__(self, megacli_path):
        self.megacli_path = megacli_path

    def run_command(self, *args):
        exit_re = re.compile('^Exit Code: (.*)$')
        p = subprocess.Popen(
            [self.megacli_path] + list(args),
            shell=False,
            stdout=subprocess.PIPE
        )
        for line in p.communicate()[0].split('\n'):
            emd = exit_re.match(line)
            if emd:
                # exit code is usually meaningless
                rv = int(emd.groups()[0], 16)
                rv = rv
            else:
                yield line

    def extract_by_regex(self, regex, lines, one_only=False):
        if isinstance(regex, basestring):
            regex = re.compile(regex)
        result = []
        for line in lines:
            md = regex.match(line)
            if md:
                if one_only:
                    return md.groups()
                else:
                    result.append(md.groups)
        if one_only:
            raise Exception('Expected one match for %s, got %d. Input was %s' % (
                regex,
                len(result),
                '\n'.join(lines)
            ))
        return result

    @property
    def controller_count(self):
        return int(self.extract_by_regex(
            r'^Controller Count: ([^.]+)\.$',
            self.run_command('-adpCount'),
            one_only=True
        )[0])

    @property
    def controllers(self):
        for i in range(self.controller_count):
            yield MegaCLIController(i, self)

class Component(object):
    REQUIRED_FIELDS = tuple()

    def __init__(self, parent, props=None):
        if props is None:
            self.props = {}
        else:
            self.props = props
        self.parent = parent

    def __setitem__(self, key, value):
        self.props[key] = value

    def __getitem__(self, key):
        return self.props[key]

    @property
    def identifier(self):
        return NotImplementedError()

    @property
    def health_status(self):
        return NotImplementedError()

    @property
    def health_messages(self):
        for bad_key, bad_value in sorted(self.health_status[0].items()):
            yield '%s was unexpectedly %r' % (bad_key, bad_value)

    @property
    def healthy(self):
        return self.health_status[1]

    def __str__(self):
        return self.identifier

    @classmethod
    def from_output(kls, lines, parent):
        lds = []
        for d in kls.PARSER.parse(lines):
            args = []
            for f in kls.REQUIRED_FIELDS:
                args.append(d.pop(f))
            args.extend([parent, d])
            lds.append(kls(*args))
        return lds

class Disk(Component):
    ERROR_COUNT_KEYS = (
        'Media Error Count',
        'Predictive Failure Count',
    )
    ERROR_BOOL_KEYS = ('Drive has flagged a S.M.A.R.T alert', )
    REQUIRED_FIELDS = ('Enclosure Device ID', 'Slot Number')

    PARSER = BlockParser(rules=[
        once_per_block(colon_field('Enclosure Device ID', int)),
        rule(colon_field('Slot Number', int)),
        rule(colon_field('Other Error Count', int)),
        rule(colon_field('Predictive Failure Count', int)),
        rule(colon_field('Media Error Count', int)),
        rule(colon_field('Drive has flagged a S.M.A.R.T alert', yesnobool)),
    ], default_constructor=colon_field(None, str))

    def __init__(self, enclosure_id, slot_number, parent, props=None):
        self.enclosure_id = enclosure_id
        self.slot_number = slot_number
        self.thresholds = dict(
            (k, 0)
            for k
            in self.ERROR_COUNT_KEYS
        )
        super(Disk, self).__init__(parent, props)

    def set_threshold(self, key, value):
        self.thresholds[key] = value

    @property
    def identifier(self):
        return 'PhysDrv [%d:%d]' % (self.enclosure_id, self.slot_number)

    @property
    def health_status(self):
        status = {}
        overall_status = True
        for key, value in self.thresholds.items():
            if self.props.get(key, 0) > value:
                status[key] = self.props[key]
                overall_status = False
        for key in self.ERROR_BOOL_KEYS:
            if self.props.get(key, 0) != 0:
                status[key] = self.props[key]
                overall_status = False
        if 'Online' not in self['Firmware state']:
            status['Firmware state'] = self['Firmware state']
            overall_status = False
        return status, overall_status

class LogicalDevice(Component):
    PARSER = BlockParser(rules=[
        once_per_block(colon_field('Virtual Drive', lambda s: int(s.split(' ')[0]))),
        rule(colon_field('Bad Blocks Exist', yesnobool)),
        rule(colon_field('Size', parse_bytes))
    ], default_constructor=colon_field(None, str))

    REQUIRED_FIELDS = ('Name', )

    def __init__(self, name, parent, props=None):
        self.name = name
        super(LogicalDevice, self).__init__(parent, props)

    @property
    def identifier(self):
        return 'VD %r' % self.name

    @property
    def health_status(self):
        status = {}
        if self['State'] != 'Optimal':
            status['State'] = self['State']
        return status, not bool(status)

class BBU(Component):
    BAD_KEYS = (
        'Pack is about to fail & should be replaced',
        'Remaining Capacity Low',
        'Battery Pack Missing',
        'Battery Replacement required',
        'I2c Errors Detected',
    )
    UNEXPECTED_KEYS = ('Temperature',)

    PARSER = BlockParser(rules=[
        ignore_rule(colon_field('BBU status for Adapter')),
        once_per_block(colon_field('BatteryType')),
        rule(colon_field('Voltage', oknokbool)),
        rule(colon_field('Temperature', oknokbool)),
        rule(colon_field('Learn Cycle Requested', yesnobool)),
        rule(colon_field('Learn Cycle Active', yesnobool)),
        rule(colon_field('Learn Cycle Status', oknokbool)),
        rule(colon_field('Learn Cycle Timeout', yesnobool)),
        rule(colon_field('I2c Errors Detected', yesnobool)),
        rule(colon_field('Battery Pack Missing', yesnobool)),
        rule(colon_field('Battery Replacement required', yesnobool)),
        rule(colon_field('Remaining Capacity Low', yesnobool)),
        rule(colon_field('Periodic Learn Required', yesnobool)),
        rule(colon_field('Transparent Learn', yesnobool)),
        rule(colon_field('No space to cache offload', yesnobool)),
        rule(colon_field('Pack is about to fail & should be replaced', yesnobool)),
        rule(colon_field('Cache Offload premium feature required', yesnobool)),
        rule(colon_field('Module microcode update required', yesnobool)),
    ], default_constructor=colon_field(None, str))

    @property
    def identifier(self):
        return self.props['BatteryType']

    @property
    def health_status(self):
        status = {}
        if self['Battery State'] != 'Optimal':
            status['Battery State'] = self['State']
        for key in self.BAD_KEYS:
            if self[key]:
                status[key] = self[key]
        for key in self.UNEXPECTED_KEYS:
            if not self[key]:
                status[key] = 'not ok'
        return status, not bool(status)

class MegaCLIController(object):
    def __init__(self, controller_number, parent):
        self.controller_number = controller_number
        self.parent = parent

    @property
    def patrol_read_status(self):
        """patrol reads are the background disk re-reads that constantly
        happen to detect failed blocks."""
        parser = BlockParser(rules=[
            rule(colon_field('Patrol Read Mode')),
            rule(colon_field('Patrol Read Execution Delay', parse_time)),
            rule(colon_field('Number of iterations completed', int)),
            rule(colon_field('Current State')),
        ])
        return parser.parse(
            self.parent.run_command('-AdpPR', '-Info', '-a%d' % self.controller_number)
        )[0]

    @property
    def PDs(self):
        return Disk.from_output(self.parent.run_command(
            '-PDList', 'a%d' % self.controller_number
        ), self)

    @property
    def LDs(self):
        return LogicalDevice.from_output(self.parent.run_command(
            '-LDInfo', '-Lall', '-a%d' % self.controller_number
        ), self)

    @property
    def BBUs(self):
        return BBU.from_output(self.parent.run_command(
            '-AdpBbuCmd -GetBbuStatus', '-a%d' % self.controller_number
        ), self)
<commit_msg>add a log arg to MegaCLIBase, to verbosely show megacli cmds being executed<commit_after>import subprocess
import re

from .parser import BlockParser
from .parser import colon_field
from .parser import once_per_block
from .parser import ignore_rule
from .parser import rule
from .parser import yesnobool
from .parser import oknokbool
from .parser import parse_bytes
from .parser import parse_time

class MegaCLIBase(object):
    def __init__(self, megacli_path, log=None):
        self.megacli_path = megacli_path
        self.log = log

    def run_command(self, *args):
        exit_re = re.compile('^Exit Code: (.*)$')
        cmd = [self.megacli_path] + list(args)
        def quotey(input_str):
            if ' ' not in input_str:
                return input_str
            if '"' in input_str and "'" in input_str:
                raise Exception('could not quotey %s' % repr(input_str))
            if '"' in input_str and "'" not in input_str:
                return "'" + input_str + "'"
            elif "'" in input_str and '"' not in input_str:
                return '"' + input_str + '"'
            else:
                return "'" + input_str + "'"
        if self.log:
            try:
                self.log.debug('executing: ' + ' '.join(map(quotey, cmd)))
            except Exception:
                self.log.debug('executing: ' + repr(cmd))
        p = subprocess.Popen(
            cmd,
            shell=False,
            stdout=subprocess.PIPE
        )
        for line in p.communicate()[0].split('\n'):
            emd = exit_re.match(line)
            if emd:
                # exit code is usually meaningless
                rv = int(emd.groups()[0], 16)
                rv = rv
            else:
                yield line

    def extract_by_regex(self, regex, lines, one_only=False):
        if isinstance(regex, basestring):
            regex = re.compile(regex)
        result = []
        for line in lines:
            md = regex.match(line)
            if md:
                if one_only:
                    return md.groups()
                else:
                    result.append(md.groups)
        if one_only:
            raise Exception('Expected one match for %s, got %d. Input was %s' % (
                regex,
                len(result),
                '\n'.join(lines)
            ))
        return result

    @property
    def controller_count(self):
        return int(self.extract_by_regex(
            r'^Controller Count: ([^.]+)\.$',
            self.run_command('-adpCount'),
            one_only=True
        )[0])

    @property
    def controllers(self):
        for i in range(self.controller_count):
            yield MegaCLIController(i, self)

class Component(object):
    REQUIRED_FIELDS = tuple()

    def __init__(self, parent, props=None):
        if props is None:
            self.props = {}
        else:
            self.props = props
        self.parent = parent

    def __setitem__(self, key, value):
        self.props[key] = value

    def __getitem__(self, key):
        return self.props[key]

    @property
    def identifier(self):
        return NotImplementedError()

    @property
    def health_status(self):
        return NotImplementedError()

    @property
    def health_messages(self):
        for bad_key, bad_value in sorted(self.health_status[0].items()):
            yield '%s was unexpectedly %r' % (bad_key, bad_value)

    @property
    def healthy(self):
        return self.health_status[1]

    def __str__(self):
        return self.identifier

    @classmethod
    def from_output(kls, lines, parent):
        lds = []
        for d in kls.PARSER.parse(lines):
            args = []
            for f in kls.REQUIRED_FIELDS:
                args.append(d.pop(f))
            args.extend([parent, d])
            lds.append(kls(*args))
        return lds

class Disk(Component):
    ERROR_COUNT_KEYS = (
        'Media Error Count',
        'Predictive Failure Count',
    )
    ERROR_BOOL_KEYS = ('Drive has flagged a S.M.A.R.T alert', )
    REQUIRED_FIELDS = ('Enclosure Device ID', 'Slot Number')

    PARSER = BlockParser(rules=[
        once_per_block(colon_field('Enclosure Device ID', int)),
        rule(colon_field('Slot Number', int)),
        rule(colon_field('Other Error Count', int)),
        rule(colon_field('Predictive Failure Count', int)),
        rule(colon_field('Media Error Count', int)),
        rule(colon_field('Drive has flagged a S.M.A.R.T alert', yesnobool)),
    ], default_constructor=colon_field(None, str))

    def __init__(self, enclosure_id, slot_number, parent, props=None):
        self.enclosure_id = enclosure_id
        self.slot_number = slot_number
        self.thresholds = dict(
            (k, 0)
            for k
            in self.ERROR_COUNT_KEYS
        )
        super(Disk, self).__init__(parent, props)

    def set_threshold(self, key, value):
        self.thresholds[key] = value

    @property
    def identifier(self):
        return 'PhysDrv [%d:%d]' % (self.enclosure_id, self.slot_number)

    @property
    def health_status(self):
        status = {}
        overall_status = True
        for key, value in self.thresholds.items():
            if self.props.get(key, 0) > value:
                status[key] = self.props[key]
                overall_status = False
        for key in self.ERROR_BOOL_KEYS:
            if self.props.get(key, 0) != 0:
                status[key] = self.props[key]
                overall_status = False
        if 'Online' not in self['Firmware state']:
            status['Firmware state'] = self['Firmware state']
            overall_status = False
        return status, overall_status

class LogicalDevice(Component):
    PARSER = BlockParser(rules=[
        once_per_block(colon_field('Virtual Drive', lambda s: int(s.split(' ')[0]))),
        rule(colon_field('Bad Blocks Exist', yesnobool)),
        rule(colon_field('Size', parse_bytes))
    ], default_constructor=colon_field(None, str))

    REQUIRED_FIELDS = ('Name', )

    def __init__(self, name, parent, props=None):
        self.name = name
        super(LogicalDevice, self).__init__(parent, props)

    @property
    def identifier(self):
        return 'VD %r' % self.name

    @property
    def health_status(self):
        status = {}
        if self['State'] != 'Optimal':
            status['State'] = self['State']
        return status, not bool(status)

class BBU(Component):
    BAD_KEYS = (
        'Pack is about to fail & should be replaced',
        'Remaining Capacity Low',
        'Battery Pack Missing',
        'Battery Replacement required',
        'I2c Errors Detected',
    )
    UNEXPECTED_KEYS = ('Temperature',)

    PARSER = BlockParser(rules=[
        ignore_rule(colon_field('BBU status for Adapter')),
        once_per_block(colon_field('BatteryType')),
        rule(colon_field('Voltage', oknokbool)),
        rule(colon_field('Temperature', oknokbool)),
        rule(colon_field('Learn Cycle Requested', yesnobool)),
        rule(colon_field('Learn Cycle Active', yesnobool)),
        rule(colon_field('Learn Cycle Status', oknokbool)),
        rule(colon_field('Learn Cycle Timeout', yesnobool)),
        rule(colon_field('I2c Errors Detected', yesnobool)),
        rule(colon_field('Battery Pack Missing', yesnobool)),
        rule(colon_field('Battery Replacement required', yesnobool)),
        rule(colon_field('Remaining Capacity Low', yesnobool)),
        rule(colon_field('Periodic Learn Required', yesnobool)),
        rule(colon_field('Transparent Learn', yesnobool)),
        rule(colon_field('No space to cache offload', yesnobool)),
        rule(colon_field('Pack is about to fail & should be replaced', yesnobool)),
        rule(colon_field('Cache Offload premium feature required', yesnobool)),
        rule(colon_field('Module microcode update required', yesnobool)),
    ], default_constructor=colon_field(None, str))

    @property
    def identifier(self):
        return self.props['BatteryType']

    @property
    def health_status(self):
        status = {}
        if self['Battery State'] != 'Optimal':
            status['Battery State'] = self['State']
        for key in self.BAD_KEYS:
            if self[key]:
                status[key] = self[key]
        for key in self.UNEXPECTED_KEYS:
            if not self[key]:
                status[key] = 'not ok'
        return status, not bool(status)

class MegaCLIController(object):
    def __init__(self, controller_number, parent):
        self.controller_number = controller_number
        self.parent = parent

    @property
    def patrol_read_status(self):
        """patrol reads are the background disk re-reads that constantly
        happen to detect failed blocks."""
        parser = BlockParser(rules=[
            rule(colon_field('Patrol Read Mode')),
            rule(colon_field('Patrol Read Execution Delay', parse_time)),
            rule(colon_field('Number of iterations completed', int)),
            rule(colon_field('Current State')),
        ])
        return parser.parse(
            self.parent.run_command('-AdpPR', '-Info', '-a%d' % self.controller_number)
        )[0]

    @property
    def PDs(self):
        return Disk.from_output(self.parent.run_command(
            '-PDList', 'a%d' % self.controller_number
        ), self)

    @property
    def LDs(self):
        return LogicalDevice.from_output(self.parent.run_command(
            '-LDInfo', '-Lall', '-a%d' % self.controller_number
        ), self)

    @property
    def BBUs(self):
        return BBU.from_output(self.parent.run_command(
            '-AdpBbuCmd -GetBbuStatus', '-a%d' % self.controller_number
        ), self)
<|endoftext|>


<commit_before>import traceback
from tkinter import Frame, Button, W, E, N, S, Label, RAISED, StringVar, Radiobutton
from tkinter.filedialog import askopenfilename, sys
from tkinter.messagebox import showerror

from PIL import ImageTk, Image

from pyminutiaeviewer.minutiae_drawing import draw_minutiae
from pyminutiaeviewer.minutiae_reader import MinutiaeReader, MinutiaeFileFormat

class Root(Frame):
    def __init__(self):
        Frame.__init__(self)
        self.master.title("Py Minutiae Viewer")
        self.grid(sticky=W + E + N + S)

        self.minutiae_file_format = StringVar()
        self.minutiae = None

        self.control_panel = ControlPanel(self, self.load_fingerprint_image, self.load_minutiae_file, self.draw_minutiae, self.minutiae_file_format)
        self.control_panel.grid(row=0, column=0, sticky=W + E + N + S)

        self.image_raw = Image.new('RGBA', (512, 512), (255, 255, 255, 255))
        self.image_minutiae = None
        self.image = ImageTk.PhotoImage(self.image_raw)
        self.image_label = Label(self, image=self.image)
        self.image_label.grid(row=0, column=1, sticky=N)

    def load_fingerprint_image(self):
        file_path = askopenfilename(filetypes=(("Image files", ('*.bmp', '*.jpeg', '*.jpg', '*.png')),
                                               ("All files", "*.*")))
        if file_path:
            self.image_raw = Image.open(file_path).convert("RGBA")
            self.image = ImageTk.PhotoImage(self.image_raw)
            self.image_label.config(image=[self.image])
            self.update_idletasks()

    def load_minutiae_file(self):
        file_path = askopenfilename(filetypes=(("Text files", ('*.txt', '*.min')),
                                               ("All files", "*.*")))
        if file_path:
            if self.minutiae_file_format.get() == "Simple":
                reader = MinutiaeReader(MinutiaeFileFormat.SIMPLE)
            elif self.minutiae_file_format.get() == "NBIST":
                reader = MinutiaeReader(MinutiaeFileFormat.NBIST)
            else:
                print("The File Type was not set", file=sys.stderr)
                return

            try:
                self.minutiae = reader.read(file_path)
            except Exception as e:
                traceback.print_exc()
                showerror("Read Minutiae File", "There was an error in reading the minutiae file. Are you sure you "
                                                "selected the right format?\n\nThe error message was:\n{}".format(e))

    def draw_minutiae(self):
        if self.minutiae is None:
            showerror("Draw Minutiae", "The minutiae file has not been set.")
            return
        if self.image_raw is None:
            showerror("Draw Minutiae", "The image file has not been set.")
            return

        self.image_minutiae = draw_minutiae(self.image_raw, self.minutiae)

        self.image = ImageTk.PhotoImage(self.image_minutiae)
        self.image_label.config(image=[self.image])
        self.update_idletasks()

class ControlPanel(Frame):
    def __init__(self, parent, load_fingerprint_func, load_minutiae_func, draw_minutiae_func, minutiae_format):
        Frame.__init__(self, parent, relief=RAISED, borderwidth=1)
        self.mode_label = Label(self, text="Display Pre-calculated Minutiae")
        self.mode_label.grid(row=0, column=0, sticky=N)

        self.mode_button = Button(self, text="Change Mode", command=None)
        self.mode_button.grid(row=1, column=0, sticky=N+W+E)

        self.open_fingerprint_image_button = Button(self, text="Open Fingerprint Image", command=load_fingerprint_func)
        self.open_fingerprint_image_button.grid(row=2, column=0, sticky=N+W+E)

        self.radio_label = Label(self, text="Minutiae File Format:")
        self.radio_label.grid(row=3, column=0, sticky=N)
        self.radio_simple = Radiobutton(self, text="Simple", variable=minutiae_format, value="Simple")
        self.radio_simple.grid(row=4, column=0, sticky=W)
        self.radio_nbist = Radiobutton(self, text="NBIST/MINDTC", variable=minutiae_format, value="NBIST")
        self.radio_nbist.grid(row=5, column=0, sticky=W)
        minutiae_format.set("NBIST")

        self.open_minutiae_txt_button = Button(self, text="Open Minutiae File", command=load_minutiae_func)
        self.open_minutiae_txt_button.grid(row=6, column=0, sticky=N+W+E)

        self.draw_minutiae = Button(self, text="Draw Minutiae", command=draw_minutiae_func)
        self.draw_minutiae.grid(row=7, column=0, sticky=N+W+E)
<commit_msg>Using themed tkinter<commit_after>import traceback
from tkinter.ttk import Frame, Button, Label, Radiobutton, Style
from tkinter import W, E, N, S,  RAISED, StringVar
from tkinter.filedialog import askopenfilename, sys
from tkinter.messagebox import showerror

from PIL import ImageTk, Image

from pyminutiaeviewer.minutiae_drawing import draw_minutiae
from pyminutiaeviewer.minutiae_reader import MinutiaeReader, MinutiaeFileFormat

class Root(Frame):
    def __init__(self):
        Frame.__init__(self)
        s = Style()
        s.theme_use("clam")
        self.master.title("Py Minutiae Viewer")
        self.grid(sticky=W + E + N + S)

        self.minutiae_file_format = StringVar()
        self.minutiae = None

        self.control_panel = ControlPanel(self, self.load_fingerprint_image, self.load_minutiae_file, self.draw_minutiae, self.minutiae_file_format)
        self.control_panel.grid(row=0, column=0, sticky=W + E + N + S)

        self.image_raw = Image.new('RGBA', (512, 512), (255, 255, 255, 255))
        self.image_minutiae = None
        self.image = ImageTk.PhotoImage(self.image_raw)
        self.image_label = Label(self, image=self.image)
        self.image_label.grid(row=0, column=1, sticky=N)

    def load_fingerprint_image(self):
        file_path = askopenfilename(filetypes=(("Image files", ('*.bmp', '*.jpeg', '*.jpg', '*.png')),
                                               ("All files", "*.*")))
        if file_path:
            self.image_raw = Image.open(file_path).convert("RGBA")
            self.image = ImageTk.PhotoImage(self.image_raw)
            self.image_label.config(image=[self.image])
            self.update_idletasks()

    def load_minutiae_file(self):
        file_path = askopenfilename(filetypes=(("Text files", ('*.txt', '*.min')),
                                               ("All files", "*.*")))
        if file_path:
            if self.minutiae_file_format.get() == "Simple":
                reader = MinutiaeReader(MinutiaeFileFormat.SIMPLE)
            elif self.minutiae_file_format.get() == "NBIST":
                reader = MinutiaeReader(MinutiaeFileFormat.NBIST)
            else:
                print("The File Type was not set", file=sys.stderr)
                return

            try:
                self.minutiae = reader.read(file_path)
            except Exception as e:
                traceback.print_exc()
                showerror("Read Minutiae File", "There was an error in reading the minutiae file. Are you sure you "
                                                "selected the right format?\n\nThe error message was:\n{}".format(e))

    def draw_minutiae(self):
        if self.minutiae is None:
            showerror("Draw Minutiae", "The minutiae file has not been set.")
            return
        if self.image_raw is None:
            showerror("Draw Minutiae", "The image file has not been set.")
            return

        self.image_minutiae = draw_minutiae(self.image_raw, self.minutiae)

        self.image = ImageTk.PhotoImage(self.image_minutiae)
        self.image_label.config(image=[self.image])
        self.update_idletasks()

class ControlPanel(Frame):
    def __init__(self, parent, load_fingerprint_func, load_minutiae_func, draw_minutiae_func, minutiae_format):
        Frame.__init__(self, parent, relief=RAISED, borderwidth=1)
        self.mode_label = Label(self, text="Display Pre-calculated Minutiae")
        self.mode_label.grid(row=0, column=0, sticky=N)

        self.mode_button = Button(self, text="Change Mode", command=None)
        self.mode_button.grid(row=1, column=0, sticky=N+W+E)

        self.open_fingerprint_image_button = Button(self, text="Open Fingerprint Image", command=load_fingerprint_func)
        self.open_fingerprint_image_button.grid(row=2, column=0, sticky=N+W+E)

        self.radio_label = Label(self, text="Minutiae File Format:")
        self.radio_label.grid(row=3, column=0, sticky=N)
        self.radio_simple = Radiobutton(self, text="Simple", variable=minutiae_format, value="Simple")
        self.radio_simple.grid(row=4, column=0, sticky=W)
        self.radio_nbist = Radiobutton(self, text="NBIST/MINDTC", variable=minutiae_format, value="NBIST")
        self.radio_nbist.grid(row=5, column=0, sticky=W)
        minutiae_format.set("NBIST")

        self.open_minutiae_txt_button = Button(self, text="Open Minutiae File", command=load_minutiae_func)
        self.open_minutiae_txt_button.grid(row=6, column=0, sticky=N+W+E)

        self.draw_minutiae = Button(self, text="Draw Minutiae", command=draw_minutiae_func)
        self.draw_minutiae.grid(row=7, column=0, sticky=N+W+E)
<|endoftext|>