<commit_before>from binascii import hexlify
from control_flow import control_flow_graph
from copy import copy
from decompile_table import decompile_table
from debug import debug_sprint
from itertools import izip, starmap, repeat, count
import libdisassemble.opcode86 as opcode86
import re

def find(f, seq):
    for item in seq:
        if f(item):
            return item
    raise LookupError

class Indent():
    def __init__(self, level=0):
        self.level = level

    def inc(self):
        new = copy(self)
        new.level += 1
        return new

    def out(self):
        return '\t'*self.level

def decompile(cfg):
    return '\n'.join(map(decompile_vertex, sorted(cfg.vertices().iteritems(), lambda ((a,b),c), ((d,e),f): cmp(b,e))))

def decompile_vertex((t, v), indent=None):
    block_type, block_start = t
    if indent is None:
        indent = Indent(1)
    if block_type == 'block':
        return '\n'.join(starmap(decompile_ins, izip(v, repeat(indent))))

    elif block_type == 'if':
        condition, block = v
        return '{indent}if ({cond})\n{indent}{{\n{block}\n{indent}}}\n'.format(
            cond=condition, block=decompile_vertex(block, indent.inc()), indent=indent.out())

    elif block_type == 'ifelse':
        condition, true, false = v
        return '{indent}if ({cond})\n{indent}{{\n{true}\n{indent}}}\n{indent}else\n{indent}{{\n{false}\n{indent}}}\n'.format(
            cond=condition, true=decompile_vertex(true, indent.inc()), false=decompile_vertex(false, indent.inc()),
            indent=indent.out())

    elif block_type == 'while':
        condition, pre, loop = v
        return '{indent}while ({cond})\n{indent}{{\n{loop}\n{pre}\n{indent}}}\n'.format(
            indent=indent.out(), cond=condition,
            pre=decompile_vertex(pre, indent.inc()), loop=decompile_vertex(loop, indent.inc())
        )

    elif block_type == 'cons':
        out = ''
        for b in v:
            out += decompile_vertex(b, indent)
        return out

    return ''

def is_constant(x):
    return re.match("-?0x.*", x)

def decompile_ins(ins, indent):
    opcode = ins['ins'][0]
    extra_lambda = None

    def backward_ins(x):
        print x
        for i, arg in enumerate(x[1:], 1):
            try:
                x[i] = arg['name']
            except KeyError:
                x[i] = arg['origin']
            except TypeError:
                x[i] = arg
        return x

    ins['ins'] = backward_ins(ins['ins'])

    # special instructions
    if opcode[0] == '!':
        if opcode == '!label':
            fmt = '{i[1]}:'

    elif opcode[0] == 'j':
        return ''

    else:
        try:
            i, fmt, extra_lambda = find(lambda t: t[0] == opcode, decompile_table)
        except LookupError:
            fmt = '// {env[instr]}'

        fmt = indent.out()+fmt

    fmt += debug_sprint('\t\t/* {env[loc]:x}: {env[length]} ({env[bin]}) {env[prefix]} */', 'misc')
    env = {
        'loc': ins['loc'],
        'length': ins['length'],
        'bin': hexlify(ins['bin']),
        'instr': " ".join(ins['ins']),
        'ins': ins['ins'],
        'prefix': ins['prefix'],
    }
    extra = ''
    if extra_lambda:
        extra = extra_lambda(env)

    return fmt.format(i=ins['ins'], extra=extra, env=env)

def get_labels(functions):
    labels = {}
    for function in functions:
        for ins in functions[function]:
            if ins['ins'][0][0] == 'j': # all jumps and only them
                addr = ins['loc']+ins['length']+int(ins['ins'][1],16)
                labels[addr] = 'loc_{0:x}'.format(addr)

    return labels

def infer_signature(asm):
    return ('int', [])

def output_signature(signature, name):
    fun_type, args = signature

    def f(arg):
        i, type = arg
        return type + ' arg' + i

    pre = fun_type + ' ' + name + '(' + ', '.join(map(f, enumerate(args))) + ')\n{\n'
    post = '\n}'
    return pre, post

def labelize_functions(functions, labels):
    for name, function in functions.iteritems():
        f = []
        for ins in function:
#            if ins['loc'] in labels:
#                f.append({'prefix': '', 'bin': '', 'loc': ins['loc'], 'length': 0, 'ins': ['!label', labels[ins['loc']]]})
            f.append(ins)

        functions[name] = f

    return functions

def is_register(x):
    registers = map(lambda (x,y,z): x, opcode86.regs)
    return x in registers

def variable_inference(asm, labels):
    var_names = new_var_name()
    temp_names = new_temp_name()
    vars = {}

    for line, opcode in enumerate(asm):
        for i, arg_mod in enumerate(izip(opcode['ins'][1:], opcode['r'], opcode['w']), 1):
            arg, readable, writable = arg_mod
            if arg in vars:
                asm[line]['ins'][i] = vars[arg]
            else:
                if writable:
                    if is_register(arg):
                        vars[arg] = {'type':'temp', 'name':temp_names.next(), 'origin':arg}
                    else:
                        vars[arg] = {'type':'var', 'name':var_names.next(), 'origin':arg}
                    asm[line]['ins'][i] = vars[arg]

                if readable:
                    if is_constant(arg):
                        asm[line]['ins'][i] = {'type':'const', 'origin':arg}
                    else:
                        print "Error: reading nonexistant variable " + arg
                        if is_register(arg):
                            vars[arg] = {'type':'temp', 'name':temp_names.next(), 'origin':arg}
                        else:
                            vars[arg] = {'type':'var', 'name':var_names.next(), 'origin':arg}
                        asm[line]['ins'][i] = vars[arg]

    return asm

def new_var_name():
    for n in count(0):
        yield "var_{0}".format(n)

def new_temp_name():
    for n in count(0):
        yield "temp_{0}".format(n)

def decompile_function(asm, labels, name):
    signature = infer_signature(asm)
    pre, post = output_signature(signature, name)

    asm = variable_inference(asm, labels)
    cfg = control_flow_graph(asm, labels, name)

    return pre + decompile(cfg) + post

def decompile_functions(functions, symbols):
    labels = get_labels(functions)
    rfunctions = labelize_functions(functions, labels)

    output = ''
    for name, symbol in symbols.iteritems():
        output += decompile_function(functions[name], labels, name)
        output += '\n'

    return output
<commit_msg>removed debug output<commit_after>from binascii import hexlify
from control_flow import control_flow_graph
from copy import copy
from decompile_table import decompile_table
from debug import debug_sprint
from itertools import izip, starmap, repeat, count
import libdisassemble.opcode86 as opcode86
import re

def find(f, seq):
    for item in seq:
        if f(item):
            return item
    raise LookupError

class Indent():
    def __init__(self, level=0):
        self.level = level

    def inc(self):
        new = copy(self)
        new.level += 1
        return new

    def out(self):
        return '\t'*self.level

def decompile(cfg):
    return '\n'.join(map(decompile_vertex, sorted(cfg.vertices().iteritems(), lambda ((a,b),c), ((d,e),f): cmp(b,e))))

def decompile_vertex((t, v), indent=None):
    block_type, block_start = t
    if indent is None:
        indent = Indent(1)
    if block_type == 'block':
        return '\n'.join(starmap(decompile_ins, izip(v, repeat(indent))))

    elif block_type == 'if':
        condition, block = v
        return '{indent}if ({cond})\n{indent}{{\n{block}\n{indent}}}\n'.format(
            cond=condition, block=decompile_vertex(block, indent.inc()), indent=indent.out())

    elif block_type == 'ifelse':
        condition, true, false = v
        return '{indent}if ({cond})\n{indent}{{\n{true}\n{indent}}}\n{indent}else\n{indent}{{\n{false}\n{indent}}}\n'.format(
            cond=condition, true=decompile_vertex(true, indent.inc()), false=decompile_vertex(false, indent.inc()),
            indent=indent.out())

    elif block_type == 'while':
        condition, pre, loop = v
        return '{indent}while ({cond})\n{indent}{{\n{loop}\n{pre}\n{indent}}}\n'.format(
            indent=indent.out(), cond=condition,
            pre=decompile_vertex(pre, indent.inc()), loop=decompile_vertex(loop, indent.inc())
        )

    elif block_type == 'cons':
        out = ''
        for b in v:
            out += decompile_vertex(b, indent)
        return out

    return ''

def is_constant(x):
    return re.match("-?0x.*", x)

def decompile_ins(ins, indent):
    opcode = ins['ins'][0]
    extra_lambda = None

    def backward_ins(x):
        #print x
        for i, arg in enumerate(x[1:], 1):
            try:
                x[i] = arg['name']
            except KeyError:
                x[i] = arg['origin']
            except TypeError:
                x[i] = arg
        return x

    ins['ins'] = backward_ins(ins['ins'])

    # special instructions
    if opcode[0] == '!':
        if opcode == '!label':
            fmt = '{i[1]}:'

    elif opcode[0] == 'j':
        return ''

    else:
        try:
            i, fmt, extra_lambda = find(lambda t: t[0] == opcode, decompile_table)
        except LookupError:
            fmt = '// {env[instr]}'

        fmt = indent.out()+fmt

    fmt += debug_sprint('\t\t/* {env[loc]:x}: {env[length]} ({env[bin]}) {env[prefix]} */', 'misc')
    env = {
        'loc': ins['loc'],
        'length': ins['length'],
        'bin': hexlify(ins['bin']),
        'instr': " ".join(ins['ins']),
        'ins': ins['ins'],
        'prefix': ins['prefix'],
    }
    extra = ''
    if extra_lambda:
        extra = extra_lambda(env)

    return fmt.format(i=ins['ins'], extra=extra, env=env)

def get_labels(functions):
    labels = {}
    for function in functions:
        for ins in functions[function]:
            if ins['ins'][0][0] == 'j': # all jumps and only them
                addr = ins['loc']+ins['length']+int(ins['ins'][1],16)
                labels[addr] = 'loc_{0:x}'.format(addr)

    return labels

def infer_signature(asm):
    return ('int', [])

def output_signature(signature, name):
    fun_type, args = signature

    def f(arg):
        i, type = arg
        return type + ' arg' + i

    pre = fun_type + ' ' + name + '(' + ', '.join(map(f, enumerate(args))) + ')\n{\n'
    post = '\n}'
    return pre, post

def labelize_functions(functions, labels):
    for name, function in functions.iteritems():
        f = []
        for ins in function:
#            if ins['loc'] in labels:
#                f.append({'prefix': '', 'bin': '', 'loc': ins['loc'], 'length': 0, 'ins': ['!label', labels[ins['loc']]]})
            f.append(ins)

        functions[name] = f

    return functions

def is_register(x):
    registers = map(lambda (x,y,z): x, opcode86.regs)
    return x in registers

def variable_inference(asm, labels):
    var_names = new_var_name()
    temp_names = new_temp_name()
    vars = {}

    for line, opcode in enumerate(asm):
        for i, arg_mod in enumerate(izip(opcode['ins'][1:], opcode['r'], opcode['w']), 1):
            arg, readable, writable = arg_mod
            if arg in vars:
                asm[line]['ins'][i] = vars[arg]
            else:
                if writable:
                    if is_register(arg):
                        vars[arg] = {'type':'temp', 'name':temp_names.next(), 'origin':arg}
                    else:
                        vars[arg] = {'type':'var', 'name':var_names.next(), 'origin':arg}
                    asm[line]['ins'][i] = vars[arg]

                if readable:
                    if is_constant(arg):
                        asm[line]['ins'][i] = {'type':'const', 'origin':arg}
                    else:
                        print "Error: reading nonexistant variable " + arg
                        if is_register(arg):
                            vars[arg] = {'type':'temp', 'name':temp_names.next(), 'origin':arg}
                        else:
                            vars[arg] = {'type':'var', 'name':var_names.next(), 'origin':arg}
                        asm[line]['ins'][i] = vars[arg]

    return asm

def new_var_name():
    for n in count(0):
        yield "var_{0}".format(n)

def new_temp_name():
    for n in count(0):
        yield "temp_{0}".format(n)

def decompile_function(asm, labels, name):
    signature = infer_signature(asm)
    pre, post = output_signature(signature, name)

    asm = variable_inference(asm, labels)
    cfg = control_flow_graph(asm, labels, name)

    return pre + decompile(cfg) + post

def decompile_functions(functions, symbols):
    labels = get_labels(functions)
    rfunctions = labelize_functions(functions, labels)

    output = ''
    for name, symbol in symbols.iteritems():
        output += decompile_function(functions[name], labels, name)
        output += '\n'

    return output
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Execute with sudo python defendARP.py
#
#
import time
import sys
import socket
import os
import re
import time
import logging
import subprocess

from optparse 	import OptionParser

try:
	# Import scapy while suppressing warnings
	logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
	from scapy.all import ICMP, IP, ARP, Ether, send
	logging.getLogger("scapy.runtime").setLevel(logging.WARNING)
except:
	print("This tool requires Scapy, see www.secdev.org/projects/scapy")
	sys.exit()

###############################################################
##################### MAIN DEFENSE LOGIC ######################

def startDefense(ipAddress, my_ip, interface):
	'''
	ipAddress = IP to defend.
	my_ip = IP on the device running the script.
	interface = Network interface we are defending.
	'''

	# Remove given IP Address from local ARP table.
	print("INITIALIZING...")
	ping(ipAddress)
	print("Removing %s from the ARP table.") % (ipAddress)
	os.system("arp -d " + ipAddress)
	print("OK.\n")

	# Ping the IP to establish it's correct MAC address.
	# NOTE: The ARP could still be poisoned if an attacker sends poison packets while we are pinging.
	print("Obtainting MAC address.")
	packet = ping(ipAddress)
	targetMac = packet[Ether].dst
	print("MAC address found: %s") % (targetMac)

	# Find the specified IP in the ARP table and corresponding MAC address
	print("Gathering MAC address from IP address in ARP Table.")
	process = subprocess.Popen(['arp', '-a'], stdout=subprocess.PIPE)
	for line in iter(process.stdout.readline, ''):
		tokens = line.split()
		for index in range(len(tokens)):
			if targetMac == tokens[index]:
				print("Found MAC address in ARP table for target IP address.")
				# Save IP address and MAC address, get rid of surrounding '()'s
				tempAddr = tokens[1][1:-1]
				tempMac = tokens[3]
				tempLine = tokens

	# Confirm the physical address of target
	print("Is %s the correct MAC address for %s (Y/N)?") % (tempMac, ipAddress)
	valid = False
	while valid != True:
		answer = str(raw_input("> "))
		print(answer)
		if answer == "N" or answer == "n":
			print("If this is not the correct MAC then you have already been poisoned")
			print("You must start this script in a 'safe' state.")
			sys.exit()
		elif answer == "Y" or answer == "y":
			print("OK.\n")
			print("Monitoring your ARP table...\n")
			goodMac = tempMac
			valid = True
		else:
			print("Invalid Answer. Try again.")

	# Set monitor loop
	monitor = True
	while monitor == True:
		process = subprocess.Popen(['arp', '-a'], stdout=subprocess.PIPE)
		for line in iter(process.stdout.readline, ''):
			tokens = line.split()
			for index in range(len(tokens)):
				if targetMac == tokens[index]:
					# Save IP address and MAC address, get rid of surrounding '()'s
					tempAddr = tokens[1][1:-1]
					tempMac = tokens[3]
					tempLine = tokens

		# Check to make sure our good MAC address matches the one in the ARP table
		if goodMac != tempMac:
			# This should make a BEEP sound
			subprocess.Popen(['echo', '-en', '"\007"'])
			print("ARP POISONED!")
			print("Spoofed IP: %s") % (tempAddr)
			print("%s actual Physical Address: %s") % (goodMac)
			print("Attacker's Physical Address: %s") % (tempMac)
			print("Attempting to reset the correct Physical Address...")

			# Attempt to reset the ARP table. This will not work if we are continually being poisoned
			process = subprocess.Popen(['arp', '-d', tempAddr], stdout=subprocess.PIPE)
			# Re-ping the target
			packet = ping(ipAddress)
			print("ARP Table reset.")
			print("Monitoring your ARP table...")

		# Wait for 5 seconds
		time.sleep(5)

###############################################################
###################### UTILITY FUNCTIONS ######################

# Grab the IP address on a specific interface
def getMyIp(interface):
	# This is ok because we validate the output (check to make sure it is an IP address). But could be used to exfultrate info.
	p = subprocess.Popen("ifconfig " + interface + " | grep 'inet addr' | awk -F: '{print $2}' | awk '{print $1}'", shell=True, stdout=subprocess.PIPE)
	output = p.communicate()[0].rstrip()

	try:
	    socket.inet_aton(output)
	    return output
	except socket.error:
	    return ''

# Find the name of the interface we are going to use
def getInterface():
	# This is ok because there is not user input. Do NOT trust user input in this function. Use call() instead.
	p = subprocess.Popen("ifconfig  | grep 'Link encap' |  awk  '{print $1}' | head -1", shell=True, stdout=subprocess.PIPE)
	output = p.communicate()[0].rstrip()
	return output

def ping(ip):
	# Use scapy ping
	e = Ether()
	i = IP()
	i.dst = ip
	packet = e/i/ICMP()
	return packet

# Print the header information
def printHeader():
	print("SUMMARY")
	print("\tDeletes the specified IP from the ARP table, then pings the IP to")
	print("\textablish the correct Physical Address.")
	print("\tThe script will then continually monitor the specified IP's entry in")
	print("\tthe ARP table. If the IP's ARP table ever changes or is removed, the")
	print("\tscript will BEEP and set the Physical Address back to the correct value.")
	print("AUTHORS")
	print("\tAlan Reed <alreed13@gmail.com>")
	print("\tSam Cappella <sjcappella@gmail.com>")
	print("\tPlease contact the authors with any questions or questions.")
	print("CONTRIBUTE")
	print("\thttps://github.com/alan-reed/ARP-Defense")
	print("LICENSE")
	print("\tCopyright 2013. Apache license 2.0")
	print("SYNTAX")
	print("\tUse: python defendARP.py -h for help.")
	sys.exit()

# Print basic usage info
def printUsageInfo():
	print("Usage:")
	print("\tpython defendARP.py -a <ip_addr_to_monitor>")
	print("\tpython defendARP.py --address=<ip_addr_to_monitor>")
	print("Help:")
	print("\tpython defendARP.py --help")
	sys.exit()

def isUnix():
	if os.name == "posix":
		return 1
	else:
		return 0

def printOsRequirements():
	print("ERROR:");
	print("\tThis script only works on Unix systems.")
	print("\tAn equivalent script for Windows can be found at https://github.com/alan-reed/ARP-Defense/blob/master/defendAPR.bat")
	sys.exit()

###############################################################
############################ MAIN #############################

def main(argv):

	# Check OS (must be unix)
	if not isUnix():
		printOsRequirements()

	# Create option parser
	parser = OptionParser()
	# Define options
	parser.add_option("-a", "--address", dest="ip_addr", help="IP address to monitor.")
	parser.add_option("-f", "--interface", dest="interface",  help="Interface to defend.")
	parser.add_option("-i", "--info", action="store_true", dest="showInfo",  help="Show the copyright and about information.")
	(options, args) = parser.parse_args()

	# Validate arguments
	if options.showInfo == True:
		printHeader()
	if options.ip_addr == None:
		printUsageInfo()
	if options.interface == None:
		interface = getInterface()
		my_ip = getMyIp(interface)
	else:
		my_ip = getMyIp(interface)
	if options.ip_addr == my_ip:
		print("Error: Cannot monitor your own IP Address -- Try using the Default Gateway.\n")
		printUsageInfo()

	#TODO
	# Make sure the IP address is reachable
	packet = ping(options.ip_addr)
	# A packet with a destination MAC of 'ff:ff:ff:ff:ff:ff' will be returned if the target is not found
	if packet[Ether].dst == "ff:ff:ff:ff:ff:ff":
		print("Target can not be found! Please make sure you are using the correct IP address.")
		sys.exit()

	# Call main defense logic
	startDefense(options.ip_addr, my_ip, interface)

if __name__ == "__main__":
	main(sys.argv)<commit_msg>Script is now functional<commit_after>#!/usr/bin/env python
#
# Execute with sudo python defendARP.py
#
#
import time
import sys
import socket
import os
import re
import time
import logging
import subprocess
from optparse 	import OptionParser

###############################################################
##################### MAIN DEFENSE LOGIC ######################

def startDefense(ipAddress, my_ip, interface):
	'''
	ipAddress = IP to defend.
	my_ip = IP on the device running the script.
	interface = Network interface we are defending.
	'''

	# Remove given IP Address from local ARP table.
	print("\nINITIALIZING...")
	print("Removing %s from the ARP table.") % (ipAddress)
	os.system("arp -d " + ipAddress)
	print("OK.")

	# Ping the IP to establish it's correct MAC address.
	# NOTE: The ARP could still be poisoned if an attacker sends poison packets while we are pinging.

	print("Obtainting MAC address.")

	ping(ipAddress)

	mac = getMAC(ipAddress)

	print("MAC address found: %s") % (mac)

	# Confirm the physical address of target
	valid = False
	while valid != True:
		print("Is %s the correct MAC address for %s (y/n)?") % (mac,ipAddress)
		answer = str(raw_input("> "))
		if answer == "N" or answer == "n":
			print("If this is not the correct MAC then you have already been poisoned.")
			print("You must start this script in a 'safe' state.")
			sys.exit()
		elif answer == "Y" or answer == "y":
			print("OK.\n")
			print("Monitoring your ARP table...\n")
			goodMac = mac
			valid = True

	# Set monitor loop
	monitor = True
	while monitor == True:
		mac = getMAC(ipAddress)

		# Check to make sure our good MAC address matches the one in the ARP table
		if goodMac != mac:
			beep()
			print("ARP POISONED!")
			print("Spoofed IP: %s") % (ipAddress)
			attackersIP = ''
			attackersIP = getAttackerIP(ipAddress, mac)
			print("Attacker is sending your traffic to %s") % (attackersIP)
			print("%s's actual Physical Address: %s") % (ipAddress, goodMac)
			print("Attacker's Physical Address: %s") % (mac)
			print("Attempting to reset the correct Physical Address...")

			deleteMAC(ipAddress)

			# Re-ping the target to establish correct MAC
			ping(ipAddress)

			mac = getMAC(ipAddress)

			print("ARP Table reset.")
			print("\nMonitoring your ARP table...\n")

		# Wait for 2 seconds
		time.sleep(2)

###############################################################
###################### UTILITY FUNCTIONS ######################

# Grab the IP address on a specific interface
def getMyIp(interface):
	# This is ok because we validate the output (check to make sure it is an IP address). But could be used to exfultrate info.
	p = subprocess.Popen("ifconfig " + interface + " | grep 'inet addr' | awk -F: '{print $2}' | awk '{print $1}'", shell=True, stdout=subprocess.PIPE)
	output = p.communicate()[0].rstrip()

	try:
	    socket.inet_aton(output)
	    return output
	except socket.error:
	    return ''

# play beep sound
def beep():
	print("\a")

# Remove a IP/MAC pair from ARP table where IP == $1
def deleteMAC(ipAddress):
	p = subprocess.Popen("arp -d " + ipAddress, shell=True, stdout=subprocess.PIPE)
	output = p.communicate()[0].rstrip()

# Get duplicate IP from ARP table
def getAttackerIP(goodIP, mac):
	command = "arp -a | grep '" + mac + "' | grep -v '(" + goodIP + ")' |  awk  '{print $2}'"
	p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
	output = p.communicate()[0].rstrip()
	output = output.replace('(', '').replace(')', '')
	return output

# Get MAC of IP from ARP Table
def getMAC(ip):
	p = subprocess.Popen("arp -a | grep  '(" + ip + ")' |  awk  '{print $4}'", shell=True, stdout=subprocess.PIPE)
	output = p.communicate()[0].rstrip()
	return output

# Find the name of the interface we are going to use
def getInterface():
	# This is ok because there is not user input. Do NOT trust user input in this function. Use call() instead.
	p = subprocess.Popen("ifconfig  | grep 'Link encap' |  awk  '{print $1}' | head -1", shell=True, stdout=subprocess.PIPE)
	output = p.communicate()[0].rstrip()
	return output

def ping(ip):
	# return == 1: OK.
	# return == 0: Failed.
	p = subprocess.Popen("ping -c 1 " + ip, shell=True, stdout=subprocess.PIPE)
	output = p.communicate()[0].rstrip()
	if "1 received" in output:
		return 1
	else:
		return 0

# Print the header information
def printHeader():
	print("SUMMARY")
	print("\tDeletes the specified IP from the ARP table, then pings the IP to")
	print("\textablish the correct Physical Address.")
	print("\tThe script will then continually monitor the specified IP's entry in")
	print("\tthe ARP table. If the IP's ARP table ever changes or is removed, the")
	print("\tscript will BEEP and set the Physical Address back to the correct value.")
	print("AUTHORS")
	print("\tAlan Reed <alreed13@gmail.com>")
	print("\tSam Cappella <sjcappella@gmail.com>")
	print("\tPlease contact the authors with any questions or questions.")
	print("CONTRIBUTE")
	print("\thttps://github.com/alan-reed/ARP-Defense")
	print("LICENSE")
	print("\tCopyright 2013. Apache license 2.0")
	print("SYNTAX")
	print("\tUse: python defendARP.py -h for help.")
	sys.exit()

# Print basic usage info
def printUsageInfo():
	print("Usage:")
	print("\tpython defendARP.py -a <ip_addr_to_monitor>")
	print("\tpython defendARP.py --address=<ip_addr_to_monitor>")
	print("Help:")
	print("\tpython defendARP.py --help")
	sys.exit()

def isUnix():
	if os.name == "posix":
		return 1
	else:
		return 0

def printOsRequirements():
	print("ERROR:");
	print("\tThis script only works on Unix systems.")
	print("\tAn equivalent script for Windows can be found at https://github.com/alan-reed/ARP-Defense/blob/master/defendAPR.bat")
	sys.exit()

###############################################################
############################ MAIN #############################

def main(argv):

	# Check OS (must be unix)
	if not isUnix():
		printOsRequirements()

	# Create option parser
	parser = OptionParser()

	# Define options
	parser.add_option("-a", "--address", dest="ip_addr", help="IP address to monitor.")
	parser.add_option("-f", "--interface", dest="interface",  help="Interface to defend.")
	parser.add_option("-i", "--info", action="store_true", dest="showInfo",  help="Show the copyright and about information.")
	(options, args) = parser.parse_args()

	# Validate arguments
	if options.showInfo == True:
		printHeader()
	if options.ip_addr == None:
		printUsageInfo()
	if options.interface == None:
		interface = getInterface()
		my_ip = getMyIp(interface)
	else:
		my_ip = getMyIp(interface)
	if options.ip_addr == my_ip:
		print("Error: Cannot monitor your own IP Address -- Try using the Default Gateway's IP.\n")
		printUsageInfo()

	# Make sure the IP address is reachable
	res = ping(options.ip_addr)
	if res == 0:
		print("Address unreachable.");
		sys.exit()

	# Call main defense logic
	startDefense(options.ip_addr, my_ip, interface)

if __name__ == "__main__":
	main(sys.argv)<|endoftext|>


<commit_before>#
# Copyright (c) 2004 Specifix, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

import deps
import os

def flags_ix86(baseArch):
    ofInterest = { "cmov"   : True,
		   "mmx"    : True,
		   "sse"    : True,
		   "sse2"   : True,
		   "3dnow"  : True }
    try:
        lines = open("/proc/cpuinfo").read().split("\n")
    except IOError:
        lines=[]
    rc = [ (baseArch, deps.FLAG_SENSE_REQUIRED) ]
    for line in lines:
	if not line.startswith("flags"): continue
	fields = line.split()
	if fields[0] != "flags": continue

	for flag in fields[2:]:
	    if ofInterest.has_key(flag):
                rc.append((flag, deps.FLAG_SENSE_REQUIRED))

	return deps.Dependency('x86', rc)

    return deps.Dependency('x86')

def flags_i686():
    return flags_ix86(baseArch = 'i686')

def flags_i586():
    return flags_ix86(baseArch = 'i586')

def flags_mips64():
    return deps.Dependency('mipseb', [ ('mips64', deps.FLAG_SENSE_REQUIRED) ])

def flags_x86_64():
    return deps.Dependency('x86_64', [] )

def current():
    return currentArch

def any():
    return "any"

def canInstall(other):
    return other == "any" or other == currentArch

baseArch = os.uname()[4]
localNamespace = locals()
if localNamespace.has_key("flags_" + baseArch):
    currentArch = localNamespace["flags_" + baseArch]()
else:
    currentArch = deps.Dependency(baseArch)

del localNamespace
<commit_msg>arch flags are preferred, not required<commit_after>#
# Copyright (c) 2004 Specifix, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any waranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

import deps
import os

def flags_ix86(baseArch):
    ofInterest = { "cmov"   : True,
		   "mmx"    : True,
		   "sse"    : True,
		   "sse2"   : True,
		   "3dnow"  : True }
    try:
        lines = open("/proc/cpuinfo").read().split("\n")
    except IOError:
        lines=[]
    rc = [ (baseArch, deps.FLAG_SENSE_PREFERRED) ]
    for line in lines:
	if not line.startswith("flags"): continue
	fields = line.split()
	if fields[0] != "flags": continue

	for flag in fields[2:]:
	    if ofInterest.has_key(flag):
                rc.append((flag, deps.FLAG_SENSE_PREFERRED))

	return deps.Dependency('x86', rc)

    return deps.Dependency('x86')

def flags_i686():
    return flags_ix86(baseArch = 'i686')

def flags_i586():
    return flags_ix86(baseArch = 'i586')

def flags_mips64():
    return deps.Dependency('mipseb', [ ('mips64', deps.FLAG_SENSE_REQUIRED) ])

def flags_x86_64():
    return deps.Dependency('x86_64', [] )

def current():
    return currentArch

def any():
    return "any"

def canInstall(other):
    return other == "any" or other == currentArch

baseArch = os.uname()[4]
localNamespace = locals()
if localNamespace.has_key("flags_" + baseArch):
    currentArch = localNamespace["flags_" + baseArch]()
else:
    currentArch = deps.Dependency(baseArch)

del localNamespace
<|endoftext|>


<commit_before># ansisql.py
# Copyright (C) 2005 Michael Bayer mike_mp@zzzcomputing.com
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.

"""defines ANSI SQL operations."""

import sqlalchemy.schema as schema

from sqlalchemy.schema import *
import sqlalchemy.sql as sql
import sqlalchemy.engine
from sqlalchemy.sql import *
from sqlalchemy.util import *
import string

def engine(**params):
    return ANSISQLEngine(**params)

class ANSISQLEngine(sqlalchemy.engine.SQLEngine):

    def tableimpl(self, table):
        return ANSISQLTableImpl(table)

    def schemagenerator(self, proxy, **params):
        return ANSISchemaGenerator(proxy, **params)

    def schemadropper(self, proxy, **params):
        return ANSISchemaDropper(proxy, **params)

    def connect_args(self):
        return ([],{})

    def dbapi(self):
        return object()

    def compile(self, statement, bindparams):
        compiler = ANSICompiler(statement, bindparams)

        statement.accept_visitor(compiler)
        return compiler

class ANSICompiler(sql.Compiled):
    def __init__(self, parent, bindparams):
        self.binds = {}
        self.bindparams = bindparams
        self.parent = parent
        self.froms = {}
        self.wheres = {}
        self.strings = {}

    def get_from_text(self, obj):
        return self.froms[obj]

    def get_str(self, obj):
        return self.strings[obj]

    def get_whereclause(self, obj):
        return self.wheres.get(obj, None)

    def get_params(self, **params):
        d = {}
        for key, value in params.iteritems():
            try:
                b = self.binds[key]
            except KeyError:
                raise "No such bind param in statement '%s': %s" % (str(self), key)
            d[b.key] = value

        for b in self.binds.values():
            if not d.has_key(b.key):
                d[b.key] = b.value

        return d

    def visit_column(self, column):
        if column.table.name is None:
            self.strings[column] = column.name
        else:
            self.strings[column] = "%s.%s" % (column.table.name, column.name)

    def visit_fromclause(self, fromclause):
        self.froms[fromclause] = fromclause.from_name

    def visit_textclause(self, textclause):
        if textclause.parens and len(textclause.text):
            self.strings[textclause] = "(" + textclause.text + ")"
        else:
            self.strings[textclause] = textclause.text

    def visit_compound(self, compound):
        if compound.operator is None:
            sep = " "
        else:
            sep = " " + compound.operator + " "

        if compound.parens:
            self.strings[compound] = "(" + string.join([self.get_str(c) for c in compound.clauses], sep) + ")"
        else:
            self.strings[compound] = string.join([self.get_str(c) for c in compound.clauses], sep)

    def visit_clauselist(self, list):
        self.strings[list] = string.join([self.get_str(c) for c in list.clauses], ', ')

    def visit_binary(self, binary):

        if binary.parens:
           self.strings[binary] = "(" + self.get_str(binary.left) + " " + str(binary.operator) + " " + self.get_str(binary.right) + ")"
        else:
            self.strings[binary] = self.get_str(binary.left) + " " + str(binary.operator) + " " + self.get_str(binary.right)

    def visit_bindparam(self, bindparam):
        self.binds[bindparam.shortname] = bindparam

        count = 1
        key = bindparam.key

        while self.binds.setdefault(key, bindparam) is not bindparam:
            key = "%s_%d" % (bindparam.key, count)
            count += 1

        self.strings[bindparam] = ":" + key

    def visit_alias(self, alias):
        self.froms[alias] = self.get_from_text(alias.selectable) + " " + alias.name

    def visit_select(self, select):
        inner_columns = []

        for c in select._raw_columns:
            for co in c.columns:
                inner_columns.append(co)

        if select.use_labels:
            collist = string.join(["%s AS %s" % (c.fullname, c.label) for c in inner_columns], ', ')
        else:
            collist = string.join([c.fullname for c in inner_columns], ', ')

        text = "SELECT " + collist + " FROM "

        whereclause = select.whereclause

        froms = []
        for f in select.froms.values():

            # special thingy used by oracle to redefine a join
            w = self.get_whereclause(f)
            if w is not None:
                # TODO: move this more into the oracle module
                whereclause = sql.and_(w, whereclause)
                self.visit_compound(whereclause)

            t = self.get_from_text(f)
            if t is not None:
                froms.append(t)

        text += string.join(froms, ', ')

        if whereclause is not None:
            t = self.get_str(whereclause)
            if t:
                text += " WHERE " + t

        for tup in select._clauses:
            text += " " + tup[0] + " " + self.get_str(tup[1])

        self.strings[select] = text
        self.froms[select] = "(" + text + ")"

    def visit_table(self, table):
        self.froms[table] = table.name

    def visit_join(self, join):
        if join.isouter:
            self.froms[join] = ("(" + self.get_from_text(join.left) + " LEFT OUTER JOIN " + self.get_from_text(join.right) +
            " ON " + self.get_str(join.onclause) + ")")
        else:
            self.froms[join] = ("(" + self.get_from_text(join.left) + " JOIN " + self.get_from_text(join.right) +
            " ON " + self.get_str(join.onclause) + ")")

    def visit_insert(self, insert_stmt):
        colparams = insert_stmt.get_colparams(self.bindparams)

        for c in colparams:
            b = c[1]
            self.binds[b.key] = b
            self.binds[b.shortname] = b

        text = ("INSERT INTO " + insert_stmt.table.name + " (" + string.join([c[0].name for c in colparams], ', ') + ")" +
         " VALUES (" + string.join([":" + c[1].key for c in colparams], ', ') + ")")

        self.strings[insert_stmt] = text

    def visit_update(self, update_stmt):
        colparams = update_stmt.get_colparams(self.bindparams)

        for c in colparams:
            b = c[1]
            self.binds[b.key] = b
            self.binds[b.shortname] = b

        text = "UPDATE " + update_stmt.table.name + " SET " + string.join(["%s=:%s" % (c[0].name, c[1].key) for c in colparams], ', ')

        if update_stmt.whereclause:
            text += " WHERE " + self.get_str(update_stmt.whereclause)

        self.strings[update_stmt] = text

    def visit_delete(self, delete_stmt):
        text = "DELETE FROM " + delete_stmt.table.name

        if delete_stmt.whereclause:
            text += " WHERE " + self.get_str(delete_stmt.whereclause)

        self.strings[delete_stmt] = text

    def __str__(self):
        return self.get_str(self.parent)

class ANSISQLTableImpl(sql.TableImpl):
    """Selectable implementation that gets attached to a schema.Table object."""

    def __init__(self, table):
        sql.TableImpl.__init__(self)
        self.table = table
        self.id = self.table.name

    def get_from_text(self):
        return self.table.name

class ANSISchemaGenerator(sqlalchemy.engine.SchemaIterator):

    def visit_table(self, table):
        self.append("\nCREATE TABLE " + table.name + "(")

        separator = "\n"

        for column in table.columns:
            self.append(separator)
            separator = ", \n"
            self.append("\t" + column._get_specification())

        self.append("\n)\n\n")
        self.execute()

    def visit_column(self, column):
        pass

class ANSISchemaDropper(sqlalchemy.engine.SchemaIterator):
    def visit_table(self, table):
        self.append("\nDROP TABLE " + table.name)
        self.execute()

<commit_msg><commit_after># ansisql.py
# Copyright (C) 2005 Michael Bayer mike_mp@zzzcomputing.com
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.

"""defines ANSI SQL operations."""

import sqlalchemy.schema as schema

from sqlalchemy.schema import *
import sqlalchemy.sql as sql
import sqlalchemy.engine
from sqlalchemy.sql import *
from sqlalchemy.util import *
import string

def engine(**params):
    return ANSISQLEngine(**params)

class ANSISQLEngine(sqlalchemy.engine.SQLEngine):

    def tableimpl(self, table):
        return ANSISQLTableImpl(table)

    def schemagenerator(self, proxy, **params):
        return ANSISchemaGenerator(proxy, **params)

    def schemadropper(self, proxy, **params):
        return ANSISchemaDropper(proxy, **params)

    def connect_args(self):
        return ([],{})

    def dbapi(self):
        return object()

    def compile(self, statement, bindparams):
        compiler = ANSICompiler(statement, bindparams)

        statement.accept_visitor(compiler)
        return compiler

class ANSICompiler(sql.Compiled):
    def __init__(self, parent, bindparams):
        self.binds = {}
        self.bindparams = bindparams
        self.parent = parent
        self.froms = {}
        self.wheres = {}
        self.strings = {}

    def get_from_text(self, obj):
        return self.froms[obj]

    def get_str(self, obj):
        return self.strings[obj]

    def get_whereclause(self, obj):
        return self.wheres.get(obj, None)

    def get_params(self, **params):
        d = {}
        for key, value in params.iteritems():
            try:
                b = self.binds[key]
            except KeyError:
                raise "No such bind param in statement '%s': %s" % (str(self), key)
            d[b.key] = value

        for b in self.binds.values():
            if not d.has_key(b.key):
                d[b.key] = b.value

        return d

    def visit_column(self, column):
        if column.table.name is None:
            self.strings[column] = column.name
        else:
            self.strings[column] = "%s.%s" % (column.table.name, column.name)

    def visit_fromclause(self, fromclause):
        self.froms[fromclause] = fromclause.from_name

    def visit_textclause(self, textclause):
        if textclause.parens and len(textclause.text):
            self.strings[textclause] = "(" + textclause.text + ")"
        else:
            self.strings[textclause] = textclause.text

    def visit_compound(self, compound):
        if compound.operator is None:
            sep = " "
        else:
            sep = " " + compound.operator + " "

        if compound.parens:
            self.strings[compound] = "(" + string.join([self.get_str(c) for c in compound.clauses], sep) + ")"
        else:
            self.strings[compound] = string.join([self.get_str(c) for c in compound.clauses], sep)

    def visit_clauselist(self, list):
        self.strings[list] = string.join([self.get_str(c) for c in list.clauses], ', ')

    def visit_binary(self, binary):

        if binary.parens:
           self.strings[binary] = "(" + self.get_str(binary.left) + " " + str(binary.operator) + " " + self.get_str(binary.right) + ")"
        else:
            self.strings[binary] = self.get_str(binary.left) + " " + str(binary.operator) + " " + self.get_str(binary.right)

    def visit_bindparam(self, bindparam):
        self.binds[bindparam.shortname] = bindparam

        count = 1
        key = bindparam.key

        while self.binds.setdefault(key, bindparam) is not bindparam:
            key = "%s_%d" % (bindparam.key, count)
            count += 1

        self.strings[bindparam] = ":" + key

    def visit_alias(self, alias):
        self.froms[alias] = self.get_from_text(alias.selectable) + " " + alias.name

    def visit_select(self, select):
        inner_columns = []

        for c in select._raw_columns:
            for co in c.columns:
                inner_columns.append(co)

        if select.use_labels:
            collist = string.join(["%s AS %s" % (c.fullname, c.label) for c in inner_columns], ', ')
        else:
            collist = string.join([c.fullname for c in inner_columns], ', ')

        text = "SELECT " + collist + " FROM "

        whereclause = select.whereclause

        froms = []
        for f in select.froms.values():

            # special thingy used by oracle to redefine a join
            w = self.get_whereclause(f)
            if w is not None:
                # TODO: move this more into the oracle module
                whereclause = sql.and_(w, whereclause)
                self.visit_compound(whereclause)

            t = self.get_from_text(f)
            if t is not None:
                froms.append(t)

        text += string.join(froms, ', ')

        if whereclause is not None:
            t = self.get_str(whereclause)
            if t:
                text += " WHERE " + t

        for tup in select._clauses:
            text += " " + tup[0] + " " + self.get_str(tup[1])

        self.strings[select] = text
        self.froms[select] = "(" + text + ")"

    def visit_table(self, table):
        self.froms[table] = table.name

    def visit_join(self, join):
        if join.isouter:
            self.froms[join] = (self.get_from_text(join.left) + " LEFT OUTER JOIN " + self.get_from_text(join.right) +
            " ON " + self.get_str(join.onclause))
        else:
            self.froms[join] = (self.get_from_text(join.left) + " JOIN " + self.get_from_text(join.right) +
            " ON " + self.get_str(join.onclause))

    def visit_insert(self, insert_stmt):
        colparams = insert_stmt.get_colparams(self.bindparams)

        for c in colparams:
            b = c[1]
            self.binds[b.key] = b
            self.binds[b.shortname] = b

        text = ("INSERT INTO " + insert_stmt.table.name + " (" + string.join([c[0].name for c in colparams], ', ') + ")" +
         " VALUES (" + string.join([":" + c[1].key for c in colparams], ', ') + ")")

        self.strings[insert_stmt] = text

    def visit_update(self, update_stmt):
        colparams = update_stmt.get_colparams(self.bindparams)

        for c in colparams:
            b = c[1]
            self.binds[b.key] = b
            self.binds[b.shortname] = b

        text = "UPDATE " + update_stmt.table.name + " SET " + string.join(["%s=:%s" % (c[0].name, c[1].key) for c in colparams], ', ')

        if update_stmt.whereclause:
            text += " WHERE " + self.get_str(update_stmt.whereclause)

        self.strings[update_stmt] = text

    def visit_delete(self, delete_stmt):
        text = "DELETE FROM " + delete_stmt.table.name

        if delete_stmt.whereclause:
            text += " WHERE " + self.get_str(delete_stmt.whereclause)

        self.strings[delete_stmt] = text

    def __str__(self):
        return self.get_str(self.parent)

class ANSISQLTableImpl(sql.TableImpl):
    """Selectable implementation that gets attached to a schema.Table object."""

    def __init__(self, table):
        sql.TableImpl.__init__(self)
        self.table = table
        self.id = self.table.name

    def get_from_text(self):
        return self.table.name

class ANSISchemaGenerator(sqlalchemy.engine.SchemaIterator):

    def visit_table(self, table):
        self.append("\nCREATE TABLE " + table.name + "(")

        separator = "\n"

        for column in table.columns:
            self.append(separator)
            separator = ", \n"
            self.append("\t" + column._get_specification())

        self.append("\n)\n\n")
        self.execute()

    def visit_column(self, column):
        pass

class ANSISchemaDropper(sqlalchemy.engine.SchemaIterator):
    def visit_table(self, table):
        self.append("\nDROP TABLE " + table.name)
        self.execute()

<|endoftext|>


<commit_before>""" This file contains defines parameters for DIPY that we use to fill
settings in setup.py, the DIPY top-level docstring, and for building the
docs.  In setup.py in particular, we exec this file, so it cannot import dipy
"""

# DIPY version information.  An empty _version_extra corresponds to a
# full release.  '.dev' as a _version_extra string means this is a development
# version
_version_major = 1
_version_minor = 0
_version_micro = 0
_version_extra = 'dev'
# _version_extra = ''

# Format expected by setup.py and doc/source/conf.py: string of form "X.Y.Z"
__version__ = "%s.%s.%s%s" % (_version_major,
                              _version_minor,
                              _version_micro,
                              _version_extra)

CLASSIFIERS = ["Development Status :: 3 - Alpha",
               "Environment :: Console",
               "Intended Audience :: Science/Research",
               "License :: OSI Approved :: BSD License",
               "Operating System :: OS Independent",
               "Programming Language :: Python",
               "Topic :: Scientific/Engineering"]

description  = 'Diffusion MRI utilities in python'

# Note: this long_description is actually a copy/paste from the top-level
# README.rst, so that it shows up nicely on PyPI.  So please remember to edit
# it only in one place and sync it correctly.
long_description = """
======
 DIPY
======

DIPY is a python toolbox for analysis of MR diffusion imaging.

DIPY is for research only; please do not use results from DIPY for
clinical decisions.

Website
=======

Current information can always be found from the DIPY website - http://dipy.org

Mailing Lists
=============

Please see the developer's list at
http://mail.scipy.org/mailman/listinfo/nipy-devel

Code
====

You can find our sources and single-click downloads:

* `Main repository`_ on Github.
* Documentation_ for all releases and current development tree.
* Download as a tar/zip file the `current trunk`_.

.. _main repository: http://github.com/nipy/dipy
.. _Documentation: http://dipy.org
.. _current trunk: https://github.com/nipy/dipy/archive/master.zip

License
=======

DIPY is licensed under the terms of the BSD license.
Please see the LICENSE file in the dipy distribution.

DIPY uses other libraries also licensed under the BSD or the
MIT licenses.
"""

# versions for dependencies
# Check these versions against .travis.yml and requirements.txt
CYTHON_MIN_VERSION = '0.29'
NUMPY_MIN_VERSION = '1.9.0'
SCIPY_MIN_VERSION = '1.0'
NIBABEL_MIN_VERSION = '2.4.0'
H5PY_MIN_VERSION = '2.4.0'

# Main setup parameters
NAME                = 'dipy'
MAINTAINER          = "Eleftherios Garyfallidis"
MAINTAINER_EMAIL    = "neuroimaging@python.org"
DESCRIPTION         = description
LONG_DESCRIPTION    = long_description
URL                 = "http://dipy.org"
DOWNLOAD_URL        = "http://github.com/nipy/dipy/archives/master"
LICENSE             = "BSD license"
CLASSIFIERS         = CLASSIFIERS
AUTHOR              = "dipy developers"
AUTHOR_EMAIL        = "neuroimaging@python.org"
PLATFORMS           = "OS Independent"
MAJOR               = _version_major
MINOR               = _version_minor
MICRO               = _version_micro
ISRELEASE           = _version_extra == ''
VERSION             = __version__
PROVIDES            = ["dipy"]
REQUIRES            = ["numpy (>=%s)" % NUMPY_MIN_VERSION,
                       "scipy (>=%s)" % SCIPY_MIN_VERSION,
                       "nibabel (>=%s)" % NIBABEL_MIN_VERSION,
                       "h5py (>=%s)" % H5PY_MIN_VERSION]
<commit_msg>Release 1.0<commit_after>""" This file contains defines parameters for DIPY that we use to fill
settings in setup.py, the DIPY top-level docstring, and for building the
docs.  In setup.py in particular, we exec this file, so it cannot import dipy
"""

# DIPY version information.  An empty _version_extra corresponds to a
# full release.  '.dev' as a _version_extra string means this is a development
# version
_version_major = 1
_version_minor = 0
_version_micro = 0
# _version_extra = 'dev'
_version_extra = ''

# Format expected by setup.py and doc/source/conf.py: string of form "X.Y.Z"
__version__ = "%s.%s.%s%s" % (_version_major,
                              _version_minor,
                              _version_micro,
                              _version_extra)

CLASSIFIERS = ["Development Status :: 3 - Alpha",
               "Environment :: Console",
               "Intended Audience :: Science/Research",
               "License :: OSI Approved :: BSD License",
               "Operating System :: OS Independent",
               "Programming Language :: Python",
               "Topic :: Scientific/Engineering"]

description  = 'Diffusion MRI utilities in python'

# Note: this long_description is actually a copy/paste from the top-level
# README.rst, so that it shows up nicely on PyPI.  So please remember to edit
# it only in one place and sync it correctly.
long_description = """
======
 DIPY
======

DIPY is a python toolbox for analysis of MR diffusion imaging.

DIPY is for research only; please do not use results from DIPY for
clinical decisions.

Website
=======

Current information can always be found from the DIPY website - http://dipy.org

Mailing Lists
=============

Please see the developer's list at
http://mail.scipy.org/mailman/listinfo/nipy-devel

Code
====

You can find our sources and single-click downloads:

* `Main repository`_ on Github.
* Documentation_ for all releases and current development tree.
* Download as a tar/zip file the `current trunk`_.

.. _main repository: http://github.com/nipy/dipy
.. _Documentation: http://dipy.org
.. _current trunk: https://github.com/nipy/dipy/archive/master.zip

License
=======

DIPY is licensed under the terms of the BSD license.
Please see the LICENSE file in the dipy distribution.

DIPY uses other libraries also licensed under the BSD or the
MIT licenses.
"""

# versions for dependencies
# Check these versions against .travis.yml and requirements.txt
CYTHON_MIN_VERSION = '0.29'
NUMPY_MIN_VERSION = '1.9.0'
SCIPY_MIN_VERSION = '1.0'
NIBABEL_MIN_VERSION = '2.4.0'
H5PY_MIN_VERSION = '2.4.0'

# Main setup parameters
NAME                = 'dipy'
MAINTAINER          = "Eleftherios Garyfallidis"
MAINTAINER_EMAIL    = "neuroimaging@python.org"
DESCRIPTION         = description
LONG_DESCRIPTION    = long_description
URL                 = "http://dipy.org"
DOWNLOAD_URL        = "http://github.com/nipy/dipy/archives/master"
LICENSE             = "BSD license"
CLASSIFIERS         = CLASSIFIERS
AUTHOR              = "dipy developers"
AUTHOR_EMAIL        = "neuroimaging@python.org"
PLATFORMS           = "OS Independent"
MAJOR               = _version_major
MINOR               = _version_minor
MICRO               = _version_micro
ISRELEASE           = _version_extra == ''
VERSION             = __version__
PROVIDES            = ["dipy"]
REQUIRES            = ["numpy (>=%s)" % NUMPY_MIN_VERSION,
                       "scipy (>=%s)" % SCIPY_MIN_VERSION,
                       "nibabel (>=%s)" % NIBABEL_MIN_VERSION,
                       "h5py (>=%s)" % H5PY_MIN_VERSION]
<|endoftext|>


<commit_before>from collections import namedtuple

from django.db import migrations

def add_users_to_groups_based_on_users_permissions(apps, schema_editor):
    """Add every user to group with "user_permissions" if exists, else create new one.

    For each user, if the group with the exact scope of permissions exists,
    add the user to it, else create a new group with this scope of permissions
    and add the user to it.
    """
    User = apps.get_model("account", "User")
    Group = apps.get_model("auth", "Group")
    GroupData = namedtuple("GroupData", ["users", "group_name"])

    groups = Group.objects.all().prefetch_related("permissions")

    mapping = create_permissions_mapping(User, GroupData)
    for perms, group_data in mapping.items():
        group = get_group_with_given_permissions(perms, groups)
        users = group_data.users
        if group:
            group.user_set.add(*users)
            continue
        group = create_group_with_given_permissions(perms, group_data.group_name, Group)
        group.user_set.add(*users)

def create_permissions_mapping(User, GroupData):
    """Create mapping permissions to users and potential new group name."""
    mapping = {}
    users = User.objects.filter(user_permissions__isnull=False).prefetch_related(
        "user_permissions"
    )
    for user in users:
        permissions = user.user_permissions.all()
        perm_pks = (perm.pk for perm in permissions)
        if perm_pks not in mapping:
            group_name = create_group_name(permissions)
            mapping[perm_pks] = GroupData({user.pk}, group_name)
        else:
            mapping[perm_pks].users.add(user.pk)
        user.user_permissions.clear()
    return mapping

def create_group_name(permissions):
    """Create group name based on permissions."""
    formatted_names = [perm.name.rstrip(".").lower() for perm in permissions]
    group_name = ", ".join(formatted_names).capitalize()
    return group_name

def get_group_with_given_permissions(permissions, groups):
    """Get group with given set of permissions."""
    for group in groups:
        group_perm_pks = {perm.pk for perm in group.permissions.all()}
        if group_perm_pks == set(permissions):
            return group

def create_group_with_given_permissions(perm_pks, group_name, Group):
    """Create new group with given set of permissions."""
    group = Group.objects.create(name=group_name)
    group.permissions.add(*perm_pks)
    return group

class Migration(migrations.Migration):

    dependencies = [
        ("account", "0040_auto_20200415_0443"),
    ]
    operations = [
        migrations.RunPython(
            add_users_to_groups_based_on_users_permissions, migrations.RunPython.noop
        ),
    ]
<commit_msg>Fix permissions to groups mutation<commit_after>import re
from collections import defaultdict

from django.db import migrations

def add_users_to_groups_based_on_users_permissions(apps, schema_editor):
    """Add every user to group with "user_permissions" if exists, else create new one.

    For each user, if the group with the exact scope of permissions exists,
    add the user to it, else create a new group with this scope of permissions
    and add the user to it.
    """
    User = apps.get_model("account", "User")
    Group = apps.get_model("auth", "Group")

    groups = Group.objects.all().prefetch_related("permissions")
    counter = get_counter_value(Group)
    mapping = create_permissions_mapping(User)
    for perms, users in mapping.items():
        group = get_group_with_given_permissions(perms, groups)
        if group:
            group.user_set.add(*users)
            continue
        group = create_group_with_given_permissions(perms, counter, Group)
        group.user_set.add(*users)
        counter += 1

def get_counter_value(Group):
    """Get the number of next potential group."""
    pattern = r"^Group (\d+)$"
    group = Group.objects.filter(name__iregex=pattern).order_by("name").last()
    if not group:
        return 1
    return int(re.match(pattern, group.name).group(1)) + 1

def create_permissions_mapping(User):
    """Create mapping permissions to users and potential new group name."""
    mapping = defaultdict(set)
    users = (
        User.objects.filter(user_permissions__isnull=False)
        .distinct()
        .prefetch_related("user_permissions")
    )
    for user in users:
        permissions = user.user_permissions.all().order_by("pk")
        perm_pks = tuple([perm.pk for perm in permissions])
        mapping[perm_pks].add(user.pk)
        user.user_permissions.clear()
    return mapping

def get_group_with_given_permissions(permissions, groups):
    """Get group with given set of permissions."""
    for group in groups:
        group_perm_pks = {perm.pk for perm in group.permissions.all()}
        if group_perm_pks == set(permissions):
            return group

def create_group_with_given_permissions(perm_pks, counter, Group):
    """Create new group with given set of permissions."""
    group_name = f"Group {counter:03d}"
    group = Group.objects.create(name=group_name)
    group.permissions.add(*perm_pks)
    return group

class Migration(migrations.Migration):

    dependencies = [
        ("account", "0040_auto_20200415_0443"),
    ]
    operations = [
        migrations.RunPython(
            add_users_to_groups_based_on_users_permissions, migrations.RunPython.noop
        ),
    ]
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
import kfp
from kfp import components
from kfp import dsl
import os
import subprocess

diagnose_me_op = components.load_component_from_url(
    'https://raw.githubusercontent.com/kubeflow/pipelines/566dddfdfc0a6a725b6e50ea85e73d8d5578bbb9/components/diagnostics/diagnose_me/component.yaml')

confusion_matrix_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/1.3.0/components/local/confusion_matrix/component.yaml')

roc_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/1.3.0/components/local/roc/component.yaml')

dataproc_create_cluster_op = components.load_component_from_url(
    'https://raw.githubusercontent.com/kubeflow/pipelines/1.3.0/components/gcp/dataproc/create_cluster/component.yaml')

dataproc_delete_cluster_op = components.load_component_from_url(
    'https://raw.githubusercontent.com/kubeflow/pipelines/1.3.0/components/gcp/dataproc/delete_cluster/component.yaml')

dataproc_submit_pyspark_op = components.load_component_from_url(
    'https://raw.githubusercontent.com/kubeflow/pipelines/1.3.0/components/gcp/dataproc/submit_pyspark_job/component.yaml'
)

dataproc_submit_spark_op = components.load_component_from_url(
    'https://raw.githubusercontent.com/kubeflow/pipelines/1.3.0/components/gcp/dataproc/submit_spark_job/component.yaml'
)

_PYSRC_PREFIX = 'gs://ml-pipeline/sample-pipeline/xgboost' # Common path to python src.

_XGBOOST_PKG = 'gs://ml-pipeline/sample-pipeline/xgboost/xgboost4j-example-0.8-SNAPSHOT-jar-with-dependencies.jar'

_TRAINER_MAIN_CLS = 'ml.dmlc.xgboost4j.scala.example.spark.XGBoostTrainer'

_PREDICTOR_MAIN_CLS = 'ml.dmlc.xgboost4j.scala.example.spark.XGBoostPredictor'

def delete_directory_from_gcs(dir_path):
  """Delete a GCS dir recursively. Ignore errors."""
  try:
    subprocess.call(['gsutil', '-m', 'rm', '-r', dir_path])
  except:
    pass

# ! Please do not forget to enable the Dataproc API in your cluster https://console.developers.google.com/apis/api/dataproc.googleapis.com/overview

# ================================================================
# The following classes should be provided by components provider.

def dataproc_analyze_op(
    project,
    region,
    cluster_name,
    schema,
    train_data,
    output):
  """Submit dataproc analyze as a pyspark job.

  :param project: GCP project ID.
  :param region: Which zone to run this analyze.
  :param cluster_name: Name of the cluster.
  :param schema: GCS path to the schema.
  :param train_data: GCS path to the training data.
  :param output: GCS path to store the output.
  """
  return dataproc_submit_pyspark_op(
      project_id=project,
      region=region,
      cluster_name=cluster_name,
      main_python_file_uri=os.path.join(_PYSRC_PREFIX, 'analyze_run.py'),
      args=['--output', str(output), '--train', str(train_data), '--schema', str(schema)]
  )

def dataproc_transform_op(
    project,
    region,
    cluster_name,
    train_data,
    eval_data,
    target,
    analysis,
    output
):
  """Submit dataproc transform as a pyspark job.

  :param project: GCP project ID.
  :param region: Which zone to run this analyze.
  :param cluster_name: Name of the cluster.
  :param train_data: GCS path to the training data.
  :param eval_data: GCS path of the eval csv file.
  :param target: Target column name.
  :param analysis: GCS path of the analysis results
  :param output: GCS path to use for output.
  """

  # Remove existing [output]/train and [output]/eval if they exist.
  delete_directory_from_gcs(os.path.join(output, 'train'))
  delete_directory_from_gcs(os.path.join(output, 'eval'))

  return dataproc_submit_pyspark_op(
      project_id=project,
      region=region,
      cluster_name=cluster_name,
      main_python_file_uri=os.path.join(_PYSRC_PREFIX,
                                        'transform_run.py'),
      args=[
        '--output',
        str(output),
        '--analysis',
        str(analysis),
        '--target',
        str(target),
        '--train',
        str(train_data),
        '--eval',
        str(eval_data)
      ])

def dataproc_train_op(
    project,
    region,
    cluster_name,
    train_data,
    eval_data,
    target,
    analysis,
    workers,
    rounds,
    output,
    is_classification=True
):

  if is_classification:
    config='gs://ml-pipeline/sample-data/xgboost-config/trainconfcla.json'
  else:
    config='gs://ml-pipeline/sample-data/xgboost-config/trainconfreg.json'

  return dataproc_submit_spark_op(
      project_id=project,
      region=region,
      cluster_name=cluster_name,
      main_class=_TRAINER_MAIN_CLS,
      spark_job=json.dumps({'jarFileUris': [_XGBOOST_PKG]}),
      args=json.dumps([
        str(config),
        str(rounds),
        str(workers),
        str(analysis),
        str(target),
        str(train_data),
        str(eval_data),
        str(output)
      ]))

def dataproc_predict_op(
    project,
    region,
    cluster_name,
    data,
    model,
    target,
    analysis,
    output
):

  return dataproc_submit_spark_op(
      project_id=project,
      region=region,
      cluster_name=cluster_name,
      main_class=_PREDICTOR_MAIN_CLS,
      spark_job=json.dumps({'jarFileUris': [_XGBOOST_PKG]}),
      args=json.dumps([
        str(model),
        str(data),
        str(analysis),
        str(target),
        str(output)
      ]))

# =======================================================================

@dsl.pipeline(
    name='XGBoost Trainer',
    description='A trainer that does end-to-end distributed training for XGBoost models.'
)
def xgb_train_pipeline(
    output='gs://{{kfp-default-bucket}}',
    project='{{kfp-project-id}}',
    diagnostic_mode='HALT_ON_ERROR',
    rounds=5,
):
    output_template = str(output) + '/' + dsl.RUN_ID_PLACEHOLDER + '/data'
    region='us-central1'
    workers=2
    quota_check=[{'region':region,'metric':'CPUS','quota_needed':12.0}]
    train_data='gs://ml-pipeline/sample-data/sfpd/train.csv'
    eval_data='gs://ml-pipeline/sample-data/sfpd/eval.csv'
    schema='gs://ml-pipeline/sample-data/sfpd/schema.json'
    true_label='ACTION'
    target='resolution'
    required_apis='dataproc.googleapis.com'
    cluster_name='xgb-%s' % dsl.RUN_ID_PLACEHOLDER

    # Current GCP pyspark/spark op do not provide outputs as return values, instead,
    # we need to use strings to pass the uri around.
    analyze_output = output_template
    transform_output_train = os.path.join(output_template, 'train', 'part-*')
    transform_output_eval = os.path.join(output_template, 'eval', 'part-*')
    train_output = os.path.join(output_template, 'train_output')
    predict_output = os.path.join(output_template, 'predict_output')

    _diagnose_me_op = diagnose_me_op(
        bucket=output,
        execution_mode=diagnostic_mode,
        project_id=project,
        target_apis=required_apis,
        quota_check=quota_check)

    with dsl.ExitHandler(exit_op=dataproc_delete_cluster_op(
        project_id=project,
        region=region,
        name=cluster_name
    )):
        _create_cluster_op = dataproc_create_cluster_op(
            project_id=project,
            region=region,
            name=cluster_name,
            initialization_actions=[
              os.path.join(_PYSRC_PREFIX,
                           'initialization_actions.sh'),
            ],
            image_version='1.2'
        ).after(_diagnose_me_op)

        _analyze_op = dataproc_analyze_op(
            project=project,
            region=region,
            cluster_name=cluster_name,
            schema=schema,
            train_data=train_data,
            output=output_template
        ).after(_create_cluster_op).set_display_name('Analyzer')

        _transform_op = dataproc_transform_op(
            project=project,
            region=region,
            cluster_name=cluster_name,
            train_data=train_data,
            eval_data=eval_data,
            target=target,
            analysis=analyze_output,
            output=output_template
        ).after(_analyze_op).set_display_name('Transformer')

        _train_op = dataproc_train_op(
            project=project,
            region=region,
            cluster_name=cluster_name,
            train_data=transform_output_train,
            eval_data=transform_output_eval,
            target=target,
            analysis=analyze_output,
            workers=workers,
            rounds=rounds,
            output=train_output
        ).after(_transform_op).set_display_name('Trainer')

        _predict_op = dataproc_predict_op(
            project=project,
            region=region,
            cluster_name=cluster_name,
            data=transform_output_eval,
            model=train_output,
            target=target,
            analysis=analyze_output,
            output=predict_output
        ).after(_train_op).set_display_name('Predictor')

        _cm_op = confusion_matrix_op(
            predictions=os.path.join(predict_output, 'part-*.csv'),
            output_dir=output_template
        ).after(_predict_op)

        _roc_op = roc_op(
            predictions_dir=os.path.join(predict_output, 'part-*.csv'),
            true_class=true_label,
            true_score_column=true_label,
            output_dir=output_template
        ).after(_predict_op)

if __name__ == '__main__':
    kfp.compiler.Compiler().compile(xgb_train_pipeline, __file__ + '.yaml')
<commit_msg>fix(samples): Bump dataproc image version to 1.5 in xgboost_training_cm.py (#5062)<commit_after>#!/usr/bin/env python3
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json
import kfp
from kfp import components
from kfp import dsl
import os
import subprocess

diagnose_me_op = components.load_component_from_url(
    'https://raw.githubusercontent.com/kubeflow/pipelines/566dddfdfc0a6a725b6e50ea85e73d8d5578bbb9/components/diagnostics/diagnose_me/component.yaml')

confusion_matrix_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/1.3.0/components/local/confusion_matrix/component.yaml')

roc_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/1.3.0/components/local/roc/component.yaml')

dataproc_create_cluster_op = components.load_component_from_url(
    'https://raw.githubusercontent.com/kubeflow/pipelines/1.3.0/components/gcp/dataproc/create_cluster/component.yaml')

dataproc_delete_cluster_op = components.load_component_from_url(
    'https://raw.githubusercontent.com/kubeflow/pipelines/1.3.0/components/gcp/dataproc/delete_cluster/component.yaml')

dataproc_submit_pyspark_op = components.load_component_from_url(
    'https://raw.githubusercontent.com/kubeflow/pipelines/1.3.0/components/gcp/dataproc/submit_pyspark_job/component.yaml'
)

dataproc_submit_spark_op = components.load_component_from_url(
    'https://raw.githubusercontent.com/kubeflow/pipelines/1.3.0/components/gcp/dataproc/submit_spark_job/component.yaml'
)

_PYSRC_PREFIX = 'gs://ml-pipeline/sample-pipeline/xgboost' # Common path to python src.

_XGBOOST_PKG = 'gs://ml-pipeline/sample-pipeline/xgboost/xgboost4j-example-0.8-SNAPSHOT-jar-with-dependencies.jar'

_TRAINER_MAIN_CLS = 'ml.dmlc.xgboost4j.scala.example.spark.XGBoostTrainer'

_PREDICTOR_MAIN_CLS = 'ml.dmlc.xgboost4j.scala.example.spark.XGBoostPredictor'

def delete_directory_from_gcs(dir_path):
  """Delete a GCS dir recursively. Ignore errors."""
  try:
    subprocess.call(['gsutil', '-m', 'rm', '-r', dir_path])
  except:
    pass

# ! Please do not forget to enable the Dataproc API in your cluster https://console.developers.google.com/apis/api/dataproc.googleapis.com/overview

# ================================================================
# The following classes should be provided by components provider.

def dataproc_analyze_op(
    project,
    region,
    cluster_name,
    schema,
    train_data,
    output):
  """Submit dataproc analyze as a pyspark job.

  :param project: GCP project ID.
  :param region: Which zone to run this analyze.
  :param cluster_name: Name of the cluster.
  :param schema: GCS path to the schema.
  :param train_data: GCS path to the training data.
  :param output: GCS path to store the output.
  """
  return dataproc_submit_pyspark_op(
      project_id=project,
      region=region,
      cluster_name=cluster_name,
      main_python_file_uri=os.path.join(_PYSRC_PREFIX, 'analyze_run.py'),
      args=['--output', str(output), '--train', str(train_data), '--schema', str(schema)]
  )

def dataproc_transform_op(
    project,
    region,
    cluster_name,
    train_data,
    eval_data,
    target,
    analysis,
    output
):
  """Submit dataproc transform as a pyspark job.

  :param project: GCP project ID.
  :param region: Which zone to run this analyze.
  :param cluster_name: Name of the cluster.
  :param train_data: GCS path to the training data.
  :param eval_data: GCS path of the eval csv file.
  :param target: Target column name.
  :param analysis: GCS path of the analysis results
  :param output: GCS path to use for output.
  """

  # Remove existing [output]/train and [output]/eval if they exist.
  delete_directory_from_gcs(os.path.join(output, 'train'))
  delete_directory_from_gcs(os.path.join(output, 'eval'))

  return dataproc_submit_pyspark_op(
      project_id=project,
      region=region,
      cluster_name=cluster_name,
      main_python_file_uri=os.path.join(_PYSRC_PREFIX,
                                        'transform_run.py'),
      args=[
        '--output',
        str(output),
        '--analysis',
        str(analysis),
        '--target',
        str(target),
        '--train',
        str(train_data),
        '--eval',
        str(eval_data)
      ])

def dataproc_train_op(
    project,
    region,
    cluster_name,
    train_data,
    eval_data,
    target,
    analysis,
    workers,
    rounds,
    output,
    is_classification=True
):

  if is_classification:
    config='gs://ml-pipeline/sample-data/xgboost-config/trainconfcla.json'
  else:
    config='gs://ml-pipeline/sample-data/xgboost-config/trainconfreg.json'

  return dataproc_submit_spark_op(
      project_id=project,
      region=region,
      cluster_name=cluster_name,
      main_class=_TRAINER_MAIN_CLS,
      spark_job=json.dumps({'jarFileUris': [_XGBOOST_PKG]}),
      args=json.dumps([
        str(config),
        str(rounds),
        str(workers),
        str(analysis),
        str(target),
        str(train_data),
        str(eval_data),
        str(output)
      ]))

def dataproc_predict_op(
    project,
    region,
    cluster_name,
    data,
    model,
    target,
    analysis,
    output
):

  return dataproc_submit_spark_op(
      project_id=project,
      region=region,
      cluster_name=cluster_name,
      main_class=_PREDICTOR_MAIN_CLS,
      spark_job=json.dumps({'jarFileUris': [_XGBOOST_PKG]}),
      args=json.dumps([
        str(model),
        str(data),
        str(analysis),
        str(target),
        str(output)
      ]))

# =======================================================================

@dsl.pipeline(
    name='XGBoost Trainer',
    description='A trainer that does end-to-end distributed training for XGBoost models.'
)
def xgb_train_pipeline(
    output='gs://{{kfp-default-bucket}}',
    project='{{kfp-project-id}}',
    diagnostic_mode='HALT_ON_ERROR',
    rounds=5,
):
    output_template = str(output) + '/' + dsl.RUN_ID_PLACEHOLDER + '/data'
    region='us-central1'
    workers=2
    quota_check=[{'region':region,'metric':'CPUS','quota_needed':12.0}]
    train_data='gs://ml-pipeline/sample-data/sfpd/train.csv'
    eval_data='gs://ml-pipeline/sample-data/sfpd/eval.csv'
    schema='gs://ml-pipeline/sample-data/sfpd/schema.json'
    true_label='ACTION'
    target='resolution'
    required_apis='dataproc.googleapis.com'
    cluster_name='xgb-%s' % dsl.RUN_ID_PLACEHOLDER

    # Current GCP pyspark/spark op do not provide outputs as return values, instead,
    # we need to use strings to pass the uri around.
    analyze_output = output_template
    transform_output_train = os.path.join(output_template, 'train', 'part-*')
    transform_output_eval = os.path.join(output_template, 'eval', 'part-*')
    train_output = os.path.join(output_template, 'train_output')
    predict_output = os.path.join(output_template, 'predict_output')

    _diagnose_me_op = diagnose_me_op(
        bucket=output,
        execution_mode=diagnostic_mode,
        project_id=project,
        target_apis=required_apis,
        quota_check=quota_check)

    with dsl.ExitHandler(exit_op=dataproc_delete_cluster_op(
        project_id=project,
        region=region,
        name=cluster_name
    )):
        _create_cluster_op = dataproc_create_cluster_op(
            project_id=project,
            region=region,
            name=cluster_name,
            initialization_actions=[
              os.path.join(_PYSRC_PREFIX,
                           'initialization_actions.sh'),
            ],
            image_version='1.5'
        ).after(_diagnose_me_op)

        _analyze_op = dataproc_analyze_op(
            project=project,
            region=region,
            cluster_name=cluster_name,
            schema=schema,
            train_data=train_data,
            output=output_template
        ).after(_create_cluster_op).set_display_name('Analyzer')

        _transform_op = dataproc_transform_op(
            project=project,
            region=region,
            cluster_name=cluster_name,
            train_data=train_data,
            eval_data=eval_data,
            target=target,
            analysis=analyze_output,
            output=output_template
        ).after(_analyze_op).set_display_name('Transformer')

        _train_op = dataproc_train_op(
            project=project,
            region=region,
            cluster_name=cluster_name,
            train_data=transform_output_train,
            eval_data=transform_output_eval,
            target=target,
            analysis=analyze_output,
            workers=workers,
            rounds=rounds,
            output=train_output
        ).after(_transform_op).set_display_name('Trainer')

        _predict_op = dataproc_predict_op(
            project=project,
            region=region,
            cluster_name=cluster_name,
            data=transform_output_eval,
            model=train_output,
            target=target,
            analysis=analyze_output,
            output=predict_output
        ).after(_train_op).set_display_name('Predictor')

        _cm_op = confusion_matrix_op(
            predictions=os.path.join(predict_output, 'part-*.csv'),
            output_dir=output_template
        ).after(_predict_op)

        _roc_op = roc_op(
            predictions_dir=os.path.join(predict_output, 'part-*.csv'),
            true_class=true_label,
            true_score_column=true_label,
            output_dir=output_template
        ).after(_predict_op)

if __name__ == '__main__':
    kfp.compiler.Compiler().compile(xgb_train_pipeline, __file__ + '.yaml')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''HealthCheckHandler
'''

from tornado.web import HTTPError

from twork_app.web.action.base import BaseHandler

class HealthCheckHandler(BaseHandler):
    '''HealthCheckHandler, RESTFUL SUPPORTED.
    '''

    ST_ITEM = 'HEALTH_CHECK'

    def post(self, *args, **kwargs):
        raise HTTPError(405)

    def delete(self, *args, **kwargs):
        raise HTTPError(405)

    def get(self, *args, **kwargs):
        self.api_response('It works!')
<commit_msg>Rewrite put method for health_check handler<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''HealthCheckHandler
'''

from tornado.web import HTTPError

from twork_app.web.action.base import BaseHandler

class HealthCheckHandler(BaseHandler):
    '''HealthCheckHandler, RESTFUL SUPPORTED.
    '''

    ST_ITEM = 'HEALTH_CHECK'

    def post(self, *args, **kwargs):
        raise HTTPError(405)

    def put(self, *args, **kwargs):
        raise HTTPError(405)

    def delete(self, *args, **kwargs):
        raise HTTPError(405)

    def get(self, *args, **kwargs):
        self.api_response('It works!')
<|endoftext|>


<commit_before>import openpnm as op
from openpnm.phases import mixtures
import numpy as np
ws = op.Workspace()
proj = ws.new_project()
# ws.settings['loglevel'] = 20

"""
    Details about the continum and numerical model equations can be found on:
    Agnaou, M., Sadeghi, M. A., Tranter, T. G., & Gostick, J. (2020).
    Modeling transport of charged species in pore networks: solution of the
    Nernst-Planck equations coupled with fluid flow and charge conservation
    equations.
    Computers & Geosciences, 104505.
"""

# network, geometry, phase
np.random.seed(0)

net = op.network.Cubic(shape=[8, 8, 1], spacing=9e-4, project=proj)
prs = (net['pore.back'] * net['pore.right'] + net['pore.back']
       * net['pore.left'] + net['pore.front'] * net['pore.right']
       + net['pore.front'] * net['pore.left'])
thrts = net['throat.surface']
op.topotools.trim(network=net, pores=net.Ps[prs], throats=net.Ts[thrts])

geo = op.geometry.StickAndBall(network=net, pores=net.Ps, throats=net.Ts)
pore_d = op.models.misc.constant
throat_d = op.models.misc.constant
geo.add_model(propname='pore.diameter', model=pore_d, value=1.5e-4)
geo.add_model(propname='throat.diameter', model=throat_d, value=1e-4)
geo.regenerate_models()

sw = mixtures.SalineWater(network=net)
# Retrieve handles to each species for use below
Na = sw.components['Na_' + sw.name]
Cl = sw.components['Cl_' + sw.name]
H2O = sw.components['H2O_' + sw.name]

# physics
phys = op.physics.GenericPhysics(network=net, phase=sw, geometry=geo)

flow = op.models.physics.hydraulic_conductance.hagen_poiseuille
phys.add_model(propname='throat.hydraulic_conductance',
               pore_viscosity='pore.viscosity',
               throat_viscosity='throat.viscosity',
               model=flow, regen_mode='normal')

current = op.models.physics.ionic_conductance.electroneutrality
phys.add_model(propname='throat.ionic_conductance', ions=[Na.name, Cl.name],
               model=current, regen_mode='normal')

eA_dif = op.models.physics.diffusive_conductance.ordinary_diffusion
phys.add_model(propname='throat.diffusive_conductance.' + Na.name,
               pore_diffusivity='pore.diffusivity.' + Na.name,
               throat_diffusivity='throat.diffusivity.' + Na.name,
               model=eA_dif, regen_mode='normal')

eB_dif = op.models.physics.diffusive_conductance.ordinary_diffusion
phys.add_model(propname='throat.diffusive_conductance.' + Cl.name,
               pore_diffusivity='pore.diffusivity.' + Cl.name,
               throat_diffusivity='throat.diffusivity.' + Cl.name,
               model=eB_dif, regen_mode='normal')

# settings for algorithms
setts1 = {'solver_max_iter': 5, 'solver_tol': 1e-08, 'solver_rtol': 1e-08,
          'nlin_max_iter': 10}
setts2 = {'g_tol': 1e-4, 'g_max_iter': 4, 't_output': 5000, 't_step': 500,
          't_final': 20000, 't_scheme': 'implicit'}

# algorithms
sf = op.algorithms.StokesFlow(network=net, phase=sw, settings=setts1)
sf.set_value_BC(pores=net.pores('back'), values=0.01)
sf.set_value_BC(pores=net.pores('front'), values=0.00)
sf.run()
sw.update(sf.results())

p = op.algorithms.TransientIonicConduction(network=net, phase=sw,
                                           settings=setts1)
p.set_value_BC(pores=net.pores('left'), values=0.1)
p.set_value_BC(pores=net.pores('right'), values=0.00)
p.settings['charge_conservation'] = 'electroneutrality'

eA = op.algorithms.TransientNernstPlanck(network=net, phase=sw, ion=Na.name,
                                         settings=setts1)
eA.set_value_BC(pores=net.pores('back'), values=100)
eA.set_value_BC(pores=net.pores('front'), values=90)

eB = op.algorithms.TransientNernstPlanck(network=net, phase=sw, ion=Cl.name,
                                         settings=setts1)
eB.set_value_BC(pores=net.pores('back'), values=100)
eB.set_value_BC(pores=net.pores('front'), values=90)

ad_dif_mig_Na = op.models.physics.ad_dif_mig_conductance.ad_dif_mig
phys.add_model(propname='throat.ad_dif_mig_conductance.' + Na.name,
               pore_pressure=sf.settings['quantity'],
               model=ad_dif_mig_Na, ion=Na.name,
               s_scheme='powerlaw')

ad_dif_mig_Cl = op.models.physics.ad_dif_mig_conductance.ad_dif_mig
phys.add_model(propname='throat.ad_dif_mig_conductance.' + Cl.name,
               pore_pressure=sf.settings['quantity'],
               model=ad_dif_mig_Cl, ion=Cl.name,
               s_scheme='powerlaw')

it = op.algorithms.TransientNernstPlanckMultiphysicsSolver(network=net,
                                                           phase=sw,
                                                           settings=setts2)
it.setup(potential_field=p.name, ions=[eA.name, eB.name])
it.run()

sw.update(sf.results())
sw.update(p.results())
sw.update(eA.results())
sw.update(eB.results())

# output results to a vtk file for visualization on Paraview
# proj.export_data(phases=[sw], filename='OUT', filetype='xdmf')
<commit_msg>updated transient pnp example<commit_after>import openpnm as op
from openpnm.phases import mixtures
import numpy as np
ws = op.Workspace()
proj = ws.new_project()
# ws.settings['loglevel'] = 20

"""
    Details about the continum and numerical model equations can be found on:
    Agnaou, M., Sadeghi, M. A., Tranter, T. G., & Gostick, J. (2020).
    Modeling transport of charged species in pore networks: solution of the
    Nernst-Planck equations coupled with fluid flow and charge conservation
    equations.
    Computers & Geosciences, 104505.
"""

# network, geometry, phase
np.random.seed(0)

net = op.network.Cubic(shape=[8, 8, 1], spacing=9e-4, project=proj)
prs = (net['pore.back'] * net['pore.right'] + net['pore.back']
       * net['pore.left'] + net['pore.front'] * net['pore.right']
       + net['pore.front'] * net['pore.left'])
thrts = net['throat.surface']
op.topotools.trim(network=net, pores=net.Ps[prs], throats=net.Ts[thrts])

geo = op.geometry.StickAndBall(network=net, pores=net.Ps, throats=net.Ts)
pore_d = op.models.misc.constant
throat_d = op.models.misc.constant
geo.add_model(propname='pore.diameter', model=pore_d, value=1.5e-4)
geo.add_model(propname='throat.diameter', model=throat_d, value=1e-4)
geo.regenerate_models()

sw = mixtures.SalineWater(network=net)
# Retrieve handles to each species for use below
Na = sw.components['Na_' + sw.name]
Cl = sw.components['Cl_' + sw.name]
H2O = sw.components['H2O_' + sw.name]

# physics
phys = op.physics.GenericPhysics(network=net, phase=sw, geometry=geo)

flow = op.models.physics.hydraulic_conductance.hagen_poiseuille
phys.add_model(propname='throat.hydraulic_conductance',
               pore_viscosity='pore.viscosity',
               throat_viscosity='throat.viscosity',
               model=flow, regen_mode='normal')

current = op.models.physics.ionic_conductance.electroneutrality
phys.add_model(propname='throat.ionic_conductance', ions=[Na.name, Cl.name],
               model=current, regen_mode='normal')

eA_dif = op.models.physics.diffusive_conductance.ordinary_diffusion
phys.add_model(propname='throat.diffusive_conductance.' + Na.name,
               pore_diffusivity='pore.diffusivity.' + Na.name,
               throat_diffusivity='throat.diffusivity.' + Na.name,
               model=eA_dif, regen_mode='normal')

eB_dif = op.models.physics.diffusive_conductance.ordinary_diffusion
phys.add_model(propname='throat.diffusive_conductance.' + Cl.name,
               pore_diffusivity='pore.diffusivity.' + Cl.name,
               throat_diffusivity='throat.diffusivity.' + Cl.name,
               model=eB_dif, regen_mode='normal')

s_scheme = 'powerlaw'
ad_dif_mig_Na = op.models.physics.ad_dif_mig_conductance.ad_dif_mig
phys.add_model(propname='throat.ad_dif_mig_conductance.' + Na.name,
               pore_pressure='pore.pressure', model=ad_dif_mig_Na,
               ion=Na.name, s_scheme=s_scheme)

ad_dif_mig_Cl = op.models.physics.ad_dif_mig_conductance.ad_dif_mig
phys.add_model(propname='throat.ad_dif_mig_conductance.' + Cl.name,
               pore_pressure='pore.pressure', model=ad_dif_mig_Cl,
               ion=Cl.name, s_scheme=s_scheme)

# settings for algorithms
setts1 = {'solver_max_iter': 5, 'solver_tol': 1e-08, 'solver_rtol': 1e-08,
          'nlin_max_iter': 10, 'cache_A': False, 'cache_b': False}
setts2 = {'g_tol': 1e-4, 'g_max_iter': 4, 't_output': 5000, 't_step': 500,
          't_final': 20000, 't_scheme': 'implicit', 'cache_A': False,
          'cache_b': False}

# algorithms
sf = op.algorithms.StokesFlow(network=net, phase=sw, settings=setts1)
sf.set_value_BC(pores=net.pores('back'), values=0.01)
sf.set_value_BC(pores=net.pores('front'), values=0.00)
sf.run()
sw.update(sf.results())

p = op.algorithms.TransientIonicConduction(network=net, phase=sw,
                                           settings=setts1)
p.set_value_BC(pores=net.pores('left'), values=0.1)
p.set_value_BC(pores=net.pores('right'), values=0.00)
p.settings['charge_conservation'] = 'electroneutrality'

eA = op.algorithms.TransientNernstPlanck(network=net, phase=sw, ion=Na.name,
                                         settings=setts1)
eA.set_value_BC(pores=net.pores('back'), values=100)
eA.set_value_BC(pores=net.pores('front'), values=90)

eB = op.algorithms.TransientNernstPlanck(network=net, phase=sw, ion=Cl.name,
                                         settings=setts1)
eB.set_value_BC(pores=net.pores('back'), values=100)
eB.set_value_BC(pores=net.pores('front'), values=90)

it = op.algorithms.TransientNernstPlanckMultiphysicsSolver(network=net,
                                                           phase=sw,
                                                           settings=setts2)
it.setup(potential_field=p.name, ions=[eA.name, eB.name])
it.run()

sw.update(sf.results())
sw.update(p.results())
sw.update(eA.results())
sw.update(eB.results())

# output results to a vtk file for visualization on Paraview
# proj.export_data(phases=[sw], filename='OUT', filetype='xdmf')
<|endoftext|>