<commit_before>#!/usr/bin/env python

# simple pyCan Example for can4linux
import time
import datetime
import signal
import sys
import subprocess
import struct
import json
import os

sys.path.insert(0, '/root/can4linux-code/can4linux-examples')
import pyCan

# the power expressed in (mW)
# default is around 660
# it must be an Integer greater then 45
# 45mW allows to detect tags at aroud 50cm distance
antennaPower = 2000

datafolder = '/root/slope-data/'
logFolder = '/root/slope-log/'

def get_incremental_number():
	maxcounter = 0
	listFiles = os.listdir(datafolder)
	for file in listFiles:
		if os.path.isfile(datafolder + file):
			name = file.split('_')
			if len(name) > 0:
				counter = name[0]
				if counter.isdigit():
					if counter > maxcounter:
				 		maxcounter = counter
	return maxcounter + 1

incremental = str(get_incremental_number())
msgFilePath = datafolder + incremental + '_canbus-messages.txt'
logFilePath = logFolder + incremental + '_slope-canbus.log'

streamFilePath = datafolder + incremental + '_slope-stream.txt'

def get_lifting_status(status):
	if status == 0:
		return "stop"
	elif status == 1:
		return "going up"
	elif status == 2:
		return "going down"
	else:
		return "error"

def get_translation_status(status):
	if status == 0:
		return "stop"
	elif status == 1:
		return "going to load"
	elif status == 2:
		return "going to release"
	elif status == 3:
		return "antenna down"
	else:
		return "error"

def write_log(text):
	logfile = open(logFilePath, 'a', 1)
	now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
	logfile.write('[' + now + '] ' + text + '\n')
	logfile.close()

def write_msg(text):
	msgfile = open(msgFilePath, 'a+', 1)
	# now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
	msgfile.write(text + '\n')
	msgfile.close()

def write_stream(text):
	msgfile = open(streamFilePath, 'a+', 1)
	# now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
	msgfile.write(text + '\n')
	msgfile.close()

def get_timestamp():
	return str(int(round(time.time() * 1000)))

def sigterm_handler(_signo, _stack_frame):
	# Raises SystemExit(0):
	sys.exit(0)

signal.signal(signal.SIGTERM, sigterm_handler)

#print 'Python wrapper loaded'
#print datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
write_log('Slope-Canbus Version 1.0')
write_log('Python wrapper loaded')

# setting the device number
device = 0
defaultBaudrate = '250'

# open the can interface /dev/can0
try:
	fileBaud = open('/proc/sys/dev/Can/Baud', 'r+')
	baudRates = fileBaud.read().split()
	baudRates[device] = defaultBaudrate
	fileBaud.write(' '.join(baudRates))
	fileBaud.close()

except IOError:
	write_log('Could not set the new bitrate')
	exit()

except IndexError:
	write_log('No proper entry for bitrate of can' + str(device) + ' found')
	exit()

# print 'open can'+ str(device) + ' none blocking'
can_fd = pyCan.open(device)

if can_fd == -1:
	write_log('Error opening CAN device /dev/can' + str(device))
	exit()

try:
	write_log('Wait for message...')
	count = 0
	flip = '0'
	lastMsg30 = ""
	lastMsg31 = ""
	lastLifting = 0
	while True:
		data = pyCan.read(can_fd)
		#write_stream(data) #debug: write the canbus stream as is
		arrMess = data.split(':', 3)
		messId = arrMess[0][:len(arrMess[0]) - 1]
		if messId.isdigit():
			messId = int(messId)
		if messId == 30:
			# print messId
			start = arrMess[1].find("(")
			end = arrMess[1].find(")")
			bytesNum = arrMess[1][start + 1:end]
			# print bytesNum
			messBytes = arrMess[2][1:]
			if messBytes != lastMsg30:  # discard new message if is equal to last one
				lastMsg30 = messBytes
				arrBytes = messBytes.split()
				if bytesNum.isdigit():
					bytesNum = int(bytesNum)
				if bytesNum == len(arrBytes):
					weight = int(struct.unpack('<h', "".join(map(chr, [int(arrBytes[0]), int(arrBytes[1])])))[0])
					# (kg) kilogrammes
					position = int(struct.unpack('<h', "".join(map(chr, [int(arrBytes[2]), int(arrBytes[3])])))[0])
					# (m) meters
					speed = float(struct.unpack('<f', "".join(map(chr, [int(arrBytes[4]), int(arrBytes[5]), int(arrBytes[6]), int(arrBytes[7])])))[0])
					# (m/s) meters at second

					# decide what to do...
					# strOutput = str(weight) + "kg " + str(position) + "m " + str(speed) + "m/s "
					# write_msg("%d:%s" % (messId, strOutput))
					objOutput = {"weight": weight, "position": position, "speed": speed, "timestamp": get_timestamp()}
					write_msg(json.dumps(objOutput))

		if messId == 31:
			# print messId
			start = arrMess[1].find("(")
			end = arrMess[1].find(")")
			bytesNum = arrMess[1][start + 1:end]
			# print bytesNum
			messBytes = arrMess[2][1:]
			if messBytes != lastMsg31:  # discard new message if is equal to last one
				lastMsg31 = messBytes
				arrBytes = messBytes.split()
				if bytesNum.isdigit():
					bytesNum = int(bytesNum)
				if bytesNum == len(arrBytes):
					axleX = int(arrBytes[0])
					# (%) percentage of X axle grade (frontal)
					axleY = int(arrBytes[1])
					# (%) percentage of Y axle grade (lateral)
					consumption = int(struct.unpack('<h', "".join(map(chr, [int(arrBytes[2]), int(arrBytes[3])])))[0])
					# (l/h) liters at hour
					lifting = get_lifting_status(int(arrBytes[4]))
					# lifting status (0='stop', 1='going up', 2='going down', 3='antenna down')
					translation = get_translation_status(int(arrBytes[5]))
					# translation status (0='stop', 1='going to load', 2='going to release')

					# decide what to do...
					# strOutput = str(axleX) + "% " + str(axleY) + "% " + str(consumption) + "l/h " + lifting + " " + translation
					# write_msg("%d:%s" % (messId, strOutput))
					objOutput = {"axleX": axleX, "axleY": axleY, "consumption": consumption, "lifting": lifting, "translation": translation, "timestamp": get_timestamp()}
					write_msg(json.dumps(objOutput))
					lastLifting = int(arrBytes[4])

		count += 1

		#every 0.5 sec see below (10 msg per second)
		if count % 5 == 0:
			try:
				if lastLifting == 3:
					subprocess.check_call("java -jar /root/slope-bananapro/TagsReader.jar " + str(antennaPower) + " " + incremental, shell=True)
					lastLifting = 0
			except subprocess.CalledProcessError:
				write_log('Error executing lib: TagsReader.jar')

		#every 5 sec upload data and send watchdog
		if count == 50:
			count = 0
			pyCan.send(can_fd, 1, '60:' + flip)
			if flip == '0':
				flip = '1'
			if flip == '1':
				flip = '0'
			try:
				subprocess.check_call("python upload-data.py", shell=True)
			except subprocess.CalledProcessError:
				write_log('Error executing: Upload-data.py')

		time.sleep(0.1)

except KeyboardInterrupt:
	write_log('Program exits with ctrl+c')

finally:
	pyCan.close(can_fd)
	write_log('/dev/can' + str(device) + ' closed in finally')

# print 'Wait ' + str(timeout) + ' sec for an message.....'
# print pyCan.read1(can_fd, timeout * 1000000)
# print 'Wait default timeout for an message.....'
# print pyCan.read1(can_fd)

# pyCan.close(can_fd)
# write_log('/dev/can' + str(device) + ' closed at the end')
# logFile.close()
# msgFile.close()
# exit()
<commit_msg>added logs<commit_after>#!/usr/bin/env python

# simple pyCan Example for can4linux
import time
import datetime
import signal
import sys
import subprocess
import struct
import json
import os

sys.path.insert(0, '/root/can4linux-code/can4linux-examples')
import pyCan

# the power expressed in (mW)
# default is around 660
# it must be an Integer greater then 45
# 45mW allows to detect tags at aroud 50cm distance
antennaPower = 2000

datafolder = '/root/slope-data/'
logFolder = '/root/slope-log/'

def get_incremental_number():
	maxcounter = 0
	listFiles = os.listdir(datafolder)
	for file in listFiles:
		if os.path.isfile(datafolder + file):
			name = file.split('_')
			if len(name) > 0:
				counter = name[0]
				if counter.isdigit():
					if counter > maxcounter:
				 		maxcounter = counter
	return maxcounter + 1

incremental = str(get_incremental_number())
msgFilePath = datafolder + incremental + '_canbus-messages.txt'
logFilePath = logFolder + 'slope-canbus.log'

print 'Slope Service Started: Incremental Number ' + incremental

def get_lifting_status(status):
	if status == 0:
		return "stop"
	elif status == 1:
		return "going up"
	elif status == 2:
		return "going down"
	else:
		return "error"

def get_translation_status(status):
	if status == 0:
		return "stop"
	elif status == 1:
		return "going to load"
	elif status == 2:
		return "going to release"
	elif status == 3:
		return "antenna down"
	else:
		return "error"

def write_log(text):
	logfile = open(logFilePath, 'a', 1)
	now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
	logfile.write('[' + now + '] ' + text + '\n')
	logfile.close()

def write_msg(text):
	msgfile = open(msgFilePath, 'a+', 1)
	# now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
	msgfile.write(text + '\n')
	msgfile.close()

def get_timestamp():
	return str(int(round(time.time() * 1000)))

def sigterm_handler(_signo, _stack_frame):
	# Raises SystemExit(0):
	sys.exit(0)

signal.signal(signal.SIGTERM, sigterm_handler)

#print 'Python wrapper loaded'
#print datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
write_log('Slope-Canbus Version 1.0')
write_log('Python wrapper loaded')
write_log('Incremental Log Number: ' + incremental)

# setting the device number
device = 0
defaultBaudrate = '250'

# open the can interface /dev/can0
try:
	fileBaud = open('/proc/sys/dev/Can/Baud', 'r+')
	baudRates = fileBaud.read().split()
	baudRates[device] = defaultBaudrate
	fileBaud.write(' '.join(baudRates))
	fileBaud.close()

except IOError:
	write_log('Could not set the new bitrate')
	exit()

except IndexError:
	write_log('No proper entry for bitrate of can' + str(device) + ' found')
	exit()

# print 'open can'+ str(device) + ' none blocking'
can_fd = pyCan.open(device)

if can_fd == -1:
	write_log('Error opening CAN device /dev/can' + str(device))
	exit()

try:
	write_log('Wait for message...')
	count = 0
	flip = '0'
	lastMsg30 = ""
	lastMsg31 = ""
	lastLifting = 0
	while True:
		data = pyCan.read(can_fd)
		arrMess = data.split(':', 3)
		messId = arrMess[0][:len(arrMess[0]) - 1]
		if messId.isdigit():
			messId = int(messId)
		if messId == 30:
			# print messId
			start = arrMess[1].find("(")
			end = arrMess[1].find(")")
			bytesNum = arrMess[1][start + 1:end]
			# print bytesNum
			messBytes = arrMess[2][1:]
			if messBytes != lastMsg30:  # discard new message if is equal to last one
				lastMsg30 = messBytes
				arrBytes = messBytes.split()
				if bytesNum.isdigit():
					bytesNum = int(bytesNum)
				if bytesNum == len(arrBytes):
					weight = int(struct.unpack('<h', "".join(map(chr, [int(arrBytes[0]), int(arrBytes[1])])))[0])
					# (kg) kilogrammes
					position = int(struct.unpack('<h', "".join(map(chr, [int(arrBytes[2]), int(arrBytes[3])])))[0])
					# (m) meters
					speed = float(struct.unpack('<f', "".join(map(chr, [int(arrBytes[4]), int(arrBytes[5]), int(arrBytes[6]), int(arrBytes[7])])))[0])
					# (m/s) meters at second

					# decide what to do...
					# strOutput = str(weight) + "kg " + str(position) + "m " + str(speed) + "m/s "
					# write_msg("%d:%s" % (messId, strOutput))
					objOutput = {"weight": weight, "position": position, "speed": speed, "timestamp": get_timestamp()}
					write_msg(json.dumps(objOutput))

		if messId == 31:
			# print messId
			start = arrMess[1].find("(")
			end = arrMess[1].find(")")
			bytesNum = arrMess[1][start + 1:end]
			# print bytesNum
			messBytes = arrMess[2][1:]
			if messBytes != lastMsg31:  # discard new message if is equal to last one
				lastMsg31 = messBytes
				arrBytes = messBytes.split()
				if bytesNum.isdigit():
					bytesNum = int(bytesNum)
				if bytesNum == len(arrBytes):
					axleX = int(arrBytes[0])
					# (%) percentage of X axle grade (frontal)
					axleY = int(arrBytes[1])
					# (%) percentage of Y axle grade (lateral)
					consumption = int(struct.unpack('<h', "".join(map(chr, [int(arrBytes[2]), int(arrBytes[3])])))[0])
					# (l/h) liters at hour
					lifting = get_lifting_status(int(arrBytes[4]))
					# lifting status (0='stop', 1='going up', 2='going down', 3='antenna down')
					translation = get_translation_status(int(arrBytes[5]))
					# translation status (0='stop', 1='going to load', 2='going to release')

					# decide what to do...
					# strOutput = str(axleX) + "% " + str(axleY) + "% " + str(consumption) + "l/h " + lifting + " " + translation
					# write_msg("%d:%s" % (messId, strOutput))
					objOutput = {"axleX": axleX, "axleY": axleY, "consumption": consumption, "lifting": lifting, "translation": translation, "timestamp": get_timestamp()}
					write_msg(json.dumps(objOutput))
					lastLifting = int(arrBytes[4])

		count += 1

		#every 0.5 sec see below (10 msg per second)
		if count % 5 == 0:
			try:
				if lastLifting == 3:
					subprocess.check_call("java -jar /root/slope-bananapro/TagsReader.jar " + str(antennaPower) + " " + incremental, shell=True)
					lastLifting = 0
			except subprocess.CalledProcessError:
				write_log('Error executing lib: TagsReader.jar')

		#every 5 sec upload data and send watchdog
		if count == 50:
			count = 0
			pyCan.send(can_fd, 1, '60:' + flip)
			if flip == '0':
				flip = '1'
			if flip == '1':
				flip = '0'
			try:
				subprocess.check_call("python upload-data.py", shell=True)
			except subprocess.CalledProcessError:
				write_log('Error executing: Upload-data.py')

		time.sleep(0.1)

except KeyboardInterrupt:
	write_log('Program exits with ctrl+c')

finally:
	pyCan.close(can_fd)
	write_log('/dev/can' + str(device) + ' closed in finally')

# print 'Wait ' + str(timeout) + ' sec for an message.....'
# print pyCan.read1(can_fd, timeout * 1000000)
# print 'Wait default timeout for an message.....'
# print pyCan.read1(can_fd)

# pyCan.close(can_fd)
# write_log('/dev/can' + str(device) + ' closed at the end')
# logFile.close()
# msgFile.close()
# exit()
<|endoftext|>


<commit_before>import os
import sys
import runpy
import builtins

from .. import parse
from .. import compile

# Preprocess the traceback given its root frame.
#
# :param trace: `Exception.__traceback__`
#
# :return: a new traceback.
#
def traceback(trace):

    uses_runpy = False
    runpy_code = {runpy._run_module_as_main.__code__, runpy._run_code.__code__}

    # Skip over all entries in `runpy` module.
    while trace and trace.tb_frame.f_code in runpy_code:

        trace = trace.tb_next
        uses_runpy = True

    # If `runpy` was used, the next line is in `__main__`.
    trace = trace.tb_next if trace and uses_runpy else trace

    # If the next lines are in this module, skip them, too.
    while trace and trace.tb_frame.f_code is dg.__code__:

        trace = trace.tb_next

    return trace

# Start an interactive shell.
#
# :param name: name of the module.
#
# :return: runs indefinitely.
#
def dg(fd, name='__main__'):

    parser   = parse.r()
    compiler = compile.r()
    environ  = {'__name__': name, '__builtins__': __builtins__}

    if not fd.isatty():

        p = parser.reset(fd.read(), fd.name)
        c = compiler.compile(next(p), name='<module>')
        eval(c, environ)
        return exit(0)

    if fd is sys.stdin:

        # Run PYTHONSTARTUP first.
        st = os.environ.get('PYTHONSTARTUP', '')
        st and eval(builtins.compile(open(st).read(), st, 'exec'), environ)

    sys.ps1 = getattr(sys, 'ps1', '>>> ')
    sys.ps2 = getattr(sys, 'ps2', '... ')
    sys.stdin = fd

    while True:

        try:

            buf  = ''
            code = None

            while not code:

                buf += '\n' + input(sys.ps2 if buf else sys.ps1)
                tree = parser.compile_command(buf)
                code = tree is not None and compiler.compile(tree, name='<module>')

            sys.displayhook(eval(code, environ))

        except SystemExit:

            raise

        except EOFError:

            exit()

        except BaseException as e:

            sys.excepthook(type(e), e, traceback(e.__traceback__))
<commit_msg>Fix line number tables in interactive shell.<commit_after>import os
import sys
import runpy
import builtins

from .. import parse
from .. import compile

# Preprocess the traceback given its root frame.
#
# :param trace: `Exception.__traceback__`
#
# :return: a new traceback.
#
def traceback(trace):

    uses_runpy = False
    runpy_code = {runpy._run_module_as_main.__code__, runpy._run_code.__code__}

    # Skip over all entries in `runpy` module.
    while trace and trace.tb_frame.f_code in runpy_code:

        trace = trace.tb_next
        uses_runpy = True

    # If `runpy` was used, the next line is in `__main__`.
    trace = trace.tb_next if trace and uses_runpy else trace

    # If the next lines are in this module, skip them, too.
    while trace and trace.tb_frame.f_code is dg.__code__:

        trace = trace.tb_next

    return trace

# Start an interactive shell.
#
# :param name: name of the module.
#
# :return: runs indefinitely.
#
def dg(fd, name='__main__'):

    parser   = parse.r()
    compiler = compile.r()
    environ  = {'__name__': name, '__builtins__': __builtins__}

    if not fd.isatty():

        p = parser.reset(fd.read(), fd.name)
        c = compiler.compile(next(p), name='<module>')
        eval(c, environ)
        return exit(0)

    if fd is sys.stdin:

        # Run PYTHONSTARTUP first.
        st = os.environ.get('PYTHONSTARTUP', '')
        st and eval(builtins.compile(open(st).read(), st, 'exec'), environ)

    sys.ps1 = getattr(sys, 'ps1', '>>> ')
    sys.ps2 = getattr(sys, 'ps2', '... ')
    sys.stdin = fd

    while True:

        try:

            buf  = ''
            code = None

            while not code:

                buf += input(sys.ps2 if buf else sys.ps1)
                tree = parser.compile_command(buf)
                code = tree is not None and compiler.compile(tree, name='<module>')
                buf += '\n'

            sys.displayhook(eval(code, environ))

        except SystemExit:

            raise

        except EOFError:

            exit()

        except BaseException as e:

            sys.excepthook(type(e), e, traceback(e.__traceback__))
<|endoftext|>


<commit_before>"""ShutIt module. See http://shutit.tk
"""
#Copyright (C) 2014 OpenBet Limited
#
#
#Permission is hereby granted, free of charge, to any person obtaining a copy
#of this software and associated documentation files (the "Software"), to deal
#in the Software without restriction, including without limitation the rights
#to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#copies of the Software, and to permit persons to whom the Software is furnished
#to do so, subject to the following conditions:
#
#The above copyright notice and this permission notice shall be included in all
#copies or substantial portions of the Software.
#
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
#FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
#COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
#IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
#CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from shutit_module import ShutItModule
import util
import os

class template(ShutItModule):
	# Determines whether the module has been built in this container
	# already.
	#
	# Should return True if it is certain it's there, else False.
	def is_installed(self,shutit):
		"""Docstring
		"""
		shutit.set_default_expect(shutit.cfg['expect_prompts']['root_prompt'])
		return shutit.file_exists('/tmp/container_touched.sh') and shutit.file_exists('/tmp/README.md')

	# Run the build part of the module, which should ensure the module
	# has been set up.
	# If is_installed determines that the module is already there,
	# this is not run.
	#
	# Should return True if it has succeeded in building, else False.
	def build(self,shutit):
		"""Docstring
		"""
		shutit.set_default_expect(shutit.cfg['expect_prompts']['root_prompt'])
		# Line number 49 should be the next one (so bash scripts can be inserted properly)

		# DELETE THIS SECTION WHEN UNDERSTOOD - BEGIN
		shutit.send_and_expect('touch /tmp/deleteme')
		shutit.send_and_expect('touch /tmp/container_touched.sh')
		shutit.add_line_to_file('#This line should only appear once in the file','/tmp/container_touched.sh')
		shutit.add_line_to_file('#This line should only appear once in the file','/tmp/container_touched.sh')
		# We add the match_regexp because the string has a quote in it.
		shutit.add_line_to_file('echo "hello container"','/tmp/container_touched.sh',match_regexp='echo .hello container.')
		shutit.add_line_to_file('sleep 10000000 &','/tmp/container_touched.sh')
		shutit.send_and_expect('chmod +x /tmp/container_touched.sh')
		# Make sure passwd is installed
		shutit.install('passwd')
		# Install mlocate
		shutit.install('mlocate')
		# Example of distro-specific password update
		# record_command=False prevents the password being output at the end on debug. However, if it matches a password stored in config,
		# it will automatically redact it.
		if shutit.cfg['container']['install_type'] == 'apt': # apt-based password update
			# "check_exit" checks the exit code of the command you are running.
			# We can't check the exit code as the command will not return when we see 'Retype new' on the terminal.
			shutit.send_and_expect('passwd','Enter new',check_exit=False)
			shutit.send_and_expect(shutit.cfg['container']['password'],'Retype new',check_exit=False,record_command=False)
			shutit.send_and_expect(shutit.cfg['container']['password'])
		elif shutit.cfg['container']['install_type'] == 'yum': # yum-based password update
			# Check_exit checks the exit code of the command you are running.
			# We can't check the exit code as the command will not return when we see 'ew password' on the terminal.
			shutit.send_and_expect('passwd','ew password',check_exit=False)
			shutit.send_and_expect(shutit.cfg['container']['password'],'ew password',check_exit=False,record_command=False)
			shutit.send_and_expect(shutit.cfg['container']['password'])
		# You can put multiple items you might expect in a list and handle accordingly:
		res = shutit.send_and_expect('sudo ls',['assword',shutit.cfg['expect_prompts']['root_prompt']],check_exit=False)
		if res == 0:
			shutit.send_and_expect(shutit.cfg['container']['password'])
		elif res == 1:
			pass
		else:
			# We fail out on this case, as it's not expected.
			util.fail('res: ' + str(res) + ' not handled')
		# example of resource use (simple file, copy README.md into the container)
		shutit.send_file('/tmp/copiedfile',file.read(file(os.path.abspath(os.path.dirname(__file__)) + '/README.md')))
		# example of bespoke config use
		if shutit.cfg[self.module_id]['example_bool']:
			shutit.add_line_to_file('# ' + shutit.cfg[self.module_id]['example'],'/tmp/container_touched.sh')
		# Example of login/logout handling
		# When logging in, use the base prompt to attempt to match all prompts
		# Note that we don't check_exit, because the exit value won't be meaningful.
		shutit.send_and_expect('su',shutit.cfg['expect_prompts']['base_prompt'],check_exit=False)
		# Then call setup_prompt to set and get the bespoke prompt for the session
		shutit.setup_prompt('test_tmp_prompt')
		shutit.set_default_expect(shutit.cfg['expect_prompts']['test_tmp_prompt'])
		shutit.send_and_expect('echo "a command and some output"')
		# Make sure we're expecting the right thing after we exit this subshell
		shutit.set_default_expect(shutit.cfg['expect_prompts']['root_prompt'])
		shutit.send_and_expect('exit')

		# Example of temporary debugging
		orig_debug_val = shutit.cfg['build']['debug']
		shutit.cfg['build']['debug'] = True
		shutit.send_and_expect('echo "Command to look at debug output"')
		shutit.send_and_expect('echo "Another command to look at debug output"')
		shutit.cfg['build']['debug'] = orig_debug_val
		shutit.pause_point("""
				The doubling up of text is expected; try and ignore it :)
				When you're done, "Ctrl" and "]" at the same time.
				You should then see your inputted lines in the output.
				Add them to the module file before the pause_point line as
				"util.send_and_expect" lines (copy and paste), following
				the examples above this one in the file.
				Then "Ctrl" and "]" at the same time again to return to the script.
				Eventually you'll get a docker export to a tar file in your
				configured resources folder.""")
		# DELETE THIS SECTION WHEN UNDERSTOOD - DONE
		return True

	# get_config
	#
	# each object can handle config here
	# OPTIONAL part of lifecycle - uncomment to include
	def get_config(self,shutit):
		"""Docstring
		"""
		cp = shutit.cfg['config_parser']
		# Bring the example config into the config dictionary.
		shutit.cfg[self.module_id]['example']      = cp.get(self.module_id,'example')
		shutit.cfg[self.module_id]['example_bool'] = cp.getboolean(self.module_id,'example_bool')
		return True

	# check_ready
	#
	# Check whether we are ready to build this module.
	#
	# This is called before the build, to ensure modules have
	# their requirements in place (eg files required to be mounted
	# in /resources). Checking whether the build will happen (and
	# therefore whether the check should take place) will be
	# determined by the framework.
	#
	# Should return True if it ready, else False.
	# OPTIONAL part of lifecycle - uncomment to include
	#def check_ready(self,shutit):
	#	"""Docstring
	#	"""
	#	shutit.set_default_expect(shutit.cfg['expect_prompts']['root_prompt'])
	#	return shutit.file_exists('/resources/README.md')

	# start
	#
	# Run when module should be installed (is_installed() or configured to build is true)
	# Run after repo work.
	# OPTIONAL part of lifecycle - uncomment to include
	#def start(self,shutit):
	#	"""Docstring
	#	"""
	#	shutit.set_default_expect(shutit.cfg['expect_prompts']['root_prompt'])
	#	# example of starting something
	#	shutit.send_and_expect('cat /tmp/container_touched.sh')
	#	shutit.send_and_expect('sh /tmp/container_touched.sh')
	#	return True

	# stop
	#
	# Run when module should be stopped.
	# Run before repo work, and before finalize is called.
	# OPTIONAL part of lifecycle - uncomment to include
	#def stop(self,shutit):
	#	"""Docstring
	#	"""
	#	shutit.set_default_expect(shutit.cfg['expect_prompts']['root_prompt'])
	#	# example of stopping something
	#	shutit.send_and_expect("""ps -ef | grep -v grep | grep container_touched.sh | awk '{print $1}' | sed 's/\([0-9][0-9]*\)/kill \\1/' | sh""")
	#	return True

	# cleanup
	#
	# Cleanup the module, ie clear up stuff not needed for the rest of the build, eg tar files removed, apt-get cleans.
	# Should return True if all is OK, else False.
	# Note that this is only run if the build phase was actually run.
	# OPTIONAL part of lifecycle - uncomment to include
	#def cleanup(self,shutit):
	#	"""Docstring
	#	"""
	#	shutit.set_default_expect(shutit.cfg['expect_prompts']['root_prompt'])
	#	shutit.send_and_expect('rm -f /tmp/deleteme')
	#	return True

	# finalize
	#
	# Finalize the module, ie do things that need doing before we exit.
	# OPTIONAL part of lifecycle - uncomment to include
	#def finalize(self,shutit):
	#	"""Docstring
	#	"""
	#	shutit.set_default_expect(shutit.cfg['expect_prompts']['root_prompt'])
	#	# Right at the end we want to ensure the locate db is up to date.
	#	shutit.send_and_expect('updatedb')
	#	return True

	# remove
	#
	# Remove the module, which should ensure the module has been deleted
	# from the system.
	# OPTIONAL part of lifecycle - uncomment to include
	#def remove(self,shutit):
	#	"""Docstring
	#	"""
	#	shutit.set_default_expect(shutit.cfg['expect_prompts']['root_prompt'])
	#	shutit.send_and_expect('rm -f /tmp/container_touched.sh')
	#	shutit.send_and_expect('rm -f /tmp/README.md')
	#	# TODO: remove the installed apps DEPENDS on install tracking being available.
	#	return True

	# test
	#
	# Test the module is OK.
	# Should return True if all is OK, else False.
	# This is run regardless of whether the module is installed or not.
	# OPTIONAL part of lifecycle - uncomment to include
	#def test(self,shutit):
	#	"""Docstring
	#	"""
	#	shutit.set_default_expect(shutit.cfg['expect_prompts']['root_prompt'])
	#	return shutit.package_installed('mlocate') and shutit.package_installed('passwd')

# template(string,float)
# string : Any string you believe to identify this module uniquely,
#          eg com.my_corp.my_module_dir.my_module
# float:   Float value for ordering module builds, must be > 0.0
def module():
	#	"""Docstring
	#	"""
	return template(
		GLOBALLY_UNIQUE_STRING, FLOAT,
		depends=['shutit.tk.setup']
	)

<commit_msg>Remove lots of no-op set_default_prompts and poinless docstrings<commit_after>"""ShutIt module. See http://shutit.tk
"""
#Copyright (C) 2014 OpenBet Limited
#
#
#Permission is hereby granted, free of charge, to any person obtaining a copy
#of this software and associated documentation files (the "Software"), to deal
#in the Software without restriction, including without limitation the rights
#to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#copies of the Software, and to permit persons to whom the Software is furnished
#to do so, subject to the following conditions:
#
#The above copyright notice and this permission notice shall be included in all
#copies or substantial portions of the Software.
#
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
#FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
#COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
#IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
#CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from shutit_module import ShutItModule
import util
import os

class template(ShutItModule):
	# Determines whether the module has been built in this container
	# already.
	#
	# Should return True if it is certain it's there, else False.
	def is_installed(self,shutit):
		return shutit.file_exists('/tmp/container_touched.sh') and shutit.file_exists('/tmp/README.md')

	# Run the build part of the module, which should ensure the module
	# has been set up.
	# If is_installed determines that the module is already there,
	# this is not run.
	#
	# Should return True if it has succeeded in building, else False.
	def build(self,shutit):
		# Line number 49 should be the next one (so bash scripts can be inserted properly)

		# DELETE THIS SECTION WHEN UNDERSTOOD - BEGIN
		shutit.send_and_expect('touch /tmp/deleteme')
		shutit.send_and_expect('touch /tmp/container_touched.sh')
		shutit.add_line_to_file('#This line should only appear once in the file','/tmp/container_touched.sh')
		shutit.add_line_to_file('#This line should only appear once in the file','/tmp/container_touched.sh')
		# We add the match_regexp because the string has a quote in it.
		shutit.add_line_to_file('echo "hello container"','/tmp/container_touched.sh',match_regexp='echo .hello container.')
		shutit.add_line_to_file('sleep 10000000 &','/tmp/container_touched.sh')
		shutit.send_and_expect('chmod +x /tmp/container_touched.sh')
		# Make sure passwd is installed
		shutit.install('passwd')
		# Install mlocate
		shutit.install('mlocate')
		# Example of distro-specific password update
		# record_command=False prevents the password being output at the end on debug. However, if it matches a password stored in config,
		# it will automatically redact it.
		if shutit.cfg['container']['install_type'] == 'apt': # apt-based password update
			# "check_exit" checks the exit code of the command you are running.
			# We can't check the exit code as the command will not return when we see 'Retype new' on the terminal.
			shutit.send_and_expect('passwd','Enter new',check_exit=False)
			shutit.send_and_expect(shutit.cfg['container']['password'],'Retype new',check_exit=False,record_command=False)
			shutit.send_and_expect(shutit.cfg['container']['password'])
		elif shutit.cfg['container']['install_type'] == 'yum': # yum-based password update
			# Check_exit checks the exit code of the command you are running.
			# We can't check the exit code as the command will not return when we see 'ew password' on the terminal.
			shutit.send_and_expect('passwd','ew password',check_exit=False)
			shutit.send_and_expect(shutit.cfg['container']['password'],'ew password',check_exit=False,record_command=False)
			shutit.send_and_expect(shutit.cfg['container']['password'])
		# You can put multiple items you might expect in a list and handle accordingly:
		res = shutit.send_and_expect('sudo ls',['assword',shutit.cfg['expect_prompts']['root_prompt']],check_exit=False)
		if res == 0:
			shutit.send_and_expect(shutit.cfg['container']['password'])
		elif res == 1:
			pass
		else:
			# We fail out on this case, as it's not expected.
			util.fail('res: ' + str(res) + ' not handled')
		# example of resource use (simple file, copy README.md into the container)
		shutit.send_file('/tmp/copiedfile',file.read(file(os.path.abspath(os.path.dirname(__file__)) + '/README.md')))
		# example of bespoke config use
		if shutit.cfg[self.module_id]['example_bool']:
			shutit.add_line_to_file('# ' + shutit.cfg[self.module_id]['example'],'/tmp/container_touched.sh')
		# Example of login/logout handling
		# When logging in, use the base prompt to attempt to match all prompts
		# Note that we don't check_exit, because the exit value won't be meaningful.
		shutit.send_and_expect('su',shutit.cfg['expect_prompts']['base_prompt'],check_exit=False)
		# Then call setup_prompt to set and get the bespoke prompt for the session
		shutit.setup_prompt('test_tmp_prompt')
		shutit.set_default_expect(shutit.cfg['expect_prompts']['test_tmp_prompt'])
		shutit.send_and_expect('echo "a command and some output"')
		# Make sure we're expecting the right thing after we exit this subshell
		shutit.set_default_expect(shutit.cfg['expect_prompts']['root_prompt'])
		shutit.send_and_expect('exit')

		# Example of temporary debugging
		orig_debug_val = shutit.cfg['build']['debug']
		shutit.cfg['build']['debug'] = True
		shutit.send_and_expect('echo "Command to look at debug output"')
		shutit.send_and_expect('echo "Another command to look at debug output"')
		shutit.cfg['build']['debug'] = orig_debug_val
		shutit.pause_point("""
				The doubling up of text is expected; try and ignore it :)
				When you're done, "Ctrl" and "]" at the same time.
				You should then see your inputted lines in the output.
				Add them to the module file before the pause_point line as
				"util.send_and_expect" lines (copy and paste), following
				the examples above this one in the file.
				Then "Ctrl" and "]" at the same time again to return to the script.
				Eventually you'll get a docker export to a tar file in your
				configured resources folder.""")
		# DELETE THIS SECTION WHEN UNDERSTOOD - DONE
		return True

	# get_config
	#
	# each object can handle config here
	# OPTIONAL part of lifecycle - uncomment to include
	def get_config(self,shutit):
		cp = shutit.cfg['config_parser']
		# Bring the example config into the config dictionary.
		shutit.cfg[self.module_id]['example']      = cp.get(self.module_id,'example')
		shutit.cfg[self.module_id]['example_bool'] = cp.getboolean(self.module_id,'example_bool')
		return True

	# check_ready
	#
	# Check whether we are ready to build this module.
	#
	# This is called before the build, to ensure modules have
	# their requirements in place (eg files required to be mounted
	# in /resources). Checking whether the build will happen (and
	# therefore whether the check should take place) will be
	# determined by the framework.
	#
	# Should return True if it ready, else False.
	# OPTIONAL part of lifecycle - uncomment to include
	#def check_ready(self,shutit):
	#	return shutit.file_exists('/resources/README.md')

	# start
	#
	# Run when module should be installed (is_installed() or configured to build is true)
	# Run after repo work.
	# OPTIONAL part of lifecycle - uncomment to include
	#def start(self,shutit):
	#	# example of starting something
	#	shutit.send_and_expect('cat /tmp/container_touched.sh')
	#	shutit.send_and_expect('sh /tmp/container_touched.sh')
	#	return True

	# stop
	#
	# Run when module should be stopped.
	# Run before repo work, and before finalize is called.
	# OPTIONAL part of lifecycle - uncomment to include
	#def stop(self,shutit):
	#	# example of stopping something
	#	shutit.send_and_expect("""ps -ef | grep -v grep | grep container_touched.sh | awk '{print $1}' | sed 's/\([0-9][0-9]*\)/kill \\1/' | sh""")
	#	return True

	# cleanup
	#
	# Cleanup the module, ie clear up stuff not needed for the rest of the build, eg tar files removed, apt-get cleans.
	# Should return True if all is OK, else False.
	# Note that this is only run if the build phase was actually run.
	# OPTIONAL part of lifecycle - uncomment to include
	#def cleanup(self,shutit):
	#	shutit.send_and_expect('rm -f /tmp/deleteme')
	#	return True

	# finalize
	#
	# Finalize the module, ie do things that need doing before we exit.
	# OPTIONAL part of lifecycle - uncomment to include
	#def finalize(self,shutit):
	#	# Right at the end we want to ensure the locate db is up to date.
	#	shutit.send_and_expect('updatedb')
	#	return True

	# remove
	#
	# Remove the module, which should ensure the module has been deleted
	# from the system.
	# OPTIONAL part of lifecycle - uncomment to include
	#def remove(self,shutit):
	#	shutit.send_and_expect('rm -f /tmp/container_touched.sh')
	#	shutit.send_and_expect('rm -f /tmp/README.md')
	#	# TODO: remove the installed apps DEPENDS on install tracking being available.
	#	return True

	# test
	#
	# Test the module is OK.
	# Should return True if all is OK, else False.
	# This is run regardless of whether the module is installed or not.
	# OPTIONAL part of lifecycle - uncomment to include
	#def test(self,shutit):
	#	return shutit.package_installed('mlocate') and shutit.package_installed('passwd')

# template(string,float)
# string : Any string you believe to identify this module uniquely,
#          eg com.my_corp.my_module_dir.my_module
# float:   Float value for ordering module builds, must be > 0.0
def module():
	return template(
		GLOBALLY_UNIQUE_STRING, FLOAT,
		depends=['shutit.tk.setup']
	)

<|endoftext|>


<commit_before>import argparse
import teamcity
import os
import subprocess
import re

def get_cppcheck_path():
    if os.environ.get("ProgramFiles(x86)") is not None:
        return os.path.join(os.environ["ProgramFiles(x86)"], "cppcheck", "cppcheck.exe")
    else:
        return os.path.join(os.environ["ProgramFiles"], "cppcheck", "cppcheck.exe")

def create_argument(arg_name, file_handle):
    def handler(line):
        return " " + arg_name + line.strip('\n')
    return "".join([handler(x) for x in file_handle.readlines()])

def create_include_directories_argument(file_handle):
    return create_argument("-I", file_handle)

def create_exclude_paths_argument(file_handle):
    return create_argument("-i", file_handle)

def create_exclude_defines_argument(file_handle):
    return create_argument("-U", file_handle)

def create_include_defines_argument(file_handle):
    return create_argument("-D", file_handle)

def create_include_paths_argument(file_handle):
    return create_argument("", file_handle)

def handle_output_line(line):
    if line != "":
        line = line.strip('\n')
        m = re.search("^(##teamcity[^']*\s+description=')(.*)('[^']*)$", line)
        if m is None:
            print line
        else:
            escaped = re.sub("(['|\[\]])", "|\\1", m.group(2))
            print m.group(1) + escaped + m.group(3)
        return True
    else:
        return False

def main():
    parser = argparse.ArgumentParser(description="Run cppcheck")
    parser.add_argument("--idir", type=file, help="Path of file with all include directories", default="include_directories.txt")
    parser.add_argument("--ip", type=file, help="Path of file with directories to analyze", default="include_paths.txt")
    parser.add_argument("--idef", type=file, help="Path of file with included definitions", default="include_defines.txt")
    parser.add_argument("--xp", type=file, help="Path of file with directories or files to exclude from analysis", default="exclude_paths.txt")
    parser.add_argument("--xdef", type=file, help="Path of file with definitions to exclude", default="exclude_defines.txt")
    parser.add_argument("--s", type=str, help="Path of file with warnings to suppress", default="suppressions.txt")
    parser.add_argument("--cc", type=bool, help="Specifies whether to check the configuration of cppcheck", default=False)
    parser.add_argument("--ot", type=str, help="The output template", default=None)

    # get all data from command line
    args = parser.parse_args()

    # if the output format is None identify whether under TC or not and set message format accordingly
    # if format set to TC will also need to escape messages
    if args.ot is None:
        if teamcity.is_running_under_teamcity():
            args.ot = "tc"
        else:
            args.ot = "vs"

    arguments = " --inline-suppr --error-exitcode=-1 --enable=all" + \
                create_exclude_defines_argument(args.xdef) + \
                create_include_defines_argument(args.idef) + \
                create_include_paths_argument(args.ip) + \
                create_include_directories_argument(args.idir) + \
                create_exclude_paths_argument(args.xp) + \
                " --template=" + ('"##teamcity[buildProblem description=\'{file}:{line}: {severity}: {message}\']"' if args.ot == "tc" else args.ot) + \
                " --suppressions " + args.s + \
                ("" if args.cc is False else " --check-config")

    # run the process and redirect both stdout and stderr for further processing if needed
    if args.ot == "tc":
        process = subprocess.Popen(get_cppcheck_path() + arguments, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        while True:
            more = handle_output_line(process.stdout.readline())
            if not more:
                break
        return process.returncode
    else:
        return subprocess.call(get_cppcheck_path() + arguments)

if __name__ == "__main__":
    main()
<commit_msg>treat the boolean cc parameter correctly<commit_after>import argparse
import teamcity
import os
import subprocess
import re

def get_cppcheck_path():
    if os.environ.get("ProgramFiles(x86)") is not None:
        return os.path.join(os.environ["ProgramFiles(x86)"], "cppcheck", "cppcheck.exe")
    else:
        return os.path.join(os.environ["ProgramFiles"], "cppcheck", "cppcheck.exe")

def create_argument(arg_name, file_handle):
    def handler(line):
        return " " + arg_name + line.strip('\n')
    return "".join([handler(x) for x in file_handle.readlines()])

def create_include_directories_argument(file_handle):
    return create_argument("-I", file_handle)

def create_exclude_paths_argument(file_handle):
    return create_argument("-i", file_handle)

def create_exclude_defines_argument(file_handle):
    return create_argument("-U", file_handle)

def create_include_defines_argument(file_handle):
    return create_argument("-D", file_handle)

def create_include_paths_argument(file_handle):
    return create_argument("", file_handle)

def handle_output_line(line):
    if line != "":
        line = line.strip('\n')
        m = re.search("^(##teamcity[^']*\s+description=')(.*)('[^']*)$", line)
        if m is None:
            print line
        else:
            escaped = re.sub("(['|\[\]])", "|\\1", m.group(2))
            print m.group(1) + escaped + m.group(3)
        return True
    else:
        return False

def main():
    parser = argparse.ArgumentParser(description="Run cppcheck")
    parser.add_argument("--idir", type=file, help="Path of file with all include directories", default="include_directories.txt")
    parser.add_argument("--ip", type=file, help="Path of file with directories to analyze", default="include_paths.txt")
    parser.add_argument("--idef", type=file, help="Path of file with included definitions", default="include_defines.txt")
    parser.add_argument("--xp", type=file, help="Path of file with directories or files to exclude from analysis", default="exclude_paths.txt")
    parser.add_argument("--xdef", type=file, help="Path of file with definitions to exclude", default="exclude_defines.txt")
    parser.add_argument("--s", type=str, help="Path of file with warnings to suppress", default="suppressions.txt")
    parser.add_argument("--cc", dest="cc", help="Check the configuration of cppcheck", action="store_true")
    parser.add_argument("--no-cc", dest="cc", help="Don't heck the configuration of cppcheck", action="store_false")
    parser.set_defaults(cc=False)
    parser.add_argument("--ot", type=str, help="The output template", default=None)

    # get all data from command line
    args = parser.parse_args()

    # if the output format is None identify whether under TC or not and set message format accordingly
    # if format set to TC will also need to escape messages
    if args.ot is None:
        if teamcity.is_running_under_teamcity():
            args.ot = "tc"
        else:
            args.ot = "vs"

    arguments = " --inline-suppr --error-exitcode=-1 --enable=all" + \
                create_exclude_defines_argument(args.xdef) + \
                create_include_defines_argument(args.idef) + \
                create_include_paths_argument(args.ip) + \
                create_include_directories_argument(args.idir) + \
                create_exclude_paths_argument(args.xp) + \
                " --template=" + ('"##teamcity[buildProblem description=\'{file}:{line}: {severity}: {message}\']"' if args.ot == "tc" else args.ot) + \
                " --suppressions " + args.s + \
                ("" if args.cc is False else " --check-config")

    # run the process and redirect both stdout and stderr for further processing if needed
    if args.ot == "tc":
        process = subprocess.Popen(get_cppcheck_path() + arguments, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        while True:
            more = handle_output_line(process.stdout.readline())
            if not more:
                break
        return process.returncode
    else:
        return subprocess.call(get_cppcheck_path() + arguments)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>"""
Command line module for access to GMDSP converters

This will convert the specified input to an xml file in RDF format

General specification is:

>>> python main.py converter -i input -o output

Where converter is one of the following supported converters:
- allotments
- gritting
- parking
- planning
- recycling
- streetlight

For example:

>>> python main.py allotments -i ./sourcedata/allotments/allotments.csv -o ./output/allotments.rdf

"""
__author__ = 'jond'

import argparse
import importlib
from gmdspconverters import utils

parser = argparse.ArgumentParser(description="Command line module for access to the GMDSP converters.")
parser.add_argument(
    'converter',
    choices=[
        'allotments', 'gritting', 'parking', 'planning', 'recycling', 'streetlight',
    ],
    help="The converter to use"
)
parser.add_argument("-i", type=str, required=True, help="The input file.")
parser.add_argument("-o", type=str, required=True, help="The output file.")

args = parser.parse_args()

MODULE_MAP = {
    'allotments': 'allotments',
    'gritting': 'gritting',
    'parking': 'parking',
    'planning': 'planning',
    'recycling': 'recycling',
    'streetlight': 'streetlighting',
}

def main(convertertype, inputfile_path, outputfile_path):
    m = importlib.import_module('gmdspconverters.' + MODULE_MAP[convertertype])
    g = utils.create_graph()
    m.convert(g, inputfile_path)
    utils.output_graph(g, outputfile_path)

if __name__ == "__main__":
    main(args.converter, args.i, args.o)
<commit_msg>added ability to generate allotment stats<commit_after>"""
Command line module for access to GMDSP converters

This will convert the specified input to an xml file in RDF format

General specification is:

>>> python main.py converter -i input -o output

Where converter is one of the following supported converters:
- allotments
- gritting
- parking
- planning
- recycling
- streetlight

For example:

>>> python main.py allotments -i ./sourcedata/allotments/allotments.csv -o ./output/allotments.rdf

"""
__author__ = 'jond'

import argparse
import importlib
from gmdspconverters import utils

parser = argparse.ArgumentParser(description="Command line module for access to the GMDSP converters.")
parser.add_argument(
    'converter',
    choices=[
        'allotments', 'allotment-stats', 'gritting', 'parking', 'planning', 'recycling', 'streetlight',
    ],
    help="The converter to use"
)
parser.add_argument("-i", type=str, required=True, help="The input file.")
parser.add_argument("-o", type=str, required=True, help="The output file.")

args = parser.parse_args()

MODULE_MAP = {
    'allotments': 'allotments',
    'allotment-stats': 'allotmentstatistics',
    'gritting': 'gritting',
    'parking': 'parking',
    'planning': 'planning',
    'recycling': 'recycling',
    'streetlight': 'streetlighting',
}

def main(convertertype, inputfile_path, outputfile_path):
    m = importlib.import_module('gmdspconverters.' + MODULE_MAP[convertertype])
    g = utils.create_graph()
    m.convert(g, inputfile_path)
    utils.output_graph(g, outputfile_path)

if __name__ == "__main__":
    main(args.converter, args.i, args.o)
<|endoftext|>


<commit_before>from django.http import Http404, HttpResponseRedirect, HttpResponseBadRequest
from django.shortcuts import get_object_or_404, render_to_response
from django.core.paginator import Paginator, InvalidPage
from django.contrib.auth.models import User
from django.template import RequestContext
from annoying.decorators import render_to
from annoying.functions import get_object_or_None
from snipts.models import Favorite, Snipt
from django.db.models import Count
from django.conf import settings
from django.db.models import Q
from taggit.models import Tag

from haystack.forms import ModelSearchForm
from haystack.query import EmptySearchQuerySet, SearchQuerySet

RESULTS_PER_PAGE = getattr(settings, 'HAYSTACK_SEARCH_RESULTS_PER_PAGE', 20)

def home(request):
    if request.user.is_authenticated():
        return HttpResponseRedirect('/%s/' % request.user.username)
    else:
        return list_public(request)

@render_to('snipts/list-public.html')
def list_public(request, tag_slug=None):

    tags = Tag.objects.filter(snipt__public=True)
    tags = tags.annotate(count=Count('taggit_taggeditem_items__id'))
    tags = tags.order_by('-count')[:20]
    tags = sorted(tags, key=lambda tag: tag.name)

    snipts = Snipt.objects.filter(public=True).order_by('-created')

    if tag_slug:
        snipts = snipts.filter(tags__slug__in=[tag_slug])
        tag = get_object_or_404(Tag, slug=tag_slug)
    else:
        tag = None

    context = {
        'has_snipts': True,
        'public': True,
        'snipts': snipts,
        'tags': tags,
        'tag': tag,
    }

    if 'rss' in request.GET:
        context['snipts'] = context['snipts'][:20]
        return rss(request, context)

    return context

@render_to('snipts/list-user.html')
def list_user(request, username_or_custom_slug, tag_slug=None):

    user = get_object_or_None(User, username=username_or_custom_slug)

    if user is None:
        snipt = get_object_or_404(Snipt, custom_slug=username_or_custom_slug)
        return detail(request, snipt.user, snipt.slug)

    tags = Tag.objects
    snipts = Snipt.objects

    if user == request.user or (request.GET.get('api_key') == user.api_key.key):
        public = False

        favorites = Favorite.objects.filter(user=user).values('snipt')
        favorites = [f['snipt'] for f in favorites]
        snipts = snipts.filter(Q(user=user) | Q(pk__in=favorites))

        tags = tags.filter(Q(snipt__user=user) | Q(snipt__pk__in=favorites))
    else:
        tags = tags.filter(snipt__user=user, snipt__public=True)
        snipts = snipts.filter(user=user, public=True)
        public = True

    tags = tags.annotate(count=Count('taggit_taggeditem_items__id'))
    tags = tags.order_by('name')
    snipts = snipts.order_by('-created')

    if tag_slug:
        snipts = snipts.filter(tags__slug__in=[tag_slug])
        tag = get_object_or_404(Tag, slug=tag_slug)
    else:
        tag = None

    context = {
        'has_snipts': True,
        'public': public,
        'public_user': (public and user),
        'snipts': snipts,
        'tags': tags,
        'tag': tag,
        'user': user,
    }

    if 'rss' in request.GET:
        context['snipts'] = context['snipts'][:20]
        return rss(request, context)

    return context

@render_to('snipts/detail.html')
def detail(request, username, snipt_slug):

    snipt = get_object_or_404(Snipt, user__username=username, slug=snipt_slug)
    user = snipt.user

    if user != request.user:
        if not snipt.public:
            if 'key' not in request.GET:
                raise Http404
            else:
                if request.GET.get('key') != snipt.key:
                    raise Http404

    tags = Tag.objects

    if user == request.user:
        tags = tags.filter(snipt__user=user)
        public = False
    else:
        tags = tags.filter(snipt__user=user, snipt__public=True)
        public = True

    tags = tags.annotate(count=Count('taggit_taggeditem_items__id'))
    tags = tags.order_by('-count', 'name')

    return {
        'detail': True,
        'has_snipts': True,
        'public': public,
        'snipt': snipt,
        'tags': tags,
        'user': user,
    }

def embed(request, snipt_key):
    snipt = get_object_or_404(Snipt, key=snipt_key)

    lines = snipt.embedded.split('\n')
    return render_to_response('snipts/embed.html',
                              {'lines': lines, 'snipt': snipt},
                              context_instance=RequestContext(request),
                              mimetype='application/javascript')

def raw(request, snipt_key):
    snipt = get_object_or_404(Snipt, key=snipt_key)

    if snipt.lexer != 'js':
        return HttpResponseBadRequest()

    return render_to_response('snipts/raw.html',
                              {'snipt': snipt},
                              context_instance=RequestContext(request),
                              mimetype='text/javascript')

def rss(request, context):
    return render_to_response(
            'rss.xml',
            context,
            context_instance=RequestContext(request),
            mimetype="application/rss+xml"
        )

def search(request, template='search/search.html', load_all=True, form_class=ModelSearchForm, searchqueryset=None, context_class=RequestContext, extra_context=None, results_per_page=None):
    query = ''
    results = EmptySearchQuerySet()

    if request.GET.get('q'):
        searchqueryset = SearchQuerySet().filter(Q(public=True) | Q(author=request.user)).order_by('-pub_date')
        form = ModelSearchForm(request.GET, searchqueryset=searchqueryset, load_all=load_all)

        if form.is_valid():
            query = form.cleaned_data['q']
            results = form.search()
    else:
        form = form_class(searchqueryset=searchqueryset, load_all=load_all)

    paginator = Paginator(results, results_per_page or RESULTS_PER_PAGE)

    try:
        page = paginator.page(int(request.GET.get('page', 1)))
    except InvalidPage:
        raise Http404("No such page of results!")

    context = {
        'form': form,
        'page': page,
        'paginator': paginator,
        'query': query,
        'suggestion': None,
    }

    if results.query.backend.include_spelling:
        context['suggestion'] = form.get_suggestion()

    if extra_context:
        context.update(extra_context)

    return render_to_response(template, context, context_instance=context_class(request))
<commit_msg>Only let superusers serve raw JS from snipt.<commit_after>from django.http import Http404, HttpResponseRedirect, HttpResponseBadRequest
from django.shortcuts import get_object_or_404, render_to_response
from django.core.paginator import Paginator, InvalidPage
from django.contrib.auth.models import User
from django.template import RequestContext
from annoying.decorators import render_to
from annoying.functions import get_object_or_None
from snipts.models import Favorite, Snipt
from django.db.models import Count
from django.conf import settings
from django.db.models import Q
from taggit.models import Tag

from haystack.forms import ModelSearchForm
from haystack.query import EmptySearchQuerySet, SearchQuerySet

RESULTS_PER_PAGE = getattr(settings, 'HAYSTACK_SEARCH_RESULTS_PER_PAGE', 20)

def home(request):
    if request.user.is_authenticated():
        return HttpResponseRedirect('/%s/' % request.user.username)
    else:
        return list_public(request)

@render_to('snipts/list-public.html')
def list_public(request, tag_slug=None):

    tags = Tag.objects.filter(snipt__public=True)
    tags = tags.annotate(count=Count('taggit_taggeditem_items__id'))
    tags = tags.order_by('-count')[:20]
    tags = sorted(tags, key=lambda tag: tag.name)

    snipts = Snipt.objects.filter(public=True).order_by('-created')

    if tag_slug:
        snipts = snipts.filter(tags__slug__in=[tag_slug])
        tag = get_object_or_404(Tag, slug=tag_slug)
    else:
        tag = None

    context = {
        'has_snipts': True,
        'public': True,
        'snipts': snipts,
        'tags': tags,
        'tag': tag,
    }

    if 'rss' in request.GET:
        context['snipts'] = context['snipts'][:20]
        return rss(request, context)

    return context

@render_to('snipts/list-user.html')
def list_user(request, username_or_custom_slug, tag_slug=None):

    user = get_object_or_None(User, username=username_or_custom_slug)

    if user is None:
        snipt = get_object_or_404(Snipt, custom_slug=username_or_custom_slug)
        return detail(request, snipt.user, snipt.slug)

    tags = Tag.objects
    snipts = Snipt.objects

    if user == request.user or (request.GET.get('api_key') == user.api_key.key):
        public = False

        favorites = Favorite.objects.filter(user=user).values('snipt')
        favorites = [f['snipt'] for f in favorites]
        snipts = snipts.filter(Q(user=user) | Q(pk__in=favorites))

        tags = tags.filter(Q(snipt__user=user) | Q(snipt__pk__in=favorites))
    else:
        tags = tags.filter(snipt__user=user, snipt__public=True)
        snipts = snipts.filter(user=user, public=True)
        public = True

    tags = tags.annotate(count=Count('taggit_taggeditem_items__id'))
    tags = tags.order_by('name')
    snipts = snipts.order_by('-created')

    if tag_slug:
        snipts = snipts.filter(tags__slug__in=[tag_slug])
        tag = get_object_or_404(Tag, slug=tag_slug)
    else:
        tag = None

    context = {
        'has_snipts': True,
        'public': public,
        'public_user': (public and user),
        'snipts': snipts,
        'tags': tags,
        'tag': tag,
        'user': user,
    }

    if 'rss' in request.GET:
        context['snipts'] = context['snipts'][:20]
        return rss(request, context)

    return context

@render_to('snipts/detail.html')
def detail(request, username, snipt_slug):

    snipt = get_object_or_404(Snipt, user__username=username, slug=snipt_slug)
    user = snipt.user

    if user != request.user:
        if not snipt.public:
            if 'key' not in request.GET:
                raise Http404
            else:
                if request.GET.get('key') != snipt.key:
                    raise Http404

    tags = Tag.objects

    if user == request.user:
        tags = tags.filter(snipt__user=user)
        public = False
    else:
        tags = tags.filter(snipt__user=user, snipt__public=True)
        public = True

    tags = tags.annotate(count=Count('taggit_taggeditem_items__id'))
    tags = tags.order_by('-count', 'name')

    return {
        'detail': True,
        'has_snipts': True,
        'public': public,
        'snipt': snipt,
        'tags': tags,
        'user': user,
    }

def embed(request, snipt_key):
    snipt = get_object_or_404(Snipt, key=snipt_key)

    lines = snipt.embedded.split('\n')
    return render_to_response('snipts/embed.html',
                              {'lines': lines, 'snipt': snipt},
                              context_instance=RequestContext(request),
                              mimetype='application/javascript')

def raw(request, snipt_key):
    snipt = get_object_or_404(Snipt, key=snipt_key)

    if snipt.lexer != 'js' or not snipt.user.is_superuser:
        return HttpResponseBadRequest()

    return render_to_response('snipts/raw.html',
                              {'snipt': snipt},
                              context_instance=RequestContext(request),
                              mimetype='text/javascript')

def rss(request, context):
    return render_to_response(
            'rss.xml',
            context,
            context_instance=RequestContext(request),
            mimetype="application/rss+xml"
        )

def search(request, template='search/search.html', load_all=True, form_class=ModelSearchForm, searchqueryset=None, context_class=RequestContext, extra_context=None, results_per_page=None):
    query = ''
    results = EmptySearchQuerySet()

    if request.GET.get('q'):
        searchqueryset = SearchQuerySet().filter(Q(public=True) | Q(author=request.user)).order_by('-pub_date')
        form = ModelSearchForm(request.GET, searchqueryset=searchqueryset, load_all=load_all)

        if form.is_valid():
            query = form.cleaned_data['q']
            results = form.search()
    else:
        form = form_class(searchqueryset=searchqueryset, load_all=load_all)

    paginator = Paginator(results, results_per_page or RESULTS_PER_PAGE)

    try:
        page = paginator.page(int(request.GET.get('page', 1)))
    except InvalidPage:
        raise Http404("No such page of results!")

    context = {
        'form': form,
        'page': page,
        'paginator': paginator,
        'query': query,
        'suggestion': None,
    }

    if results.query.backend.include_spelling:
        context['suggestion'] = form.get_suggestion()

    if extra_context:
        context.update(extra_context)

    return render_to_response(template, context, context_instance=context_class(request))
<|endoftext|>


<commit_before>"""
Created 03/06/17
For executation of the Salmon quantification step
To be run with three arguements
    * basedir - top level output directory
    * input directory - contains folders with .fastq.gz files
    * max_threads - how many threads to allocate to salmon
Returns salmon quantifications and associated log files to a directory
within the top level output dir.
"""

# --- packages

import os
import sys
from subprocess import call

# --- variables using sys.argv

basedir = sys.argv[1]
inputdirectory = sys.argv[2]
max_threads = sys.argv[3]
processed = basedir + "salmon/"

# --- functions

def salmon_call(read1):
    dividing = read1.split(".")
    basename = dividing[0].replace("1P", "")
    read2 = read1.replace("1P", "2P")
    call("echo salmon quant -i aux_files/GRCh38transcriptome_sal.idx " +
        "-l A --numBootstraps 100 -1 " + inputdirectory + read1 + " -2 " +
        inputdirectory + read2 + " -p " + max_threads + " -o " + basename,
        shell=True)

# --- main call

if __name__ == "__main__":
    # --- check dirs and create if neccessary
    if not os.path.exists(processed):
        os.makedirs(processed)
    # --- create list of read1 pair file names
    read_list = []
    for fname in os.listdir(inputdirectory):
        if "1P" in fname:
            read_list.append(fname)
    # --- salmon_call on each read pair sequentially
    for read in read_list:
        salmon_call(read)
<commit_msg>changed salmon<commit_after>"""
Created 03/06/17
For executation of the Salmon quantification step
To be run with three arguements
    * basedir - top level output directory
    * input directory - contains folders with .fastq.gz files
    * max_threads - how many threads to allocate to salmon
Returns salmon quantifications and associated log files to a directory
within the top level output dir.
"""

# --- packages

import os
import sys
from subprocess import call

# --- variables using sys.argv

basedir = sys.argv[1]
inputdirectory = sys.argv[2]
max_threads = sys.argv[3]
processed = basedir + "salmon/"

# --- functions

def salmon_call(read1):
    dividing = read1.split(".")
    basename = dividing[0].replace("1P", "")
    read2 = read1.replace("1P", "2P")
    call("echo salmon quant -i aux_files/GRCh38transcriptome_sal.idx " +
        "-l A --numBootstraps 100 -1 " + inputdirectory + read1 + " -2 " +
        inputdirectory + read2 + " -p " + max_threads + " -o " +processed +
        basename, shell=True)

# --- main call

if __name__ == "__main__":
    # --- check dirs and create if neccessary
    if not os.path.exists(processed):
        os.makedirs(processed)
    # --- create list of read1 pair file names
    read_list = []
    for fname in os.listdir(inputdirectory):
        if "1P" in fname:
            read_list.append(fname)
    # --- salmon_call on each read pair sequentially
    for read in read_list:
        salmon_call(read)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Salt compatibility code
'''
# pylint: disable=import-error,unused-import,invalid-name,W0231,W0233

# Import python libs
from __future__ import absolute_import, unicode_literals, print_function
import sys
import types
import logging

# Import 3rd-party libs
from salt.exceptions import SaltException
from salt.ext.six import binary_type, string_types, text_type, integer_types
from salt.ext.six.moves import cStringIO, StringIO

log = logging.getLogger(__name__)

try:
    # Python >2.5
    import xml.etree.cElementTree as ElementTree
except Exception:
    try:
        # Python >2.5
        import xml.etree.ElementTree as ElementTree
    except Exception:
        try:
            # normal cElementTree install
            import elementtree.cElementTree as ElementTree
        except Exception:
            try:
                # normal ElementTree install
                import elementtree.ElementTree as ElementTree
            except Exception:
                ElementTree = None

# True if we are running on Python 3.
PY3 = sys.version_info.major == 3

if PY3:
    import builtins
    exceptions = builtins
else:
    import exceptions

if ElementTree is not None:
    if not hasattr(ElementTree, 'ParseError'):
        class ParseError(Exception):
            '''
            older versions of ElementTree do not have ParseError
            '''

        ElementTree.ParseError = ParseError

def text_(s, encoding='latin-1', errors='strict'):
    '''
    If ``s`` is an instance of ``binary_type``, return
    ``s.decode(encoding, errors)``, otherwise return ``s``
    '''
    return s.decode(encoding, errors) if isinstance(s, binary_type) else s

def bytes_(s, encoding='latin-1', errors='strict'):
    '''
    If ``s`` is an instance of ``text_type``, return
    ``s.encode(encoding, errors)``, otherwise return ``s``
    '''
    return s.encode(encoding, errors) if isinstance(s, text_type) else s

def ascii_native_(s):
    '''
    Python 3: If ``s`` is an instance of ``text_type``, return
    ``s.encode('ascii')``, otherwise return ``str(s, 'ascii', 'strict')``

    Python 2: If ``s`` is an instance of ``text_type``, return
    ``s.encode('ascii')``, otherwise return ``str(s)``
    '''
    if isinstance(s, text_type):
        s = s.encode('ascii')

    return str(s, 'ascii', 'strict') if PY3 else s

def native_(s, encoding='latin-1', errors='strict'):
    '''
    Python 3: If ``s`` is an instance of ``text_type``, return ``s``, otherwise
    return ``str(s, encoding, errors)``

    Python 2: If ``s`` is an instance of ``text_type``, return
    ``s.encode(encoding, errors)``, otherwise return ``str(s)``
    '''
    if PY3:
        out = s if isinstance(s, text_type) else str(s, encoding, errors)
    else:
        out = s.encode(encoding, errors) if isinstance(s, text_type) else str(s)

    return out

def string_io(data=None):  # cStringIO can't handle unicode
    '''
    Pass data through to stringIO module and return result
    '''
    try:
        return cStringIO(bytes(data))
    except (UnicodeEncodeError, TypeError):
        return StringIO(data)

try:
    if PY3:
        import ipaddress
    else:
        import salt.ext.ipaddress as ipaddress
except ImportError:
    ipaddress = None

class IPv6AddressScoped(ipaddress.IPv6Address):
    '''
    Represent and manipulate single IPv6 Addresses.
    Scope-aware version
    '''
    def __init__(self, address):
        '''
        Instantiate a new IPv6 address object. Scope is moved to an attribute 'scope'.

        Args:
            address: A string or integer representing the IP

              Additionally, an integer can be passed, so
              IPv6Address('2001:db8::') == IPv6Address(42540766411282592856903984951653826560)
              or, more generally
              IPv6Address(int(IPv6Address('2001:db8::'))) == IPv6Address('2001:db8::')

        Raises:
            AddressValueError: If address isn't a valid IPv6 address.

        :param address:
        '''
        if isinstance(address, string_types) and '%' in address:
            buff = address.split('%')
            if len(buff) != 2:
                raise SaltException('Invalid IPv6 address: "{}"'.format(address))
            address, self.__scope = buff
        else:
            self.__scope = None

        if sys.version_info.major == 2:
            ipaddress._BaseAddress.__init__(self, address)
            ipaddress._BaseV6.__init__(self, address)
        else:
            # Python 3.4 fix. Versions higher are simply not affected
            # https://github.com/python/cpython/blob/3.4/Lib/ipaddress.py#L543-L544
            self._version = 6
            self._max_prefixlen = ipaddress.IPV6LENGTH

        # Efficient constructor from integer.
        if isinstance(address, integer_types):
            self._check_int_address(address)
            self._ip = address
        elif self._is_packed_binary(address):
            self._check_packed_address(address, 16)
            self._ip = ipaddress._int_from_bytes(address, 'big')
        else:
            address = str(address)
            if '/' in address:
                raise ipaddress.AddressValueError("Unexpected '/' in {}".format(address))
            self._ip = self._ip_int_from_string(address)

    def _is_packed_binary(self, data):
        '''
        Check if data is hexadecimal packed

        :param data:
        :return:
        '''
        packed = False
        if isinstance(data, bytes) and len(data) == 16 and b':' not in data:
            try:
                packed = bool(int(str(bytearray(data)).encode('hex'), 16))
            except ValueError:
                pass

        return packed

    @property
    def scope(self):
        '''
        Return scope of IPv6 address.

        :return:
        '''
        return self.__scope

    def __str__(self):
        return text_type(self._string_from_ip_int(self._ip) +
                         ('%' + self.scope if self.scope is not None else ''))

class IPv6InterfaceScoped(ipaddress.IPv6Interface, IPv6AddressScoped):
    '''
    Update
    '''
    def __init__(self, address):
        if isinstance(address, (bytes, int)):
            IPv6AddressScoped.__init__(self, address)
            self.network = ipaddress.IPv6Network(self._ip)
            self._prefixlen = self._max_prefixlen
            return

        addr = ipaddress._split_optional_netmask(address)
        IPv6AddressScoped.__init__(self, addr[0])
        self.network = ipaddress.IPv6Network(address, strict=False)
        self.netmask = self.network.netmask
        self._prefixlen = self.network._prefixlen
        self.hostmask = self.network.hostmask

if ipaddress:
    ipaddress.IPv6Address = IPv6AddressScoped
    if sys.version_info.major == 2:
        ipaddress.IPv6Interface = IPv6InterfaceScoped
<commit_msg>Roll back some changes to _compat.py<commit_after># -*- coding: utf-8 -*-
'''
Salt compatibility code
'''
# pylint: disable=import-error,unused-import,invalid-name,W0231,W0233

# Import python libs
from __future__ import absolute_import, unicode_literals, print_function
import sys
import types
import logging

# Import 3rd-party libs
from salt.exceptions import SaltException
from salt.ext.six import binary_type, string_types, text_type, integer_types
from salt.ext.six.moves import cStringIO, StringIO

log = logging.getLogger(__name__)

try:
    # Python >2.5
    import xml.etree.cElementTree as ElementTree
except Exception:
    try:
        # Python >2.5
        import xml.etree.ElementTree as ElementTree
    except Exception:
        try:
            # normal cElementTree install
            import elementtree.cElementTree as ElementTree
        except Exception:
            try:
                # normal ElementTree install
                import elementtree.ElementTree as ElementTree
            except Exception:
                ElementTree = None

# True if we are running on Python 3.
PY3 = sys.version_info.major == 3

if PY3:
    import builtins
    exceptions = builtins
else:
    import exceptions

if ElementTree is not None:
    if not hasattr(ElementTree, 'ParseError'):
        class ParseError(Exception):
            '''
            older versions of ElementTree do not have ParseError
            '''

        ElementTree.ParseError = ParseError

def text_(s, encoding='latin-1', errors='strict'):
    '''
    If ``s`` is an instance of ``binary_type``, return
    ``s.decode(encoding, errors)``, otherwise return ``s``
    '''
    return s.decode(encoding, errors) if isinstance(s, binary_type) else s

def bytes_(s, encoding='latin-1', errors='strict'):
    '''
    If ``s`` is an instance of ``text_type``, return
    ``s.encode(encoding, errors)``, otherwise return ``s``
    '''
    return s.encode(encoding, errors) if isinstance(s, text_type) else s

def ascii_native_(s):
    '''
    Python 3: If ``s`` is an instance of ``text_type``, return
    ``s.encode('ascii')``, otherwise return ``str(s, 'ascii', 'strict')``

    Python 2: If ``s`` is an instance of ``text_type``, return
    ``s.encode('ascii')``, otherwise return ``str(s)``
    '''
    if isinstance(s, text_type):
        s = s.encode('ascii')

    return str(s, 'ascii', 'strict') if PY3 else s

def native_(s, encoding='latin-1', errors='strict'):
    '''
    Python 3: If ``s`` is an instance of ``text_type``, return ``s``, otherwise
    return ``str(s, encoding, errors)``

    Python 2: If ``s`` is an instance of ``text_type``, return
    ``s.encode(encoding, errors)``, otherwise return ``str(s)``
    '''
    if PY3:
        out = s if isinstance(s, text_type) else str(s, encoding, errors)
    else:
        out = s.encode(encoding, errors) if isinstance(s, text_type) else str(s)

    return out

def string_io(data=None):  # cStringIO can't handle unicode
    '''
    Pass data through to stringIO module and return result
    '''
    try:
        return cStringIO(bytes(data))
    except (UnicodeEncodeError, TypeError):
        return StringIO(data)

try:
    if PY3:
        import ipaddress
    else:
        import salt.ext.ipaddress as ipaddress
except ImportError:
    ipaddress = None

class IPv6AddressScoped(ipaddress.IPv6Address):
    '''
    Represent and manipulate single IPv6 Addresses.
    Scope-aware version
    '''
    def __init__(self, address):
        '''
        Instantiate a new IPv6 address object. Scope is moved to an attribute 'scope'.

        Args:
            address: A string or integer representing the IP

              Additionally, an integer can be passed, so
              IPv6Address('2001:db8::') == IPv6Address(42540766411282592856903984951653826560)
              or, more generally
              IPv6Address(int(IPv6Address('2001:db8::'))) == IPv6Address('2001:db8::')

        Raises:
            AddressValueError: If address isn't a valid IPv6 address.

        :param address:
        '''
        # pylint: disable-all
        if not hasattr(self, '_is_packed_binary'):
            # This method (below) won't be around for some Python 3 versions
            # and we need check this differently anyway
            self._is_packed_binary = lambda p: isinstance(p, bytes)
        # pylint: enable-all
        if isinstance(address, string_types) and '%' in address:
            buff = address.split('%')
            if len(buff) != 2:
                raise SaltException('Invalid IPv6 address: "{}"'.format(address))
            address, self.__scope = buff
        else:
            self.__scope = None

        if sys.version_info.major == 2:
            ipaddress._BaseAddress.__init__(self, address)
            ipaddress._BaseV6.__init__(self, address)
        else:
            # Python 3.4 fix. Versions higher are simply not affected
            # https://github.com/python/cpython/blob/3.4/Lib/ipaddress.py#L543-L544
            self._version = 6
            self._max_prefixlen = ipaddress.IPV6LENGTH

        # Efficient constructor from integer.
        if isinstance(address, integer_types):
            self._check_int_address(address)
            self._ip = address
        elif self._is_packed_binary(address):
            self._check_packed_address(address, 16)
            self._ip = ipaddress._int_from_bytes(address, 'big')
        else:
            address = str(address)
            if '/' in address:
                raise ipaddress.AddressValueError("Unexpected '/' in {}".format(address))
            self._ip = self._ip_int_from_string(address)

    def _is_packed_binary(self, data):
        '''
        Check if data is hexadecimal packed

        :param data:
        :return:
        '''
        packed = False
        if isinstance(data, bytes) and len(data) == 16 and b':' not in data:
            try:
                packed = bool(int(str(bytearray(data)).encode('hex'), 16))
            except ValueError:
                pass

        return packed

    @property
    def scope(self):
        '''
        Return scope of IPv6 address.

        :return:
        '''
        return self.__scope

    def __str__(self):
        return text_type(self._string_from_ip_int(self._ip) +
                         ('%' + self.scope if self.scope is not None else ''))

class IPv6InterfaceScoped(ipaddress.IPv6Interface, IPv6AddressScoped):
    '''
    Update
    '''
    def __init__(self, address):
        if isinstance(address, (bytes, int)):
            IPv6AddressScoped.__init__(self, address)
            self.network = ipaddress.IPv6Network(self._ip)
            self._prefixlen = self._max_prefixlen
            return

        addr = ipaddress._split_optional_netmask(address)
        IPv6AddressScoped.__init__(self, addr[0])
        self.network = ipaddress.IPv6Network(address, strict=False)
        self.netmask = self.network.netmask
        self._prefixlen = self.network._prefixlen
        self.hostmask = self.network.hostmask

if ipaddress:
    ipaddress.IPv6Address = IPv6AddressScoped
    if sys.version_info.major == 2:
        ipaddress.IPv6Interface = IPv6InterfaceScoped
<|endoftext|>


<commit_before>'''
Many aspects of the salt payload need to be managed, from the return of
encrypted keys to general payload dynamics and packaging, these happen
in here
'''

# Import python libs
import sys

# Import salt libs
import salt.log
import salt.crypt
from salt.exceptions import SaltReqTimeoutError
from salt._compat import pickle
from salt.utils.odict import OrderedDict

# Import third party libs
try:
    import zmq
except:
    # No need for zeromq in local mode
    pass

log = salt.log.logging.getLogger(__name__)

try:
    # Attempt to import msgpack
    import msgpack
    # There is a serialization issue on ARM and potentially other platforms
    # for some msgpack bindings, check for it
    if msgpack.loads(msgpack.dumps([1, 2, 3]), use_list=True) is None:
        raise ImportError
except ImportError:
    # Fall back to msgpack_pure
    try:
        import msgpack_pure as msgpack
    except ImportError:
        # TODO: Come up with a sane way to get a configured logfile
        #       and write to the logfile when this error is hit also
        LOG_FORMAT = '[%(levelname)-8s] %(message)s'
        salt.log.setup_console_logger(log_format=LOG_FORMAT)
        log.fatal('Unable to import msgpack or msgpack_pure python modules')
        sys.exit(1)

def package(payload):
    '''
    This method for now just wraps msgpack.dumps, but it is here so that
    we can make the serialization a custom option in the future with ease.
    '''
    return msgpack.dumps(payload)

def unpackage(package_):
    '''
    Unpackages a payload
    '''
    return msgpack.loads(package_, use_list=True)

def format_payload(enc, **kwargs):
    '''
    Pass in the required arguments for a payload, the enc type and the cmd,
    then a list of keyword args to generate the body of the load dict.
    '''
    payload = {'enc': enc}
    load = {}
    for key in kwargs:
        load[key] = kwargs[key]
    payload['load'] = load
    return package(payload)

class Serial(object):
    '''
    Create a serialization object, this object manages all message
    serialization in Salt
    '''
    def __init__(self, opts):
        if isinstance(opts, dict):
            self.serial = opts.get('serial', 'msgpack')
        elif isinstance(opts, str):
            self.serial = opts
        else:
            self.serial = 'msgpack'

    def loads(self, msg):
        '''
        Run the correct loads serialization format
        '''
        if self.serial == 'msgpack':
            return msgpack.loads(msg, use_list=True)
        elif self.serial == 'pickle':
            try:
                return pickle.loads(msg)
            except Exception:
                return msgpack.loads(msg, use_list=True)

    def load(self, fn_):
        '''
        Run the correct serialization to load a file
        '''
        data = fn_.read()
        fn_.close()
        return self.loads(data)

    def dumps(self, msg):
        '''
        Run the correct dumps serialization format
        '''
        if self.serial == 'pickle':
            return pickle.dumps(msg)
        else:
            try:
                return msgpack.dumps(msg)
            except TypeError:
                if msgpack.version >= (0, 2, 0):
                    # Should support OrderedDict serialization, so, let's
                    # raise the exception
                    raise

                # msgpack is < 0.2.0, let's make it's life easier
                # Since OrderedDict is identified as a dictionary, we can't
                # make use of msgpack custom types, we will need to convert by
                # hand.
                # This means iterating through all elements of a dictionary or
                # list/tuple
                def odict_encoder(obj):
                    if isinstance(obj, OrderedDict):
                        return dict(obj)
                    return obj

                if isinstance(msg, dict):
                    for k, v in msg.copy().iteritems():
                        msg[k] = odict_encoder(v)
                elif isinstance(msg, (list, tuple)):
                    msg = list(msg)
                    for idx, entry in enumerate(msg):
                        msg[idx] = odict_encoder(entry)
                return msgpack.dumps(msg)

    def dump(self, msg, fn_):
        '''
        Serialize the correct data into the named file object
        '''
        fn_.write(self.dumps(msg))
        fn_.close()

class SREQ(object):
    '''
    Create a generic interface to wrap salt zeromq req calls.
    '''
    def __init__(self, master, id_='', serial='msgpack', linger=0):
        self.master = master
        self.serial = Serial(serial)
        self.context = zmq.Context()
        self.socket = self.context.socket(zmq.REQ)
        if hasattr(zmq, 'RECONNECT_IVL_MAX'):
            self.socket.setsockopt(
                zmq.RECONNECT_IVL_MAX, 5000
            )

        if master.startswith('tcp://[') and hasattr(zmq, 'IPV4ONLY'):
            # IPv6 sockets work for both IPv6 and IPv4 addresses
            self.socket.setsockopt(zmq.IPV4ONLY, 0)
        self.socket.linger = linger
        if id_:
            self.socket.setsockopt(zmq.IDENTITY, id_)
        self.socket.connect(master)
        self.poller = zmq.Poller()

    def send(self, enc, load, tries=1, timeout=60):
        '''
        Takes two arguments, the encryption type and the base payload
        '''
        payload = {'enc': enc}
        payload['load'] = load
        pkg = self.serial.dumps(payload)
        self.socket.send(pkg)
        self.poller.register(self.socket, zmq.POLLIN)
        tried = 0
        while True:
            polled = self.poller.poll(timeout * 1000)
            tried += 1
            if polled:
                break
            elif tried >= tries:
                raise SaltReqTimeoutError(
                    'Waited {0} seconds'.format(
                        timeout * tried
                    )
                )
        return self.serial.loads(self.socket.recv())

    def send_auto(self, payload, tries=1, timeout=60):
        '''
        Detect the encryption type based on the payload
        '''
        enc = payload.get('enc', 'clear')
        load = payload.get('load', {})
        return self.send(enc, load, tries, timeout)

    def destroy(self):
        for socket in self.poller.sockets.keys():
            if socket.closed is False:
                socket.setsockopt(zmq.LINGER, 1)
                socket.close()
            self.poller.unregister(socket)
        if self.socket.closed is False:
            self.socket.setsockopt(zmq.LINGER, 1)
            self.socket.close()
        if self.context.closed is False:
            self.context.term()

    def __del__(self):
        self.destroy()
<commit_msg>salt.payload: fix pylint W0702 "No exception type(s) specified"<commit_after>'''
Many aspects of the salt payload need to be managed, from the return of
encrypted keys to general payload dynamics and packaging, these happen
in here
'''

# Import python libs
import sys

# Import salt libs
import salt.log
import salt.crypt
from salt.exceptions import SaltReqTimeoutError
from salt._compat import pickle
from salt.utils.odict import OrderedDict

# Import third party libs
try:
    import zmq
except ImportError:
    # No need for zeromq in local mode
    pass

log = salt.log.logging.getLogger(__name__)

try:
    # Attempt to import msgpack
    import msgpack
    # There is a serialization issue on ARM and potentially other platforms
    # for some msgpack bindings, check for it
    if msgpack.loads(msgpack.dumps([1, 2, 3]), use_list=True) is None:
        raise ImportError
except ImportError:
    # Fall back to msgpack_pure
    try:
        import msgpack_pure as msgpack
    except ImportError:
        # TODO: Come up with a sane way to get a configured logfile
        #       and write to the logfile when this error is hit also
        LOG_FORMAT = '[%(levelname)-8s] %(message)s'
        salt.log.setup_console_logger(log_format=LOG_FORMAT)
        log.fatal('Unable to import msgpack or msgpack_pure python modules')
        sys.exit(1)

def package(payload):
    '''
    This method for now just wraps msgpack.dumps, but it is here so that
    we can make the serialization a custom option in the future with ease.
    '''
    return msgpack.dumps(payload)

def unpackage(package_):
    '''
    Unpackages a payload
    '''
    return msgpack.loads(package_, use_list=True)

def format_payload(enc, **kwargs):
    '''
    Pass in the required arguments for a payload, the enc type and the cmd,
    then a list of keyword args to generate the body of the load dict.
    '''
    payload = {'enc': enc}
    load = {}
    for key in kwargs:
        load[key] = kwargs[key]
    payload['load'] = load
    return package(payload)

class Serial(object):
    '''
    Create a serialization object, this object manages all message
    serialization in Salt
    '''
    def __init__(self, opts):
        if isinstance(opts, dict):
            self.serial = opts.get('serial', 'msgpack')
        elif isinstance(opts, str):
            self.serial = opts
        else:
            self.serial = 'msgpack'

    def loads(self, msg):
        '''
        Run the correct loads serialization format
        '''
        if self.serial == 'msgpack':
            return msgpack.loads(msg, use_list=True)
        elif self.serial == 'pickle':
            try:
                return pickle.loads(msg)
            except Exception:
                return msgpack.loads(msg, use_list=True)

    def load(self, fn_):
        '''
        Run the correct serialization to load a file
        '''
        data = fn_.read()
        fn_.close()
        return self.loads(data)

    def dumps(self, msg):
        '''
        Run the correct dumps serialization format
        '''
        if self.serial == 'pickle':
            return pickle.dumps(msg)
        else:
            try:
                return msgpack.dumps(msg)
            except TypeError:
                if msgpack.version >= (0, 2, 0):
                    # Should support OrderedDict serialization, so, let's
                    # raise the exception
                    raise

                # msgpack is < 0.2.0, let's make it's life easier
                # Since OrderedDict is identified as a dictionary, we can't
                # make use of msgpack custom types, we will need to convert by
                # hand.
                # This means iterating through all elements of a dictionary or
                # list/tuple
                def odict_encoder(obj):
                    if isinstance(obj, OrderedDict):
                        return dict(obj)
                    return obj

                if isinstance(msg, dict):
                    for k, v in msg.copy().iteritems():
                        msg[k] = odict_encoder(v)
                elif isinstance(msg, (list, tuple)):
                    msg = list(msg)
                    for idx, entry in enumerate(msg):
                        msg[idx] = odict_encoder(entry)
                return msgpack.dumps(msg)

    def dump(self, msg, fn_):
        '''
        Serialize the correct data into the named file object
        '''
        fn_.write(self.dumps(msg))
        fn_.close()

class SREQ(object):
    '''
    Create a generic interface to wrap salt zeromq req calls.
    '''
    def __init__(self, master, id_='', serial='msgpack', linger=0):
        self.master = master
        self.serial = Serial(serial)
        self.context = zmq.Context()
        self.socket = self.context.socket(zmq.REQ)
        if hasattr(zmq, 'RECONNECT_IVL_MAX'):
            self.socket.setsockopt(
                zmq.RECONNECT_IVL_MAX, 5000
            )

        if master.startswith('tcp://[') and hasattr(zmq, 'IPV4ONLY'):
            # IPv6 sockets work for both IPv6 and IPv4 addresses
            self.socket.setsockopt(zmq.IPV4ONLY, 0)
        self.socket.linger = linger
        if id_:
            self.socket.setsockopt(zmq.IDENTITY, id_)
        self.socket.connect(master)
        self.poller = zmq.Poller()

    def send(self, enc, load, tries=1, timeout=60):
        '''
        Takes two arguments, the encryption type and the base payload
        '''
        payload = {'enc': enc}
        payload['load'] = load
        pkg = self.serial.dumps(payload)
        self.socket.send(pkg)
        self.poller.register(self.socket, zmq.POLLIN)
        tried = 0
        while True:
            polled = self.poller.poll(timeout * 1000)
            tried += 1
            if polled:
                break
            elif tried >= tries:
                raise SaltReqTimeoutError(
                    'Waited {0} seconds'.format(
                        timeout * tried
                    )
                )
        return self.serial.loads(self.socket.recv())

    def send_auto(self, payload, tries=1, timeout=60):
        '''
        Detect the encryption type based on the payload
        '''
        enc = payload.get('enc', 'clear')
        load = payload.get('load', {})
        return self.send(enc, load, tries, timeout)

    def destroy(self):
        for socket in self.poller.sockets.keys():
            if socket.closed is False:
                socket.setsockopt(zmq.LINGER, 1)
                socket.close()
            self.poller.unregister(socket)
        if self.socket.closed is False:
            self.socket.setsockopt(zmq.LINGER, 1)
            self.socket.close()
        if self.context.closed is False:
            self.context.term()

    def __del__(self):
        self.destroy()
<|endoftext|>