<commit_before>#! /usr/bin/env python
"""
Spatial adjust and test precision

"""

import numpy as np

from sklearn import linear_model
from sklearn import cross_validation
from sklearn.metrics import explained_variance_score
from sklearn.metrics import mean_squared_error
import argparse, sys, csv, os, time

def getArgs():
	parser = argparse.ArgumentParser(
		description = """Affine spatial transformation with simulated error"""
	)

	parser.add_argument(
		"-c",
		"--controlPoints",
		type = str,
		required = True,
		help = "Control points csv file. See README doc for formating instructions."
	)

	parser.add_argument(
		"-u",
		"--unprojectPoints",
		type = str,
		required = True,
		help = "Unprojected points on plot coordinate system"
	)

	parser.add_argument(
		"-x",
		"--xname",
		type = str,
		required = True,
		help = "field name for x-coordinate in unprojected points file"

	)

	parser.add_argument(
		"-y",
		"--yname",
		type = str,
		required = True,
		help = "field name for y-coordinate in unprojected points file"

	)

	parser.add_argument(
		"-i",
		"--uid",
		type = str,
		required = True,
		help = "field name for unique identifier of point in unprojected points file"

	)

	parser.add_argument(
		"-n",
		"--nsims",
		type = str,
		required = False,
		help = """Number of simulations"""
	)

	parser.add_argument(
		"-o",
		"--output",
		type = str,
		required = True,
		help = """Output file prefix"""
	)

	parser.add_argument(
		"-v",
		"--verbose",
		action = "store_true",
		help = "Print status updates while executing"
	)

	return parser.parse_args()

def getIndex(header, item):
	"""Get index of variable
	"""
	for i, elem in enumerate(header):
		if elem.lower() == item.lower():
			return i
	return None

def getData(path, fields):
	"""Read unprojected points file as numpy array"""
	data = []
	with open(path, 'rUb') as f:
		indata = csv.reader(f)
		var_indices = []
		for i, line in enumerate(indata):
			if i == 0:
				header = line
				continue
			if 'NA' in line:
				for j, elem in enumerate(line):
					line[j] = np.nan
			data.append(line)
	data = np.array(data, dtype=np.float64)
	indices = np.array([getIndex(header, item) for item in fields])
	header = [header[i] for i in indices]
	data = data[:, indices]
	return header, data

def getControl(path):
	"""Read control points file as numpy array
	"""
	data = []
	with open(path, "rU") as f:
		for i, line in enumerate(f):
			ls = line.strip().split(',')
			if i == 0:
				continue
			data.append(ls)
	data = np.array(data, dtype=np.float64)
	return data

def writeOut(data, header, filename, verbose):
	"""Write data to a file.
	"""
	if verbose:
		print "\nInitializing {0} file output".format(filename)
	if header is not None:

		with open(filename, "w") as f:
			header_str = ",".join(header)+"\n"
			f.write(header_str)
			for line in data:
				row = ",".join([str(elem) for elem in line])+"\n"
				f.write(row)
	else:
		with open(filename, "a") as f:
			for line in data:
				row = ",".join([str(elem) for elem in line])+"\n"
				f.write(row)
	if verbose:
		print "Wrote {0} to disk\n".format(filename)

def fit(X, y):
	"""Fit using matrix algebra
	"""
	XTXinv = inv(np.dot(X.T, X))
	XTy = np.dot(X.T, y)
	B = np.dot(XTXinv, XTy)
	return B

def affine_parameterization(utm_e, utm_n, x, y):
	"""Parameterize affine function
	"""
	X = np.column_stack((x, y, np.ones(len(x))))
	affine x = fit(X, utm_e)
	affine_y = fit(X, utm_n)
	return affine_x, affine_y

def affine_transformation(X_unprj, affine_x, affine_y, args, header):
	"""Predict with affine function
	"""
	x_pred = np.dot(X_unprj, affine_x)
	y_pred = np.dot(X_unprj, affine_y)
	return x_pred, y_pred

def main():
	t_i = time.time()
	args = getArgs()
	initDir = os.getcwd()

	fields = [
		args.uid,
		args.xname,
		args.yname
	]

	header, data = getData(args.unprojectPoints, fields)
	if args.verbose:
		print header, data

	np.random.seed(10)

	cp = getControl(args.controlPoints)
	print cp
	print cp.shape

	if cp.shape[1] == 6 and args.nsims is not None:
		sim = True
		if args.verbose:
			print "Will simulate {} realizations of corners.".format(args.nsims)
	elif args.nsims is None:
		sim = False
	elif cp.shape[1] == 4:
		sim = False
		if args.nsims is not None:
			if args.verbose:
				print "Cannot simulate error because no utm_e and utm_n se's"
				print  "are included in control points file."
	else:
		print "Incorrect dimensions for control points csv. Exiting"
		sys.exit(1)

	# Affine Spatial Transformation Parameterization
	# x' = Ax + By + C
	# y' = Dx + Ey + F
	utm_e, utm_n, x, y = cp[:,0], cp[:,1], cp[:,2], cp[:,3]
	affine_x, affine_y = affine_parameterization(utm_e, utm_n, x, y ) # utm_e, utm_n, x, y

	uids = data[:, getIndex(header, args.uid)]
	gx = data[:, getIndex(header, args.xname)]
	gy = data[:, getIndex(header, args.yname)]
	X_unprj = np.column_stack(
		(
			gx,
			gy
		)
	)
	x_pred, y_pred = affine_transformation(X_unprj, affine_x, affine_y, args, header)

	projected_data = np.column_stack(
		(
			uids,
			gx,
			gy,
			x_pred,
			y_pred
		)
	)

	projected_data_header = ["uid", "gx", "gy", "x_pred", "y_pred"]
	writeOut(projected_data, projected_data_header, args.output+"-projected.csv", args.verbose) # data, header, filename

	# Simulated locations
	if sim:
		# error cols 4 and 5 from cp csv
		utm_se_e = cp[:,4]
		utm_se_n = cp[:,5]
		sim_data_header = ["iter", "uid", "gx", "gy", "x_pred", "y_pred"]
		for i in range(int(args.nsims)):
			if args.verbose:
				print "Simulation number {}".format(i+1)
			evec = np.zeros(cp.shape[0])
			nvec = np.zeros(cp.shape[0])
			for j, row in enumerate(cp):
				evec[j] = np.random.normal(row[0],row[4], 1)
				nvec[j] = np.random.normal(row[1],row[5], 1)
			affine_x, affine_y = affine_parameterization(evec, nvec, x, y)
			x_pred, y_pred = affine_transformation(X_unprj, affine_x, affine_y, args, header)
			iteration = i + np.zeros(x_pred.shape[0])
			sim_data = np.column_stack(
				(
				iteration,
				uids,
				gx,
				gy,
				x_pred,
				y_pred
				)
			)
			if i == 0:
				writeOut(sim_data, sim_data_header, args.output+"-sim.csv", args.verbose) # data, header, filename
			else:
				writeOut(sim_data, None, args.output+"-sim.csv", args.verbose) # data, header, filename
	t_f = time.time()
	if args.verbose:
		print "Spatially adjusted data in {} seconds".format(t_f - t_i)

if __name__ == "__main__":
	main()
<commit_msg>add verbose print conditional<commit_after>#! /usr/bin/env python
"""
Spatial adjust and test precision

"""

import numpy as np

from sklearn import linear_model
from sklearn import cross_validation
from sklearn.metrics import explained_variance_score
from sklearn.metrics import mean_squared_error
import argparse, sys, csv, os, time

def getArgs():
	parser = argparse.ArgumentParser(
		description = """Affine spatial transformation with simulated error"""
	)

	parser.add_argument(
		"-c",
		"--controlPoints",
		type = str,
		required = True,
		help = "Control points csv file. See README doc for formating instructions."
	)

	parser.add_argument(
		"-u",
		"--unprojectPoints",
		type = str,
		required = True,
		help = "Unprojected points on plot coordinate system"
	)

	parser.add_argument(
		"-x",
		"--xname",
		type = str,
		required = True,
		help = "field name for x-coordinate in unprojected points file"

	)

	parser.add_argument(
		"-y",
		"--yname",
		type = str,
		required = True,
		help = "field name for y-coordinate in unprojected points file"

	)

	parser.add_argument(
		"-i",
		"--uid",
		type = str,
		required = True,
		help = "field name for unique identifier of point in unprojected points file"

	)

	parser.add_argument(
		"-n",
		"--nsims",
		type = str,
		required = False,
		help = """Number of simulations"""
	)

	parser.add_argument(
		"-o",
		"--output",
		type = str,
		required = True,
		help = """Output file prefix"""
	)

	parser.add_argument(
		"-v",
		"--verbose",
		action = "store_true",
		help = "Print status updates while executing"
	)

	return parser.parse_args()

def getIndex(header, item):
	"""Get index of variable
	"""
	for i, elem in enumerate(header):
		if elem.lower() == item.lower():
			return i
	return None

def getData(path, fields):
	"""Read unprojected points file as numpy array"""
	data = []
	with open(path, 'rUb') as f:
		indata = csv.reader(f)
		var_indices = []
		for i, line in enumerate(indata):
			if i == 0:
				header = line
				continue
			if 'NA' in line:
				for j, elem in enumerate(line):
					line[j] = np.nan
			data.append(line)
	data = np.array(data, dtype=np.float64)
	indices = np.array([getIndex(header, item) for item in fields])
	header = [header[i] for i in indices]
	data = data[:, indices]
	return header, data

def getControl(path):
	"""Read control points file as numpy array
	"""
	data = []
	with open(path, "rU") as f:
		for i, line in enumerate(f):
			ls = line.strip().split(',')
			if i == 0:
				continue
			data.append(ls)
	data = np.array(data, dtype=np.float64)
	return data

def writeOut(data, header, filename, verbose):
	"""Write data to a file.
	"""
	if verbose:
		print "\nInitializing {0} file output".format(filename)
	if header is not None:

		with open(filename, "w") as f:
			header_str = ",".join(header)+"\n"
			f.write(header_str)
			for line in data:
				row = ",".join([str(elem) for elem in line])+"\n"
				f.write(row)
	else:
		with open(filename, "a") as f:
			for line in data:
				row = ",".join([str(elem) for elem in line])+"\n"
				f.write(row)
	if verbose:
		print "Wrote {0} to disk\n".format(filename)

def fit(X, y):
	"""Fit using matrix algebra
	"""
	XTXinv = inv(np.dot(X.T, X))
	XTy = np.dot(X.T, y)
	B = np.dot(XTXinv, XTy)
	return B

def affine_parameterization(utm_e, utm_n, x, y):
	"""Parameterize affine function
	"""
	X = np.column_stack((x, y, np.ones(len(x))))
	affine x = fit(X, utm_e)
	affine_y = fit(X, utm_n)
	return affine_x, affine_y

def affine_transformation(X_unprj, affine_x, affine_y, args, header):
	"""Predict with affine function
	"""
	x_pred = np.dot(X_unprj, affine_x)
	y_pred = np.dot(X_unprj, affine_y)
	return x_pred, y_pred

def main():
	t_i = time.time()
	args = getArgs()
	initDir = os.getcwd()

	fields = [
		args.uid,
		args.xname,
		args.yname
	]

	header, data = getData(args.unprojectPoints, fields)
	if args.verbose:
		print header, data

	np.random.seed(10)

	cp = getControl(args.controlPoints)
	if args.verbose:
		print cp
		print cp.shape

	if cp.shape[1] == 6 and args.nsims is not None:
		sim = True
		if args.verbose:
			print "Will simulate {} realizations of corners.".format(args.nsims)
	elif args.nsims is None:
		sim = False
	elif cp.shape[1] == 4:
		sim = False
		if args.nsims is not None:
			if args.verbose:
				print "Cannot simulate error because no utm_e and utm_n se's"
				print  "are included in control points file."
	else:
		print "Incorrect dimensions for control points csv. Exiting"
		sys.exit(1)

	# Affine Spatial Transformation Parameterization
	# x' = Ax + By + C
	# y' = Dx + Ey + F
	utm_e, utm_n, x, y = cp[:,0], cp[:,1], cp[:,2], cp[:,3]
	affine_x, affine_y = affine_parameterization(utm_e, utm_n, x, y )

	uids = data[:, getIndex(header, args.uid)]
	gx = data[:, getIndex(header, args.xname)]
	gy = data[:, getIndex(header, args.yname)]
	X_unprj = np.column_stack(
		(
			gx,
			gy
		)
	)
	x_pred, y_pred = affine_transformation(X_unprj, affine_x, affine_y, args, header)

	projected_data = np.column_stack(
		(
			uids,
			gx,
			gy,
			x_pred,
			y_pred
		)
	)

	projected_data_header = ["uid", "gx", "gy", "x_pred", "y_pred"]
	writeOut(projected_data, projected_data_header, args.output+"-projected.csv", args.verbose) # data, header, filename

	# Simulated locations
	if sim:
		# error cols 4 and 5 from cp csv
		utm_se_e = cp[:,4]
		utm_se_n = cp[:,5]
		sim_data_header = ["iter", "uid", "gx", "gy", "x_pred", "y_pred"]
		for i in range(int(args.nsims)):
			if args.verbose:
				print "Simulation number {}".format(i+1)
			evec = np.zeros(cp.shape[0])
			nvec = np.zeros(cp.shape[0])
			for j, row in enumerate(cp):
				evec[j] = np.random.normal(row[0],row[4], 1)
				nvec[j] = np.random.normal(row[1],row[5], 1)
			affine_x, affine_y = affine_parameterization(evec, nvec, x, y)
			x_pred, y_pred = affine_transformation(X_unprj, affine_x, affine_y, args, header)
			iteration = i + np.zeros(x_pred.shape[0])
			sim_data = np.column_stack(
				(
				iteration,
				uids,
				gx,
				gy,
				x_pred,
				y_pred
				)
			)
			if i == 0:
				writeOut(sim_data, sim_data_header, args.output+"-sim.csv", args.verbose) # data, header, filename
			else:
				writeOut(sim_data, None, args.output+"-sim.csv", args.verbose) # data, header, filename
	t_f = time.time()
	if args.verbose:
		print "Spatially adjusted data in {} seconds".format(t_f - t_i)

if __name__ == "__main__":
	main()
<|endoftext|>


<commit_before>import cupy
from cupy import _core

_packbits_kernel = _core.ElementwiseKernel(
    'raw T myarray, raw int32 myarray_size', 'uint8 packed',
    '''for (int j = 0; j < 8; ++j) {
        int k = i * 8 + j;
        int bit = k < myarray_size && myarray[k] != 0;
        packed |= bit << (7 - j);
    }''',
    'cupy_packbits_kernel'
)

def packbits(myarray):
    """Packs the elements of a binary-valued array into bits in a uint8 array.

    This function currently does not support ``axis`` option.

    Args:
        myarray (cupy.ndarray): Input array.

    Returns:
        cupy.ndarray: The packed array.

    .. note::
        When the input array is empty, this function returns a copy of it,
        i.e., the type of the output array is not necessarily always uint8.
        This exactly follows the NumPy's behaviour (as of version 1.11),
        alghough this is inconsistent to the documentation.

    .. seealso:: :func:`numpy.packbits`
    """
    if myarray.dtype.kind not in 'biu':
        raise TypeError(
            'Expected an input array of integer or boolean data type')

    myarray = myarray.ravel()
    packed_size = (myarray.size + 7) // 8
    packed = cupy.zeros((packed_size,), dtype=cupy.uint8)
    return _packbits_kernel(myarray, myarray.size, packed)

_unpackbits_kernel = {'big': _core.ElementwiseKernel(
    'raw uint8 myarray', 'T unpacked',
    'unpacked = (myarray[i / 8] >> (7 - i % 8)) & 1;',
    'cupy_unpackbits_kernel'
),
    'little': _core.ElementwiseKernel(
    'raw uint8 myarray', 'T unpacked',
    'unpacked = (myarray[i / 8] >> (i % 8)) & 1;',
    'cupy_unpackbits_kernel'
)}

def unpackbits(myarray, bitorder='big'):
    """Unpacks elements of a uint8 array into a binary-valued output array.

    This function currently does not support ``axis`` option.

    Args:
        myarray (cupy.ndarray): Input array.

        bitorder (str, optional): bit order to use when unpacking the array, allowed values are `'little'` and `'big'`. Defaults to `'big'`.

    Returns:
        cupy.ndarray: The unpacked array.

    .. seealso:: :func:`numpy.unpackbits`
    """
    if myarray.dtype != cupy.uint8:
        raise TypeError('Expected an input array of unsigned byte data type')

    if bitorder not in ['big', 'little']:
        raise ValueError("bitorder must be either 'big' or 'little'")

    unpacked = cupy.ndarray((myarray.size * 8), dtype=cupy.uint8)
    return _unpackbits_kernel[bitorder](myarray, unpacked)
<commit_msg>fix flake8/autopep8 errors<commit_after>import cupy
from cupy import _core

_packbits_kernel = _core.ElementwiseKernel(
    'raw T myarray, raw int32 myarray_size', 'uint8 packed',
    '''for (int j = 0; j < 8; ++j) {
        int k = i * 8 + j;
        int bit = k < myarray_size && myarray[k] != 0;
        packed |= bit << (7 - j);
    }''',
    'cupy_packbits_kernel'
)

def packbits(myarray):
    """Packs the elements of a binary-valued array into bits in a uint8 array.

    This function currently does not support ``axis`` option.

    Args:
        myarray (cupy.ndarray): Input array.

    Returns:
        cupy.ndarray: The packed array.

    .. note::
        When the input array is empty, this function returns a copy of it,
        i.e., the type of the output array is not necessarily always uint8.
        This exactly follows the NumPy's behaviour (as of version 1.11),
        alghough this is inconsistent to the documentation.

    .. seealso:: :func:`numpy.packbits`
    """
    if myarray.dtype.kind not in 'biu':
        raise TypeError(
            'Expected an input array of integer or boolean data type')

    myarray = myarray.ravel()
    packed_size = (myarray.size + 7) // 8
    packed = cupy.zeros((packed_size,), dtype=cupy.uint8)
    return _packbits_kernel(myarray, myarray.size, packed)

_unpackbits_kernel = {'big': _core.ElementwiseKernel(
    'raw uint8 myarray', 'T unpacked',
    'unpacked = (myarray[i / 8] >> (7 - i % 8)) & 1;',
    'cupy_unpackbits_kernel'
),
    'little': _core.ElementwiseKernel(
    'raw uint8 myarray', 'T unpacked',
    'unpacked = (myarray[i / 8] >> (i % 8)) & 1;',
    'cupy_unpackbits_kernel'
)}

def unpackbits(myarray, bitorder='big'):
    """Unpacks elements of a uint8 array into a binary-valued output array.

    This function currently does not support ``axis`` option.

    Args:
        myarray (cupy.ndarray): Input array.

        bitorder (str, optional): bit order to use when unpacking the array,

        allowed values are `'little'` and `'big'`. Defaults to `'big'`.

    Returns:
        cupy.ndarray: The unpacked array.

    .. seealso:: :func:`numpy.unpackbits`
    """
    if myarray.dtype != cupy.uint8:
        raise TypeError('Expected an input array of unsigned byte data type')

    if bitorder not in ['big', 'little']:
        raise ValueError("bitorder must be either 'big' or 'little'")

    unpacked = cupy.ndarray((myarray.size * 8), dtype=cupy.uint8)
    return _unpackbits_kernel[bitorder](myarray, unpacked)
<|endoftext|>


<commit_before>from math import atan, pi

def number_of_pages(id, network_size):

    if network_size < 1000:
        return id % 2
    else:
        bin_number = (id/network_size)*1000
        if 995 <= bin_number:
            return 50
        if 985 <= bin_number < 995:
            return 20
        if 965 <= bin_number < 985:
            return 10
        if 935 <= bin_number < 965:
            return 5
        if 735 <= bin_number < 935:
            return 1
        else:
            return 0

def number_of_friends(id, network_size, scaling=1):

    bin_number = (id/network_size)*100

    if bin_number <= 90:
        #No growth
        return 2
    if 89 < bin_number <= 99:
        avg_friends = 338 #steady state for active users
        #S-shaped growth on network size
        return avg_friends*(atan((network_size-250000000*scaling)/150000000*scaling) + pi/2)/pi
    else:
        #Linear growth on network size
        proportion_active_users = .17
        return .25*network_size*proportion_active_users

def number_of_activities(id, network_size):

    bin_number = (id/network_size)*100

    if bin_number <= 89:
        return 0
    if 89 < bin_number <= 98:
        return 1
    else:
        return 2.5 #super rough estimate using Obama's twitter account

<commit_msg>Added functions for number of friends, number of activities, number of pages with pseudo random proportions<commit_after>from math import atan, pi

def number_of_pages(user_id, network_size):

    if network_size < 1000:
        return user_id % 2
    else:
        bin_number = (user_id/network_size)*1000
        if 995 <= bin_number:
            return 50
        if 985 <= bin_number < 995:
            return 20
        if 965 <= bin_number < 985:
            return 10
        if 935 <= bin_number < 965:
            return 5
        if 735 <= bin_number < 935:
            return 1
        else:
            return 0

def number_of_friends(user_id, network_size, scaling=1):

    bin_number = (user_id/network_size)*100

    if bin_number <= 90:
        #No growth
        return 2
    if 89 < bin_number <= 99:
        avg_friends = 338 #steady state for active users
        #S-shaped growth on network size
        return avg_friends*(atan((network_size-250000000*scaling)/150000000*scaling) + pi/2)/pi
    else:
        #Linear growth on network size
        proportion_active_users = .17
        return .25*network_size*proportion_active_users

def number_of_activities(user_id, network_size):

    bin_number = (user_id/network_size)*100

    if bin_number <= 89:
        return 0
    if 89 < bin_number <= 98:
        return 1
    else:
        return 2.5 #super rough estimate using Obama's twitter account

<|endoftext|>


<commit_before>"""f90nml.namelist
   ===============

   Tools for creating Fortran namelist files from Python ``dict``s.

   :copyright: Copyright 2014 Marshall Ward, see AUTHORS for details.
   :license: Apache License, Version 2.0, see LICENSE for details.
"""
from __future__ import print_function

import os
try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict

class Namelist(OrderedDict):
    """Case-insensitive Python dict"""

    def __init__(self, *args, **kwds):

        # If using (unordered) dict, then resort the keys for reproducibility
        s_args = list(args)
        if (args and not isinstance(args[0], OrderedDict) and
                isinstance(args[0], dict)):
            s_args[0] = sorted(args[0].items())

        super(Namelist, self).__init__(*s_args, **kwds)

        # Convert any internal dicts to Namelists
        for key, val in self.items():
            if isinstance(val, dict):
                self[key] = Namelist(val)

        # Formatting properties
        self._colwidth = 72
        self._indent = 4 * ' '
        self._end_comma = False
        self._uppercase = False
        self._floatformat = ''
        self._logical_repr = {False: '.false.', True: '.true.'}

        # Namelist group spacing flag
        self._newline = False

        # Representatation functions
        self.f90str = {
            bool:
                lambda x: self.logical_repr[x],
            int:
                lambda x: str(x),
            float:
                lambda x: '{0:{fmt}}'.format(x, fmt=self.floatformat),
            complex:
                lambda x: '({0}, {1})'.format(x.real, x.imag),
            str:
                lambda x: repr(x).replace("\\'", "''").replace('\\"', '""').replace('\\\\', '\\'),
            type(None):
                lambda x: ''
        }

    def __contains__(self, key):
        return super(Namelist, self).__contains__(key.lower())

    def __delitem__(self, key):
        return super(Namelist, self).__delitem__(key.lower())

    def __getitem__(self, key):
        return super(Namelist, self).__getitem__(key.lower())

    def __setitem__(self, key, value):
        super(Namelist, self).__setitem__(key.lower(), value)

    # Format configuration

    # Column width
    @property
    def colwidth(self):
        """Return the target column width of the namelist file."""
        return self._colwidth

    @colwidth.setter
    def colwidth(self, width):
        """Validate and set the column width."""
        if isinstance(width, int):
            if width >= 0:
                self._colwidth = width
            else:
                raise ValueError('Column width must be nonnegative.')
        else:
            raise TypeError('Column width must be a nonnegative integer.')

    # Variable indent
    @property
    def indent(self):
        """Return the indentation string within namelist group entries."""
        return self._indent

    @indent.setter
    def indent(self, value):
        """Validate and set the indent width, either as an explicit whitespace
        string or by the number of whitespace characters.
        """

        # Explicit indent setting
        if isinstance(value, str):
            if value.isspace():
                self._indent = value
            else:
                raise ValueError('String indentation can only contain '
                                 'whitespace.')

        # Set indent width
        elif isinstance(value, int):
            if value >= 0:
                self._indent = value * ' '
            else:
                raise ValueError('Indentation spacing must be nonnegative.')

        else:
            raise TypeError('Indentation must be specified by string or space '
                            'width.')

    # Terminal comma
    @property
    def end_comma(self):
        """Return True if entries are terminated with commas."""
        return self._end_comma

    @end_comma.setter
    def end_comma(self, value):
        """Validate and set the comma termination flag."""
        if not isinstance(value, bool):
            raise TypeError('end_comma attribute must be a logical type.')
        self._end_comma = value

    # Uppercase
    @property
    def uppercase(self):
        """Return True if names are displayed in upper case."""
        return self._uppercase

    @uppercase.setter
    def uppercase(self, value):
        """Validate and set the upper case flag."""
        if not isinstance(value, bool):
            raise TypeError('uppercase attribute must be a logical type.')
        self._uppercase = value

    # Float format
    @property
    def floatformat(self):
        """Return the current floating point format code."""
        return self._floatformat

    @floatformat.setter
    def floatformat(self, value):
        """Validate and set the upper case flag."""
        if isinstance(value, str):
            # Duck-test the format string; raise ValueError on fail
            '{0:{1}}'.format(1.23, value)

            self._floatformat = value
        else:
            raise TypeError('Floating point format code must be a string.')

    # Logical representation
    # NOTE: This presumes that bools and ints are identical as dict keys
    @property
    def logical_repr(self):
        """Return the namelist representations of logical values."""
        return self._logical_repr

    @logical_repr.setter
    def logical_repr(self, value):
        """Set the namelist representations of logical values."""

        if not any(isinstance(value, t) for t in (list, tuple)):
            raise TypeError("Logical representation must be a tuple with "
                            "a valid true and false value.")
        if not len(value) == 2:
            raise ValueError("List must contain two values.")

        self.false_repr = value[0]
        self.true_repr = value[1]

    @property
    def true_repr(self):
        """Return the namelist representation of logical true."""
        return self._logical_repr[1]

    @true_repr.setter
    def true_repr(self, value):
        """Validate and set the logical true representation."""
        if isinstance(value, str):
            if not (value.lower().startswith('t') or
                    value.lower().startswith('.t')):
                raise ValueError("Logical true representation must start with "
                                 "'T' or '.T'.")
            else:
                self._logical_repr[1] = value
        else:
            raise TypeError('Logical true representation must be a string.')

    @property
    def false_repr(self):
        """Return the namelist representation of logical false."""
        return self._logical_repr[0]

    @false_repr.setter
    def false_repr(self, value):
        """Validate and set the logical false representation."""
        if isinstance(value, str):
            if not (value.lower().startswith('f') or
                    value.lower().startswith('.f')):
                raise ValueError("Logical false representation must start "
                                 "with 'F' or '.F'.")
            else:
                self._logical_repr[0] = value
        else:
            raise TypeError('Logical false representation must be a string.')

    # File output

    def write(self, nml_path, force=False):
        """Output dict to a Fortran 90 namelist file."""

        # Reset newline flag
        self._newline = False

        nml_is_file = hasattr(nml_path, 'read')
        if not force and not nml_is_file and os.path.isfile(nml_path):
            raise IOError('File {0} already exists.'.format(nml_path))

        nml_file = nml_path if nml_is_file else open(nml_path, 'w')
        try:
            for grp_name, grp_vars in self.items():
                # Check for repeated namelist records (saved as lists)
                if isinstance(grp_vars, list):
                    for g_vars in grp_vars:
                        self.write_nmlgrp(grp_name, g_vars, nml_file)
                else:
                    self.write_nmlgrp(grp_name, grp_vars, nml_file)
        finally:
            if not nml_is_file:
                nml_file.close()

        ## collect individual (name, namelist) pairs, including repeated names
        #groups = []
        #for (name,vals) in self.items():
        #    if isinstance(vals, list): # repeated namelist
        #        groups.extend((name, v) for v in vals)
        #    else:
        #        groups.append((name, vals))

        #nml_file = nml_path if nml_is_file else open(nml_path, 'w')
        #try:
        #    if len(groups) > 0:
        #        first_name, first_vals = groups[0]
        #        self.write_nmlgrp(first_name, first_vals, nml_file)

        #        for (name, vals) in groups[1:]:
        #            print(file=nml_file) # double-space between groups
        #            self.write_nmlgrp(name, vals, nml_file)
        #finally:
        #    if not nml_is_file:
        #        nml_file.close()

    def write_nmlgrp(self, grp_name, grp_vars, nml_file):
        """Write namelist group to target file."""

        if self._newline:
            print(file=nml_file)
        self._newline = True

        if self.uppercase:
            grp_name = grp_name.upper()

        print('&{0}'.format(grp_name), file=nml_file)

        for v_name, v_val in grp_vars.items():

            for v_str in self.var_strings(v_name, v_val):
                nml_line = self.indent + '{0}'.format(v_str)
                print(nml_line, file=nml_file)

        print('/', file=nml_file)

    def var_strings(self, v_name, v_values, v_idx=None):
        """Convert namelist variable to list of fixed-width strings."""

        if self.uppercase:
            v_name = v_name.upper()

        var_strs = []

        # Parse derived type contents
        if isinstance(v_values, dict):
            for f_name, f_vals in v_values.items():
                v_title = '%'.join([v_name, f_name])

                v_strs = self.var_strings(v_title, f_vals)
                var_strs.extend(v_strs)

        # Parse an array of derived types
        elif (isinstance(v_values, list) and
              any(isinstance(v, dict) for v in v_values) and
              all((isinstance(v, dict) or v is None) for v in v_values)):
            for idx, val in enumerate(v_values, start=1):

                if val is None:
                    continue

                v_title = v_name + '({0})'.format(idx)

                v_strs = self.var_strings(v_title, val)
                var_strs.extend(v_strs)

        # Parse a multidimensional array
        # TODO: Merge with the array of derived types
        elif (isinstance(v_values, list) and
              any(isinstance(v, list) for v in v_values) and
              all((isinstance(v, list) or v is None) for v in v_values)):

            if not v_idx:
                v_idx = []

            v_title = v_name
            for idx, val in enumerate(v_values, start=1):

                v_idx_new = v_idx + [idx]
                v_strs = self.var_strings(v_title, val, v_idx_new)
                var_strs.extend(v_strs)

        else:
            if not isinstance(v_values, list):
                v_values = [v_values]

            if v_idx:
                v_name += '(:, ' + ', '.join(str(i) for i in v_idx[::-1]) + ')'

            # Split output across multiple lines (if necessary)
            val_strs = []

            val_line = ''
            for v_val in v_values:

                v_width = self.colwidth - len(self.indent + v_name + ' = ')

                if len(val_line) < v_width:
                    val_line += self.f90repr(v_val) + ', '

                if len(val_line) >= v_width:
                    val_strs.append(val_line.rstrip())
                    val_line = ''

            # Append any remaining values
            if val_line:
                if self.end_comma or v_values[-1] is None:
                    val_strs.append(val_line)
                else:
                    val_strs.append(val_line[:-2])

            # Complete the set of values
            if val_strs:
                var_strs.append('{0} = {1}'
                                ''.format(v_name, val_strs[0]).strip())

                for v_str in val_strs[1:]:
                    var_strs.append(' ' * (len(v_name + ' = ')) + v_str)

        return var_strs

    def f90repr(self, value):
        """Convert primitive Python types to equivalent Fortran strings."""

        try:
            return self.f90str[type(value)](value)
        except KeyError:
            raise ValueError('Type {0} of {1} cannot be converted to a '
                             'Fortran type.'.format(type(value), value))
<commit_msg>Comment removal<commit_after>"""f90nml.namelist
   ===============

   Tools for creating Fortran namelist files from Python ``dict``s.

   :copyright: Copyright 2014 Marshall Ward, see AUTHORS for details.
   :license: Apache License, Version 2.0, see LICENSE for details.
"""
from __future__ import print_function

import os
try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict

class Namelist(OrderedDict):
    """Case-insensitive Python dict"""

    def __init__(self, *args, **kwds):

        # If using (unordered) dict, then resort the keys for reproducibility
        s_args = list(args)
        if (args and not isinstance(args[0], OrderedDict) and
                isinstance(args[0], dict)):
            s_args[0] = sorted(args[0].items())

        super(Namelist, self).__init__(*s_args, **kwds)

        # Convert any internal dicts to Namelists
        for key, val in self.items():
            if isinstance(val, dict):
                self[key] = Namelist(val)

        # Formatting properties
        self._colwidth = 72
        self._indent = 4 * ' '
        self._end_comma = False
        self._uppercase = False
        self._floatformat = ''
        self._logical_repr = {False: '.false.', True: '.true.'}

        # Namelist group spacing flag
        self._newline = False

        # Representatation functions
        self.f90str = {
            bool:
                lambda x: self.logical_repr[x],
            int:
                lambda x: str(x),
            float:
                lambda x: '{0:{fmt}}'.format(x, fmt=self.floatformat),
            complex:
                lambda x: '({0}, {1})'.format(x.real, x.imag),
            str:
                lambda x: repr(x).replace("\\'", "''").replace('\\"', '""').replace('\\\\', '\\'),
            type(None):
                lambda x: ''
        }

    def __contains__(self, key):
        return super(Namelist, self).__contains__(key.lower())

    def __delitem__(self, key):
        return super(Namelist, self).__delitem__(key.lower())

    def __getitem__(self, key):
        return super(Namelist, self).__getitem__(key.lower())

    def __setitem__(self, key, value):
        super(Namelist, self).__setitem__(key.lower(), value)

    # Format configuration

    # Column width
    @property
    def colwidth(self):
        """Return the target column width of the namelist file."""
        return self._colwidth

    @colwidth.setter
    def colwidth(self, width):
        """Validate and set the column width."""
        if isinstance(width, int):
            if width >= 0:
                self._colwidth = width
            else:
                raise ValueError('Column width must be nonnegative.')
        else:
            raise TypeError('Column width must be a nonnegative integer.')

    # Variable indent
    @property
    def indent(self):
        """Return the indentation string within namelist group entries."""
        return self._indent

    @indent.setter
    def indent(self, value):
        """Validate and set the indent width, either as an explicit whitespace
        string or by the number of whitespace characters.
        """

        # Explicit indent setting
        if isinstance(value, str):
            if value.isspace():
                self._indent = value
            else:
                raise ValueError('String indentation can only contain '
                                 'whitespace.')

        # Set indent width
        elif isinstance(value, int):
            if value >= 0:
                self._indent = value * ' '
            else:
                raise ValueError('Indentation spacing must be nonnegative.')

        else:
            raise TypeError('Indentation must be specified by string or space '
                            'width.')

    # Terminal comma
    @property
    def end_comma(self):
        """Return True if entries are terminated with commas."""
        return self._end_comma

    @end_comma.setter
    def end_comma(self, value):
        """Validate and set the comma termination flag."""
        if not isinstance(value, bool):
            raise TypeError('end_comma attribute must be a logical type.')
        self._end_comma = value

    # Uppercase
    @property
    def uppercase(self):
        """Return True if names are displayed in upper case."""
        return self._uppercase

    @uppercase.setter
    def uppercase(self, value):
        """Validate and set the upper case flag."""
        if not isinstance(value, bool):
            raise TypeError('uppercase attribute must be a logical type.')
        self._uppercase = value

    # Float format
    @property
    def floatformat(self):
        """Return the current floating point format code."""
        return self._floatformat

    @floatformat.setter
    def floatformat(self, value):
        """Validate and set the upper case flag."""
        if isinstance(value, str):
            # Duck-test the format string; raise ValueError on fail
            '{0:{1}}'.format(1.23, value)

            self._floatformat = value
        else:
            raise TypeError('Floating point format code must be a string.')

    # Logical representation
    # NOTE: This presumes that bools and ints are identical as dict keys
    @property
    def logical_repr(self):
        """Return the namelist representations of logical values."""
        return self._logical_repr

    @logical_repr.setter
    def logical_repr(self, value):
        """Set the namelist representations of logical values."""

        if not any(isinstance(value, t) for t in (list, tuple)):
            raise TypeError("Logical representation must be a tuple with "
                            "a valid true and false value.")
        if not len(value) == 2:
            raise ValueError("List must contain two values.")

        self.false_repr = value[0]
        self.true_repr = value[1]

    @property
    def true_repr(self):
        """Return the namelist representation of logical true."""
        return self._logical_repr[1]

    @true_repr.setter
    def true_repr(self, value):
        """Validate and set the logical true representation."""
        if isinstance(value, str):
            if not (value.lower().startswith('t') or
                    value.lower().startswith('.t')):
                raise ValueError("Logical true representation must start with "
                                 "'T' or '.T'.")
            else:
                self._logical_repr[1] = value
        else:
            raise TypeError('Logical true representation must be a string.')

    @property
    def false_repr(self):
        """Return the namelist representation of logical false."""
        return self._logical_repr[0]

    @false_repr.setter
    def false_repr(self, value):
        """Validate and set the logical false representation."""
        if isinstance(value, str):
            if not (value.lower().startswith('f') or
                    value.lower().startswith('.f')):
                raise ValueError("Logical false representation must start "
                                 "with 'F' or '.F'.")
            else:
                self._logical_repr[0] = value
        else:
            raise TypeError('Logical false representation must be a string.')

    # File output

    def write(self, nml_path, force=False):
        """Output dict to a Fortran 90 namelist file."""

        # Reset newline flag
        self._newline = False

        nml_is_file = hasattr(nml_path, 'read')
        if not force and not nml_is_file and os.path.isfile(nml_path):
            raise IOError('File {0} already exists.'.format(nml_path))

        nml_file = nml_path if nml_is_file else open(nml_path, 'w')
        try:
            for grp_name, grp_vars in self.items():
                # Check for repeated namelist records (saved as lists)
                if isinstance(grp_vars, list):
                    for g_vars in grp_vars:
                        self.write_nmlgrp(grp_name, g_vars, nml_file)
                else:
                    self.write_nmlgrp(grp_name, grp_vars, nml_file)
        finally:
            if not nml_is_file:
                nml_file.close()

    def write_nmlgrp(self, grp_name, grp_vars, nml_file):
        """Write namelist group to target file."""

        if self._newline:
            print(file=nml_file)
        self._newline = True

        if self.uppercase:
            grp_name = grp_name.upper()

        print('&{0}'.format(grp_name), file=nml_file)

        for v_name, v_val in grp_vars.items():

            for v_str in self.var_strings(v_name, v_val):
                nml_line = self.indent + '{0}'.format(v_str)
                print(nml_line, file=nml_file)

        print('/', file=nml_file)

    def var_strings(self, v_name, v_values, v_idx=None):
        """Convert namelist variable to list of fixed-width strings."""

        if self.uppercase:
            v_name = v_name.upper()

        var_strs = []

        # Parse derived type contents
        if isinstance(v_values, dict):
            for f_name, f_vals in v_values.items():
                v_title = '%'.join([v_name, f_name])

                v_strs = self.var_strings(v_title, f_vals)
                var_strs.extend(v_strs)

        # Parse an array of derived types
        elif (isinstance(v_values, list) and
              any(isinstance(v, dict) for v in v_values) and
              all((isinstance(v, dict) or v is None) for v in v_values)):
            for idx, val in enumerate(v_values, start=1):

                if val is None:
                    continue

                v_title = v_name + '({0})'.format(idx)

                v_strs = self.var_strings(v_title, val)
                var_strs.extend(v_strs)

        # Parse a multidimensional array
        # TODO: Merge with the array of derived types
        elif (isinstance(v_values, list) and
              any(isinstance(v, list) for v in v_values) and
              all((isinstance(v, list) or v is None) for v in v_values)):

            if not v_idx:
                v_idx = []

            v_title = v_name
            for idx, val in enumerate(v_values, start=1):

                v_idx_new = v_idx + [idx]
                v_strs = self.var_strings(v_title, val, v_idx_new)
                var_strs.extend(v_strs)

        else:
            if not isinstance(v_values, list):
                v_values = [v_values]

            if v_idx:
                v_name += '(:, ' + ', '.join(str(i) for i in v_idx[::-1]) + ')'

            # Split output across multiple lines (if necessary)
            val_strs = []

            val_line = ''
            for v_val in v_values:

                v_width = self.colwidth - len(self.indent + v_name + ' = ')

                if len(val_line) < v_width:
                    val_line += self.f90repr(v_val) + ', '

                if len(val_line) >= v_width:
                    val_strs.append(val_line.rstrip())
                    val_line = ''

            # Append any remaining values
            if val_line:
                if self.end_comma or v_values[-1] is None:
                    val_strs.append(val_line)
                else:
                    val_strs.append(val_line[:-2])

            # Complete the set of values
            if val_strs:
                var_strs.append('{0} = {1}'
                                ''.format(v_name, val_strs[0]).strip())

                for v_str in val_strs[1:]:
                    var_strs.append(' ' * (len(v_name + ' = ')) + v_str)

        return var_strs

    def f90repr(self, value):
        """Convert primitive Python types to equivalent Fortran strings."""

        try:
            return self.f90str[type(value)](value)
        except KeyError:
            raise ValueError('Type {0} of {1} cannot be converted to a '
                             'Fortran type.'.format(type(value), value))
<|endoftext|>


<commit_before>#!/usr/bin/env python
import json
import sys
import yaml

from docker import Client
from docker import errors

def add_subparser(subparsers):
    list_parser = subparsers.add_parser(
        'test-run',
        description='Test run service Docker container (not implemented)',
        help='Test run service Docker container (not implemented)')

    list_parser.add_argument('-s', '--service',
                             help='Name of service for which you wish to '
                             + 'upload a docker image. Leading "services-'
                             + '", as included in a Jenkins job name, will'
                             + ' be stripped.',
                             required=True,
                             )
    list_parser.add_argument('-p', '--path',
                             help='Path to Dockerfile which you want to test-run',
                             required=False,
                             )
    list_parser.add_argument('-t', '--tty',
                             help='Run Docker container with --tty=true',
                             required=False,
                             )
    list_parser.add_argument('-c', '--cmd',
                             help='Run Docker container with particular command, o'
                             + 'for instance bash',
                             required=False,
                             )
    list_parser.add_argument('-C', '--canary',
                             help='Run Docker container with CPU/memory limit set '
                             + 'for canary cluster',
                             required=False,
                             )
    list_parser.add_argument('-v', '--verbose',
                             help='Show Docker commands output',
                             required=False,
                             )
    list_parser.add_argument('-e', '--cluster',
                             help='Specify Marathon cluster',
                             required=False,
                             )

    list_parser.set_defaults(command=paasta_test_run)

def read_marathon_config():
    """
    Read Marathon configs to get cluster info and volumes
    that we need to bind when runngin a container.
    """
    config_path = '/etc/paasta_tools/paasta.json'

    config = json.loads(open(config_path).read())

    volumes = list()

    for volume in config['docker_volumes']:
        volumes.append(volume['hostPath'] + ':' + volume['containerPath'] + ':' + volume['mode'].lower())

    result = dict()

    result['cluster'] = config['cluster']
    result['volumes'] = volumes
    return result

def read_service_manifest(service, cluster, canary):
    """
    Read service manifest to get info about the environment.
    """
    path = '/nail/etc/services/' + service + '/marathon-' + cluster + '.yaml'

    manifest = yaml.load(file(path, 'r'))

    if canary:
        return manifest['main']
    else:
        return manifest['main']

def run_docker_container(docker_client, docker_hash, args):
    """
    Run Docker container by image hash with args set in command line.
    Function prints the output of run command in stdout.
    """
    marathon_config = read_marathon_config()

    service_manifest = read_service_manifest(args.service, marathon_config['cluster'], args.canary)

    print docker_hash

    command = ''
    if args.cmd:
        command = args.cmd

    tty = False
    if args.tty:
        tty = args.tty

    create_result = docker_client.create_container(image=docker_hash,
                                                   command=command,
                                                   tty=tty,
                                                   volumes=marathon_config['volumes'],
                                                   mem_limit=str(service_manifest['mem']) + 'm',
                                                   cpu_shares=service_manifest['cpus'],
                                                   ports=[8888],
                                                   stdin_open=True,
                                                   )

    docker_client.start(create_result['Id'], port_bindings={8888: None})

    for line in docker_client.attach(create_result['Id'], stream=True, logs=True):
        sys.stdout.write(line)

def build_docker_container(docker_client, args):
    """
    Build Docker container from Dockerfile in the current directory or
    specified in command line args. Returns result image hash.
    """
    result = ''

    dockerfile_path = './'

    if args.path:
        dockerfile_path = args.path

    for line in docker_client.build(path=dockerfile_path, tag='latest'):
        line_dict = json.loads(line)

        try:
            stream_line = line_dict['stream']

            if args.verbose:
                sys.stdout.write(stream_line)

            if stream_line.startswith('Successfully built '):
                """Strip the beginning of a string and \n in the end."""
                result = stream_line[len('Successfully built '):]
                result = result[:len(result) - 1]
        except:
            pass

    return result

def paasta_test_run(args):
    docker_client = Client(base_url='unix://var/run/docker.sock')

    try:
        docker_hash = build_docker_container(docker_client, args)
    except errors.APIError as e:
        sys.stderr.write('Can\'t build Docker container with message: ' + str(e) + '\n')
        sys.exit(1)

    try:
        run_docker_container(docker_client, docker_hash, args)
    except errors.APIError as e:
        sys.stderr.write('Can\'t run Docker container with message: ' + str(e) + '\n')
        sys.exit(1)
<commit_msg>Added service name validation.<commit_after>#!/usr/bin/env python
import json
import sys
import yaml

from docker import Client
from docker import errors

from paasta_tools.paasta_cli.utils import validate_service_name

def add_subparser(subparsers):
    list_parser = subparsers.add_parser(
        'test-run',
        description='Test run service Docker container (not implemented)',
        help='Test run service Docker container (not implemented)')

    list_parser.add_argument('-s', '--service',
                             help='Name of service for which you wish to '
                             + 'upload a docker image. Leading "services-'
                             + '", as included in a Jenkins job name, will'
                             + ' be stripped.',
                             required=True,
                             )
    list_parser.add_argument('-p', '--path',
                             help='Path to Dockerfile which you want to test-run',
                             required=False,
                             )
    list_parser.add_argument('-t', '--tty',
                             help='Run Docker container with --tty=true',
                             required=False,
                             )
    list_parser.add_argument('-c', '--cmd',
                             help='Run Docker container with particular command, o'
                             + 'for instance bash',
                             required=False,
                             )
    list_parser.add_argument('-C', '--canary',
                             help='Run Docker container with CPU/memory limit set '
                             + 'for canary cluster',
                             required=False,
                             )
    list_parser.add_argument('-v', '--verbose',
                             help='Show Docker commands output',
                             required=False,
                             )
    list_parser.add_argument('-e', '--cluster',
                             help='Specify Marathon cluster',
                             required=False,
                             )

    list_parser.set_defaults(command=paasta_test_run)

def read_marathon_config():
    """
    Read Marathon configs to get cluster info and volumes
    that we need to bind when runngin a container.
    """
    config_path = '/etc/paasta_tools/paasta.json'

    config = json.loads(open(config_path).read())

    volumes = list()

    for volume in config['docker_volumes']:
        volumes.append(volume['hostPath'] + ':' + volume['containerPath'] + ':' + volume['mode'].lower())

    result = dict()

    result['cluster'] = config['cluster']
    result['volumes'] = volumes
    return result

def read_service_manifest(service, cluster, canary):
    """
    Read service manifest to get info about the environment.
    """
    service_name = service
    if service_name and service_name.startswith('services-'):
        service_name = service_name.split('services-', 1)[1]
    validate_service_name(service_name)

    path = '/nail/etc/services/' + service + '/marathon-' + cluster + '.yaml'

    manifest = yaml.load(file(path, 'r'))

    if canary:
        return manifest['main']
    else:
        return manifest['main']

def run_docker_container(docker_client, docker_hash, args):
    """
    Run Docker container by image hash with args set in command line.
    Function prints the output of run command in stdout.
    """
    marathon_config = read_marathon_config()

    service_manifest = read_service_manifest(args.service, marathon_config['cluster'], args.canary)

    print docker_hash

    command = ''
    if args.cmd:
        command = args.cmd

    tty = False
    if args.tty:
        tty = args.tty

    create_result = docker_client.create_container(image=docker_hash,
                                                   command=command,
                                                   tty=tty,
                                                   volumes=marathon_config['volumes'],
                                                   mem_limit=str(service_manifest['mem']) + 'm',
                                                   cpu_shares=service_manifest['cpus'],
                                                   ports=[8888],
                                                   stdin_open=True,
                                                   )

    docker_client.start(create_result['Id'], port_bindings={8888: None})

    for line in docker_client.attach(create_result['Id'], stream=True, logs=True):
        sys.stdout.write(line)

def build_docker_container(docker_client, args):
    """
    Build Docker container from Dockerfile in the current directory or
    specified in command line args. Returns result image hash.
    """
    result = ''

    dockerfile_path = './'

    if args.path:
        dockerfile_path = args.path

    for line in docker_client.build(path=dockerfile_path, tag='latest'):
        line_dict = json.loads(line)

        try:
            stream_line = line_dict['stream']

            if args.verbose:
                sys.stdout.write(stream_line)

            if stream_line.startswith('Successfully built '):
                """Strip the beginning of a string and \n in the end."""
                result = stream_line[len('Successfully built '):]
                result = result[:len(result) - 1]
        except:
            pass

    return result

def paasta_test_run(args):
    docker_client = Client(base_url='unix://var/run/docker.sock')

    try:
        docker_hash = build_docker_container(docker_client, args)
    except errors.APIError as e:
        sys.stderr.write('Can\'t build Docker container with message: ' + str(e) + '\n')
        sys.exit(1)

    try:
        run_docker_container(docker_client, docker_hash, args)
    except errors.APIError as e:
        sys.stderr.write('Can\'t run Docker container with message: ' + str(e) + '\n')
        sys.exit(1)
<|endoftext|>


<commit_before><commit_msg>a lot: better concordancing, conc saving, resizing windows, etc<commit_after><|endoftext|>


<commit_before>"""
Usage:

    ./manage.py import_tfl_stops < data/tfl/bus-stops.csv
"""

from titlecase import titlecase

from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned

from busstops.management.import_from_csv import ImportFromCSVCommand
from busstops.models import StopPoint

class Command(ImportFromCSVCommand):

    def handle_row(self, row):
        if row['Naptan_Atco'] in (None, '', 'NONE'):
            return None

        try:
            stop = StopPoint.objects.get(pk=row['Naptan_Atco'])
        except ObjectDoesNotExist:
            try:
                stop = StopPoint.objects.get(pk__contains=row['Naptan_Atco'])
            except (ObjectDoesNotExist, MultipleObjectsReturned) as e:
                print e, row
                return None

        if row['Heading'] != '':
            stop.heading = row['Heading']
        stop.tfl = True

        if stop.street.isupper():
            stop.street = titlecase(stop.street)
        if stop.landmark.isupper():
            stop.landmark = titlecase(stop.landmark)

        stop.save()
<commit_msg>Fetch TfL stop names<commit_after>"""
Usage:

    ./manage.py import_tfl_stops < data/tfl/bus-stops.csv
"""

import requests
from titlecase import titlecase

from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned

from busstops.management.import_from_csv import ImportFromCSVCommand
from busstops.models import StopPoint

class Command(ImportFromCSVCommand):

    @staticmethod
    def get_name(atco_code):
        """
        Given a stop's ATCO code, returns the best-formatted version of its common name from the
        TfL API
        """
        data = requests.get('https://api.tfl.gov.uk/StopPoint/%s' % atco_code).json()
        return data.get('commonName')

    def handle_row(self, row):
        if row['Naptan_Atco'] in (None, '', 'NONE'):
            return None

        try:
            stop = StopPoint.objects.get(pk=row['Naptan_Atco'])
        except ObjectDoesNotExist:
            try:
                stop = StopPoint.objects.get(pk__contains=row['Naptan_Atco'])
            except (ObjectDoesNotExist, MultipleObjectsReturned) as e:
                print e, row
                return None

        if row['Heading'] != '':
            stop.heading = row['Heading']
        stop.common_name = self.get_name(stop.atco_code)
        stop.tfl = True

        if stop.street.isupper():
            stop.street = titlecase(stop.street)
        if stop.landmark.isupper():
            stop.landmark = titlecase(stop.landmark)

        stop.save()
<|endoftext|>


<commit_before>import binascii
import json
import logging
import os
from datetime import timedelta
from functools import update_wrapper
from logging.handlers import TimedRotatingFileHandler
from multiprocessing import Process
import subprocess

import click
import git
import pushover
import yaml
from flask import Flask, Response, abort, current_app, jsonify, make_response, request
from utilkit import fileutil

import settings

app = Flask(__name__)
app.debug = settings.DEBUG

logger = logging.getLogger('webhaak')
logger.setLevel(logging.DEBUG)
# Log will rotate daily with a max history of LOG_BACKUP_COUNT
fh = TimedRotatingFileHandler(
    settings.LOG_LOCATION,
    when='d',
    interval=1,
    backupCount=settings.LOG_BACKUP_COUNT
)
fh.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
logger.addHandler(fh)

# Load the configuration of the various projects/hooks
with open(settings.PROJECTS_FILE, 'r') as pf:
    projects = fileutil.yaml_ordered_load(pf, yaml.SafeLoader)

def notify_user(result, config):
    """Send a PushOver message if configured, after git and command have run

    result is a dictionary with fields:
      command_result
      status: 'OK' | 'error'
      type: 'commanderror'
      message
    """
    try:
        projectname = config[0]
        triggerconfig = config[1]
        title = ''
        branch = 'master'
        if 'repo_branch' in triggerconfig:
            branch = triggerconfig['repo_branch']
        message = 'repo: {}\nbranch: {}\ncommand: {}'.format(triggerconfig['repo'], branch, triggerconfig['command'])
        if result['status'] == 'OK':
            title = "Hook for {} ran successfully".format(projectname)
        else:
            title = "Hook for {} failed: {}".format(projectname, result['type'])
            message = message + '\n\n{}'.format(result['message'])
        logging.debug(message)
        logging.info('Sending notification...')
        client = pushover.Client(settings.PUSHOVER_USERKEY, api_token=settings.PUSHOVER_APPTOKEN)
        client.send_message(message, title=title)
        logging.info('Notification sent')
    except AttributeError:
        logging.warning('Notification through PushOver failed because of missing configuration')

def gettriggersettings(appkey, triggerkey):
    """Look up the trigger and return the repo and command to be updated and fired"""
    for project in projects:
        if projects[project]['appkey'] == appkey:
            for trigger in projects[project]['triggers']:
                if projects[project]['triggers'][trigger]['triggerkey'] == triggerkey:
                    return (project, projects[project]['triggers'][trigger])
    return None

def get_repo_basename(repo_url):
    """Extract repository basename from its url, as that will be the name of directory it will be cloned into"""
    result = os.path.basename(repo_url)
    filename, file_extension = os.path.splitext(result)
    if file_extension == '.git':
        # Strip the .git from the name, as Git will do the same on non-bare checkouts
        result = filename
    return result

def fetchinfo_to_str(fetchinfo):
    """git.remote.FetchInfo to human readable representation"""
    result = fetchinfo[0].note
    return result

def update_repo(config):
    """Update (pull) the Git repo"""
    projectname = config[0]
    triggerconfig = config[1]

    repo_url = triggerconfig['repo']
    repo_parent = settings.REPOS_CACHE_DIR
    if 'repoparent' in triggerconfig and triggerconfig['repoparent']:
        repo_parent = triggerconfig['repoparent']

    logger.info('[%s] Updating %s', projectname, repo_url)
    logger.info('[%s] Repo parent %s', projectname, repo_parent)

    # Ensure cache dir for webhaak exists and is writable
    fileutil.ensure_dir_exists(repo_parent)  # throws OSError if repo_parent is not writable

    # TODO: check whether dir exists with different repository
    repo_dir = os.path.join(repo_parent, get_repo_basename(repo_url))
    logger.info('[%s] Repo dir %s', projectname, repo_dir)
    if os.path.isdir(repo_dir):
        # Repo already exists locally, do a pull
        logger.info('[%s] Repo exists, pull', projectname)

        apprepo = git.Repo(repo_dir)
        origin = apprepo.remote('origin')
        result = fetchinfo_to_str(origin.fetch())  # assure we actually have data. fetch() returns useful information
        logger.info('[%s] Fetch result: %s', projectname, result)
        origin.pull()
        logger.info('[%s] Done pulling, checkout()', projectname)
        result += ' ' + str(apprepo.git.checkout())
    else:
        # Repo needs to be cloned
        logger.info('[%s] Repo does not exist yet, clone', projectname)
        apprepo = git.Repo.init(repo_dir)
        origin = apprepo.create_remote('origin', repo_url)
        origin.fetch()                  # assure we actually have data. fetch() returns useful information
        # Setup a local tracking branch of a remote branch
        apprepo.create_head('master', origin.refs.master).set_tracking_branch(origin.refs.master)
        # push and pull behaves similarly to `git push|pull`
        result = origin.pull()
        logger.info('[%s] Done pulling, checkout()', projectname)
        result += ' ' + str(apprepo.git.checkout())
    return result

def run_command(config):
    """Run the command(s) defined for this trigger"""
    projectname = config[0]
    triggerconfig = config[1]
    if 'command' not in triggerconfig:
        # No command to execute, return
        logger.info('[%s] No command to execute', projectname)
        return None
    command = triggerconfig['command']
    # Replace some placeholders to be used in executing scripts from one of the repos
    command = command.replace('REPODIR', os.path.join(settings.REPOS_CACHE_DIR, projectname))
    command = command.replace('CACHEDIR', settings.REPOS_CACHE_DIR)
    command = command.strip()  # ensure no weird linefeeds and superfluous whitespace are there
    logger.info('[%s] Executing `%s`', projectname, command)

    # TODO: capture_output is new in Python 3.7, replaces stdout and stderr
    # result = subprocess.run(command_parts, capture_output=True, check=True, shell=True, universal_newlines=True)
    result = subprocess.run(
        command,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        shell=True,
        universal_newlines=True
    )
    return result

def do_pull_andor_command(config):
    """Asynchronous task, performing the git pulling and the specified scripting inside a Process"""
    projectname = config[0]
    result = {'application': projectname}
    result['trigger'] = config[1]
    if 'repo' in config[1]:
        try:
            result['repo_result'] = update_repo(config)
            logger.info('[%s] result repo: %s', projectname, str(result['repo_result']))
        except git.GitCommandError as e:
            result = {'status': 'error', 'type': 'giterror', 'message': str(e)}
            logger.error('[%s] giterror: %s', projectname, str(e))
            notify_user(result, config)
        except (OSError, KeyError) as e:
            result = {'status': 'error', 'type': 'oserror', 'message': str(e)}
            logger.error('[%s] oserror: %s', projectname, str(e))
            notify_user(result, config)

    cmdresult = run_command(config)
    if cmdresult.returncode == 0:
        logger.info('[%s] success for command: %s', projectname, str(cmdresult.stdout))
        result['status'] = 'OK'
    else:
        result['status'] = 'error'
        result['type'] = 'commanderror'
        result['message'] = cmdresult.stderr.strip()
        # TODO: seperate logfiles per job? Filename then based on appkey_triggerkey_timestamp.log
        logger.error(
            '[%s] commanderror with returncode %s: %s',
            projectname,
            str(cmdresult.returncode),
            cmdresult.stderr
        )
        logger.error('[%s] stdout: %s', projectname, cmdresult.stdout)
        logger.error('[%s] stderr: %s', projectname, cmdresult.stderr)

    notify_user(result, config)

# == API request support functions/mixins ======

def crossdomain(origin=None, methods=None, headers=None,
                max_age=21600, attach_to_all=True,
                automatic_options=True):
    """Decorator to send the correct cross-domain headers
    src: https://blog.skyred.fi/articles/better-crossdomain-snippet-for-flask.html
    """
    if methods is not None:
        methods = ', '.join(sorted(x.upper() for x in methods))
    if headers is not None and not isinstance(headers, str):
        headers = ', '.join(x.upper() for x in headers)
    if not isinstance(origin, str):
        origin = ', '.join(origin)
    if isinstance(max_age, timedelta):
        max_age = max_age.total_seconds()

    def get_methods():
        if methods is not None:
            return methods

        options_resp = current_app.make_default_options_response()
        return options_resp.headers['allow']

    def decorator(f):
        def wrapped_function(*args, **kwargs):
            if automatic_options and request.method == 'OPTIONS':
                resp = current_app.make_default_options_response()
            else:
                resp = make_response(f(*args, **kwargs))
            if not attach_to_all and request.method != 'OPTIONS':
                return resp

            h = resp.headers
            h['Access-Control-Allow-Origin'] = origin
            h['Access-Control-Allow-Methods'] = get_methods()
            h['Access-Control-Max-Age'] = str(max_age)
            h['Access-Control-Allow-Credentials'] = 'true'
            h['Access-Control-Allow-Headers'] = \
                "Origin, X-Requested-With, Content-Type, Accept, Authorization"
            if headers is not None:
                h['Access-Control-Allow-Headers'] = headers
            return resp

        f.provide_automatic_options = False
        return update_wrapper(wrapped_function, f)
    return decorator

class APIException(Exception):
    status_code = 400

    def __init__(self, message, status_code=None, payload=None):
        Exception.__init__(self)
        self.message = message
        if status_code is not None:
            self.status_code = status_code
        self.payload = payload

    def to_dict(self):
        rv = dict(self.payload or ())
        rv['message'] = self.message
        rv['status_code'] = self.status_code
        return rv

class InvalidAPIUsage(APIException):
    status_code = 400

@app.errorhandler(404)
def page_not_found(e):
    return jsonify(error=404, text=str(e)), 404

@app.errorhandler(InvalidAPIUsage)
def handle_invalid_usage(error):
    response = jsonify(error.to_dict())
    response.status_code = error.status_code
    response.mimetype = 'application/json'
    return response

# == Web app endpoints ======

@app.route('/')
def indexpage():
    logger.debug('Root page requested')
    return 'Welcome to Webhaak, see the documentation to how to setup and use webhooks.'

@app.route('/admin/<secretkey>/list', methods=['GET'])
def listtriggers(secretkey):
    """List the appkeys and triggerkeys"""
    print(secretkey)
    print(settings.SECRETKEY)
    try:
        if secretkey != settings.SECRETKEY:
            abort(404)
    except AttributeError:
        abort(404)

    result = {}
    for project in projects:
        result[project] = {
            'title': project,
            'appkey': projects[project]['appkey'],
            'triggers': [],
        }
        for trigger in projects[project]['triggers']:
            result[project]['triggers'].append(
                {
                    'title': trigger,
                    'triggerkey': projects[project]['triggers'][trigger]['triggerkey']
                }
            )
    return Response(
        json.dumps({'projects': result}), status=200, mimetype='application/json'
    )

@app.route('/app/<appkey>/<triggerkey>', methods=['GET', 'OPTIONS', 'POST'])
#@crossdomain(origin='*', max_age=settings.MAX_CACHE_AGE)
def apptrigger(appkey, triggerkey):
    """Fire the trigger described by the configuration under `triggerkey`"""
    logger.info('%s on appkey: %s triggerkey: %s', request.method, appkey, triggerkey)
    if request.method == 'POST':
        # Likely some ping was sent, check if so
        if request.headers.get('X-GitHub-Event') == "ping":
            payload = request.get_json()
            logger.info(
                'received GitHub ping for %s hook: %s ',
                payload['repository']['full_name'],
                payload['hook']['url']
            )
            return json.dumps({'msg': 'Hi!'})
        elif request.headers.get('X-GitHub-Event') != "push":
            payload = request.get_json()
            logger.info(
                'received wrong event type from GitHub for %s hook: %s',
                payload['repository']['full_name'],
                payload['hook']['url']
            )
            return json.dumps({'msg': "wrong event type"})
        else:
            payload = request.get_json()
            event_info = 'received push from GitHub for '
            if 'repository' in payload:
                event_info += payload['repository']['full_name']
            if 'pusher' in payload:
                event_info += ' by ' + payload['pusher']['name']
            if 'compare' in payload:
                event_info += ', compare: ' + payload['compare']
            logger.info(payload)
            logger.info(event_info)

    config = gettriggersettings(appkey, triggerkey)
    if config is None:
        logger.error('appkey/triggerkey combo not found')
        return Response(json.dumps({'status': 'Error'}), status=404, mimetype='application/json')
    p = Process(target=do_pull_andor_command, args=(config,))
    p.start()
    return Response(
        json.dumps({
            'status': 'OK',
            'message': 'Command accepted and will be run in the background'
        }), status=200, mimetype='application/json'
    )

@app.route('/monitor')
#@app.route('/monitor/')
#@app.route('/monitor/monitor.html')
def monitor():
    """Monitoring ping"""
    result = 'OK'
    return result

def generatekey():
    """Generate a random ascii string to be used as identifier"""
    return binascii.hexlify(os.urandom(24))

@app.cli.command()
def printappkey():
    """Generate new appkey"""
    print(generatekey())

@app.route('/getappkey')
def getappkey():
    """Generate new appkey"""
    return Response(json.dumps({'key': generatekey().decode('utf-8')}), status=200, mimetype='application/json')

if __name__ == '__main__':
    if not settings.DEBUG:
        app.run(port=settings.PORT, debug=settings.DEBUG)
    else:
        app.run(host='0.0.0.0', port=settings.PORT, debug=settings.DEBUG)
<commit_msg>Pre-emptive fix for doing jsonp calls<commit_after>import binascii
import json
import logging
import os
from datetime import timedelta
from functools import update_wrapper
from logging.handlers import TimedRotatingFileHandler
from multiprocessing import Process
import subprocess

import click
import git
import pushover
import yaml
from flask import Flask, Response, abort, current_app, jsonify, make_response, request
from utilkit import fileutil

import settings

app = Flask(__name__)
app.debug = settings.DEBUG

logger = logging.getLogger('webhaak')
logger.setLevel(logging.DEBUG)
# Log will rotate daily with a max history of LOG_BACKUP_COUNT
fh = TimedRotatingFileHandler(
    settings.LOG_LOCATION,
    when='d',
    interval=1,
    backupCount=settings.LOG_BACKUP_COUNT
)
fh.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
logger.addHandler(fh)

# Load the configuration of the various projects/hooks
with open(settings.PROJECTS_FILE, 'r') as pf:
    projects = fileutil.yaml_ordered_load(pf, yaml.SafeLoader)

def notify_user(result, config):
    """Send a PushOver message if configured, after git and command have run

    result is a dictionary with fields:
      command_result
      status: 'OK' | 'error'
      type: 'commanderror'
      message
    """
    try:
        projectname = config[0]
        triggerconfig = config[1]
        title = ''
        branch = 'master'
        if 'repo_branch' in triggerconfig:
            branch = triggerconfig['repo_branch']
        message = 'repo: {}\nbranch: {}\ncommand: {}'.format(triggerconfig['repo'], branch, triggerconfig['command'])
        if result['status'] == 'OK':
            title = "Hook for {} ran successfully".format(projectname)
        else:
            title = "Hook for {} failed: {}".format(projectname, result['type'])
            message = message + '\n\n{}'.format(result['message'])
        logging.debug(message)
        logging.info('Sending notification...')
        client = pushover.Client(settings.PUSHOVER_USERKEY, api_token=settings.PUSHOVER_APPTOKEN)
        client.send_message(message, title=title)
        logging.info('Notification sent')
    except AttributeError:
        logging.warning('Notification through PushOver failed because of missing configuration')

def gettriggersettings(appkey, triggerkey):
    """Look up the trigger and return the repo and command to be updated and fired"""
    for project in projects:
        if projects[project]['appkey'] == appkey:
            for trigger in projects[project]['triggers']:
                if projects[project]['triggers'][trigger]['triggerkey'] == triggerkey:
                    return (project, projects[project]['triggers'][trigger])
    return None

def get_repo_basename(repo_url):
    """Extract repository basename from its url, as that will be the name of directory it will be cloned into"""
    result = os.path.basename(repo_url)
    filename, file_extension = os.path.splitext(result)
    if file_extension == '.git':
        # Strip the .git from the name, as Git will do the same on non-bare checkouts
        result = filename
    return result

def fetchinfo_to_str(fetchinfo):
    """git.remote.FetchInfo to human readable representation"""
    result = fetchinfo[0].note
    return result

def update_repo(config):
    """Update (pull) the Git repo"""
    projectname = config[0]
    triggerconfig = config[1]

    repo_url = triggerconfig['repo']
    repo_parent = settings.REPOS_CACHE_DIR
    if 'repoparent' in triggerconfig and triggerconfig['repoparent']:
        repo_parent = triggerconfig['repoparent']

    logger.info('[%s] Updating %s', projectname, repo_url)
    logger.info('[%s] Repo parent %s', projectname, repo_parent)

    # Ensure cache dir for webhaak exists and is writable
    fileutil.ensure_dir_exists(repo_parent)  # throws OSError if repo_parent is not writable

    # TODO: check whether dir exists with different repository
    repo_dir = os.path.join(repo_parent, get_repo_basename(repo_url))
    logger.info('[%s] Repo dir %s', projectname, repo_dir)
    if os.path.isdir(repo_dir):
        # Repo already exists locally, do a pull
        logger.info('[%s] Repo exists, pull', projectname)

        apprepo = git.Repo(repo_dir)
        origin = apprepo.remote('origin')
        result = fetchinfo_to_str(origin.fetch())  # assure we actually have data. fetch() returns useful information
        logger.info('[%s] Fetch result: %s', projectname, result)
        origin.pull()
        logger.info('[%s] Done pulling, checkout()', projectname)
        result += ' ' + str(apprepo.git.checkout())
    else:
        # Repo needs to be cloned
        logger.info('[%s] Repo does not exist yet, clone', projectname)
        apprepo = git.Repo.init(repo_dir)
        origin = apprepo.create_remote('origin', repo_url)
        origin.fetch()                  # assure we actually have data. fetch() returns useful information
        # Setup a local tracking branch of a remote branch
        apprepo.create_head('master', origin.refs.master).set_tracking_branch(origin.refs.master)
        # push and pull behaves similarly to `git push|pull`
        result = origin.pull()
        logger.info('[%s] Done pulling, checkout()', projectname)
        result += ' ' + str(apprepo.git.checkout())
    return result

def run_command(config):
    """Run the command(s) defined for this trigger"""
    projectname = config[0]
    triggerconfig = config[1]
    if 'command' not in triggerconfig:
        # No command to execute, return
        logger.info('[%s] No command to execute', projectname)
        return None
    command = triggerconfig['command']
    # Replace some placeholders to be used in executing scripts from one of the repos
    command = command.replace('REPODIR', os.path.join(settings.REPOS_CACHE_DIR, projectname))
    command = command.replace('CACHEDIR', settings.REPOS_CACHE_DIR)
    command = command.strip()  # ensure no weird linefeeds and superfluous whitespace are there
    logger.info('[%s] Executing `%s`', projectname, command)

    # TODO: capture_output is new in Python 3.7, replaces stdout and stderr
    # result = subprocess.run(command_parts, capture_output=True, check=True, shell=True, universal_newlines=True)
    result = subprocess.run(
        command,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        shell=True,
        universal_newlines=True
    )
    return result

def do_pull_andor_command(config):
    """Asynchronous task, performing the git pulling and the specified scripting inside a Process"""
    projectname = config[0]
    result = {'application': projectname}
    result['trigger'] = config[1]
    if 'repo' in config[1]:
        try:
            result['repo_result'] = update_repo(config)
            logger.info('[%s] result repo: %s', projectname, str(result['repo_result']))
        except git.GitCommandError as e:
            result = {'status': 'error', 'type': 'giterror', 'message': str(e)}
            logger.error('[%s] giterror: %s', projectname, str(e))
            notify_user(result, config)
        except (OSError, KeyError) as e:
            result = {'status': 'error', 'type': 'oserror', 'message': str(e)}
            logger.error('[%s] oserror: %s', projectname, str(e))
            notify_user(result, config)

    cmdresult = run_command(config)
    if cmdresult.returncode == 0:
        logger.info('[%s] success for command: %s', projectname, str(cmdresult.stdout))
        result['status'] = 'OK'
    else:
        result['status'] = 'error'
        result['type'] = 'commanderror'
        result['message'] = cmdresult.stderr.strip()
        # TODO: seperate logfiles per job? Filename then based on appkey_triggerkey_timestamp.log
        logger.error(
            '[%s] commanderror with returncode %s: %s',
            projectname,
            str(cmdresult.returncode),
            cmdresult.stderr
        )
        logger.error('[%s] stdout: %s', projectname, cmdresult.stdout)
        logger.error('[%s] stderr: %s', projectname, cmdresult.stderr)

    notify_user(result, config)

# == API request support functions/mixins ======

def crossdomain(origin=None, methods=None, headers=None,
                max_age=21600, attach_to_all=True,
                automatic_options=True):
    """Decorator to send the correct cross-domain headers
    src: https://blog.skyred.fi/articles/better-crossdomain-snippet-for-flask.html
    """
    if methods is not None:
        methods = ', '.join(sorted(x.upper() for x in methods))
    if headers is not None and not isinstance(headers, str):
        headers = ', '.join(x.upper() for x in headers)
    if not isinstance(origin, str):
        origin = ', '.join(origin)
    if isinstance(max_age, timedelta):
        max_age = max_age.total_seconds()

    def get_methods():
        if methods is not None:
            return methods

        options_resp = current_app.make_default_options_response()
        return options_resp.headers['allow']

    def decorator(f):
        def wrapped_function(*args, **kwargs):
            if automatic_options and request.method == 'OPTIONS':
                resp = current_app.make_default_options_response()
            else:
                resp = make_response(f(*args, **kwargs))
            if not attach_to_all and request.method != 'OPTIONS':
                return resp

            h = resp.headers
            h['Access-Control-Allow-Origin'] = origin
            h['Access-Control-Allow-Methods'] = get_methods()
            h['Access-Control-Max-Age'] = str(max_age)
            h['Access-Control-Allow-Credentials'] = 'true'
            h['Access-Control-Allow-Headers'] = \
                "Origin, X-Requested-With, Content-Type, Accept, Authorization"
            if headers is not None:
                h['Access-Control-Allow-Headers'] = headers
            return resp

        f.provide_automatic_options = False
        return update_wrapper(wrapped_function, f)
    return decorator

class APIException(Exception):
    status_code = 400

    def __init__(self, message, status_code=None, payload=None):
        Exception.__init__(self)
        self.message = message
        if status_code is not None:
            self.status_code = status_code
        self.payload = payload

    def to_dict(self):
        rv = dict(self.payload or ())
        rv['message'] = self.message
        rv['status_code'] = self.status_code
        return rv

class InvalidAPIUsage(APIException):
    status_code = 400

@app.errorhandler(404)
def page_not_found(e):
    return jsonify(error=404, text=str(e)), 404

@app.errorhandler(InvalidAPIUsage)
def handle_invalid_usage(error):
    response = jsonify(error.to_dict())
    response.status_code = error.status_code
    response.mimetype = 'application/json'
    return response

# == Web app endpoints ======

@app.route('/')
def indexpage():
    logger.debug('Root page requested')
    return 'Welcome to Webhaak, see the documentation to how to setup and use webhooks.'

@app.route('/admin/<secretkey>/list', methods=['GET'])
@crossdomain(origin='*')
def listtriggers(secretkey):
    """List the appkeys and triggerkeys"""
    print(secretkey)
    print(settings.SECRETKEY)
    try:
        if secretkey != settings.SECRETKEY:
            abort(404)
    except AttributeError:
        abort(404)

    result = {}
    for project in projects:
        result[project] = {
            'title': project,
            'appkey': projects[project]['appkey'],
            'triggers': [],
        }
        for trigger in projects[project]['triggers']:
            result[project]['triggers'].append(
                {
                    'title': trigger,
                    'triggerkey': projects[project]['triggers'][trigger]['triggerkey']
                }
            )
    return Response(
        json.dumps({'projects': result}), status=200, mimetype='application/json'
    )

@app.route('/app/<appkey>/<triggerkey>', methods=['GET', 'OPTIONS', 'POST'])
#@crossdomain(origin='*', max_age=settings.MAX_CACHE_AGE)
def apptrigger(appkey, triggerkey):
    """Fire the trigger described by the configuration under `triggerkey`"""
    logger.info('%s on appkey: %s triggerkey: %s', request.method, appkey, triggerkey)
    if request.method == 'POST':
        # Likely some ping was sent, check if so
        if request.headers.get('X-GitHub-Event') == "ping":
            payload = request.get_json()
            logger.info(
                'received GitHub ping for %s hook: %s ',
                payload['repository']['full_name'],
                payload['hook']['url']
            )
            return json.dumps({'msg': 'Hi!'})
        elif request.headers.get('X-GitHub-Event') != "push":
            payload = request.get_json()
            logger.info(
                'received wrong event type from GitHub for %s hook: %s',
                payload['repository']['full_name'],
                payload['hook']['url']
            )
            return json.dumps({'msg': "wrong event type"})
        else:
            payload = request.get_json()
            event_info = 'received push from GitHub for '
            if 'repository' in payload:
                event_info += payload['repository']['full_name']
            if 'pusher' in payload:
                event_info += ' by ' + payload['pusher']['name']
            if 'compare' in payload:
                event_info += ', compare: ' + payload['compare']
            logger.info(payload)
            logger.info(event_info)

    config = gettriggersettings(appkey, triggerkey)
    if config is None:
        logger.error('appkey/triggerkey combo not found')
        return Response(json.dumps({'status': 'Error'}), status=404, mimetype='application/json')
    p = Process(target=do_pull_andor_command, args=(config,))
    p.start()
    return Response(
        json.dumps({
            'status': 'OK',
            'message': 'Command accepted and will be run in the background'
        }), status=200, mimetype='application/json'
    )

@app.route('/monitor')
#@app.route('/monitor/')
#@app.route('/monitor/monitor.html')
def monitor():
    """Monitoring ping"""
    result = 'OK'
    return result

def generatekey():
    """Generate a random ascii string to be used as identifier"""
    return binascii.hexlify(os.urandom(24))

@app.cli.command()
def printappkey():
    """Generate new appkey"""
    print(generatekey())

@app.route('/getappkey')
def getappkey():
    """Generate new appkey"""
    return Response(json.dumps({'key': generatekey().decode('utf-8')}), status=200, mimetype='application/json')

if __name__ == '__main__':
    if not settings.DEBUG:
        app.run(port=settings.PORT, debug=settings.DEBUG)
    else:
        app.run(host='0.0.0.0', port=settings.PORT, debug=settings.DEBUG)
<|endoftext|>


<commit_before>import re

from django import forms
from django.core.exceptions import ValidationError

from tower import ugettext as _
from tower import ugettext_lazy as _lazy

from mozillians.groups.helpers import stringify_groups
from mozillians.groups.models import Group

class SortForm(forms.Form):
    """Group Index Sort Form."""
    sort = forms.ChoiceField(required=False,
                             choices=(('name', _lazy(u'Name A-Z')),
                                      ('-num_members', _lazy(u'Most Members')),
                                      ('num_members', _lazy(u'Fewest Members'))))

    def clean_sort(self):
        if self.cleaned_data['sort'] == '':
            return 'name'
        return self.cleaned_data['sort']

class GroupWidget(forms.TextInput):

    def render(self, name, value, attrs=None):
        if not (value is None or isinstance(value, basestring)):
            value = stringify_groups(Group.objects.get(pk=v) for v in value)

        return super(GroupWidget, self).render(name, value, attrs)

class GroupField(forms.CharField):
    widget = GroupWidget

    def clean(self, value):
        """Groups are saved in lowercase because it's easy and
        consistent.

        """
        value = super(GroupField, self).clean(value)

        if not re.match(r'^[a-zA-Z0-9 .:,-]*$', value):
            raise ValidationError(_(u'Groups can only contain alphanumeric '
                                    'characters, dashes, spaces.'))

        values = [g.strip() for g in value.lower().split(',')
                  if g and ',' not in g]

        groups = []
        for g in values:
            (group, created) = Group.objects.get_or_create(name=g)

            if not group.system:
                groups.append(group)

        return groups
<commit_msg>Remove some unused code from groups/forms.py<commit_after>from django import forms

from tower import ugettext_lazy as _lazy

class SortForm(forms.Form):
    """Group Index Sort Form."""
    sort = forms.ChoiceField(required=False,
                             choices=(('name', _lazy(u'Name A-Z')),
                                      ('-num_members', _lazy(u'Most Members')),
                                      ('num_members', _lazy(u'Fewest Members'))))

    def clean_sort(self):
        if self.cleaned_data['sort'] == '':
            return 'name'
        return self.cleaned_data['sort']
<|endoftext|>