<commit_before>'''
Collection of algorithms for PyRate.
Author: Ben Davies, ANUSF
'''

from math import pi
from itertools import product

from numpy import sin, cos, radians, unique, histogram, std, mean
from numpy import float32, nan, isnan, sum as nsum, array, ndarray
from pygraph.classes.graph import graph
from pygraph.algorithms.minmax import minimal_spanning_tree

import config
from shared import EpochList

# constants
MM_PER_METRE = 1000

def wavelength_to_mm(data, wavelength):
	"""Converts ROIPAC phase from metres to millimetres"""
	return data * MM_PER_METRE * (wavelength / (4 * pi))

def los_conversion(phase_data, unit_vec_component):
	'''Converts phase from LOS to horizontal/vertical components. Args are
	numpy arrays.'''

	# NB: currently not tested as implementation is too simple
	return phase_data * unit_vec_component

def unit_vector(incidence, azimuth):
	vertical = cos(incidence)
	north_south = sin(incidence) * sin(azimuth)
	east_west = sin(incidence) * cos(azimuth)
	return east_west, north_south, vertical

def get_epochs(ifgs):
	masters = [i.MASTER for i in ifgs]
	slaves = [i.SLAVE for i in ifgs]

	combined = masters + slaves
	dates, n = unique(combined, False, True)
	repeat, _ = histogram(n, bins=len(set(n)))

	# absolute span for each date from the zero/start point
	span = [ (dates[i] - dates[0]).days / 365.25 for i in range(len(dates)) ]
	return EpochList(dates, repeat, span)

def ref_pixel(params, ifgs):
	'''Return (y,x) reference pixel coordinate given open Ifgs.'''

	head = ifgs[0]
	refx = params.get(config.REFX, 0)
	refy = params.get(config.REFY, 0)

	# sanity check any specified ref pixel settings
	if refx != 0 or refy != 0:
		if refx < 1 or refx > head.WIDTH - 1:
			raise ValueError("Invalid reference pixel X coordinate: %s" % refx)
		if refy < 1 or refy > head.FILE_LENGTH - 1:
			raise ValueError("Invalid reference pixel Y coordinate: %s" % refy)
		return (refy, refx)  # reuse preset ref pixel

	check_ref_pixel_params(params, head)

	# pre-calculate useful amounts
	refnx = params[config.REFNX]
	refny = params[config.REFNY]
	chipsize = params[config.REF_CHIP_SIZE]
	radius = chipsize / 2
	phase_stack = array([i.phase_data for i in ifgs]) # TODO: mem efficiencies?
	thresh = params[config.REF_MIN_FRAC] * chipsize * chipsize
	min_sd = float("inf") # dummy start value

	# do window searches across dataset, central pixel of stack with smallest mean
	# is the reference pixel
	for y in _step(head.FILE_LENGTH, refny, radius):
		for x in _step(head.WIDTH, refnx, radius):
			data = phase_stack[:, y-radius:y+radius+1, x-radius:x+radius+1]
			valid = [nsum(~isnan(i)) > thresh for i in data]

			if all(valid): # ignore stack if 1+ ifgs have too many incoherent cells
				sd = [std( i[~isnan(i)] ) for i in data]
				mean_sd = mean(sd)
				if mean_sd < min_sd:
					min_sd = mean_sd
					refy, refx = y, x

	if (refy, refx) == (0, 0):
		raise RefPixelError("Could not find a reference pixel")
	return refy, refx

def check_ref_pixel_params(params, head):
	'''Validates reference pixel search parameters. head is any Ifg.'''

	def missing_option_error(option):
		msg = "Missing '%s' in configuration options" % option
		raise config.ConfigException(msg)

	# sanity check chipsize setting
	chipsize = params.get(config.REF_CHIP_SIZE)
	if chipsize is None:
		missing_option_error(config.REF_CHIP_SIZE)

	if chipsize < 3 or chipsize > head.WIDTH or (chipsize % 2 == 0):
		raise ValueError("Chipsize setting must be >=3 and at least <= grid width")

	# sanity check minimum fraction
	min_frac = params.get(config.REF_MIN_FRAC)
	if min_frac is None:
		missing_option_error(config.REF_MIN_FRAC)

	if min_frac < 0.0 or min_frac > 1.0:
		raise ValueError("Minimum fraction setting must be >= 0.0 and <= 1.0 ")

	# sanity check X|Y steps
	refnx = params.get(config.REFNX)
	if refnx is None:
		missing_option_error(config.REFNX)

	max_width = (head.WIDTH - (chipsize-1))
	if refnx < 1 or refnx > max_width:
		raise ValueError("Invalid refnx setting, must be > 0 and < %s" % max_width)

	refny = params.get(config.REFNY)
	if refny is None:
		missing_option_error(config.REFNY)

	max_rows = (head.FILE_LENGTH - (chipsize-1))
	if refny < 1 or refny > max_rows:
		raise ValueError("Invalid refny setting, must be > 0 and < %s" % max_rows)

def _step(dim, ref, radius):
	'''Returns xrange obj of axis indicies for a search window. dim is the total
	length of the grid dimension. ref is the desired number of steps. radius is #
	cells out from the centre of the chip, or (chipsize / 2).'''

	if ref == 1:
		# centre a single search step
		return xrange(dim // 2, dim, dim) # fake step to ensure single xrange value

	max_dim = dim - (2*radius) # max possible number for refn(x|y)
	if ref == 2: # handle 2 search windows, method below doesn't cover the case
		return [radius, dim-radius-1]

	step = max_dim // (ref-1)
	return xrange(radius, dim, step)

def _remove_root_node(mst):
	"""Discard pygraph's root node from MST dict to conserve memory."""
	for k in mst.keys():
		if mst[k] is None:
			del mst[k]

def mst_matrix(ifgs, epochs):
	'''Returns array of minimum spanning trees for the Ifgs.'''

	# TODO: implement rows memory saving option/ row by row access?

	# locally cache all edges/weights for on-the-fly graph modification
	edges = [i.DATE12 for i in ifgs]
	weights = [i.nan_fraction for i in ifgs]

	# make default MST to optimise result when no Ifg cells in a stack are nans
	g = graph()
	g.add_nodes(epochs.dates) # each acquisition is a node
	for edge, weight in zip(edges, weights):
		g.add_edge(edge, wt=weight)

	default_mst = minimal_spanning_tree(g)
	_remove_root_node(default_mst)

	# prepare source and dest data arrays
	# [i.phase_data for i in ifgs]
	num_ifgs = len(ifgs)
	data_stack = array([i.phase_data for i in ifgs], dtype=float32)
	mst_result = ndarray(shape=(i.FILE_LENGTH, i.WIDTH), dtype=object)

	# create MSTs for each pixel in the ifg data stack
	for y, x in product(xrange(i.FILE_LENGTH), xrange(i.WIDTH)):
		values = data_stack[:,y,x] # select stack of all ifg values for a pixel
		nc = sum(isnan(values))

		# optimisations: use precreated results for all nans/no nans
		if nc == 0:
			mst_result[y,x] = default_mst
			continue
		elif nc == num_ifgs:
			mst_result[y,x] = nan
			continue

		# otherwise dynamically adjust graph, skipping edges where pixels are NaN
		for value, edge, weight in zip(values, edges, weights):
			if not isnan(value):
				if not g.has_edge(edge):
					g.add_edge(edge, wt=weight)
			else:
				if g.has_edge(edge):
					g.del_edge(edge)

		mst = minimal_spanning_tree(g)
		_remove_root_node(mst)
		mst_result[y,x] = mst

	return mst_result

class RefPixelError(Exception):
	pass
<commit_msg>Fixed error message for invalid refnx|y parameters.<commit_after>'''
Collection of algorithms for PyRate.
Author: Ben Davies, ANUSF
'''

from math import pi
from itertools import product

from numpy import sin, cos, radians, unique, histogram, std, mean
from numpy import float32, nan, isnan, sum as nsum, array, ndarray
from pygraph.classes.graph import graph
from pygraph.algorithms.minmax import minimal_spanning_tree

import config
from shared import EpochList

# constants
MM_PER_METRE = 1000

def wavelength_to_mm(data, wavelength):
	"""Converts ROIPAC phase from metres to millimetres"""
	return data * MM_PER_METRE * (wavelength / (4 * pi))

def los_conversion(phase_data, unit_vec_component):
	'''Converts phase from LOS to horizontal/vertical components. Args are
	numpy arrays.'''

	# NB: currently not tested as implementation is too simple
	return phase_data * unit_vec_component

def unit_vector(incidence, azimuth):
	vertical = cos(incidence)
	north_south = sin(incidence) * sin(azimuth)
	east_west = sin(incidence) * cos(azimuth)
	return east_west, north_south, vertical

def get_epochs(ifgs):
	masters = [i.MASTER for i in ifgs]
	slaves = [i.SLAVE for i in ifgs]

	combined = masters + slaves
	dates, n = unique(combined, False, True)
	repeat, _ = histogram(n, bins=len(set(n)))

	# absolute span for each date from the zero/start point
	span = [ (dates[i] - dates[0]).days / 365.25 for i in range(len(dates)) ]
	return EpochList(dates, repeat, span)

def ref_pixel(params, ifgs):
	'''Return (y,x) reference pixel coordinate given open Ifgs.'''

	head = ifgs[0]
	refx = params.get(config.REFX, 0)
	refy = params.get(config.REFY, 0)

	# sanity check any specified ref pixel settings
	if refx != 0 or refy != 0:
		if refx < 1 or refx > head.WIDTH - 1:
			raise ValueError("Invalid reference pixel X coordinate: %s" % refx)
		if refy < 1 or refy > head.FILE_LENGTH - 1:
			raise ValueError("Invalid reference pixel Y coordinate: %s" % refy)
		return (refy, refx)  # reuse preset ref pixel

	check_ref_pixel_params(params, head)

	# pre-calculate useful amounts
	refnx = params[config.REFNX]
	refny = params[config.REFNY]
	chipsize = params[config.REF_CHIP_SIZE]
	radius = chipsize / 2
	phase_stack = array([i.phase_data for i in ifgs]) # TODO: mem efficiencies?
	thresh = params[config.REF_MIN_FRAC] * chipsize * chipsize
	min_sd = float("inf") # dummy start value

	# do window searches across dataset, central pixel of stack with smallest mean
	# is the reference pixel
	for y in _step(head.FILE_LENGTH, refny, radius):
		for x in _step(head.WIDTH, refnx, radius):
			data = phase_stack[:, y-radius:y+radius+1, x-radius:x+radius+1]
			valid = [nsum(~isnan(i)) > thresh for i in data]

			if all(valid): # ignore stack if 1+ ifgs have too many incoherent cells
				sd = [std( i[~isnan(i)] ) for i in data]
				mean_sd = mean(sd)
				if mean_sd < min_sd:
					min_sd = mean_sd
					refy, refx = y, x

	if (refy, refx) == (0, 0):
		raise RefPixelError("Could not find a reference pixel")
	return refy, refx

def check_ref_pixel_params(params, head):
	'''Validates reference pixel search parameters. head is any Ifg.'''

	def missing_option_error(option):
		msg = "Missing '%s' in configuration options" % option
		raise config.ConfigException(msg)

	# sanity check chipsize setting
	chipsize = params.get(config.REF_CHIP_SIZE)
	if chipsize is None:
		missing_option_error(config.REF_CHIP_SIZE)

	if chipsize < 3 or chipsize > head.WIDTH or (chipsize % 2 == 0):
		raise ValueError("Chipsize setting must be >=3 and at least <= grid width")

	# sanity check minimum fraction
	min_frac = params.get(config.REF_MIN_FRAC)
	if min_frac is None:
		missing_option_error(config.REF_MIN_FRAC)

	if min_frac < 0.0 or min_frac > 1.0:
		raise ValueError("Minimum fraction setting must be >= 0.0 and <= 1.0 ")

	# sanity check X|Y steps
	refnx = params.get(config.REFNX)
	if refnx is None:
		missing_option_error(config.REFNX)

	max_width = (head.WIDTH - (chipsize-1))
	if refnx < 1 or refnx > max_width:
		raise ValueError("Invalid refnx setting, must be > 0 and <= %s" % max_width)

	refny = params.get(config.REFNY)
	if refny is None:
		missing_option_error(config.REFNY)

	max_rows = (head.FILE_LENGTH - (chipsize-1))
	if refny < 1 or refny > max_rows:
		raise ValueError("Invalid refny setting, must be > 0 and <= %s" % max_rows)

def _step(dim, ref, radius):
	'''Returns xrange obj of axis indicies for a search window. dim is the total
	length of the grid dimension. ref is the desired number of steps. radius is #
	cells out from the centre of the chip, or (chipsize / 2).'''

	if ref == 1:
		# centre a single search step
		return xrange(dim // 2, dim, dim) # fake step to ensure single xrange value

	max_dim = dim - (2*radius) # max possible number for refn(x|y)
	if ref == 2: # handle 2 search windows, method below doesn't cover the case
		return [radius, dim-radius-1]

	step = max_dim // (ref-1)
	return xrange(radius, dim, step)

def _remove_root_node(mst):
	"""Discard pygraph's root node from MST dict to conserve memory."""
	for k in mst.keys():
		if mst[k] is None:
			del mst[k]

def mst_matrix(ifgs, epochs):
	'''Returns array of minimum spanning trees for the Ifgs.'''

	# TODO: implement rows memory saving option/ row by row access?

	# locally cache all edges/weights for on-the-fly graph modification
	edges = [i.DATE12 for i in ifgs]
	weights = [i.nan_fraction for i in ifgs]

	# make default MST to optimise result when no Ifg cells in a stack are nans
	g = graph()
	g.add_nodes(epochs.dates) # each acquisition is a node
	for edge, weight in zip(edges, weights):
		g.add_edge(edge, wt=weight)

	default_mst = minimal_spanning_tree(g)
	_remove_root_node(default_mst)

	# prepare source and dest data arrays
	# [i.phase_data for i in ifgs]
	num_ifgs = len(ifgs)
	data_stack = array([i.phase_data for i in ifgs], dtype=float32)
	mst_result = ndarray(shape=(i.FILE_LENGTH, i.WIDTH), dtype=object)

	# create MSTs for each pixel in the ifg data stack
	for y, x in product(xrange(i.FILE_LENGTH), xrange(i.WIDTH)):
		values = data_stack[:,y,x] # select stack of all ifg values for a pixel
		nc = sum(isnan(values))

		# optimisations: use precreated results for all nans/no nans
		if nc == 0:
			mst_result[y,x] = default_mst
			continue
		elif nc == num_ifgs:
			mst_result[y,x] = nan
			continue

		# otherwise dynamically adjust graph, skipping edges where pixels are NaN
		for value, edge, weight in zip(values, edges, weights):
			if not isnan(value):
				if not g.has_edge(edge):
					g.add_edge(edge, wt=weight)
			else:
				if g.has_edge(edge):
					g.del_edge(edge)

		mst = minimal_spanning_tree(g)
		_remove_root_node(mst)
		mst_result[y,x] = mst

	return mst_result

class RefPixelError(Exception):
	pass
<|endoftext|>


<commit_before>#! /usr/bin/env python2
"""Script to see if some standard files have changed

A 'change' is any change to size, mtime or md5sum
"""

from __future__ import print_function
import os
import csv
import md5
import sys
import argparse
from datetime import datetime

from bdb import BdbQuit
from dotsite import paths

__version__ = '1.0.0'

def run_args(args, methods):
    """Run any methods eponymous with args"""
    if not args:
        return False
    valuable_args = {k for k, v in args.__dict__.items() if v}
    arg_methods = {methods[a] for a in valuable_args if a in methods}
    for method in arg_methods:
        method(args)

def version(args):
    print('%s %s' % (args, __version__))
    raise SystemExit

def Use_debugger(_args):
    try:
        import pudb as pdb
    except ImportError:
        import pdb
    pdb.set_trace()

def parse_args(methods):
    """Parse out command line arguments"""
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument('-v', '--version', action='store_true',
                        help='Show version')
    parser.add_argument('-U', '--Use_debugger', action='store_true',
                        help='Run the script with pdb (or pudb if available)')
    args = parser.parse_args()
    run_args(args, methods)
    return args

def plastic_date():
    """Looks like a date, quacks like a date. Not a date"""
    return 'Zun, 99 Zun 9999 99:61:61'

class Signature(object):
    """Identifying details for a file

    Includes time, size and md5sum
    """
    def __init__(self, t, z, s):
        self._time = t
        self._size = z
        self._sum = s

    def __str__(self):
        return ', '.join([self._time_str(), str(self._size)])

    def __nonzero__(self):
        return bool(self._time) or self._size > -1 or bool(self._sum)

    def __cmp__(self, other):
        return cmp(self.strings(), other.strings())

    def _time_str(self):
        """A formatted string for the time"""
        try:
            if not self._time:
                raise ValueError
            format_ = '%a, %d %b %Y %H:%M:%S'
            return datetime.fromtimestamp(float(self._time)).strftime(format_)
        except ValueError:
            return plastic_date()

    def strings(self):
        """A list of time, size and sum as strings"""
        return [str(self._time), str(self._size), str(self._sum)]

class EmptySignature(Signature):
    def __init__(self):
        super(EmptySignature, self).__init__(0, -1, '')

def pad_keys(items, keys):
    """Make sure all keys are in items

    If any key is missing add an empty signature"""
    for key in keys:
        if key not in items:
            items[key] = EmptySignature()
    return items

def path_to_data():
    """Where we store old values"""
    return paths.environ_path('JAB') / 'local/login_sums.csv'

def write_files(items, path):
    """Write items to csv file"""
    with path.open('wb') as stream:
        writer = csv.writer(stream)
        for k, v in items.iteritems():
            if not v:
                continue
            row = [k] + v.strings()
            writer.writerow(row)

def default_values():
    """Gives a dict of default values for the default basenames"""
    return pad_keys({}, default_basenames())

def write_default_values():
    """Write default values to the data file"""
    values = default_values()
    write_files(values, path_to_data())
    return values

def read_old_values(basenames):
    """Read old date, size and md5sum for those basenames"""
    result = {}
    with path_to_data().open() as stream:
        reader = csv.reader(stream)
        for row in reader:
            result[row[0]] = Signature(*row[1:])
    return pad_keys(result, basenames)

def old_values(basenames):
    """Read old date, size and md5sum for those basenames

    If old values are not present, write defaults
    """
    if not path_to_data().isfile():
        return write_default_values()
    else:
        return read_old_values(basenames)

def new_values(basenames):
    """Get date, size and md5sum for those basenames in $HOME"""
    result = {}
    for basename in basenames:
        p = paths.home() / basename
        if not p.isfile():
            continue
        size = '%d' % p.size
        mtime = '%0.8f' % p.mtime
        m = md5.new()
        m.update(p.text())
        result[basename] = Signature(mtime, size, m.hexdigest())
    return pad_keys(result, basenames)

def default_basenames():
    return [
        '.bashrc',
        '.vimrc',
        '.gitconfig',
        '.netrc',
        '.profile',
    ]

def script(args):
    """Compare date, size and md5sum for known files in $HOME"""
    basenames = default_basenames()
    old = old_values(basenames)
    new = new_values(basenames)
    result = True
    for basename in basenames:
        if old[basename] == new[basename]:
            continue
        print('%s changed:' % basename)
        print('\tOld: %s' % old[basename])
        print('\tNew: %s' % new[basename])
        result = False
    if not result:
        print('')
        write_files(new, path_to_data())
    return result

def main():
    """Run the script"""
    try:
        args = parse_args(globals())
        return os.EX_OK if script(args) else not os.EX_OK
    except BdbQuit:
        pass
    except SystemExit as e:
        return e.code
    except Exception, e:  # pylint: disable=broad-except
        if __version__[0] < '1':
            raise
        print(e, sys.stderr)
        return not os.EX_OK
    return os.EX_OK

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Show newly found files as "registered"<commit_after>#! /usr/bin/env python2
"""Script to see if some standard files have changed

A 'change' is any change to size, mtime or md5sum
"""

from __future__ import print_function
import os
import csv
import md5
import sys
import argparse
from datetime import datetime

from bdb import BdbQuit
from dotsite import paths

__version__ = '1.0.0'

def run_args(args, methods):
    """Run any methods eponymous with args"""
    if not args:
        return False
    valuable_args = {k for k, v in args.__dict__.items() if v}
    arg_methods = {methods[a] for a in valuable_args if a in methods}
    for method in arg_methods:
        method(args)

def version(args):
    print('%s %s' % (args, __version__))
    raise SystemExit

def Use_debugger(_args):
    try:
        import pudb as pdb
    except ImportError:
        import pdb
    pdb.set_trace()

def parse_args(methods):
    """Parse out command line arguments"""
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument('-v', '--version', action='store_true',
                        help='Show version')
    parser.add_argument('-U', '--Use_debugger', action='store_true',
                        help='Run the script with pdb (or pudb if available)')
    args = parser.parse_args()
    run_args(args, methods)
    return args

def plastic_date():
    """Looks like a date, quacks like a date. Not a date"""
    return 'Zun, 99 Zun 9999 99:61:61'

class Signature(object):
    """Identifying details for a file

    Includes time, size and md5sum
    """
    def __init__(self, t, z, s):
        self._time = t
        self._size = z
        self._sum = s

    def __str__(self):
        return ', '.join([self._time_str(), str(self._size)])

    def __nonzero__(self):
        return bool(self._time) or self._size > -1 or bool(self._sum)

    def __cmp__(self, other):
        return cmp(self.strings(), other.strings())

    def _time_str(self):
        """A formatted string for the time"""
        try:
            if not self._time:
                raise ValueError
            format_ = '%a, %d %b %Y %H:%M:%S'
            return datetime.fromtimestamp(float(self._time)).strftime(format_)
        except ValueError:
            return plastic_date()

    def strings(self):
        """A list of time, size and sum as strings"""
        return [str(self._time), str(self._size), str(self._sum)]

class EmptySignature(Signature):
    def __init__(self):
        super(EmptySignature, self).__init__(0, -1, '')

def pad_keys(items, keys):
    """Make sure all keys are in items

    If any key is missing add an empty signature"""
    for key in keys:
        if key not in items:
            items[key] = EmptySignature()
    return items

def path_to_data():
    """Where we store old values"""
    return paths.environ_path('JAB') / 'local/login_sums.csv'

def write_files(items, path):
    """Write items to csv file"""
    with path.open('wb') as stream:
        writer = csv.writer(stream)
        for k, v in items.iteritems():
            if not v:
                continue
            row = [k] + v.strings()
            writer.writerow(row)

def default_values():
    """Gives a dict of default values for the default basenames"""
    return pad_keys({}, default_basenames())

def write_default_values():
    """Write default values to the data file"""
    values = default_values()
    write_files(values, path_to_data())
    return values

def read_old_values(basenames):
    """Read old date, size and md5sum for those basenames"""
    result = {}
    with path_to_data().open() as stream:
        reader = csv.reader(stream)
        for row in reader:
            result[row[0]] = Signature(*row[1:])
    return pad_keys(result, basenames)

def old_values(basenames):
    """Read old date, size and md5sum for those basenames

    If old values are not present, write defaults
    """
    if not path_to_data().isfile():
        return write_default_values()
    else:
        return read_old_values(basenames)

def new_values(basenames):
    """Get date, size and md5sum for those basenames in $HOME"""
    result = {}
    for basename in basenames:
        p = paths.home() / basename
        if not p.isfile():
            continue
        size = '%d' % p.size
        mtime = '%0.8f' % p.mtime
        m = md5.new()
        m.update(p.text())
        result[basename] = Signature(mtime, size, m.hexdigest())
    return pad_keys(result, basenames)

def default_basenames():
    return [
        '.bashrc',
        '.vimrc',
        '.gitconfig',
        '.netrc',
        '.profile',
    ]

def script(args):
    """Compare date, size and md5sum for known files in $HOME"""
    basenames = default_basenames()
    old = old_values(basenames)
    new = new_values(basenames)
    result = True
    for basename in basenames:
        if old[basename] == new[basename]:
            continue
        if old[basename]:
            print('%s changed:' % basename)
            print('\tOld: %s' % old[basename])
        else:
            print('%s registered:' % basename)
            print('\tNew: %s' % new[basename])
        result = False
    if not result:
        print('')
        write_files(new, path_to_data())
    return result

def main():
    """Run the script"""
    try:
        args = parse_args(globals())
        return os.EX_OK if script(args) else not os.EX_OK
    except BdbQuit:
        pass
    except SystemExit as e:
        return e.code
    except Exception, e:  # pylint: disable=broad-except
        if __version__[0] < '1':
            raise
        print(e, sys.stderr)
        return not os.EX_OK
    return os.EX_OK

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>import shap
import numpy as np

def test_null_model():
    explainer = shap.KernelExplainer(lambda x: np.zeros(x.shape[0]), np.ones((2,10)), nsamples=100)#knn.predict
    e = explainer.explain(np.ones((1,10)))

def test_front_page_model_agnostic():
    from shap import KernelExplainer, DenseData, visualize, initjs
    from sklearn import datasets,neighbors
    from numpy import random, arange

    # print the JS visualization code to the notebook
    initjs()

    # train a k-nearest neighbors classifier on a random subset
    iris = datasets.load_iris()
    random.seed(2)
    inds = arange(len(iris.target))
    random.shuffle(inds)
    knn = neighbors.KNeighborsClassifier()
    knn.fit(iris.data, iris.target == 0)

    # use Shap to explain a single prediction
    background = DenseData(iris.data[inds[:100],:], iris.feature_names) # name the features
    explainer = KernelExplainer(knn.predict, background, nsamples=100)#knn.predict
    x = iris.data[inds[102:103],:]
    visualize(explainer.explain(x))
<commit_msg>One more test<commit_after>import shap
import numpy as np

def test_null_model_small():
    explainer = shap.KernelExplainer(lambda x: np.zeros(x.shape[0]), np.ones((2,4)), nsamples=100)#knn.predict
    e = explainer.explain(np.ones((1,4)))
    assert np.sum(np.abs(e.effects)) < 1e-8

def test_null_model():
    explainer = shap.KernelExplainer(lambda x: np.zeros(x.shape[0]), np.ones((2,10)), nsamples=100)#knn.predict
    e = explainer.explain(np.ones((1,10)))

def test_front_page_model_agnostic():
    from shap import KernelExplainer, DenseData, visualize, initjs
    from sklearn import datasets,neighbors
    from numpy import random, arange

    # print the JS visualization code to the notebook
    initjs()

    # train a k-nearest neighbors classifier on a random subset
    iris = datasets.load_iris()
    random.seed(2)
    inds = arange(len(iris.target))
    random.shuffle(inds)
    knn = neighbors.KNeighborsClassifier()
    knn.fit(iris.data, iris.target == 0)

    # use Shap to explain a single prediction
    background = DenseData(iris.data[inds[:100],:], iris.feature_names) # name the features
    explainer = KernelExplainer(knn.predict, background, nsamples=100)#knn.predict
    x = iris.data[inds[102:103],:]
    visualize(explainer.explain(x))
<|endoftext|>


<commit_before>#! /usr/bin/env python

import argparse
from clint.textui.colored import red, green, blue

from time import sleep
from clint.textui import progress
import csv
import os
import sys

class Usage(Exception):
    def __init__(self, msg, data):
        self.msg = msg
        self.data = data

def processManifest(args):
    # print "Hello World!"
    # print args
    manifestPath = os.path.join(args.baseDir, 'sprites.mf')
    if not os.path.exists(manifestPath):
        raise Usage("Manifest not found at " + manifestPath)
    lineCount = len(open(manifestPath).readlines())

    manifest = csv.DictReader(open(manifestPath), skipinitialspace=True)
    manifest.fieldnames = ['filename', 'spritesheet']
    spritesheets = {}

    for line in progress.bar(manifest,
                             label="Reading Manifest: ",
                             expected_size=lineCount):
        sheet = line['spritesheet']
        image = line['filename']
        imagePath = os.path.join(args.baseDir, image)
        if not os.path.exists(imagePath):
            raise Usage("Image not found at %s from %s, %s." %
                        (red(imagePath, bold=True),
                         blue(manifestPath, bold=True),
                         blue('line ' + str(manifest.line_num), bold=True)),
                        (imagePath, manifestPath, manifest.line_num))
        spritesheets.setdefault(sheet, []).append(image)
    return spritesheets

def main(argv=None):
    if argv is None:
        argv = sys.argv
    try:
        parser = argparse.ArgumentParser(
            add_help=True,
            description='A tool to merge svg files, and generate pngs.'
        )
        parser.add_argument(
            'baseDir',
            action="store",
            help='The directory to find the images and the manifests in.')
        args = parser.parse_args(argv[1:])
        spritesheets = processManifest(args)
        print spritesheets
    except Usage, err:
        print >>sys.stderr
        print >>sys.stderr, red("Error:", bold=True)
        print >>sys.stderr, err.msg
        print >>sys.stderr
        print >>sys.stderr, green("for help use --help", bold=True)
        return 2

if __name__ == "__main__":
    sys.exit(main())
<commit_msg>Fix the Usage error when there’s a missing manifest.<commit_after>#! /usr/bin/env python

import argparse
from clint.textui.colored import red, green, blue

from time import sleep
from clint.textui import progress
import csv
import os
import sys

class Usage(Exception):
    def __init__(self, msg, data):
        self.msg = msg
        self.data = data

def processManifest(args):
    # print "Hello World!"
    # print args
    manifestPath = os.path.join(args.baseDir, 'sprites.mf')
    if not os.path.exists(manifestPath):
        raise Usage("Manifest not found at %s." %
                    (red(manifestPath, bold=True)),
                    (manifestPath,))
    lineCount = len(open(manifestPath).readlines())

    manifest = csv.DictReader(open(manifestPath), skipinitialspace=True)
    manifest.fieldnames = ['filename', 'spritesheet']
    spritesheets = {}

    for line in progress.bar(manifest,
                             label="Reading Manifest: ",
                             expected_size=lineCount):
        sheet = line['spritesheet']
        image = line['filename']
        imagePath = os.path.join(args.baseDir, image)
        if not os.path.exists(imagePath):
            raise Usage("Image not found at %s from %s, %s." %
                        (red(imagePath, bold=True),
                         blue(manifestPath, bold=True),
                         blue('line ' + str(manifest.line_num), bold=True)),
                        (imagePath, manifestPath, manifest.line_num))
        spritesheets.setdefault(sheet, []).append(image)
    return spritesheets

def main(argv=None):
    if argv is None:
        argv = sys.argv
    try:
        parser = argparse.ArgumentParser(
            add_help=True,
            description='A tool to merge svg files, and generate pngs.'
        )
        parser.add_argument(
            'baseDir',
            action="store",
            help='The directory to find the images and the manifests in.')
        args = parser.parse_args(argv[1:])
        spritesheets = processManifest(args)
        print spritesheets
    except Usage, err:
        print >>sys.stderr
        print >>sys.stderr, red("Error:", bold=True)
        print >>sys.stderr, err.msg
        print >>sys.stderr
        print >>sys.stderr, green("for help use --help", bold=True)
        return 2

if __name__ == "__main__":
    sys.exit(main())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import (absolute_import, print_function, unicode_literals)
from acli.output import (output_ascii_table, dash_if_none)
from colorclass import Color, Windows
Windows.enable(auto_colors=True, reset_atexit=True)

def get_tag(name=None, tags=None):
    if tags:
        for tag in tags:
            if tag.get('Key') == name:
                return tag.get('Value')

def split_string(the_string=None, chunk_size=None):
    length = 0
    output = str()
    chunks = the_string.split()
    for chunk in chunks:
        length += len(chunk)
        output += chunk
        if length >= chunk_size:
            output += "\n"
            length = 0
        else:
            output += " "
    return output.rstrip()

def output_secgroup_list(output_media=None, secgroups=None):
    """
    @type output_media: unicode
    @type secgroups: dict
    """
    if output_media == 'console':
        td = list()
        td.append([Color('{autoblue}group id{/autoblue}'),
                   Color('{autoblue}group name{/autoblue}'),
                   Color('{autoblue}description{/autoblue}')])
        for secgroup in secgroups.get('SecurityGroups'):
            td.append([secgroup.get('GroupId'),
                      split_string(secgroup.get('GroupName'), chunk_size=30),
                      split_string(secgroup.get('Description'), chunk_size=40)])
        output_ascii_table(table_title=Color('{autowhite}security groups{/autowhite}'),
                           table_data=td,
                           inner_heading_row_border=True)
    exit(0)

def get_ip_ranges(ip_ranges):
    out = str()
    for ip_range in ip_ranges:
        out += "{0}\n".format(ip_range.get('CidrIp'))
    return out.rstrip()

def get_uid_group_pairs(uid_gps=None):
    out = str()
    for uid_gp in uid_gps:
        out += '{0} - {1}\n'.format(uid_gp.get('UserId'), uid_gp.get('GroupId'))
    return out.strip()

def output_secgroup_info(output_media=None, secgroup=None):
    """
    @type output_media: unicode
    @type secgroup: ec2.SecurityGroup
    """
    if secgroup:
        if output_media == 'console':
            td = list()
            td.append([Color('{autoblue}group id{/autoblue}'), secgroup.get('GroupId')])
            td.append([Color('{autoblue}group name{/autoblue}'), secgroup.get('GroupName')])
            td.append([Color('{autoblue}description{/autoblue}'), secgroup.get('Description')])
            td.append([Color('{autoblue}vpc id{/autoblue}'), dash_if_none(secgroup.get('VpcId'))])
            if secgroup.get('IpPermissions'):
                td.append(['ip permissions', '{0}'.format("-" * 30)])
                for ip_perm in secgroup.get('IpPermissions'):
                    td.append([Color('{autoblue}from port{/autoblue}'), str(ip_perm.get('FromPort'))])
                    td.append([Color('{autoblue}to port{/autoblue}'), str(ip_perm.get('ToPort'))])
                    td.append([Color('{autoblue}ip ranges{/autoblue}'), dash_if_none(get_ip_ranges(ip_perm.get('IpRanges')))])
                    td.append([Color('{autoblue}ip protocol{/autoblue}'), str(ip_perm.get('IpProtocol'))])
                    if ip_perm.get('UserIdGroupPairs'):
                        td.append(['user id group pairs', '{0}'.format("-" * 30)])
                        td.append(['', get_uid_group_pairs(uid_gps=ip_perm.get('UserIdGroupPairs'))])

            if secgroup.get('IpPermissionsEgress'):
                td.append(['ip permissions egress', '{0}'.format("-" * 30)])
                for ip_perm_egress in secgroup.get('IpPermissions'):
                    td.append([Color('{autoblue}prefix list ids{/autoblue}'), dash_if_none(ip_perm_egress.get('PrefixListIds'))])
                    td.append([Color('{autoblue}from port{/autoblue}'), str(ip_perm_egress.get('FromPort'))])
                    td.append([Color('{autoblue}to port{/autoblue}'), str(ip_perm_egress.get('ToPort'))])
                    td.append([Color('{autoblue}ip ranges{/autoblue}'), dash_if_none(get_ip_ranges(ip_perm_egress.get('IpRanges')))])
                    td.append([Color('{autoblue}ip protocol{/autoblue}'), str(ip_perm_egress.get('IpProtocol'))])
                    if ip_perm_egress.get('UserIdGroupPairs'):
                        td.append(['user id group pairs', '{0}'.format("-" * 30)])
                        td.append(['', get_uid_group_pairs(uid_gps=ip_perm_egress.get('UserIdGroupPairs'))])
        output_ascii_table(table_title=Color('{autowhite}security group info{/autowhite}'),
                           table_data=td)
    else:
        exit('Security group does not exist.')
    exit(0)
<commit_msg>fix egress output and tidy up.<commit_after># -*- coding: utf-8 -*-
from __future__ import (absolute_import, print_function, unicode_literals)
from acli.output import (output_ascii_table, dash_if_none)
from colorclass import Color, Windows
Windows.enable(auto_colors=True, reset_atexit=True)

def get_tag(name=None, tags=None):
    if tags:
        for tag in tags:
            if tag.get('Key') == name:
                return tag.get('Value')

def split_string(the_string=None, chunk_size=None):
    length = 0
    output = str()
    chunks = the_string.split()
    for chunk in chunks:
        length += len(chunk)
        output += chunk
        if length >= chunk_size:
            output += "\n"
            length = 0
        else:
            output += " "
    return output.rstrip()

def output_secgroup_list(output_media=None, secgroups=None):
    """
    @type output_media: unicode
    @type secgroups: dict
    """
    if output_media == 'console':
        td = list()
        td.append([Color('{autoblue}group id{/autoblue}'),
                   Color('{autoblue}group name{/autoblue}'),
                   Color('{autoblue}description{/autoblue}')])
        for secgroup in secgroups.get('SecurityGroups'):
            td.append([secgroup.get('GroupId'),
                      split_string(secgroup.get('GroupName'), chunk_size=30),
                      split_string(secgroup.get('Description'), chunk_size=40)])
        output_ascii_table(table_title=Color('{autowhite}security groups{/autowhite}'),
                           table_data=td,
                           inner_heading_row_border=True)
    exit(0)

def get_ip_ranges(ip_ranges):
    out = str()
    for ip_range in ip_ranges:
        out += "{0}\n".format(ip_range.get('CidrIp'))
    return out.rstrip()

def get_uid_group_pairs(uid_gps=None):
    out = str()
    for uid_gp in uid_gps:
        out += '{0} - {1}\n'.format(uid_gp.get('UserId'), uid_gp.get('GroupId'))
    return out.strip()

def output_secgroup_info(output_media=None, secgroup=None):
    """
    @type output_media: unicode
    @type secgroup: ec2.SecurityGroup
    """
    if secgroup:
        if output_media == 'console':
            td = list()
            td.append([Color('{autoblue}group id{/autoblue}'), secgroup.get('GroupId')])
            td.append([Color('{autoblue}group name{/autoblue}'), secgroup.get('GroupName')])
            td.append([Color('{autoblue}description{/autoblue}'), secgroup.get('Description')])
            td.append([Color('{autoblue}vpc id{/autoblue}'), dash_if_none(secgroup.get('VpcId'))])
            if secgroup.get('IpPermissions'):
                td.append(['ip permissions', '{0}'.format("-" * 30)])
                for ip_perm in secgroup.get('IpPermissions'):
                    td.append([Color('{autoblue}from port{/autoblue}'),
                               dash_if_none(ip_perm.get('FromPort'))])
                    td.append([Color('{autoblue}to port{/autoblue}'),
                               dash_if_none(ip_perm.get('ToPort'))])
                    td.append([Color('{autoblue}ip ranges{/autoblue}'),
                               dash_if_none(get_ip_ranges(ip_perm.get('IpRanges')))])
                    td.append([Color('{autoblue}ip protocol{/autoblue}'),
                               dash_if_none(ip_perm.get('IpProtocol'))])
                    if ip_perm.get('UserIdGroupPairs'):
                        td.append(['user id group pairs', '{0}'.format("-" * 30)])
                        td.append(['', get_uid_group_pairs(uid_gps=ip_perm.get('UserIdGroupPairs'))])

            if secgroup.get('IpPermissionsEgress'):
                td.append(['ip permissions egress', '{0}'.format("-" * 30)])
                for ip_perm_egress in secgroup.get('IpPermissionsEgress'):
                    td.append([Color('{autoblue}prefix list ids{/autoblue}'),
                               dash_if_none(ip_perm_egress.get('PrefixListIds'))])
                    td.append([Color('{autoblue}from port{/autoblue}'),
                               dash_if_none(ip_perm_egress.get('FromPort'))])
                    td.append([Color('{autoblue}to port{/autoblue}'),
                               dash_if_none(ip_perm_egress.get('ToPort'))])
                    td.append([Color('{autoblue}ip ranges{/autoblue}'),
                               dash_if_none(get_ip_ranges(ip_perm_egress.get('IpRanges')))])
                    td.append([Color('{autoblue}ip protocol{/autoblue}'),
                               dash_if_none(ip_perm_egress.get('IpProtocol'))])
                    if ip_perm_egress.get('UserIdGroupPairs'):
                        td.append(['user id group pairs', '{0}'.format("-" * 30)])
                        td.append(['', get_uid_group_pairs(uid_gps=ip_perm_egress.get('UserIdGroupPairs'))])
        output_ascii_table(table_title=Color('{autowhite}security group info{/autowhite}'),
                           table_data=td)
    else:
        exit('Security group does not exist.')
    exit(0)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import boto
import boto.route53
import boto.route53.record
import boto.ec2.elb
import boto.rds2
import time
from argparse import ArgumentParser, RawTextHelpFormatter
import datetime
import sys
from vpcutil import rds_subnet_group_name_for_stack_name, all_stack_names
import os

description = """

   Creates a new RDS instance in a VPC using restore
   from point in time using the latest available backup.
   The new db will be the same size as the original.
   The name of the db will remain the same, the master db password
   will be changed and is set on the command line.

   New db name defaults to "from-<source db name>-<human date>-<ts>"
   A new DNS entry will be created for the RDS when provided
   on the command line

"""

RDS_SIZES = [
    'db.m1.small',
    'db.m1.large',
    'db.m1.xlarge',
    'db.m2.xlarge',
    'db.m2.2xlarge',
    'db.m2.4xlarg',
]

# These are the groups for the different
# stack names that will be assigned once
# the corresponding db is cloned

SG_GROUPS = {
    'stage-edx': 'sg-d2f623b7',
}

# This group must already be created
# and allows for full access to port
# 3306. this group is assigned temporarily
# for cleaning the db

SG_GROUPS_FULL = {
    'stage-edx': 'sg-0abf396f',
}

def parse_args(args=sys.argv[1:]):

    stack_names = all_stack_names()
    rds = boto.rds2.connect_to_region('us-east-1')
    dbs = [db['DBInstanceIdentifier']
           for db in rds.describe_db_instances()['DescribeDBInstancesResponse']['DescribeDBInstancesResult']['DBInstances']]

    parser = ArgumentParser(description=description, formatter_class=RawTextHelpFormatter)
    parser.add_argument('-s', '--stack-name', choices=stack_names,
                        default='stage-edx',
                        help='Stack name for where you want this RDS instance launched')
    parser.add_argument('-t', '--type', choices=RDS_SIZES,
                        default='db.m1.small', help='RDS size to create instances of')
    parser.add_argument('-d', '--db-source', choices=dbs,
                        default=u'stage-edx', help="source db to clone")
    parser.add_argument('-p', '--password',
                        help="password for the new database", metavar="NEW PASSWORD")
    parser.add_argument('-r', '--region', default='us-east-1',
                        help="region to connect to")
    parser.add_argument('--dns',
                        help="dns entry for the new rds instance")
    parser.add_argument('--security-group', action="store_true",
                        default=False,
                        help="add sg group from SG_GROUPS")
    parser.add_argument('--clean-wwc', action="store_true",
                        default=False,
                        help="clean the wwc db after launching it into the vpc, removing sensitive data")
    parser.add_argument('--clean-prod-grader', action="store_true",
                        default=False,
                        help="clean the prod_grader db after launching it into the vpc, removing sensitive data")
    parser.add_argument('--dump', action="store_true",
                        default=False,
                        help="create a sql dump after launching it into the vpc")
    parser.add_argument('--secret-var-file',
                        help="using a secret var file run ansible against the host to update db users")

    return parser.parse_args(args)

def wait_on_db_status(db_name, region='us-east-1', wait_on='available', aws_id=None, aws_secret=None):
    rds = boto.rds2.connect_to_region(region)
    while True:
        statuses = rds.describe_db_instances(db_name)['DescribeDBInstancesResponse']['DescribeDBInstancesResult']['DBInstances']
        if len(statuses) > 1:
            raise Exception("More than one instance returned for {0}".format(db_name))
        if statuses[0]['DBInstanceStatus'] == wait_on:
            break
        sys.stdout.write(".")
        sys.stdout.flush()
        time.sleep(2)
    return

if __name__ == '__main__':
    args = parse_args()
    sanitize_wwc_sql_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sanitize-db-wwc.sql")
    sanitize_prod_grader_sql_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sanitize-db-prod_grader.sql")
    play_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../playbooks/edx-east")

    rds = boto.rds2.connect_to_region(args.region)
    subnet_name = rds_subnet_group_name_for_stack_name(args.stack_name)
    restore_dbid = 'from-{0}-{1}-{2}'.format(args.db_source, datetime.date.today(), int(time.time()))
    rds.restore_db_instance_to_point_in_time(
        source_db_instance_identifier=args.db_source,
        target_db_instance_identifier=restore_dbid,
        use_latest_restorable_time=True,
        db_instance_class=args.type,
        db_subnet_group_name=subnet_name)
    wait_on_db_status(restore_dbid)

    db_host = rds.describe_db_instances(restore_dbid)['DescribeDBInstancesResponse']['DescribeDBInstancesResult']['DBInstances'][0]['Endpoint']['Address']

    if args.password or args.security_group:
        modify_args = dict(
            apply_immediately=True
        )
        if args.password:
            modify_args['master_user_password'] = args.password
        if args.security_group:
            modify_args['vpc_security_group_ids'] = [SG_GROUPS[args.stack_name], SG_GROUPS_FULL[args.stack_name]]

        # Update the db immediately
        rds.modify_db_instance(restore_dbid, **modify_args)

    if args.clean_wwc:
        # Run the mysql clean sql file
        sanitize_cmd = """mysql -u root -p{root_pass} -h{db_host} < {sanitize_wwc_sql_file} """.format(
            root_pass=args.password,
            db_host=db_host,
            sanitize_wwc_sql_file=sanitize_wwc_sql_file)
        print("Running {}".format(sanitize_cmd))
        os.system(sanitize_cmd)

    if args.clean_prod_grader:
        # Run the mysql clean sql file
        sanitize_cmd = """mysql -u root -p{root_pass} -h{db_host} < {sanitize_prod_grader_sql_file} """.format(
            root_pass=args.password,
            db_host=db_host,
            sanitize_prod_grader_sql_file=sanitize_prod_grader_sql_file)
        print("Running {}".format(sanitize_cmd))
        os.system(sanitize_cmd)

    if args.secret_var_file:
        db_cmd = """cd {play_path} && ansible-playbook -c local -i 127.0.0.1, update_edxapp_db_users.yml """ \
            """-e @{secret_var_file} -e "edxapp_db_root_user=root edxapp_db_root_pass={root_pass} """ \
            """EDXAPP_MYSQL_HOST={db_host}" """.format(
            root_pass=args.password,
            secret_var_file=args.secret_var_file,
            db_host=db_host,
            play_path=play_path)
        print("Running {}".format(db_cmd))
        os.system(db_cmd)

    if args.dns:
        dns_cmd = """cd {play_path} && ansible-playbook -c local -i 127.0.0.1, create_cname.yml """ \
            """-e "dns_zone=edx.org dns_name={dns} sandbox={db_host}" """.format(
            play_path=play_path,
            dns=args.dns,
            db_host=db_host)
        print("Running {}".format(dns_cmd))
        os.system(dns_cmd)

    if args.security_group:
        # remove full mysql access from within the vpc
        rds.modify_db_instance(restore_dbid, vpc_security_group_ids=[SG_GROUPS[args.stack_name]])
<commit_msg>setting db name<commit_after>#!/usr/bin/env python
import boto
import boto.route53
import boto.route53.record
import boto.ec2.elb
import boto.rds2
import time
from argparse import ArgumentParser, RawTextHelpFormatter
import datetime
import sys
from vpcutil import rds_subnet_group_name_for_stack_name, all_stack_names
import os

description = """

   Creates a new RDS instance in a VPC using restore
   from point in time using the latest available backup.
   The new db will be the same size as the original.
   The name of the db will remain the same, the master db password
   will be changed and is set on the command line.

   New db name defaults to "from-<source db name>-<human date>-<ts>"
   A new DNS entry will be created for the RDS when provided
   on the command line

"""

RDS_SIZES = [
    'db.m1.small',
    'db.m1.large',
    'db.m1.xlarge',
    'db.m2.xlarge',
    'db.m2.2xlarge',
    'db.m2.4xlarg',
]

# These are the groups for the different
# stack names that will be assigned once
# the corresponding db is cloned

SG_GROUPS = {
    'stage-edx': 'sg-d2f623b7',
}

# This group must already be created
# and allows for full access to port
# 3306. this group is assigned temporarily
# for cleaning the db

SG_GROUPS_FULL = {
    'stage-edx': 'sg-0abf396f',
}

def parse_args(args=sys.argv[1:]):

    stack_names = all_stack_names()
    rds = boto.rds2.connect_to_region('us-east-1')
    dbs = [db['DBInstanceIdentifier']
           for db in rds.describe_db_instances()['DescribeDBInstancesResponse']['DescribeDBInstancesResult']['DBInstances']]

    parser = ArgumentParser(description=description, formatter_class=RawTextHelpFormatter)
    parser.add_argument('-s', '--stack-name', choices=stack_names,
                        default='stage-edx',
                        help='Stack name for where you want this RDS instance launched')
    parser.add_argument('-t', '--type', choices=RDS_SIZES,
                        default='db.m1.small', help='RDS size to create instances of')
    parser.add_argument('-d', '--db-source', choices=dbs,
                        default=u'stage-edx', help="source db to clone")
    parser.add_argument('-p', '--password',
                        help="password for the new database", metavar="NEW PASSWORD")
    parser.add_argument('-r', '--region', default='us-east-1',
                        help="region to connect to")
    parser.add_argument('--dns',
                        help="dns entry for the new rds instance")
    parser.add_argument('--security-group', action="store_true",
                        default=False,
                        help="add sg group from SG_GROUPS")
    parser.add_argument('--clean-wwc', action="store_true",
                        default=False,
                        help="clean the wwc db after launching it into the vpc, removing sensitive data")
    parser.add_argument('--clean-prod-grader', action="store_true",
                        default=False,
                        help="clean the prod_grader db after launching it into the vpc, removing sensitive data")
    parser.add_argument('--dump', action="store_true",
                        default=False,
                        help="create a sql dump after launching it into the vpc")
    parser.add_argument('--secret-var-file',
                        help="using a secret var file run ansible against the host to update db users")

    return parser.parse_args(args)

def wait_on_db_status(db_name, region='us-east-1', wait_on='available', aws_id=None, aws_secret=None):
    rds = boto.rds2.connect_to_region(region)
    while True:
        statuses = rds.describe_db_instances(db_name)['DescribeDBInstancesResponse']['DescribeDBInstancesResult']['DBInstances']
        if len(statuses) > 1:
            raise Exception("More than one instance returned for {0}".format(db_name))
        if statuses[0]['DBInstanceStatus'] == wait_on:
            break
        sys.stdout.write(".")
        sys.stdout.flush()
        time.sleep(2)
    return

if __name__ == '__main__':
    args = parse_args()
    sanitize_wwc_sql_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sanitize-db-wwc.sql")
    sanitize_prod_grader_sql_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "sanitize-db-prod_grader.sql")
    play_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../playbooks/edx-east")

    rds = boto.rds2.connect_to_region(args.region)
    subnet_name = rds_subnet_group_name_for_stack_name(args.stack_name)
    restore_dbid = 'from-{0}-{1}-{2}'.format(args.db_source, datetime.date.today(), int(time.time()))
    rds.restore_db_instance_to_point_in_time(
        source_db_instance_identifier=args.db_source,
        target_db_instance_identifier=restore_dbid,
        use_latest_restorable_time=True,
        db_instance_class=args.type,
        db_subnet_group_name=subnet_name)
    wait_on_db_status(restore_dbid)

    db_host = rds.describe_db_instances(restore_dbid)['DescribeDBInstancesResponse']['DescribeDBInstancesResult']['DBInstances'][0]['Endpoint']['Address']

    if args.password or args.security_group:
        modify_args = dict(
            apply_immediately=True
        )
        if args.password:
            modify_args['master_user_password'] = args.password
        if args.security_group:
            modify_args['vpc_security_group_ids'] = [SG_GROUPS[args.stack_name], SG_GROUPS_FULL[args.stack_name]]

        # Update the db immediately
        rds.modify_db_instance(restore_dbid, **modify_args)

    if args.clean_wwc:
        # Run the mysql clean sql file
        sanitize_cmd = """mysql -u root -p{root_pass} -h{db_host} wwc < {sanitize_wwc_sql_file} """.format(
            root_pass=args.password,
            db_host=db_host,
            sanitize_wwc_sql_file=sanitize_wwc_sql_file)
        print("Running {}".format(sanitize_cmd))
        os.system(sanitize_cmd)

    if args.clean_prod_grader:
        # Run the mysql clean sql file
        sanitize_cmd = """mysql -u root -p{root_pass} -h{db_host} prod_grader < {sanitize_prod_grader_sql_file} """.format(
            root_pass=args.password,
            db_host=db_host,
            sanitize_prod_grader_sql_file=sanitize_prod_grader_sql_file)
        print("Running {}".format(sanitize_cmd))
        os.system(sanitize_cmd)

    if args.secret_var_file:
        db_cmd = """cd {play_path} && ansible-playbook -c local -i 127.0.0.1, update_edxapp_db_users.yml """ \
            """-e @{secret_var_file} -e "edxapp_db_root_user=root edxapp_db_root_pass={root_pass} """ \
            """EDXAPP_MYSQL_HOST={db_host}" """.format(
            root_pass=args.password,
            secret_var_file=args.secret_var_file,
            db_host=db_host,
            play_path=play_path)
        print("Running {}".format(db_cmd))
        os.system(db_cmd)

    if args.dns:
        dns_cmd = """cd {play_path} && ansible-playbook -c local -i 127.0.0.1, create_cname.yml """ \
            """-e "dns_zone=edx.org dns_name={dns} sandbox={db_host}" """.format(
            play_path=play_path,
            dns=args.dns,
            db_host=db_host)
        print("Running {}".format(dns_cmd))
        os.system(dns_cmd)

    if args.security_group:
        # remove full mysql access from within the vpc
        rds.modify_db_instance(restore_dbid, vpc_security_group_ids=[SG_GROUPS[args.stack_name]])
<|endoftext|>


<commit_before>from pathlib import Path
from setuptools_scm import get_version
import logging

def main():
    # get version
    version = get_version(root='..', relative_to=__file__)
    version = str(version).replace('+g', '')
    logging.info('Version: {}'.format(version))

    # get root path
    root_path = Path(__file__).parents[1]
    logging.info('Root path: {}'.format(root_path.resolve()))

    # dump version
    version_path = root_path / 'VERSION'
    logging.info('Dumping version to: {}'.format(version_path.resolve()))

    with open(str(version_path), 'w') as f:
        f.write(version)
    logging.info('Version dumped')

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    main()
<commit_msg>ensuring pep440 version compliance<commit_after>from pathlib import Path
from setuptools_scm import get_version
import logging

def main():
    # get version
    version = get_version(root='..', relative_to=__file__)
    logging.info('Version: {}'.format(version))

    # ensure PEP440 compliance
    version_segments = version.split('.dev')
    if len(version_segments) > 1:
        logging.info('Cleaning version for PEP440 compliance')
        tag_version_segment = version_segments[0]
        dev_version_segment = version.split('.dev')[1]
        dev_version_segment = abs(hash(dev_version_segment))

        version = '{}.dev{}'.format(tag_version_segment, dev_version_segment)
        logging.info('PEP440 version: {}'.format(version))

    # get root path
    root_path = Path(__file__).parents[1]
    logging.info('Root path: {}'.format(root_path.resolve()))

    # dump version
    version_path = root_path / 'VERSION'
    logging.info('Dumping version to: {}'.format(version_path.resolve()))

    with open(str(version_path), 'w') as f:
        f.write(version)
    logging.info('Version dumped')

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    main()
<|endoftext|>


<commit_before># utils/SwiftBuildSupport.py - Utilities for Swift build scripts -*- python -*-
#
# This source file is part of the Swift.org open source project
#
# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
# Licensed under Apache License v2.0 with Runtime Library Exception
#
# See http://swift.org/LICENSE.txt for license information
# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors

from __future__ import print_function

try:
    # Python 2
    import ConfigParser
except ImportError:
    # Python 3
    import configparser as ConfigParser

import os
import pipes
import subprocess
import sys

HOME = os.environ.get("HOME", "/")

def _get_default_source_root():
    result = ""

    # Are we in a Swift checkout? Start from this file and check its parent
    # directories.
    #
    # $SWIFT_SOURCE_ROOT/swift/utils/SwiftBuildSupport.py
    (swift_path, parent_dirname) = os.path.split(os.path.dirname(__file__))
    if parent_dirname != "utils":
        return result
    if not os.path.exists(os.path.join(swift_path, 'CMakeLists.txt')):
        return result
    result = os.path.dirname(swift_path)

    # Are we in an LLVM checkout? Start from the Swift checkout and check /its/
    # parent directories.
    #
    # $SWIFT_SOURCE_ROOT/llvm/tools/swift/utils/SwiftBuildSupport.py
    (llvm_path, parent_dirname) = os.path.split(result)
    if parent_dirname != "tools":
        return result
    if not os.path.exists(os.path.join(llvm_path, 'CMakeLists.txt')):
        return result
    result = os.path.dirname(llvm_path)

    return result

# Set SWIFT_SOURCE_ROOT in your environment to control where the sources
# are found.
SWIFT_SOURCE_ROOT = os.environ.get(
    "SWIFT_SOURCE_ROOT", _get_default_source_root())

# Set SWIFT_BUILD_ROOT to a directory that will contain a subdirectory
# for each build configuration
SWIFT_BUILD_ROOT = os.environ.get(
    "SWIFT_BUILD_ROOT", os.path.join(SWIFT_SOURCE_ROOT, "build"))

def print_with_argv0(message):
    print(sys.argv[0] + ": " + message)

def quote_shell_command(args):
    return " ".join([pipes.quote(a) for a in args])

def check_call(args, print_command=False, verbose=False):
    if print_command:
        print(os.getcwd() + "$ " + quote_shell_command(args))
    try:
        return subprocess.check_call(args)
    except subprocess.CalledProcessError as e:
        if verbose:
            print_with_argv0(e.strerror)
        else:
            print_with_argv0(
                "command terminated with a non-zero exit status " +
                str(e.returncode) + ", aborting")
        sys.stdout.flush()
        sys.exit(1)
    except OSError as e:
        print_with_argv0("could not execute '" + quote_shell_command(args) +
            "': " + e.strerror)
        sys.stdout.flush()
        sys.exit(1)

def check_output(args, print_command=False, verbose=False):
    if print_command:
        print(os.getcwd() + "$ " + quote_shell_command(args))
    try:
        return subprocess.check_output(args)
    except subprocess.CalledProcessError as e:
        if verbose:
            print_with_argv0(e.strerror)
        else:
            print_with_argv0(
                "command terminated with a non-zero exit status " +
                str(e.returncode) + ", aborting")
        sys.stdout.flush()
        sys.exit(1)
    except OSError as e:
        print_with_argv0("could not execute '" + quote_shell_command(args) +
            "': " + e.strerror)
        sys.stdout.flush()
        sys.exit(1)

def _load_preset_files_impl(preset_file_names, substitutions={}):
    config = ConfigParser.SafeConfigParser(substitutions, allow_no_value=True)
    if config.read(preset_file_names) == []:
        print_with_argv0(
            "preset file not found (tried " + str(preset_file_names) + ")")
        sys.exit(1)
    return config

_PRESET_PREFIX = "preset: "

def _get_preset_options_impl(config, substitutions, preset_name):
    section_name = _PRESET_PREFIX + preset_name
    if section_name not in config.sections():
        return (None, None, None)

    build_script_opts = []
    build_script_impl_opts = []
    missing_opts = []
    dash_dash_seen = False

    for o in config.options(section_name):
        try:
            a = config.get(section_name, o)
        except ConfigParser.InterpolationMissingOptionError as e:
            # e.reference contains the correctly formatted option
            missing_opts.append(e.reference)
            continue

        if not a:
            a = ""

        if o in substitutions:
            continue

        opt = None
        if o == "mixin-preset":
            # Split on newlines and filter out empty lines.
            mixins = filter(None, [m.strip() for m in a.splitlines()])
            for mixin in mixins:
                (base_build_script_opts,
                    base_build_script_impl_opts,
                    base_missing_opts) = \
                    _get_preset_options_impl(config, substitutions, mixin)
                build_script_opts += base_build_script_opts
                build_script_impl_opts += base_build_script_impl_opts
                missing_opts += base_missing_opts
        elif o == "dash-dash":
            dash_dash_seen = True
        elif a == "":
            opt = "--" + o
        else:
            opt = "--" + o + "=" + a

        if opt:
            if not dash_dash_seen:
                build_script_opts.append(opt)
            else:
                build_script_impl_opts.append(opt)

    return (build_script_opts, build_script_impl_opts, missing_opts)

def get_preset_options(substitutions, preset_file_names, preset_name):
    config = _load_preset_files_impl(preset_file_names, substitutions)

    (build_script_opts, build_script_impl_opts, missing_opts) = \
        _get_preset_options_impl(config, substitutions, preset_name)
    if not build_script_opts:
        print_with_argv0("preset '" + preset_name + "' not found")
        sys.exit(1)
    if missing_opts:
        print_with_argv0("missing option(s) for preset '" + preset_name +
                         "': " + ", ".join(missing_opts))
        sys.exit(1)

    return build_script_opts + ["--"] + build_script_impl_opts

def get_all_preset_names(preset_file_names):
    config = _load_preset_files_impl(preset_file_names)
    return [name[len(_PRESET_PREFIX):] for name in config.sections()
            if name.startswith(_PRESET_PREFIX)]

# A context manager for changing the current working directory.
#
#     with WorkingDirectory('/tmp'):
#         ... do work in /tmp...
class WorkingDirectory(object):
    def __init__(self, new_cwd):
        self.new_cwd = new_cwd

    def __enter__(self):
        self.old_cwd = os.getcwd()
        os.chdir(self.new_cwd)

    def __exit__(self, type, value, traceback):
        os.chdir(self.old_cwd)
<commit_msg>[Python] Fix AttributeError: CalledProcessError.strerror does not exist<commit_after># utils/SwiftBuildSupport.py - Utilities for Swift build scripts -*- python -*-
#
# This source file is part of the Swift.org open source project
#
# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
# Licensed under Apache License v2.0 with Runtime Library Exception
#
# See http://swift.org/LICENSE.txt for license information
# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors

from __future__ import print_function

try:
    # Python 2
    import ConfigParser
except ImportError:
    # Python 3
    import configparser as ConfigParser

import os
import pipes
import subprocess
import sys

HOME = os.environ.get("HOME", "/")

def _get_default_source_root():
    result = ""

    # Are we in a Swift checkout? Start from this file and check its parent
    # directories.
    #
    # $SWIFT_SOURCE_ROOT/swift/utils/SwiftBuildSupport.py
    (swift_path, parent_dirname) = os.path.split(os.path.dirname(__file__))
    if parent_dirname != "utils":
        return result
    if not os.path.exists(os.path.join(swift_path, 'CMakeLists.txt')):
        return result
    result = os.path.dirname(swift_path)

    # Are we in an LLVM checkout? Start from the Swift checkout and check /its/
    # parent directories.
    #
    # $SWIFT_SOURCE_ROOT/llvm/tools/swift/utils/SwiftBuildSupport.py
    (llvm_path, parent_dirname) = os.path.split(result)
    if parent_dirname != "tools":
        return result
    if not os.path.exists(os.path.join(llvm_path, 'CMakeLists.txt')):
        return result
    result = os.path.dirname(llvm_path)

    return result

# Set SWIFT_SOURCE_ROOT in your environment to control where the sources
# are found.
SWIFT_SOURCE_ROOT = os.environ.get(
    "SWIFT_SOURCE_ROOT", _get_default_source_root())

# Set SWIFT_BUILD_ROOT to a directory that will contain a subdirectory
# for each build configuration
SWIFT_BUILD_ROOT = os.environ.get(
    "SWIFT_BUILD_ROOT", os.path.join(SWIFT_SOURCE_ROOT, "build"))

def print_with_argv0(message):
    print(sys.argv[0] + ": " + message)

def quote_shell_command(args):
    return " ".join([pipes.quote(a) for a in args])

def check_call(args, print_command=False, verbose=False):
    if print_command:
        print(os.getcwd() + "$ " + quote_shell_command(args))
    try:
        return subprocess.check_call(args)
    except subprocess.CalledProcessError as e:
        print_with_argv0(
            "command terminated with a non-zero exit status " +
            str(e.returncode) + ", aborting")
        sys.stdout.flush()
        sys.exit(1)
    except OSError as e:
        print_with_argv0("could not execute '" + quote_shell_command(args) +
            "': " + e.strerror)
        sys.stdout.flush()
        sys.exit(1)

def check_output(args, print_command=False, verbose=False):
    if print_command:
        print(os.getcwd() + "$ " + quote_shell_command(args))
    try:
        return subprocess.check_output(args)
    except subprocess.CalledProcessError as e:
        print_with_argv0(
            "command terminated with a non-zero exit status " +
            str(e.returncode) + ", aborting")
        sys.stdout.flush()
        sys.exit(1)
    except OSError as e:
        print_with_argv0("could not execute '" + quote_shell_command(args) +
            "': " + e.strerror)
        sys.stdout.flush()
        sys.exit(1)

def _load_preset_files_impl(preset_file_names, substitutions={}):
    config = ConfigParser.SafeConfigParser(substitutions, allow_no_value=True)
    if config.read(preset_file_names) == []:
        print_with_argv0(
            "preset file not found (tried " + str(preset_file_names) + ")")
        sys.exit(1)
    return config

_PRESET_PREFIX = "preset: "

def _get_preset_options_impl(config, substitutions, preset_name):
    section_name = _PRESET_PREFIX + preset_name
    if section_name not in config.sections():
        return (None, None, None)

    build_script_opts = []
    build_script_impl_opts = []
    missing_opts = []
    dash_dash_seen = False

    for o in config.options(section_name):
        try:
            a = config.get(section_name, o)
        except ConfigParser.InterpolationMissingOptionError as e:
            # e.reference contains the correctly formatted option
            missing_opts.append(e.reference)
            continue

        if not a:
            a = ""

        if o in substitutions:
            continue

        opt = None
        if o == "mixin-preset":
            # Split on newlines and filter out empty lines.
            mixins = filter(None, [m.strip() for m in a.splitlines()])
            for mixin in mixins:
                (base_build_script_opts,
                    base_build_script_impl_opts,
                    base_missing_opts) = \
                    _get_preset_options_impl(config, substitutions, mixin)
                build_script_opts += base_build_script_opts
                build_script_impl_opts += base_build_script_impl_opts
                missing_opts += base_missing_opts
        elif o == "dash-dash":
            dash_dash_seen = True
        elif a == "":
            opt = "--" + o
        else:
            opt = "--" + o + "=" + a

        if opt:
            if not dash_dash_seen:
                build_script_opts.append(opt)
            else:
                build_script_impl_opts.append(opt)

    return (build_script_opts, build_script_impl_opts, missing_opts)

def get_preset_options(substitutions, preset_file_names, preset_name):
    config = _load_preset_files_impl(preset_file_names, substitutions)

    (build_script_opts, build_script_impl_opts, missing_opts) = \
        _get_preset_options_impl(config, substitutions, preset_name)
    if not build_script_opts:
        print_with_argv0("preset '" + preset_name + "' not found")
        sys.exit(1)
    if missing_opts:
        print_with_argv0("missing option(s) for preset '" + preset_name +
                         "': " + ", ".join(missing_opts))
        sys.exit(1)

    return build_script_opts + ["--"] + build_script_impl_opts

def get_all_preset_names(preset_file_names):
    config = _load_preset_files_impl(preset_file_names)
    return [name[len(_PRESET_PREFIX):] for name in config.sections()
            if name.startswith(_PRESET_PREFIX)]

# A context manager for changing the current working directory.
#
#     with WorkingDirectory('/tmp'):
#         ... do work in /tmp...
class WorkingDirectory(object):
    def __init__(self, new_cwd):
        self.new_cwd = new_cwd

    def __enter__(self):
        self.old_cwd = os.getcwd()
        os.chdir(self.new_cwd)

    def __exit__(self, type, value, traceback):
        os.chdir(self.old_cwd)
<|endoftext|>


<commit_before>import sys
import os
sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'models'))

#print(sys.path)

from mysql_python import MysqlPython
from models.User import User
from models.Customer import Customer
import MySQLdb
from DBUtils.PooledDB import PooledDB
import hashlib
import time

from dbutils import *

def insert_customer(uid, customer):
    if (True == if_customer_exists(customer)):
        return update_customer_info(uid, customer)
    else:
        if (is_customer_deleted(customer)):
            if (is_customer_need_restore(customer)):
                restore_customer(customer)
                return update_customer_info(uid, customer)
            else:
                return True

        else:
            try:
                conn = g_dbPool.connection()
                cur=conn.cursor()
                count = cur.execute("insert into customer(id, uid, name, group_name, spell, address, longitude, latitude, boss, phone, email, description, update_date) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) " \
                                    , (customer.id, uid, customer.name, customer.group_name, customer.spell, customer.address, customer.longitude, customer.latitude, customer.boss, customer.phone, customer.email, customer.description, customer.update_date))
                conn.commit()
                if (1 == count):
                    return True
                else:
                    return False

            except MySQLdb.Error,e:
                return False
            finally:
                cur.close()

def if_customer_exists(customer):
    conn = g_dbPool.connection()
    cur=conn.cursor(MySQLdb.cursors.DictCursor)
    try:
        cur.execute("select * from customer where id=%s and is_deleted=0" , (customer.id, ))

        rows=cur.fetchall()
        if (len(rows) < 1):
            return False
        else:
            return True

    except MySQLdb.Error,e:
        return False
    finally:
        cur.close()

def is_customer_deleted(customer):
    conn = g_dbPool.connection()
    cur=conn.cursor(MySQLdb.cursors.DictCursor)
    try:
        cur.execute("select * from customer where id=%s and is_deleted=1" , (customer.id, ))

        rows=cur.fetchall()
        if (len(rows) < 1):
            return False
        else:
            return True

    except MySQLdb.Error,e:
        return False
    finally:
        cur.close()

def is_customer_need_restore(customer):
    conn = g_dbPool.connection()
    cur=conn.cursor(MySQLdb.cursors.DictCursor)
    try:
        if (is_customer_deleted(customer)):
            #check the update_date and confirm if need restore group
            cur.execute("select * from customer where id=%s and is_deleted=1 and update_date < %s" , (customer.id, customer.update_date) )
            rows=cur.fetchall()
            if (len(rows) < 1):
                return False
            else:
                return True
        else:
            return False

    except MySQLdb.Error,e:
        return False
    finally:
        cur.close()

def restore_customer(customer):
    conn = g_dbPool.connection()
    cur=conn.cursor()
    try:
        count = cur.execute("update customer set update_date=%s, is_deleted=0 where id=%s " \
                            , (customer.update_date, customer.id))
        conn.commit()
        if (count >= 0):
            return True
        else:
            return False

    except MySQLdb.Error,e:
        return False
    finally:
        cur.close()

def update_customer_info(uid, customer):
    conn = g_dbPool.connection()
    cur=conn.cursor()
    try:
        count = cur.execute("update customer set uid = %s, name = %s, group_name = %s, spell = %s, address = %s, longitude = %s, latitude = %s, boss = %s, phone = %s, email = %s, description = %s, update_date = %s where id = %s and update_date <= %s" \
                    , (uid, customer.name, customer.group_name, customer.spell, customer.address, customer.longitude, customer.latitude, customer.boss, customer.phone, customer.email, customer.description, customer.update_date, customer.id, customer.update_date))

        conn.commit()
        if (count >= 0):
            return True
        else:
            return False

    except MySQLdb.Error,e:
        return False
    finally:
        cur.close()

def init_customer(row):
    customer = Customer()
    customer.id = row['id']
    customer.uid = row['uid']
    customer.name = row['name']
    customer.group_name = row['group_name']
    customer.spell = row['spell']

    customer.address = row['address']
    customer.longitude = row['longitude']
    customer.latitude = row['latitude']
    customer.boss = row['boss']
    customer.phone = row['phone']

    customer.email = row['email']
    customer.description = row['description']
    customer.update_date = row['update_date']
    customer.is_deleted = row['is_deleted']

    return customer

def select_customer_list(uid, type = 0):
    if (type.isdigit() and 0 == int(type)):
        return select_all_customer_list(uid)
    else:
        return select_exists_customer_list(uid)

def select_all_customer_list(uid):
    conn = g_dbPool.connection()
    cur=conn.cursor(MySQLdb.cursors.DictCursor)
    try:
        cur.execute("select * from customer where uid=%s" , (uid, ))

        rows=cur.fetchall()

        lstCustomer = []
        for row in rows:
            lstCustomer.append(init_customer(row))
            return lstCustomer

    except MySQLdb.Error,e:
        return False
    finally:
        cur.close()

def select_exists_customer_list(uid):
    conn = g_dbPool.connection()
    cur=conn.cursor(MySQLdb.cursors.DictCursor)
    try:
        cur.execute("select * from customer where uid=%s and is_deleted=0" , (uid, ))

        rows=cur.fetchall()

        lstCustomer = []
        for row in rows:
            lstCustomer.append(init_customer(row))

        return lstCustomer

    except MySQLdb.Error,e:
        return False
    finally:
        cur.close()

def select_customer(uid, id):
    conn = g_dbPool.connection()
    cur=conn.cursor(MySQLdb.cursors.DictCursor)
    try:
        cur.execute("select * from customer where id=%s and uid=%s and is_deleted=0" , (id, uid))

        rows=cur.fetchall()
        if (len(rows) < 1):
            return None

        row = rows[0]
        customer = init_customer(row)
        return customer

    except MySQLdb.Error,e:
        return False
    finally:
        cur.close()
<commit_msg>update select customer list<commit_after>import sys
import os
sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'models'))

#print(sys.path)

from mysql_python import MysqlPython
from models.User import User
from models.Customer import Customer
import MySQLdb
from DBUtils.PooledDB import PooledDB
import hashlib
import time

from dbutils import *

def insert_customer(uid, customer):
    if (True == if_customer_exists(customer)):
        return update_customer_info(uid, customer)
    else:
        if (is_customer_deleted(customer)):
            if (is_customer_need_restore(customer)):
                restore_customer(customer)
                return update_customer_info(uid, customer)
            else:
                return True

        else:
            try:
                conn = g_dbPool.connection()
                cur=conn.cursor()
                count = cur.execute("insert into customer(id, uid, name, group_name, spell, address, longitude, latitude, boss, phone, email, description, update_date) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) " \
                                    , (customer.id, uid, customer.name, customer.group_name, customer.spell, customer.address, customer.longitude, customer.latitude, customer.boss, customer.phone, customer.email, customer.description, customer.update_date))
                conn.commit()
                if (1 == count):
                    return True
                else:
                    return False

            except MySQLdb.Error,e:
                return False
            finally:
                cur.close()

def if_customer_exists(customer):
    conn = g_dbPool.connection()
    cur=conn.cursor(MySQLdb.cursors.DictCursor)
    try:
        cur.execute("select * from customer where id=%s and is_deleted=0" , (customer.id, ))

        rows=cur.fetchall()
        if (len(rows) < 1):
            return False
        else:
            return True

    except MySQLdb.Error,e:
        return False
    finally:
        cur.close()

def is_customer_deleted(customer):
    conn = g_dbPool.connection()
    cur=conn.cursor(MySQLdb.cursors.DictCursor)
    try:
        cur.execute("select * from customer where id=%s and is_deleted=1" , (customer.id, ))

        rows=cur.fetchall()
        if (len(rows) < 1):
            return False
        else:
            return True

    except MySQLdb.Error,e:
        return False
    finally:
        cur.close()

def is_customer_need_restore(customer):
    conn = g_dbPool.connection()
    cur=conn.cursor(MySQLdb.cursors.DictCursor)
    try:
        if (is_customer_deleted(customer)):
            #check the update_date and confirm if need restore group
            cur.execute("select * from customer where id=%s and is_deleted=1 and update_date < %s" , (customer.id, customer.update_date) )
            rows=cur.fetchall()
            if (len(rows) < 1):
                return False
            else:
                return True
        else:
            return False

    except MySQLdb.Error,e:
        return False
    finally:
        cur.close()

def restore_customer(customer):
    conn = g_dbPool.connection()
    cur=conn.cursor()
    try:
        count = cur.execute("update customer set update_date=%s, is_deleted=0 where id=%s " \
                            , (customer.update_date, customer.id))
        conn.commit()
        if (count >= 0):
            return True
        else:
            return False

    except MySQLdb.Error,e:
        return False
    finally:
        cur.close()

def update_customer_info(uid, customer):
    conn = g_dbPool.connection()
    cur=conn.cursor()
    try:
        count = cur.execute("update customer set uid = %s, name = %s, group_name = %s, spell = %s, address = %s, longitude = %s, latitude = %s, boss = %s, phone = %s, email = %s, description = %s, update_date = %s where id = %s and update_date <= %s" \
                    , (uid, customer.name, customer.group_name, customer.spell, customer.address, customer.longitude, customer.latitude, customer.boss, customer.phone, customer.email, customer.description, customer.update_date, customer.id, customer.update_date))

        conn.commit()
        if (count >= 0):
            return True
        else:
            return False

    except MySQLdb.Error,e:
        return False
    finally:
        cur.close()

def init_customer(row):
    customer = Customer()
    customer.id = row['id']
    customer.uid = row['uid']
    customer.name = row['name']
    customer.group_name = row['group_name']
    customer.spell = row['spell']

    customer.address = row['address']
    customer.longitude = row['longitude']
    customer.latitude = row['latitude']
    customer.boss = row['boss']
    customer.phone = row['phone']

    customer.email = row['email']
    customer.description = row['description']
    customer.update_date = row['update_date']
    customer.is_deleted = row['is_deleted']

    return customer

def select_customer_list(uid, type = 0):
    if (type.isdigit() and 0 == int(type)):
        return select_all_customer_list(uid)
    else:
        return select_exists_customer_list(uid)

def select_all_customer_list(uid):
    conn = g_dbPool.connection()
    cur=conn.cursor(MySQLdb.cursors.DictCursor)
    try:
        cur.execute("select * from customer where uid=%s" , (uid, ))

        rows=cur.fetchall()

        lstCustomer = []
        for row in rows:
            lstCustomer.append(init_customer(row))

        return lstCustomer

    except MySQLdb.Error,e:
        return False
    finally:
        cur.close()

def select_exists_customer_list(uid):
    conn = g_dbPool.connection()
    cur=conn.cursor(MySQLdb.cursors.DictCursor)
    try:
        cur.execute("select * from customer where uid=%s and is_deleted=0" , (uid, ))

        rows=cur.fetchall()

        lstCustomer = []
        for row in rows:
            lstCustomer.append(init_customer(row))

        return lstCustomer

    except MySQLdb.Error,e:
        return False
    finally:
        cur.close()

def select_customer(uid, id):
    conn = g_dbPool.connection()
    cur=conn.cursor(MySQLdb.cursors.DictCursor)
    try:
        cur.execute("select * from customer where id=%s and uid=%s and is_deleted=0" , (id, uid))

        rows=cur.fetchall()
        if (len(rows) < 1):
            return None

        row = rows[0]
        customer = init_customer(row)
        return customer

    except MySQLdb.Error,e:
        return False
    finally:
        cur.close()
<|endoftext|>