<commit_before>"""Array printing function

$Id: arrayprint.py,v 1.9 2005/09/13 13:58:44 teoliphant Exp $
"""
__all__ = ["set_summary", "summary_off", "set_precision", "set_line_width",
           "array2string"]

#
# Written by Konrad Hinsen <hinsenk@ere.umontreal.ca>
# last revision: 1996-3-13
# modified by Jim Hugunin 1997-3-3 for repr's and str's (and other details)
# and by Perry Greenfield 2000-4-1 for numarray
# and by Travis Oliphant  2005-8-22 for numpy

import sys
import numeric      as _gen
import numerictypes as _nt
import umath        as _uf
from multiarray import format_longfloat
from fromnumeric import ravel
_nc = _gen

# The following functions are emergency substitutes for numeric functions
# which sometimes get broken during development.

def product(x, y): return x*y

def _maximum_reduce(arr):
    maximum = arr[0]
    for i in xrange(1, arr.nelements()):
        if arr[i] > maximum: maximum = arr[i]
    return maximum

def _minimum_reduce(arr):
    minimum = arr[0]
    for i in xrange(1, arr.nelements()):
        if arr[i] < minimum: minimum = arr[i]
    return minimum

def _numeric_compress(arr):
    nonzero = 0
    for i in xrange(arr.nelements()):
        if arr[i] != 0: nonzero += 1
    retarr = _nc.zeros((nonzero,))
    nonzero = 0
    for i in xrange(arr.nelements()):
        if arr[i] != 0:
            retarr[nonzero] = abs(arr[i])
            nonzero += 1
    return retarr

_failsafe = 0
if _failsafe:
    max_reduce = _maximum_reduce
    min_reduce = _minimum_reduce
else:
    max_reduce = _uf.maximum.reduce
    min_reduce = _uf.minimum.reduce

_summaryEdgeItems = 3     # repr N leading and trailing items of each dimension
_summaryThreshhold = 1000 # total items > triggers array summarization

_float_output_precision = 8
_float_output_suppress_small = False
_line_width = 75

def set_printoptions(precision=None, threshold=None, edgeitems=None,
                     linewidth=None, suppress=None):
    """Set options associated with printing.

    precision  the default number of digits of precision for floating
                   point output
                   (default 8)
    threshold  total number of array elements which trigger summarization
                   rather than full repr.
                   (default 1000)
    edgeitems  number of array items in summary at beginning and end of
                   each dimension.
                   (default 3)
    linewidth  the number of characters per line for the purpose of inserting
                   line breaks.
                   (default 75)
    suppress    Boolean value indicating whether or not suppress printing
                   of small floating point values using scientific notation
                   (default False)
    """

    global _summaryThreshhold, _summaryEdgeItems, _float_output_precision, \
           _line_width, _float_output_suppress_small
    if (linewidth is not None):
        _line_width = linewidth
    if (threshold is not None):
        _summaryThreshhold = threshold
    if (edgeitems is not None):
        _summaryEdgeItems = edgeitems
    if (precision is not None):
        _float_output_precision = precision
    if (suppress is not None):
        _float_output_suppress_small = not not suppress
    return

def get_printoptions():
    return _float_output_precision, _summaryThreshhold, _summaryEdgeItems, \
           _line_width, _float_output_suppress_small

def _leading_trailing(a):
    if a.ndim == 1:
        if len(a) > 2*_summaryEdgeItems:
            b = _gen.concatenate((a[:_summaryEdgeItems],
                                     a[-_summaryEdgeItems:]))
        else:
            b = a
    else:
        if len(a) > 2*_summaryEdgeItems:
            l = [_leading_trailing(a[i]) for i in range(
                min(len(a), _summaryEdgeItems))]
            l.extend([_leading_trailing(a[-i]) for i in range(
                min(len(a), _summaryEdgeItems),0,-1)])
        else:
            l = [_leading_trailing(a[i]) for i in range(0, len(a))]
        b = _gen.concatenate(tuple(l))
    return b

def _array2string(a, max_line_width, precision, suppress_small, separator=' ',
                  prefix=""):

    if max_line_width is None:
        max_line_width = _line_width

    if precision is None:
        precision = _float_output_precision

    if suppress_small is None:
        suppress_small = _float_output_suppress_small

    if a.size > _summaryThreshhold:
        summary_insert = "..., "
        data = _leading_trailing(a)
    else:
        summary_insert = ""
        data = ravel(a)

    try:
        format_function = a._format
    except AttributeError:
        dtype = a.dtype.type
        if issubclass(dtype, _nt.bool):
            format = "%s"
            format_function = lambda x: format % x
        if issubclass(dtype, _nt.integer):
            max_str_len = max(len(str(max_reduce(data))),
                              len(str(min_reduce(data))))
            format = '%' + str(max_str_len) + 'd'
            format_function = lambda x: _formatInteger(x, format)
        elif issubclass(dtype, _nt.floating):
            if issubclass(dtype, _nt.longfloat):
                format_function = _longfloatFormatter(precision)
            else:
                format = _floatFormat(data, precision, suppress_small)
                format_function = lambda x: _formatFloat(x, format)
        elif issubclass(dtype, _nt.complexfloating):
            if issubclass(dtype, _nt.clongfloat):
                format_function = _clongfloatFormatter(precision)
            else:
                real_format = _floatFormat(
                    data.real, precision, suppress_small, sign=0)
                imag_format = _floatFormat(
                    data.imag, precision, suppress_small, sign=1)
                format_function = lambda x: \
                              _formatComplex(x, real_format, imag_format)
        elif issubclass(dtype, _nt.unicode_):
            format = "%s"
            format_function = lambda x: repr(x)
        else:
            format = '%s'
            format_function = lambda x: format % str(x)

    next_line_prefix = " " # skip over "["
    next_line_prefix += " "*len(prefix)                  # skip over array(

    lst = _formatArray(a, format_function, len(a.shape), max_line_width,
                       next_line_prefix, separator,
                       _summaryEdgeItems, summary_insert)[:-1]

    return lst

def array2string(a, max_line_width = None, precision = None,
                 suppress_small = None, separator=' ', prefix="",
                 style=repr):

    if a.shape == ():
        x = a.item()
        try:
            lst = a._format(x)
        except AttributeError:
            lst = style(x)
    elif reduce(product, a.shape) == 0:
        # treat as a null array if any of shape elements == 0
        lst = "[]"
    else:
        lst = _array2string(a, max_line_width, precision, suppress_small,
                            separator, prefix)
    return lst

def _extendLine(s, line, word, max_line_len, next_line_prefix):
    if len(line.rstrip()) + len(word.rstrip()) >= max_line_len:
        s += line.rstrip() + "\n"
        line = next_line_prefix
    line += word
    return s, line

def _formatArray(a, format_function, rank, max_line_len,
                 next_line_prefix, separator, edge_items, summary_insert):
    """formatArray is designed for two modes of operation:

    1. Full output

    2. Summarized output

    """
    if rank == 0:
        return str(a.item())

    if summary_insert and 2*edge_items < len(a):
        leading_items, trailing_items, summary_insert1 = \
                       edge_items, edge_items, summary_insert
    else:
        leading_items, trailing_items, summary_insert1 = 0, len(a), ""

    if rank == 1:

        s = ""
        line = next_line_prefix
        for i in xrange(leading_items):
            word = format_function(a[i]) + separator
            s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)

        if summary_insert1:
            s, line = _extendLine(s, line, summary_insert1, max_line_len, next_line_prefix)

        for i in xrange(trailing_items, 1, -1):
            word = format_function(a[-i]) + separator
            s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)

        word = format_function(a[-1])
        s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)
        s += line + "]\n"
        s = '[' + s[len(next_line_prefix):]
    else:
        s = '['
        sep = separator.rstrip()
        for i in xrange(leading_items):
            if i > 0:
                s += next_line_prefix
            s += _formatArray(a[i], format_function, rank-1, max_line_len,
                              " " + next_line_prefix, separator, edge_items,
                              summary_insert)
            s = s.rstrip() + sep.rstrip() + '\n'*max(rank-1,1)

        if summary_insert1:
            s += next_line_prefix + summary_insert1 + "\n"

        for i in xrange(trailing_items, 1, -1):
            if leading_items or i != trailing_items:
                s += next_line_prefix
            s += _formatArray(a[-i], format_function, rank-1, max_line_len,
                              " " + next_line_prefix, separator, edge_items,
                              summary_insert)
            s = s.rstrip() + sep.rstrip() + '\n'*max(rank-1,1)
        if leading_items or trailing_items > 1:
            s += next_line_prefix
        s += _formatArray(a[-1], format_function, rank-1, max_line_len,
                          " " + next_line_prefix, separator, edge_items,
                          summary_insert).rstrip()+']\n'
    return s

def _floatFormat(data, precision, suppress_small, sign = 0):
    exp_format = 0
    non_zero = _uf.absolute(data.compress(_uf.not_equal(data, 0)))
    ##non_zero = _numeric_compress(data) ##
    if len(non_zero) == 0:
        max_val = 0.
        min_val = 0.
    else:
        max_val = max_reduce(non_zero)
        min_val = min_reduce(non_zero)
        if max_val >= 1.e8:
            exp_format = 1
        if not suppress_small and (min_val < 0.0001
                                   or max_val/min_val > 1000.):
            exp_format = 1
    if exp_format:
        large_exponent = 0 < min_val < 1e-99 or max_val >= 1e100
        max_str_len = 8 + precision + large_exponent
        if sign: format = '%+'
        else: format = '%'
        format = format + str(max_str_len) + '.' + str(precision) + 'e'
        if large_exponent: format = format + '3'
    else:
        format = '%.' + str(precision) + 'f'
        precision = min(precision, max([_digits(x, precision, format)
                                        for x in data]))
        max_str_len = len(str(int(max_val))) + precision + 2
        if sign: format = '%#+'
        else: format = '%#'
        format = format + str(max_str_len) + '.' + str(precision) + 'f'
    return format

def _digits(x, precision, format):
    s = format % x
    zeros = len(s)
    while s[zeros-1] == '0': zeros = zeros-1
    return precision-len(s)+zeros

_MAXINT = sys.maxint
_MININT = -sys.maxint-1
def _formatInteger(x, format):
    if (x < _MAXINT) and (x > _MININT):
        return format % x
    else:
        return "%s" % x

def _longfloatFormatter(precision):
    def formatter(x):
        return format_longfloat(x, precision)
    return formatter

def _formatFloat(x, format, strip_zeros = 1):
    if format[-1] == '3':
        # 3-digit exponent
        format = format[:-1]
        s = format % x
        third = s[-3]
        if third == '+' or third == '-':
            s = s[1:-2] + '0' + s[-2:]
    elif format[-1] == 'e':
        # 2-digit exponent
        s = format % x
        if s[-3] == '0':
            s = ' ' + s[:-3] + s[-2:]
    elif format[-1] == 'f':
        s = format % x
        if strip_zeros:
            zeros = len(s)
            while s[zeros-1] == '0': zeros = zeros-1
            s = s[:zeros] + (len(s)-zeros)*' '
    else:
        s = format % x
    return s

def _clongfloatFormatter(precision):
    def formatter(x):
        r = format_longfloat(x.real, precision)
        i = format_longfloat(x.imag, precision)
        if x.imag < 0:
            i = '-' + i
        else:
            i = '+' + i
        return r + i + 'j'
    return formatter

def _formatComplex(x, real_format, imag_format):
    r = _formatFloat(x.real, real_format)
    i = _formatFloat(x.imag, imag_format, 0)
    if imag_format[-1] == 'f':
        zeros = len(i)
        while zeros > 2 and i[zeros-1] == '0': zeros = zeros-1
        i = i[:zeros] + 'j' + (len(i)-zeros)*' '
    else:
        i = i + 'j'
    return r + i

def _formatGeneral(x):
    return str(x) + ' '

if __name__ == '__main__':
    a = _nc.arange(10)
    print array2string(a)
    print array2string(_nc.array([[],[]]))
<commit_msg>Remove unused names from __all__ in arrayprint.<commit_after>"""Array printing function

$Id: arrayprint.py,v 1.9 2005/09/13 13:58:44 teoliphant Exp $
"""
__all__ = ["array2string", "set_printoptions", "get_printoptions"]

#
# Written by Konrad Hinsen <hinsenk@ere.umontreal.ca>
# last revision: 1996-3-13
# modified by Jim Hugunin 1997-3-3 for repr's and str's (and other details)
# and by Perry Greenfield 2000-4-1 for numarray
# and by Travis Oliphant  2005-8-22 for numpy

import sys
import numeric      as _gen
import numerictypes as _nt
import umath        as _uf
from multiarray import format_longfloat
from fromnumeric import ravel
_nc = _gen

# The following functions are emergency substitutes for numeric functions
# which sometimes get broken during development.

def product(x, y): return x*y

def _maximum_reduce(arr):
    maximum = arr[0]
    for i in xrange(1, arr.nelements()):
        if arr[i] > maximum: maximum = arr[i]
    return maximum

def _minimum_reduce(arr):
    minimum = arr[0]
    for i in xrange(1, arr.nelements()):
        if arr[i] < minimum: minimum = arr[i]
    return minimum

def _numeric_compress(arr):
    nonzero = 0
    for i in xrange(arr.nelements()):
        if arr[i] != 0: nonzero += 1
    retarr = _nc.zeros((nonzero,))
    nonzero = 0
    for i in xrange(arr.nelements()):
        if arr[i] != 0:
            retarr[nonzero] = abs(arr[i])
            nonzero += 1
    return retarr

_failsafe = 0
if _failsafe:
    max_reduce = _maximum_reduce
    min_reduce = _minimum_reduce
else:
    max_reduce = _uf.maximum.reduce
    min_reduce = _uf.minimum.reduce

_summaryEdgeItems = 3     # repr N leading and trailing items of each dimension
_summaryThreshhold = 1000 # total items > triggers array summarization

_float_output_precision = 8
_float_output_suppress_small = False
_line_width = 75

def set_printoptions(precision=None, threshold=None, edgeitems=None,
                     linewidth=None, suppress=None):
    """Set options associated with printing.

    precision  the default number of digits of precision for floating
                   point output
                   (default 8)
    threshold  total number of array elements which trigger summarization
                   rather than full repr.
                   (default 1000)
    edgeitems  number of array items in summary at beginning and end of
                   each dimension.
                   (default 3)
    linewidth  the number of characters per line for the purpose of inserting
                   line breaks.
                   (default 75)
    suppress    Boolean value indicating whether or not suppress printing
                   of small floating point values using scientific notation
                   (default False)
    """

    global _summaryThreshhold, _summaryEdgeItems, _float_output_precision, \
           _line_width, _float_output_suppress_small
    if (linewidth is not None):
        _line_width = linewidth
    if (threshold is not None):
        _summaryThreshhold = threshold
    if (edgeitems is not None):
        _summaryEdgeItems = edgeitems
    if (precision is not None):
        _float_output_precision = precision
    if (suppress is not None):
        _float_output_suppress_small = not not suppress
    return

def get_printoptions():
    return _float_output_precision, _summaryThreshhold, _summaryEdgeItems, \
           _line_width, _float_output_suppress_small

def _leading_trailing(a):
    if a.ndim == 1:
        if len(a) > 2*_summaryEdgeItems:
            b = _gen.concatenate((a[:_summaryEdgeItems],
                                     a[-_summaryEdgeItems:]))
        else:
            b = a
    else:
        if len(a) > 2*_summaryEdgeItems:
            l = [_leading_trailing(a[i]) for i in range(
                min(len(a), _summaryEdgeItems))]
            l.extend([_leading_trailing(a[-i]) for i in range(
                min(len(a), _summaryEdgeItems),0,-1)])
        else:
            l = [_leading_trailing(a[i]) for i in range(0, len(a))]
        b = _gen.concatenate(tuple(l))
    return b

def _array2string(a, max_line_width, precision, suppress_small, separator=' ',
                  prefix=""):

    if max_line_width is None:
        max_line_width = _line_width

    if precision is None:
        precision = _float_output_precision

    if suppress_small is None:
        suppress_small = _float_output_suppress_small

    if a.size > _summaryThreshhold:
        summary_insert = "..., "
        data = _leading_trailing(a)
    else:
        summary_insert = ""
        data = ravel(a)

    try:
        format_function = a._format
    except AttributeError:
        dtype = a.dtype.type
        if issubclass(dtype, _nt.bool):
            format = "%s"
            format_function = lambda x: format % x
        if issubclass(dtype, _nt.integer):
            max_str_len = max(len(str(max_reduce(data))),
                              len(str(min_reduce(data))))
            format = '%' + str(max_str_len) + 'd'
            format_function = lambda x: _formatInteger(x, format)
        elif issubclass(dtype, _nt.floating):
            if issubclass(dtype, _nt.longfloat):
                format_function = _longfloatFormatter(precision)
            else:
                format = _floatFormat(data, precision, suppress_small)
                format_function = lambda x: _formatFloat(x, format)
        elif issubclass(dtype, _nt.complexfloating):
            if issubclass(dtype, _nt.clongfloat):
                format_function = _clongfloatFormatter(precision)
            else:
                real_format = _floatFormat(
                    data.real, precision, suppress_small, sign=0)
                imag_format = _floatFormat(
                    data.imag, precision, suppress_small, sign=1)
                format_function = lambda x: \
                              _formatComplex(x, real_format, imag_format)
        elif issubclass(dtype, _nt.unicode_):
            format = "%s"
            format_function = lambda x: repr(x)
        else:
            format = '%s'
            format_function = lambda x: format % str(x)

    next_line_prefix = " " # skip over "["
    next_line_prefix += " "*len(prefix)                  # skip over array(

    lst = _formatArray(a, format_function, len(a.shape), max_line_width,
                       next_line_prefix, separator,
                       _summaryEdgeItems, summary_insert)[:-1]

    return lst

def array2string(a, max_line_width = None, precision = None,
                 suppress_small = None, separator=' ', prefix="",
                 style=repr):

    if a.shape == ():
        x = a.item()
        try:
            lst = a._format(x)
        except AttributeError:
            lst = style(x)
    elif reduce(product, a.shape) == 0:
        # treat as a null array if any of shape elements == 0
        lst = "[]"
    else:
        lst = _array2string(a, max_line_width, precision, suppress_small,
                            separator, prefix)
    return lst

def _extendLine(s, line, word, max_line_len, next_line_prefix):
    if len(line.rstrip()) + len(word.rstrip()) >= max_line_len:
        s += line.rstrip() + "\n"
        line = next_line_prefix
    line += word
    return s, line

def _formatArray(a, format_function, rank, max_line_len,
                 next_line_prefix, separator, edge_items, summary_insert):
    """formatArray is designed for two modes of operation:

    1. Full output

    2. Summarized output

    """
    if rank == 0:
        return str(a.item())

    if summary_insert and 2*edge_items < len(a):
        leading_items, trailing_items, summary_insert1 = \
                       edge_items, edge_items, summary_insert
    else:
        leading_items, trailing_items, summary_insert1 = 0, len(a), ""

    if rank == 1:

        s = ""
        line = next_line_prefix
        for i in xrange(leading_items):
            word = format_function(a[i]) + separator
            s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)

        if summary_insert1:
            s, line = _extendLine(s, line, summary_insert1, max_line_len, next_line_prefix)

        for i in xrange(trailing_items, 1, -1):
            word = format_function(a[-i]) + separator
            s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)

        word = format_function(a[-1])
        s, line = _extendLine(s, line, word, max_line_len, next_line_prefix)
        s += line + "]\n"
        s = '[' + s[len(next_line_prefix):]
    else:
        s = '['
        sep = separator.rstrip()
        for i in xrange(leading_items):
            if i > 0:
                s += next_line_prefix
            s += _formatArray(a[i], format_function, rank-1, max_line_len,
                              " " + next_line_prefix, separator, edge_items,
                              summary_insert)
            s = s.rstrip() + sep.rstrip() + '\n'*max(rank-1,1)

        if summary_insert1:
            s += next_line_prefix + summary_insert1 + "\n"

        for i in xrange(trailing_items, 1, -1):
            if leading_items or i != trailing_items:
                s += next_line_prefix
            s += _formatArray(a[-i], format_function, rank-1, max_line_len,
                              " " + next_line_prefix, separator, edge_items,
                              summary_insert)
            s = s.rstrip() + sep.rstrip() + '\n'*max(rank-1,1)
        if leading_items or trailing_items > 1:
            s += next_line_prefix
        s += _formatArray(a[-1], format_function, rank-1, max_line_len,
                          " " + next_line_prefix, separator, edge_items,
                          summary_insert).rstrip()+']\n'
    return s

def _floatFormat(data, precision, suppress_small, sign = 0):
    exp_format = 0
    non_zero = _uf.absolute(data.compress(_uf.not_equal(data, 0)))
    ##non_zero = _numeric_compress(data) ##
    if len(non_zero) == 0:
        max_val = 0.
        min_val = 0.
    else:
        max_val = max_reduce(non_zero)
        min_val = min_reduce(non_zero)
        if max_val >= 1.e8:
            exp_format = 1
        if not suppress_small and (min_val < 0.0001
                                   or max_val/min_val > 1000.):
            exp_format = 1
    if exp_format:
        large_exponent = 0 < min_val < 1e-99 or max_val >= 1e100
        max_str_len = 8 + precision + large_exponent
        if sign: format = '%+'
        else: format = '%'
        format = format + str(max_str_len) + '.' + str(precision) + 'e'
        if large_exponent: format = format + '3'
    else:
        format = '%.' + str(precision) + 'f'
        precision = min(precision, max([_digits(x, precision, format)
                                        for x in data]))
        max_str_len = len(str(int(max_val))) + precision + 2
        if sign: format = '%#+'
        else: format = '%#'
        format = format + str(max_str_len) + '.' + str(precision) + 'f'
    return format

def _digits(x, precision, format):
    s = format % x
    zeros = len(s)
    while s[zeros-1] == '0': zeros = zeros-1
    return precision-len(s)+zeros

_MAXINT = sys.maxint
_MININT = -sys.maxint-1
def _formatInteger(x, format):
    if (x < _MAXINT) and (x > _MININT):
        return format % x
    else:
        return "%s" % x

def _longfloatFormatter(precision):
    def formatter(x):
        return format_longfloat(x, precision)
    return formatter

def _formatFloat(x, format, strip_zeros = 1):
    if format[-1] == '3':
        # 3-digit exponent
        format = format[:-1]
        s = format % x
        third = s[-3]
        if third == '+' or third == '-':
            s = s[1:-2] + '0' + s[-2:]
    elif format[-1] == 'e':
        # 2-digit exponent
        s = format % x
        if s[-3] == '0':
            s = ' ' + s[:-3] + s[-2:]
    elif format[-1] == 'f':
        s = format % x
        if strip_zeros:
            zeros = len(s)
            while s[zeros-1] == '0': zeros = zeros-1
            s = s[:zeros] + (len(s)-zeros)*' '
    else:
        s = format % x
    return s

def _clongfloatFormatter(precision):
    def formatter(x):
        r = format_longfloat(x.real, precision)
        i = format_longfloat(x.imag, precision)
        if x.imag < 0:
            i = '-' + i
        else:
            i = '+' + i
        return r + i + 'j'
    return formatter

def _formatComplex(x, real_format, imag_format):
    r = _formatFloat(x.real, real_format)
    i = _formatFloat(x.imag, imag_format, 0)
    if imag_format[-1] == 'f':
        zeros = len(i)
        while zeros > 2 and i[zeros-1] == '0': zeros = zeros-1
        i = i[:zeros] + 'j' + (len(i)-zeros)*' '
    else:
        i = i + 'j'
    return r + i

def _formatGeneral(x):
    return str(x) + ' '

if __name__ == '__main__':
    a = _nc.arange(10)
    print array2string(a)
    print array2string(_nc.array([[],[]]))
<|endoftext|>


<commit_before>#! /usr/bin/env python2.7
import math
import subprocess
import os, glob, shutil
import sys
from argparse import ArgumentParser

def check_dir(dir, force):
	old_files = [file for file in os.listdir(dir) if '.timing' in file and os.path.isfile(os.path.join(dir, file))]
	if len(old_files) and not force:
		raise Exception("Old data found in /{}/... stopping".format(dir))

def make_sure_path_exists(path):
    try:
        os.makedirs(path)
    except OSError as exception:
        if exception.errno != errno.EEXIST:
            raise

def run(name, diff=False, shuffle=False, force=False):
	maker = ['make', '-j24', 'DEBUG=FALSE', 'IGN=TRUE', 'PRINT=FALSE',  'LOG_OUTPUT=FALSE']
	if shuffle:
		maker.extend(["SAME_IC=FALSE", "SHUFFLE=TRUE"])
	elif different_ics:
		maker.extend(["SAME_IC=FALSE"])

	with open('src/launch_bounds.cuh') as file:
		lines = [line.strip() for line in lines]
		block_size = next(line for line in lines if 'TARGET_BLOCK_SIZE' in line)
		block_size = block_size[block_size.index('(') + 1 : block_size.index(')')]

	all_exes = []
	for file in glob.glob('*-int*'):
		all_exes.append(file)

	all_exes = sorted(all_exes, key = lambda x: 1000 if 'gpu' in x else 1)

	threads = [6, 12]
	powers = [0, 3, 7, 10, 14, 17, 20]
	Repeats = 10
	l_t = [False, True]
	l_s = [False, True]
	exppowers = [int(pow(2, exponent)) for exponent in powers]

	#make sure we don't have old data
	check_dir('output', force)
	if not shuffle and not diff:
		desc = '_same_ic'
	elif shuffle:
		desc = '_shuffled_ic'
	elif diff:
		desc = '_diff_ic'
	else:
		raise Exception('Cannot turn on Shuffle and Diff at same time')

	make_sure_path_exists(name + desc)
	results_folder = os.path.abspath(name + desc)
	check_dir(results_folder, force)

	file_list = []
	for repeat in range(repeats):
		for low_tol in l_t:
			for large_step in l_s:
				flags = maker[:]
				if low_tol:
					flags += ['LOW_TOL=TRUE', 'FAST_MATH=TRUE']
				else:
					flags += ['FAST_MATH=FALSE']
				if large_step:
					flags += ['LARGE_STEP=TRUE']
				#force remake
				subprocess.call(flags)
				for exe in all_exes:
					the_threads = threads if not 'gpu' in exe else [block_size]
					for thread in threads:
						for i in range(len(powers)):
							if powers[i] > 14 and not different_ics and not shuffle:
								continue
							if 'exp' in exe:
								continue
							filename = './output/' + exe + '_' + str(thread) +'_' + str(powers[i]) + ('_lt' if low_tol else '') \
										+ ('_ls' if large_step else '') +'.timing'
							if filename not in file_list:
								file_list.append(filename)
							mode = 'w' if repeat == 0 else 'a'
							file = open(filename, mode)
							if 'gpu' in filename:
								subprocess.call([os.path.join(os.getcwd(), exe), str(exppowers[i])], stdout=file)
							else:
								subprocess.call([os.path.join(os.getcwd(), exe), str(thread), str(exppowers[i])], stdout=file)
							file.flush()
							file.close()
	#finally move the files so we can run other cases
	for file in file_list:
		shutil.move(os.path.join('output', file), results_folder, file)

if __name__ == '__main__':
	parser = ArgumentParser(description='Runs timing runs for the various integrators')
	parser.add_arguement('-n', '--name',
						type=str,
						required=True,
						help='the name of the mechanism')
	parser.add_arguement('-f', '--force'
						required=False,
						default=False,
						action='store_true',
						help='Force reuse of past data files')
	run(args.name, diff=True, shuffle=False, args.force)
	run(args.name, diff=False, shuffle=True, args.force)<commit_msg>fixes to the runner<commit_after>#! /usr/bin/env python2.7
import math
import subprocess
import os, glob, shutil
import errno
import sys
from argparse import ArgumentParser

def check_dir(dir, force):
	old_files = [file for file in os.listdir(dir) if '.timing' in file and os.path.isfile(os.path.join(dir, file))]
	if len(old_files) and not force:
		raise Exception("Old data found in /{}/... stopping".format(dir))

def make_sure_path_exists(path):
    try:
        os.makedirs(path)
    except OSError as exception:
        if exception.errno != errno.EEXIST:
            raise

def run(name, diff=False, shuffle=False, force=False):
	maker = ['make', '-j24', 'DEBUG=FALSE', 'IGN=TRUE', 'PRINT=FALSE',  'LOG_OUTPUT=FALSE']
	if shuffle:
		maker.extend(["SAME_IC=FALSE", "SHUFFLE=TRUE"])
	elif diff:
		maker.extend(["SAME_IC=FALSE"])

	with open('src/launch_bounds.cuh') as file:
		lines = [line.strip() for line in file.readlines()]
		block_size = next(line for line in lines if 'TARGET_BLOCK_SIZE' in line)
		block_size = block_size[block_size.index('(') + 1 : block_size.index(')')]

	all_exes = []
	for file in glob.glob('*-int*'):
		all_exes.append(file)

	all_exes = sorted(all_exes, key = lambda x: 1000 if 'gpu' in x else 1)

	threads = [6, 12]
	powers = [0, 3, 7, 10, 14, 17, 20]
	repeats = 5
	l_t = [False, True]
	l_s = [False, True]
	exppowers = [int(pow(2, exponent)) for exponent in powers]

	#make sure we don't have old data
	check_dir('output', force)
	if not shuffle and not diff:
		desc = '_same_ic'
	elif shuffle:
		desc = '_shuffled_ic'
	elif diff:
		desc = '_diff_ic'
	else:
		raise Exception('Cannot turn on Shuffle and Diff at same time')

	make_sure_path_exists(name + desc)
	results_folder = os.path.abspath(name + desc)
	check_dir(results_folder, force)

	file_list = []
	for repeat in range(repeats):
		for low_tol in l_t:
			for large_step in l_s:
				flags = maker[:]
				if low_tol:
					flags += ['LOW_TOL=TRUE', 'FAST_MATH=TRUE']
				else:
					flags += ['FAST_MATH=FALSE']
				if large_step:
					flags += ['LARGE_STEP=TRUE']
				#force remake
				subprocess.call(flags)
				for exe in all_exes:
					the_threads = threads if not 'gpu' in exe else [block_size]
					for thread in the_threads:
						for i in range(len(powers)):
							if powers[i] > 14 and not diff and not shuffle:
								continue
							if 'exp' in exe:
								continue
							filename = './output/' + exe + '_' + str(thread) +'_' + str(powers[i]) + ('_lt' if low_tol else '') \
										+ ('_ls' if large_step else '') +'.timing'
							if filename not in file_list:
								file_list.append(filename)
							mode = 'a' if repeat != 0 else 'w'
							file = open(filename, mode)
							if 'gpu' in filename:
								subprocess.call([os.path.join(os.getcwd(), exe), str(exppowers[i])], stdout=file)
							else:
								subprocess.call([os.path.join(os.getcwd(), exe), str(thread), str(exppowers[i])], stdout=file)
							file.flush()
							file.close()
	#finally move the files so we can run other cases
	for file in file_list:
		shutil.move(os.path.join('output', file), os.path.join(results_folder, file))

if __name__ == '__main__':
	parser = ArgumentParser(description='Runs timing runs for the various integrators')
	parser.add_argument('-n', '--name',
						type=str,
						required=True,
						help='the name of the mechanism')
	parser.add_argument('-f', '--force',
						required=False,
						default=False,
						action='store_true',
						help='Force reuse of past data files')
	args = parser.parse_args()

	run(args.name, diff=True, shuffle=False, force=args.force)
	args.force = False
	run(args.name, diff=False, shuffle=True, force=args.force)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
#
#    Authors: Adrien Peiffer
#    Copyright (c) 2014 Acsone SA/NV (http://www.acsone.eu)
#    All Rights Reserved
#
#    WARNING: This program as such is intended to be used by professional
#    programmers who take the whole responsibility of assessing all potential
#    consequences resulting from its eventual inadequacies and bugs.
#    End users who are looking for a ready-to-use solution with commercial
#    guarantees and support are strongly advised to contact a Free Software
#    Service Company.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#

import openerp.tests.common as common
from openerp import workflow
from openerp.osv import orm
from datetime import datetime, timedelta

DB = common.DB
ADMIN_USER_ID = common.ADMIN_USER_ID

def get_simple_product_id(self):
    return self.registry('product.product').create(self.cr,
                                                   self.uid,
                                                   {'name': 'product_test_01',
                                                    'lst_price': 2000.00,
                                                    }, context={})

def get_journal_check(self, value):
    sale_journal_id = self.ref('account.sales_journal')
    journal_id = self.registry('account.journal').copy(self.cr,
                                                       self.uid,
                                                       sale_journal_id,
                                                       {},
                                                       context={})
    self.registry('account.journal').write(self.cr,
                                           self.uid,
                                           [journal_id],
                                           {'check_chronology': value},
                                           context={})
    return journal_id

def get_simple_account_invoice_line_values(self, product_id):
    return {'name': 'test',
            'account_id': self.ref('account.a_sale'),
            'price_unit': 2000.00,
            'quantity': 1,
            'product_id': product_id,
            }

def create_simple_invoice(self, journal_id, date):
    partner_id = self.ref('base.res_partner_2')
    product_id = get_simple_product_id(self)
    return self.env['account.invoice']\
        .create({'partner_id': partner_id,
                 'account_id':
                 self.ref('account.a_recv'),
                 'journal_id':
                 journal_id,
                 'date_invoice': date,
                 'invoice_line': [(0, 0, {'name': 'test',
                                          'account_id':
                                          self.ref('account.a_sale'),
                                          'price_unit': 2000.00,
                                          'quantity': 1,
                                          'product_id': product_id,
                                          }
                                   )
                                  ],
                 })

class TestAccountConstraintChronology(common.TransactionCase):

    def setUp(self):
        super(TestAccountConstraintChronology, self).setUp()

    def test_invoice_draft(self):
        journal_id = get_journal_check(self, True)
        today = datetime.now()
        yesterday = today - timedelta(days=1)
        date = yesterday.strftime('%Y-%m-%d')
        create_simple_invoice(self, journal_id, date)
        date = today.strftime('%Y-%m-%d')
        invoice_2 = create_simple_invoice(self, journal_id, date)
        self.assertRaises(orm.except_orm, workflow.trg_validate, self.uid,
                          'account.invoice', invoice_2.id, 'invoice_open',
                          self.cr)

    def test_invoice_validate(self):
        journal_id = get_journal_check(self, True)
        today = datetime.now()
        tomorrow = today + timedelta(days=1)
        date = tomorrow.strftime('%Y-%m-%d')
        invoice = create_simple_invoice(self, journal_id, date)
        workflow.trg_validate(self.uid, 'account.invoice', invoice.id,
                              'invoice_open', self.cr)
        date = today.strftime('%Y-%m-%d')
        invoice_2 = create_simple_invoice(self, journal_id, date)
        self.assertRaises(orm.except_orm, workflow.trg_validate, self.uid,
                          'account.invoice', invoice_2.id, 'invoice_open',
                          self.cr)

    def test_invoice_without_date(self):
        journal_id = get_journal_check(self, True)
        today = datetime.now()
        yesterday = today - timedelta(days=1)
        date = yesterday.strftime('%Y-%m-%d')
        create_simple_invoice(self, journal_id, date)
        invoice_2 = create_simple_invoice(self, journal_id, False)
        self.assertRaises(orm.except_orm, workflow.trg_validate, self.uid,
                          'account.invoice', invoice_2.id, 'invoice_open',
                          self.cr)
<commit_msg>[IMP] DEFAULT_SERVER_DATE_FORMAT<commit_after># -*- coding: utf-8 -*-
#
#
#    Authors: Adrien Peiffer
#    Copyright (c) 2014 Acsone SA/NV (http://www.acsone.eu)
#    All Rights Reserved
#
#    WARNING: This program as such is intended to be used by professional
#    programmers who take the whole responsibility of assessing all potential
#    consequences resulting from its eventual inadequacies and bugs.
#    End users who are looking for a ready-to-use solution with commercial
#    guarantees and support are strongly advised to contact a Free Software
#    Service Company.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#

import openerp.tests.common as common
from openerp import workflow
from openerp.osv import orm
from datetime import datetime, timedelta
from openerp.tools import DEFAULT_SERVER_DATE_FORMAT

DB = common.DB
ADMIN_USER_ID = common.ADMIN_USER_ID

def get_simple_product_id(self):
    return self.registry('product.product').create(self.cr,
                                                   self.uid,
                                                   {'name': 'product_test_01',
                                                    'lst_price': 2000.00,
                                                    }, context={})

def get_journal_check(self, value):
    sale_journal_id = self.ref('account.sales_journal')
    journal_id = self.registry('account.journal').copy(self.cr,
                                                       self.uid,
                                                       sale_journal_id,
                                                       {},
                                                       context={})
    self.registry('account.journal').write(self.cr,
                                           self.uid,
                                           [journal_id],
                                           {'check_chronology': value},
                                           context={})
    return journal_id

def get_simple_account_invoice_line_values(self, product_id):
    return {'name': 'test',
            'account_id': self.ref('account.a_sale'),
            'price_unit': 2000.00,
            'quantity': 1,
            'product_id': product_id,
            }

def create_simple_invoice(self, journal_id, date):
    partner_id = self.ref('base.res_partner_2')
    product_id = get_simple_product_id(self)
    return self.env['account.invoice']\
        .create({'partner_id': partner_id,
                 'account_id':
                 self.ref('account.a_recv'),
                 'journal_id':
                 journal_id,
                 'date_invoice': date,
                 'invoice_line': [(0, 0, {'name': 'test',
                                          'account_id':
                                          self.ref('account.a_sale'),
                                          'price_unit': 2000.00,
                                          'quantity': 1,
                                          'product_id': product_id,
                                          }
                                   )
                                  ],
                 })

class TestAccountConstraintChronology(common.TransactionCase):

    def setUp(self):
        super(TestAccountConstraintChronology, self).setUp()

    def test_invoice_draft(self):
        journal_id = get_journal_check(self, True)
        today = datetime.now()
        yesterday = today - timedelta(days=1)
        date = yesterday.strftime(DEFAULT_SERVER_DATE_FORMAT)
        create_simple_invoice(self, journal_id, date)
        date = today.strftime(DEFAULT_SERVER_DATE_FORMAT)
        invoice_2 = create_simple_invoice(self, journal_id, date)
        self.assertRaises(orm.except_orm, workflow.trg_validate, self.uid,
                          'account.invoice', invoice_2.id, 'invoice_open',
                          self.cr)

    def test_invoice_validate(self):
        journal_id = get_journal_check(self, True)
        today = datetime.now()
        tomorrow = today + timedelta(days=1)
        date = tomorrow.strftime(DEFAULT_SERVER_DATE_FORMAT)
        invoice = create_simple_invoice(self, journal_id, date)
        workflow.trg_validate(self.uid, 'account.invoice', invoice.id,
                              'invoice_open', self.cr)
        date = today.strftime(DEFAULT_SERVER_DATE_FORMAT)
        invoice_2 = create_simple_invoice(self, journal_id, date)
        self.assertRaises(orm.except_orm, workflow.trg_validate, self.uid,
                          'account.invoice', invoice_2.id, 'invoice_open',
                          self.cr)

    def test_invoice_without_date(self):
        journal_id = get_journal_check(self, True)
        today = datetime.now()
        yesterday = today - timedelta(days=1)
        date = yesterday.strftime(DEFAULT_SERVER_DATE_FORMAT)
        create_simple_invoice(self, journal_id, date)
        invoice_2 = create_simple_invoice(self, journal_id, False)
        self.assertRaises(orm.except_orm, workflow.trg_validate, self.uid,
                          'account.invoice', invoice_2.id, 'invoice_open',
                          self.cr)
<|endoftext|>


<commit_before>#! /usr/bin/env python
# Last Change: Sat Jul 05 04:00 PM 2008 J
"""This module defines the default tools for each platform, as well as the
default compiler configurations."""

# This is a copy of scons/Tools/__init__.py, because scons does not offer any
# public api for this
def tool_list(platform):
    """platform should be the value returned by enbv['PLATFORM'], not
    sys.platform !!!!."""
    # Here, we set the list of default tools as used by numpy.distutils.scons.
    # This is ripped of scons, because scons does not provide a way to get
    # separate default lists for separate tools (e.g linker, C compiler,
    # etc...)

    if str(platform) == 'win32':
        # prefer Microsoft tools on Windows
        linkers = ['mslink', 'gnulink', 'ilink', 'linkloc', 'ilink32' ]
        c_compilers = ['msvc', 'mingw', 'gcc', 'intelc', 'icl', 'icc', 'cc',
                       'bcc32' ]
        cxx_compilers = ['msvc', 'intelc', 'icc', 'g++', 'c++', 'bcc32' ]
        assemblers = ['masm', 'nasm', 'gas', '386asm' ]
        fortran_compilers = ['g77', 'ifl', 'cvf', 'f95', 'f90', 'fortran']
        ars = ['mslib', 'ar', 'tlib']
    elif str(platform) == 'os2':
        # prefer IBM tools on OS/2
        linkers = ['ilink', 'gnulink', 'mslink']
        c_compilers = ['icc', 'gcc', 'msvc', 'cc']
        cxx_compilers = ['icc', 'g++', 'msvc', 'c++']
        assemblers = ['nasm', 'masm', 'gas']
        fortran_compilers = ['ifl', 'g77']
        ars = ['ar', 'mslib']
    elif str(platform) == 'irix':
        # prefer MIPSPro on IRIX
        linkers = ['sgilink', 'gnulink']
        c_compilers = ['sgicc', 'gcc', 'cc']
        cxx_compilers = ['sgic++', 'g++', 'c++']
        assemblers = ['as', 'gas']
        fortran_compilers = ['f95', 'f90', 'f77', 'g77', 'fortran']
        ars = ['sgiar']
    elif str(platform) == 'sunos':
        # prefer Forte tools on SunOS
        linkers = ['sunlink', 'gnulink']
        c_compilers = ['suncc', 'gcc', 'cc']
        cxx_compilers = ['sunc++', 'g++', 'c++']
        assemblers = ['as', 'gas']
        fortran_compilers = ['f95', 'f90', 'f77', 'g77', 'fortran']
        ars = ['sunar']
    elif str(platform) == 'hpux':
        # prefer aCC tools on HP-UX
        linkers = ['hplink', 'gnulink']
        c_compilers = ['hpcc', 'gcc', 'cc']
        cxx_compilers = ['hpc++', 'g++', 'c++']
        assemblers = ['as', 'gas']
        fortran_compilers = ['f95', 'f90', 'f77', 'g77', 'fortran']
        ars = ['ar']
    elif str(platform) == 'aix':
        # prefer AIX Visual Age tools on AIX
        linkers = ['aixlink', 'gnulink']
        c_compilers = ['aixcc', 'gcc', 'cc']
        cxx_compilers = ['aixc++', 'g++', 'c++']
        assemblers = ['as', 'gas']
        fortran_compilers = ['f95', 'f90', 'aixf77', 'g77', 'fortran']
        ars = ['ar']
    elif str(platform) == 'darwin':
        # prefer GNU tools on Mac OS X, except for some linkers and IBM tools
        linkers = ['applelink', 'gnulink']
        c_compilers = ['gcc', 'cc']
        cxx_compilers = ['g++', 'c++']
        assemblers = ['as']
        fortran_compilers = ['f95', 'f90', 'g77']
        ars = ['ar']
    else:
        # prefer GNU tools on all other platforms
        linkers = ['gnulink', 'mslink', 'ilink']
        c_compilers = ['gcc', 'msvc', 'intelc', 'icc', 'cc']
        cxx_compilers = ['g++', 'msvc', 'intelc', 'icc', 'c++']
        assemblers = ['gas', 'nasm', 'masm']
        fortran_compilers = ['f95', 'f90', 'g77', 'ifort', 'ifl', 'fortran']
        ars = ['ar', 'mslib']

    other_tools = ['BitKeeper', 'CVS', 'dmd', 'dvipdf', 'dvips', 'gs', 'jar',
            'javac', 'javah', 'latex', 'lex', 'm4', 'midl', 'msvs', 'pdflatex',
            'pdftex', 'Perforce', 'RCS', 'rmic', 'rpcgen', 'SCCS', 'swig',
            'tar', 'tex', 'yacc', 'zip']
    return linkers, c_compilers, cxx_compilers, assemblers, fortran_compilers, \
           ars, other_tools
<commit_msg>Clean up default tools list.<commit_after>#! /usr/bin/env python
# Last Change: Wed Jul 30 02:00 PM 2008 J
"""This module defines the default tools for each platform, as well as the
default compiler configurations."""

# This is a copy of scons/Tools/__init__.py, because scons does not offer any
# public api for this
def tool_list(platform):
    """platform should be the value returned by enbv['PLATFORM'], not
    sys.platform. """

    if str(platform) == 'win32':
        # prefer Microsoft tools on Windows
        linkers = ['mslink', 'gnulink', 'ilink', 'linkloc', 'ilink32' ]
        c_compilers = ['msvc', 'mingw', 'gcc', 'intelc', 'icl', 'icc', 'cc',
                       'bcc32' ]
        cxx_compilers = ['msvc', 'intelc', 'icc', 'g++', 'c++', 'bcc32' ]
        assemblers = ['masm', 'nasm', 'gas', '386asm' ]
        fortran_compilers = ['g77', 'ifl', 'cvf', 'f95', 'f90', 'fortran']
        ars = ['mslib', 'ar', 'tlib']
    elif str(platform) == 'sunos':
        # prefer Forte tools on SunOS
        linkers = ['sunlink', 'gnulink']
        c_compilers = ['suncc', 'gcc', 'cc']
        cxx_compilers = ['sunc++', 'g++', 'c++']
        assemblers = ['as', 'gas']
        fortran_compilers = ['f95', 'f90', 'f77', 'g77', 'fortran']
        ars = ['sunar']
    elif str(platform) == 'hpux':
        # prefer aCC tools on HP-UX
        linkers = ['hplink', 'gnulink']
        c_compilers = ['hpcc', 'gcc', 'cc']
        cxx_compilers = ['hpc++', 'g++', 'c++']
        assemblers = ['as', 'gas']
        fortran_compilers = ['f95', 'f90', 'f77', 'g77', 'fortran']
        ars = ['ar']
    elif str(platform) == 'aix':
        # prefer AIX Visual Age tools on AIX
        linkers = ['aixlink', 'gnulink']
        c_compilers = ['aixcc', 'gcc', 'cc']
        cxx_compilers = ['aixc++', 'g++', 'c++']
        assemblers = ['as', 'gas']
        fortran_compilers = ['f95', 'f90', 'aixf77', 'g77', 'fortran']
        ars = ['ar']
    elif str(platform) == 'darwin':
        # prefer GNU tools on Mac OS X, except for some linkers and IBM tools
        linkers = ['applelink', 'gnulink']
        c_compilers = ['gcc', 'cc']
        cxx_compilers = ['g++', 'c++']
        assemblers = ['as']
        fortran_compilers = ['f95', 'f90', 'g77']
        ars = ['ar']
    else:
        # prefer GNU tools on all other platforms
        linkers = ['gnulink', 'mslink', 'ilink']
        c_compilers = ['gcc', 'msvc', 'intelc', 'icc', 'cc']
        cxx_compilers = ['g++', 'msvc', 'intelc', 'icc', 'c++']
        assemblers = ['gas', 'nasm', 'masm']
        fortran_compilers = ['f95', 'f90', 'g77', 'ifort', 'ifl', 'fortran']
        ars = ['ar', 'mslib']

    other_tools = ['BitKeeper', 'CVS', 'dmd', 'dvipdf', 'dvips', 'gs', 'jar',
            'javac', 'javah', 'latex', 'lex', 'm4', 'midl', 'msvs', 'pdflatex',
            'pdftex', 'Perforce', 'RCS', 'rmic', 'rpcgen', 'SCCS', 'swig',
            'tar', 'tex', 'yacc', 'zip']
    return linkers, c_compilers, cxx_compilers, assemblers, fortran_compilers, \
           ars, other_tools
<|endoftext|>


<commit_before>#!/usr/bin/python

# An openstack ansible install runner

import curses
import curses.textpad
import datetime
import fcntl
import json
import os
import re
import select
import subprocess
import sys
import textwrap
import time

class Step(object):
    def __init__(self, name, depends=None):
        self.name = name
        self.depends = depends
        self.attempts = 0

class SimpleCommandStep(Step):
    def __init__(self, name, command, depends=None, cwd=None, env=None):
        super(SimpleCommandStep, self).__init__(name, depends)
        self.command = command
        self.cwd = cwd

        self.env = os.environ
        self.env.update(env)

    def __str__(self):
        return 'step %s, depends on %s' %(self.name, self.depends)

    def run(self, emit, screen):
        emit.emit('Running %s' % self)
        emit.emit('# %s\n' % self.command)
        self.attempts += 1

        if self.attempts > 5:
            emit.emit('... repeatedly failed step, giving up')
            sys.exit(1)

        obj = subprocess.Popen(self.command,
                               stdin=subprocess.PIPE,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE,
                               shell=True,
                               cwd=self.cwd,
                               env=self.env)

        flags = fcntl.fcntl(obj.stdout, fcntl.F_GETFL)
        fcntl.fcntl(obj.stdout, fcntl.F_SETFL, flags | os.O_NONBLOCK)

        flags = fcntl.fcntl(obj.stderr, fcntl.F_GETFL)
        fcntl.fcntl(obj.stderr, fcntl.F_SETFL, flags | os.O_NONBLOCK)

        obj.stdin.close()
        while obj.poll() is None:
            readable, _, _ = select.select([obj.stderr, obj.stdout], [], [], 0)
            for f in readable:
                emit.emit(os.read(f.fileno(), 10000))

        emit.emit('... process complete')
        returncode = obj.returncode
        emit.emit('... exit code %d' % returncode)
        return returncode == 0

class QuestionStep(Step):
    def __init__(self, name, title, helpful, prompt, depends=None):
        super(QuestionStep, self).__init__(name, depends)
        self.title = title
        self.help = helpful
        self.prompt = prompt

    def run(self, emit, screen):
        emit.emit('%s' % self.title)
        emit.emit('%s\n' % ('=' * len(self.title)))
        emit.emit('%s\n' % self.help)
        return emit.getstr('>> ')

class EnforceScreenStep(Step):
    def __init__(self):
        super(EnforceScreenStep, self).__init__('enforce-screen')

    def run(self, emit, screen):
        if not os.environ['TERM'].startswith('screen'):
            emit.emit('Only run ostrich in a screen session please')
            sys.exit(1)
        return True

class RegexpEditorStep(Step):
    def __init__(self, name, path, search, replace, cwd=None, env=None):
        super(RegexpEditorStep, self).__init__(name)
        self.path = path
        if cwd:
            self.path = os.path.join(cwd, path)

        self.search = search
        self.replace = replace

    def run(self, emit, screen):
        output = []
        changes = 0

        with open(self.path, 'r') as f:
            for line in f.readlines():
                line = line.rstrip()
                newline = re.sub(self.search, self.replace, line)
                emit.emit(newline)
                output.append(newline)
                if newline != line:
                    changes += 1

        with open(self.path, 'w') as f:
            f.write('\n'.join(output))

        return 'Changed %d lines' % changes

class Emitter(object):
    def __init__(self, output):
        self.output = output
        self.logfile = None

    def clear(self):
	self.output.clear()

    def logger(self, logfile):
        if self.logfile:
            self.logfile.close()
        self.logfile = open(os.path.expanduser('~/.ostrich/%s' % logfile), 'w')

    def emit(self, s):
        height, width = self.output.getmaxyx()

        for line in s.split('\n'):
            line = ''.join([i if ord(i) < 128 else ' ' for i in line])

            if self.logfile:
                self.logfile.write('%s %s\n' %(datetime.datetime.now(), line))
                self.logfile.flush()

            for l in textwrap.wrap(line, width - 3):
                if len(l) > 0:
                    self.output.scroll()
                    self.output.addstr(height - 2, 1, ' ' * (width - 1))

                    try:
                        self.output.addstr(height - 2, 2, l)
                    except Exception as e:
                        print 'Exception: %s' % e
                        print '>>%s<<' % line
                        sys.exit(1)

        self.output.border()
        self.output.refresh()

    def getstr(self, s):
        height, width = self.output.getmaxyx()

        self.emit(s)
        curses.echo()
        answer = self.output.getstr(height - 2, len(s) + 2)
        curses.noecho()
        return answer

class Runner(object):
    def __init__(self, screen):
        self.screen = screen

        self.steps = {}

        self.state_path = os.path.expanduser('~/.ostrich/state.json')
        if not os.path.exists(os.path.expanduser('~/.ostrich')):
            os.mkdir(os.path.expanduser('~/.ostrich'))

        self.complete = {}
        self.counter = 0
        if os.path.exists(self.state_path):
            with open(self.state_path, 'r') as f:
                state = json.loads(f.read())
                self.complete = state.get('complete', {})
                self.counter = state.get('counter', 0)

    def load_step(self, step):
        self.steps[step.name] = step

    def load_dependancy_chain(self, steps, depends=None):
        depend = depends
        for step in steps:
            step.depends = depend
            depend = step.name
            self.load_step(step)

    def resolve_steps(self):
        # Setup curses windows for the steps view
        height, width = self.screen.getmaxyx()
        progress = curses.newwin(3, width, 0, 0)
        progress.border()
        progress.refresh()

        output = curses.newwin(height - 4, width, 3, 0)
        output.scrollok(True)
        output.border()
        output.refresh()
        emitter = Emitter(output)

        for step_name in self.complete:
            if step_name in self.steps:
                del self.steps[step_name]

        run = [True]
        complete = []

        while len(run) > 0:
            run = []
            complete = []

            for step_name in self.steps:
                step = self.steps[step_name]

                if not step.depends or self.complete.get(step.depends, False):
                    progress.clear()
                    progress.addstr(1, 3, '%s %d steps to run, running %s' %(datetime.datetime.now(), len(self.steps), step_name))
                    progress.border()
                    progress.refresh()

                    run.append(step_name)
                    emitter.clear()
                    emitter.logger('%06d-%s' %(self.counter, step_name))
                    outcome = step.run(emitter, self.screen)
                    self.counter += 1

                    if outcome:
                        self.complete[step_name] = outcome
                        complete.append(step_name)

                    with open(self.state_path, 'w') as f:
                        f.write(json.dumps({'complete': self.complete,
                                            'counter': self.counter},
                                           indent=4, sort_keys=True))

            for step_name in complete:
                del self.steps[step_name]

def main(screen):
    screen.nodelay(False)
    r = Runner(screen)

    # We really like screen around here
    r.load_step(SimpleCommandStep('apt-daily', 'while [ `ps -ef | grep apt.systemd.daily | grep -vc "grep"` -gt 0 ]; do   echo "Waiting for daily apt run to end";   sleep 10; done'))
    r.load_step(EnforceScreenStep())
    r.resolve_steps()

    # git proxies
    r.load_dependancy_chain(
        [QuestionStep('git-mirror-github',
                      'Are you running a local github.com mirror?',
                      'Mirroring github.com speeds up setup on slow and unreliable networks, but means that you have to maintain a mirror somewhere on your corporate network. If you are unsure, just enter "git://github.com" here. Otherwise, we need an answer in the form of <protocol>://<server>, for example git://gitmirror.example.com',
                      'Mirror URL'),
         QuestionStep('git-mirror-openstack',
                      'Are you running a local git.openstack.org mirror?',
                      'Mirroring git.openstack.org speeds up setup on slow and unreliable networks, but means that you have to maintain a mirror somewhere on your corporate network. If you are unsure, just enter "git://git.openstack.org" here. Otherwise, we need an answer in the form of <protocol>://<server>, for example git://gitmirror.example.com',
                      'Mirror URL'),
         QuestionStep('osa-branch',
                      'What OSA branch (or commit SHA) would you like to use?',
                      'Use stable/newton unless you know what you are doing.',
                      'OSA branch'),
         QuestionStep('http-proxy',
                      'Are you running a local http proxy?',
                      'OSA will download large objects such as LXC base images. If you have a slow network, or are on a corporate network which requires a proxy, configure it here with a URL like http://cache.example.com:3128',
                      'HTTP Proxy'),
         ])

    # APT commands
    r.load_dependancy_chain(
        [SimpleCommandStep('apt-update', 'apt-get update'),
         SimpleCommandStep('apt-upgrade', 'apt-get upgrade -y'),
         SimpleCommandStep('apt-dist-upgrade', 'apt-get dist-upgrade -y'),
         SimpleCommandStep('apt-useful',
                           'apt-get install -y screen ack-grep git expect')
         ])

    # Do the thing
    r.resolve_steps()

    # Steps requiring data from earlier
    r.load_dependancy_chain(
        [SimpleCommandStep('git-clone-osa', 'git clone %s/openstack/openstack-ansible /opt/openstack-ansible' % r.complete['git-mirror-openstack']),
         ])

    # Steps where we now have the OSA checkout
    kwargs = {'cwd': '/opt/openstack-ansible',
              'env': {'ANSIBLE_ROLE_FETCH_MODE': 'git-clone',
                      'http_proxy': r.complete['http-proxy'],
                      'https_proxy': r.complete['http-proxy']}}
    r.load_dependancy_chain(
         [SimpleCommandStep('git-checkout-osa', 'git checkout %s' % r.complete['osa-branch'], **kwargs),
          SimpleCommandStep('fixup-add-ironic', 'sed -i -e "/- name: heat.yml.aio/ a \        - name: ironic.yml.aio"  tests/bootstrap-aio.yml', **kwargs),
          SimpleCommandStep('fixup-virt-ironic', 'echo "nova_virt_type: ironic" >> etc/openstack_deploy/user_variables.yml', **kwargs),
          RegexpEditorStep('ansible-role-assignments-github-mirror', 'ansible-role-requirements.yml',
                           '(http|https|git)://github.com', r.complete['git-mirror-github'], **kwargs),
          RegexpEditorStep('ansible-role-assignments-openstack-mirror', 'ansible-role-requirements.yml',
                           '(http|https|git)://git.openstack.org', r.complete['git-mirror-openstack'], **kwargs),
          RegexpEditorStep('ansible-no-loopback-swap', '/opt/openstack-ansible/tests/roles/bootstrap-host/tasks/prepare_loopback_swap.yml',
                           'command: grep /openstack/swap.img /proc/swaps', 'command: /bin/true', **kwargs),
          RegexpEditorStep('ansible-from-mirror', '/opt/openstack-ansible/scripts/bootstrap-ansible.sh',
                           'https://github.com/ansible/ansible', '%s/ansible/ansible' % r.complete['git-mirror-github']),
          SimpleCommandStep('bootstrap-ansible', './scripts/bootstrap-ansible.sh', **kwargs),
          SimpleCommandStep('bootstrap-aio', './scripts/bootstrap-aio.sh', **kwargs),
          ],
        depends='git-clone-osa')

    # We also need to re-write git repos in a large number of roles
    for root, _, files in os.walk('/etc/ansible'):
        for filename in files:
            path = os.path.join(root, filename)
            if path.endswith('ansible-role-requirements.yml'):
                r.add_step(RegexpEditorStep('%s-github-mirror' % path, path,
                                            '(http|https|git)://github.com', r.complete['git-mirror-github'], **kwargs))
                r.add_step(RegexpEditorStep('%s-openstack-mirror' % path, path,
                                            '(http|https|git)://git.openstack.org', r.complete['git-mirror-openstack'], **kwargs))
    r.resolve_steps()

    kwargs['cwd'] = os.path.join(kwargs['cwd'], 'playbooks')
    r.load_dependancy_chain(
         [SimpleCommandStep('bootstrap-setup-everything', 'openstack-ansible setup-everything.yml', **kwargs),
          ],
        depends='bootstrap-aio')

    # Do the more things
    r.resolve_steps()

if __name__ == '__main__':
    curses.wrapper(main)
<commit_msg>Make paths logable, fix wrong method name.<commit_after>#!/usr/bin/python

# An openstack ansible install runner

import curses
import curses.textpad
import datetime
import fcntl
import json
import os
import re
import select
import subprocess
import sys
import textwrap
import time

class Step(object):
    def __init__(self, name, depends=None):
        self.name = name
        self.depends = depends
        self.attempts = 0

class SimpleCommandStep(Step):
    def __init__(self, name, command, depends=None, cwd=None, env=None):
        super(SimpleCommandStep, self).__init__(name, depends)
        self.command = command
        self.cwd = cwd

        self.env = os.environ
        self.env.update(env)

    def __str__(self):
        return 'step %s, depends on %s' %(self.name, self.depends)

    def run(self, emit, screen):
        emit.emit('Running %s' % self)
        emit.emit('# %s\n' % self.command)
        self.attempts += 1

        if self.attempts > 5:
            emit.emit('... repeatedly failed step, giving up')
            sys.exit(1)

        obj = subprocess.Popen(self.command,
                               stdin=subprocess.PIPE,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE,
                               shell=True,
                               cwd=self.cwd,
                               env=self.env)

        flags = fcntl.fcntl(obj.stdout, fcntl.F_GETFL)
        fcntl.fcntl(obj.stdout, fcntl.F_SETFL, flags | os.O_NONBLOCK)

        flags = fcntl.fcntl(obj.stderr, fcntl.F_GETFL)
        fcntl.fcntl(obj.stderr, fcntl.F_SETFL, flags | os.O_NONBLOCK)

        obj.stdin.close()
        while obj.poll() is None:
            readable, _, _ = select.select([obj.stderr, obj.stdout], [], [], 0)
            for f in readable:
                emit.emit(os.read(f.fileno(), 10000))

        emit.emit('... process complete')
        returncode = obj.returncode
        emit.emit('... exit code %d' % returncode)
        return returncode == 0

class QuestionStep(Step):
    def __init__(self, name, title, helpful, prompt, depends=None):
        super(QuestionStep, self).__init__(name, depends)
        self.title = title
        self.help = helpful
        self.prompt = prompt

    def run(self, emit, screen):
        emit.emit('%s' % self.title)
        emit.emit('%s\n' % ('=' * len(self.title)))
        emit.emit('%s\n' % self.help)
        return emit.getstr('>> ')

class EnforceScreenStep(Step):
    def __init__(self):
        super(EnforceScreenStep, self).__init__('enforce-screen')

    def run(self, emit, screen):
        if not os.environ['TERM'].startswith('screen'):
            emit.emit('Only run ostrich in a screen session please')
            sys.exit(1)
        return True

class RegexpEditorStep(Step):
    def __init__(self, name, path, search, replace, cwd=None, env=None):
        super(RegexpEditorStep, self).__init__(name)
        self.path = path
        if cwd:
            self.path = os.path.join(cwd, path)

        self.search = search
        self.replace = replace

    def run(self, emit, screen):
        output = []
        changes = 0

        with open(self.path, 'r') as f:
            for line in f.readlines():
                line = line.rstrip()
                newline = re.sub(self.search, self.replace, line)
                emit.emit(newline)
                output.append(newline)
                if newline != line:
                    changes += 1

        with open(self.path, 'w') as f:
            f.write('\n'.join(output))

        return 'Changed %d lines' % changes

class Emitter(object):
    def __init__(self, output):
        self.output = output
        self.logfile = None

    def clear(self):
	self.output.clear()

    def logger(self, logfile):
        if self.logfile:
            self.logfile.close()
        self.logfile = open(os.path.expanduser('~/.ostrich/%s' % logfile), 'w')

    def emit(self, s):
        height, width = self.output.getmaxyx()

        for line in s.split('\n'):
            line = ''.join([i if ord(i) < 128 else ' ' for i in line])

            if self.logfile:
                self.logfile.write('%s %s\n' %(datetime.datetime.now(), line))
                self.logfile.flush()

            for l in textwrap.wrap(line, width - 3):
                if len(l) > 0:
                    self.output.scroll()
                    self.output.addstr(height - 2, 1, ' ' * (width - 1))

                    try:
                        self.output.addstr(height - 2, 2, l)
                    except Exception as e:
                        print 'Exception: %s' % e
                        print '>>%s<<' % line
                        sys.exit(1)

        self.output.border()
        self.output.refresh()

    def getstr(self, s):
        height, width = self.output.getmaxyx()

        self.emit(s)
        curses.echo()
        answer = self.output.getstr(height - 2, len(s) + 2)
        curses.noecho()
        return answer

class Runner(object):
    def __init__(self, screen):
        self.screen = screen

        self.steps = {}

        self.state_path = os.path.expanduser('~/.ostrich/state.json')
        if not os.path.exists(os.path.expanduser('~/.ostrich')):
            os.mkdir(os.path.expanduser('~/.ostrich'))

        self.complete = {}
        self.counter = 0
        if os.path.exists(self.state_path):
            with open(self.state_path, 'r') as f:
                state = json.loads(f.read())
                self.complete = state.get('complete', {})
                self.counter = state.get('counter', 0)

    def load_step(self, step):
        self.steps[step.name] = step

    def load_dependancy_chain(self, steps, depends=None):
        depend = depends
        for step in steps:
            step.depends = depend
            depend = step.name
            self.load_step(step)

    def resolve_steps(self):
        # Setup curses windows for the steps view
        height, width = self.screen.getmaxyx()
        progress = curses.newwin(3, width, 0, 0)
        progress.border()
        progress.refresh()

        output = curses.newwin(height - 4, width, 3, 0)
        output.scrollok(True)
        output.border()
        output.refresh()
        emitter = Emitter(output)

        for step_name in self.complete:
            if step_name in self.steps:
                del self.steps[step_name]

        run = [True]
        complete = []

        while len(run) > 0:
            run = []
            complete = []

            for step_name in self.steps:
                step = self.steps[step_name]

                if not step.depends or self.complete.get(step.depends, False):
                    progress.clear()
                    progress.addstr(1, 3, '%s %d steps to run, running %s' %(datetime.datetime.now(), len(self.steps), step_name))
                    progress.border()
                    progress.refresh()

                    run.append(step_name)
                    emitter.clear()
                    emitter.logger('%06d-%s' %(self.counter, step_name))
                    outcome = step.run(emitter, self.screen)
                    self.counter += 1

                    if outcome:
                        self.complete[step_name] = outcome
                        complete.append(step_name)

                    with open(self.state_path, 'w') as f:
                        f.write(json.dumps({'complete': self.complete,
                                            'counter': self.counter},
                                           indent=4, sort_keys=True))

            for step_name in complete:
                del self.steps[step_name]

def main(screen):
    screen.nodelay(False)
    r = Runner(screen)

    # We really like screen around here
    r.load_step(SimpleCommandStep('apt-daily', 'while [ `ps -ef | grep apt.systemd.daily | grep -vc "grep"` -gt 0 ]; do   echo "Waiting for daily apt run to end";   sleep 10; done'))
    r.load_step(EnforceScreenStep())
    r.resolve_steps()

    # git proxies
    r.load_dependancy_chain(
        [QuestionStep('git-mirror-github',
                      'Are you running a local github.com mirror?',
                      'Mirroring github.com speeds up setup on slow and unreliable networks, but means that you have to maintain a mirror somewhere on your corporate network. If you are unsure, just enter "git://github.com" here. Otherwise, we need an answer in the form of <protocol>://<server>, for example git://gitmirror.example.com',
                      'Mirror URL'),
         QuestionStep('git-mirror-openstack',
                      'Are you running a local git.openstack.org mirror?',
                      'Mirroring git.openstack.org speeds up setup on slow and unreliable networks, but means that you have to maintain a mirror somewhere on your corporate network. If you are unsure, just enter "git://git.openstack.org" here. Otherwise, we need an answer in the form of <protocol>://<server>, for example git://gitmirror.example.com',
                      'Mirror URL'),
         QuestionStep('osa-branch',
                      'What OSA branch (or commit SHA) would you like to use?',
                      'Use stable/newton unless you know what you are doing.',
                      'OSA branch'),
         QuestionStep('http-proxy',
                      'Are you running a local http proxy?',
                      'OSA will download large objects such as LXC base images. If you have a slow network, or are on a corporate network which requires a proxy, configure it here with a URL like http://cache.example.com:3128',
                      'HTTP Proxy'),
         ])

    # APT commands
    r.load_dependancy_chain(
        [SimpleCommandStep('apt-update', 'apt-get update'),
         SimpleCommandStep('apt-upgrade', 'apt-get upgrade -y'),
         SimpleCommandStep('apt-dist-upgrade', 'apt-get dist-upgrade -y'),
         SimpleCommandStep('apt-useful',
                           'apt-get install -y screen ack-grep git expect')
         ])

    # Do the thing
    r.resolve_steps()

    # Steps requiring data from earlier
    r.load_dependancy_chain(
        [SimpleCommandStep('git-clone-osa', 'git clone %s/openstack/openstack-ansible /opt/openstack-ansible' % r.complete['git-mirror-openstack']),
         ])

    # Steps where we now have the OSA checkout
    kwargs = {'cwd': '/opt/openstack-ansible',
              'env': {'ANSIBLE_ROLE_FETCH_MODE': 'git-clone',
                      'http_proxy': r.complete['http-proxy'],
                      'https_proxy': r.complete['http-proxy']}}
    r.load_dependancy_chain(
         [SimpleCommandStep('git-checkout-osa', 'git checkout %s' % r.complete['osa-branch'], **kwargs),
          SimpleCommandStep('fixup-add-ironic', 'sed -i -e "/- name: heat.yml.aio/ a \        - name: ironic.yml.aio"  tests/bootstrap-aio.yml', **kwargs),
          SimpleCommandStep('fixup-virt-ironic', 'echo "nova_virt_type: ironic" >> etc/openstack_deploy/user_variables.yml', **kwargs),
          RegexpEditorStep('ansible-role-assignments-github-mirror', 'ansible-role-requirements.yml',
                           '(http|https|git)://github.com', r.complete['git-mirror-github'], **kwargs),
          RegexpEditorStep('ansible-role-assignments-openstack-mirror', 'ansible-role-requirements.yml',
                           '(http|https|git)://git.openstack.org', r.complete['git-mirror-openstack'], **kwargs),
          RegexpEditorStep('ansible-no-loopback-swap', '/opt/openstack-ansible/tests/roles/bootstrap-host/tasks/prepare_loopback_swap.yml',
                           'command: grep /openstack/swap.img /proc/swaps', 'command: /bin/true', **kwargs),
          RegexpEditorStep('ansible-from-mirror', '/opt/openstack-ansible/scripts/bootstrap-ansible.sh',
                           'https://github.com/ansible/ansible', '%s/ansible/ansible' % r.complete['git-mirror-github']),
          SimpleCommandStep('bootstrap-ansible', './scripts/bootstrap-ansible.sh', **kwargs),
          SimpleCommandStep('bootstrap-aio', './scripts/bootstrap-aio.sh', **kwargs),
          ],
        depends='git-clone-osa')

    # We also need to re-write git repos in a large number of roles
    for root, _, files in os.walk('/etc/ansible'):
        for filename in files:
            path = os.path.join(root, filename)
            jobpath = path.replace('.yml', '').replace('/', '_')
            if path.endswith('ansible-role-requirements.yml'):
                r.load_step(RegexpEditorStep('%s-github-mirror' % jobpath, path,
                                             '(http|https|git)://github.com', r.complete['git-mirror-github'], **kwargs))
                r.load_step(RegexpEditorStep('%s-openstack-mirror' % jobpath, path,
                                             '(http|https|git)://git.openstack.org', r.complete['git-mirror-openstack'], **kwargs))
    r.resolve_steps()

    kwargs['cwd'] = os.path.join(kwargs['cwd'], 'playbooks')
    r.load_dependancy_chain(
         [SimpleCommandStep('bootstrap-setup-everything', 'openstack-ansible setup-everything.yml', **kwargs),
          ],
        depends='bootstrap-aio')

    # Do the more things
    r.resolve_steps()

if __name__ == '__main__':
    curses.wrapper(main)
<|endoftext|>


<commit_before># Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# pylint: disable=invalid-name
"""
Hangouts Chat bot that responds to events and messages from a room asynchronously.
"""

# [START async-bot]

import logging
from google.cloud import datastore
from googleapiclient.discovery import build
from flask import Flask, render_template, request, json
import google.auth

app = Flask(__name__)

scopes = ['https://www.googleapis.com/auth/chat.bot']
credentials, project_id = google.auth.default()
credentials = credentials.with_scopes(scopes=scopes)
chat = build('chat', 'v1', credentials=credentials)

# phase number (key) : [phase name, phase specific help message]
phase_map = {
        1 : ["name", "The name that will be associated with your dynamic search ad campaign estimate."],
        2 : ["startDate", "The date that your campaign simulation starts. Must be in the form mm-dd-yyyy"],
        3 : ["endDate", "The date that your campaign simulation ends. Must be in the form mm-dd-yyyy"],
        4 : ["dailyBudget", "Amount you are willing to spend each day the ad is displayed."],
        5 : ["locations", "The locations that your ad will target. Ex: CA, USA (Must follow this form!)"],
        6 : ["negativeLocations", "The locations that your ad will not target. Ex: CA, USA (Must follow this form!)"],
        7 : ["domain", "The domain of your webpage being advertised. (Ex: http://google.com)"],
        8 : ["targets", "Target pages of your domain that you would like to specifically advertise. (Ex: http://google.com/shoes)",
        9 : ["manualCPC", "Maximum amount being spent on each click. (Cost per click)"]
}

@app.route('/', methods=['POST'])
def home_post():
  """Respond to POST requests to this endpoint.
  All requests sent to this endpoint from Hangouts Chat are POST
  requests.
  """

  event_data = request.get_json()

  resp = None

  # If the bot is removed from the space, it doesn't post a message
  # to the space. Instead, log a message showing that the bot was removed.
  if event_data['type'] == 'REMOVED_FROM_SPACE':
    logging.info('Bot removed from  %s', event_data['space']['name'])
    return json.jsonify({})

  resp = format_response(event_data)
  space_name = event_data['space']['name']
  send_async_response(resp, space_name)

  # Return empty jsom respomse simce message already sent via REST API
  return json.jsonify({})

# [START async-response]

def send_async_response(response, space_name):
  """Sends a response back to the Hangouts Chat room asynchronously.
  Args:
    response: the response payload
    space_name: The URL of the Hangouts Chat room
  """
  chat.spaces().messages().create(
    parent=space_name,
    body=response).execute()

# [END async-response]

def format_response(event):
  """Determine what response to provide based upon event data.
  Args:
    event: A dictionary with the event data.
  """

  event_type = event['type']

  text = ""
  sender_name = event['user']['displayName']

  # Case 1: The bot was added to a room
  if event_type == 'ADDED_TO_SPACE' and event['space']['type'] == 'ROOM':
    text = 'Thanks for adding me to {}!'.format(event['space']['displayName'])

  # Case 2: The bot was added to a DM
  elif event_type == 'ADDED_TO_SPACE' and event['space']['type'] == 'DM':
    text = 'Thanks for adding me to a DM, {}! To start creating a campaign, type \'start\''.format(sender_name)

  elif event_type == 'MESSAGE':
    if (event['message']['text'] == 'start') {
      client_key = datastore_client.key('User', sender_name)
      task = datastore.Entity(key=client_key)
      task['phase-num'] = 0
    }
    datastore_client = datastore.Client()

    # The kind for the new entity
    kind = 'TestMessage'
    # The name/ID for the new entity
    name = 'msg1'
    # The Cloud Datastore key for the new entity
    task_key = datastore_client.key(kind, name)

    # Prepares the new entity
    task = datastore.Entity(key=task_key)
    task['test-message'] = event['message']['text']

    # Saves the entity
    datastore_client.put(task)

    text = 'Your message, {}: "{}"'.format(sender_name, event['message']['text'])

  response = {'text': text}

  # The following three lines of code update the thread that raised the event.
  # Delete them if you want to send the message in a new thread.
  if event_type == 'MESSAGE' and event['message']['thread'] is not None:
    thread_id = event['message']['thread']
    response['thread'] = thread_id

  return response

# [END async-bot]

@app.route('/', methods=['GET'])
def home_get():
  """Respond to GET requests to this endpoint.
  This function responds to requests with a simple HTML landing page for this
  App Engine instance.
  """

  return render_template('home.html')

if __name__ == '__main__':
  # This is used when running locally. Gunicorn is used to run the
  # application on Google App Engine. See entrypoint in app.yaml.
  app.run(host='127.0.0.1', port=8080, debug=True)

def handle_message(user_name, message) {
  if (message == 'start') {
    # TODO
  }
}<commit_msg>Add phase map to be used by the bot.<commit_after># Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# pylint: disable=invalid-name
"""
Hangouts Chat bot that responds to events and messages from a room asynchronously.
"""

# [START async-bot]

import logging
from google.cloud import datastore
from googleapiclient.discovery import build
from flask import Flask, render_template, request, json
import google.auth

app = Flask(__name__)

scopes = ['https://www.googleapis.com/auth/chat.bot']
credentials, project_id = google.auth.default()
credentials = credentials.with_scopes(scopes=scopes)
chat = build('chat', 'v1', credentials=credentials)

# phase number (key) : [phase name, phase specific help message]
phase_map = {
            1 : ['name', 'The name that will be associated with your dynamic search ad campaign estimate.'],
            2 : ['startDate', 'The date that your campaign simulation starts. Must be in the form mm-dd-yyyy.'],
            3 : ['endDate', 'The date that your campaign simulation ends. Must be in the form mm-dd-yyyy.'],
            4 : ['dailyBudget', 'Amount you are willing to spend each day the ad is displayed.'],
            5 : ['locations', 'The locations that your ad will target. Ex: CA, USA (Must follow this form!)'],
            6 : ['negativeLocations', 'The locations that your ad will not target. Ex: CA, USA (Must follow this format!)'],
            7 : ['domain', 'The domain of your webpage being advertised. (Ex: http://google.com)'],
            8 : ['targets', 'Target pages of your domain that you would like to specifically advertise. (Ex: http://google.com/somepage)'],
            9 : ['manualCPC', 'Maximum amount being spent on each click. (Cost per click)']
}

@app.route('/', methods=['POST'])
def home_post():
  """Respond to POST requests to this endpoint.
  All requests sent to this endpoint from Hangouts Chat are POST
  requests.
  """

  event_data = request.get_json()

  resp = None

  # If the bot is removed from the space, it doesn't post a message
  # to the space. Instead, log a message showing that the bot was removed.
  if event_data['type'] == 'REMOVED_FROM_SPACE':
    logging.info('Bot removed from  %s', event_data['space']['name'])
    return json.jsonify({})

  resp = format_response(event_data)
  space_name = event_data['space']['name']
  send_async_response(resp, space_name)

  # Return empty jsom respomse simce message already sent via REST API
  return json.jsonify({})

# [START async-response]

def send_async_response(response, space_name):
  """Sends a response back to the Hangouts Chat room asynchronously.
  Args:
    response: the response payload
    space_name: The URL of the Hangouts Chat room
  """
  chat.spaces().messages().create(
    parent=space_name,
    body=response).execute()

# [END async-response]
test = False

def format_response(event):
  """Determine what response to provide based upon event data.
  Args:
    event: A dictionary with the event data.
  """
  global test
  event_type = event['type']

  text = ""
  sender_name = event['user']['displayName']

  # Case 1: The bot was added to a room
  if event_type == 'ADDED_TO_SPACE' and event['space']['type'] == 'ROOM':
    text = 'Thanks for adding me to {}!'.format(event['space']['displayName'])

  # Case 2: The bot was added to a DM
  elif event_type == 'ADDED_TO_SPACE' and event['space']['type'] == 'DM':
    text = 'Thanks for adding me to a DM, {}! To start creating a campaign, type \'start\''.format(sender_name)
    test = True
  elif event_type == 'MESSAGE':
    datastore_client = datastore.Client()
    if (event['message']['text'] == 'start'):
      """event['user']['name'] always follows the form users/###############,
          need the id after users/ """
      userId = event['user']['name'][len('users/'):]

      client_key = datastore_client.key('User', userId)
      userEntity = datastore.Entity(key=client_key)
      userEntity['phase-num'] = 0

    # The kind for the new entity
    kind = 'TestMessage'

    # The name/ID for the new entity
    name = event['user']['name'][len('users/'):]

    # The Cloud Datastore key for the new entity
    task_key = datastore_client.key(kind, name)

    # Prepares the new entity
    task = datastore.Entity(key=task_key)
    task['test-message'] = event['message']['text']

    # Saves the entity
    datastore_client.put(task)
    text = 'Your message, {}: "{}" and {}'.format(sender_name, event['message']['text'], test)

  response = {'text': text}

  # The following three lines of code update the thread that raised the event.
  # Delete them if you want to send the message in a new thread.
  if event_type == 'MESSAGE' and event['message']['thread'] is not None:
    thread_id = event['message']['thread']
    response['thread'] = thread_id

  return response

# [END async-bot]

@app.route('/', methods=['GET'])
def home_get():
  """Respond to GET requests to this endpoint.
  This function responds to requests with a simple HTML landing page for this
  App Engine instance.
  """

  return render_template('home.html')

if __name__ == '__main__':
  # This is used when running locally. Gunicorn is used to run the
  # application on Google App Engine. See entrypoint in app.yaml.
  app.run(host='127.0.0.1', port=8080, debug=True)

def handle_message(user_name, message):
  if (message == 'start'):
    # TODO: fix start element
    print('start process')<|endoftext|>