<commit_before>from itertools import product

from lasagne.layers import get_output
from lasagne.layers import get_output_shape
from lasagne.objectives import binary_crossentropy
import matplotlib.pyplot as plt
import numpy as np
import theano
import theano.tensor as T
import io
import lasagne
import pydotplus as pydot
from IPython.display import Image

def plot_loss(net):
    train_loss = [row['train_loss'] for row in net.train_history_]
    valid_loss = [row['valid_loss'] for row in net.train_history_]
    plt.plot(train_loss, label='train loss')
    plt.plot(valid_loss, label='valid loss')
    plt.xlabel('epoch')
    plt.ylabel('loss')
    plt.legend(loc='best')
    return plt

def plot_conv_weights(layer, figsize=(6, 6)):
    """Plot the weights of a specific layer.

    Only really makes sense with convolutional layers.

    Parameters
    ----------
    layer : lasagne.layers.Layer

    """
    W = layer.W.get_value()
    shape = W.shape
    nrows = np.ceil(np.sqrt(shape[0])).astype(int)
    ncols = nrows

    for feature_map in range(shape[1]):
        figs, axes = plt.subplots(nrows, ncols, figsize=figsize, squeeze=False)

        for ax in axes.flatten():
            ax.set_xticks([])
            ax.set_yticks([])
            ax.axis('off')

        for i, (r, c) in enumerate(product(range(nrows), range(ncols))):
            if i >= shape[0]:
                break
            axes[r, c].imshow(W[i, feature_map], cmap='gray',
                              interpolation='none')
    return plt

def plot_conv_activity(layer, x, figsize=(6, 8)):
    """Plot the acitivities of a specific layer.

    Only really makes sense with layers that work 2D data (2D
    convolutional layers, 2D pooling layers ...).

    Parameters
    ----------
    layer : lasagne.layers.Layer

    x : numpy.ndarray
      Only takes one sample at a time, i.e. x.shape[0] == 1.

    """
    if x.shape[0] != 1:
        raise ValueError("Only one sample can be plotted at a time.")

    # compile theano function
    xs = T.tensor4('xs').astype(theano.config.floatX)
    get_activity = theano.function([xs], get_output(layer, xs))

    activity = get_activity(x)
    shape = activity.shape
    nrows = np.ceil(np.sqrt(shape[1])).astype(int)
    ncols = nrows

    figs, axes = plt.subplots(nrows + 1, ncols, figsize=figsize, squeeze=False)
    axes[0, ncols // 2].imshow(1 - x[0][0], cmap='gray',
                               interpolation='none')
    axes[0, ncols // 2].set_title('original')

    for ax in axes.flatten():
        ax.set_xticks([])
        ax.set_yticks([])
        ax.axis('off')

    for i, (r, c) in enumerate(product(range(nrows), range(ncols))):
        if i >= shape[1]:
            break
        ndim = activity[0][i].ndim
        if ndim != 2:
            raise ValueError("Wrong number of dimensions, image data should "
                             "have 2, instead got {}".format(ndim))
        axes[r + 1, c].imshow(-activity[0][i], cmap='gray',
                              interpolation='none')
    return plt

def occlusion_heatmap(net, x, target, square_length=7):
    """An occlusion test that checks an image for its critical parts.

    In this function, a square part of the image is occluded (i.e. set
    to 0) and then the net is tested for its propensity to predict the
    correct label. One should expect that this propensity shrinks of
    critical parts of the image are occluded. If not, this indicates
    overfitting.

    Depending on the depth of the net and the size of the image, this
    function may take awhile to finish, since one prediction for each
    pixel of the image is made.

    Currently, all color channels are occluded at the same time. Also,
    this does not really work if images are randomly distorted by the
    batch iterator.

    See paper: Zeiler, Fergus 2013

    Parameters
    ----------
    net : NeuralNet instance
      The neural net to test.

    x : np.array
      The input data, should be of shape (1, c, x, y). Only makes
      sense with image data.

    target : int
      The true value of the image. If the net makes several
      predictions, say 10 classes, this indicates which one to look
      at.

    square_length : int (default=7)
      The length of the side of the square that occludes the image.
      Must be an odd number.

    Results
    -------
    heat_array : np.array (with same size as image)
      An 2D np.array that at each point (i, j) contains the predicted
      probability of the correct class if the image is occluded by a
      square with center (i, j).

    """
    if (x.ndim != 4) or x.shape[0] != 1:
        raise ValueError("This function requires the input data to be of "
                         "shape (1, c, x, y), instead got {}".format(x.shape))
    if square_length % 2 == 0:
        raise ValueError("Square length has to be an odd number, instead "
                         "got {}.".format(square_length))

    num_classes = get_output_shape(net.layers_[-1])[1]
    img = x[0].copy()
    bs, col, s0, s1 = x.shape

    heat_array = np.zeros((s0, s1))
    pad = square_length // 2 + 1
    x_occluded = np.zeros((s1, col, s0, s1), dtype=img.dtype)
    probs = np.zeros((s0, s1, num_classes))

    # generate occluded images
    for i in range(s0):
        # batch s1 occluded images for faster prediction
        for j in range(s1):
            x_pad = np.pad(img, ((0, 0), (pad, pad), (pad, pad)), 'constant')
            x_pad[:, i:i + square_length, j:j + square_length] = 0.
            x_occluded[j] = x_pad[:, pad:-pad, pad:-pad]
        y_proba = net.predict_proba(x_occluded)
        probs[i] = y_proba.reshape(s1, num_classes)

    # from predicted probabilities, pick only those of target class
    for i in range(s0):
        for j in range(s1):
            heat_array[i, j] = probs[i, j, target]
    return heat_array

def _plot_heat_map(net, X, figsize, get_heat_image):
    if (X.ndim != 4):
        raise ValueError("This function requires the input data to be of "
                         "shape (b, c, x, y), instead got {}".format(X.shape))

    num_images = X.shape[0]
    if figsize[1] is None:
        figsize = (figsize[0], num_images * figsize[0] / 3)
    figs, axes = plt.subplots(num_images, 3, figsize=figsize)

    for ax in axes.flatten():
        ax.set_xticks([])
        ax.set_yticks([])
        ax.axis('off')

    for n in range(num_images):
        heat_img = get_heat_image(net, X[n:n + 1, :, :, :], n)

        ax = axes if num_images == 1 else axes[n]
        img = X[n, :, :, :].mean(0)
        ax[0].imshow(-img, interpolation='nearest', cmap='gray')
        ax[0].set_title('image')
        ax[1].imshow(-heat_img, interpolation='nearest', cmap='Reds')
        ax[1].set_title('critical parts')
        ax[2].imshow(-img, interpolation='nearest', cmap='gray')
        ax[2].imshow(-heat_img, interpolation='nearest', cmap='Reds',
                     alpha=0.6)
        ax[2].set_title('super-imposed')
    return plt

def plot_occlusion(net, X, target, square_length=7, figsize=(9, None)):
    """Plot which parts of an image are particularly import for the
    net to classify the image correctly.

    See paper: Zeiler, Fergus 2013

    Parameters
    ----------
    net : NeuralNet instance
      The neural net to test.

    X : numpy.array
      The input data, should be of shape (b, c, 0, 1). Only makes
      sense with image data.

    target : list or numpy.array of ints
      The true values of the image. If the net makes several
      predictions, say 10 classes, this indicates which one to look
      at. If more than one sample is passed to X, each of them needs
      its own target.

    square_length : int (default=7)
      The length of the side of the square that occludes the image.
      Must be an odd number.

    figsize : tuple (int, int)
      Size of the figure.

    Plots
    -----
    Figure with 3 subplots: the original image, the occlusion heatmap,
    and both images super-imposed.

    """
    return _plot_heat_map(
        net, X, figsize, lambda net, X, n: occlusion_heatmap(
            net, X, target[n], square_length))

def saliency_map(input, output, pred, X):
    score = -binary_crossentropy(output[:, pred], np.array([1])).sum()
    return np.abs(T.grad(score, input).eval({input: X}))

def saliency_map_net(net, X):
    input = net.layers_[0].input_var
    output = get_output(net.layers_[-1])
    pred = output.eval({input: X}).argmax(axis=1)
    return saliency_map(input, output, pred, X)[0].transpose(1, 2, 0).squeeze()

def plot_saliency(net, X, figsize=(9, None)):
    return _plot_heat_map(
        net, X, figsize, lambda net, X, n: -saliency_map_net(net, X))

def get_hex_color(layer_type):
    """
    Determines the hex color for a layer.
    :parameters:
        - layer_type : string
            Class name of the layer
    :returns:
        - color : string containing a hex color for filling block.
    """
    COLORS = ['#4A88B3', '#98C1DE', '#6CA2C8', '#3173A2', '#17649B',
              '#FFBB60', '#FFDAA9', '#FFC981', '#FCAC41', '#F29416',
              '#C54AAA', '#E698D4', '#D56CBE', '#B72F99', '#B0108D',
              '#75DF54', '#B3F1A0', '#91E875', '#5DD637', '#3FCD12']

    hashed = int(hash(layer_type)) % 5

    if "conv" in layer_type.lower():
        return COLORS[:5][hashed]
    if layer_type in lasagne.layers.pool.__all__:
        return COLORS[5:10][hashed]
    if layer_type in lasagne.layers.recurrent.__all__:
        return COLORS[10:15][hashed]
    else:
        return COLORS[15:20][hashed]

def make_pydot_graph(layers, output_shape=True, verbose=False):
    """
    :parameters:
        - layers : list
            List of the layers, as obtained from lasagne.layers.get_all_layers
        - output_shape: (default `True`)
            If `True`, the output shape of each layer will be displayed.
        - verbose: (default `False`)
            If `True`, layer attributes like filter shape, stride, etc.
            will be displayed.
    :returns:
        - pydot_graph : PyDot object containing the graph
    """

    pydot_graph = pydot.Dot('Network', graph_type='digraph')
    pydot_nodes = {}
    pydot_edges = []
    for i, layer in enumerate(layers):
        layer_type = '{0}'.format(layer.__class__.__name__)
        key = repr(layer)
        label = layer_type
        color = get_hex_color(layer_type)
        if verbose:
            for attr in ['num_filters', 'num_units', 'ds',
                         'filter_shape', 'stride', 'strides', 'p']:
                if hasattr(layer, attr):
                    label += '\n{0}: {1}'.format(attr, getattr(layer, attr))
            if hasattr(layer, 'nonlinearity'):
                try:
                    nonlinearity = layer.nonlinearity.__name__
                except AttributeError:
                    nonlinearity = layer.nonlinearity.__class__.__name__
                label += '\nnonlinearity: {0}'.format(nonlinearity)

        if output_shape:
            label += '\nOutput shape: {0}'.format(layer.output_shape)

        pydot_nodes[key] = pydot.Node(
            key, label=label, shape='record', fillcolor=color, style='filled')

        if hasattr(layer, 'input_layers'):
            for input_layer in layer.input_layers:
                pydot_edges.append([repr(input_layer), key])

        if hasattr(layer, 'input_layer'):
            pydot_edges.append([repr(layer.input_layer), key])

    for node in pydot_nodes.values():
        pydot_graph.add_node(node)

    for edges in pydot_edges:
        pydot_graph.add_edge(
            pydot.Edge(pydot_nodes[edges[0]], pydot_nodes[edges[1]]))
    return pydot_graph

def draw_to_file(layers, filename, **kwargs):
    """
    Draws a network diagram to a file
    :parameters:
        - layers : list or NeuralNet instance
            List of layers or the neural net to draw.
        - filename : string
            The filename to save output to
        - **kwargs: see docstring of make_pydot_graph for other options
    """
    layers = (layers.get_all_layers() if hasattr(layers, 'get_all_layers')
              else layers)
    dot = make_pydot_graph(layers, **kwargs)
    ext = filename[filename.rfind('.') + 1:]
    with io.open(filename, 'wb') as fid:
        fid.write(dot.create(format=ext))

def draw_to_notebook(layers, **kwargs):
    """
    Draws a network diagram in an IPython notebook
    :parameters:
        - layers : list or NeuralNet instance
            List of layers or the neural net to draw.
        - **kwargs : see the docstring of make_pydot_graph for other options
    """
    layers = (layers.get_all_layers() if hasattr(layers, 'get_all_layers')
              else layers)
    dot = make_pydot_graph(layers, **kwargs)
    return Image(dot.create_png())
<commit_msg>remove hard dependancy of pydotplus and ipython. (#279)<commit_after>from itertools import product

from lasagne.layers import get_output
from lasagne.layers import get_output_shape
from lasagne.objectives import binary_crossentropy
import matplotlib.pyplot as plt
import numpy as np
import theano
import theano.tensor as T
import io
import lasagne

def plot_loss(net):
    train_loss = [row['train_loss'] for row in net.train_history_]
    valid_loss = [row['valid_loss'] for row in net.train_history_]
    plt.plot(train_loss, label='train loss')
    plt.plot(valid_loss, label='valid loss')
    plt.xlabel('epoch')
    plt.ylabel('loss')
    plt.legend(loc='best')
    return plt

def plot_conv_weights(layer, figsize=(6, 6)):
    """Plot the weights of a specific layer.

    Only really makes sense with convolutional layers.

    Parameters
    ----------
    layer : lasagne.layers.Layer

    """
    W = layer.W.get_value()
    shape = W.shape
    nrows = np.ceil(np.sqrt(shape[0])).astype(int)
    ncols = nrows

    for feature_map in range(shape[1]):
        figs, axes = plt.subplots(nrows, ncols, figsize=figsize, squeeze=False)

        for ax in axes.flatten():
            ax.set_xticks([])
            ax.set_yticks([])
            ax.axis('off')

        for i, (r, c) in enumerate(product(range(nrows), range(ncols))):
            if i >= shape[0]:
                break
            axes[r, c].imshow(W[i, feature_map], cmap='gray',
                              interpolation='none')
    return plt

def plot_conv_activity(layer, x, figsize=(6, 8)):
    """Plot the acitivities of a specific layer.

    Only really makes sense with layers that work 2D data (2D
    convolutional layers, 2D pooling layers ...).

    Parameters
    ----------
    layer : lasagne.layers.Layer

    x : numpy.ndarray
      Only takes one sample at a time, i.e. x.shape[0] == 1.

    """
    if x.shape[0] != 1:
        raise ValueError("Only one sample can be plotted at a time.")

    # compile theano function
    xs = T.tensor4('xs').astype(theano.config.floatX)
    get_activity = theano.function([xs], get_output(layer, xs))

    activity = get_activity(x)
    shape = activity.shape
    nrows = np.ceil(np.sqrt(shape[1])).astype(int)
    ncols = nrows

    figs, axes = plt.subplots(nrows + 1, ncols, figsize=figsize, squeeze=False)
    axes[0, ncols // 2].imshow(1 - x[0][0], cmap='gray',
                               interpolation='none')
    axes[0, ncols // 2].set_title('original')

    for ax in axes.flatten():
        ax.set_xticks([])
        ax.set_yticks([])
        ax.axis('off')

    for i, (r, c) in enumerate(product(range(nrows), range(ncols))):
        if i >= shape[1]:
            break
        ndim = activity[0][i].ndim
        if ndim != 2:
            raise ValueError("Wrong number of dimensions, image data should "
                             "have 2, instead got {}".format(ndim))
        axes[r + 1, c].imshow(-activity[0][i], cmap='gray',
                              interpolation='none')
    return plt

def occlusion_heatmap(net, x, target, square_length=7):
    """An occlusion test that checks an image for its critical parts.

    In this function, a square part of the image is occluded (i.e. set
    to 0) and then the net is tested for its propensity to predict the
    correct label. One should expect that this propensity shrinks of
    critical parts of the image are occluded. If not, this indicates
    overfitting.

    Depending on the depth of the net and the size of the image, this
    function may take awhile to finish, since one prediction for each
    pixel of the image is made.

    Currently, all color channels are occluded at the same time. Also,
    this does not really work if images are randomly distorted by the
    batch iterator.

    See paper: Zeiler, Fergus 2013

    Parameters
    ----------
    net : NeuralNet instance
      The neural net to test.

    x : np.array
      The input data, should be of shape (1, c, x, y). Only makes
      sense with image data.

    target : int
      The true value of the image. If the net makes several
      predictions, say 10 classes, this indicates which one to look
      at.

    square_length : int (default=7)
      The length of the side of the square that occludes the image.
      Must be an odd number.

    Results
    -------
    heat_array : np.array (with same size as image)
      An 2D np.array that at each point (i, j) contains the predicted
      probability of the correct class if the image is occluded by a
      square with center (i, j).

    """
    if (x.ndim != 4) or x.shape[0] != 1:
        raise ValueError("This function requires the input data to be of "
                         "shape (1, c, x, y), instead got {}".format(x.shape))
    if square_length % 2 == 0:
        raise ValueError("Square length has to be an odd number, instead "
                         "got {}.".format(square_length))

    num_classes = get_output_shape(net.layers_[-1])[1]
    img = x[0].copy()
    bs, col, s0, s1 = x.shape

    heat_array = np.zeros((s0, s1))
    pad = square_length // 2 + 1
    x_occluded = np.zeros((s1, col, s0, s1), dtype=img.dtype)
    probs = np.zeros((s0, s1, num_classes))

    # generate occluded images
    for i in range(s0):
        # batch s1 occluded images for faster prediction
        for j in range(s1):
            x_pad = np.pad(img, ((0, 0), (pad, pad), (pad, pad)), 'constant')
            x_pad[:, i:i + square_length, j:j + square_length] = 0.
            x_occluded[j] = x_pad[:, pad:-pad, pad:-pad]
        y_proba = net.predict_proba(x_occluded)
        probs[i] = y_proba.reshape(s1, num_classes)

    # from predicted probabilities, pick only those of target class
    for i in range(s0):
        for j in range(s1):
            heat_array[i, j] = probs[i, j, target]
    return heat_array

def _plot_heat_map(net, X, figsize, get_heat_image):
    if (X.ndim != 4):
        raise ValueError("This function requires the input data to be of "
                         "shape (b, c, x, y), instead got {}".format(X.shape))

    num_images = X.shape[0]
    if figsize[1] is None:
        figsize = (figsize[0], num_images * figsize[0] / 3)
    figs, axes = plt.subplots(num_images, 3, figsize=figsize)

    for ax in axes.flatten():
        ax.set_xticks([])
        ax.set_yticks([])
        ax.axis('off')

    for n in range(num_images):
        heat_img = get_heat_image(net, X[n:n + 1, :, :, :], n)

        ax = axes if num_images == 1 else axes[n]
        img = X[n, :, :, :].mean(0)
        ax[0].imshow(-img, interpolation='nearest', cmap='gray')
        ax[0].set_title('image')
        ax[1].imshow(-heat_img, interpolation='nearest', cmap='Reds')
        ax[1].set_title('critical parts')
        ax[2].imshow(-img, interpolation='nearest', cmap='gray')
        ax[2].imshow(-heat_img, interpolation='nearest', cmap='Reds',
                     alpha=0.6)
        ax[2].set_title('super-imposed')
    return plt

def plot_occlusion(net, X, target, square_length=7, figsize=(9, None)):
    """Plot which parts of an image are particularly import for the
    net to classify the image correctly.

    See paper: Zeiler, Fergus 2013

    Parameters
    ----------
    net : NeuralNet instance
      The neural net to test.

    X : numpy.array
      The input data, should be of shape (b, c, 0, 1). Only makes
      sense with image data.

    target : list or numpy.array of ints
      The true values of the image. If the net makes several
      predictions, say 10 classes, this indicates which one to look
      at. If more than one sample is passed to X, each of them needs
      its own target.

    square_length : int (default=7)
      The length of the side of the square that occludes the image.
      Must be an odd number.

    figsize : tuple (int, int)
      Size of the figure.

    Plots
    -----
    Figure with 3 subplots: the original image, the occlusion heatmap,
    and both images super-imposed.

    """
    return _plot_heat_map(
        net, X, figsize, lambda net, X, n: occlusion_heatmap(
            net, X, target[n], square_length))

def saliency_map(input, output, pred, X):
    score = -binary_crossentropy(output[:, pred], np.array([1])).sum()
    return np.abs(T.grad(score, input).eval({input: X}))

def saliency_map_net(net, X):
    input = net.layers_[0].input_var
    output = get_output(net.layers_[-1])
    pred = output.eval({input: X}).argmax(axis=1)
    return saliency_map(input, output, pred, X)[0].transpose(1, 2, 0).squeeze()

def plot_saliency(net, X, figsize=(9, None)):
    return _plot_heat_map(
        net, X, figsize, lambda net, X, n: -saliency_map_net(net, X))

def get_hex_color(layer_type):
    """
    Determines the hex color for a layer.
    :parameters:
        - layer_type : string
            Class name of the layer
    :returns:
        - color : string containing a hex color for filling block.
    """
    COLORS = ['#4A88B3', '#98C1DE', '#6CA2C8', '#3173A2', '#17649B',
              '#FFBB60', '#FFDAA9', '#FFC981', '#FCAC41', '#F29416',
              '#C54AAA', '#E698D4', '#D56CBE', '#B72F99', '#B0108D',
              '#75DF54', '#B3F1A0', '#91E875', '#5DD637', '#3FCD12']

    hashed = int(hash(layer_type)) % 5

    if "conv" in layer_type.lower():
        return COLORS[:5][hashed]
    if layer_type in lasagne.layers.pool.__all__:
        return COLORS[5:10][hashed]
    if layer_type in lasagne.layers.recurrent.__all__:
        return COLORS[10:15][hashed]
    else:
        return COLORS[15:20][hashed]

def make_pydot_graph(layers, output_shape=True, verbose=False):
    """
    :parameters:
        - layers : list
            List of the layers, as obtained from lasagne.layers.get_all_layers
        - output_shape: (default `True`)
            If `True`, the output shape of each layer will be displayed.
        - verbose: (default `False`)
            If `True`, layer attributes like filter shape, stride, etc.
            will be displayed.
    :returns:
        - pydot_graph : PyDot object containing the graph
    """
    import pydotplus as pydot
    pydot_graph = pydot.Dot('Network', graph_type='digraph')
    pydot_nodes = {}
    pydot_edges = []
    for i, layer in enumerate(layers):
        layer_type = '{0}'.format(layer.__class__.__name__)
        key = repr(layer)
        label = layer_type
        color = get_hex_color(layer_type)
        if verbose:
            for attr in ['num_filters', 'num_units', 'ds',
                         'filter_shape', 'stride', 'strides', 'p']:
                if hasattr(layer, attr):
                    label += '\n{0}: {1}'.format(attr, getattr(layer, attr))
            if hasattr(layer, 'nonlinearity'):
                try:
                    nonlinearity = layer.nonlinearity.__name__
                except AttributeError:
                    nonlinearity = layer.nonlinearity.__class__.__name__
                label += '\nnonlinearity: {0}'.format(nonlinearity)

        if output_shape:
            label += '\nOutput shape: {0}'.format(layer.output_shape)

        pydot_nodes[key] = pydot.Node(
            key, label=label, shape='record', fillcolor=color, style='filled')

        if hasattr(layer, 'input_layers'):
            for input_layer in layer.input_layers:
                pydot_edges.append([repr(input_layer), key])

        if hasattr(layer, 'input_layer'):
            pydot_edges.append([repr(layer.input_layer), key])

    for node in pydot_nodes.values():
        pydot_graph.add_node(node)

    for edges in pydot_edges:
        pydot_graph.add_edge(
            pydot.Edge(pydot_nodes[edges[0]], pydot_nodes[edges[1]]))
    return pydot_graph

def draw_to_file(layers, filename, **kwargs):
    """
    Draws a network diagram to a file
    :parameters:
        - layers : list or NeuralNet instance
            List of layers or the neural net to draw.
        - filename : string
            The filename to save output to
        - **kwargs: see docstring of make_pydot_graph for other options
    """
    layers = (layers.get_all_layers() if hasattr(layers, 'get_all_layers')
              else layers)
    dot = make_pydot_graph(layers, **kwargs)
    ext = filename[filename.rfind('.') + 1:]
    with io.open(filename, 'wb') as fid:
        fid.write(dot.create(format=ext))

def draw_to_notebook(layers, **kwargs):
    """
    Draws a network diagram in an IPython notebook
    :parameters:
        - layers : list or NeuralNet instance
            List of layers or the neural net to draw.
        - **kwargs : see the docstring of make_pydot_graph for other options
    """
    from IPython.display import Image
    layers = (layers.get_all_layers() if hasattr(layers, 'get_all_layers')
              else layers)
    dot = make_pydot_graph(layers, **kwargs)
    return Image(dot.create_png())
<|endoftext|>


<commit_before>#!/usr/bin/env python
#-*- coding: utf-8 -*-

from botinfo import *
from math import sqrt, sin, cos, tan, pow, exp, pi, ceil, floor, e, factorial, log, log2
from math import radians, degrees, acos, asin, atan, atan2, erf, gamma, gcd
from bs4 import BeautifulSoup as BS
from requests import get as re_get
from random import randint
from os import mkdir
from os.path import isdir, join

bot_name = "dumb-bot"
client = commands.Bot(command_prefix=".", description="A bot that doesn't do a whole lot")
logger = create_logger(bot_name)

@client.event
async def on_ready():
    """
    The function that is consumed upon startup
    Set vars, check for bot folder existence, etc
    """
    if not setup_bot_data(bot_name, logger):
        logger("Failed to set up {}'s folder".format(bot_name))
        client.close()
    return logger("Connection status: {}".format(client.is_logged_in))

@client.command()
async def source():
    """Print out a link to the source code"""
    return await client.say("https://github.com/sleibrock/discord-bots")

@client.command()
async def commits():
    """Print out a `git log --graph --decorate=full --oneline | head -n 5`"""
    return await client.say(pre_text(call("git log --decorate=full --graph --oneline | head -n 5")))

@client.command()
async def authors():
    """Return a `git shortlog -sn` output to print authors"""
    return await client.say(pre_text(call("git shortlog -sn")))

@client.command()
async def update():
    """
    Execute a `git pull` to update the code
    If there was a successful pull, dumb-bot will restart his own thread
    """
    # TODO: detect files updated and kill their respected threads?
    result = call("git pull")
    await client.say(pre_text(result))
    if result.strip() == "Already up-to-date.": # figure out something non-string-comp
        return
    await client.say("Upgrading myself senpai")
    client.close()
    return logger("Killing own thread for upgrade")

@client.command()
async def uname():
    """Return the system info of the host"""
    return await client.say(pre_text(call("uname -a")))

@client.command()
async def uptime():
    """Return the uptime of the host machine"""
    return await client.say(pre_text(call(["uptime"])))

@client.command()
async def free():
    """See how much memory is available"""
    return await client.say(pre_text(call(["free"])))

@client.command()
async def servers():
    """See the servers the bot is connected to"""
    return await client.say(pre_text(", ".join(map(str, client.servers))))

@client.command()
async def comp(*expression):
    """
    Evaluate an expression with Python syntax
    Non-expressions like keywords are not allowed

    Math functions:
     * pow, exp, sqrt, ceil, floor, sin, cos, tan, factorial
     * radians, degrees, acos, asin, atan, atan2, erf, gamma, gcd
    """
    s = " ".join(expression)
    if len(expression) > 80:
        return await client.say("Input was too large, sorry")
    try:
        # TODO: add some kind of timeout condition to prevent inf recursion
        output = eval(s)
        if len(output) > 100:
            await client.say("Output was too large, sorry")
        else:
            await client.say(pre_text(">>> {}".format(output)))
    except Exception as ex:
        await client.say("Error: {}".format(ex))
    return

@client.command()
async def rtd(*string):
    """
    Roll a d<N> die <X> number of times
    Example: .rtd 2d10 - rolls two d10 dice
    """
    dice_str = "".join(string).strip()
    logger(dice_str)
    if dice_str == "":
        return await client.say("You didn't say anything!")
    try:
        times, sides = list(map(int, dice_str.lower().split("d")))
        logger("roll {} d{}".format(times, sides))
        res = [randint(1, sides) for x in range(times)]
        return await client.say(", ".join(map(str, res)))
    except Exception as ex:
        logger("Error: {}".format(ex))
    return await client.say("Error: bad input arg")

@client.command()
async def ddg(*search):
    """
    Search DuckDuckGo and post the first result
    Example: .ddg let me google that for you
    """
    try:
        search_str = " ".join(search)
        if search_str.strip() == "":
            return await client.say("You didn't search for anything!")
        search_str.replace(" ", "%20") # replace spaces
        url = "https://duckduckgo.com/html/?q={0}".format(search_str)
        bs = BS(re_get(url).text, "html.parser")
        results = bs.find_all("div", class_="web-result")
        if not results:
            return await client.say("Couldn't find anything")
        a = results[0].find("a", class_="result__a")
        title = a.text
        link = a["href"]
        return await client.say("{} - {}".format(title, link))
    except Exception as ex:
        logger("Fail: {}".format(ex))
    return await client.say("Failed to get the search")

@client.command()
async def yt(*search):
    """
    Do a youtube search and yield the first result
    Example: .yt how do I take a screenshot
    """
    try:
        search_str = " ".join(search).lower()
        if search_str.strip() == "":
            return await client.say("You didn't search for anything!")
        search_str.replace(" ", "+")
        url = "https://www.youtube.com/results?search_query={}".format(search_str)
        bs = BS(re_get(url).text, "html.parser")
        items = bs.find("div", id="results").find_all("div", class_="yt-lockup-content")
        if not items:
            return await client.say("Couldn't find any results")
        href = items[0].find("a", class_="yt-uix-sessionlink")["href"]

        # Check if the first result was a Google Adsense link
        if not href.startswith("/watch"):
            href = items[1].find("a", class_="yt-uix-sessionlink")["href"]
        return await client.say("https://youtube.com{}".format(href))
    except Exception as ex:
        logger("Fail: {}".format(ex))
    return await client.say("Failed to request the search")

if __name__ == "__main__":
    try:
        argv.pop(0)
        key = argv.pop(0)
        client.run(read_key(key))
    except Exception as e:
        logger("Whoops! {}".format(e))
    except SystemExit:
        logger("Leaving this existence behind")
    except KeyboardInterrupt:
        logger("Ouch!")
    quit()

# end

<commit_msg>Removing evaluation command<commit_after>#!/usr/bin/env python
#-*- coding: utf-8 -*-

from botinfo import *
from math import sqrt, sin, cos, tan, pow, exp, pi, ceil, floor, e, factorial, log, log2
from math import radians, degrees, acos, asin, atan, atan2, erf, gamma, gcd
from bs4 import BeautifulSoup as BS
from requests import get as re_get
from random import randint
from os import mkdir
from os.path import isdir, join

bot_name = "dumb-bot"
client = commands.Bot(command_prefix=".", description="A bot that doesn't do a whole lot")
logger = create_logger(bot_name)

@client.event
async def on_ready():
    """
    The function that is consumed upon startup
    Set vars, check for bot folder existence, etc
    """
    if not setup_bot_data(bot_name, logger):
        logger("Failed to set up {}'s folder".format(bot_name))
        client.close()
    return logger("Connection status: {}".format(client.is_logged_in))

@client.command()
async def source():
    """Print out a link to the source code"""
    return await client.say("https://github.com/sleibrock/discord-bots")

@client.command()
async def commits():
    """Print out a `git log --graph --decorate=full --oneline | head -n 5`"""
    return await client.say(pre_text(call("git log --decorate=full --graph --oneline | head -n 5")))

@client.command()
async def authors():
    """Return a `git shortlog -sn` output to print authors"""
    return await client.say(pre_text(call("git shortlog -sn")))

@client.command()
async def update():
    """
    Execute a `git pull` to update the code
    If there was a successful pull, dumb-bot will restart his own thread
    """
    # TODO: detect files updated and kill their respected threads?
    result = call("git pull")
    await client.say(pre_text(result))
    if result.strip() == "Already up-to-date.": # figure out something non-string-comp
        return
    await client.say("Upgrading myself senpai")
    client.close()
    return logger("Killing own thread for upgrade")

@client.command()
async def uname():
    """Return the system info of the host"""
    return await client.say(pre_text(call("uname -a")))

@client.command()
async def uptime():
    """Return the uptime of the host machine"""
    return await client.say(pre_text(call(["uptime"])))

@client.command()
async def free():
    """See how much memory is available"""
    return await client.say(pre_text(call(["free"])))

@client.command()
async def servers():
    """See the servers the bot is connected to"""
    return await client.say(pre_text(", ".join(map(str, client.servers))))

@client.command()
async def rtd(*string):
    """
    Roll a d<N> die <X> number of times
    Example: .rtd 2d10 - rolls two d10 dice
    """
    dice_str = "".join(string).strip()
    logger(dice_str)
    if dice_str == "":
        return await client.say("You didn't say anything!")
    try:
        times, sides = list(map(int, dice_str.lower().split("d")))
        logger("roll {} d{}".format(times, sides))
        res = [randint(1, sides) for x in range(times)]
        return await client.say(", ".join(map(str, res)))
    except Exception as ex:
        logger("Error: {}".format(ex))
    return await client.say("Error: bad input arg")

@client.command()
async def ddg(*search):
    """
    Search DuckDuckGo and post the first result
    Example: .ddg let me google that for you
    """
    try:
        search_str = " ".join(search)
        if search_str.strip() == "":
            return await client.say("You didn't search for anything!")
        search_str.replace(" ", "%20") # replace spaces
        url = "https://duckduckgo.com/html/?q={0}".format(search_str)
        bs = BS(re_get(url).text, "html.parser")
        results = bs.find_all("div", class_="web-result")
        if not results:
            return await client.say("Couldn't find anything")
        a = results[0].find("a", class_="result__a")
        title = a.text
        link = a["href"]
        return await client.say("{} - {}".format(title, link))
    except Exception as ex:
        logger("Fail: {}".format(ex))
    return await client.say("Failed to get the search")

@client.command()
async def yt(*search):
    """
    Do a youtube search and yield the first result
    Example: .yt how do I take a screenshot
    """
    try:
        search_str = " ".join(search).lower()
        if search_str.strip() == "":
            return await client.say("You didn't search for anything!")
        search_str.replace(" ", "+")
        url = "https://www.youtube.com/results?search_query={}".format(search_str)
        bs = BS(re_get(url).text, "html.parser")
        items = bs.find("div", id="results").find_all("div", class_="yt-lockup-content")
        if not items:
            return await client.say("Couldn't find any results")
        href = items[0].find("a", class_="yt-uix-sessionlink")["href"]

        # Check if the first result was a Google Adsense link
        if not href.startswith("/watch"):
            href = items[1].find("a", class_="yt-uix-sessionlink")["href"]
        return await client.say("https://youtube.com{}".format(href))
    except Exception as ex:
        logger("Fail: {}".format(ex))
    return await client.say("Failed to request the search")

if __name__ == "__main__":
    try:
        argv.pop(0)
        key = argv.pop(0)
        client.run(read_key(key))
    except Exception as e:
        logger("Whoops! {}".format(e))
    except SystemExit:
        logger("Leaving this existence behind")
    except KeyboardInterrupt:
        logger("Ouch!")
    quit()

# end

<|endoftext|>


<commit_before>import boto3

from noopy.utils import to_pascal_case
from noopy.endpoint import Endpoint

def endpoint(path, method):
    def decorator(func):
        import settings
        lambda_settings = settings.LAMBDA
        client = boto3.client('lambda')
        function_prefix = 'arn:aws:lambda:{}:{}:{}'.format(
                client._client_config.region_name,
                settings.ACCOUNT_ID,
                lambda_settings['Prefix']
        )
        func.name_for_lambda = '{}{}'.format(function_prefix, to_pascal_case(func.func_name))
        endpoint_ = Endpoint(path, method)
        Endpoint.endpoints[endpoint_] = func
        return func

    return decorator
<commit_msg>Function prefix for lambda has ":function" after account id<commit_after>import boto3

from noopy.utils import to_pascal_case
from noopy.endpoint import Endpoint

def endpoint(path, method):
    def decorator(func):
        import settings
        lambda_settings = settings.LAMBDA
        client = boto3.client('lambda')
        function_prefix = 'arn:aws:lambda:{}:{}:function:{}'.format(
                client._client_config.region_name,
                settings.ACCOUNT_ID,
                lambda_settings['Prefix']
        )
        func.name_for_lambda = '{}{}'.format(function_prefix, to_pascal_case(func.func_name))
        endpoint_ = Endpoint(path, method)
        Endpoint.endpoints[endpoint_] = func
        return func

    return decorator
<|endoftext|>


<commit_before>#Alexa
Security_Profile_Description = os.environ['SECURITY_PROFILE_DESCRIPTION']
Security_Profile_ID  = os.environ['SECURITY_PROFILE_ID']
Client_ID = os.environ['CLIENT_ID']
Client_Secret = os.environ['CLIENT_SECRET']

#Facebook
Facebook_Token= os.environ['FACEBOOK_TOKEN']

#wit.ai
Wit_Token= os.environ['WIT_TOKEN']

#Redis
redis_url = os.environ['REDIS_URL']
<commit_msg>import os<commit_after>import os

#Alexa
Security_Profile_Description = os.environ['SECURITY_PROFILE_DESCRIPTION']
Security_Profile_ID  = os.environ['SECURITY_PROFILE_ID']
Client_ID = os.environ['CLIENT_ID']
Client_Secret = os.environ['CLIENT_SECRET']

#Facebook
Facebook_Token= os.environ['FACEBOOK_TOKEN']

#wit.ai
Wit_Token= os.environ['WIT_TOKEN']

#Redis
redis_url = os.environ['REDIS_URL']
<|endoftext|>


<commit_before>import json

__version__ = '0.1.1'

class VersionConflictionException(Exception):
    def __init__(self, n):
        super().__init__('Version {} has already been specified'.format(n))

def database():
    def instance():
        versions = {}
        latest = -1

        def version(n=0):
            def wrapper(cls):
                if n in versions:
                    raise VersionConflictionException(n)
                versions[n] = cls
                nonlocal latest
                latest = max(latest, n)
                return cls
            return wrapper

        def record(jstr):
            jobj = json.loads(jstr)
            n = jobj['_ver']
            while n < latest:
                cls = versions[n + 1]
                cls.migrate(jobj)
                n += 1
            jobj['_ver'] = latest
            return versions[latest](jobj)

        return version, record

    return instance()
<commit_msg>[build] update version<commit_after>import json

__version__ = '0.1.2'

class VersionConflictionException(Exception):
    def __init__(self, n):
        super().__init__('Version {} has already been specified'.format(n))

def database():
    def instance():
        versions = {}
        latest = -1

        def version(n=0):
            def wrapper(cls):
                if n in versions:
                    raise VersionConflictionException(n)
                versions[n] = cls
                nonlocal latest
                latest = max(latest, n)
                return cls
            return wrapper

        def record(jstr):
            jobj = json.loads(jstr)
            n = jobj['_ver']
            while n < latest:
                cls = versions[n + 1]
                cls.migrate(jobj)
                n += 1
            jobj['_ver'] = latest
            return versions[latest](jobj)

        return version, record

    return instance()
<|endoftext|>


<commit_before>#  Copyright 2008-2011 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

from __future__ import with_statement
import os
import sys
import inspect
if sys.platform.startswith('java'):
    from java.lang.System import getProperty

from robot.errors import DataError

from .error import get_error_details
from .robotpath import abspath, normpath

# TODO:
# - don't import test library modules if library is in cache
# - test can variable files be implemented with java/python classes nowadays
#   (possibly returning class when importing by path is bwic anyway)

class Importer(object):

    def __init__(self, type=None, logger=None):
        if not logger:
            from robot.output import LOGGER as logger
        self._type = type or ''
        self._logger = logger
        self._importers = (ByPathImporter(logger),
                           NonDottedImporter(logger),
                           DottedImporter(logger))
        self._by_path_importer = self._importers[0]

    def import_class_or_module(self, name):
        """Imports Python class/module or Java class with given name.

        Class can either live in a module/package or be standalone Java class.
        In the former case the name is something like 'MyClass' and in the
        latter it could be 'your.package.YourLibrary'. Python classes always
        live in a module, but if the module name is exactly same as the class
        name then simple 'MyLibrary' will import a class.

        Python modules can be imported both using format 'MyModule' and
        'mymodule.submodule'.

        `name` can also be a path to the imported file/directory. In that case
        importing is done using `import_class_or_module_by_path` method.
        """
        try:
            imported, source = self._import_class_or_module(name)
        except DataError, err:
            self._raise_import_failed(name, err)
        else:
            self._log_import_succeeded(imported, name, source)
            return imported

    def _import_class_or_module(self, name):
        for importer in self._importers:
            if importer.handles(name):
                return importer.import_(name)

    def import_class_or_module_by_path(self, path):
        """Import a Python module or Java class using a file system path.

        When importing a Python file, the path must end with '.py' and the
        actual file must also exist. When importing a Python module implemented
        as a directory, the path must end with '/' or, on Windows, with '\\'.

        When importing Java classes, the path must end with '.java' or '.class'.
        The class file must exist in both cases and in the former case also
        the source file must exist.
        """
        try:
            imported, source = self._by_path_importer.import_(path)
        except DataError, err:
            self._raise_import_failed(path, err)
        else:
            self._log_import_succeeded(imported, imported.__name__, source)
            return imported

    def _raise_import_failed(self, name, error):
        import_type = '%s ' % self._type if self._type else ''
        msg = "Importing %s'%s' failed: %s" % (import_type, name, error.message)
        if not error.details:
            raise DataError(msg)
        msg = [msg, error.details]
        msg.extend(self._get_items_in('PYTHONPATH', sys.path))
        if sys.platform.startswith('java'):
            classpath = getProperty('java.class.path').split(os.path.pathsep)
            msg.extend(self._get_items_in('CLASSPATH', classpath))
        raise DataError('\n'.join(msg))

    def _get_items_in(self, type, items):
        yield '%s:' % type
        for item in items:
            if item:
                yield '  %s' % item

    def _log_import_succeeded(self, item, name, source):
        import_type = '%s ' % self._type if self._type else ''
        item_type = 'module' if inspect.ismodule(item) else 'class'
        location = ("'%s'" % source) if source else 'unknown location'
        self._logger.info("Imported %s%s '%s' from %s."
                          % (import_type, item_type, name, location))

class _Importer(object):

    def __init__(self, logger):
        self._logger = logger

    def _import(self, name, fromlist=None, retry=True):
        try:
            try:
                return __import__(name, fromlist=fromlist)
            except ImportError:
                # Hack to support standalone Jython:
                # http://code.google.com/p/robotframework/issues/detail?id=515
                if fromlist and retry and sys.platform.startswith('java'):
                    __import__(name)
                    return self._import(name, fromlist, retry=False)
                raise
        except:
            raise DataError(*get_error_details())

    def _verify_type(self, imported):
        if inspect.isclass(imported) or inspect.ismodule(imported):
            return imported
        raise DataError('Expected class or module, got <%s>.' % type(imported).__name__)

    def _get_class_from_module(self, module):
        klass = getattr(module, module.__name__, None)
        return klass if inspect.isclass(klass) else None

    def _get_source(self, module):
        source = getattr(module, '__file__', None)
        return abspath(source) if source else None

class ByPathImporter(_Importer):
    _valid_import_extensions = ('.py', '.java', '.class', '')

    def handles(self, path):
        return os.path.isabs(path)

    def import_(self, path):
        self._verify_import_path(path)
        self._remove_wrong_module_from_sys_modules(path)
        module = self._import_by_path(path)
        imported = self._get_class_from_module(module) or module
        return self._verify_type(imported), path

    def _verify_import_path(self, path):
        if not os.path.exists(path):
            raise DataError('File or directory does not exist.')
        if not os.path.isabs(path):
            raise DataError('Import path must be absolute.')
        if not os.path.splitext(path)[1] in self._valid_import_extensions:
            raise DataError('Not a valid file or directory to import.')

    def _remove_wrong_module_from_sys_modules(self, path):
        importing_from, name = self._split_path_to_module(path)
        importing_package = os.path.splitext(path)[1] == ''
        if self._wrong_module_imported(name, importing_from, importing_package):
            del sys.modules[name]
            self._logger.info("Removed module '%s' from sys.modules to import "
                              "fresh module." % name)

    def _split_path_to_module(self, path):
        module_dir, module_file = os.path.split(abspath(path))
        module_name = os.path.splitext(module_file)[0]
        if module_name.endswith('$py'):
            module_name = module_name[:-3]
        return module_dir, module_name

    def _wrong_module_imported(self, name, importing_from, importing_package):
        module = sys.modules.get(name)
        if not module:
            return False
        source = getattr(module, '__file__', None)
        if not source:  # play safe (occurs at least with java based modules)
            return True
        imported_from, imported_package = self._get_import_information(source)
        return ((normpath(importing_from), importing_package) !=
                (normpath(imported_from), imported_package))

    def _get_import_information(self, source):
        imported_from, imported_file = self._split_path_to_module(source)
        imported_package = imported_file == '__init__'
        if imported_package:
            imported_from = os.path.dirname(imported_from)
        return imported_from, imported_package

    def _import_by_path(self, path):
        module_dir, module_name = self._split_path_to_module(path)
        sys.path.insert(0, module_dir)
        try:
            return self._import(module_name)
        finally:
            sys.path.pop(0)

class NonDottedImporter(_Importer):

    def handles(self, name):
        return '.' not in name

    def import_(self, name):
        module = self._import(name)
        imported = self._get_class_from_module(module) or module
        return self._verify_type(imported), self._get_source(module)

class DottedImporter(_Importer):

    def handles(self, name):
        return '.' in name

    def import_(self, name):
        parent_name, lib_name = name.rsplit('.', 1)
        parent = self._import(parent_name, [str(lib_name)])
        try:
            imported = getattr(parent, lib_name)
        except AttributeError:
            raise DataError("Module '%s' does not contain '%s'."
                            % (parent_name, lib_name))
        return self._verify_type(imported), self._get_source(parent)
<commit_msg>rm todos. first one cannot be implemented and second is issue 1021.<commit_after>#  Copyright 2008-2011 Nokia Siemens Networks Oyj
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

from __future__ import with_statement
import os
import sys
import inspect
if sys.platform.startswith('java'):
    from java.lang.System import getProperty

from robot.errors import DataError

from .error import get_error_details
from .robotpath import abspath, normpath

class Importer(object):

    def __init__(self, type=None, logger=None):
        if not logger:
            from robot.output import LOGGER as logger
        self._type = type or ''
        self._logger = logger
        self._importers = (ByPathImporter(logger),
                           NonDottedImporter(logger),
                           DottedImporter(logger))
        self._by_path_importer = self._importers[0]

    def import_class_or_module(self, name):
        """Imports Python class/module or Java class with given name.

        Class can either live in a module/package or be standalone Java class.
        In the former case the name is something like 'MyClass' and in the
        latter it could be 'your.package.YourLibrary'. Python classes always
        live in a module, but if the module name is exactly same as the class
        name then simple 'MyLibrary' will import a class.

        Python modules can be imported both using format 'MyModule' and
        'mymodule.submodule'.

        `name` can also be a path to the imported file/directory. In that case
        importing is done using `import_class_or_module_by_path` method.
        """
        try:
            imported, source = self._import_class_or_module(name)
        except DataError, err:
            self._raise_import_failed(name, err)
        else:
            self._log_import_succeeded(imported, name, source)
            return imported

    def _import_class_or_module(self, name):
        for importer in self._importers:
            if importer.handles(name):
                return importer.import_(name)

    def import_class_or_module_by_path(self, path):
        """Import a Python module or Java class using a file system path.

        When importing a Python file, the path must end with '.py' and the
        actual file must also exist. When importing a Python module implemented
        as a directory, the path must end with '/' or, on Windows, with '\\'.

        When importing Java classes, the path must end with '.java' or '.class'.
        The class file must exist in both cases and in the former case also
        the source file must exist.
        """
        try:
            imported, source = self._by_path_importer.import_(path)
        except DataError, err:
            self._raise_import_failed(path, err)
        else:
            self._log_import_succeeded(imported, imported.__name__, source)
            return imported

    def _raise_import_failed(self, name, error):
        import_type = '%s ' % self._type if self._type else ''
        msg = "Importing %s'%s' failed: %s" % (import_type, name, error.message)
        if not error.details:
            raise DataError(msg)
        msg = [msg, error.details]
        msg.extend(self._get_items_in('PYTHONPATH', sys.path))
        if sys.platform.startswith('java'):
            classpath = getProperty('java.class.path').split(os.path.pathsep)
            msg.extend(self._get_items_in('CLASSPATH', classpath))
        raise DataError('\n'.join(msg))

    def _get_items_in(self, type, items):
        yield '%s:' % type
        for item in items:
            if item:
                yield '  %s' % item

    def _log_import_succeeded(self, item, name, source):
        import_type = '%s ' % self._type if self._type else ''
        item_type = 'module' if inspect.ismodule(item) else 'class'
        location = ("'%s'" % source) if source else 'unknown location'
        self._logger.info("Imported %s%s '%s' from %s."
                          % (import_type, item_type, name, location))

class _Importer(object):

    def __init__(self, logger):
        self._logger = logger

    def _import(self, name, fromlist=None, retry=True):
        try:
            try:
                return __import__(name, fromlist=fromlist)
            except ImportError:
                # Hack to support standalone Jython:
                # http://code.google.com/p/robotframework/issues/detail?id=515
                if fromlist and retry and sys.platform.startswith('java'):
                    __import__(name)
                    return self._import(name, fromlist, retry=False)
                raise
        except:
            raise DataError(*get_error_details())

    def _verify_type(self, imported):
        if inspect.isclass(imported) or inspect.ismodule(imported):
            return imported
        raise DataError('Expected class or module, got <%s>.' % type(imported).__name__)

    def _get_class_from_module(self, module):
        klass = getattr(module, module.__name__, None)
        return klass if inspect.isclass(klass) else None

    def _get_source(self, module):
        source = getattr(module, '__file__', None)
        return abspath(source) if source else None

class ByPathImporter(_Importer):
    _valid_import_extensions = ('.py', '.java', '.class', '')

    def handles(self, path):
        return os.path.isabs(path)

    def import_(self, path):
        self._verify_import_path(path)
        self._remove_wrong_module_from_sys_modules(path)
        module = self._import_by_path(path)
        imported = self._get_class_from_module(module) or module
        return self._verify_type(imported), path

    def _verify_import_path(self, path):
        if not os.path.exists(path):
            raise DataError('File or directory does not exist.')
        if not os.path.isabs(path):
            raise DataError('Import path must be absolute.')
        if not os.path.splitext(path)[1] in self._valid_import_extensions:
            raise DataError('Not a valid file or directory to import.')

    def _remove_wrong_module_from_sys_modules(self, path):
        importing_from, name = self._split_path_to_module(path)
        importing_package = os.path.splitext(path)[1] == ''
        if self._wrong_module_imported(name, importing_from, importing_package):
            del sys.modules[name]
            self._logger.info("Removed module '%s' from sys.modules to import "
                              "fresh module." % name)

    def _split_path_to_module(self, path):
        module_dir, module_file = os.path.split(abspath(path))
        module_name = os.path.splitext(module_file)[0]
        if module_name.endswith('$py'):
            module_name = module_name[:-3]
        return module_dir, module_name

    def _wrong_module_imported(self, name, importing_from, importing_package):
        module = sys.modules.get(name)
        if not module:
            return False
        source = getattr(module, '__file__', None)
        if not source:  # play safe (occurs at least with java based modules)
            return True
        imported_from, imported_package = self._get_import_information(source)
        return ((normpath(importing_from), importing_package) !=
                (normpath(imported_from), imported_package))

    def _get_import_information(self, source):
        imported_from, imported_file = self._split_path_to_module(source)
        imported_package = imported_file == '__init__'
        if imported_package:
            imported_from = os.path.dirname(imported_from)
        return imported_from, imported_package

    def _import_by_path(self, path):
        module_dir, module_name = self._split_path_to_module(path)
        sys.path.insert(0, module_dir)
        try:
            return self._import(module_name)
        finally:
            sys.path.pop(0)

class NonDottedImporter(_Importer):

    def handles(self, name):
        return '.' not in name

    def import_(self, name):
        module = self._import(name)
        imported = self._get_class_from_module(module) or module
        return self._verify_type(imported), self._get_source(module)

class DottedImporter(_Importer):

    def handles(self, name):
        return '.' in name

    def import_(self, name):
        parent_name, lib_name = name.rsplit('.', 1)
        parent = self._import(parent_name, [str(lib_name)])
        try:
            imported = getattr(parent, lib_name)
        except AttributeError:
            raise DataError("Module '%s' does not contain '%s'."
                            % (parent_name, lib_name))
        return self._verify_type(imported), self._get_source(parent)
<|endoftext|>


<commit_before>import logging
import requests
import sys
import time

import sys
root = logging.getLogger()
root.setLevel(logging.DEBUG)

ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
root.addHandler(ch)

#HOST = 'http://localhost:5000'
HOST = 'http://silver-python2.appspot.com'
response = requests.get("{}/publish".format(HOST))
task_id = response.content

SLEEP_DURATION_S = 5
logging.info("Waiting for {} seconds".format(SLEEP_DURATION_S))
time.sleep(SLEEP_DURATION_S)

response = requests.get("{}/status/{}".format(HOST, task_id))
assert(response.content == "PROCESSED")

<commit_msg>Add Task Id<commit_after>import logging
import requests
import sys
import time

import sys
root = logging.getLogger()
root.setLevel(logging.DEBUG)

ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
root.addHandler(ch)

#HOST = 'http://localhost:5000'
HOST = 'http://silver-python2.appspot.com'
response = requests.get("{}/publish".format(HOST))
task_id = response.content

logging.info("Processing task id {}".format(task_id))
SLEEP_DURATION_S = 5
logging.info("Waiting for {} seconds".format(SLEEP_DURATION_S))
time.sleep(SLEEP_DURATION_S)

response = requests.get("{}/status/{}".format(HOST, task_id))
assert(response.content == "PROCESSED")

<|endoftext|>


<commit_before># Copyright 2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Thanks to : jmathison

"""Scripted agents."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy

from pysc2.agents import base_agent
from pysc2.lib import actions
from pysc2.lib import features

_PLAYER_RELATIVE = features.SCREEN_FEATURES.player_relative.index
_PLAYER_FRIENDLY = 1
_PLAYER_NEUTRAL = 3  # beacon/minerals
_PLAYER_HOSTILE = 4
_NO_OP = actions.FUNCTIONS.no_op.id
_MOVE_SCREEN = actions.FUNCTIONS.Move_screen.id
_ATTACK_SCREEN = actions.FUNCTIONS.Attack_screen.id
_SELECT_ARMY = actions.FUNCTIONS.select_army.id
_NOT_QUEUED = [0]
_SELECT_ALL = [0]
_HAL_ADEPT = actions.FUNCTIONS.Hallucination_Adept_quick.id
_HAL_ARCHON = actions.FUNCTIONS.Hallucination_Archon_quick.id
_HAL_COL = actions.FUNCTIONS.Hallucination_Colossus_quick.id
_HAL_DISRUP = actions.FUNCTIONS.Hallucination_Disruptor_quick.id
_HAL_HIGTEM = actions.FUNCTIONS.Hallucination_HighTemplar_quick.id
_HAL_IMN = actions.FUNCTIONS.Hallucination_Immortal_quick.id
_HAL_PHOENIX = actions.FUNCTIONS.Hallucination_Phoenix_quick.id
_HAL_STALKER = actions.FUNCTIONS.Hallucination_Stalker_quick.id
_HAL_VOIDRAID = actions.FUNCTIONS.Hallucination_VoidRay_quick.id
_HAL_ZEALOT = actions.FUNCTIONS.Hallucination_Zealot_quick.id
_FORCE_FIELD = actions.FUNCTIONS.Effect_ForceField_screen.id
_GUARD_FIELD = actions.FUNCTIONS.Effect_GuardianShield_quick.id

Hallucinations = [_HAL_ADEPT, _HAL_ARCHON, _HAL_DISRUP, _HAL_HIGTEM, _HAL_IMN, _HAL_PHOENIX, _HAL_STALKER, _HAL_VOIDRAID, _HAL_ZEALOT]

class Hallucination(base_agent.BaseAgent):
  """An agent specifically for solving the ForceField map."""

  def step(self, obs):
    super(Hallucination, self).step(obs)
    if Hallucinations= in obs.observation["available_actions"]:
      player_relative = obs.observation["screen"][_PLAYER_RELATIVE]
      hellion_y, hellion_x = (player_relative == _PLAYER_HOSTILE).nonzero()
      if not hydralisk_y.any():
        return actions.FunctionCall(_NO_OP, [])
      index = numpy.argmax(hydralisk_y)
      target = [hydralisk_x[index], hydralisk_y[index]]
      return actions.FunctionCall(Hallucinations(np.random.randint(0,len(Hallucinations)), [_NOT_QUEUED])
    elif _SELECT_ARMY in obs.observation["available_actions"]:
      return actions.FunctionCall(_SELECT_ARMY, [_SELECT_ALL])
    else:
      return actions.FunctionCall(_NO_OP, [])

class SentryForceField(base_agent.BaseAgent):
  """An agent specifically for solving the ForceField map."""

  def step(self, obs):
    super(SentryForceField, self).step(obs)
    if _FORCE_FIELD= in obs.observation["available_actions"]:
      player_relative = obs.observation["screen"][_PLAYER_RELATIVE]
      hydralisk_y, hydralisk_x = (player_relative == _PLAYER_HOSTILE).nonzero()
      if not hydralisk_y.any():
        return actions.FunctionCall(_NO_OP, [])
      index = numpy.argmax(hydralisk_y)
      target = [hydralisk_x[index], hydralisk_y[index]]
      return actions.FunctionCall(_FORCE_FIELD, [_NOT_QUEUED, target])
    elif _SELECT_ARMY in obs.observation["available_actions"]:
      return actions.FunctionCall(_SELECT_ARMY, [_SELECT_ALL])
    else:
      return actions.FunctionCall(_NO_OP, [])
<commit_msg>Debug and add collaborator<commit_after># Copyright 2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Thanks to : jmathison
#             Simon0xzx

"""Scripted agents."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy

from pysc2.agents import base_agent
from pysc2.lib import actions
from pysc2.lib import features

_PLAYER_RELATIVE = features.SCREEN_FEATURES.player_relative.index
_PLAYER_FRIENDLY = 1
_PLAYER_NEUTRAL = 3  # beacon/minerals
_PLAYER_HOSTILE = 4
_NO_OP = actions.FUNCTIONS.no_op.id
_MOVE_SCREEN = actions.FUNCTIONS.Move_screen.id
_ATTACK_SCREEN = actions.FUNCTIONS.Attack_screen.id
_SELECT_ARMY = actions.FUNCTIONS.select_army.id
_NOT_QUEUED = [0]
_SELECT_ALL = [0]
_HAL_ADEPT = actions.FUNCTIONS.Hallucination_Adept_quick.id
_HAL_ARCHON = actions.FUNCTIONS.Hallucination_Archon_quick.id
_HAL_COL = actions.FUNCTIONS.Hallucination_Colossus_quick.id
_HAL_DISRUP = actions.FUNCTIONS.Hallucination_Disruptor_quick.id
_HAL_HIGTEM = actions.FUNCTIONS.Hallucination_HighTemplar_quick.id
_HAL_IMN = actions.FUNCTIONS.Hallucination_Immortal_quick.id
_HAL_PHOENIX = actions.FUNCTIONS.Hallucination_Phoenix_quick.id
_HAL_STALKER = actions.FUNCTIONS.Hallucination_Stalker_quick.id
_HAL_VOIDRAID = actions.FUNCTIONS.Hallucination_VoidRay_quick.id
_HAL_ZEALOT = actions.FUNCTIONS.Hallucination_Zealot_quick.id
_FORCE_FIELD = actions.FUNCTIONS.Effect_ForceField_screen.id
_GUARD_FIELD = actions.FUNCTIONS.Effect_GuardianShield_quick.id

Hallucinations = [_HAL_ADEPT, _HAL_ARCHON, _HAL_DISRUP, _HAL_HIGTEM, _HAL_IMN, _HAL_PHOENIX, _HAL_STALKER, _HAL_VOIDRAID, _HAL_ZEALOT]

class Hallucination(base_agent.BaseAgent):
  """An agent specifically for solving the ForceField map."""

  def step(self, obs):
    super(Hallucination, self).step(obs)
    if Hallucinations in obs.observation["available_actions"]:
      player_relative = obs.observation["screen"][_PLAYER_RELATIVE]
      hellion_y, hellion_x = (player_relative == _PLAYER_HOSTILE).nonzero()
      if not hydralisk_y.any():
        return actions.FunctionCall(_NO_OP, [])
      index = numpy.argmax(hydralisk_y)
      target = [hydralisk_x[index], hydralisk_y[index]]
      return actions.FunctionCall(Hallucinations(np.random.randint(0,len(Hallucinations)), [_NOT_QUEUED]))
    elif _SELECT_ARMY in obs.observation["available_actions"]:
      return actions.FunctionCall(_SELECT_ARMY, [_SELECT_ALL])
    else:
      return actions.FunctionCall(_NO_OP, [])

class SentryForceField(base_agent.BaseAgent):
  """An agent specifically for solving the ForceField map."""

  def step(self, obs):
    super(SentryForceField, self).step(obs)
    if _FORCE_FIELD= in obs.observation["available_actions"]:
      player_relative = obs.observation["screen"][_PLAYER_RELATIVE]
      hydralisk_y, hydralisk_x = (player_relative == _PLAYER_HOSTILE).nonzero()
      if not hydralisk_y.any():
        return actions.FunctionCall(_NO_OP, [])
      index = numpy.argmax(hydralisk_y)
      target = [hydralisk_x[index], hydralisk_y[index]]
      return actions.FunctionCall(_FORCE_FIELD, [_NOT_QUEUED, target])
    elif _SELECT_ARMY in obs.observation["available_actions"]:
      return actions.FunctionCall(_SELECT_ARMY, [_SELECT_ALL])
    else:
      return actions.FunctionCall(_NO_OP, [])
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
A python script to normalize a wide variety of value notations
from electronics engineering.

Examples of valid notations include:
    1,234.56kΩ
    1k234
    1k234Ω
    1,234.56Ω
    4µA
    4e6A
    4e6nA

Originally published at techoverflow.net.
"""
import re
import math

__author__ = "Uli Koehler"
__license__ = "CC0 1.0 Universal"
__version__ = "1.0"

# Suffices handled by the library
siSuffices = [["f"], ["p"], ["n"], ["u", "µ"], ["m"], [],
              ["k"], ["M"], ["G"], ["T"]]
siSuffixMult = -15  # The multiplier for the first suffix

# Valid unit designators. Ensure no SI suffix is added here
units = ["F", "A", "Ω", "W", "H", "C", "F", "K"]

# (Not implemented yet) Allowable Unit prefixes
unitPrefixes = ["Δ", "°"]

def getSuffixMultiplier(suffix):
    """
    For a given character, get either the multiplier or None if not found.
    The multiplier is returned as base-10 exponent integral. This avoids
    IEEE754 inaccuracies.

    >>> getSuffixMultiplier("f")
    -15
    >>> getSuffixMultiplier("k")
    3
    >>> getSuffixMultiplier("u")
    -6
    >>> getSuffixMultiplier("µ")
    -6
    >>> getSuffixMultiplier("T")
    12
    >>> getSuffixMultiplier("")
    0
    """
    if not suffix:
        return 0
    multiplier = siSuffixMult
    for suffixList in siSuffices:
        for siSuffix in suffixList:
            if siSuffix == suffix:
                return multiplier
        multiplier += 3
    return None

def isValidSuffix(suffix):
    """
    Check if the given character is a valid suffix

    >>> isValidSuffix("f")
    True
    >>> isValidSuffix("k")
    True
    >>> isValidSuffix("T")
    True
    >>> isValidSuffix("µ")
    True
    >>> isValidSuffix("B")
    False
    """
    return getSuffixMultiplier(suffix) is not None

def normalizeCommaToPoint(s):
    """
    Normalize comma to point for float conversion.
    Correctly handles thousands separators.

    Note that cases like "1,234" are undecidable between
    "1234" and "1.234". They are treated as "1.234".

    Only points and commata are potentially modified.
    Other characters and digits are not handled.

    >>> normalizeCommaToPoint("1234")
    '1234'
    >>> normalizeCommaToPoint("123.4")
    '123.4'
    >>> normalizeCommaToPoint("123,4")
    '123.4'
    >>> normalizeCommaToPoint("1,234.5")
    '1234.5'
    >>> normalizeCommaToPoint("1.234,5")
    '1234.5'
    >>> normalizeCommaToPoint("1.234,5k")
    '1234.5k'
    """
    foundComma = False
    foundPoint = False
    foundCommaFirst = False
    for ch in s:
        if ch == ",":
            # Set flag if this is the first comma and no
            # point has been encountered so far
            if not (foundPoint or foundComma):
                foundCommaFirst = True
            foundComma = True
        elif ch == ".":
            foundPoint = True
    # Found nothing or only point -> no modification required
    if not (foundComma or foundPoint) or (foundPoint and not foundComma):
        return s
    # Only comma -> replace and exit339.photobucket.com/user/ulitronic/library
    if foundComma and not foundPoint:
        return s.replace(",", ".")
    # foundComma and foundPoint
    if foundCommaFirst:  # Comma likely used as thousands separators
        # Just remove commata
        return s.replace(",", "")
    else:  # Point used as thousands separator
        return s.replace(".", "").replace(",", ".")

def splitSuffixSeparator(s):
    """
    Separate a string into a 3-tuple (number, suffix, unit).
    returns None if the string could not be parsed.

    The tuple will never contain None but empty strings if some
    element is not present. The number must be present for the string
    to be considered valid.

    Units always need to be a suffix. Instead
    Thousands separators or commata instead of points may be used
    (see normalizeCommaToPoint documentation).

    Thousands separators and suffix-as-decimal-separators may NOT
    be mixed. Whitespace is removed automatically.

    >>> splitSuffixSeparator("1234")
    ('1234', '', '')
    >>> splitSuffixSeparator("1234k")
    ('1234', 'k', '')
    >>> splitSuffixSeparator("1234kΩ")
    ('1234', 'k', 'Ω')
    >>> splitSuffixSeparator("1.234kΩ")
    ('1.234', 'k', 'Ω')
    >>> splitSuffixSeparator("1,234kΩ")
    ('1.234', 'k', 'Ω')
    >>> splitSuffixSeparator("1,234.56kΩ")
    ('1234.56', 'k', 'Ω')
    >>> splitSuffixSeparator("1k234")
    ('1.234', 'k', '')
    >>> splitSuffixSeparator("1k234Ω")
    ('1.234', 'k', 'Ω')
    >>> splitSuffixSeparator("1,234.56Ω")
    ('1234.56', '', 'Ω')
    >>> splitSuffixSeparator("1A")
    ('1', '', 'A')
    >>> splitSuffixSeparator("1")
    ('1', '', '')
    >>> splitSuffixSeparator("1k234 Ω")
    ('1.234', 'k', 'Ω')
    >>> splitSuffixSeparator("1")
    ('1', '', '')
    >>> splitSuffixSeparator("-1,234.56kΩ")
    ('-1234.56', 'k', 'Ω')
    >>> splitSuffixSeparator("-1e3kΩ")
    ('-1e3', 'k', 'Ω')
    >>> splitSuffixSeparator("1e-3kΩ")
    ('1e-3', 'k', 'Ω')
    >>> splitSuffixSeparator("-4e6nA")
    ('-4e6', 'n', 'A')
    >>> splitSuffixSeparator("1,234.56kfA")
    >>> splitSuffixSeparator("1.23k45A")
    >>> splitSuffixSeparator("")
    >>> splitSuffixSeparator("1,234k56Ω")
    >>> splitSuffixSeparator("foobar")
    >>> splitSuffixSeparator(None)
    """
    if not s:
        return None
    s = normalizeCommaToPoint(s).replace(" ", "")
    # Ensure we have at least one character
    if not s:
        return None
    # If there is a unit, it MUST be a suffix and 1 char only
    unit = s[-1] if s[-1] in units else ""
    if unit:  # Remove unit from current string
        s = s[:-1]
        # The stripped string must be non-empty
        if not s:
            return None
    # Try to find SI suffix at the end or in the middle
    if isValidSuffix(s[-1]):
        suffix = s[-1]
        s = s[:-1]
    else:  # Try to find unit anywhere
        isSuffixList = [isValidSuffix(ch) for ch in s]
        # Ensure only ONE unit occurs in the string
        suffixCount = isSuffixList.count(True)
        if suffixCount > 1:
            return None
        elif suffixCount == 0:
            suffix = ""
        else:  # suffixCount == 1 --> correct
            # Suffix-as-decimal-separator --> no other decimal separator
            if "." in s:  # Commas already handled by normalizeCommaToPoint
                return None
            suffixIndex = isSuffixList.index(True)
            # Suffix must NOT be first character
            if suffixIndex == 0:
                return None
            suffix = s[suffixIndex]
            s = s.replace(suffix, ".")
    # Final check: Is there any number left and is it valid?
    if not s:
        return None
    if not all([ch.isdigit() or ch in [".", "-", "e"] for ch in s]):
        return None
    return (s, suffix, unit)

def normalizeEngineerInput(s):
    """
    Converts an engineer's input of a wide variety of formats to a numeric
    value.

    Returns a pair (number, unit)

    See splitSuffixSeparator() for further details on supported formats
    """
    (num, suffix, unit) = splitSuffixSeparator(s)
    val = float(num) * (10 ** getSuffixMultiplier(suffix))
    return (val, unit)

def _formatWithSuffix(v, suffix):
    """
    Format a given value with a given suffix.
    This helper function formats the value to 3 visible digits.
    v must be pre-multiplied by the factor implied by the suffix

    >>> _formatWithSuffix(1.01, "A")
    '1.01 A'
    >>> _formatWithSuffix(1, "A")
    '1.00 A'
    >>> _formatWithSuffix(101, "A")
    '101 A'
    >>> _formatWithSuffix(99.9, "A")
    '99.9 A'
    """
    if v < 10:
        res = "%.2f" % v
    elif v < 100:
        res = "%.1f" % v
    else:  # Should only happen if v < 1000
        res = "%d" % int(v)
    #Avoid appending whitespace if there is no suffix
    if suffix:
        return "%s %s" % (res, suffix)
    else:
        return res

def formatValue(v, unit=""):
    """
    Format v using SI suffices with optional units.
    Produces a string with 3 visible digits.

    >>> formatValue(1.0e-15, "V")
    '1.00 fV'
    >>> formatValue(234.6789e-3, "V")
    '234 mV'
    >>> formatValue(234.6789, "V")
    '234 V'
    >>> formatValue(2345.6789, "V")
    '2.35 kV'
    >>> formatValue(2345.6789e6, "V")
    '2.35 GV'
    >>> formatValue(2345.6789e12, "V")
    '2.35 EV'
    """
    suffixMap = {
        -5: "f", -4: "p", -3: "n", -2: "μ", -1: "m",
        0: "", 1: "k", 2: "M", 3: "G", 4: "T", 5: "E"
    }
    #Suffix map is indexed by one third of the decadic logarithm.
    exp = 0 if v == 0.0 else math.log(v, 10.0)
    suffixMapIdx = int(math.floor(exp / 3.0))
    #Ensure we're in range
    if suffixMapIdx < -5:
        suffixMapIdx = -5
    elif suffixMapIdx > 5:
        suffixMapIdx = 5
    #Pre-multiply the value
    v = v * (10.0 ** -(suffixMapIdx * 3))
    #Delegate the rest of the task to the helper
    return _formatWithSuffix(v, suffixMap[suffixMapIdx] + unit)

if __name__ == "__main__":
    import doctest
    doctest.testmod()
    # Usage example
    print(normalizeEngineerInput("1µ234 Ω"))
<commit_msg>Enable parsing of multi-character units<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
A python script to normalize a wide variety of value notations
from electronics engineering.

Examples of valid notations include:
    1,234.56kΩ
    1k234
    1k234Ω
    1,234.56Ω
    4µA
    4e6A
    4e6nA

Originally published at techoverflow.net.
"""
import re
import math
import sys

__author__ = "Uli Koehler"
__license__ = "CC0 1.0 Universal"
__version__ = "1.0"

# This script will NOT work properly with Python 2.x!
if sys.version_info[0] < 3:
    print("This script requires Python version 3.x")
    sys.exit(1)

# Suffices handled by the library
siSuffices = [["f"], ["p"], ["n"], ["u", "µ"], ["m"], [],
              ["k"], ["M"], ["G"], ["T"]]
siSuffixMult = -15  # The multiplier for the first suffix

# Valid unit designators. Ensure no SI suffix is added here
units = frozenset(["F", "A", "Ω", "W", "H", "C", "F", "K", "Hz"])

# (Not implemented yet) Allowable Unit prefixes
unitPrefixes = ["Δ", "°"]

def getSuffixMultiplier(suffix):
    """
    For a given character, get either the multiplier or None if not found.
    The multiplier is returned as base-10 exponent integral. This avoids
    IEEE754 inaccuracies.

    >>> getSuffixMultiplier("f")
    -15
    >>> getSuffixMultiplier("k")
    3
    >>> getSuffixMultiplier("u")
    -6
    >>> getSuffixMultiplier("µ")
    -6
    >>> getSuffixMultiplier("T")
    12
    >>> getSuffixMultiplier("")
    0
    """
    if not suffix:
        return 0
    multiplier = siSuffixMult
    for suffixList in siSuffices:
        for siSuffix in suffixList:
            if siSuffix == suffix:
                return multiplier
        multiplier += 3
    return None

def isValidSuffix(suffix):
    """
    Check if the given character is a valid suffix

    >>> isValidSuffix("f")
    True
    >>> isValidSuffix("k")
    True
    >>> isValidSuffix("T")
    True
    >>> isValidSuffix("µ")
    True
    >>> isValidSuffix("B")
    False
    """
    return getSuffixMultiplier(suffix) is not None

def normalizeCommaToPoint(s):
    """
    Normalize comma to point for float conversion.
    Correctly handles thousands separators.

    Note that cases like "1,234" are undecidable between
    "1234" and "1.234". They are treated as "1.234".

    Only points and commata are potentially modified.
    Other characters and digits are not handled.

    >>> normalizeCommaToPoint("1234")
    '1234'
    >>> normalizeCommaToPoint("123.4")
    '123.4'
    >>> normalizeCommaToPoint("123,4")
    '123.4'
    >>> normalizeCommaToPoint("1,234.5")
    '1234.5'
    >>> normalizeCommaToPoint("1.234,5")
    '1234.5'
    >>> normalizeCommaToPoint("1.234,5k")
    '1234.5k'
    """
    foundComma = False
    foundPoint = False
    foundCommaFirst = False
    for ch in s:
        if ch == ",":
            # Set flag if this is the first comma and no
            # point has been encountered so far
            if not (foundPoint or foundComma):
                foundCommaFirst = True
            foundComma = True
        elif ch == ".":
            foundPoint = True
    # Found nothing or only point -> no modification required
    if not (foundComma or foundPoint) or (foundPoint and not foundComma):
        return s
    # Only comma -> replace and exit339.photobucket.com/user/ulitronic/library
    if foundComma and not foundPoint:
        return s.replace(",", ".")
    # foundComma and foundPoint
    if foundCommaFirst:  # Comma likely used as thousands separators
        # Just remove commata
        return s.replace(",", "")
    else:  # Point used as thousands separator
        return s.replace(".", "").replace(",", ".")

def splitSuffixSeparator(s):
    """
    Separate a string into a 3-tuple (number, suffix, unit).
    returns None if the string could not be parsed.

    The tuple will never contain None but empty strings if some
    element is not present. The number must be present for the string
    to be considered valid.

    Units always need to be a suffix. Instead
    Thousands separators or commata instead of points may be used
    (see normalizeCommaToPoint documentation).

    Thousands separators and suffix-as-decimal-separators may NOT
    be mixed. Whitespace is removed automatically.

    >>> splitSuffixSeparator("1234")
    ('1234', '', '')
    >>> splitSuffixSeparator("1234k")
    ('1234', 'k', '')
    >>> splitSuffixSeparator("1234kΩ")
    ('1234', 'k', 'Ω')
    >>> splitSuffixSeparator("1.234kΩ")
    ('1.234', 'k', 'Ω')
    >>> splitSuffixSeparator("1,234kΩ")
    ('1.234', 'k', 'Ω')
    >>> splitSuffixSeparator("1,234.56kΩ")
    ('1234.56', 'k', 'Ω')
    >>> splitSuffixSeparator("1k234")
    ('1.234', 'k', '')
    >>> splitSuffixSeparator("1k234Ω")
    ('1.234', 'k', 'Ω')
    >>> splitSuffixSeparator("1,234.56Ω")
    ('1234.56', '', 'Ω')
    >>> splitSuffixSeparator("1A")
    ('1', '', 'A')
    >>> splitSuffixSeparator("1")
    ('1', '', '')
    >>> splitSuffixSeparator("1k234 Ω")
    ('1.234', 'k', 'Ω')
    >>> splitSuffixSeparator("1")
    ('1', '', '')
    >>> splitSuffixSeparator("-1,234.56kΩ")
    ('-1234.56', 'k', 'Ω')
    >>> splitSuffixSeparator("-1e3kΩ")
    ('-1e3', 'k', 'Ω')
    >>> splitSuffixSeparator("1e-3kΩ")
    ('1e-3', 'k', 'Ω')
    >>> splitSuffixSeparator("-4e6nA")
    ('-4e6', 'n', 'A')
    >>> splitSuffixSeparator("3.2 MHz")
    ('3.2', 'M', 'Hz')
    >>> splitSuffixSeparator("1,234.56kfA")
    >>> splitSuffixSeparator("1.23k45A")
    >>> splitSuffixSeparator("")
    >>> splitSuffixSeparator("1,234k56Ω")
    >>> splitSuffixSeparator("foobar")
    >>> splitSuffixSeparator(None)
    """
    if not s:
        return None
    s = normalizeCommaToPoint(s).replace(" ", "")
    # Ensure we have at least one character
    if not s:
        return None
    # Handle 2-character units (MUST be a suffix)
    if len(s) > 2 and s[-2:] in units:
        unit = s[-2:]
        s = s[:-2]
    else: #Handle 1-char units
        # If this is executed, the unit MUST be a suffix and 1 char only
        unit = s[-1] if s[-1] in units else ""
        if unit: # Strip unit from string
            s = s[:-1]
    # The string with possibly the unit removed must be non-empty
    if not s:
        return None
    # Try to find SI suffix at the end or in the middle
    if isValidSuffix(s[-1]):
        suffix = s[-1]
        s = s[:-1]
    else:  # Try to find unit anywhere
        isSuffixList = [isValidSuffix(ch) for ch in s]
        # Ensure only ONE unit occurs in the string
        suffixCount = isSuffixList.count(True)
        if suffixCount > 1:
            return None
        elif suffixCount == 0:
            suffix = ""
        else:  # suffixCount == 1 --> correct
            # Suffix-as-decimal-separator --> no other decimal separator
            if "." in s:  # Commas already handled by normalizeCommaToPoint
                return None
            suffixIndex = isSuffixList.index(True)
            # Suffix must NOT be first character
            if suffixIndex == 0:
                return None
            suffix = s[suffixIndex]
            s = s.replace(suffix, ".")
    # Final check: Is there any number left and is it valid?
    if not s:
        return None
    if not all([ch.isdigit() or ch in [".", "-", "e"] for ch in s]):
        return None
    return (s, suffix, unit)

def normalizeEngineerInput(s):
    """
    Converts an engineer's input of a wide variety of formats to a numeric
    value.

    Returns a pair (number, unit)

    See splitSuffixSeparator() for further details on supported formats
    """
    (num, suffix, unit) = splitSuffixSeparator(s)
    val = float(num) * (10 ** getSuffixMultiplier(suffix))
    return (val, unit)

def _formatWithSuffix(v, suffix):
    """
    Format a given value with a given suffix.
    This helper function formats the value to 3 visible digits.
    v must be pre-multiplied by the factor implied by the suffix

    >>> _formatWithSuffix(1.01, "A")
    '1.01 A'
    >>> _formatWithSuffix(1, "A")
    '1.00 A'
    >>> _formatWithSuffix(101, "A")
    '101 A'
    >>> _formatWithSuffix(99.9, "A")
    '99.9 A'
    """
    if v < 10:
        res = "%.2f" % v
    elif v < 100:
        res = "%.1f" % v
    else:  # Should only happen if v < 1000
        res = "%d" % int(v)
    #Avoid appending whitespace if there is no suffix
    if suffix:
        return "%s %s" % (res, suffix)
    else:
        return res

def formatValue(v, unit=""):
    """
    Format v using SI suffices with optional units.
    Produces a string with 3 visible digits.

    >>> formatValue(1.0e-15, "V")
    '1.00 fV'
    >>> formatValue(234.6789e-3, "V")
    '234 mV'
    >>> formatValue(234.6789, "V")
    '234 V'
    >>> formatValue(2345.6789, "V")
    '2.35 kV'
    >>> formatValue(2345.6789e6, "V")
    '2.35 GV'
    >>> formatValue(2345.6789e12, "V")
    '2.35 EV'
    """
    suffixMap = {
        -5: "f", -4: "p", -3: "n", -2: "μ", -1: "m",
        0: "", 1: "k", 2: "M", 3: "G", 4: "T", 5: "E"
    }
    #Suffix map is indexed by one third of the decadic logarithm.
    exp = 0 if v == 0.0 else math.log(v, 10.0)
    suffixMapIdx = int(math.floor(exp / 3.0))
    #Ensure we're in range
    if suffixMapIdx < -5:
        suffixMapIdx = -5
    elif suffixMapIdx > 5:
        suffixMapIdx = 5
    #Pre-multiply the value
    v = v * (10.0 ** -(suffixMapIdx * 3))
    #Delegate the rest of the task to the helper
    return _formatWithSuffix(v, suffixMap[suffixMapIdx] + unit)

if __name__ == "__main__":
    import doctest
    doctest.testmod()
    # Usage example
    print(normalizeEngineerInput("1µ234 Ω"))
<|endoftext|>


<commit_before>import os
import sys
import time
import random
from subprocess import Popen
import tweepy
from config import TWITTER_SETTINGS, GROOVESHARK_SETTINGS, APP_SETTINGS
import grooveshark

mpg123_path = APP_SETTINGS['mpg123_path']

class TabbyPlayer():
    def __init__(self):
        self.api = None
        self.proc = None
        self.tag = None
        self.mention_from = None
        self.in_reply_to_status_id = None
        self.artist = None
        self.title = None

    def tabbyLogger(self, msg, level='INFO'):
        '''
        Python logging module sucks, I only print simple messages
        with timestamp in console
        '''
        timestamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        print '[%s %s] %s' % (level, timestamp, msg)

    def tweetCurrentSong(self):
        '''
        Post tweet of current playing song
        '''
        tweet = None
        try:
            verbs =['dancing','jamming', 'shaking']
            tweet = '@' + self.mention_from + ' I am ' + verbs[random.randint(0,2)] + ' to ' + self.title + ' by ' + self.artist + ' right meow!'
            self.api.update_status(tweet, self.in_reply_to_status_id)
        except Exception, e:
            self.tabbyLogger(str(e), 'ERROR')
            return
        self.tabbyLogger(tweet)

    def play(self, songUrl):
        '''
        Play song with mpg123 asynchronously
        '''
        self.proc = Popen([mpg123_path, songUrl])
        self.tabbyLogger('Playing...' + songUrl)

    def isPlaying(self):
        '''
        Return True when there is music playing
        '''
        # process not started yet
        if self.proc is None:
            return False

        retcode = self.proc.poll()

        # process not finished, meaning music is playing
        if retcode is None:
            return True

        # return code received, meaning music is finished playing
        return False

    def parseTweet(self, tweet):
        '''
        Parse tweet and send query to GrooveShark
        '''
        if not tweet:
            return

        # split that tweet into list
        cmd = tweet.text.lower().split()

        # pass tweet and only match the following command(s)
        # play song: '@wktabby play <song/musician name>'
        if cmd[1] == 'play':
            query = ' '.join(cmd[2:])
            self.tabbyLogger(query)
            self.mention_from = tweet.user.screen_name
            self.in_reply_to_status_id = tweet.id
            self.grooveStream(query)

    def grooveStream(self, query):
        '''
        Retrive URL, artist, title with query from GrooveShark
        '''
        grooveshark.init(GROOVESHARK_SETTINGS['key'], GROOVESHARK_SETTINGS['secret'])
        songUrl, self.artist, self.title = grooveshark.get_stream_from_query(query)
        if songUrl:
            self.play(songUrl)
            self.tweetCurrentSong()
        else:
            tweet = '@' + self.mention_from + ' Sorry I didn\'t find ' + '"' + query + '"'
            try:
                self.api.update_status(tweet, self.in_reply_to_status_id)
            except Exception, e:
                self.tabbyLogger(str(e), 'ERROR')
                return
            self.tabbyLogger(tweet)

    def playLastestMentionSong(self, tweet):
        '''
        Parse tweet and play song
        '''
        self.tabbyLogger(':playLastestMentionSong')
        if self.api == None:
            self.api = tweepy.API(auth)
        self.parseTweet(tweet)

class TwitterStreamListener(tweepy.StreamListener):
    def on_status(self, status):
        print('@' + status.user.screen_name + ' ' + status.text)
        player = TabbyPlayer()
        player.playLastestMentionSong(status)

    def on_error(self, status_code):
        print >> sys.stderr, 'Encountered error with status code:', status_code
        return True # Don't kill the stream

    def on_timeout(self):
        print >> sys.stderr, 'Timeout...'
        return True # Don't kill the stream

if __name__ == '__main__':

    ### 1. Authentication setup ###

    auth = tweepy.OAuthHandler(TWITTER_SETTINGS['consumer_key'], TWITTER_SETTINGS['consumer_secret'])
    auth.set_access_token(TWITTER_SETTINGS['access_token_key'], TWITTER_SETTINGS['access_token_secret'])

    ### 2. Listener to search and process twitter @mention ###

    sapi = tweepy.streaming.Stream(auth, TwitterStreamListener())
    sapi.filter(track=['@wktabby'])
<commit_msg>only play one song at a time<commit_after>import os
import sys
import time
import random
from subprocess import Popen
import tweepy
from config import TWITTER_SETTINGS, GROOVESHARK_SETTINGS, APP_SETTINGS
import grooveshark

mpg123_path = APP_SETTINGS['mpg123_path']

class TabbyPlayer():
    def __init__(self):
        self.api = None
        self.proc = None
        self.tag = None
        self.mention_from = None
        self.in_reply_to_status_id = None
        self.artist = None
        self.title = None

    @staticmethod
    def tabbyLogger(msg, level='INFO'):
        '''
        Python logging module sucks, I only print simple messages
        with timestamp in console
        '''
        timestamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        print '[%s %s] %s' % (level, timestamp, msg)

    def tweetCurrentSong(self):
        '''
        Post tweet of current playing song
        '''
        tweet = None
        try:
            verbs =['dancing','jamming', 'shaking']
            tweet = '@' + self.mention_from + ' I am ' + verbs[random.randint(0,2)] + ' to ' + self.title + ' by ' + self.artist + ' right meow!'
            self.api.update_status(tweet, self.in_reply_to_status_id)
        except Exception, e:
            self.tabbyLogger(str(e), 'ERROR')
            return
        self.tabbyLogger('SEND: ' + tweet)

    def play(self, songUrl):
        '''
        Play song with mpg123 asynchronously
        '''
        self.proc = Popen([mpg123_path, songUrl])
        self.tabbyLogger('Playing...' + songUrl)

    def isPlaying(self):
        '''
        Return True when there is music playing
        '''
        # process not started yet
        if self.proc is None:
            return False

        retcode = self.proc.poll()

        # process not finished, meaning music is playing
        if retcode is None:
            return True

        # return code received, meaning music is finished playing
        return False

    def parseTweet(self, tweet):
        '''
        Parse tweet and send query to GrooveShark
        '''
        if not tweet:
            return

        # split that tweet into list
        cmd = tweet.text.lower().split()

        # pass tweet and only match the following command(s)
        # play song: '@wktabby play <song/musician name>'
        if cmd[1] == 'play':
            query = ' '.join(cmd[2:])
            self.tabbyLogger('SEARCH: ' + query)
            self.mention_from = tweet.user.screen_name
            self.in_reply_to_status_id = tweet.id
            self.grooveStream(query)

    def grooveStream(self, query):
        '''
        Retrive URL, artist, title with query from GrooveShark
        '''
        grooveshark.init(GROOVESHARK_SETTINGS['key'], GROOVESHARK_SETTINGS['secret'])
        songUrl, self.artist, self.title = grooveshark.get_stream_from_query(query)
        if songUrl:
            self.play(songUrl)
            self.tweetCurrentSong()
        else:
            tweet = '@' + self.mention_from + ' Sorry I didn\'t find ' + '"' + query + '"'
            try:
                self.api.update_status(tweet, self.in_reply_to_status_id)
            except Exception, e:
                self.tabbyLogger(str(e), 'ERROR')
                return
            self.tabbyLogger('SEND: ' + tweet)

    def playLastestMentionSong(self, tweet):
        '''
        Parse tweet and play song
        '''
        # only check @mentions when player is idle
        if self.isPlaying():
            self.tabbyLogger('Song is playing...')
            return

        if self.api == None:
            self.api = tweepy.API(auth)
        self.parseTweet(tweet)

class TwitterStreamListener(tweepy.StreamListener):
    # class variable
    player = None

    def on_status(self, status):
        TabbyPlayer.tabbyLogger('RECEIVE: '+ '@' + status.user.screen_name + ' ' + status.text)
        if self.player is None:
            self.player = TabbyPlayer()
        self.player.playLastestMentionSong(status)

    def on_error(self, status_code):
        print >> sys.stderr, 'Encountered error with status code:', status_code
        return True # Don't kill the stream

    def on_timeout(self):
        print >> sys.stderr, 'Timeout...'
        return True # Don't kill the stream

if __name__ == '__main__':

    ### 1. Authentication setup ###

    auth = tweepy.OAuthHandler(TWITTER_SETTINGS['consumer_key'], TWITTER_SETTINGS['consumer_secret'])
    auth.set_access_token(TWITTER_SETTINGS['access_token_key'], TWITTER_SETTINGS['access_token_secret'])

    ### 2. Listener to search and process twitter @mention ###

    sapi = tweepy.streaming.Stream(auth, TwitterStreamListener())
    sapi.filter(track=['@wktabby'])
<|endoftext|>