<commit_before>import numpy as np
from scipy import optimize

__author__ = 'John Bucknam & Bill Clark'

class ForwardNN(object):
    """A feedforward neural network that capable of regression or classification.

    Requires Numpy to run and takes in ndarray inputs.

    """
    # CONSTANTS
    SUM_OF_SQUARES_TYPE = "Sum of Squares"
    CROSS_ENTROPY_TYPE = "Cross Entropy"

    #Variables
    inputLayerSize = None
    outputLayerSize = None
    hiddenLayerSizes = None
    weight = None

    def __init__(self, layer_sizes):
        """Constructor

        Params:
            tuple: Floats; First element is the input layer, last element is the output layer,
            and every other layer acts as an input layer
        """
        # Define each layer
        self.inputLayerSize = layer_sizes[0]
        self.outputLayerSize = layer_sizes[len(layer_sizes) - 1]
        self.hiddenLayerSizes = layer_sizes[1:len(layer_sizes) - 1]

        # Set each weight depending on the number of each layer being paired
        self.weight = []

        # If there are hidden layers
        if len(self.hiddenLayerSizes) != 0:
            # Add random weights from input layer to hidden layer 0
            self.weight.append(np.random.randn(self.inputLayerSize, self.hiddenLayerSizes[0]))
            # For each hidden layer
            for i in range(len(self.hiddenLayerSizes) - 1):
                # Add random weights between each hidden layer
                self.weight.append(np.random.randn(self.hiddenLayerSizes[i], self.hiddenLayerSizes[i + 1]))
            # Add random weights between the last hidden layer and output layer
            self.weight.append(
                np.random.randn(self.hiddenLayerSizes[len(self.hiddenLayerSizes) - 1], self.outputLayerSize))
        else:
            # Add random weights between the input layer and output layer
            self.weight.append(np.random.randn(self.inputLayerSize, self.outputLayerSize))

    def forward(self, input_matrix):
        """Feeds the input forward through the neural network

        Returns:
             ndarray: Output of our inputted matrix that is [n, 10] (n being the # of inputs)
        """
        # Values taken from each set of weights * input summed together
        self.inputSum = []
        # The inputSums inserted into the threshold function (sigmoid)
        self.threshold = []

        # Append each inputSum value matrix
        self.inputSum.append(np.dot(input_matrix, self.weight[0]))
        for i in range(len(self.weight) - 1):
            # Append each A value from the sigmoid(Z)
            self.threshold.append(self.sigmoid(self.inputSum[i]))
            self.inputSum.append(np.dot(self.threshold[i], self.weight[i + 1]))
        y_hat = self.sigmoid(self.inputSum[len(self.inputSum) - 1])
        return y_hat

    @staticmethod
    def sigmoid(z):
        """Static Sigmoid Function

        Params:
            ndarray: floats

        Returns:
            ndarray: floats
        """
        return 1 / (1 + np.exp(-(z.clip(-100, 100))))

    def sigmoid_prime(self, z):
        """Derivative of the Sigmoid Function

        Params:
            ndarray: floats

        Returns:
            ndarray: floats
        """
        return np.multiply(self.sigmoid(z), (1 - self.sigmoid(z)))

    @staticmethod
    def tanh(z):
        """Static Tanh Function

        Params:
            ndarray: floats

        Returns:
            ndarray: floats
        """
        return np.tanh(z)

    @staticmethod
    def tanh_prime(z):
        """Static Tanh Derivative Function

        Params:
            ndarray: floats

        Returns:
            ndarray: floats
        """
        return np.multiply(((2 * np.cosh(z))/(np.cosh(2 * z) + 1)), ((2 * np.cosh(z))/(np.cosh(2 * z) + 1)))

    def cost_function(self, x, y):
        """The cost function that compares the expected output with the actual output for training

        Params:
            ndarray: Input; Should match [n, InputLayerSize] where n is the number of inputs >= to 1.
            ndarray: Output; Should match [n, OutputLayerSize] where n is the number of outputs >= to 1.

            The n value for input and output should be the same.

        Returns:
            None: If invalid training data
            float: Cost of the training data
        """
        y_hat = self.forward(x)
        if y.shape[0] == x.shape[0]:
            type = self.cost_function_type(x, y)
            if type == self.SUM_OF_SQUARES_TYPE:
                return self._sum_of_squares(y, y_hat)
            elif type == self.CROSS_ENTROPY_TYPE:
                return self._cross_entropy(y, y_hat)
        return None

    def cost_function_type(self, x, y):
        if y.shape[1] == 1:
            return self.SUM_OF_SQUARES_TYPE
        elif y.shape[1] > 1:
            return self.CROSS_ENTROPY_TYPE
        else:
            return None

    def _cross_entropy(self, y, y_hat):
        """A type of cost function used for classification greater than 2.

        Returns:
            float: Cost of current neural network.
        """
        return -1 * np.sum(np.multiply(y, self.safe_log(y_hat)) + np.multiply(1 - y, self.safe_log(1 - y_hat))) / y.shape[0]

    @staticmethod
    def safe_log(x, min_val=0.0000000001):
        """A log function used to cap ndarray x when having low or nan elements

        Returns:
            ndarray: Natural log of x matrix
        """
        return np.log(x.clip(min=min_val))

    @staticmethod
    def _sum_of_squares(y, y_hat):
        """A cost function used for regression or binary classification

        Returns:
            float: Cost of current neural network.
        """
        return np.sum(np.power(y - y_hat, 2)) / y.shape[0]

    def cost_function_prime(self, x, y):
        """The derivative of the cost function

        Uses the derived cost function to retrieve the gradient for each component

        Returns:
            list: ndarray; Returns a list of matricies where each matrix represent a specific component
        """
        y_hat = self.forward(x);

        delta = []
        derived = []
        if len(self.weight) > 1:
            delta.append(np.multiply(-(y-y_hat), self.sigmoid_prime(self.inputSum[len(self.inputSum) - 1])))
            derived.append(np.dot(self.threshold[len(self.threshold) - 1].T, delta[len(delta) - 1]))

            for i in range(2, len(self.inputSum)):
                delta.append(np.array(np.dot(delta[len(delta) - 1], self.weight[len(self.weight) - i + 1].T)) *
                             np.array(self.sigmoid_prime(self.inputSum[len(self.inputSum) - i])))
                derived.append(np.dot(self.threshold[len(self.threshold) - i].T, delta[len(delta) - 1]))

            if len(self.weight) > 1:
                delta.append(np.array(np.dot(delta[len(delta) - 1], self.weight[1].T)) *
                             np.array(self.sigmoid_prime(self.inputSum[0])))
                derived.append(np.dot(x.T, delta[len(delta) - 1]))
        else:
            delta.append(np.multiply(-(y-y_hat), self.sigmoid_prime(self.inputSum[len(self.inputSum) - 1])))
            derived.append(np.dot(x.T, delta[len(delta) - 1]))
        return derived

    def compute_gradients(self, x, y):
        """Returns the gradients from each layer of computation

        Returns:
            ndarray: 1-D array of floats containing the gradient values
        """
        # Obtains the derived costs over each derived weight set
        derived = self.cost_function_prime(x, y)

        # Unravels the derived cost for each set of weights and concatenates them all together
        params = derived[len(derived) - 1].ravel();

        # Concatenates the gradients
        for i in range(len(derived) - 1):
            params = np.concatenate((params.ravel(), derived[len(derived) - 2 - i].ravel()))
        return params

    def get_params(self):
        """Returns the weights in a 1-D array

        Returns:
            ndarray: 1-D array of floats containing the weights. Size of array is [1, n]
                        where n = InputLayerSize * HiddenLayerSize[0] +
                                    range(0,lastHiddenLayer - 1) for HiddenLayerSize[i-1] * HiddenLayerSize[i+1] +
                                    HiddenLayerSize[lastHiddenLayer] * OutputLayerSize
        """
        params = None
        for i in range(len(self.weight)):
            if params is None:
                params = self.weight[i].ravel()
            elif len(params.shape) == 1:
                params = np.concatenate((params.ravel(), self.weight[i].ravel()))
            else:
                params = np.concatenate((params.ravel(), self.weight[i].ravel()), axis=1)
        if len(params.shape) == 2 and params.shape[0] == 1:
            return params.T
        return params

    def set_params(self, params):
        """Sets the weights of the Neural Network from a 1-D array

        Params:
            ndarray: 1-D array of floats that are the weights. Size of array is [1, n]
                        where n = InputLayerSize * HiddenLayerSize[0] +
                                    range(0,lastHiddenLayer - 1) for HiddenLayerSize[i-1] * HiddenLayerSize[i+1] +
                                    HiddenLayerSize[lastHiddenLayer] * OutputLayerSize
        """
        # Starting position of first set of weights
        hiddenStart = 0
        if len(self.weight) > 1:
            # Ending position of first set of weights
            hiddenEnd = self.hiddenLayerSizes[0] * self.inputLayerSize
            # Sets the first set of weights
            self.weight[0] = np.reshape(params[hiddenStart:hiddenEnd], (self.inputLayerSize, self.hiddenLayerSizes[0]))

            # for each hidden layer
            for layer in range(0, len(self.hiddenLayerSizes) - 1):
                # new start position is the previous end position
                hiddenStart = hiddenEnd
                # set new end position
                hiddenEnd = hiddenStart + self.hiddenLayerSizes[layer] * self.hiddenLayerSizes[layer + 1]
                # Sets the set of weights to weight list
                self.weight[layer + 1] = np.reshape(params[hiddenStart:hiddenEnd],
                                           (self.hiddenLayerSizes[layer], self.hiddenLayerSizes[layer + 1]))
            # Setting the final set of weights to output
            hiddenStart = hiddenEnd
            hiddenEnd = hiddenStart + self.hiddenLayerSizes[len(self.hiddenLayerSizes) - 1] * self.outputLayerSize
            self.weight[len(self.weight) - 1] = np.reshape(params[hiddenStart:hiddenEnd],
                                            (self.hiddenLayerSizes[len(self.hiddenLayerSizes) - 1], self.outputLayerSize))
        else:
            hiddenEnd = self.inputLayerSize * self.outputLayerSize
            self.weight[0] = np.reshape(params[hiddenStart:hiddenEnd], (self.inputLayerSize, self.outputLayerSize))

class Trainer(object):
    def __init__(self, N):
        """Constructor

        Params:
            NeuralNetwork: Sets neural network to be trained.
        """
        self.neural_net = N

    def cost_function_wrapper(self, params, x, y):
        """Used to set the parameters of the Neural Network being trained

        Returns:
            float: Cost of the current Neural Network
            ndarray: 1-D array of gradients
        """
        # Sets the parameters
        self.neural_net.set_params(params)
        # Gets the cost
        cost = self.neural_net.cost_function(x, y)
        # Obtain the derived cost for each derived weights
        grad = self.neural_net.compute_gradients(x, y)

        return cost, grad

    def train(self, x, y):
        """Trains the Neural Network to fit the training data

        Uses the training method to minimize the cost function and set the weights of the Neural Network.

        Params:
            ndarray: Input value for the Neural Network
            ndarray: Expected output value from the Neural Network
        """
        # Parameters of weights from the Neural Network
        params = self.neural_net.get_params()

        # Options: maximum # of iterations and show information display after minimizing
        opt = {'maxiter': 200, 'disp': False}

        # jac: Jacobian (Defines that cost_function_wrapper returns gradients)
        # BFGS: Uses BFGS method of training and gradient descent
        # callback: Return values acts as parameter for set_params
        optimize.minimize(self.cost_function_wrapper, params, jac=True, method='BFGS',
                          args=(x, y), options=opt, callback=self.neural_net.set_params)
<commit_msg>Updated ForwardNN.py with comments<commit_after>import numpy as np
from scipy import optimize

__author__ = 'John Bucknam & Bill Clark'

class ForwardNN(object):
    """A feedforward neural network that capable of regression or classification.

    Requires Numpy to run and takes in ndarray inputs.

    """
    # CONSTANTS
    SUM_OF_SQUARES_TYPE = "Sum of Squares"
    CROSS_ENTROPY_TYPE = "Cross Entropy"

    #Variables
    inputLayerSize = None
    outputLayerSize = None
    hiddenLayerSizes = None
    weight = None

    def __init__(self, layer_sizes):
        """Constructor

        Params:
            tuple: Floats; First element is the input layer, last element is the output layer,
            and every other layer acts as an input layer
        """
        # Define each layer
        self.inputLayerSize = layer_sizes[0]
        self.outputLayerSize = layer_sizes[len(layer_sizes) - 1]
        self.hiddenLayerSizes = layer_sizes[1:len(layer_sizes) - 1]

        # Set each weight depending on the number of each layer being paired
        self.weight = []

        # If there are hidden layers
        if len(self.hiddenLayerSizes) != 0:
            # Add random weights from input layer to hidden layer 0
            self.weight.append(np.random.randn(self.inputLayerSize, self.hiddenLayerSizes[0]))
            # For each hidden layer
            for i in range(len(self.hiddenLayerSizes) - 1):
                # Add random weights between each hidden layer
                self.weight.append(np.random.randn(self.hiddenLayerSizes[i], self.hiddenLayerSizes[i + 1]))
            # Add random weights between the last hidden layer and output layer
            self.weight.append(
                np.random.randn(self.hiddenLayerSizes[len(self.hiddenLayerSizes) - 1], self.outputLayerSize))
        else:
            # Add random weights between the input layer and output layer
            self.weight.append(np.random.randn(self.inputLayerSize, self.outputLayerSize))

    def forward(self, input_matrix):
        """Feeds the input forward through the neural network

        Returns:
             ndarray: Output of our inputted matrix that is [n, 10] (n being the # of inputs)
        """
        # Values taken from each set of weights * input summed together
        self.inputSum = []
        # The inputSums inserted into the threshold function (sigmoid)
        self.threshold = []

        # Append each inputSum value matrix
        self.inputSum.append(np.dot(input_matrix, self.weight[0]))
        for i in range(len(self.weight) - 1):
            # Append each A value from the sigmoid(Z)
            self.threshold.append(self.sigmoid(self.inputSum[i]))
            self.inputSum.append(np.dot(self.threshold[i], self.weight[i + 1]))
        y_hat = self.sigmoid(self.inputSum[len(self.inputSum) - 1])
        return y_hat

    @staticmethod
    def sigmoid(z):
        """Static Sigmoid Function

        Params:
            ndarray: floats

        Returns:
            ndarray: floats
        """
        return 1 / (1 + np.exp(-(z.clip(-100, 100))))

    def sigmoid_prime(self, z):
        """Derivative of the Sigmoid Function

        Params:
            ndarray: floats

        Returns:
            ndarray: floats
        """
        return np.multiply(self.sigmoid(z), (1 - self.sigmoid(z)))

    @staticmethod
    def tanh(z):
        """Static Tanh Function

        Params:
            ndarray: floats

        Returns:
            ndarray: floats
        """
        return np.tanh(z)

    @staticmethod
    def tanh_prime(z):
        """Static Tanh Derivative Function

        Params:
            ndarray: floats

        Returns:
            ndarray: floats
        """
        return np.multiply(((2 * np.cosh(z))/(np.cosh(2 * z) + 1)), ((2 * np.cosh(z))/(np.cosh(2 * z) + 1)))

    def cost_function(self, x, y):
        """The cost function that compares the expected output with the actual output for training

        Params:
            ndarray: Input; Should match [n, InputLayerSize] where n is the number of inputs >= to 1.
            ndarray: Output; Should match [n, OutputLayerSize] where n is the number of outputs >= to 1.

            The n value for input and output should be the same.

        Returns:
            None: If invalid training data
            float: Cost of the training data
        """
        y_hat = self.forward(x)
        if y.shape[0] == x.shape[0]:
            type = self.cost_function_type(x, y)
            if type == self.SUM_OF_SQUARES_TYPE:
                return self._sum_of_squares(y, y_hat)
            elif type == self.CROSS_ENTROPY_TYPE:
                return self._cross_entropy(y, y_hat)
        return None

    def cost_function_type(self, x, y):
        if y.shape[1] == 1:
            return self.SUM_OF_SQUARES_TYPE
        elif y.shape[1] > 1:
            return self.CROSS_ENTROPY_TYPE
        else:
            return None

    def _cross_entropy(self, y, y_hat):
        """A type of cost function used for classification greater than 2.

        Returns:
            float: Cost of current neural network.
        """
        return -1 * np.sum(np.multiply(y, self.safe_log(y_hat)) + np.multiply(1 - y, self.safe_log(1 - y_hat))) / y.shape[0]

    @staticmethod
    def safe_log(x, min_val=0.0000000001):
        """A log function used to cap ndarray x when having low or nan elements

        Returns:
            ndarray: Natural log of x matrix
        """
        return np.log(x.clip(min=min_val))

    @staticmethod
    def _sum_of_squares(y, y_hat):
        """A cost function used for regression or binary classification

        Returns:
            float: Cost of current neural network.
        """
        return np.sum(np.power(y - y_hat, 2)) / y.shape[0]

    def cost_function_prime(self, x, y):
        """The derivative of the cost function

        Uses the derived cost function to retrieve the gradient for each component

        Returns:
            list: ndarray; Returns a list of matricies where each matrix represent a specific component
        """
        y_hat = self.forward(x);

        delta = []
        derived = []
        if len(self.weight) > 1:
            # Derivative of cost function * derivative of threshold function(z)
            delta.append(np.multiply(-(y-y_hat), self.sigmoid_prime(self.inputSum[len(self.inputSum) - 1])))
            derived.append(np.dot(self.threshold[len(self.threshold) - 1].T, delta[len(delta) - 1]))

            # Loop for each set of weights
            for i in range(2, len(self.inputSum)):
                delta.append(np.array(np.dot(delta[len(delta) - 1], self.weight[len(self.weight) - i + 1].T)) *
                             np.array(self.sigmoid_prime(self.inputSum[len(self.inputSum) - i])))
                derived.append(np.dot(self.threshold[len(self.threshold) - i].T, delta[len(delta) - 1]))

            # Final set of weights with input
            delta.append(np.array(np.dot(delta[len(delta) - 1], self.weight[1].T)) *
                         np.array(self.sigmoid_prime(self.inputSum[0])))
            derived.append(np.dot(x.T, delta[len(delta) - 1]))
        else:
            # Derivative of cost function * derivative of threshold function(z)
            delta.append(np.multiply(-(y-y_hat), self.sigmoid_prime(self.inputSum[len(self.inputSum) - 1])))
            derived.append(np.dot(x.T, delta[len(delta) - 1]))
        return derived

    def compute_gradients(self, x, y):
        """Returns the gradients from each layer of computation

        Returns:
            ndarray: 1-D array of floats containing the gradient values
        """
        # Obtains the derived costs over each derived weight set
        derived = self.cost_function_prime(x, y)

        # Unravels the derived cost for each set of weights and concatenates them all together
        params = derived[len(derived) - 1].ravel();

        # Concatenates the gradients
        for i in range(len(derived) - 1):
            params = np.concatenate((params.ravel(), derived[len(derived) - 2 - i].ravel()))
        return params

    def get_params(self):
        """Returns the weights in a 1-D array

        Returns:
            ndarray: 1-D array of floats containing the weights. Size of array is [1, n]
                        where n = InputLayerSize * HiddenLayerSize[0] +
                                    range(0,lastHiddenLayer - 1) for HiddenLayerSize[i-1] * HiddenLayerSize[i+1] +
                                    HiddenLayerSize[lastHiddenLayer] * OutputLayerSize
        """
        params = None
        for i in range(len(self.weight)):
            if params is None:
                params = self.weight[i].ravel()
            elif len(params.shape) == 1:
                params = np.concatenate((params.ravel(), self.weight[i].ravel()))
            else:
                params = np.concatenate((params.ravel(), self.weight[i].ravel()), axis=1)
        if len(params.shape) == 2 and params.shape[0] == 1:
            return params.T
        return params

    def set_params(self, params):
        """Sets the weights of the Neural Network from a 1-D array

        Params:
            ndarray: 1-D array of floats that are the weights. Size of array is [1, n]
                        where n = InputLayerSize * HiddenLayerSize[0] +
                                    range(0,lastHiddenLayer - 1) for HiddenLayerSize[i-1] * HiddenLayerSize[i+1] +
                                    HiddenLayerSize[lastHiddenLayer] * OutputLayerSize
        """
        # Starting position of first set of weights
        hiddenStart = 0
        if len(self.weight) > 1:
            # Ending position of first set of weights
            hiddenEnd = self.hiddenLayerSizes[0] * self.inputLayerSize
            # Sets the first set of weights
            self.weight[0] = np.reshape(params[hiddenStart:hiddenEnd], (self.inputLayerSize, self.hiddenLayerSizes[0]))

            # for each hidden layer
            for layer in range(0, len(self.hiddenLayerSizes) - 1):
                # new start position is the previous end position
                hiddenStart = hiddenEnd
                # set new end position
                hiddenEnd = hiddenStart + self.hiddenLayerSizes[layer] * self.hiddenLayerSizes[layer + 1]
                # Sets the set of weights to weight list
                self.weight[layer + 1] = np.reshape(params[hiddenStart:hiddenEnd],
                                           (self.hiddenLayerSizes[layer], self.hiddenLayerSizes[layer + 1]))
            # Setting the final set of weights to output
            hiddenStart = hiddenEnd
            hiddenEnd = hiddenStart + self.hiddenLayerSizes[len(self.hiddenLayerSizes) - 1] * self.outputLayerSize
            self.weight[len(self.weight) - 1] = np.reshape(params[hiddenStart:hiddenEnd],
                                            (self.hiddenLayerSizes[len(self.hiddenLayerSizes) - 1], self.outputLayerSize))
        else:
            hiddenEnd = self.inputLayerSize * self.outputLayerSize
            self.weight[0] = np.reshape(params[hiddenStart:hiddenEnd], (self.inputLayerSize, self.outputLayerSize))

class Trainer(object):
    def __init__(self, N):
        """Constructor

        Params:
            NeuralNetwork: Sets neural network to be trained.
        """
        self.neural_net = N

    def cost_function_wrapper(self, params, x, y):
        """Used to set the parameters of the Neural Network being trained

        Returns:
            float: Cost of the current Neural Network
            ndarray: 1-D array of gradients
        """
        # Sets the parameters
        self.neural_net.set_params(params)
        # Gets the cost
        cost = self.neural_net.cost_function(x, y)
        # Obtain the derived cost for each derived weights
        grad = self.neural_net.compute_gradients(x, y)

        return cost, grad

    def train(self, x, y):
        """Trains the Neural Network to fit the training data

        Uses the training method to minimize the cost function and set the weights of the Neural Network.

        Params:
            ndarray: Input value for the Neural Network
            ndarray: Expected output value from the Neural Network
        """
        # Parameters of weights from the Neural Network
        params = self.neural_net.get_params()

        # Options: maximum # of iterations and show information display after minimizing
        opt = {'maxiter': 200, 'disp': False}

        # jac: Jacobian (Defines that cost_function_wrapper returns gradients)
        # BFGS: Uses BFGS method of training and gradient descent
        # callback: Return values acts as parameter for set_params
        optimize.minimize(self.cost_function_wrapper, params, jac=True, method='BFGS',
                          args=(x, y), options=opt, callback=self.neural_net.set_params)
<|endoftext|>


<commit_before>def is_number_prime(number_to_check):
    is_prime = True
    # I'm staring with the no.2, because we already know that no.1 isn't prime, because it can only divided by itself
    for numberToDivideBy in range(2, number_to_check):
        # use modulo to see if there is a remainder.
        # if the remainder is zero, then the number is NOT prime
        if number_to_check % numberToDivideBy == 0:
            is_prime = False
            break  # no need to continue checking all the other numbers
    return is_prime

########################################################################################################################
# assume the number is prime. if its got a remainder after being divided by something, then its not prime
for numberToCheck in range(1, 10000):
    the_number_is_prime = is_number_prime(numberToCheck)

    if the_number_is_prime:
        print("%d is prime" % numberToCheck)
<commit_msg>added prime number optimisation<commit_after>import time

def now():
    return time.time() * 1000.0

start = now()
optimise = True
list_primes = []

def is_number_prime(the_number_to_check):
    is_prime = True
    if optimise:
        # justin says this is faster
        nums_to_divide_by = list_primes
    else:
        # Im staring with the no.2, because we already know that no.1 isn't prime, because it can only divided by itself
        nums_to_divide_by = range(2, the_number_to_check)

    for number_to_divide_by in nums_to_divide_by:
        # use modulo to see if there is a remainder.
        # if the remainder is zero, then the number is NOT prime
        if the_number_to_check % number_to_divide_by == 0:
            is_prime = False
            break  # no need to continue checking all the other numbers
    return is_prime

########################################################################################################################
# assume the number is prime. if its got a remainder after being divided by something, then its not prime
for number_to_check in range(2, 10000):
    the_number_is_prime = is_number_prime(number_to_check)

    if the_number_is_prime:
        list_primes.append(number_to_check)

print("Here are the prime numbers:")
print(list_primes)
print("Took %d milliseconds" % (now() - start))
<|endoftext|>


<commit_before>from GPy.kern import RBF
from GPy.util.linalg import mdot
import numpy as np
from numpy.core.umath_tests import inner1d

class ExtRBF(RBF):
    def get_gradients_Kn(self, dL_dK, X, X2=None):
        """
        Given the derivative of the objective wrt the covariance matrix
        (dL_dK), compute the gradient wrt the parameters of this kernel,
        and store in the parameters object as e.g. self.variance.gradient
        """
        variance_gradient = inner1d(self.K(X, X2), dL_dK) *  1./ self.variance

        dL_dr = (self.dK_dr_via_X(X, X2) * dL_dK)
        if self.ARD:
            tmp = dL_dr * self._inv_dist(X, X2)
            if X2 is None: X2 = X
            lengthscale_gradient = np.array([np.einsum('ij,ij,...->i', tmp, np.square(X[:,q:q+1] - X2[:,q:q+1].T), -1./self.lengthscale[q]**3) for q in xrange(self.input_dim)])
        else:
            r = self._scaled_dist(X, X2)
            lengthscale_gradient = -np.sum(dL_dr*r, axis=1)/self.lengthscale
            lengthscale_gradient = lengthscale_gradient[np.newaxis, :]

        return np.hstack((variance_gradient[:, np.newaxis], lengthscale_gradient.T))

    def get_gradients_Kdiagn(self, X):

        variance_gradient = self.Kdiag(X) * 1./self.variance
        return np.hstack((variance_gradient[:, np.newaxis], np.zeros((X.shape[0], self.lengthscale.shape[0]))))

    def get_gradients_Kzz(self, S, D, X, X2=None):
        variance_gradient = mdot(S, self.K(X, X2), D) * 1./self.variance

        if X2 is None: X2 = X
        rinv = self._inv_dist(X, X2)
        d =  X[:, None, :] - X2[None, :, :]
        if self.ARD:
            x_xl3 = np.square(d) * (rinv * self.dK_dr_via_X(X, X2))[:,:,None]
        else:
            x_xl3 = np.square(d).sum(axis=2)[:, :, np.newaxis] * (rinv * self.dK_dr_via_X(X, X2))[:,:,None]
        lengthscale_gradient = -np.tensordot(D, np.tensordot(S, x_xl3, (1,0)), (0,1)) / self.lengthscale**3
        return np.hstack((np.diagonal(variance_gradient)[:, np.newaxis], np.diagonal(lengthscale_gradient).T))
<commit_msg>made lenght scale more memory efficient<commit_after>from GPy.kern import RBF
from GPy.util.linalg import mdot
import numpy as np
from numpy.core.umath_tests import inner1d

class ExtRBF(RBF):
    def get_gradients_Kn(self, dL_dK, X, X2=None):
        """
        Given the derivative of the objective wrt the covariance matrix
        (dL_dK), compute the gradient wrt the parameters of this kernel,
        and store in the parameters object as e.g. self.variance.gradient
        """
        variance_gradient = inner1d(self.K(X, X2), dL_dK) *  1./ self.variance

        dL_dr = (self.dK_dr_via_X(X, X2) * dL_dK)
        if self.ARD:
            tmp = dL_dr * self._inv_dist(X, X2)
            if X2 is None: X2 = X
            lengthscale_gradient = np.array([np.einsum('ij,ij,...->i', tmp, np.square(X[:,q:q+1] - X2[:,q:q+1].T), -1./self.lengthscale[q]**3) for q in xrange(self.input_dim)])
        else:
            r = self._scaled_dist(X, X2)
            lengthscale_gradient = -np.sum(dL_dr*r, axis=1)/self.lengthscale
            lengthscale_gradient = lengthscale_gradient[np.newaxis, :]

        return np.hstack((variance_gradient[:, np.newaxis], lengthscale_gradient.T))

    def get_gradients_Kdiagn(self, X):

        variance_gradient = self.Kdiag(X) * 1./self.variance
        return np.hstack((variance_gradient[:, np.newaxis], np.zeros((X.shape[0], self.lengthscale.shape[0]))))

    def get_gradients_Kzz(self, S, D, X, X2=None):
        variance_gradient = mdot(S, self.K(X, X2), D) * 1./self.variance

        if X2 is None: X2 = X
        if self.ARD:
            rinv = self._inv_dist(X, X2)
            d =  X[:, None, :] - X2[None, :, :]
            x_xl3 = np.square(d) * (rinv * self.dK_dr_via_X(X, X2))[:,:,None]
            lengthscale_gradient = -np.tensordot(D, np.tensordot(S, x_xl3, (1,0)), (0,1)) / self.lengthscale**3
            lengthscale_gradient = np.diagonal(lengthscale_gradient).T
        else:
            lengthscale_gradient = np.diagonal(-mdot(S, (self._scaled_dist(X, X2) * self.dK_dr_via_X(X, X2)).T, D) / self.lengthscale)[:, np.newaxis]

        return np.hstack((np.diagonal(variance_gradient)[:, np.newaxis], lengthscale_gradient))
<|endoftext|>


<commit_before>#simple scrolling textbox using Pygame

import pygame
import sys
import threading
import time

screen_width=300
screen_height=200
text_height=14
background_color = (255,255,255)
text_color=(0,0,0)

pygame.init()
screen = pygame.display.set_mode((screen_width, screen_height))
screen.fill(background_color)
pygame.display.update()

scratch = screen.copy()
font = pygame.font.Font(None, 14)
linelen = screen_width/font.size('a')[0]
keepRect = pygame.Rect((0, text_height), (screen_width, screen_width-text_height))
scrollingRect = pygame.Rect((0, 0), (screen_width, screen_height - text_height))
writeRect = pygame.Rect((0, screen_height-text_height), (screen_width, text_height))

def update(text):
    while len(text) > linelen:
        cutoff = text.rfind(' ', 0, linelen)
        updateLine(text[0:cutoff])
        text = text[cutoff + 1:]
    updateLine(text)

def updateLine(line):
    lineSurf = font.render(line, True, text_color)
    screen.fill(background_color)
    screen.blit(scratch, scrollingRect, keepRect)
    screen.blit(lineSurf, writeRect)
    scratch.fill(background_color)
    scratch.blit(screen, screen.get_rect())
    pygame.display.update()

text =  """\
To be, or not to be: that is the question:
Whether 'tis nobler in the mind to suffer
The slings and arrows of outrageous fortune,
Or to take arms against a sea of troubles,
And by opposing end them? To die: to sleep;
No more; and by a sleep to say we end
The heart-ache and the thousand natural shocks
That flesh is heir to, 'tis a consummation
Devoutly to be wish'd. To die, to sleep;
To sleep: perchance to dream: ay, there's the rub;
For in that sleep of death what dreams may come
When we have shuffled off this mortal coil,
Must give us pause: there's the respect
That makes calamity of so long life;
For who would bear the whips and scorns of time,
The oppressor's wrong, the proud man's contumely,
The pangs of despised love, the law's delay,
The insolence of office and the spurns
That patient merit of the unworthy takes,
When he himself might his quietus make With a bare bodkin? who would fardels bear,
To grunt and sweat under a weary life,
But that the dread of something after death,
The undiscover'd country from whose bourn
No traveller returns, puzzles the will
And makes us rather bear those ills we have
Than fly to others that we know not of?
Thus conscience does make cowards of us all;
And thus the native hue of resolution
Is sicklied o'er with the pale cast of thought,
And enterprises of great pith and moment
With this regard their currents turn awry,
And lose the name of action. - Soft you now!
The fair Ophelia! Nymph, in thy orisons
Be all my sins remember'd.
"""

def datasource():
    lines = text.split('\n')
    for one_line in lines:
        yield one_line

def write_to_screen():
    data = datasource()
    while True:
        try:
            time.sleep(0.5)
            line = data.next()
            update(line)
            yield 1
        except StopIteration:
            break

for _ in write_to_screen():
    pass
<commit_msg>now a component<commit_after>#simple scrolling textbox using Pygame

import pygame
import sys
import threading
import time
from Axon.Component import component

#the long lines are there on purpose, to see if the component wraps text correctly.
text =  """\
To be, or not to be: that is the question:
Whether 'tis nobler in the mind to suffer
The slings and arrows of outrageous fortune,
Or to take arms against a sea of troubles,
And by opposing end them? To die: to sleep;
No more; and by a sleep to say we end
The heart-ache and the thousand natural shocks That flesh is heir to, 'tis a consummation Devoutly to be wish'd. To die, to sleep;
To sleep: perchance to dream: ay, there's the rub;
For in that sleep of death what dreams may come
When we have shuffled off this mortal coil,
Must give us pause: there's the respect
That makes calamity of so long life;
For who would bear the whips and scorns of time,
The oppressor's wrong, the proud man's contumely,
The pangs of despised love, the law's delay,
The insolence of office and the spurns
That patient merit of the unworthy takes,
When he himself might his quietus make With a bare bodkin? who would fardels bear,
To grunt and sweat under a weary life,
But that the dread of something after death,
The undiscover'd country from whose bourn
No traveller returns, puzzles the will
And makes us rather bear those ills we have
Than fly to others that we know not of?
Thus conscience does make cowards of us all;
And thus the native hue of resolution
Is sicklied o'er with the pale cast of thought,
And enterprises of great pith and moment
With this regard their currents turn awry,
And lose the name of action. - Soft you now!
The fair Ophelia! Nymph, in thy orisons
Be all my sins remember'd.
"""

def datasource():
    lines = text.split('\n')
    for one_line in lines:
        yield one_line

class writer(component):
    def __init__(self, source, screen_width=300, screen_height=200, text_height=14,
                 background_color = (255,255,255), text_color=(0,0,0)):
        super(writer, self).__init__()
        self.source = source
        self.screen_width = screen_width
        self.screen_height = screen_height
        self.text_height = text_height
        self.background_color = background_color
        self.text_color = text_color

        pygame.init()
        self.screen = pygame.display.set_mode((screen_width, screen_height))
        self.screen.fill(background_color)
        pygame.display.update()

        self.scratch = self.screen.copy()
        self.font = pygame.font.Font(None, 14)
        self.linelen = self.screen_width/self.font.size('a')[0]
        self.keepRect = pygame.Rect((0, text_height), (screen_width, screen_width-text_height))
        self.scrollingRect = pygame.Rect((0, 0), (screen_width, screen_height - text_height))
        self.writeRect = pygame.Rect((0, screen_height-text_height), (screen_width, text_height))

    def main(self):
        while True:
            try:
                time.sleep(0.5)
                line = self.source.next()
                print line
                self.update(line)
                yield 1
            except StopIteration:
                break

    def update(self, text):
        while len(text) > self.linelen:
            cutoff = text.rfind(' ', 0, self.linelen)
            self.updateLine(text[0:cutoff])
            text = text[cutoff + 1:]
        self.updateLine(text)

    def updateLine(self, line):
        lineSurf = self.font.render(line, True, self.text_color)
        self.screen.fill(self.background_color)
        self.screen.blit(self.scratch, self.scrollingRect, self.keepRect)
        self.screen.blit(lineSurf, self.writeRect)
        self.scratch.fill(self.background_color)
        self.scratch.blit(self.screen, self.screen.get_rect())
        pygame.display.update()

def tests():
    source = datasource()
    writer(source).run()

if __name__ == '__main__':
    tests()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import os
import io
import sys
import json
import ctypes

from django.test import TestCase
from django.utils.html import escape
from django.utils.translation import ugettext_lazy as _

from .models import TestModel, TestCharFieldModel, TestTextFieldModel, Message

UTF8_OK_BUT_TOO_LONG_FILE = os.path.join('dev_example', 'tests', 'utf8_ok_but_too_long.txt')
UTF8_OK_FILE = os.path.join('dev_example', 'tests', 'utf8_ok.txt')
UTF8_NOK_FILE = os.path.join('dev_example', 'tests', 'utf8_nok.txt')
UTF8_NOK_ELS_FILE = os.path.join('dev_example', 'tests', 'els_awesome_file.txt')
BINARY_FILE = os.path.join('dev_example', 'tests', 'binky.png')

class RestTests(TestCase):
    def setUp(self):
        self.url = '/api/message/'

    def test_add_view_exists(self):
        response = self.client.get(self.url)
        self.assertEqual(response['Content-Type'], 'application/json')

    def test_max_content_length(self):
        with open(UTF8_OK_BUT_TOO_LONG_FILE, 'rb') as fp:
            response = self.client.post(
                self.url,
                {'file': fp},
                follow=True
            )

            self.assertEqual(Message.objects.count(), 0)

            if sys.version_info >= (3, 0):
                result = json.loads(response.content.decode('utf-8'))
            else:
                result = json.loads(response.content)

            self.assertTrue(result['file'][0])

            self.assertEqual(response['Content-Type'], 'application/json')
            self.assertEqual(response.status_code, 400)

    def test_add_view_shows_error_when_submitting_els_content_text_field(self):
        if sys.version_info >= (3, 0):
            with io.open(UTF8_NOK_ELS_FILE, 'r', encoding='utf8') as fp:
                content = fp.read()
                fp.seek(0)
                response = self.client.post(
                    self.url,
                    {
                        'text': content,
                        'char': content,
                        'file': fp

                     },
                    follow=True
                )

                result = json.loads(response.content)
                self.assertEqual(result['text'][0], _('4 Byte UTF8-characters detected'))
                self.assertEqual(result['char'][0], _('4 Byte UTF8-characters detected'))
                self.assertEqual(result['file'][0], _('4 Byte UTF8-characters detected'))
        else:
            with open(UTF8_NOK_ELS_FILE, 'rb') as fp:
                content = str(fp.read())
                fp.seek(0)
                response = self.client.post(
                    self.url,
                    {
                        'text': content,
                        'char': content,
                        'file': fp,
                     },
                    follow=True
                )

                result = json.loads(response.content.decode('utf-8'))
                self.assertEqual(result['text'][0], _('Non UTF8-content detected'))
                self.assertEqual(result['char'][0], _('Non UTF8-content detected'))
                self.assertEqual(result['file'][0], _('4 Byte UTF8-characters detected'))

        self.assertEqual(response['Content-Type'], 'application/json')
        self.assertEqual(response.status_code, 400)
        self.assertEqual(Message.objects.count(), 0)

    def test_check_for_null_character(self):
        content = ctypes.create_unicode_buffer('String\0Other')

        response = self.client.post(
            self.url,
            {'text': content, },
            follow=True
        )

        result = json.loads(response.content.decode('utf-8'))

        self.assertEqual(result['text'][0], _('NULL character detected'))
        self.assertEqual(response['Content-Type'], 'application/json')
        self.assertEqual(response.status_code, 400)
        self.assertEqual(TestTextFieldModel.objects.count(), 0)

class TextFieldTests(TestCase):
    def setUp(self):
        self.url = '/text-field/'

    def test_add_view_exists(self):
        response = self.client.get(self.url)
        self.assertContains(response, '<input', status_code=200)

    def test_check_for_null_character(self):
        content = ctypes.create_unicode_buffer('String\0Other')

        response = self.client.post(
            self.url,
            {'text': content, },
            follow=True
        )

        self.assertContains(response, escape(_('NULL character detected')))
        self.assertEqual(TestTextFieldModel.objects.count(), 0)

    def test_add_view_works(self):
        with open(UTF8_OK_FILE, 'rb') as fp:
            response = self.client.post(
                self.url,
                {
                    'text': fp.read()
                },
                follow=True
            )
            self.assertEqual(TestTextFieldModel.objects.count(), 1)

    def test_add_view_shows_error_when_submitting_non_utf8_content(self):
        if sys.version_info < (3, 0):
            with open(UTF8_NOK_FILE, 'rb') as fp:
                content = fp.read()
                response = self.client.post(
                    self.url,
                    {'text': str(content), },
                    follow=True
                )
                self.assertEqual(TestTextFieldModel.objects.count(), 0)
                self.assertContains(response, escape(_('Non UTF8-content detected')))

    def test_check_for_null_character_second_method(self):
        content = 'abcd\x01\x00cdefg'

        response = self.client.post(
            self.url,
            {'text': content, },
            follow=True
        )

        self.assertContains(response, escape(_('NULL character detected')))
        self.assertEqual(TestTextFieldModel.objects.count(), 0)

    def test_add_view_shows_error_when_submitting_els_content(self):
        if sys.version_info >= (3, 0):
            with io.open(UTF8_NOK_ELS_FILE, 'r', encoding='utf8') as fp:
                response = self.client.post(
                    self.url,
                    {'text': str(fp.read()), },
                    follow=True
                )
                self.assertContains(response, escape(_('4 Byte UTF8-characters detected')))
                self.assertEqual(TestTextFieldModel.objects.count(), 0)

        else:
            with open(UTF8_NOK_ELS_FILE, 'rb') as fp:
                response = self.client.post(
                    self.url,
                    {'text': str(fp.read()), },
                    follow=True
                )
                self.assertEqual(TestTextFieldModel.objects.count(), 0)
                self.assertContains(response, escape(_('Non UTF8-content detected')))

class CharFieldTests(TestCase):
    def setUp(self):
        self.url = '/char-field/'

    def test_add_view_exists(self):
        response = self.client.get(self.url)
        self.assertContains(response, '<input', status_code=200)

    def test_check_for_null_character(self):
        content = ctypes.create_unicode_buffer('String\0Other')

        response = self.client.post(
            self.url,
            {'text': content, },
            follow=True
        )

        self.assertContains(response, escape(_('NULL character detected')))
        self.assertEqual(TestTextFieldModel.objects.count(), 0)

    def test_check_for_null_character_second_method(self):
        content = 'abcd\x01\x00cdefg'

        response = self.client.post(
            self.url,
            {'text': content, },
            follow=True
        )

        self.assertContains(response, escape(_('NULL character detected')))
        self.assertEqual(TestTextFieldModel.objects.count(), 0)

    def test_add_view_works(self):
        with open(UTF8_OK_FILE, 'rb') as fp:
            response = self.client.post(
                self.url,
                {
                    'text': fp.read()
                },
                follow=True
            )

            self.assertEqual(TestCharFieldModel.objects.count(), 1)

    def test_add_view_shows_error_when_submitting_non_utf8_content(self):
        if sys.version_info < (3, 0):
            with open(UTF8_NOK_FILE, 'rb') as fp:
                content = fp.read()
                response = self.client.post(
                    self.url,
                    {'text': str(content), },
                    follow=True
                )
                self.assertEqual(TestCharFieldModel.objects.count(), 0)
                self.assertContains(response, escape(_('Non UTF8-content detected')))

    def test_add_view_shows_error_when_submitting_els_content(self):
        if sys.version_info >= (3, 0):
            with io.open(UTF8_NOK_ELS_FILE, 'r', encoding='utf8') as fp:
                response = self.client.post(
                    self.url,
                    {'text': str(fp.read()), },
                    follow=True
                )
                self.assertEqual(TestCharFieldModel.objects.count(), 0)
                self.assertContains(response, escape(_('4 Byte UTF8-characters detected')))
        else:
            with open(UTF8_NOK_ELS_FILE, 'rb') as fp:
                response = self.client.post(
                    self.url,
                    {'text': str(fp.read()), },
                    follow=True
                )
                self.assertEqual(TestCharFieldModel.objects.count(), 0)
                self.assertContains(response, escape(_('Non UTF8-content detected')))

class FileTests(TestCase):
    def setUp(self):
        self.url = '/'
        self.max_content_length_url = '/max-content-length/'

    def test_add_view_exists(self):
        response = self.client.get(self.url)
        self.assertContains(response, '<input', status_code=200, count=3)

    def test_add_view_works(self):
        with open(UTF8_OK_FILE, 'rb') as fp:
            response = self.client.post(self.url, {'file': fp, }, follow=True)
            self.assertEqual(TestModel.objects.count(), 1)

    def test_when_sending_no_file_when_required_utf8_stuff_should_not_be_triggered(self):
        response = self.client.post(self.url)
        self.assertEqual(response.status_code, 200)
        self.assertContains(response, escape(_('This field is required.')))
        self.assertEqual(TestModel.objects.count(), 0)

    def test_add_view_shows_error_when_submitting_non_utf8_file(self):
        with open(UTF8_NOK_FILE, 'rb') as fp:
            response = self.client.post(
                self.url,
                {'file': fp, },
                follow=True
            )
            self.assertEqual(TestModel.objects.count(), 0)
            self.assertContains(response, escape(_('Non UTF8-content detected')))

    def test_add_view_shows_error_when_submitting_els_utf8_file(self):
        with open(UTF8_NOK_ELS_FILE, 'rb') as fp:
            response = self.client.post(
                self.url,
                {'file': fp, },
                follow=True
            )
            self.assertEqual(TestModel.objects.count(), 0)
            self.assertContains(response, escape(_('4 Byte UTF8-characters detected')))

    def test_add_view_shows_error_when_submitting_binary_file(self):
        with open(BINARY_FILE, 'rb') as fp:
            response = self.client.post(
                self.url,
                {'file': fp, },
                follow=True
            )
            self.assertEqual(TestModel.objects.count(), 0)
            self.assertContains(response, escape(_('Non UTF8-content detected')))

    def test_max_content_length(self):
        with open(UTF8_OK_BUT_TOO_LONG_FILE, 'rb') as fp:
            response = self.client.post(
                self.max_content_length_url,
                {'file': fp, },
                follow=True
            )

            self.assertEqual(TestModel.objects.count(), 0)
            self.assertContains(
                response,
                escape(
                    _('The content of the text file cannot be longer then %(max_content_length)s characters.' % {
                        'max_content_length': 1000})
                ),
                status_code=200
            )
<commit_msg>Fix for Python 3.5, reading str as json is different, apprently<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import os
import io
import sys
import json
import ctypes

from django.test import TestCase
from django.utils.html import escape
from django.utils.translation import ugettext_lazy as _

from .models import TestModel, TestCharFieldModel, TestTextFieldModel, Message

UTF8_OK_BUT_TOO_LONG_FILE = os.path.join('dev_example', 'tests', 'utf8_ok_but_too_long.txt')
UTF8_OK_FILE = os.path.join('dev_example', 'tests', 'utf8_ok.txt')
UTF8_NOK_FILE = os.path.join('dev_example', 'tests', 'utf8_nok.txt')
UTF8_NOK_ELS_FILE = os.path.join('dev_example', 'tests', 'els_awesome_file.txt')
BINARY_FILE = os.path.join('dev_example', 'tests', 'binky.png')

class RestTests(TestCase):
    def setUp(self):
        self.url = '/api/message/'

    def test_add_view_exists(self):
        response = self.client.get(self.url)
        self.assertEqual(response['Content-Type'], 'application/json')

    def test_max_content_length(self):
        with open(UTF8_OK_BUT_TOO_LONG_FILE, 'rb') as fp:
            response = self.client.post(
                self.url,
                {'file': fp},
                follow=True
            )

            self.assertEqual(Message.objects.count(), 0)

            if sys.version_info >= (3, 0):
                result = json.loads(response.content.decode('utf-8'))
            else:
                result = json.loads(response.content)

            self.assertTrue(result['file'][0])

            self.assertEqual(response['Content-Type'], 'application/json')
            self.assertEqual(response.status_code, 400)

    def test_add_view_shows_error_when_submitting_els_content_text_field(self):
        if sys.version_info >= (3, 0):
            with io.open(UTF8_NOK_ELS_FILE, 'r', encoding='utf8') as fp:
                content = fp.read()
                fp.seek(0)
                response = self.client.post(
                    self.url,
                    {
                        'text': content,
                        'char': content,
                        'file': fp

                     },
                    follow=True
                )

                result = json.loads(response.content.decode('utf-8'))

                self.assertEqual(result['text'][0], _('4 Byte UTF8-characters detected'))
                self.assertEqual(result['char'][0], _('4 Byte UTF8-characters detected'))
                self.assertEqual(result['file'][0], _('4 Byte UTF8-characters detected'))
        else:
            with open(UTF8_NOK_ELS_FILE, 'rb') as fp:
                content = str(fp.read())
                fp.seek(0)
                response = self.client.post(
                    self.url,
                    {
                        'text': content,
                        'char': content,
                        'file': fp,
                     },
                    follow=True
                )

                result = json.loads(response.content.decode('utf-8'))
                self.assertEqual(result['text'][0], _('Non UTF8-content detected'))
                self.assertEqual(result['char'][0], _('Non UTF8-content detected'))
                self.assertEqual(result['file'][0], _('4 Byte UTF8-characters detected'))

        self.assertEqual(response['Content-Type'], 'application/json')
        self.assertEqual(response.status_code, 400)
        self.assertEqual(Message.objects.count(), 0)

    def test_check_for_null_character(self):
        content = ctypes.create_unicode_buffer('String\0Other')

        response = self.client.post(
            self.url,
            {'text': content, },
            follow=True
        )

        result = json.loads(response.content.decode('utf-8'))

        self.assertEqual(result['text'][0], _('NULL character detected'))
        self.assertEqual(response['Content-Type'], 'application/json')
        self.assertEqual(response.status_code, 400)
        self.assertEqual(TestTextFieldModel.objects.count(), 0)

class TextFieldTests(TestCase):
    def setUp(self):
        self.url = '/text-field/'

    def test_add_view_exists(self):
        response = self.client.get(self.url)
        self.assertContains(response, '<input', status_code=200)

    def test_check_for_null_character(self):
        content = ctypes.create_unicode_buffer('String\0Other')

        response = self.client.post(
            self.url,
            {'text': content, },
            follow=True
        )

        self.assertContains(response, escape(_('NULL character detected')))
        self.assertEqual(TestTextFieldModel.objects.count(), 0)

    def test_add_view_works(self):
        with open(UTF8_OK_FILE, 'rb') as fp:
            response = self.client.post(
                self.url,
                {
                    'text': fp.read()
                },
                follow=True
            )
            self.assertEqual(TestTextFieldModel.objects.count(), 1)

    def test_add_view_shows_error_when_submitting_non_utf8_content(self):
        if sys.version_info < (3, 0):
            with open(UTF8_NOK_FILE, 'rb') as fp:
                content = fp.read()
                response = self.client.post(
                    self.url,
                    {'text': str(content), },
                    follow=True
                )
                self.assertEqual(TestTextFieldModel.objects.count(), 0)
                self.assertContains(response, escape(_('Non UTF8-content detected')))

    def test_check_for_null_character_second_method(self):
        content = 'abcd\x01\x00cdefg'

        response = self.client.post(
            self.url,
            {'text': content, },
            follow=True
        )

        self.assertContains(response, escape(_('NULL character detected')))
        self.assertEqual(TestTextFieldModel.objects.count(), 0)

    def test_add_view_shows_error_when_submitting_els_content(self):
        if sys.version_info >= (3, 0):
            with io.open(UTF8_NOK_ELS_FILE, 'r', encoding='utf8') as fp:
                response = self.client.post(
                    self.url,
                    {'text': str(fp.read()), },
                    follow=True
                )
                self.assertContains(response, escape(_('4 Byte UTF8-characters detected')))
                self.assertEqual(TestTextFieldModel.objects.count(), 0)

        else:
            with open(UTF8_NOK_ELS_FILE, 'rb') as fp:
                response = self.client.post(
                    self.url,
                    {'text': str(fp.read()), },
                    follow=True
                )
                self.assertEqual(TestTextFieldModel.objects.count(), 0)
                self.assertContains(response, escape(_('Non UTF8-content detected')))

class CharFieldTests(TestCase):
    def setUp(self):
        self.url = '/char-field/'

    def test_add_view_exists(self):
        response = self.client.get(self.url)
        self.assertContains(response, '<input', status_code=200)

    def test_check_for_null_character(self):
        content = ctypes.create_unicode_buffer('String\0Other')

        response = self.client.post(
            self.url,
            {'text': content, },
            follow=True
        )

        self.assertContains(response, escape(_('NULL character detected')))
        self.assertEqual(TestTextFieldModel.objects.count(), 0)

    def test_check_for_null_character_second_method(self):
        content = 'abcd\x01\x00cdefg'

        response = self.client.post(
            self.url,
            {'text': content, },
            follow=True
        )

        self.assertContains(response, escape(_('NULL character detected')))
        self.assertEqual(TestTextFieldModel.objects.count(), 0)

    def test_add_view_works(self):
        with open(UTF8_OK_FILE, 'rb') as fp:
            response = self.client.post(
                self.url,
                {
                    'text': fp.read()
                },
                follow=True
            )

            self.assertEqual(TestCharFieldModel.objects.count(), 1)

    def test_add_view_shows_error_when_submitting_non_utf8_content(self):
        if sys.version_info < (3, 0):
            with open(UTF8_NOK_FILE, 'rb') as fp:
                content = fp.read()
                response = self.client.post(
                    self.url,
                    {'text': str(content), },
                    follow=True
                )
                self.assertEqual(TestCharFieldModel.objects.count(), 0)
                self.assertContains(response, escape(_('Non UTF8-content detected')))

    def test_add_view_shows_error_when_submitting_els_content(self):
        if sys.version_info >= (3, 0):
            with io.open(UTF8_NOK_ELS_FILE, 'r', encoding='utf8') as fp:
                response = self.client.post(
                    self.url,
                    {'text': str(fp.read()), },
                    follow=True
                )
                self.assertEqual(TestCharFieldModel.objects.count(), 0)
                self.assertContains(response, escape(_('4 Byte UTF8-characters detected')))
        else:
            with open(UTF8_NOK_ELS_FILE, 'rb') as fp:
                response = self.client.post(
                    self.url,
                    {'text': str(fp.read()), },
                    follow=True
                )
                self.assertEqual(TestCharFieldModel.objects.count(), 0)
                self.assertContains(response, escape(_('Non UTF8-content detected')))

class FileTests(TestCase):
    def setUp(self):
        self.url = '/'
        self.max_content_length_url = '/max-content-length/'

    def test_add_view_exists(self):
        response = self.client.get(self.url)
        self.assertContains(response, '<input', status_code=200, count=3)

    def test_add_view_works(self):
        with open(UTF8_OK_FILE, 'rb') as fp:
            response = self.client.post(self.url, {'file': fp, }, follow=True)
            self.assertEqual(TestModel.objects.count(), 1)

    def test_when_sending_no_file_when_required_utf8_stuff_should_not_be_triggered(self):
        response = self.client.post(self.url)
        self.assertEqual(response.status_code, 200)
        self.assertContains(response, escape(_('This field is required.')))
        self.assertEqual(TestModel.objects.count(), 0)

    def test_add_view_shows_error_when_submitting_non_utf8_file(self):
        with open(UTF8_NOK_FILE, 'rb') as fp:
            response = self.client.post(
                self.url,
                {'file': fp, },
                follow=True
            )
            self.assertEqual(TestModel.objects.count(), 0)
            self.assertContains(response, escape(_('Non UTF8-content detected')))

    def test_add_view_shows_error_when_submitting_els_utf8_file(self):
        with open(UTF8_NOK_ELS_FILE, 'rb') as fp:
            response = self.client.post(
                self.url,
                {'file': fp, },
                follow=True
            )
            self.assertEqual(TestModel.objects.count(), 0)
            self.assertContains(response, escape(_('4 Byte UTF8-characters detected')))

    def test_add_view_shows_error_when_submitting_binary_file(self):
        with open(BINARY_FILE, 'rb') as fp:
            response = self.client.post(
                self.url,
                {'file': fp, },
                follow=True
            )
            self.assertEqual(TestModel.objects.count(), 0)
            self.assertContains(response, escape(_('Non UTF8-content detected')))

    def test_max_content_length(self):
        with open(UTF8_OK_BUT_TOO_LONG_FILE, 'rb') as fp:
            response = self.client.post(
                self.max_content_length_url,
                {'file': fp, },
                follow=True
            )

            self.assertEqual(TestModel.objects.count(), 0)
            self.assertContains(
                response,
                escape(
                    _('The content of the text file cannot be longer then %(max_content_length)s characters.' % {
                        'max_content_length': 1000})
                ),
                status_code=200
            )
<|endoftext|>


<commit_before>"""The parameters dictionary contains global parameter settings."""

from collections import OrderedDict
from os import environ

__all__ = ['configuration', 'init_configuration', 'print_defaults', 'print_state',
           'add_sub_configuration']

# Be EXTREMELY careful when writing to a Parameters dictionary
# Read here for reference: http://wiki.c2.com/?GlobalVariablesAreBad
# https://softwareengineering.stackexchange.com/questions/148108/why-is-global-state-so-evil
# If any issues related to global state arise, the following class should
# be made immutable. It shall only be written to at application startup
# and never modified.

class Parameters(OrderedDict):
    """
    A dictionary-like class to hold global configuration parameters for devito
    On top of a normal dict, this provides the option to provide callback functions
    so that any interested module can be informed when the configuration changes.
    """
    def __init__(self, name=None, **kwargs):
        super(Parameters, self).__init__(**kwargs)
        self._name = name
        self._accepted = {}
        self._defaults = {}
        self._update_functions = {}
        if kwargs is not None:
            for key, value in kwargs.items():
                self[key] = value

    def __setitem__(self, key, value):
        super(Parameters, self).__setitem__(key, value)
        self._updated(key, value)

    def _updated(self, key, value):
        """
        Call any provided update functions so that the other modules know we've
        been updated.
        """
        if key in self._update_functions:
            retval = self._update_functions[key](value)
            if retval is not None:
                super(Parameters, self).__setitem__(key, retval)

    def add(self, key, value, accepted=None, callback=None):
        """
        Add a new parameter ``key`` with default value ``value``.

        Associate ``key`` with a list of ``accepted`` values.

        If provided, make sure ``callback`` is executed when the value of ``key``
        changes.
        """
        super(Parameters, self).__setitem__(key, value)
        self._accepted[key] = accepted
        self._defaults[key] = value
        if callable(callback):
            self._update_functions[key] = callback

    def update(self, key, value):
        """
        Update the parameter ``key`` to ``value``. This is different from
        ``self[key] = value`` as the callback, if any, is bypassed.
        """
        assert key in self
        super(Parameters, self).__setitem__(key, value)

    def initialize(self):
        """
        Execute all callbacks in ``self._update_functions``. Should be invoked
        once right after all entries have been set.
        """
        for k, v in self.items():
            self._updated(k, v)

    @property
    def name(self):
        return self._name

env_vars_mapper = {
    'DEVITO_ARCH': 'compiler',
    'DEVITO_ISA': 'isa',
    'DEVITO_PLATFORM': 'platform',
    'DEVITO_BACKEND': 'backend',
    'DEVITO_DSE': 'dse',
    'DEVITO_DLE': 'dle',
    'DEVITO_DLE_OPTIONS': 'dle_options',
    'DEVITO_OPENMP': 'openmp',
    'DEVITO_LOGGING': 'log_level',
    'DEVITO_FIRST_TOUCH': 'first_touch',
    'DEVITO_DEBUG_COMPILER': 'debug_compiler',
}

configuration = Parameters("Devito-Configuration")
"""The Devito configuration parameters."""

def init_configuration(configuration=configuration, env_vars_mapper=env_vars_mapper):
    # Populate /configuration/ with user-provided options
    if environ.get('DEVITO_CONFIG') is None:
        # Try env variables, otherwise stick to defaults
        for k, v in sorted(env_vars_mapper.items()):
            configuration.update(v, environ.get(k, configuration._defaults[v]))
    else:
        # Attempt reading from the specified configuration file
        raise NotImplementedError("Devito doesn't support configuration via file yet.")

    # Parameters validation
    for k, v in list(configuration.items()):
        try:
            items = v.split(';')
            # Env variable format: 'var=k1:v1;k2:v2:k3:v3:...'
            keys, values = zip(*[i.split(':') for i in items])
            # Casting
            values = [eval(i) for i in values]
        except AttributeError:
            # Env variable format: 'var=v', 'v' is not a string
            keys = [v]
            values = []
        except ValueError:
            # Env variable format: 'var=k1;k2:v2...' or even just 'var=v'
            keys = [i.split(':')[0] for i in items]
            values = []
            # Cast to integer
            for i, j in enumerate(list(keys)):
                try:
                    keys[i] = int(j)
                except (TypeError, ValueError):
                    keys[i] = j
        accepted = configuration._accepted[k]
        if accepted is not None and any(i not in accepted for i in keys):
            raise ValueError("Illegal configuration parameter (%s, %s). "
                             "Accepted: %s" % (k, v, str(accepted)))
        if len(keys) == len(values):
            configuration.update(k, dict(zip(keys, values)))
        elif len(keys) == 1:
            configuration.update(k, keys[0])
        else:
            configuration.update(k, keys)

    configuration.initialize()

def add_sub_configuration(sub_configuration, sub_env_vars_mapper=None):
    init_configuration(sub_configuration, sub_env_vars_mapper or {})
    setattr(configuration, sub_configuration.name, sub_configuration)

def print_defaults():
    """Print the environment variables accepted by Devito, their default value,
    as well as all of the accepted values."""
    from devito.logger import info
    for k, v in env_vars_mapper.items():
        info('%s: %s. Default: %s' % (k, configuration._accepted[v],
                                      configuration._defaults[v]))

def print_state():
    """Print the current configuration state."""
    from devito.logger import info
    for k, v in configuration.items():
        info('%s: %s' % (k, v))
<commit_msg>Improve global configuration mechanism<commit_after>"""The parameters dictionary contains global parameter settings."""

from collections import OrderedDict
from os import environ

__all__ = ['configuration', 'init_configuration', 'print_defaults', 'print_state',
           'add_sub_configuration']

# Be EXTREMELY careful when writing to a Parameters dictionary
# Read here for reference: http://wiki.c2.com/?GlobalVariablesAreBad
# https://softwareengineering.stackexchange.com/questions/148108/why-is-global-state-so-evil
# If any issues related to global state arise, the following class should
# be made immutable. It shall only be written to at application startup
# and never modified.

class Parameters(OrderedDict):
    """
    A dictionary-like class to hold global configuration parameters for devito
    On top of a normal dict, this provides the option to provide callback functions
    so that any interested module can be informed when the configuration changes.
    """
    def __init__(self, name=None, **kwargs):
        super(Parameters, self).__init__(**kwargs)
        self._name = name
        self._accepted = {}
        self._defaults = {}
        self._update_functions = {}
        if kwargs is not None:
            for key, value in kwargs.items():
                self[key] = value

    def _check_key_value(func):
        def wrapper(self, key, value):
            accepted = self._accepted[key]
            if accepted is not None:
                tocheck = list(value) if isinstance(value, dict) else [value]
                if any(i not in accepted for i in tocheck):
                    raise ValueError("Illegal configuration parameter (%s, %s). "
                                     "Accepted: %s" % (key, value, str(accepted)))
            func(self, key, value)
        return wrapper

    def _updated(self, key, value):
        """
        Call any provided update functions so that the other modules know we've
        been updated.
        """
        if key in self._update_functions:
            retval = self._update_functions[key](value)
            if retval is not None:
                super(Parameters, self).__setitem__(key, retval)

    @_check_key_value
    def __setitem__(self, key, value):
        super(Parameters, self).__setitem__(key, value)
        self._updated(key, value)

    @_check_key_value
    def update(self, key, value):
        """
        Update the parameter ``key`` to ``value``. This is different from
        ``self[key] = value`` as the callback, if any, is bypassed.
        """
        super(Parameters, self).__setitem__(key, value)

    def add(self, key, value, accepted=None, callback=None):
        """
        Add a new parameter ``key`` with default value ``value``.

        Associate ``key`` with a list of ``accepted`` values.

        If provided, make sure ``callback`` is executed when the value of ``key``
        changes.
        """
        super(Parameters, self).__setitem__(key, value)
        self._accepted[key] = accepted
        self._defaults[key] = value
        if callable(callback):
            self._update_functions[key] = callback

    def initialize(self):
        """
        Execute all callbacks in ``self._update_functions``. Should be invoked
        once right after all entries have been set.
        """
        for k, v in self.items():
            self._updated(k, v)

    @property
    def name(self):
        return self._name

env_vars_mapper = {
    'DEVITO_ARCH': 'compiler',
    'DEVITO_ISA': 'isa',
    'DEVITO_PLATFORM': 'platform',
    'DEVITO_BACKEND': 'backend',
    'DEVITO_DSE': 'dse',
    'DEVITO_DLE': 'dle',
    'DEVITO_DLE_OPTIONS': 'dle_options',
    'DEVITO_OPENMP': 'openmp',
    'DEVITO_LOGGING': 'log_level',
    'DEVITO_FIRST_TOUCH': 'first_touch',
    'DEVITO_DEBUG_COMPILER': 'debug_compiler',
}

configuration = Parameters("Devito-Configuration")
"""The Devito configuration parameters."""

def init_configuration(configuration=configuration, env_vars_mapper=env_vars_mapper):
    # Populate /configuration/ with user-provided options
    if environ.get('DEVITO_CONFIG') is None:
        # Try env variables, otherwise stick to defaults
        unprocessed = OrderedDict([(v, environ.get(k, configuration._defaults[v]))
                                   for k, v in sorted(env_vars_mapper.items())])
    else:
        # Attempt reading from the specified configuration file
        raise NotImplementedError("Devito doesn't support configuration via file yet.")

    # Parameters validation
    for k, v in unprocessed.items():
        try:
            items = v.split(';')
            # Env variable format: 'var=k1:v1;k2:v2:k3:v3:...'
            keys, values = zip(*[i.split(':') for i in items])
            # Casting
            values = [eval(i) for i in values]
        except AttributeError:
            # Env variable format: 'var=v', 'v' is not a string
            keys = [v]
            values = []
        except ValueError:
            # Env variable format: 'var=k1;k2:v2...' or even just 'var=v'
            keys = [i.split(':')[0] for i in items]
            values = []
            # Cast to integer
            for i, j in enumerate(list(keys)):
                try:
                    keys[i] = int(j)
                except (TypeError, ValueError):
                    keys[i] = j
        if len(keys) == len(values):
            configuration.update(k, dict(zip(keys, values)))
        elif len(keys) == 1:
            configuration.update(k, keys[0])
        else:
            configuration.update(k, keys)

    configuration.initialize()

def add_sub_configuration(sub_configuration, sub_env_vars_mapper=None):
    init_configuration(sub_configuration, sub_env_vars_mapper or {})
    # For use from within a backend (i.e., inside Devito)
    setattr(configuration, sub_configuration.name, sub_configuration)
    # For use in user code, when the backend is a runtime choice and some
    # options are in common between the supported backends
    setattr(configuration, 'backend', sub_configuration)

def print_defaults():
    """Print the environment variables accepted by Devito, their default value,
    as well as all of the accepted values."""
    from devito.logger import info
    for k, v in env_vars_mapper.items():
        info('%s: %s. Default: %s' % (k, configuration._accepted[v],
                                      configuration._defaults[v]))

def print_state():
    """Print the current configuration state."""
    from devito.logger import info
    for k, v in configuration.items():
        info('%s: %s' % (k, v))
<|endoftext|>


<commit_before>import os
import sys
import stat
import tempfile
import shutil
import ipaddr

from devops.model import Node, Network
from devops.network import IpNetworksPool

class Controller:
    def __init__(self, driver):
        self.driver = driver

        self.networks_pool = IpNetworksPool()
        self._reserve_networks()

        self.home_dir = os.environ.get('DEVOPS_HOME') or os.path.join(os.environ['HOME'], ".devops")
        try:
            os.makedirs(os.path.join(self.home_dir, 'environments'), 0755)
        except OSError:
            sys.exc_clear()

    def build_environment(self, environment):
        environment.work_dir = tempfile.mkdtemp(prefix=os.path.join(self.home_dir, 'environments', environment.name)+'-')
        os.chmod(environment.work_dir, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)

        environment.driver = self.driver

        for network in environment.networks:
            network.ip_addresses = self.networks_pool.get()
            self.driver.create_network(network)
            network.driver = self.driver
            network.start()

        for node in environment.nodes:
            self._build_node(environment, node)
            node.driver = self.driver

    def destroy_environment(self, environment):
        for node in environment.nodes:
            node.stop()
            self.driver.delete_node(node)
            del node.driver

        for network in environment.networks:
            network.stop()
            self.driver.delete_network(network)
            del network.driver
            self.networks_pool.put(network.ip_addresses)

        del environment.driver

        shutil.rmtree(environment.work_dir)

    def _reserve_networks(self):
        with os.popen("ip route") as f:
            for line in f:
                words = line.split()
                if len(words) == 0:
                    continue
                if words[0] == 'default':
                    continue
                address = ipaddr.IPv4Network(words[0])
                self.networks_pool.reserve(address)

    def _build_network(self, environment, network):
        network.ip_addresses = self.networks_pool.get()

        self.driver.create_network(network)

    def _build_node(self, environment, node):
        for disk in filter(lambda d: d.path is None, node.disks):
            fd, disk.path = tempfile.mkstemp(
                prefix=environment.work_dir + '/disk',
                suffix='.' + disk.format
            )
            os.close(fd)
            self.driver.create_disk(disk)

        self.driver.create_node(node)

<commit_msg>[devops] Fix permission problems for Libvirt's disk image files<commit_after>import os
import sys
import stat
import tempfile
import shutil
import ipaddr

from devops.model import Node, Network
from devops.network import IpNetworksPool

class Controller:
    def __init__(self, driver):
        self.driver = driver

        self.networks_pool = IpNetworksPool()
        self._reserve_networks()

        self.home_dir = os.environ.get('DEVOPS_HOME') or os.path.join(os.environ['HOME'], ".devops")
        try:
            os.makedirs(os.path.join(self.home_dir, 'environments'), 0755)
        except OSError:
            sys.exc_clear()

    def build_environment(self, environment):
        environment.work_dir = tempfile.mkdtemp(prefix=os.path.join(self.home_dir, 'environments', environment.name)+'-')
        os.chmod(environment.work_dir, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)

        environment.driver = self.driver

        for network in environment.networks:
            network.ip_addresses = self.networks_pool.get()
            self.driver.create_network(network)
            network.driver = self.driver
            network.start()

        for node in environment.nodes:
            self._build_node(environment, node)
            node.driver = self.driver

    def destroy_environment(self, environment):
        for node in environment.nodes:
            node.stop()
            self.driver.delete_node(node)
            del node.driver

        for network in environment.networks:
            network.stop()
            self.driver.delete_network(network)
            del network.driver
            self.networks_pool.put(network.ip_addresses)

        del environment.driver

        shutil.rmtree(environment.work_dir)

    def _reserve_networks(self):
        with os.popen("ip route") as f:
            for line in f:
                words = line.split()
                if len(words) == 0:
                    continue
                if words[0] == 'default':
                    continue
                address = ipaddr.IPv4Network(words[0])
                self.networks_pool.reserve(address)

    def _build_network(self, environment, network):
        network.ip_addresses = self.networks_pool.get()

        self.driver.create_network(network)

    def _build_node(self, environment, node):
        for disk in filter(lambda d: d.path is None, node.disks):
            fd, disk.path = tempfile.mkstemp(
                prefix=environment.work_dir + '/disk',
                suffix='.' + disk.format
            )
            os.close(fd)
            os.chmod(disk.path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH)
            self.driver.create_disk(disk)

        self.driver.create_node(node)

<|endoftext|>


<commit_before>from lxml import etree

class Element:
    def __init__(self, element):
        self.element = element
        self.tag = self.element.tag
        self.text = self.element.text

    def __getitem__(self, name, default=None):
        return self.element.get(name, default)

    def __setitem__(self, name, value):
        return self.element.set(name, value)

    def find_all(self, xpath):
        "find_all(xpath) - returns list of elements matching given XPath"
        results = []
        for e in self.element.xpath(xpath):
            if hasattr(e, 'xpath'):
                # wrap element nodes with our class
                e = Element(e)
            results.append(e)
        return results

    def find(self, xpath):
        "find(xpath) - returns first element matching given XPath or None"
        elements = self.find_all(xpath)
        if len(elements) == 0:
            return None

        return elements[0]

def parse_file(path):
    "parse_file(path) - parse file contents as XML and return XML document object."
    with file(path) as f:
        return parse_stream(f)

def parse_stream(stream):
    "parse_stream(stream) - parse stream as XML and return XML document object."
    return parse_string(stream.read())

def parse_string(s):
    "parse_string(s) - parse string as XML and return XML document object."
    return Element(etree.fromstring(s))

<commit_msg>[devops] Updated xml parser to return true strings instead of wrapper objects (helps during YAML serialization)<commit_after>from lxml import etree

class Element:
    def __init__(self, element):
        self.element = element
        self.tag = self.element.tag
        self.text = self.element.text

    def __getitem__(self, name, default=None):
        return self.element.get(name, default)

    def __setitem__(self, name, value):
        return self.element.set(name, value)

    def find_all(self, xpath):
        "find_all(xpath) - returns list of elements matching given XPath"
        results = []
        for e in self.element.xpath(xpath):
            if hasattr(e, 'xpath'):
                # wrap element nodes with our class
                e = Element(e)
            else:
                e = str(e)
            results.append(e)
        return results

    def find(self, xpath):
        "find(xpath) - returns first element matching given XPath or None"
        elements = self.find_all(xpath)
        if len(elements) == 0:
            return None

        return elements[0]

def parse_file(path):
    "parse_file(path) - parse file contents as XML and return XML document object."
    with file(path) as f:
        return parse_stream(f)

def parse_stream(stream):
    "parse_stream(stream) - parse stream as XML and return XML document object."
    return parse_string(stream.read())

def parse_string(s):
    "parse_string(s) - parse string as XML and return XML document object."
    return Element(etree.fromstring(s))

<|endoftext|>


<commit_before>import os
import signal
import subprocess
import sys
import time

import yaml

from dib2cloud import util

def sigchld_handler(signum, frame):
    os.waitpid(0, 0)

signal.signal(signal.SIGCHLD, sigchld_handler)

def from_processfile(process_type, path):
    with open(path, 'r') as fh:
        return process_type(**yaml.safe_load(fh))

class Process(object):
    def __init__(self, pid=None):
        self.pid = None

    def start(self):
        self.pid = self._run()
        return self.pid

class PythonProcess(Process):
    def __init__(self, func, *args, **kwargs):
        super(Process, self).__init__()
        self._func = func
        self._args = args
        self._kwargs = kwargs

    def _run(self):
        chpid = os.fork()
        if chpid != 0:
            # We are the parent
            return chpid
        else:
            # We are the child
            self._func(*self._args, **self._kwargs)
            os._exit(0)

class CmdProcess(Process):
    def __init__(self, cmd, stdout, stderr):
        super(Process, self).__init__()
        self._cmd = cmd
        self._stdout = stdout
        self._stderr = stderr
        self._proc = None

    def _run(self):
        self._subproc = subprocess.Popen(self._cmd,
                                         stdout=self._stdout,
                                         stderr=self._stderr)
        return self._subproc.pid

class ProcessTracker(object):
    def __init__(self, uuid, pf_dir, pid=None):
        self.uuid = uuid
        self.pf_dir = pf_dir
        self.pid = pid
        self._proc = None

    @property
    def processfile_path(self):
        util.assert_dir(self.pf_dir)
        return os.path.join(self.pf_dir, '%s.processfile' % self.uuid)

    def to_yaml_file(self, path):
        with open(path, 'w') as fh:
            out = {}
            for attr in self.process_properties + ['uuid', 'pf_dir', 'pid']:
                out[attr] = getattr(self, attr)
            yaml.safe_dump(out, fh)

    def run(self):
        if self.pid:
            raise RuntimeError('Image build for image uuid %s with name %s has'
                               ' already been run.', self.uuid, self.name)
        self._proc = self._get_process()
        self._proc.start()
        self.pid = self._proc.pid
        self.to_yaml_file(self.processfile_path)

    def wait(self, timeout=None):
        if self._proc is not None:
            self._proc.wait(timeout)
        else:
            while True:
                if self.is_running():
                    return True
                else:
                    time.sleep(.5)

    def is_running(self):
        try:
            os.kill(self.pid, 0)
        except OSError:
            return False
        return True
<commit_msg>Add info about using _exit in PythonProcess<commit_after>import os
import signal
import subprocess
import sys
import time

import yaml

from dib2cloud import util

def sigchld_handler(signum, frame):
    os.waitpid(0, 0)

signal.signal(signal.SIGCHLD, sigchld_handler)

def from_processfile(process_type, path):
    with open(path, 'r') as fh:
        return process_type(**yaml.safe_load(fh))

class Process(object):
    def __init__(self, pid=None):
        self.pid = None

    def start(self):
        self.pid = self._run()
        return self.pid

class PythonProcess(Process):
    def __init__(self, func, *args, **kwargs):
        super(Process, self).__init__()
        self._func = func
        self._args = args
        self._kwargs = kwargs

    def _run(self):
        chpid = os.fork()
        if chpid != 0:
            # We are the parent
            return chpid
        else:
            # We are the child
            self._func(*self._args, **self._kwargs)
            # Become the session and group leader
            os.setsid()
            # Use _exit so we don't call any atexit registered functions of
            # our parent. This has the downside of not flushing any stdio fd's
            # so care must be taken when using things like
            # multiprocessing.Queue which rely on a separate i/o thread
            os._exit(0)

class CmdProcess(Process):
    def __init__(self, cmd, stdout, stderr):
        super(Process, self).__init__()
        self._cmd = cmd
        self._stdout = stdout
        self._stderr = stderr
        self._proc = None

    def _run(self):
        self._subproc = subprocess.Popen(self._cmd,
                                         stdout=self._stdout,
                                         stderr=self._stderr)
        return self._subproc.pid

class ProcessTracker(object):
    def __init__(self, uuid, pf_dir, pid=None):
        self.uuid = uuid
        self.pf_dir = pf_dir
        self.pid = pid
        self._proc = None

    @property
    def processfile_path(self):
        util.assert_dir(self.pf_dir)
        return os.path.join(self.pf_dir, '%s.processfile' % self.uuid)

    def to_yaml_file(self, path):
        with open(path, 'w') as fh:
            out = {}
            for attr in self.process_properties + ['uuid', 'pf_dir', 'pid']:
                out[attr] = getattr(self, attr)
            yaml.safe_dump(out, fh)

    def run(self):
        if self.pid:
            raise RuntimeError('Image build for image uuid %s with name %s has'
                               ' already been run.', self.uuid, self.name)
        self._proc = self._get_process()
        self._proc.start()
        self.pid = self._proc.pid
        self.to_yaml_file(self.processfile_path)

    def wait(self, timeout=None):
        if self._proc is not None:
            self._proc.wait(timeout)
        else:
            while True:
                if self.is_running():
                    return True
                else:
                    time.sleep(.5)

    def is_running(self):
        try:
            os.kill(self.pid, 0)
        except OSError:
            return False
        return True
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# bl4de | <bloorq@gmail.com> | https://twitter.com/_bl4de
#
# diggit - gets .git repository
import sys
import os

# some common definitions
VERSION = "0.0.1"
BASE_DIR = ".git/"
OBJECT_DIR = BASE_DIR + "objects/"

DESTINATION_FOLDER = "/Users/bl4de/hacking/playground/diggit/test1/"

def get_object_url(object_hash):
    return OBJECT_DIR + object_hash[0:2] + "/" + object_hash[2:]

def get_object_dir_prefix(object_hash):
    return object_hash[0:2] + "/"

def save_git_object(object_hash):
    complete_url = base_url + "/" + get_object_url(object_hash)

    os.system("wget " + complete_url)
    os.system("mkdir -p " + DESTINATION_FOLDER + OBJECT_DIR +
              get_object_dir_prefix(object_hash))
    os.system(
            "cp " +
            object_hash[2:] + " " + DESTINATION_FOLDER + OBJECT_DIR +
            get_object_dir_prefix(object_hash))

    os.system("cd " + DESTINATION_FOLDER + OBJECT_DIR +
              get_object_dir_prefix(object_hash) +
              " && git cat-file -p " + object_hash)

# main program
if __name__ == "__main__":
    print "### diggit.py ver. {v} ###".format(v=VERSION)

base_url = sys.argv[1]

save_git_object('07603070376d63d911f608120eb4b5489b507692')
<commit_msg>diggit.py - get .git object (1)<commit_after>#!/usr/bin/python
#
# bl4de | <bloorq@gmail.com> | https://twitter.com/_bl4de
#
# diggit - gets .git repository
import sys
import os

# some common definitions
VERSION = "0.0.1"
BASE_DIR = ".git/"
OBJECT_DIR = BASE_DIR + "objects/"

DESTINATION_FOLDER = "/Users/bl4de/hacking/playground/diggit/test1/"

def get_object_url(object_hash):
    return OBJECT_DIR + object_hash[0:2] + "/" + object_hash[2:]

def get_object_dir_prefix(object_hash):
    return object_hash[0:2] + "/"

def save_git_object(object_hash):
    complete_url = base_url + "/" + get_object_url(object_hash)

    os.system("curl '" + complete_url + "' --create-dirs -o '" +
              DESTINATION_FOLDER +
              get_object_url(object_hash) + "'")

    git_object_type = os.popen("cd " + DESTINATION_FOLDER + OBJECT_DIR +
                               get_object_dir_prefix(object_hash) +
                               " && git cat-file -t " + object_hash).read()

    print git_object_type

# main program
if __name__ == "__main__":
    print "### diggit.py ver. {v} ###".format(v=VERSION)

base_url = sys.argv[1]

save_git_object('07603070376d63d911f608120eb4b5489b507692')
<|endoftext|>


<commit_before># Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from dino import environ
from dino import utils
from dino.config import SessionKeys

__author__ = 'Oscar Eriksson <oscar.eriks@gmail.com>'

class OnStatusHooks(object):
    @staticmethod
    def set_status(arg: tuple) -> None:
        data, activity = arg

        # todo: leave rooms on invisible/offline?
        user_id = activity.actor.id
        user_name = environ.env.session.get(SessionKeys.user_name.value, None)
        status = activity.verb

        if status == 'online':
            environ.env.db.set_user_online(user_id)
            activity_json = utils.activity_for_connect(user_id, user_name)
            environ.env.emit('gn_user_connected', activity_json, broadcast=True, include_self=False)
            # TODO: join room events
            #activity_json = utils.activity_for_connect(user_id, user_name)
            #environ.env.emit('gn_user_connected', activity_json, broadcast=True, include_self=False)

        elif status == 'invisible':
            environ.env.db.set_user_invisible(user_id)
            activity_json = utils.activity_for_disconnect(user_id, user_name)
            environ.env.emit('gn_user_disconnected', activity_json, broadcast=True, include_self=False)

        elif status == 'offline':
            environ.env.db.set_user_offline(user_id)
            activity_json = utils.activity_for_disconnect(user_id, user_name)
            environ.env.emit('gn_user_disconnected', activity_json, broadcast=True, include_self=False)

@environ.env.observer.on('on_status')
def _on_status_set_status(arg: tuple) -> None:
    OnStatusHooks.set_status(arg)
<commit_msg>send gn_user_joined events when not invisible anymore<commit_after># Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from dino import environ
from dino import utils
from dino.config import SessionKeys

__author__ = 'Oscar Eriksson <oscar.eriks@gmail.com>'

class OnStatusHooks(object):
    @staticmethod
    def set_status(arg: tuple) -> None:
        data, activity = arg

        user_id = activity.actor.id
        user_name = environ.env.session.get(SessionKeys.user_name.value, None)
        image = environ.env.session.get(SessionKeys.image.value, '')
        status = activity.verb

        if status == 'online':
            environ.env.db.set_user_online(user_id)
            activity_json = utils.activity_for_connect(user_id, user_name)
            environ.env.emit('gn_user_connected', activity_json, broadcast=True, include_self=False)

            rooms = utils.rooms_for_user(user_id)
            for room_id in rooms:
                room_name = utils.get_room_name(room_id)
                activity_json = utils.activity_for_user_joined(user_id, user_name, room_id, room_name, image)
                environ.env.emit('gn_user_joined', activity_json, room=room_id, broadcast=True, include_self=False)

        elif status == 'invisible':
            environ.env.db.set_user_invisible(user_id)
            activity_json = utils.activity_for_disconnect(user_id, user_name)
            environ.env.emit('gn_user_disconnected', activity_json, broadcast=True, include_self=False)

        elif status == 'offline':
            environ.env.db.set_user_offline(user_id)
            activity_json = utils.activity_for_disconnect(user_id, user_name)
            environ.env.emit('gn_user_disconnected', activity_json, broadcast=True, include_self=False)

@environ.env.observer.on('on_status')
def _on_status_set_status(arg: tuple) -> None:
    OnStatusHooks.set_status(arg)
<|endoftext|>