<commit_before>import logging
import math
import tensorflow as tf
import numpy as np

class SingleNeuralNet():
    '''
    A single neural network with fixed hyperparameters/topology.

    This must run in the same process in which it's created.

    This class should be considered private to this module.

    Args:
        num_params: The number of params.
        layer_dims: The number of nodes in each layer.
        layer_activations: The activation function for each layer.
        train_epochs: Epochs per train.
        batch_size: The training batch size.
        keep_prob: The dropoout keep probability.
        regularisation_coefficient: The regularisation coefficient.
    '''

    def __init__(self,
                 num_params,
                 layer_dims,
                 layer_activations,
                 train_epochs,
                 batch_size,
                 keep_prob,
                 regularisation_coefficient):
        self.log = logging.getLogger(__name__)
        self.graph = tf.Graph()
        self.tf_session = tf.Session(graph=self.graph)

        if not len(layer_dims) == len(layer_activations):
            self.log.error('len(layer_dims) != len(layer_activations)')
            raise ValueError

        # All member variables of this class are constants. The only things that change are the TF
        # variables.

        self.num_params = num_params
        self.train_epochs = train_epochs
        self.batch_size = batch_size
        self.keep_prob = keep_prob
        self.regularisation_coefficient = regularisation_coefficient

        with self.graph.as_default():
            # Inputs
            self.input_placeholder = tf.placeholder(tf.float32, shape=[None, self.num_params])
            self.output_placeholder = tf.placeholder(tf.float32, shape=[None, 1])
            self.keep_prob_placeholder = tf.placeholder_with_default(1., shape=[])
            self.regularisation_coefficient_placeholder = tf.placeholder_with_default(0., shape=[])

            # Parameters
            self.weights = []
            self.biases = []

            # Input + internal nodes
            # TODO: Use length scale for setting initial weights?
            prev_layer_dim = self.num_params
            prev_h = self.input_placeholder
            stddev=0.1
            for (i, (dim, act)) in enumerate(zip(layer_dims, layer_activations)):
                self.weights.append(tf.Variable(
                    tf.random_normal([prev_layer_dim, dim], stddev=stddev),
                    name="weight_"+str(i)))
                self.biases.append(tf.Variable(
                    tf.random_normal([dim], stddev=stddev),
                    name="bias_"+str(i)))
                prev_layer_dim = dim
                prev_h = tf.nn.dropout(
                      act(tf.matmul(prev_h, self.weights[-1]) + self.biases[-1]),
                      keep_prob=self.keep_prob_placeholder)

            # Output node
            self.weights.append(tf.Variable(
                tf.random_normal([prev_layer_dim, 1], stddev=stddev),
                name="weight_out"))
            self.biases.append(tf.Variable(
                tf.random_normal([1], stddev=stddev),
                name="bias_out"))
            self.output_var = tf.matmul(prev_h, self.weights[-1]) + self.biases[-1]

            # Loss function and training
            self.loss_func = (
                    tf.reduce_mean(tf.reduce_sum(tf.square(self.output_var - self.output_placeholder),
                                                 reduction_indices=[1]))
                    + self.regularisation_coefficient_placeholder
                            * tf.reduce_mean([tf.nn.l2_loss(W) for W in self.weights]))
            # TODO: Set learning rate based on length scale?
            self.train_step = tf.train.AdamOptimizer().minimize(self.loss_func)

            # Gradient
            self.output_var_gradient = tf.gradients(self.output_var, self.input_placeholder)

            # Initialiser for ... initialising
            self.initialiser = tf.initialize_all_variables()

            # Saver for saving and restoring params
            self.saver = tf.train.Saver(write_version=tf.train.SaverDef.V2)

    def destroy(self):
        self.tf_session.close()

    def init(self):
        '''
        Initializes the net.
        '''
        self.tf_session.run(self.initialiser)

    def load(self, archive):
        '''
        Imports the net from an archive dictionary. You must call exactly one of this and init() before calling any other methods.
        '''
        self.log.debug("Loading neural network")
        self.saver.restore(self.tf_session, "./" + str(archive['saver_path']))

    def save(self):
        '''
        Exports the net to an archive dictionary.
        '''
        # TODO: Use a proper timestamped filename, maybe?
        path = self.saver.save(self.tf_session, "net.ckpt")
        self.log.debug("Saving neural network to: " + path)
        return {'saver_path': path}

    def _loss(self, params, costs):
        '''
        Returns the loss and unregularised loss for the given params and costs.
        '''
        return (self.tf_session.run(
            self.loss_func,
            feed_dict={self.input_placeholder: params,
                       self.output_placeholder: [[c] for c in costs],
                       self.regularisation_coefficient_placeholder: self.regularisation_coefficient,
                       }),
            self.tf_session.run(
                self.loss_func,
                feed_dict={self.input_placeholder: params,
                           self.output_placeholder: [[c] for c in costs],
                           self.regularisation_coefficient_placeholder: 0,
                           }))

    def fit(self, params, costs):
        '''
        Fit the neural net to the provided data

        Args:
            params (array): array of parameter arrays
            costs (array): array of costs (associated with the corresponding parameters)
        '''
        self.log.debug('Fitting neural network')
        if len(params) == 0:
            self.log.error('No data provided.')
            raise ValueError
        if not len(params) == len(costs):
            self.log.error("Params and costs must have the same length")
            raise ValueError

        # The general training procedure is as follows:
        # - set a threshold based on the current loss
        # - train for train_epochs epochs
        # - if the new loss is greater than the threshold then we haven't improved much, so stop
        # - else start from the top
        while True:
            threshold = 0.9 * self._loss(params, costs)[0]
            self.log.debug("Training with threshold " + str(threshold))
            if threshold == 0:
                break
            tot = 0
            for i in range(self.train_epochs):
                # Split the data into random batches, and train on each batch
                indices = np.random.permutation(len(params))
                for j in range(math.ceil(len(params) / self.batch_size)):
                    batch_indices = indices[j * self.batch_size : (j + 1) * self.batch_size]
                    batch_input = [params[index] for index in batch_indices]
                    batch_output = [[costs[index]] for index in batch_indices]
                    self.tf_session.run(self.train_step,
                                        feed_dict={self.input_placeholder: batch_input,
                                                   self.output_placeholder: batch_output,
                                                   self.regularisation_coefficient_placeholder: self.regularisation_coefficient,
                                                   self.keep_prob_placeholder: self.keep_prob,
                                                   })
                (l, ul) = self._loss(params, costs)
                self.losses_list.append(l)
                tot += l
                if i % 10 == 0:
                    self.log.debug('Fit neural network with total training cost ' + str(l)
                            + ', with unregularized cost ' + str(ul))

            (l, ul) = self._loss(params, costs)
            al = tot / float(self.train_epochs)
            self.log.debug('Loss ' + str(l) + ', average loss ' + str(al))
            if l > threshold:
                break

    def cross_validation_loss(self, params, costs):
        '''
        Returns the loss of the network on a cross validation set.

        Args:
            params (array): array of parameter arrays
            costs (array): array of costs (associated with the corresponding parameters)
        '''
        return self.tf_session.run(self.loss_func,
                                  feed_dict={self.input_placeholder: params,
                                  self.output_placeholder: [[c] for c in costs],
                                  })

    def predict_cost(self,params):
        '''
        Produces a prediction of cost from the neural net at params.

        Returns:
            float : Predicted cost at parameters
        '''
        runs = 100
        # Do some runs with dropout, and return the smallest. This is kind of LCB.
        results = [y[0] for y in self.tf_session.run(self.output_var, feed_dict={
                self.input_placeholder: [params] * runs,
                self.keep_prob_placeholder: 0.99})]
        results.sort()
        return results[int(runs * 0.2)]

    def predict_cost_gradient(self,params):
        '''
        Produces a prediction of the gradient of the cost function at params.

        Returns:
            float : Predicted gradient at parameters
        '''
        return self.tf_session.run(self.output_var_gradient, feed_dict={self.input_placeholder: [params]})[0][0]

class NeuralNetImpl():
    '''
    Neural network implementation. This may actually create multiple neural networks with different
    topologies or hyperparameters, and switch between them based on the data.

    This must run in the same process in which it's created.

    Args:
        num_params (int): The number of params.
        fit_hyperparameters (bool): Whether to try to fit the hyperparameters to the data.
    '''

    def __init__(self,
                 num_params = None,
                 fit_hyperparameters = False):

        self.log = logging.getLogger(__name__)
        self.log.debug('Initialising neural network impl')
        if num_params is None:
            self.log.error("num_params must be provided")
            raise ValueError

        # Constants.
        self.num_params = num_params
        self.fit_hyperparameters = fit_hyperparameters

        # Tracking variables. These need to be set when importing and saved when exporting.
        self.last_hyperfit = 0
        self.last_net_reg = 0.01

        self.net = self._make_net(0.01)

    def _make_net(self, reg):
        '''
        Helper method to create a new net with a specified regularisation coefficient. The net is not initialised, so you must call init() or load() on it before any other method.

        Args:
            reg (float): Regularisation coefficient.
        '''
        def gelu_fast(_x):
            return 0.5 * _x * (1 + tf.tanh(tf.sqrt(2 / np.pi) * (_x + 0.044715 * tf.pow(_x, 3))))
        def amazing_abs(_x):
            return tf.maximum(1 - tf.abs(_x), 0)
        return SingleNeuralNet(
                self.num_params,
                [64]*5, [tf.nn.relu]*5,
                100, # train_epochs
                64, # batch_size
                0.99, # keep_prob
                reg)

    def init(self):
        '''
        Initializes the net.
        '''
        self.net.init()

    def load(self, archive):
        '''
        Imports the net from an archive dictionary. You must call exactly one of this and init() before calling any other methods.
        '''
        self.last_hyperfit = int(archive['last_hyperfit'])
        self.last_net_reg = float(archive['last_net_reg'])

        # Destroy the old net, and replace it with the new loaded one.
        self.net.destroy()
        self.net = self._make_net(self.last_net_reg)
        self.net.load(dict(archive['net']))

    def save(self):
        '''
        Exports the net to an archive dictionary.
        '''
        return {'last_hyperfit': self.last_hyperfit,
                'last_net_reg': self.last_net_reg,
                'net': self.net.save(),
                }

    def fit_neural_net(self, all_params, all_costs):
        '''
        Fits the neural net with the appropriate topology to the data

        Args:
            all_params (array): array of all parameter arrays
            all_costs (array): array of costs (associated with the corresponding parameters)
        '''
        self.log.debug('Fitting neural network')
        if len(all_params) == 0:
            self.log.error('No data provided.')
            raise ValueError
        if not len(all_params) == len(all_costs):
            self.log.error("Params and costs must have the same length")
            raise ValueError

        # TODO: Consider adding some kind of "cost capping". Our NNs will never predict costs going
        # off to infinity, so we could be "wasting" training cost due to totally irrelevant points.
        # If we capped the costs to some value then this might help. Note that this is really just
        # another form of cost scaling.

        if self.fit_hyperparameters:
            # Every 20 fits (starting at 5, just because), re-fit the hyperparameters
            if int(len(all_params + 5) / 20) > self.last_hyperfit:
                self.last_hyperfit = int(len(all_params + 5) / 20)

                # Fit regularisation

                # Split the data into training and cross validation
                cv_size = int(len(all_params) / 10)
                train_params = all_params[:-cv_size]
                train_costs = all_costs[:-cv_size]
                cv_params = all_params[cv_size:]
                cv_costs = all_costs[cv_size:]

                orig_cv_loss = self.net.cross_validation_loss(cv_params, cv_costs)
                best_cv_loss = orig_cv_loss

                self.log.debug("Fitting regularisation, current cv loss=" + str(orig_cv_loss))

                # Try a bunch of different regularisation parameters, switching to a new one if it
                # does significantly better on the cross validation set than the old one.
                for r in [0.001, 0.01, 0.1, 1, 10]:
                    net = self._make_net(r)
                    net.init()
                    net.fit(train_params, train_costs)
                    this_cv_loss = net.cross_validation_loss(cv_params, cv_costs)
                    if this_cv_loss < best_cv_loss and this_cv_loss < 0.1 * orig_cv_loss:
                        best_cv_loss = this_cv_loss
                        self.log.debug("Switching to reg=" + str(r) + ", cv loss=" + str(best_cv_loss))
                        self.last_net_reg = r
                        self.net.destroy()
                        self.net = net
                    else:
                        net.destroy()

                # TODO: Fit depth

        self.net.fit(all_params, all_costs)

    def predict_cost(self,params):
        '''
        Produces a prediction of cost from the neural net at params.

        Returns:
            float : Predicted cost at parameters
        '''
        return self.net.predict_cost(params)

    def predict_cost_gradient(self,params):
        '''
        Produces a prediction of the gradient of the cost function at params.

        Returns:
            float : Predicted gradient at parameters
        '''
        return self.net.predict_cost_gradient(params)
<commit_msg>Disable the LCB stuff for now<commit_after>import logging
import math
import tensorflow as tf
import numpy as np

class SingleNeuralNet():
    '''
    A single neural network with fixed hyperparameters/topology.

    This must run in the same process in which it's created.

    This class should be considered private to this module.

    Args:
        num_params: The number of params.
        layer_dims: The number of nodes in each layer.
        layer_activations: The activation function for each layer.
        train_epochs: Epochs per train.
        batch_size: The training batch size.
        keep_prob: The dropoout keep probability.
        regularisation_coefficient: The regularisation coefficient.
    '''

    def __init__(self,
                 num_params,
                 layer_dims,
                 layer_activations,
                 train_epochs,
                 batch_size,
                 keep_prob,
                 regularisation_coefficient):
        self.log = logging.getLogger(__name__)
        self.graph = tf.Graph()
        self.tf_session = tf.Session(graph=self.graph)

        if not len(layer_dims) == len(layer_activations):
            self.log.error('len(layer_dims) != len(layer_activations)')
            raise ValueError

        # All member variables of this class are constants. The only things that change are the TF
        # variables.

        self.num_params = num_params
        self.train_epochs = train_epochs
        self.batch_size = batch_size
        self.keep_prob = keep_prob
        self.regularisation_coefficient = regularisation_coefficient

        with self.graph.as_default():
            # Inputs
            self.input_placeholder = tf.placeholder(tf.float32, shape=[None, self.num_params])
            self.output_placeholder = tf.placeholder(tf.float32, shape=[None, 1])
            self.keep_prob_placeholder = tf.placeholder_with_default(1., shape=[])
            self.regularisation_coefficient_placeholder = tf.placeholder_with_default(0., shape=[])

            # Parameters
            self.weights = []
            self.biases = []

            # Input + internal nodes
            # TODO: Use length scale for setting initial weights?
            prev_layer_dim = self.num_params
            prev_h = self.input_placeholder
            stddev=0.1
            for (i, (dim, act)) in enumerate(zip(layer_dims, layer_activations)):
                self.weights.append(tf.Variable(
                    tf.random_normal([prev_layer_dim, dim], stddev=stddev),
                    name="weight_"+str(i)))
                self.biases.append(tf.Variable(
                    tf.random_normal([dim], stddev=stddev),
                    name="bias_"+str(i)))
                prev_layer_dim = dim
                prev_h = tf.nn.dropout(
                      act(tf.matmul(prev_h, self.weights[-1]) + self.biases[-1]),
                      keep_prob=self.keep_prob_placeholder)

            # Output node
            self.weights.append(tf.Variable(
                tf.random_normal([prev_layer_dim, 1], stddev=stddev),
                name="weight_out"))
            self.biases.append(tf.Variable(
                tf.random_normal([1], stddev=stddev),
                name="bias_out"))
            self.output_var = tf.matmul(prev_h, self.weights[-1]) + self.biases[-1]

            # Loss function and training
            self.loss_func = (
                    tf.reduce_mean(tf.reduce_sum(tf.square(self.output_var - self.output_placeholder),
                                                 reduction_indices=[1]))
                    + self.regularisation_coefficient_placeholder
                            * tf.reduce_mean([tf.nn.l2_loss(W) for W in self.weights]))
            # TODO: Set learning rate based on length scale?
            self.train_step = tf.train.AdamOptimizer().minimize(self.loss_func)

            # Gradient
            self.output_var_gradient = tf.gradients(self.output_var, self.input_placeholder)

            # Initialiser for ... initialising
            self.initialiser = tf.initialize_all_variables()

            # Saver for saving and restoring params
            self.saver = tf.train.Saver(write_version=tf.train.SaverDef.V2)

    def destroy(self):
        self.tf_session.close()

    def init(self):
        '''
        Initializes the net.
        '''
        self.tf_session.run(self.initialiser)

    def load(self, archive):
        '''
        Imports the net from an archive dictionary. You must call exactly one of this and init() before calling any other methods.
        '''
        self.log.debug("Loading neural network")
        self.saver.restore(self.tf_session, "./" + str(archive['saver_path']))

    def save(self):
        '''
        Exports the net to an archive dictionary.
        '''
        # TODO: Use a proper timestamped filename, maybe?
        path = self.saver.save(self.tf_session, "net.ckpt")
        self.log.debug("Saving neural network to: " + path)
        return {'saver_path': path}

    def _loss(self, params, costs):
        '''
        Returns the loss and unregularised loss for the given params and costs.
        '''
        return (self.tf_session.run(
            self.loss_func,
            feed_dict={self.input_placeholder: params,
                       self.output_placeholder: [[c] for c in costs],
                       self.regularisation_coefficient_placeholder: self.regularisation_coefficient,
                       }),
            self.tf_session.run(
                self.loss_func,
                feed_dict={self.input_placeholder: params,
                           self.output_placeholder: [[c] for c in costs],
                           self.regularisation_coefficient_placeholder: 0,
                           }))

    def fit(self, params, costs):
        '''
        Fit the neural net to the provided data

        Args:
            params (array): array of parameter arrays
            costs (array): array of costs (associated with the corresponding parameters)
        '''
        self.log.debug('Fitting neural network')
        if len(params) == 0:
            self.log.error('No data provided.')
            raise ValueError
        if not len(params) == len(costs):
            self.log.error("Params and costs must have the same length")
            raise ValueError

        # The general training procedure is as follows:
        # - set a threshold based on the current loss
        # - train for train_epochs epochs
        # - if the new loss is greater than the threshold then we haven't improved much, so stop
        # - else start from the top
        while True:
            threshold = 0.9 * self._loss(params, costs)[0]
            self.log.debug("Training with threshold " + str(threshold))
            if threshold == 0:
                break
            tot = 0
            for i in range(self.train_epochs):
                # Split the data into random batches, and train on each batch
                indices = np.random.permutation(len(params))
                for j in range(math.ceil(len(params) / self.batch_size)):
                    batch_indices = indices[j * self.batch_size : (j + 1) * self.batch_size]
                    batch_input = [params[index] for index in batch_indices]
                    batch_output = [[costs[index]] for index in batch_indices]
                    self.tf_session.run(self.train_step,
                                        feed_dict={self.input_placeholder: batch_input,
                                                   self.output_placeholder: batch_output,
                                                   self.regularisation_coefficient_placeholder: self.regularisation_coefficient,
                                                   self.keep_prob_placeholder: self.keep_prob,
                                                   })
                (l, ul) = self._loss(params, costs)
                self.losses_list.append(l)
                tot += l
                if i % 10 == 0:
                    self.log.debug('Fit neural network with total training cost ' + str(l)
                            + ', with unregularized cost ' + str(ul))

            (l, ul) = self._loss(params, costs)
            al = tot / float(self.train_epochs)
            self.log.debug('Loss ' + str(l) + ', average loss ' + str(al))
            if l > threshold:
                break

    def cross_validation_loss(self, params, costs):
        '''
        Returns the loss of the network on a cross validation set.

        Args:
            params (array): array of parameter arrays
            costs (array): array of costs (associated with the corresponding parameters)
        '''
        return self.tf_session.run(self.loss_func,
                                  feed_dict={self.input_placeholder: params,
                                  self.output_placeholder: [[c] for c in costs],
                                  })

    def predict_cost(self,params):
        '''
        Produces a prediction of cost from the neural net at params.

        Returns:
            float : Predicted cost at parameters
        '''
        return self.tf_session.run(self.output_var, feed_dict={self.input_placeholder: [params]})[0][0]
        #runs = 100
        ## Do some runs with dropout, and return the smallest. This is kind of LCB.
        #results = [y[0] for y in self.tf_session.run(self.output_var, feed_dict={
        #        self.input_placeholder: [params] * runs,
        #        self.keep_prob_placeholder: 0.99})]
        #results.sort()
        #return results[int(runs * 0.2)]

    def predict_cost_gradient(self,params):
        '''
        Produces a prediction of the gradient of the cost function at params.

        Returns:
            float : Predicted gradient at parameters
        '''
        return self.tf_session.run(self.output_var_gradient, feed_dict={self.input_placeholder: [params]})[0][0]

class NeuralNetImpl():
    '''
    Neural network implementation. This may actually create multiple neural networks with different
    topologies or hyperparameters, and switch between them based on the data.

    This must run in the same process in which it's created.

    Args:
        num_params (int): The number of params.
        fit_hyperparameters (bool): Whether to try to fit the hyperparameters to the data.
    '''

    def __init__(self,
                 num_params = None,
                 fit_hyperparameters = False):

        self.log = logging.getLogger(__name__)
        self.log.debug('Initialising neural network impl')
        if num_params is None:
            self.log.error("num_params must be provided")
            raise ValueError

        # Constants.
        self.num_params = num_params
        self.fit_hyperparameters = fit_hyperparameters

        # Tracking variables. These need to be set when importing and saved when exporting.
        self.last_hyperfit = 0
        self.last_net_reg = 0.01

        self.net = self._make_net(0.01)

    def _make_net(self, reg):
        '''
        Helper method to create a new net with a specified regularisation coefficient. The net is not initialised, so you must call init() or load() on it before any other method.

        Args:
            reg (float): Regularisation coefficient.
        '''
        def gelu_fast(_x):
            return 0.5 * _x * (1 + tf.tanh(tf.sqrt(2 / np.pi) * (_x + 0.044715 * tf.pow(_x, 3))))
        def amazing_abs(_x):
            return tf.maximum(1 - tf.abs(_x), 0)
        return SingleNeuralNet(
                self.num_params,
                [64]*5, [tf.nn.relu]*5,
                100, # train_epochs
                64, # batch_size
                0.99, # keep_prob
                reg)

    def init(self):
        '''
        Initializes the net.
        '''
        self.net.init()

    def load(self, archive):
        '''
        Imports the net from an archive dictionary. You must call exactly one of this and init() before calling any other methods.
        '''
        self.last_hyperfit = int(archive['last_hyperfit'])
        self.last_net_reg = float(archive['last_net_reg'])

        # Destroy the old net, and replace it with the new loaded one.
        self.net.destroy()
        self.net = self._make_net(self.last_net_reg)
        self.net.load(dict(archive['net']))

    def save(self):
        '''
        Exports the net to an archive dictionary.
        '''
        return {'last_hyperfit': self.last_hyperfit,
                'last_net_reg': self.last_net_reg,
                'net': self.net.save(),
                }

    def fit_neural_net(self, all_params, all_costs):
        '''
        Fits the neural net with the appropriate topology to the data

        Args:
            all_params (array): array of all parameter arrays
            all_costs (array): array of costs (associated with the corresponding parameters)
        '''
        self.log.debug('Fitting neural network')
        if len(all_params) == 0:
            self.log.error('No data provided.')
            raise ValueError
        if not len(all_params) == len(all_costs):
            self.log.error("Params and costs must have the same length")
            raise ValueError

        # TODO: Consider adding some kind of "cost capping". Our NNs will never predict costs going
        # off to infinity, so we could be "wasting" training cost due to totally irrelevant points.
        # If we capped the costs to some value then this might help. Note that this is really just
        # another form of cost scaling.

        if self.fit_hyperparameters:
            # Every 20 fits (starting at 5, just because), re-fit the hyperparameters
            if int(len(all_params + 5) / 20) > self.last_hyperfit:
                self.last_hyperfit = int(len(all_params + 5) / 20)

                # Fit regularisation

                # Split the data into training and cross validation
                cv_size = int(len(all_params) / 10)
                train_params = all_params[:-cv_size]
                train_costs = all_costs[:-cv_size]
                cv_params = all_params[cv_size:]
                cv_costs = all_costs[cv_size:]

                orig_cv_loss = self.net.cross_validation_loss(cv_params, cv_costs)
                best_cv_loss = orig_cv_loss

                self.log.debug("Fitting regularisation, current cv loss=" + str(orig_cv_loss))

                # Try a bunch of different regularisation parameters, switching to a new one if it
                # does significantly better on the cross validation set than the old one.
                for r in [0.001, 0.01, 0.1, 1, 10]:
                    net = self._make_net(r)
                    net.init()
                    net.fit(train_params, train_costs)
                    this_cv_loss = net.cross_validation_loss(cv_params, cv_costs)
                    if this_cv_loss < best_cv_loss and this_cv_loss < 0.1 * orig_cv_loss:
                        best_cv_loss = this_cv_loss
                        self.log.debug("Switching to reg=" + str(r) + ", cv loss=" + str(best_cv_loss))
                        self.last_net_reg = r
                        self.net.destroy()
                        self.net = net
                    else:
                        net.destroy()

                # TODO: Fit depth

        self.net.fit(all_params, all_costs)

    def predict_cost(self,params):
        '''
        Produces a prediction of cost from the neural net at params.

        Returns:
            float : Predicted cost at parameters
        '''
        return self.net.predict_cost(params)

    def predict_cost_gradient(self,params):
        '''
        Produces a prediction of the gradient of the cost function at params.

        Returns:
            float : Predicted gradient at parameters
        '''
        return self.net.predict_cost_gradient(params)
<|endoftext|>


<commit_before>import logging
import math
import tensorflow as tf
import numpy as np

class SingleNeuralNet():
    '''
    A single neural network with fixed hyperparameters/topology.

    This must run in the same process in which it's created.

    Args:
        num_params: The number of params.
        num_layers: The number of layers.
        layer_dim: The number of nodes in each layer.
        train_epochs: Epochs per train.
        batch_size: The training batch size.
        keep_prob: The dropoout keep probability.
        regularisation_coefficient: The regularisation coefficient.
    '''

    def __init__(self,
                 num_params,
                 num_layers,
                 layer_dim,
                 train_epochs,
                 batch_size,
                 keep_prob,
                 regularisation_coefficient):
        self.log = logging.getLogger(__name__)
        self.tf_session = tf.InteractiveSession()

        self.num_params = num_params
        self.num_layers = num_layers
        self.layer_dim = layer_dim
        self.train_epochs = train_epochs
        self.batch_size = batch_size
        self.keep_prob = keep_prob
        self.regularisation_coefficient = regularisation_coefficient

        # Inputs
        self.input_placeholder = tf.placeholder(tf.float32, shape=[None, self.num_params])
        self.output_placeholder = tf.placeholder(tf.float32, shape=[None, 1])
        self.keep_prob_placeholder = tf.placeholder_with_default(1., shape=[])
        self.regularisation_coefficient_placeholder = tf.placeholder_with_default(0., shape=[])

        # Parameters
        self.weights = []
        self.biases = []

        # Input + internal nodes
        # TODO: Use length scale for setting initial weights?
        prev_layer_dim = self.num_params
        prev_h = self.input_placeholder
        for dim in [self.layer_dim] * self.num_layers:
            self.weights.append(tf.Variable(tf.random_normal([prev_layer_dim, dim], stddev=0.1)))
            self.biases.append(tf.Variable(tf.random_normal([dim])))
            prev_layer_dim = dim
            prev_h = tf.nn.dropout(
                  tf.nn.sigmoid(tf.matmul(prev_h, self.weights[-1]) + self.biases[-1]),
                  keep_prob=self.keep_prob_placeholder)

        # Output node
        self.weights.append(tf.Variable(tf.random_normal([prev_layer_dim, 1])))
        self.biases.append(tf.Variable(tf.random_normal([1])))
        self.output_var = tf.matmul(prev_h, self.weights[-1]) + self.biases[-1]

        # Loss function and training
        self.loss_func = (
                tf.reduce_mean(tf.reduce_sum(tf.square(self.output_var - self.output_placeholder),
                                             reduction_indices=[1]))
                + self.regularisation_coefficient_placeholder
                        * sum([tf.nn.l2_loss(W) for W in self.weights]))
        self.train_step = tf.train.AdamOptimizer(1.0).minimize(self.loss_func)

        # Gradient
        self.output_var_gradient = tf.gradients(self.output_var, self.input_placeholder)

        self.tf_session.run(tf.initialize_all_variables())

    def fit(self, params, costs):
        '''
        Fit the neural net to the provided data

        Args:
            params (array): array of parameter arrays
            costs (array): array of costs (associated with the corresponding parameters)
        '''
        self.log.debug('Fitting neural network')
        if len(params) == 0:
            self.log.error('No data provided.')
            raise ValueError
        if not len(params) == len(costs):
            self.log.error("Params and costs must have the same length")
            raise ValueError

        for i in range(self.train_epochs):
            # Split the data into random batches, and train on each batch
            indices = np.random.permutation(len(params))
            for j in range(math.ceil(len(params) / self.batch_size)):
                batch_indices = indices[j * self.batch_size : (j + 1) * self.batch_size]
                batch_input = [params[index] for index in batch_indices]
                batch_output = [[costs[index]] for index in batch_indices]
                self.tf_session.run(self.train_step,
                                    feed_dict={self.input_placeholder: batch_input,
                                               self.output_placeholder: batch_output,
                                               self.regularisation_coefficient_placeholder: self.regularisation_coefficient,
                                               self.keep_prob_placeholder: self.keep_prob,
                                               })

        self.log.debug('Fit neural network with total training cost '
                + str(self.tf_session.run(
                        self.loss_func,
                        feed_dict={self.input_placeholder: params,
                                   self.output_placeholder: [[c] for c in costs],
                                   self.regularisation_coefficient_placeholder: self.regularisation_coefficient,
                                   })))

    def cross_validation_loss(self, params, costs):
        '''
        Returns the loss of the network on a cross validation set.

        Args:
            params (array): array of parameter arrays
            costs (array): array of costs (associated with the corresponding parameters)
        '''
        return self.tf_session.run(self.loss_func,
                                  feed_dict={self.input_placeholder: params,
                                  self.output_placeholder: [[c] for c in costs],
                                  })

    def predict_cost(self,params):
        '''
        Produces a prediction of cost from the neural net at params.

        Returns:
            float : Predicted cost at parameters
        '''
        return self.tf_session.run(self.output_var, feed_dict={self.input_placeholder: [params]})[0][0]

    def predict_cost_gradient(self,params):
        '''
        Produces a prediction of the gradient of the cost function at params.

        Returns:
            float : Predicted gradient at parameters
        '''
        return self.tf_session.run(self.output_var_gradient, feed_dict={self.input_placeholder: [params]})[0][0]

class NeuralNetImpl():
    '''
    Neural network implementation. This may actually create multiple neural networks with different
    topologies or hyperparameters, and switch between them based on the data.

    This must run in the same process in which it's created.

    Args:
        num_params (int): The number of params.
        fit_hyperparameters (bool): Whether to try to fit the hyperparameters to the data.
    '''

    def __init__(self,
                 num_params = None,
                 fit_hyperparameters = False):

        self.log = logging.getLogger(__name__)
        self.log.debug('Initialising neural network impl')
        if num_params is None:
            self.log.error("num_params must be provided")
            raise ValueError

        self.num_params = num_params
        self.fit_hyperparameters = fit_hyperparameters
        self.last_hyperfit = 0

        self.net = self._make_net(0.01)

    def _make_net(self, reg):
        '''
        Helper method to create a new net with a specified regularisation coefficient.

        Args:
            reg (float): Regularisation coefficient.
        '''
        return SingleNeuralNet(
                self.num_params,
                1, # num_layers
                128, # layer_dim
                1000, # train_epochs
                64, # batch_size
                1., # keep_prob
                reg)

    def fit_neural_net(self, all_params, all_costs):
        '''
        Fits the neural net with the appropriate topology to the data

        Args:
            all_params (array): array of all parameter arrays
            all_costs (array): array of costs (associated with the corresponding parameters)
        '''
        self.log.debug('Fitting neural network')
        if len(all_params) == 0:
            self.log.error('No data provided.')
            raise ValueError
        if not len(all_params) == len(all_costs):
            self.log.error("Params and costs must have the same length")
            raise ValueError

        # TODO: Consider adding some kind of "cost capping". Our NNs will never predict costs going
        # off to infinity, so we could be "wasting" training cost due to totally irrelevant points.
        # If we capped the costs to some value then this might help. Note that this is really just
        # another form of cost scaling.

        if self.fit_hyperparameters:
            # Every 20 fits (starting at 5, just because), re-fit the hyperparameters
            if int(len(all_params + 5) / 20) > self.last_hyperfit:
                self.last_hyperfit = int(len(all_params + 5) / 20)

                # Fit regularisation

                # Split the data into training and cross validation
                cv_size = int(len(all_params) / 10)
                train_params = all_params[:-cv_size]
                train_costs = all_costs[:-cv_size]
                cv_params = all_params[cv_size:]
                cv_costs = all_costs[cv_size:]

                orig_cv_loss = self.net.cross_validation_loss(cv_params, cv_costs)
                best_cv_loss = orig_cv_loss

                self.log.debug("Fitting regularisation, current cv loss=" + str(orig_cv_loss))

                # Try a bunch of different regularisation parameters, switching to a new one if it
                # does significantly better on the cross validation set than the old one.
                for r in [0.001, 0.01, 0.1, 1, 10]:
                    net = self._make_net(r)
                    net.fit(train_params, train_costs)
                    this_cv_loss = net.cross_validation_loss(cv_params, cv_costs)
                    if this_cv_loss < best_cv_loss and this_cv_loss < 0.1 * orig_cv_loss:
                        best_cv_loss = this_cv_loss
                        self.log.debug("Switching to reg=" + str(r) + ", cv loss=" + str(best_cv_loss))
                        self.net = net

                # TODO: Fit depth

        self.net.fit(all_params, all_costs)

    def predict_cost(self,params):
        '''
        Produces a prediction of cost from the neural net at params.

        Returns:
            float : Predicted cost at parameters
        '''
        return self.net.predict_cost(params)

    def predict_cost_gradient(self,params):
        '''
        Produces a prediction of the gradient of the cost function at params.

        Returns:
            float : Predicted gradient at parameters
        '''
        return self.net.predict_cost_gradient(params)
<commit_msg>Use mean instead of sum for regularization<commit_after>import logging
import math
import tensorflow as tf
import numpy as np

class SingleNeuralNet():
    '''
    A single neural network with fixed hyperparameters/topology.

    This must run in the same process in which it's created.

    Args:
        num_params: The number of params.
        num_layers: The number of layers.
        layer_dim: The number of nodes in each layer.
        train_epochs: Epochs per train.
        batch_size: The training batch size.
        keep_prob: The dropoout keep probability.
        regularisation_coefficient: The regularisation coefficient.
    '''

    def __init__(self,
                 num_params,
                 num_layers,
                 layer_dim,
                 train_epochs,
                 batch_size,
                 keep_prob,
                 regularisation_coefficient):
        self.log = logging.getLogger(__name__)
        self.tf_session = tf.InteractiveSession()

        self.num_params = num_params
        self.num_layers = num_layers
        self.layer_dim = layer_dim
        self.train_epochs = train_epochs
        self.batch_size = batch_size
        self.keep_prob = keep_prob
        self.regularisation_coefficient = regularisation_coefficient

        # Inputs
        self.input_placeholder = tf.placeholder(tf.float32, shape=[None, self.num_params])
        self.output_placeholder = tf.placeholder(tf.float32, shape=[None, 1])
        self.keep_prob_placeholder = tf.placeholder_with_default(1., shape=[])
        self.regularisation_coefficient_placeholder = tf.placeholder_with_default(0., shape=[])

        # Parameters
        self.weights = []
        self.biases = []

        # Input + internal nodes
        # TODO: Use length scale for setting initial weights?
        prev_layer_dim = self.num_params
        prev_h = self.input_placeholder
        for dim in [self.layer_dim] * self.num_layers:
            self.weights.append(tf.Variable(tf.random_normal([prev_layer_dim, dim], stddev=0.1)))
            self.biases.append(tf.Variable(tf.random_normal([dim])))
            prev_layer_dim = dim
            prev_h = tf.nn.dropout(
                  tf.nn.sigmoid(tf.matmul(prev_h, self.weights[-1]) + self.biases[-1]),
                  keep_prob=self.keep_prob_placeholder)

        # Output node
        self.weights.append(tf.Variable(tf.random_normal([prev_layer_dim, 1])))
        self.biases.append(tf.Variable(tf.random_normal([1])))
        self.output_var = tf.matmul(prev_h, self.weights[-1]) + self.biases[-1]

        # Loss function and training
        self.loss_func = (
                tf.reduce_mean(tf.reduce_sum(tf.square(self.output_var - self.output_placeholder),
                                             reduction_indices=[1]))
                + self.regularisation_coefficient_placeholder
                        * tf.reduce_mean([tf.nn.l2_loss(W) for W in self.weights]))
        self.train_step = tf.train.AdamOptimizer(1.0).minimize(self.loss_func)

        # Gradient
        self.output_var_gradient = tf.gradients(self.output_var, self.input_placeholder)

        self.tf_session.run(tf.initialize_all_variables())

    def fit(self, params, costs):
        '''
        Fit the neural net to the provided data

        Args:
            params (array): array of parameter arrays
            costs (array): array of costs (associated with the corresponding parameters)
        '''
        self.log.debug('Fitting neural network')
        if len(params) == 0:
            self.log.error('No data provided.')
            raise ValueError
        if not len(params) == len(costs):
            self.log.error("Params and costs must have the same length")
            raise ValueError

        for i in range(self.train_epochs):
            # Split the data into random batches, and train on each batch
            indices = np.random.permutation(len(params))
            for j in range(math.ceil(len(params) / self.batch_size)):
                batch_indices = indices[j * self.batch_size : (j + 1) * self.batch_size]
                batch_input = [params[index] for index in batch_indices]
                batch_output = [[costs[index]] for index in batch_indices]
                self.tf_session.run(self.train_step,
                                    feed_dict={self.input_placeholder: batch_input,
                                               self.output_placeholder: batch_output,
                                               self.regularisation_coefficient_placeholder: self.regularisation_coefficient,
                                               self.keep_prob_placeholder: self.keep_prob,
                                               })

        self.log.debug('Fit neural network with total training cost '
                + str(self.tf_session.run(
                        self.loss_func,
                        feed_dict={self.input_placeholder: params,
                                   self.output_placeholder: [[c] for c in costs],
                                   self.regularisation_coefficient_placeholder: self.regularisation_coefficient,
                                   })))

    def cross_validation_loss(self, params, costs):
        '''
        Returns the loss of the network on a cross validation set.

        Args:
            params (array): array of parameter arrays
            costs (array): array of costs (associated with the corresponding parameters)
        '''
        return self.tf_session.run(self.loss_func,
                                  feed_dict={self.input_placeholder: params,
                                  self.output_placeholder: [[c] for c in costs],
                                  })

    def predict_cost(self,params):
        '''
        Produces a prediction of cost from the neural net at params.

        Returns:
            float : Predicted cost at parameters
        '''
        return self.tf_session.run(self.output_var, feed_dict={self.input_placeholder: [params]})[0][0]

    def predict_cost_gradient(self,params):
        '''
        Produces a prediction of the gradient of the cost function at params.

        Returns:
            float : Predicted gradient at parameters
        '''
        return self.tf_session.run(self.output_var_gradient, feed_dict={self.input_placeholder: [params]})[0][0]

class NeuralNetImpl():
    '''
    Neural network implementation. This may actually create multiple neural networks with different
    topologies or hyperparameters, and switch between them based on the data.

    This must run in the same process in which it's created.

    Args:
        num_params (int): The number of params.
        fit_hyperparameters (bool): Whether to try to fit the hyperparameters to the data.
    '''

    def __init__(self,
                 num_params = None,
                 fit_hyperparameters = False):

        self.log = logging.getLogger(__name__)
        self.log.debug('Initialising neural network impl')
        if num_params is None:
            self.log.error("num_params must be provided")
            raise ValueError

        self.num_params = num_params
        self.fit_hyperparameters = fit_hyperparameters
        self.last_hyperfit = 0

        self.net = self._make_net(0.01)

    def _make_net(self, reg):
        '''
        Helper method to create a new net with a specified regularisation coefficient.

        Args:
            reg (float): Regularisation coefficient.
        '''
        return SingleNeuralNet(
                self.num_params,
                1, # num_layers
                128, # layer_dim
                1000, # train_epochs
                64, # batch_size
                1., # keep_prob
                reg)

    def fit_neural_net(self, all_params, all_costs):
        '''
        Fits the neural net with the appropriate topology to the data

        Args:
            all_params (array): array of all parameter arrays
            all_costs (array): array of costs (associated with the corresponding parameters)
        '''
        self.log.debug('Fitting neural network')
        if len(all_params) == 0:
            self.log.error('No data provided.')
            raise ValueError
        if not len(all_params) == len(all_costs):
            self.log.error("Params and costs must have the same length")
            raise ValueError

        # TODO: Consider adding some kind of "cost capping". Our NNs will never predict costs going
        # off to infinity, so we could be "wasting" training cost due to totally irrelevant points.
        # If we capped the costs to some value then this might help. Note that this is really just
        # another form of cost scaling.

        if self.fit_hyperparameters:
            # Every 20 fits (starting at 5, just because), re-fit the hyperparameters
            if int(len(all_params + 5) / 20) > self.last_hyperfit:
                self.last_hyperfit = int(len(all_params + 5) / 20)

                # Fit regularisation

                # Split the data into training and cross validation
                cv_size = int(len(all_params) / 10)
                train_params = all_params[:-cv_size]
                train_costs = all_costs[:-cv_size]
                cv_params = all_params[cv_size:]
                cv_costs = all_costs[cv_size:]

                orig_cv_loss = self.net.cross_validation_loss(cv_params, cv_costs)
                best_cv_loss = orig_cv_loss

                self.log.debug("Fitting regularisation, current cv loss=" + str(orig_cv_loss))

                # Try a bunch of different regularisation parameters, switching to a new one if it
                # does significantly better on the cross validation set than the old one.
                for r in [0.001, 0.01, 0.1, 1, 10]:
                    net = self._make_net(r)
                    net.fit(train_params, train_costs)
                    this_cv_loss = net.cross_validation_loss(cv_params, cv_costs)
                    if this_cv_loss < best_cv_loss and this_cv_loss < 0.1 * orig_cv_loss:
                        best_cv_loss = this_cv_loss
                        self.log.debug("Switching to reg=" + str(r) + ", cv loss=" + str(best_cv_loss))
                        self.net = net

                # TODO: Fit depth

        self.net.fit(all_params, all_costs)

    def predict_cost(self,params):
        '''
        Produces a prediction of cost from the neural net at params.

        Returns:
            float : Predicted cost at parameters
        '''
        return self.net.predict_cost(params)

    def predict_cost_gradient(self,params):
        '''
        Produces a prediction of the gradient of the cost function at params.

        Returns:
            float : Predicted gradient at parameters
        '''
        return self.net.predict_cost_gradient(params)
<|endoftext|>


<commit_before># Licensed under GPL3 (see LICENSE)
# coding=utf-8

"""
Classes and utility functions for communicating with SBIG cameras via the INDI protocol, http://www.indilib.org.
"""

import time
import io

from astropy.io import fits

from .indiclient import indiclient

class CCDCam(indiclient):
    """
    Wrap indiclient.indiclient with some camera-specific utility functions to simplify things like taking,
    exposures, configuring camera binning, etc.
    """
    def __init__(self, host, port, driver="CCD Simulator", debug=True):
        super(CCDCam, self).__init__(host, port)
        self.enable_blob()
        self.driver = driver
        self.debug = debug
        if not self.connected:
            self.connect()

        # run this to clear any queued events
        self.process_events()
        self.defvectorlist = []

    @property
    def connected(self):
        """
        Check connection status and return True if connected, False otherwise.
        """
        status = self.get_text(self.driver, "CONNECTION", "CONNECT")
        if status == 'On':
            return True
        else:
            return False

    def connect(self):
        """
        Enable camera connection
        """
        vec = self.set_and_send_switchvector_by_elementlabel(self.driver, "CONNECTION", "Connect")
        if self.debug:
            vec.tell()
        return vec

    def disconnect(self):
        """
        Disable camera connection
        """
        vec = self.set_and_send_switchvector_by_elementlabel(self.driver, "CONNECTION", "Disonnect")
        if self.debug:
            vec.tell()
        return vec

    def _default_def_handler(self, vector, indi):
        """
        Overload the default vector handler to do a vector.tell() so it's clear what's going on
        """
        if self.debug:
            vector.tell()
        pass

    def expose(self, exptime=1.0, exptype="Light"):
        """
        Take exposure and return FITS data
        """
        if exptype not in ["Light", "Dark", "Bias", "Flat"]:
            raise Exception("Invalid exposure type, %s. Must be one of 'Light', 'Dark', 'Bias', or 'Flat'." % exptype)

        if exptime < 0.0 or exptime > 3600.0:
            raise Exception("Invalid exposure time, %f. Must be >= 0 and <= 3600 sec." % exptime)

        ft_vec = self.set_and_send_switchvector_by_elementlabel(self.driver, "CCD_FRAME_TYPE", exptype)
        if self.debug:
            ft_vec.tell()

        exp_vec = self.set_and_send_float(self.driver, "CCD_EXPOSURE", "CCD_EXPOSURE_VALUE", exptime)
        if self.debug:
            exp_vec.tell()

        self.defvectorlist = []
        fitsdata = None
        run = True

        while run:
            self.process_receive_vector_queue()
            while self.receive_event_queue.empty() is False:
                vector = self.receive_event_queue.get()
                if vector.tag.get_type() == "BLOBVector":
                    print("reading out...")
                    blob = vector.get_first_element()
                    if blob.get_plain_format() == ".fits":
                        buf = io.BytesIO(blob.get_data())
                        fitsdata = fits.open(buf)
                    run = False
                    break
            time.sleep(0.1)
        return fitsdata

class MATCam(CCDCam):
    """
    Wrap CCDCam, set the driver to the SBIG driver, and point to the server for the MAT camera.
    """
    def __init__(self, host="192.168.1.105", port=7624):
        super(SBIGCam, self).__init__(host, port, driver="SBIG CCD")
<commit_msg>fix bug that still referred to old class name<commit_after># Licensed under GPL3 (see LICENSE)
# coding=utf-8

"""
Classes and utility functions for communicating with SBIG cameras via the INDI protocol, http://www.indilib.org.
"""

import time
import io

from astropy.io import fits

from .indiclient import indiclient

class CCDCam(indiclient):
    """
    Wrap indiclient.indiclient with some camera-specific utility functions to simplify things like taking,
    exposures, configuring camera binning, etc.
    """
    def __init__(self, host, port, driver="CCD Simulator", debug=True):
        super(CCDCam, self).__init__(host, port)
        self.enable_blob()
        self.driver = driver
        self.debug = debug
        if not self.connected:
            self.connect()

        # run this to clear any queued events
        self.process_events()
        self.defvectorlist = []

    @property
    def connected(self):
        """
        Check connection status and return True if connected, False otherwise.
        """
        status = self.get_text(self.driver, "CONNECTION", "CONNECT")
        if status == 'On':
            return True
        else:
            return False

    def connect(self):
        """
        Enable camera connection
        """
        vec = self.set_and_send_switchvector_by_elementlabel(self.driver, "CONNECTION", "Connect")
        if self.debug:
            vec.tell()
        return vec

    def disconnect(self):
        """
        Disable camera connection
        """
        vec = self.set_and_send_switchvector_by_elementlabel(self.driver, "CONNECTION", "Disonnect")
        if self.debug:
            vec.tell()
        return vec

    def _default_def_handler(self, vector, indi):
        """
        Overload the default vector handler to do a vector.tell() so it's clear what's going on
        """
        if self.debug:
            vector.tell()
        pass

    def expose(self, exptime=1.0, exptype="Light"):
        """
        Take exposure and return FITS data
        """
        if exptype not in ["Light", "Dark", "Bias", "Flat"]:
            raise Exception("Invalid exposure type, %s. Must be one of 'Light', 'Dark', 'Bias', or 'Flat'." % exptype)

        if exptime < 0.0 or exptime > 3600.0:
            raise Exception("Invalid exposure time, %f. Must be >= 0 and <= 3600 sec." % exptime)

        ft_vec = self.set_and_send_switchvector_by_elementlabel(self.driver, "CCD_FRAME_TYPE", exptype)
        if self.debug:
            ft_vec.tell()

        exp_vec = self.set_and_send_float(self.driver, "CCD_EXPOSURE", "CCD_EXPOSURE_VALUE", exptime)
        if self.debug:
            exp_vec.tell()

        self.defvectorlist = []
        fitsdata = None
        run = True

        while run:
            self.process_receive_vector_queue()
            while self.receive_event_queue.empty() is False:
                vector = self.receive_event_queue.get()
                if vector.tag.get_type() == "BLOBVector":
                    print("reading out...")
                    blob = vector.get_first_element()
                    if blob.get_plain_format() == ".fits":
                        buf = io.BytesIO(blob.get_data())
                        fitsdata = fits.open(buf)
                    run = False
                    break
            time.sleep(0.1)
        return fitsdata

class MATCam(CCDCam):
    """
    Wrap CCDCam, set the driver to the SBIG driver, and point to the server for the MAT camera.
    """
    def __init__(self, host="192.168.1.105", port=7624):
        super(MATCam, self).__init__(host, port, driver="SBIG CCD")
<|endoftext|>


<commit_before>#%matplotlib inline

import time

import numpy as np
import numpy.matlib as matlib
import matplotlib.pyplot as pyplot

from keras import utils
from keras.models import Sequential
from keras.layers import Dense, Activation
from keras.optimizers import SGD
from keras.callbacks import EarlyStopping

from keras.datasets import mnist

# Import MNIST database from Keras
(x_train, y_train), (x_test, y_test) = mnist.load_data()

print("\nShape x_train: ", x_train.shape)
print("Shape x_test: ", x_test.shape)

print("Shape y_train: ", y_train.shape)
print("Shape y_test: ", y_test.shape)

test = np.array([[1, 2, 3], [4, 4, 4], [5, 5, 5]])

print("\n", test)

print("\nTest ravel: ", np.ravel(test))

inputs = Input(shape=test.size)<commit_msg>Working mnist implementation<commit_after>#%matplotlib inline
import time

import numpy 			 as np
import numpy.matlib 	 as matlib
import matplotlib.pyplot as pyplot

from keras 				import utils
from keras.models 		import Sequential
from keras.layers 		import Dense, Activation
from keras.optimizers 	import SGD
from keras.callbacks 	import EarlyStopping
from keras.datasets 	import mnist

# Import MNIST database from Keras
(x_train, y_train), (x_test, y_test) = mnist.load_data()

print("first value sum :", x_train[0].sum())
print("y_train: ", y_train[0:10])

testSize = x_test.shape[0] # temporary

x_val = x_test[0:np.floor(testSize/2).astype(int)]
y_val = y_test[0:np.floor(testSize/2).astype(int)]

x_test = x_test[np.floor(testSize/2).astype(int):]
y_test = y_test[np.floor(testSize/2).astype(int):]

m = x_train.shape[0]		# Training data size
inputDim = x_train[0].size 	# Input dimension
testSize = x_test.shape[0]	# Test and validation data sizes
valSize  = x_val.shape[0]
K = 10						# Number of classes

print("")
print("Number of examples: ", m)
print("Input dimension: ", inputDim)

print("")
print("Shape x_train: ", x_train.shape)
print("Shape y_train: ", y_train.shape)

print("Shape x_test: ", x_test.shape)
print("Shape y_test: ", y_test.shape)

print("Shape x_val: ", x_val.shape)
print("Shape y_val: ", y_val.shape)

x_trainUnwrap = np.reshape(x_train, (m,inputDim))
#y_trainUnwrap = np.ravel(y_train)
x_testUnwrap  = np.reshape(x_test, (testSize, inputDim))
#y_testUnwrap  = np.ravel(y_test)
x_valUnwrap  = np.reshape(x_val, (valSize, inputDim))

print("")
print("Shape x_trainUnwrap: ", x_trainUnwrap.shape)
print("Shape x_testUnwrap: ", x_testUnwrap.shape)

# Unwrap labels
y_trainUnwrap = utils.to_categorical(y_train, K)
y_testUnwrap = utils.to_categorical(y_test, K)
y_valUnwrap = utils.to_categorical(y_val, K)

## Network
model = Sequential()

#Input
model.add(Dense(units=50, input_dim=inputDim))
model.add(Activation('tanh'))

#model.add(Dense(units=10))

# Output
model.add(Dense(units=10))
model.add(Activation('softmax'))

# Network hyperparameters
learningRate = 0.01
maxEpochs = 1000
batchSize = 256

# Configure optimizer
sgd = SGD(lr=learningRate, nesterov=False)
model.compile(loss="categorical_crossentropy", optimizer=sgd, metrics=['accuracy'])

# Configure callbacks
earlyStop = EarlyStopping(monitor='val_loss', min_delta=0.001, patience=6, verbose=1, mode='auto')

# Train Network
timerStart = time.time()

hist = model.fit(x_trainUnwrap, y_trainUnwrap, epochs=maxEpochs, batch_size=batchSize, callbacks=[earlyStop] ,validation_data=(x_valUnwrap, y_valUnwrap), verbose=0)
numEpochs = len(hist.history['acc'])

timerEnd = time.time()

eta = timerEnd-timerStart

# Test trained model
metrics = model.evaluate(x_testUnwrap, y_testUnwrap, batch_size=batchSize)
y_pred = model.predict(x_testUnwrap, batch_size=batchSize)

# Information
print('\n')

#print(model.summary())

print('\n')
print("Epochs: ", numEpochs)
print("Elapsed time: ", eta)
print("Elapsed time per epoch: ", eta/numEpochs)
print("Loss: ", metrics[0])
print("Accuracy: ", metrics[1])<|endoftext|>


<commit_before># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from .constants import LOG
from .wsdispatch import WSDispatch
from utils import gen_id, gen_endpoint
import json
import tornado.websocket

class WSException(Exception):
    pass

class PushWSHandler(tornado.websocket.WebSocketHandler):

    """ Core websocket Handler.

        For alternate protocols, it's recommended to superclass this core
        and add appropriate calls where needed. (e.g. device wake-up wrappers
        for flush())
    """

    uaid = None

    def __init__(self, application=None, request=None, **kw):
        self.config = kw.get('config', {})
        self.storage = kw.get('storage', {})
        self.flags = kw.get('flags', {})
        self.logger = kw.get('logger')
        self.dispatch = kw.get('dispatch', WSDispatch())
        # unit test flag.
        self.test = kw.get('test', False)
        if self.test:
            self.return_buffer = ''
        # protocol handlers
        self.funcs = {'hello': self.hello,
                      'register': self.register,
                      'unregister': self.unregister,
                      'ack': self.ack}
        if (not kw.get('test', False)):
            super(PushWSHandler, self).__init__(application, request)

    # Websocket core functions
    def open(self, *args, **kw):
        pass

    def on_message(self, message):
        """ Handle all incoming websocket messages and dispatch to
            appropriate function.
        """
        try:
            msg = json.loads(message)
            mt = msg['messageType']
            if mt not in self.funcs.keys():
                self.error('Unknown command: %s' % msg)
                return
            result, chain = self.funcs[mt](msg)
            self.send(result)
            while chain is not None:
                result, chain = chain(message)
                self.send(result)
        except Exception, e:
            self.logger.log(type='error', msg=repr(e), severity=LOG.ERROR)
            self.error('Unable to process message (see logs)')

    def on_connection_close(self):
        """ The websocket connection has been severed, either directly
            by the client or indirectly via network instability.

            Attempt to garbage collect.
        """
        if self.uaid:
            self.dispatch.release(self.uaid)
        self.uaid = None
        self.close()

    ## Protocol handler functions.
    def hello(self, msg):
        """ Register the UAID as an active listener.
        """
        status = 200
        self.uaid = msg['uaid'] or gen_id()
        if not self.dispatch.register(self.uaid, self.flush, msg):
            self.logger.log(type='error',
                            severity=LOG.DEBUG,
                            msg="Could not register uaid %s" % self.uaid)
            status = 500
            # This is a bad request. Close the channel immediately.
            try:
                self.send({"messageType": "hello",
                           "status": status,
                           "uaid": self.uaid})
            except Exception:
                pass
            self.close()
        # chain the flush function to send existing data.
        # Note: For alternate protocols (e.g. UDP, initialize the calls here.)
        return ({"messageType": "hello",
                "status": status,
                "uaid": self.uaid}, self.flush)

    def close(self):
        """ Close a websocket
        """
        if not self.test:
            return super(PushWSHandler, self).close()

    def register(self, msg):
        """ Request a new endpoint for a channelID
        """
        status = 200
        gid = '%s.%s' % (self.uaid, msg['channelID'])
        response = {"messageType": "register",
                    "status": status,
                    "pushEndpoint": gen_endpoint(self.config, gid) }
        return (response, None)

    def unregister(self, msg):
        """ Drop the channelID
        """
        status = 200
        response = {"messageType": "unregister",
                    "status": status,
                    "channelID": msg['channelID']}
        return (response, None)

    def ack(self, msg):
        """ Client is responding that has processed the channelIDs.
            Remove them from storage
        """
        content = self.storage.get_updates(self.uaid, None, self.logger)
        ids = {}
        for item in content['updates']:
            ids[item['channelID']] = item['version']
        for item in msg['expired']:
            self.storage.delete_appid(self.uaid, item, self.logger,
                                      clearOnly=True)
        for item in msg['updates']:
            try:
                if ids[item['channelID']] == item['version']:
                    self.storage.delete_appid(self.uaid, item['channelID'],
                                              self.logger, clearOnly=True)
            except KeyError:
                continue
        # if there's anything left, send it again.
        return (None, self.flush)

    ## Utility Functions.
    def flush(self, uaid=None, channelID=None, msg=None):
        """ Fetch all pending notifications and publish to client.

            This function is registered to be called whenever a new
            Notification is posted via the REST interface.

        """
        content = self.storage.get_updates(self.uaid, None, self.logger)
        # only send data if we have any.
        if len(content['updates']) or len(content['expired']):
            content['messageType'] = 'notification'
            # force a send here since we can be called outside of the chain
            try:
                self.send(content)
            except IOError:
                self.on_connection_close()
                self._retry()
        # Already sent content, and no additional action required
        # remove any retransmittal timers
        return (None, None)

    def error(self, err, send=True):
        """ Standardize error response
        """
        self.logger.log(type='error', msg=err, severity=LOG.DEBUG)
        if send:
            errmsg = {'messageType': 'error',
                      'error': err}
            try:
                self.send(json.dumps(errmsg))
            except Exception:
                self.on_connection_close()
                raise

    def send(self, message):
        """ Return a response to the client.

            Overwrite for partner specific responses
        """
        if message is None:
            return
        try:
            if not self.test:
                self.write_message(message)
            else:
                self.return_buffer += json.dumps(message)
        except Exception, e:
            self.logger.log(type="error", severity=LOG.ERROR,
                            msg='Unable to send message %s ' % repr(e))
            self.on_connection_close()
            raise
        pass

    def _retry(self):
        ## Use whatever proprietary means to wake the remote device.
        ## Note: if using something like an IP ping, may want to only
        ## attempt a wake up one or two times. IP address may be
        ## reassigned, causing us to spend a lot of effort keeping someone
        ## else's phone from sleeping.
        #
        # instance = torando.ioloop.IOLoop.instance()
        # if self.to is None:
        #   self.to = instance.add_timeout(self.config.get('timeout', 600),
        #                                  self.dispatch.wakeDevice())
        #
        pass
<commit_msg>Removed redundant close<commit_after># This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from .constants import LOG
from .wsdispatch import WSDispatch
from utils import gen_id, gen_endpoint
import json
import tornado.websocket

class WSException(Exception):
    pass

class PushWSHandler(tornado.websocket.WebSocketHandler):

    """ Core websocket Handler.

        For alternate protocols, it's recommended to superclass this core
        and add appropriate calls where needed. (e.g. device wake-up wrappers
        for flush())
    """

    uaid = None

    def __init__(self, application=None, request=None, **kw):
        self.config = kw.get('config', {})
        self.storage = kw.get('storage', {})
        self.flags = kw.get('flags', {})
        self.logger = kw.get('logger')
        self.dispatch = kw.get('dispatch', WSDispatch())
        # unit test flag.
        self.test = kw.get('test', False)
        if self.test:
            self.return_buffer = ''
        # protocol handlers
        self.funcs = {'hello': self.hello,
                      'register': self.register,
                      'unregister': self.unregister,
                      'ack': self.ack}
        if (not kw.get('test', False)):
            super(PushWSHandler, self).__init__(application, request)

    # Websocket core functions
    def open(self, *args, **kw):
        pass

    def on_message(self, message):
        """ Handle all incoming websocket messages and dispatch to
            appropriate function.
        """
        try:
            msg = json.loads(message)
            mt = msg['messageType']
            if mt not in self.funcs.keys():
                self.error('Unknown command: %s' % msg)
                return
            result, chain = self.funcs[mt](msg)
            self.send(result)
            while chain is not None:
                result, chain = chain(message)
                self.send(result)
        except Exception, e:
            self.logger.log(type='error', msg=repr(e), severity=LOG.ERROR)
            self.error('Unable to process message (see logs)')

    def on_connection_close(self):
        """ The websocket connection has been severed, either directly
            by the client or indirectly via network instability.

            Attempt to garbage collect.
        """
        if self.uaid:
            self.dispatch.release(self.uaid)
        self.uaid = None

    def close(self):
        """ Close a websocket
        """
        if not self.test:
            return super(PushWSHandler, self).close()

    ## Protocol handler functions.
    def hello(self, msg):
        """ Register the UAID as an active listener.
        """
        status = 200
        self.uaid = msg['uaid'] or gen_id()
        if not self.dispatch.register(self.uaid, self.flush, msg):
            self.logger.log(type='error',
                            severity=LOG.DEBUG,
                            msg="Could not register uaid %s" % self.uaid)
            status = 500
            # This is a bad request. Close the channel immediately.
            try:
                self.send({"messageType": "hello",
                           "status": status,
                           "uaid": self.uaid})
            except Exception:
                pass
            self.close()
        # chain the flush function to send existing data.
        # Note: For alternate protocols (e.g. UDP, initialize the calls here.)
        return ({"messageType": "hello",
                "status": status,
                "uaid": self.uaid}, self.flush)

    def register(self, msg):
        """ Request a new endpoint for a channelID
        """
        status = 200
        gid = '%s.%s' % (self.uaid, msg['channelID'])
        response = {"messageType": "register",
                    "status": status,
                    "pushEndpoint": gen_endpoint(self.config, gid) }
        return (response, None)

    def unregister(self, msg):
        """ Drop the channelID
        """
        status = 200
        response = {"messageType": "unregister",
                    "status": status,
                    "channelID": msg['channelID']}
        return (response, None)

    def ack(self, msg):
        """ Client is responding that has processed the channelIDs.
            Remove them from storage
        """
        content = self.storage.get_updates(self.uaid, None, self.logger)
        ids = {}
        for item in content['updates']:
            ids[item['channelID']] = item['version']
        for item in msg['expired']:
            self.storage.delete_appid(self.uaid, item, self.logger,
                                      clearOnly=True)
        for item in msg['updates']:
            try:
                if ids[item['channelID']] == item['version']:
                    self.storage.delete_appid(self.uaid, item['channelID'],
                                              self.logger, clearOnly=True)
            except KeyError:
                continue
        # if there's anything left, send it again.
        return (None, self.flush)

    ## Utility Functions.
    def flush(self, uaid=None, channelID=None, msg=None):
        """ Fetch all pending notifications and publish to client.

            This function is registered to be called whenever a new
            Notification is posted via the REST interface.

        """
        content = self.storage.get_updates(self.uaid, None, self.logger)
        # only send data if we have any.
        if len(content['updates']) or len(content['expired']):
            content['messageType'] = 'notification'
            # force a send here since we can be called outside of the chain
            try:
                self.send(content)
            except IOError:
                self.on_connection_close()
                self._retry()
        # Already sent content, and no additional action required
        # remove any retransmittal timers
        return (None, None)

    def error(self, err, send=True):
        """ Standardize error response
        """
        self.logger.log(type='error', msg=err, severity=LOG.DEBUG)
        if send:
            errmsg = {'messageType': 'error',
                      'error': err}
            try:
                self.send(json.dumps(errmsg))
            except Exception:
                self.on_connection_close()
                raise

    def send(self, message):
        """ Return a response to the client.

            Overwrite for partner specific responses
        """
        if message is None:
            return
        try:
            if not self.test:
                self.write_message(message)
            else:
                self.return_buffer += json.dumps(message)
        except Exception, e:
            self.logger.log(type="error", severity=LOG.ERROR,
                            msg='Unable to send message %s ' % repr(e))
            self.on_connection_close()
            raise
        pass

    def _retry(self):
        ## Use whatever proprietary means to wake the remote device.
        ## Note: if using something like an IP ping, may want to only
        ## attempt a wake up one or two times. IP address may be
        ## reassigned, causing us to spend a lot of effort keeping someone
        ## else's phone from sleeping.
        #
        # instance = torando.ioloop.IOLoop.instance()
        # if self.to is None:
        #   self.to = instance.add_timeout(self.config.get('timeout', 600),
        #                                  self.dispatch.wakeDevice())
        #
        pass
<|endoftext|>


<commit_before>"""Identify and anonymize IP addresses."""

import ipaddress
import logging
import regex

from hashlib import md5
from six import u

class tree_node():
    """Simple binary tree with a value, left node, and right node.

    This is used for holding a translation from original to anonymized IP
    addresses.
    """

    def __init__(self, value):
        """Initialize new node."""
        self.left = None
        self.right = None
        self.value = value

    def dump_to_file(self, file_out, depth=0, input_addr=0, output_addr=0):
        """Recursively traverse tree and write translations to output file."""
        # Root node value does not contribute to output_addr, so only update
        # output_addr for nodes after root (depth > 0)
        if depth > 0:
            output_addr = (output_addr << 1) + self.value

        # Only dump nodes at max depth (32) i.e. full 32bit anonymization
        if depth == 32:
            org_ip_str = str(ipaddress.IPv4Address(input_addr))
            new_ip_str = str(ipaddress.IPv4Address(output_addr))
            logging.debug('dumped {}\t{}'.format(org_ip_str, new_ip_str))
            file_out.write('{}\t{}\n'.format(org_ip_str, new_ip_str))
            return

        depth += 1
        if self.left is not None:
            left_path = (input_addr << 1)
            self.left.dump_to_file(file_out, depth, left_path, output_addr)
        if self.right is not None:
            right_path = (input_addr << 1) + 1
            self.right.dump_to_file(file_out, depth, right_path, output_addr)

    def preserve_ipv4_class(self):
        """Initialize tree to preserve IPv4 classes (call only on root node)."""
        node = self
        # IP classes are defined by the number of leading 1's in the address up
        # to the fourth 1, so setup the tree to preserve those
        for i in range(0, 4):
            node.left = tree_node(0)
            node.right = tree_node(1)
            node = node.right

def anonymize_ip_addr(my_ip_tree, line, salt):
    """Replace each IP address in the line with an anonymized IP address.

    Quad-octets that look like masks will be left unchanged.  That is, any
    quad-octet that consists solely of an initial group of 1s followed by 0s
    or initial 0s followed by 1s will be unchanged.
    """
    pattern = '((\d{1,3})\.\d{1,3}\.\d{1,3}\.\d{1,3})(?=/(\d{1,3}))?'
    matches = regex.findall(pattern, line)
    if matches is None:
        return line

    # Escape existing curly braces since string.format will be used to insert
    # anonymized IP addresses
    new_line = line.replace('{', '{{')
    new_line = new_line.replace('}', '}}')
    new_line = regex.sub(pattern, '{}', new_line)

    ip_addrs = []
    for match in matches:
        ip_str = match[0]
        first_octet = match[1]
        ip_int = _ip_to_int(ip_str)
        if _is_mask(ip_int):
            logging.debug("Skipping mask {}".format(ip_str))
            ip_addrs.append(ip_str)
        elif int(first_octet) >= 224:
            # TODO: consider just removing this and anonymizing (if preserving
            # class), but skipping anything in IP class D and class E for now
            logging.debug("Skipping addresses reserved for multicast and R&D {}"
                          .format(ip_str))
            ip_addrs.append(ip_str)
        else:
            new_ip = _convert_to_anon_ip(my_ip_tree, ip_int, salt)
            new_ip_str = str(ipaddress.IPv4Address(new_ip))
            ip_addrs.append(new_ip_str)
            logging.debug("Replaced {} with {}".format(ip_str, new_ip_str))

    return new_line.format(*ip_addrs)

def _convert_to_anon_ip(node, ip_int, salt):
    """Anonymize an IP address using an existing IP tree root node.

    The bits of a given source IP address define a branch in the binary tree,
    where each source bit selects an edge (1=right, 0=left) from the previous
    node and the value at the next node is the anonymized bit.  This process
    is repeated until all prefix bits are exhausted.  The values at each node
    are pseudorandomly generated as needed and are the inverse of their sibling.
    """
    new_ip_int = 0

    for i in range(31, -1, -1):
        # msb is the next bit to anonymize
        msb = (ip_int >> i) & 1
        if node.left is None:
            # Use a salted hash as a deterministic bit generator here so address
            # anonymization is the same regardless of order/context

            # This is a string of all source bits preceding the one to be anonymized
            preceding_bits = '{:032b}'.format(ip_int)[:31 - i]
            rand_bit = _generate_bit_from_hash(salt + preceding_bits)

            # Go ahead and populate both left and right nodes, sacrificing
            # space to simplify control flow
            node.left = tree_node(rand_bit)
            node.right = tree_node(1 - rand_bit)
        if msb:
            node = node.right
        else:
            node = node.left
        new_ip_int |= node.value << i
    return new_ip_int

def _generate_bit_from_hash(hash_input):
    """Return the last bit of the result from hashing the input string."""
    last_hash_digit = md5((hash_input).encode()).hexdigest()[-1]
    return int(last_hash_digit, 16) & 1

def _ip_to_int(ip_str):
    """Convert an IP address string to integer representation."""
    # Need to strip leading zeros so ipaddress does not assume octal notation
    ip_str = regex.sub('0*(\d+)\.0*(\d+)\.0*(\d+)\.0*(\d+)',
                       r'\1.\2.\3.\4', ip_str)
    ip_int = int(ipaddress.IPv4Address(u(ip_str)))
    return int(ip_int)

def _is_mask(possible_mask_int):
    """Determine if the input int is a mask or not.

    If the binary representation starts with all 1s and ends with all 0s
    (or starts with 0s and ends with 1s), then we assume it is a mask.

    Counting the number of times consecutive bits do not match (transitions)
    gives us a reasonable idea of whether or not something is a mask.  With 0
    or 1 transitions, assume the value is a mask.
    e.g. 1100 has only one place where consecutive bits don't match
         0000 has zero
         0110 has two
         0101 has three
    """
    prev_bit = possible_mask_int & 1
    delta_count = 0
    for pos in range(1, 32):
        cur_bit = (possible_mask_int >> pos) & 1
        delta_count += (prev_bit ^ cur_bit)
        prev_bit = cur_bit
    return (delta_count < 2)
<commit_msg>wording<commit_after>"""Identify and anonymize IP addresses."""

import ipaddress
import logging
import regex

from hashlib import md5
from six import u

class tree_node():
    """Simple binary tree with a value, left node, and right node.

    This is used for holding a translation from original to anonymized IP
    addresses.
    """

    def __init__(self, value):
        """Initialize new node."""
        self.left = None
        self.right = None
        self.value = value

    def dump_to_file(self, file_out, depth=0, input_addr=0, output_addr=0):
        """Recursively traverse tree and write translations to output file."""
        # Root node value does not contribute to output_addr, so only update
        # output_addr for nodes after root (depth > 0)
        if depth > 0:
            output_addr = (output_addr << 1) + self.value

        # Only dump nodes at max depth (32) i.e. full 32bit anonymization
        if depth == 32:
            org_ip_str = str(ipaddress.IPv4Address(input_addr))
            new_ip_str = str(ipaddress.IPv4Address(output_addr))
            logging.debug('dumped {}\t{}'.format(org_ip_str, new_ip_str))
            file_out.write('{}\t{}\n'.format(org_ip_str, new_ip_str))
            return

        depth += 1
        if self.left is not None:
            left_path = (input_addr << 1)
            self.left.dump_to_file(file_out, depth, left_path, output_addr)
        if self.right is not None:
            right_path = (input_addr << 1) + 1
            self.right.dump_to_file(file_out, depth, right_path, output_addr)

    def preserve_ipv4_class(self):
        """Initialize tree to preserve IPv4 classes (call only on root node)."""
        node = self
        # IP classes are defined by the number of leading 1's in the address up
        # to the fourth 1, so setup the tree to preserve those
        for i in range(0, 4):
            node.left = tree_node(0)
            node.right = tree_node(1)
            node = node.right

def anonymize_ip_addr(my_ip_tree, line, salt):
    """Replace each IP address in the line with an anonymized IP address.

    Quad-octets that look like masks will be left unchanged.  That is, any
    quad-octet that consists solely of an initial group of 1s followed by 0s
    or initial 0s followed by 1s will be unchanged.
    """
    pattern = '((\d{1,3})\.\d{1,3}\.\d{1,3}\.\d{1,3})(?=/(\d{1,3}))?'
    matches = regex.findall(pattern, line)
    if matches is None:
        return line

    # Escape existing curly braces since string.format will be used to insert
    # anonymized IP addresses
    new_line = line.replace('{', '{{')
    new_line = new_line.replace('}', '}}')
    new_line = regex.sub(pattern, '{}', new_line)

    ip_addrs = []
    for match in matches:
        ip_str = match[0]
        first_octet = match[1]
        ip_int = _ip_to_int(ip_str)
        if _is_mask(ip_int):
            logging.debug("Skipping mask {}".format(ip_str))
            ip_addrs.append(ip_str)
        elif int(first_octet) >= 224:
            # TODO: consider just removing this and anonymizing (if preserving
            # class), but skipping anything in IP class D and class E for now
            logging.debug("Skipping addresses reserved for multicast and R&D {}"
                          .format(ip_str))
            ip_addrs.append(ip_str)
        else:
            new_ip = _convert_to_anon_ip(my_ip_tree, ip_int, salt)
            new_ip_str = str(ipaddress.IPv4Address(new_ip))
            ip_addrs.append(new_ip_str)
            logging.debug("Replaced {} with {}".format(ip_str, new_ip_str))

    return new_line.format(*ip_addrs)

def _convert_to_anon_ip(node, ip_int, salt):
    """Anonymize an IP address using an existing IP tree root node.

    The bits of a given source IP address define a branch in the binary tree,
    where each source bit selects an edge (1=right, 0=left) from the previous
    node and the value at the next node is the anonymized bit.  This process
    is repeated until all prefix bits are exhausted.  The values at each node
    are pseudorandomly generated as needed and are the inverse of their sibling.
    """
    new_ip_int = 0

    for i in range(31, -1, -1):
        # msb is the next bit to anonymize
        msb = (ip_int >> i) & 1
        if node.left is None:
            # Use a salted hash as a deterministic bit generator here so address
            # anonymization is the same regardless of order/context

            # This is a string of all source bits preceding the one to be anonymized
            preceding_bits = '{:032b}'.format(ip_int)[:31 - i]
            anon_bit = _generate_bit_from_hash(salt + preceding_bits)

            # Go ahead and populate both left and right nodes, sacrificing
            # space to simplify control flow
            node.left = tree_node(anon_bit)
            node.right = tree_node(1 - anon_bit)
        if msb:
            node = node.right
        else:
            node = node.left
        new_ip_int |= node.value << i
    return new_ip_int

def _generate_bit_from_hash(hash_input):
    """Return the last bit of the result from hashing the input string."""
    last_hash_digit = md5((hash_input).encode()).hexdigest()[-1]
    return int(last_hash_digit, 16) & 1

def _ip_to_int(ip_str):
    """Convert an IP address string to integer representation."""
    # Need to strip leading zeros so ipaddress does not assume octal notation
    ip_str = regex.sub('0*(\d+)\.0*(\d+)\.0*(\d+)\.0*(\d+)',
                       r'\1.\2.\3.\4', ip_str)
    ip_int = int(ipaddress.IPv4Address(u(ip_str)))
    return int(ip_int)

def _is_mask(possible_mask_int):
    """Determine if the input int is a mask or not.

    If the binary representation starts with all 1s and ends with all 0s
    (or starts with 0s and ends with 1s), then we assume it is a mask.

    Counting the number of times consecutive bits do not match (transitions)
    gives us a reasonable idea of whether or not something is a mask.  With 0
    or 1 transitions, assume the value is a mask.
    e.g. 1100 has only one place where consecutive bits don't match
         0000 has zero
         0110 has two
         0101 has three
    """
    prev_bit = possible_mask_int & 1
    delta_count = 0
    for pos in range(1, 32):
        cur_bit = (possible_mask_int >> pos) & 1
        delta_count += (prev_bit ^ cur_bit)
        prev_bit = cur_bit
    return (delta_count < 2)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# My Model
from utils.ops import ops
import tensorflow as tf
import numpy as np
from  sklearn.datasets import make_blobs

#############################################
#       Deep Adaptive Separator Model       #
#############################################

class KMeans:

	def __init__(self, nb_clusters, nb_iterations=50, graph=None, input_tensor=None):

		self.nb_clusters = nb_clusters
		self.nb_iterations = nb_iterations

		if input_tensor == None:
			self.graph = tf.Graph()
		else:
			self.graph = tf.get_default_graph()

		with self.graph.as_default():
			if input_tensor == None:
				# Spectrogram, embeddings
				# shape = [batch, T*F , E ]
				self.X = tf.placeholder("float", [None, None, None])
			else:
				self.X = input_tensor

			self.input_dim = tf.shape(self.X)[1]
			self.B = tf.shape(self.X)[0]

			# Take randomly 'nb_clusters' vectors from X
			batch_range = tf.tile(tf.reshape(tf.range(self.B, dtype=tf.int32), shape=[self.B, 1, 1]), [1, self.nb_clusters, 1])
			random = tf.random_uniform([self.B, self.nb_clusters, 1], minval = 0, maxval = self.input_dim - 1, dtype = tf.int32)
			indices = tf.concat([batch_range, random], axis = 2)
			self.centroids = tf.gather_nd(self.X, indices)
			self.network

		if graph == None and input_tensor == None:
			# Create a session for this model based on the constructed graph
			self.sess = tf.Session(graph = self.graph)

	def init(self):
		with self.graph.as_default():
			self.sess.run(tf.global_variables_initializer())

	@ops.scope
	def network(self):

		i = tf.constant(0)
		cond = lambda i, m: tf.less(i, self.nb_iterations)
		_ , self.centroids = tf.while_loop(cond, self.body,[i, self.centroids], shape_invariants=[i.get_shape(), tf.TensorShape([None, None, None])])

		return self.centroids, self.get_labels(self.centroids, self.X)

	def body(self ,i, centroids):
		with tf.name_scope('iteration'):
				# Checking the closest clusters
				labels = self.get_labels(centroids, self.X)

				# Creating the matrix equality [ B , S , TF], equality[: , s, :] = [labels == s](float32)
				cluster_range = tf.range(0, tf.shape(centroids)[1])
				equality = tf.map_fn(lambda r: tf.cast(tf.equal(labels, r), tf.float32), cluster_range, dtype=tf.float32)
				equality = tf.transpose(equality, [1 , 0, 2])

				new_centroids = tf.matmul(equality, self.X)/tf.reduce_sum(equality, axis=2, keep_dims=True)
				return [i+1, new_centroids]

	def get_labels(self, centroids, X):
		centroids_ = tf.expand_dims(centroids, 1)
		X_ = tf.expand_dims(X, 2)
		return tf.argmin(tf.norm(X_ - centroids_, axis=3), axis=2, output_type=tf.int32)

	def fit(self, X_train):
		return self.sess.run(self.network, {self.X: X_train})

if __name__ == "__main__":
	nb_samples = 10000
	E = 2
	nb_clusters = 2
	# X1 = np.random.random_sample((nb_samples/2, E))
	# X2 = np.random.random_sample((nb_samples/2, E)) + 2
	# X = np.reshape(np.concatenate((X1,X2), axis=0), (1, nb_samples ,E))
	# X = np.reshape(np.concatenate((X, X), axis=0), (2, nb_samples ,E))
	# print X.shape

	X, y = make_blobs(n_samples=nb_samples, centers=nb_clusters, n_features=E)
	X = X[np.newaxis,:]
	y = y[np.newaxis,:]
	print y
	kmean = KMeans(nb_clusters)
	kmean.init()
	centroids, labels = kmean.fit(X)
	print labels
	print y
	if np.all((y-labels) == 0) or np.all((y+labels) == 1):
		print 'OK'

<commit_msg>can initialize the centroids <commit_after># -*- coding: utf-8 -*-
# My Model
from utils.ops import ops
import tensorflow as tf
import numpy as np
from  sklearn.datasets import make_blobs

#############################################
#       Deep Adaptive Separator Model       #
#############################################

class KMeans:

	def __init__(self, nb_clusters, centroids_init=None, nb_iterations=50, graph=None, input_tensor=None):

		self.nb_clusters = nb_clusters
		self.nb_iterations = nb_iterations

		if input_tensor == None:
			self.graph = tf.Graph()
		else:
			self.graph = tf.get_default_graph()

		with self.graph.as_default():
			if input_tensor == None:
				# Spectrogram, embeddings
				# shape = [batch, T*F , E ]
				self.X = tf.placeholder("float", [None, None, None])
			else:
				self.X = input_tensor

			self.input_dim = tf.shape(self.X)[1]
			self.B = tf.shape(self.X)[0]

			if centroids_init is None:
				# Take randomly 'nb_clusters' vectors from X
				batch_range = tf.tile(tf.reshape(tf.range(self.B, dtype=tf.int32), shape=[self.B, 1, 1]), [1, self.nb_clusters, 1])
				random = tf.random_uniform([self.B, self.nb_clusters, 1], minval = 0, maxval = self.input_dim - 1, dtype = tf.int32)
				indices = tf.concat([batch_range, random], axis = 2)
				self.centroids = tf.gather_nd(self.X, indices)
			else:
				self.centroids = tf.identity(centroids_init)

			self.network

		if graph == None and input_tensor == None:
			# Create a session for this model based on the constructed graph
			self.sess = tf.Session(graph = self.graph)

	def init(self):
		with self.graph.as_default():
			self.sess.run(tf.global_variables_initializer())

	@ops.scope
	def network(self):

		i = tf.constant(0)
		cond = lambda i, m: tf.less(i, self.nb_iterations)
		_ , self.centroids = tf.while_loop(cond, self.body,[i, self.centroids], shape_invariants=[i.get_shape(), tf.TensorShape([None, None, None])])

		return self.centroids, self.get_labels(self.centroids, self.X)

	def body(self ,i, centroids):
		with tf.name_scope('iteration'):
				# Checking the closest clusters
				labels = self.get_labels(centroids, self.X)

				# Creating the matrix equality [ B , S , TF], equality[: , s, :] = [labels == s](float32)
				cluster_range = tf.range(0, tf.shape(centroids)[1])
				equality = tf.map_fn(lambda r: tf.cast(tf.equal(labels, r), tf.float32), cluster_range, dtype=tf.float32)
				equality = tf.transpose(equality, [1 , 0, 2])

				new_centroids = tf.matmul(equality, self.X)/tf.reduce_sum(equality, axis=2, keep_dims=True)
				return [i+1, new_centroids]

	def get_labels(self, centroids, X):
		centroids_ = tf.expand_dims(centroids, 1)
		X_ = tf.expand_dims(X, 2)
		return tf.argmin(tf.norm(X_ - centroids_, axis=3), axis=2, output_type=tf.int32)

	def fit(self, X_train):
		return self.sess.run(self.network, {self.X: X_train})

if __name__ == "__main__":
	nb_samples = 10000
	E = 2
	nb_clusters = 2
	# X1 = np.random.random_sample((nb_samples/2, E))
	# X2 = np.random.random_sample((nb_samples/2, E)) + 2
	# X = np.reshape(np.concatenate((X1,X2), axis=0), (1, nb_samples ,E))
	# X = np.reshape(np.concatenate((X, X), axis=0), (2, nb_samples ,E))
	# print X.shape

	X, y = make_blobs(n_samples=nb_samples, centers=nb_clusters, n_features=E)
	X = X[np.newaxis,:]
	y = y[np.newaxis,:]
	print y
	kmean = KMeans(nb_clusters)
	kmean.init()
	centroids, labels = kmean.fit(X)
	print labels
	print y
	if np.all((y-labels) == 0) or np.all((y+labels) == 1):
		print 'OK'

<|endoftext|>