<commit_before>import argparse
import os
import fnmatch
import cv2
import sys
import random
import matplotlib.pyplot as plt
import cPickle as pickle
import numpy as np

from sklearn import cross_validation
from sklearn.metrics import confusion_matrix

parser = argparse.ArgumentParser(description=("make database"))
parser.add_argument('--net_file',
                     help="file where the serialized network should be saved",
                     type=str,
                     default="trained_net.p")
parser.add_argument('--path_to_data',
                     help="The path to where the training data is",
                     type=str,
                     default="")
parser.add_argument('--emotions',
                    nargs='+',
                    help='The emotions to be used to train the network.',
                    type=str)
parser.add_argument('--display_example_data',
                    dest='display_example_data',
                    action='store_true',
                    default=False,
                    help=("if true, shows a couple of the training examples."))
parser.add_argument('--cv',
                    dest='cv',
                    action='store_true',
                    default=False,
                    help=("if true, runs the cv code to try multiple hyperparameters "
                          "to find one which does best."))
args = parser.parse_args()

sys.path.append("..")

from lib import activationfunctions
from lib import deepbelief as db
from lib import common

def getFiles(cateogry, show=False):
  imgs = []

  extension = "png"
  path = os.path.join(args.path_to_data, cateogry)

  print "reading files from", path

  imageFiles = [(os.path.join(dirpath, f), f)
        for dirpath, dirnames, files in os.walk(path)
        for f in fnmatch.filter(files, '*.' + extension)]

  for fullPath, shortPath in imageFiles:
    img = cv2.imread(fullPath, 0)
    print img.reshape(-1).shape
    imgs += [img.reshape(-1)]
    if show:
      plt.imshow(img, cmap=plt.cm.gray)
      plt.show()

  return imgs

"""Create the files needed to train the network. """
def createTrainingSet(show=False):
  # Read the unsupervised data.
  unsupervised = getFiles("unsupervised")

  labels = []
  data = []

  for index, emotion in enumerate(args.emotions):
    # Read the data for this emotion faces
    print 'Reading data for emotion', emotion
    data_for_emotion = getFiles(emotion)
    data += data_for_emotion
    labels += [index] * len(data_for_emotion)

  data = np.array(data)
  print "data.shape"
  print data.shape
  data, labels = common.shuffle(data, labels)

  unsupervised = np.array(unsupervised)
  labels = common.labelsToVectors(labels, len(args.emotions))

  return unsupervised, data, labels

def seeData():
  unsupervised, data, labels = createTrainingSet()

  if unsupervised:
    print 'show unsupervised data'
    for i in xrange(5):
      plt.imshow(unsupervised[i].reshape((40,30)), cmap=plt.cm.gray)
      plt.show()

  print 'showing supervised data'
  for i in xrange(20):
    plt.imshow(data[i].reshape((40,30)), cmap=plt.cm.gray)
    plt.show()

def trainNetWithAllData():
  unsupervisedData, data, labels = createTrainingSet()

  print "data.shape"
  print data.shape
  print "labels.shape"
  print labels.shape

  data = common.scale(data)
  unsupervisedData = None

  activationFunction = activationfunctions.Rectified()
  rbmActivationFunctionVisible = activationfunctions.Identity()
  rbmActivationFunctionHidden = activationfunctions.RectifiedNoisy()

  unsupervisedLearningRate = 0.0001
  supervisedLearningRate = 0.001
  momentumMax = 0.99

  net = db.DBN(4, [1200, 1500, 1000, len(args.emotions)],
             binary=False,
             activationFunction=activationFunction,
             rbmActivationFunctionVisible=rbmActivationFunctionVisible,
             rbmActivationFunctionHidden=rbmActivationFunctionHidden,
             unsupervisedLearningRate=unsupervisedLearningRate,
             supervisedLearningRate=supervisedLearningRate,
             momentumMax=momentumMax,
             nesterovMomentum=True,
             rbmNesterovMomentum=True,
             rmsprop=True,
             miniBatchSize=20,
             hiddenDropout=0.5,
             visibleDropout=0.8,
             momentumFactorForLearningRateRBM=False,
             firstRBMheuristic=False,
             rbmVisibleDropout=1.0,
             rbmHiddenDropout=1.0,
             preTrainEpochs=10,
             sparsityConstraintRbm=False,
             sparsityRegularizationRbm=0.001,
             sparsityTragetRbm=0.01)

  net.train(data, labels, maxEpochs=200,
            validation=False,
            unsupervisedData=unsupervisedData)

  with open(args.net_file, "wb") as f:
    pickle.dump(net, f)
  return net

def trainAndTestNet():
  unsupervisedData, data, labels = createTrainingSet()

  print np.unique(np.argmax(labels, axis=1))

  print "data.shape"
  print data.shape
  print "labels.shape"
  print labels.shape

  # Random data for training and testing
  kf = cross_validation.KFold(n=len(data), k=5)
  for train, test in kf:
    break

  print data
  data = common.scale(data)
  unsupervisedData = None

  activationFunction = activationfunctions.Rectified()
  rbmActivationFunctionVisible = activationfunctions.Identity()
  rbmActivationFunctionHidden = activationfunctions.RectifiedNoisy()

  unsupervisedLearningRate = 0.0001
  supervisedLearningRate = 0.001
  momentumMax = 0.99

  trainData = data[train]
  trainLabels = labels[train]

  net = db.DBN(4, [1200, 1500, 1000, len(args.emotions)],
             binary=False,
             activationFunction=activationFunction,
             rbmActivationFunctionVisible=rbmActivationFunctionVisible,
             rbmActivationFunctionHidden=rbmActivationFunctionHidden,
             unsupervisedLearningRate=unsupervisedLearningRate,
             supervisedLearningRate=supervisedLearningRate,
             momentumMax=momentumMax,
             nesterovMomentum=True,
             rbmNesterovMomentum=True,
             rmsprop=True,
             miniBatchSize=20,
             hiddenDropout=0.5,
             visibleDropout=0.8,
             momentumFactorForLearningRateRBM=False,
             firstRBMheuristic=False,
             rbmVisibleDropout=1.0,
             rbmHiddenDropout=1.0,
             preTrainEpochs=10,
             sparsityConstraintRbm=False,
             sparsityRegularizationRbm=0.001,
             sparsityTragetRbm=0.01)

  net.train(trainData, trainLabels, maxEpochs=200,
            validation=False,
            unsupervisedData=unsupervisedData)

  probs, predicted = net.classify(data[test])

  actualLabels = labels[test]
  correct = 0
  errorCases = []

  for i in xrange(len(test)):
    actual = actualLabels[i]
    print probs[i]
    if predicted[i] == np.argmax(actual):
      correct += 1
    else:
      errorCases.append(i)

  print "correct"
  print correct

  print "percentage correct"
  print correct  * 1.0 / len(test)

  confMatrix = confusion_matrix(np.argmax(actualLabels, axis=1), predicted)
  print "confusion matrix"
  print confMatrix

  with open(args.net_file, "wb") as f:
    pickle.dump(net, f)
  return net

def getHyperParamsAndBestNet():
  unsupervisedData, data, labels = createTrainingSet()

  print np.unique(np.argmax(labels, axis=1))

  print "data.shape"
  print data.shape
  print "labels.shape"
  print labels.shape

  print data
  data = common.scale(data)
  unsupervisedData = None

  activationFunction = activationfunctions.Rectified()
  rbmActivationFunctionVisible = activationfunctions.Identity()
  rbmActivationFunctionHidden = activationfunctions.RectifiedNoisy()

  tried_params = []
  percentages = []
  best_index = 0
  index = 0
  best_correct = 0

  # Random data for training and testing
  kf = cross_validation.KFold(n=len(data), n_folds=10)
  for train, test in kf:
    unsupervisedLearningRate = random.uniform(0.0001, 0.2)
    supervisedLearningRate = random.uniform(0.0001, 0.2)
    momentumMax = random.uniform(0.7, 1)

    tried_params += [{'unsupervisedLearningRate': unsupervisedLearningRate,
                      'supervisedLearningRate': supervisedLearningRate,
                      'momentumMax': momentumMax}]

    trainData = data[train]
    trainLabels = labels[train]

    net = db.DBN(4, [1200, 1500, 1000, len(args.emotions)],
               binary=False,
               activationFunction=activationFunction,
               rbmActivationFunctionVisible=rbmActivationFunctionVisible,
               rbmActivationFunctionHidden=rbmActivationFunctionHidden,
               unsupervisedLearningRate=unsupervisedLearningRate,
               supervisedLearningRate=supervisedLearningRate,
               momentumMax=momentumMax,
               nesterovMomentum=True,
               rbmNesterovMomentum=True,
               rmsprop=True,
               miniBatchSize=20,
               hiddenDropout=0.5,
               visibleDropout=0.8,
               momentumFactorForLearningRateRBM=False,
               firstRBMheuristic=False,
               rbmVisibleDropout=1.0,
               rbmHiddenDropout=1.0,
               preTrainEpochs=10,
               sparsityConstraintRbm=False,
               sparsityRegularizationRbm=0.001,
               sparsityTragetRbm=0.01)

    net.train(trainData, trainLabels, maxEpochs=200,
              validation=False,
              unsupervisedData=unsupervisedData)

    probs, predicted = net.classify(data[test])

    actualLabels = labels[test]
    correct = 0

    for i in xrange(len(test)):
      actual = actualLabels[i]
      print probs[i]
      if predicted[i] == np.argmax(actual):
        correct += 1

    percentage_correct = correct * 1.0 / len(test)
    print "percentage correct"
    print percentage_correct

    if percentage_correct > best_correct:
      best_index = index
      best_correct = percentage_correct
      with open(args.net_file, "wb") as f:
        pickle.dump(net, f)

    percentages += [percentage_correct]
    index += 1

  print 'best params'
  print tried_params[best_index]
  print 'precision'
  print best_correct

if __name__ == '__main__':
  if args.display_example_data:
    seeData()

  if args.cv:
    getHyperParamsAndBestNet()
  else:
    trainNetWithAllData()

<commit_msg>Clean up<commit_after>import argparse
import os
import fnmatch
import cv2
import sys
import random
import matplotlib.pyplot as plt
import cPickle as pickle
import numpy as np

from sklearn import cross_validation
from sklearn.metrics import confusion_matrix

parser = argparse.ArgumentParser(description=("make database"))
parser.add_argument('--net_file',
                     help="file where the serialized network should be saved",
                     type=str,
                     default="trained_net.p")
parser.add_argument('--path_to_data',
                     help="The path to where the training data is",
                     type=str,
                     default="")
parser.add_argument('--emotions',
                    nargs='+',
                    help='The emotions to be used to train the network.',
                    type=str)
parser.add_argument('--display_example_data',
                    dest='display_example_data',
                    action='store_true',
                    default=False,
                    help=("if true, shows a couple of the training examples."))
parser.add_argument('--cv',
                    dest='cv',
                    action='store_true',
                    default=False,
                    help=("if true, runs the cv code to try multiple hyperparameters "
                          "to find one which does best."))
args = parser.parse_args()

sys.path.append("..")

from lib import activationfunctions
from lib import deepbelief as db
from lib import common

def getFiles(cateogry, show=False):
  """Returns all the png files in the subdirectory given by the input argument.

  The images are returns as a list of numpy arrays.
  """
  imgs = []

  extension = "png"
  path = os.path.join(args.path_to_data, cateogry)

  print "reading files from", path

  imageFiles = [(os.path.join(dirpath, f), f)
        for dirpath, dirnames, files in os.walk(path)
        for f in fnmatch.filter(files, '*.' + extension)]

  for fullPath, shortPath in imageFiles:
    img = cv2.imread(fullPath, 0)
    print img.reshape(-1).shape
    imgs += [img.reshape(-1)]
    if show:
      plt.imshow(img, cmap=plt.cm.gray)
      plt.show()

  return imgs

def createTrainingSet(show=False):
  # Read the unsupervised data.
  unsupervised = getFiles("unsupervised")

  labels = []
  data = []

  for index, emotion in enumerate(args.emotions):
    # Read the data for this emotion faces
    print 'Reading data for emotion', emotion
    data_for_emotion = getFiles(emotion)
    data += data_for_emotion
    labels += [index] * len(data_for_emotion)

  data = np.array(data)
  print "data.shape"
  print data.shape
  data, labels = common.shuffle(data, labels)

  unsupervised = np.array(unsupervised)
  labels = common.labelsToVectors(labels, len(args.emotions))

  return unsupervised, data, labels

def visualizeTrainingData():
  unsupervised, data, labels = createTrainingSet()

  if unsupervised:
    print 'show unsupervised data'
    for i in xrange(5):
      plt.imshow(unsupervised[i].reshape((40,30)), cmap=plt.cm.gray)
      plt.show()

  print 'showing supervised data'
  for i in xrange(20):
    plt.imshow(data[i].reshape((40,30)), cmap=plt.cm.gray)
    plt.show()

def trainNetWithAllData():
  unsupervisedData, data, labels = createTrainingSet()

  print "data.shape"
  print data.shape
  print "labels.shape"
  print labels.shape

  data = common.scale(data)
  unsupervisedData = None

  activationFunction = activationfunctions.Rectified()
  rbmActivationFunctionVisible = activationfunctions.Identity()
  rbmActivationFunctionHidden = activationfunctions.RectifiedNoisy()

  unsupervisedLearningRate = 0.0001
  supervisedLearningRate = 0.001
  momentumMax = 0.99

  net = db.DBN(4, [1200, 1500, 1000, len(args.emotions)],
             binary=False,
             activationFunction=activationFunction,
             rbmActivationFunctionVisible=rbmActivationFunctionVisible,
             rbmActivationFunctionHidden=rbmActivationFunctionHidden,
             unsupervisedLearningRate=unsupervisedLearningRate,
             supervisedLearningRate=supervisedLearningRate,
             momentumMax=momentumMax,
             nesterovMomentum=True,
             rbmNesterovMomentum=True,
             rmsprop=True,
             miniBatchSize=20,
             hiddenDropout=0.5,
             visibleDropout=0.8,
             momentumFactorForLearningRateRBM=False,
             firstRBMheuristic=False,
             rbmVisibleDropout=1.0,
             rbmHiddenDropout=1.0,
             preTrainEpochs=10,
             sparsityConstraintRbm=False,
             sparsityRegularizationRbm=0.001,
             sparsityTragetRbm=0.01)

  net.train(data, labels, maxEpochs=200,
            validation=False,
            unsupervisedData=unsupervisedData)

  with open(args.net_file, "wb") as f:
    pickle.dump(net, f)
  return net

def trainAndTestNet():
  unsupervisedData, data, labels = createTrainingSet()

  print np.unique(np.argmax(labels, axis=1))

  print "data.shape"
  print data.shape
  print "labels.shape"
  print labels.shape

  # Random data for training and testing
  kf = cross_validation.KFold(n=len(data), k=5)
  for train, test in kf:
    break

  print data
  data = common.scale(data)
  unsupervisedData = None

  activationFunction = activationfunctions.Rectified()
  rbmActivationFunctionVisible = activationfunctions.Identity()
  rbmActivationFunctionHidden = activationfunctions.RectifiedNoisy()

  unsupervisedLearningRate = 0.0001
  supervisedLearningRate = 0.001
  momentumMax = 0.99

  trainData = data[train]
  trainLabels = labels[train]

  net = db.DBN(4, [1200, 1500, 1000, len(args.emotions)],
             binary=False,
             activationFunction=activationFunction,
             rbmActivationFunctionVisible=rbmActivationFunctionVisible,
             rbmActivationFunctionHidden=rbmActivationFunctionHidden,
             unsupervisedLearningRate=unsupervisedLearningRate,
             supervisedLearningRate=supervisedLearningRate,
             momentumMax=momentumMax,
             nesterovMomentum=True,
             rbmNesterovMomentum=True,
             rmsprop=True,
             miniBatchSize=20,
             hiddenDropout=0.5,
             visibleDropout=0.8,
             momentumFactorForLearningRateRBM=False,
             firstRBMheuristic=False,
             rbmVisibleDropout=1.0,
             rbmHiddenDropout=1.0,
             preTrainEpochs=10,
             sparsityConstraintRbm=False,
             sparsityRegularizationRbm=0.001,
             sparsityTragetRbm=0.01)

  net.train(trainData, trainLabels, maxEpochs=200,
            validation=False,
            unsupervisedData=unsupervisedData)

  probs, predicted = net.classify(data[test])

  actualLabels = labels[test]
  correct = 0
  errorCases = []

  for i in xrange(len(test)):
    actual = actualLabels[i]
    print probs[i]
    if predicted[i] == np.argmax(actual):
      correct += 1
    else:
      errorCases.append(i)

  print "correct"
  print correct

  print "percentage correct"
  print correct  * 1.0 / len(test)

  confMatrix = confusion_matrix(np.argmax(actualLabels, axis=1), predicted)
  print "confusion matrix"
  print confMatrix

  with open(args.net_file, "wb") as f:
    pickle.dump(net, f)
  return net

# Performs CV to choose the best hyperparameters given the data.
def getHyperParamsAndBestNet():
  unsupervisedData, data, labels = createTrainingSet()

  print np.unique(np.argmax(labels, axis=1))

  print "data.shape"
  print data.shape
  print "labels.shape"
  print labels.shape

  print data
  data = common.scale(data)
  unsupervisedData = None

  activationFunction = activationfunctions.Rectified()
  rbmActivationFunctionVisible = activationfunctions.Identity()
  rbmActivationFunctionHidden = activationfunctions.RectifiedNoisy()

  tried_params = []
  percentages = []
  best_index = 0
  index = 0
  best_correct = 0

  # Random data for training and testing
  kf = cross_validation.KFold(n=len(data), n_folds=10)
  for train, test in kf:
    unsupervisedLearningRate = random.uniform(0.0001, 0.2)
    supervisedLearningRate = random.uniform(0.0001, 0.2)
    momentumMax = random.uniform(0.7, 1)

    tried_params += [{'unsupervisedLearningRate': unsupervisedLearningRate,
                      'supervisedLearningRate': supervisedLearningRate,
                      'momentumMax': momentumMax}]

    trainData = data[train]
    trainLabels = labels[train]

    net = db.DBN(4, [1200, 1500, 1000, len(args.emotions)],
               binary=False,
               activationFunction=activationFunction,
               rbmActivationFunctionVisible=rbmActivationFunctionVisible,
               rbmActivationFunctionHidden=rbmActivationFunctionHidden,
               unsupervisedLearningRate=unsupervisedLearningRate,
               supervisedLearningRate=supervisedLearningRate,
               momentumMax=momentumMax,
               nesterovMomentum=True,
               rbmNesterovMomentum=True,
               rmsprop=True,
               miniBatchSize=20,
               hiddenDropout=0.5,
               visibleDropout=0.8,
               momentumFactorForLearningRateRBM=False,
               firstRBMheuristic=False,
               rbmVisibleDropout=1.0,
               rbmHiddenDropout=1.0,
               preTrainEpochs=10,
               sparsityConstraintRbm=False,
               sparsityRegularizationRbm=0.001,
               sparsityTragetRbm=0.01)

    net.train(trainData, trainLabels, maxEpochs=200,
              validation=False,
              unsupervisedData=unsupervisedData)

    probs, predicted = net.classify(data[test])

    actualLabels = labels[test]
    correct = 0

    for i in xrange(len(test)):
      actual = actualLabels[i]
      print probs[i]
      if predicted[i] == np.argmax(actual):
        correct += 1

    percentage_correct = correct * 1.0 / len(test)
    print "percentage correct"
    print percentage_correct

    if percentage_correct > best_correct:
      best_index = index
      best_correct = percentage_correct
      with open(args.net_file, "wb") as f:
        pickle.dump(net, f)

    percentages += [percentage_correct]
    index += 1

  print 'best params'
  print tried_params[best_index]
  print 'precision'
  print best_correct

if __name__ == '__main__':
  if args.display_example_data:
    visualizeTrainingData()

  if args.cv:
    getHyperParamsAndBestNet()
  else:
    trainNetWithAllData()

<|endoftext|>


<commit_before># Copyright 2016 the original author or authors.
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

__author__ = "Julian Debatin"
__copyright__ = "The authors"
__license__ = "Apache 2"
__email__ = "juliandebatin@gmail.com"
__status__ = "Production"

from Core.CellType import *
from Core.ExecConfig import ExecConfig
from Core.ModelConfig import ModelConfig
from Steppable.InitializerSteppable import InitializerSteppable
from Steppable.DeathSteppable import DeathSteppable
from Steppable.GrowthMitosisSteppable import GrowthMitosisSteppable
from Steppable.GrowthSteppable import GrowthSteppable
from Steppable.CMTransformationSteppable import CMTransformationSteppable
from Steppable.UrinationSteppable import UrinationSteppable
from Logger.VolumeFitnessSteppable import VolumeFitnessSteppable
from Logger.ArrangementFitnessSteppable import ArrangementFitnessSteppable
from Logger.DummyFitnessSteppable import DummyFitnessSteppable
from Steppable.ColonySteppable import ColonySteppable
from Steppable.MutationSteppable import MutationSteppable

class SpaCdbCdiInUa(ModelConfig):
    def __init__(self, sim, simthread):
        ModelConfig.__init__(self, sim, simthread)

    def _initModel(self):
        self.name = "SpaCdbCdiInUa"
        self.cellTypes = self._createCellTypes()
        self.energyMatrix = self._createEnergyMatrix()
        self._run() # Must be the last statement.

    def _createCellTypes(self):
        cellTypes = []
        stem = Stemcell
        stem.setGrowthVolumePerDayRelVolume(0.1)
        self.stemNecrosisProb = stem.necrosisProb = 0

        basal = Basalcell
        basal.setGrowthVolumePerDayRelVolume(0.1)
        basal.apoptosisTimeInDays = 8000000000.0
        self.basalNecrosisProb = basal.necrosisProb = 0.000003

        intermediate = Intermediatecell
        intermediate.setGrowthVolumePerDayRelVolume(0.09)
        intermediate.apoptosisTimeInDays = 20000000000.0
        self.intermediateNecrosisProb = intermediate.necrosisProb = 0.000005

        umbrella = Umbrellacell
        umbrella.setGrowthVolumePerDayRelVolume(0.08)
        umbrella.apoptosisTimeInDays = 1000000000.0
        self.umbrellaNecrosisProb = umbrella.necrosisProb = 0.000008

        stem.setDescendants(0.90, [stem.id, basal.id])
        stem.setDescendants(0.05, [stem.id, stem.id])
        stem.setDescendants(0.05, [basal.id, basal.id])

        cellTypes.extend((Medium, Basalmembrane, stem, basal, intermediate, umbrella))

        return cellTypes

    def _getSteppables(self):
        steppableList = []
        steppableList.append(ColonySteppable(self.sim, self))
        steppableList.append(InitializerSteppable(self.sim, self))
        steppableList.append(GrowthSteppable(self.sim, self))
        steppableList.append(GrowthMitosisSteppable(self.sim, self))
        steppableList.append(CMTransformationSteppable(self.sim, self))
        steppableList.append(UrinationSteppable(self.sim, self, prop=0.02))
        steppableList.append(DeathSteppable(self.sim, self))
        #steppableList.append(OptimumSearchSteppable(self.sim, self))
        steppableList.append(VolumeFitnessSteppable(self.sim, self))
        steppableList.append(ArrangementFitnessSteppable(self.sim, self))
        steppableList.append(DummyFitnessSteppable(self.sim, self))
        steppableList.append(MutationSteppable(self.sim, self, self.stemNecrosisProb, self.basalNecrosisProb,
                                               self.intermediateNecrosisProb, self.umbrellaNecrosisProb))

        return steppableList

    def _createExecConfig(self):
        return ExecConfig(MCSperDay=500, #SEED=10,
                          xLength=500, yLength=150, zLength=0, voxelDensity=.8)
<commit_msg>no message<commit_after># Copyright 2016 the original author or authors.
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

__author__ = "Julian Debatin"
__copyright__ = "The authors"
__license__ = "Apache 2"
__email__ = "juliandebatin@gmail.com"
__status__ = "Production"

from Core.CellType import *
from Core.ExecConfig import ExecConfig
from Core.ModelConfig import ModelConfig
from Steppable.InitializerSteppable import InitializerSteppable
from Steppable.DeathSteppable import DeathSteppable
from Steppable.GrowthMitosisSteppable import GrowthMitosisSteppable
from Steppable.GrowthSteppable import GrowthSteppable
from Steppable.CMTransformationSteppable import CMTransformationSteppable
from Steppable.UrinationSteppable import UrinationSteppable
from Logger.VolumeFitnessSteppable import VolumeFitnessSteppable
from Logger.ArrangementFitnessSteppable import ArrangementFitnessSteppable
from Logger.DummyFitnessSteppable import DummyFitnessSteppable
from Steppable.ColonySteppable import ColonySteppable
from Steppable.MutationSteppable import MutationSteppable

class SpaCdbCdiInUa(ModelConfig):
    def __init__(self, sim, simthread):
        ModelConfig.__init__(self, sim, simthread)

    def _initModel(self):
        self.name = "SpaCdbCdiInUa"
        self.cellTypes = self._createCellTypes()
        self.energyMatrix = self._createEnergyMatrix()
        self._run() # Must be the last statement.

    def _createCellTypes(self):
        cellTypes = []
        stem = Stemcell
        stem.setGrowthVolumePerDayRelVolume(0.1)
        self.stemNecrosisProb = stem.necrosisProb = 0

        basal = Basalcell
        basal.setGrowthVolumePerDayRelVolume(0.1)
        basal.apoptosisTimeInDays = 8000000000.0
        self.basalNecrosisProb = basal.necrosisProb = 0.000001

        intermediate = Intermediatecell
        intermediate.setGrowthVolumePerDayRelVolume(0.09)
        intermediate.apoptosisTimeInDays = 20000000000.0
        self.intermediateNecrosisProb = intermediate.necrosisProb = 0.000002

        umbrella = Umbrellacell
        umbrella.setGrowthVolumePerDayRelVolume(0.08)
        umbrella.apoptosisTimeInDays = 1000000000.0
        self.umbrellaNecrosisProb = umbrella.necrosisProb = 0.000003

        stem.setDescendants(0.90, [stem.id, basal.id])
        stem.setDescendants(0.05, [stem.id, stem.id])
        stem.setDescendants(0.05, [basal.id, basal.id])

        cellTypes.extend((Medium, Basalmembrane, stem, basal, intermediate, umbrella))

        return cellTypes

    def _getSteppables(self):
        steppableList = []
        steppableList.append(ColonySteppable(self.sim, self))
        steppableList.append(InitializerSteppable(self.sim, self))
        steppableList.append(GrowthSteppable(self.sim, self))
        steppableList.append(GrowthMitosisSteppable(self.sim, self))
        steppableList.append(CMTransformationSteppable(self.sim, self))
        steppableList.append(UrinationSteppable(self.sim, self, prop=0.02))
        steppableList.append(DeathSteppable(self.sim, self))
        #steppableList.append(OptimumSearchSteppable(self.sim, self))
        steppableList.append(VolumeFitnessSteppable(self.sim, self))
        steppableList.append(ArrangementFitnessSteppable(self.sim, self))
        steppableList.append(DummyFitnessSteppable(self.sim, self))
        steppableList.append(MutationSteppable(self.sim, self, self.stemNecrosisProb, self.basalNecrosisProb,
                                               self.intermediateNecrosisProb, self.umbrellaNecrosisProb))

        return steppableList

    def _createExecConfig(self):
        return ExecConfig(MCSperDay=500, #SEED=10,
                          xLength=500, yLength=150, zLength=0, voxelDensity=.8)
<|endoftext|>


<commit_before>"""Tests for s3direct package."""

from os import environ
import sys
from distutils.version import StrictVersion
import django
from django.conf import settings

settings.configure(
    DEBUG=True,
    DATABASES={
        'default': {
            'ENGINE': 'django.db.backends.sqlite3',
        }
    },
    ROOT_URLCONF='s3direct.urls',
    INSTALLED_APPS=(
        'django.contrib.auth',
        'django.contrib.contenttypes',
        'django.contrib.sessions',
        'django.contrib.admin',
        's3direct',
    ),
    MIDDLEWARE_CLASSES=(
        'django.contrib.sessions.middleware.SessionMiddleware',
        'django.contrib.auth.middleware.AuthenticationMiddleware',
    ),
    TEMPLATES=[
        {
            'BACKEND': 'django.template.backends.django.DjangoTemplates',
            'APP_DIRS': True
        }
    ],
    AWS_ACCESS_KEY_ID=environ.get('AWS_ACCESS_KEY_ID', ''),
    AWS_SECRET_ACCESS_KEY=environ.get('AWS_SECRET_ACCESS_KEY', ''),
    AWS_STORAGE_BUCKET_NAME=environ.get(
        'AWS_STORAGE_BUCKET_NAME', 'test-bucket'),
    S3DIRECT_REGION='us-east-1',
    S3DIRECT_DESTINATIONS={
        'misc': {
            'key': lambda original_filename: 'images/unique.jpg',
        },
        'files': {
            'key': 'uploads/files',
            'auth': lambda u: u.is_staff,
        },
        'imgs': {
            'key': 'uploads/imgs',
            'auth': lambda u: True,
            'allowed': ['image/jpeg', 'image/png'],
            'content_length_range': (5000, 20000000),  # 5kb - 20mb
        },
        'vids': {
            'key': 'uploads/vids',
            'auth': lambda u: u.is_authenticated(),
            'allowed': ['video/mp4'],
        },
        'cached': {
            'key': 'uploads/vids',
            'auth': lambda u: True,
            'allowed': '*',
            'acl': 'authenticated-read',
            'bucket': 'astoragebucketname',
            'cache_control': 'max-age=2592000',
            'content_disposition': 'attachment',
            'server_side_encryption': 'AES256',
        }
    }
)

if hasattr(django, 'setup'):
    django.setup()

if django.get_version() < StrictVersion('1.6'):
    from django.test.simple import DjangoTestSuiteRunner
    test_runner = DjangoTestSuiteRunner(verbosity=1)
else:
    from django.test.runner import DiscoverRunner
    test_runner = DiscoverRunner(verbosity=1)

failures = test_runner.run_tests(['s3direct', ])

if failures:
    sys.exit(failures)
<commit_msg>Configure test Django with required context processors for 1.11.<commit_after>"""Tests for s3direct package."""

from os import environ
import sys
from distutils.version import StrictVersion
import django
from django.conf import settings

settings.configure(
    DEBUG=True,
    DATABASES={
        'default': {
            'ENGINE': 'django.db.backends.sqlite3',
        }
    },
    ROOT_URLCONF='s3direct.urls',
    INSTALLED_APPS=(
        'django.contrib.auth',
        'django.contrib.contenttypes',
        'django.contrib.sessions',
        'django.contrib.admin',
        's3direct',
    ),
    MIDDLEWARE_CLASSES=(
        'django.contrib.sessions.middleware.SessionMiddleware',
        'django.contrib.auth.middleware.AuthenticationMiddleware',
    ),
    TEMPLATES=[
        {
            'BACKEND': 'django.template.backends.django.DjangoTemplates',
            'APP_DIRS': True,
            'OPTIONS': {
                'context_processors': [
                    "django.contrib.auth.context_processors.auth",
                ]
            }
        },
    ],
    AWS_ACCESS_KEY_ID=environ.get('AWS_ACCESS_KEY_ID', ''),
    AWS_SECRET_ACCESS_KEY=environ.get('AWS_SECRET_ACCESS_KEY', ''),
    AWS_STORAGE_BUCKET_NAME=environ.get(
        'AWS_STORAGE_BUCKET_NAME', 'test-bucket'),
    S3DIRECT_REGION='us-east-1',
    S3DIRECT_DESTINATIONS={
        'misc': {
            'key': lambda original_filename: 'images/unique.jpg',
        },
        'files': {
            'key': 'uploads/files',
            'auth': lambda u: u.is_staff,
        },
        'imgs': {
            'key': 'uploads/imgs',
            'auth': lambda u: True,
            'allowed': ['image/jpeg', 'image/png'],
            'content_length_range': (5000, 20000000),  # 5kb - 20mb
        },
        'vids': {
            'key': 'uploads/vids',
            'auth': lambda u: u.is_authenticated(),
            'allowed': ['video/mp4'],
        },
        'cached': {
            'key': 'uploads/vids',
            'auth': lambda u: True,
            'allowed': '*',
            'acl': 'authenticated-read',
            'bucket': 'astoragebucketname',
            'cache_control': 'max-age=2592000',
            'content_disposition': 'attachment',
            'server_side_encryption': 'AES256',
        }
    }
)

if hasattr(django, 'setup'):
    django.setup()

if django.get_version() < StrictVersion('1.6'):
    from django.test.simple import DjangoTestSuiteRunner
    test_runner = DjangoTestSuiteRunner(verbosity=1)
else:
    from django.test.runner import DiscoverRunner
    test_runner = DiscoverRunner(verbosity=1)

failures = test_runner.run_tests(['s3direct', ])

if failures:
    sys.exit(failures)
<|endoftext|>


<commit_before># Copyright 2016 the original author or authors.
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

__author__ = "Julian Debatin"
__copyright__ = "The authors"
__license__ = "Apache 2"
__email__ = "juliandebatin@gmail.com"
__status__ = "Production"

from Core.CellType import *
from Core.ExecConfig import ExecConfig
from Core.ModelConfig import ModelConfig
from Steppable.InitializerSteppable import InitializerSteppable
from Steppable.DeathSteppable import DeathSteppable
from Steppable.GrowthMitosisSteppable import GrowthMitosisSteppable
from Steppable.GrowthSteppable import GrowthSteppable
from Steppable.CMTransformationSteppable import CMTransformationSteppable
from Steppable.UrinationSteppable import UrinationSteppable
from Logger.VolumeFitnessSteppable import VolumeFitnessSteppable
from Logger.ArrangementFitnessSteppable import ArrangementFitnessSteppable
from Logger.DummyFitnessSteppable import DummyFitnessSteppable
from Steppable.ColonySteppable import ColonySteppable
from Steppable.MutationSteppable import MutationSteppable

class SpaSdbCdiInUa(ModelConfig):
    def __init__(self, sim, simthread):
        ModelConfig.__init__(self, sim, simthread)

    def _initModel(self):
        self.name = "SpaSdbCdiInUa"
        self.cellTypes = self._createCellTypes()
        self.energyMatrix = self._createEnergyMatrix()
        self._run() # Must be the last statement.

    def _createCellTypes(self):
        cellTypes = []
        stem = Stemcell
        stem.setGrowthVolumePerDayRelVolume(0.12)
        self.stemNecrosisProb = stem.necrosisProb = 0

        basal = Basalcell
        basal.setGrowthVolumePerDayRelVolume(0.11)
        basal.apoptosisTimeInDays = 8000000000.0
        self.basalNecrosisProb = basal.necrosisProb = 0.00002

        intermediate = Intermediatecell
        intermediate.setGrowthVolumePerDayRelVolume(0.09)
        intermediate.apoptosisTimeInDays = 20000000000.0
        self.intermediateNecrosisProb = intermediate.necrosisProb = 0.00002

        umbrella = Umbrellacell
        umbrella.setGrowthVolumePerDayRelVolume(0.08)
        umbrella.apoptosisTimeInDays = 100000000000.0
        self.umbrellaNecrosisProb = umbrella.necrosisProb = 0.00004

        stem.setDescendants(0.90, [stem.id, basal.id])
        stem.setDescendants(0.05, [stem.id, stem.id])
        stem.setDescendants(0.05, [basal.id, basal.id])
        basal.setDescendants(1.0, [basal.id, intermediate.id])

        cellTypes.extend((Medium, Basalmembrane, stem, basal, intermediate, umbrella))

        return cellTypes

    def _getSteppables(self):
        steppableList = []
        steppableList.append(ColonySteppable(self.sim, self))
        steppableList.append(InitializerSteppable(self.sim, self))
        steppableList.append(GrowthSteppable(self.sim, self))
        steppableList.append(GrowthMitosisSteppable(self.sim, self))
        steppableList.append(CMTransformationSteppable(self.sim, self))
        steppableList.append(UrinationSteppable(self.sim, self, prop=0.02))
        steppableList.append(DeathSteppable(self.sim, self))
        #steppableList.append(OptimumSearchSteppable(self.sim, self))
        steppableList.append(VolumeFitnessSteppable(self.sim, self))
        steppableList.append(ArrangementFitnessSteppable(self.sim, self))
        steppableList.append(DummyFitnessSteppable(self.sim, self))
        steppableList.append(MutationSteppable(self.sim, self, self.stemNecrosisProb, self.basalNecrosisProb,
                                               self.intermediateNecrosisProb, self.umbrellaNecrosisProb))

        return steppableList

    def _createExecConfig(self):
        return ExecConfig(MCSperDay=500, #SEED=10,
                          xLength=500, yLength=150, zLength=0, voxelDensity=.8)
<commit_msg>no message<commit_after># Copyright 2016 the original author or authors.
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

__author__ = "Julian Debatin"
__copyright__ = "The authors"
__license__ = "Apache 2"
__email__ = "juliandebatin@gmail.com"
__status__ = "Production"

from Core.CellType import *
from Core.ExecConfig import ExecConfig
from Core.ModelConfig import ModelConfig
from Steppable.InitializerSteppable import InitializerSteppable
from Steppable.DeathSteppable import DeathSteppable
from Steppable.GrowthMitosisSteppable import GrowthMitosisSteppable
from Steppable.GrowthSteppable import GrowthSteppable
from Steppable.CMTransformationSteppable import CMTransformationSteppable
from Steppable.UrinationSteppable import UrinationSteppable
from Logger.VolumeFitnessSteppable import VolumeFitnessSteppable
from Logger.ArrangementFitnessSteppable import ArrangementFitnessSteppable
from Logger.DummyFitnessSteppable import DummyFitnessSteppable
from Steppable.ColonySteppable import ColonySteppable
from Steppable.MutationSteppable import MutationSteppable

class SpaSdbCdiInUa(ModelConfig):
    def __init__(self, sim, simthread):
        ModelConfig.__init__(self, sim, simthread)

    def _initModel(self):
        self.name = "SpaSdbCdiInUa"
        self.cellTypes = self._createCellTypes()
        self.energyMatrix = self._createEnergyMatrix()
        self._run() # Must be the last statement.

    def _createCellTypes(self):
        cellTypes = []
        stem = Stemcell
        stem.setGrowthVolumePerDayRelVolume(0.12)
        self.stemNecrosisProb = stem.necrosisProb = 0

        basal = Basalcell
        basal.setGrowthVolumePerDayRelVolume(0.11)
        basal.apoptosisTimeInDays = 8000000000.0
        self.basalNecrosisProb = basal.necrosisProb = 0.00001

        intermediate = Intermediatecell
        intermediate.setGrowthVolumePerDayRelVolume(0.09)
        intermediate.apoptosisTimeInDays = 20000000000.0
        self.intermediateNecrosisProb = intermediate.necrosisProb = 0.00002

        umbrella = Umbrellacell
        umbrella.setGrowthVolumePerDayRelVolume(0.08)
        umbrella.apoptosisTimeInDays = 100000000000.0
        self.umbrellaNecrosisProb = umbrella.necrosisProb = 0.00003

        stem.setDescendants(0.90, [stem.id, basal.id])
        stem.setDescendants(0.05, [stem.id, stem.id])
        stem.setDescendants(0.05, [basal.id, basal.id])
        basal.setDescendants(1.0, [basal.id, intermediate.id])

        cellTypes.extend((Medium, Basalmembrane, stem, basal, intermediate, umbrella))

        return cellTypes

    def _getSteppables(self):
        steppableList = []
        steppableList.append(ColonySteppable(self.sim, self))
        steppableList.append(InitializerSteppable(self.sim, self))
        steppableList.append(GrowthSteppable(self.sim, self))
        steppableList.append(GrowthMitosisSteppable(self.sim, self))
        steppableList.append(CMTransformationSteppable(self.sim, self))
        steppableList.append(UrinationSteppable(self.sim, self, prop=0.02))
        steppableList.append(DeathSteppable(self.sim, self))
        #steppableList.append(OptimumSearchSteppable(self.sim, self))
        steppableList.append(VolumeFitnessSteppable(self.sim, self))
        steppableList.append(ArrangementFitnessSteppable(self.sim, self))
        steppableList.append(DummyFitnessSteppable(self.sim, self))
        steppableList.append(MutationSteppable(self.sim, self, self.stemNecrosisProb, self.basalNecrosisProb,
                                               self.intermediateNecrosisProb, self.umbrellaNecrosisProb))

        return steppableList

    def _createExecConfig(self):
        return ExecConfig(MCSperDay=500, #SEED=10,
                          xLength=500, yLength=150, zLength=0, voxelDensity=.8)
<|endoftext|>


<commit_before><commit_msg>Includes __all__<commit_after>__all__ = ['phjRROR']
<|endoftext|>


<commit_before>#!/usr/bin/python

import os, sys, re, shutil, unittest, doctest

WITH_CYTHON = True
CLEANUP_WORKDIR = True

from distutils.dist import Distribution
from distutils.core import Extension
from distutils.command.build_ext import build_ext
distutils_distro = Distribution()

TEST_DIRS = ['compile', 'errors', 'run']
TEST_RUN_DIRS = ['run']

INCLUDE_DIRS = [ d for d in os.getenv('INCLUDE', '').split(os.pathsep) if d ]
CFLAGS = os.getenv('CFLAGS', '').split()

class ErrorWriter(object):
    match_error = re.compile('(?:.*:)?([-0-9]+):([-0-9]+):(.*)').match
    def __init__(self):
        self.output = []
        self.write = self.output.append

    def geterrors(self):
        s = ''.join(self.output)
        errors = []
        for line in s.split('\n'):
            match = self.match_error(line)
            if match:
                line, column, message = match.groups()
                errors.append( "%d:%d:%s" % (int(line), int(column), message.strip()) )
        return errors

class TestBuilder(object):
    def __init__(self, rootdir, workdir, selectors, annotate):
        self.rootdir = rootdir
        self.workdir = workdir
        self.selectors = selectors
        self.annotate = annotate

    def build_suite(self):
        suite = unittest.TestSuite()
        filenames = os.listdir(self.rootdir)
        filenames.sort()
        for filename in filenames:
            if not WITH_CYTHON and filename == "errors":
                # we won't get any errors without running Cython
                continue
            path = os.path.join(self.rootdir, filename)
            if os.path.isdir(path) and filename in TEST_DIRS:
                suite.addTest(
                    self.handle_directory(path, filename))
        return suite

    def handle_directory(self, path, context):
        workdir = os.path.join(self.workdir, context)
        if not os.path.exists(workdir):
            os.makedirs(workdir)
        if workdir not in sys.path:
            sys.path.insert(0, workdir)

        expect_errors = (context == 'errors')
        suite = unittest.TestSuite()
        filenames = os.listdir(path)
        filenames.sort()
        for filename in filenames:
            if not filename.endswith(".pyx"):
                continue
            module = filename[:-4]
            fqmodule = "%s.%s" % (context, module)
            if not [ 1 for match in self.selectors
                     if match(fqmodule) ]:
                continue
            if context in TEST_RUN_DIRS:
                test = CythonRunTestCase(
                    path, workdir, module, self.annotate)
            else:
                test = CythonCompileTestCase(
                    path, workdir, module, expect_errors, self.annotate)
            suite.addTest(test)
        return suite

class CythonCompileTestCase(unittest.TestCase):
    def __init__(self, directory, workdir, module,
                 expect_errors=False, annotate=False):
        self.directory = directory
        self.workdir = workdir
        self.module = module
        self.expect_errors = expect_errors
        self.annotate = annotate
        unittest.TestCase.__init__(self)

    def shortDescription(self):
        return "compiling " + self.module

    def tearDown(self):
        cleanup_c_files = WITH_CYTHON and CLEANUP_WORKDIR
        if os.path.exists(self.workdir):
            for rmfile in os.listdir(self.workdir):
                if not cleanup_c_files and rmfile[-2:] in (".c", ".h"):
                    continue
                if self.annotate and rmfile.endswith(".html"):
                    continue
                try:
                    rmfile = os.path.join(self.workdir, rmfile)
                    if os.path.isdir(rmfile):
                        shutil.rmtree(rmfile, ignore_errors=True)
                    else:
                        os.remove(rmfile)
                except IOError:
                    pass
        else:
            os.makedirs(self.workdir)

    def runTest(self):
        self.compile(self.directory, self.module, self.workdir,
                     self.directory, self.expect_errors, self.annotate)

    def split_source_and_output(self, directory, module, workdir):
        source_and_output = open(os.path.join(directory, module + '.pyx'), 'rU')
        out = open(os.path.join(workdir, module + '.pyx'), 'w')
        for line in source_and_output:
            last_line = line
            if line.startswith("_ERRORS"):
                out.close()
                out = ErrorWriter()
            else:
                out.write(line)
        try:
            geterrors = out.geterrors
        except AttributeError:
            return []
        else:
            return geterrors()

    def run_cython(self, directory, module, targetdir, incdir, annotate):
        include_dirs = INCLUDE_DIRS[:]
        if incdir:
            include_dirs.append(incdir)
        source = os.path.join(directory, module + '.pyx')
        target = os.path.join(targetdir, module + '.c')
        options = CompilationOptions(
            pyrex_default_options,
            include_path = include_dirs,
            output_file = target,
            annotate = annotate,
            use_listing_file = False, cplus = False, generate_pxi = False)
        cython_compile(source, options=options,
                       full_module_name=module)

    def run_distutils(self, module, workdir, incdir):
        cwd = os.getcwd()
        os.chdir(workdir)
        try:
            build_extension = build_ext(distutils_distro)
            build_extension.include_dirs = INCLUDE_DIRS[:]
            if incdir:
                build_extension.include_dirs.append(incdir)
            build_extension.finalize_options()

            extension = Extension(
                module,
                sources = [module + '.c'],
                extra_compile_args = CFLAGS,
                )
            build_extension.extensions = [extension]
            build_extension.build_temp = workdir
            build_extension.build_lib  = workdir
            build_extension.run()
        finally:
            os.chdir(cwd)

    def compile(self, directory, module, workdir, incdir,
                expect_errors, annotate):
        expected_errors = errors = ()
        if expect_errors:
            expected_errors = self.split_source_and_output(
                directory, module, workdir)
            directory = workdir

        if WITH_CYTHON:
            old_stderr = sys.stderr
            try:
                sys.stderr = ErrorWriter()
                self.run_cython(directory, module, workdir, incdir, annotate)
                errors = sys.stderr.geterrors()
            finally:
                sys.stderr = old_stderr

        if errors or expected_errors:
            for expected, error in zip(expected_errors, errors):
                self.assertEquals(expected, error)
            if len(errors) < len(expected_errors):
                expected_error = expected_errors[len(errors)]
                self.assertEquals(expected_error, None)
            elif len(errors) > len(expected_errors):
                unexpected_error = errors[len(expected_errors)]
                self.assertEquals(None, unexpected_error)
        else:
            self.run_distutils(module, workdir, incdir)

class CythonRunTestCase(CythonCompileTestCase):
    def shortDescription(self):
        return "compiling and running " + self.module

    def run(self, result=None):
        if result is None:
            result = self.defaultTestResult()
        result.startTest(self)
        try:
            self.runTest()
            doctest.DocTestSuite(self.module).run(result)
        except Exception:
            result.addError(self, sys.exc_info())
            result.stopTest(self)
        try:
            self.tearDown()
        except Exception:
            pass

if __name__ == '__main__':
    try:
        sys.argv.remove("--no-cython")
    except ValueError:
        WITH_CYTHON = True
    else:
        WITH_CYTHON = False

    if WITH_CYTHON:
        from Cython.Compiler.Main import \
            CompilationOptions, \
            default_options as pyrex_default_options, \
            compile as cython_compile

    from distutils.dist import Distribution
    from distutils.core import Extension
    from distutils.command.build_ext import build_ext
    distutils_distro = Distribution()

    # RUN ALL TESTS!
    ROOTDIR = os.path.join(os.getcwd(), os.path.dirname(sys.argv[0]), 'tests')
    WORKDIR = os.path.join(os.getcwd(), 'BUILD')
    if WITH_CYTHON:
        if os.path.exists(WORKDIR):
            shutil.rmtree(WORKDIR, ignore_errors=True)
    if not os.path.exists(WORKDIR):
        os.makedirs(WORKDIR)

    if WITH_CYTHON:
        from Cython.Compiler.Version import version
        from Cython.Compiler.Main import \
            CompilationOptions, \
            default_options as pyrex_default_options, \
            compile as cython_compile
        print("Running tests against Cython %s" % version)
    else:
        print("Running tests without Cython.")
    print("Python %s" % sys.version)
    print("")

    try:
        sys.argv.remove("-C")
    except ValueError:
        coverage = None
    else:
        import coverage
        coverage.erase()

    try:
        sys.argv.remove("--no-cleanup")
    except ValueError:
        CLEANUP_WORKDIR = True
    else:
        CLEANUP_WORKDIR = False

    try:
        sys.argv.remove("-a")
    except ValueError:
        annotate_source = False
    else:
        annotate_source = True

    import re
    selectors = [ re.compile(r, re.I|re.U).search for r in sys.argv[1:] ]
    if not selectors:
        selectors = [ lambda x:True ]

    tests = TestBuilder(ROOTDIR, WORKDIR, selectors, annotate_source)
    test_suite = tests.build_suite()

    if coverage is not None:
        coverage.start()

    unittest.TextTestRunner(verbosity=2).run(test_suite)

    if coverage is not None:
        coverage.stop()
        ignored_modules = ('Options', 'Version', 'DebugFlags')
        modules = [ module for name, module in sys.modules.items()
                    if module is not None and
                    name.startswith('Cython.Compiler.') and
                    name[len('Cython.Compiler.'):] not in ignored_modules ]
        coverage.report(modules, show_missing=0)
<commit_msg>require options '-v' or '-vv' for status output in test runner<commit_after>#!/usr/bin/python

import os, sys, re, shutil, unittest, doctest

WITH_CYTHON = True
CLEANUP_WORKDIR = True

from distutils.dist import Distribution
from distutils.core import Extension
from distutils.command.build_ext import build_ext
distutils_distro = Distribution()

TEST_DIRS = ['compile', 'errors', 'run']
TEST_RUN_DIRS = ['run']

INCLUDE_DIRS = [ d for d in os.getenv('INCLUDE', '').split(os.pathsep) if d ]
CFLAGS = os.getenv('CFLAGS', '').split()

class ErrorWriter(object):
    match_error = re.compile('(?:.*:)?([-0-9]+):([-0-9]+):(.*)').match
    def __init__(self):
        self.output = []
        self.write = self.output.append

    def geterrors(self):
        s = ''.join(self.output)
        errors = []
        for line in s.split('\n'):
            match = self.match_error(line)
            if match:
                line, column, message = match.groups()
                errors.append( "%d:%d:%s" % (int(line), int(column), message.strip()) )
        return errors

class TestBuilder(object):
    def __init__(self, rootdir, workdir, selectors, annotate):
        self.rootdir = rootdir
        self.workdir = workdir
        self.selectors = selectors
        self.annotate = annotate

    def build_suite(self):
        suite = unittest.TestSuite()
        filenames = os.listdir(self.rootdir)
        filenames.sort()
        for filename in filenames:
            if not WITH_CYTHON and filename == "errors":
                # we won't get any errors without running Cython
                continue
            path = os.path.join(self.rootdir, filename)
            if os.path.isdir(path) and filename in TEST_DIRS:
                suite.addTest(
                    self.handle_directory(path, filename))
        return suite

    def handle_directory(self, path, context):
        workdir = os.path.join(self.workdir, context)
        if not os.path.exists(workdir):
            os.makedirs(workdir)
        if workdir not in sys.path:
            sys.path.insert(0, workdir)

        expect_errors = (context == 'errors')
        suite = unittest.TestSuite()
        filenames = os.listdir(path)
        filenames.sort()
        for filename in filenames:
            if not filename.endswith(".pyx"):
                continue
            module = filename[:-4]
            fqmodule = "%s.%s" % (context, module)
            if not [ 1 for match in self.selectors
                     if match(fqmodule) ]:
                continue
            if context in TEST_RUN_DIRS:
                test = CythonRunTestCase(
                    path, workdir, module, self.annotate)
            else:
                test = CythonCompileTestCase(
                    path, workdir, module, expect_errors, self.annotate)
            suite.addTest(test)
        return suite

class CythonCompileTestCase(unittest.TestCase):
    def __init__(self, directory, workdir, module,
                 expect_errors=False, annotate=False):
        self.directory = directory
        self.workdir = workdir
        self.module = module
        self.expect_errors = expect_errors
        self.annotate = annotate
        unittest.TestCase.__init__(self)

    def shortDescription(self):
        return "compiling " + self.module

    def tearDown(self):
        cleanup_c_files = WITH_CYTHON and CLEANUP_WORKDIR
        if os.path.exists(self.workdir):
            for rmfile in os.listdir(self.workdir):
                if not cleanup_c_files and rmfile[-2:] in (".c", ".h"):
                    continue
                if self.annotate and rmfile.endswith(".html"):
                    continue
                try:
                    rmfile = os.path.join(self.workdir, rmfile)
                    if os.path.isdir(rmfile):
                        shutil.rmtree(rmfile, ignore_errors=True)
                    else:
                        os.remove(rmfile)
                except IOError:
                    pass
        else:
            os.makedirs(self.workdir)

    def runTest(self):
        self.compile(self.directory, self.module, self.workdir,
                     self.directory, self.expect_errors, self.annotate)

    def split_source_and_output(self, directory, module, workdir):
        source_and_output = open(os.path.join(directory, module + '.pyx'), 'rU')
        out = open(os.path.join(workdir, module + '.pyx'), 'w')
        for line in source_and_output:
            last_line = line
            if line.startswith("_ERRORS"):
                out.close()
                out = ErrorWriter()
            else:
                out.write(line)
        try:
            geterrors = out.geterrors
        except AttributeError:
            return []
        else:
            return geterrors()

    def run_cython(self, directory, module, targetdir, incdir, annotate):
        include_dirs = INCLUDE_DIRS[:]
        if incdir:
            include_dirs.append(incdir)
        source = os.path.join(directory, module + '.pyx')
        target = os.path.join(targetdir, module + '.c')
        options = CompilationOptions(
            pyrex_default_options,
            include_path = include_dirs,
            output_file = target,
            annotate = annotate,
            use_listing_file = False, cplus = False, generate_pxi = False)
        cython_compile(source, options=options,
                       full_module_name=module)

    def run_distutils(self, module, workdir, incdir):
        cwd = os.getcwd()
        os.chdir(workdir)
        try:
            build_extension = build_ext(distutils_distro)
            build_extension.include_dirs = INCLUDE_DIRS[:]
            if incdir:
                build_extension.include_dirs.append(incdir)
            build_extension.finalize_options()

            extension = Extension(
                module,
                sources = [module + '.c'],
                extra_compile_args = CFLAGS,
                )
            build_extension.extensions = [extension]
            build_extension.build_temp = workdir
            build_extension.build_lib  = workdir
            build_extension.run()
        finally:
            os.chdir(cwd)

    def compile(self, directory, module, workdir, incdir,
                expect_errors, annotate):
        expected_errors = errors = ()
        if expect_errors:
            expected_errors = self.split_source_and_output(
                directory, module, workdir)
            directory = workdir

        if WITH_CYTHON:
            old_stderr = sys.stderr
            try:
                sys.stderr = ErrorWriter()
                self.run_cython(directory, module, workdir, incdir, annotate)
                errors = sys.stderr.geterrors()
            finally:
                sys.stderr = old_stderr

        if errors or expected_errors:
            for expected, error in zip(expected_errors, errors):
                self.assertEquals(expected, error)
            if len(errors) < len(expected_errors):
                expected_error = expected_errors[len(errors)]
                self.assertEquals(expected_error, None)
            elif len(errors) > len(expected_errors):
                unexpected_error = errors[len(expected_errors)]
                self.assertEquals(None, unexpected_error)
        else:
            self.run_distutils(module, workdir, incdir)

class CythonRunTestCase(CythonCompileTestCase):
    def shortDescription(self):
        return "compiling and running " + self.module

    def run(self, result=None):
        if result is None:
            result = self.defaultTestResult()
        result.startTest(self)
        try:
            self.runTest()
            doctest.DocTestSuite(self.module).run(result)
        except Exception:
            result.addError(self, sys.exc_info())
            result.stopTest(self)
        try:
            self.tearDown()
        except Exception:
            pass

if __name__ == '__main__':
    try:
        sys.argv.remove("--no-cython")
    except ValueError:
        WITH_CYTHON = True
    else:
        WITH_CYTHON = False

    if WITH_CYTHON:
        from Cython.Compiler.Main import \
            CompilationOptions, \
            default_options as pyrex_default_options, \
            compile as cython_compile

    from distutils.dist import Distribution
    from distutils.core import Extension
    from distutils.command.build_ext import build_ext
    distutils_distro = Distribution()

    # RUN ALL TESTS!
    ROOTDIR = os.path.join(os.getcwd(), os.path.dirname(sys.argv[0]), 'tests')
    WORKDIR = os.path.join(os.getcwd(), 'BUILD')
    if WITH_CYTHON:
        if os.path.exists(WORKDIR):
            shutil.rmtree(WORKDIR, ignore_errors=True)
    if not os.path.exists(WORKDIR):
        os.makedirs(WORKDIR)

    if WITH_CYTHON:
        from Cython.Compiler.Version import version
        from Cython.Compiler.Main import \
            CompilationOptions, \
            default_options as pyrex_default_options, \
            compile as cython_compile
        print("Running tests against Cython %s" % version)
    else:
        print("Running tests without Cython.")
    print("Python %s" % sys.version)
    print("")

    try:
        sys.argv.remove("-C")
    except ValueError:
        coverage = None
    else:
        import coverage
        coverage.erase()

    try:
        sys.argv.remove("--no-cleanup")
    except ValueError:
        CLEANUP_WORKDIR = True
    else:
        CLEANUP_WORKDIR = False

    try:
        sys.argv.remove("-a")
    except ValueError:
        annotate_source = False
    else:
        annotate_source = True

    try:
        sys.argv.remove("-vv")
    except ValueError:
        try:
            sys.argv.remove("-v")
        except ValueError:
            verbosity = 0
        else:
            verbosity = 1
    else:
        verbosity = 2

    import re
    selectors = [ re.compile(r, re.I|re.U).search for r in sys.argv[1:] ]
    if not selectors:
        selectors = [ lambda x:True ]

    tests = TestBuilder(ROOTDIR, WORKDIR, selectors, annotate_source)
    test_suite = tests.build_suite()

    if coverage is not None:
        coverage.start()

    unittest.TextTestRunner(verbosity=verbosity).run(test_suite)

    if coverage is not None:
        coverage.stop()
        ignored_modules = ('Options', 'Version', 'DebugFlags')
        modules = [ module for name, module in sys.modules.items()
                    if module is not None and
                    name.startswith('Cython.Compiler.') and
                    name[len('Cython.Compiler.'):] not in ignored_modules ]
        coverage.report(modules, show_missing=0)
<|endoftext|>


<commit_before>"""
Utilities for downloading or generating datasets, splitting data, and computing accuracy metrics.
"""
import os
import numpy as np
import scipy.stats as st
import requests
from io import BytesIO
from equadratures.scalers import scaler_minmax

def gen_linear(n_observations=100, n_dim=5, n_relevent=5,bias=0.0, noise=0.0, random_seed=None):
    """ Generate a synthetic linear dataset for regression.

    Data is generated using a random linear regression model with ``n_relevent`` input dimensions.
    The remaining dimensions are "irrelevent" noise i.e. they do not affect the output.
    Gaussian noise with standard deviation ``noise`` is added.

    Parameters
    ----------
    n_observations : int, optional
        The number of observations (samples).
    n_dim : int, optional
        The total number of dimensions.
    n_relevent : int, optional
        The number of relevent input dimensions, i.e., the number of features used to build the linear model used to generate the output.
    bias : float, optional
        The bias term in the underlying linear model.
    noise : float, optional
        The standard deviation of the gaussian noise applied to the output.
    random_seed : int, optional
        Random number generator seed.

    Returns
    -------
    tuple
        Tuple (X,y) containing two numpy.ndarray's; One with shape (n_observations,n_dim) containing the inputs,
        and one with shape (n_observations,1) containing the outputs/targets.
    """
    # Generate input data
    n_relevent = min(n_dim, n_relevent)
    if np.__version__ >= '1.17':
        generator = np.random.default_rng(random_seed)
    else:
        generator = np.random.RandomState(random_seed)
    X = generator.standard_normal((n_observations,n_dim))
    X = scaler_minmax().transform(X)

    # Generate the truth model with n_relevent input dimensions
    truth_model = np.zeros((n_dim, 1))
#    truth_model[:n_relevent, :] = generator.standard_normal((n_relevent,1))
    truth_model[:n_relevent, :] = generator.uniform(-1,1,n_relevent).reshape(-1,1)
    y = scaler_minmax().transform(np.dot(X, truth_model)) + bias

    # Add noise
    if noise > 0.0:
        y += generator.normal(scale=noise, size=y.shape)

    return X, y

def gen_friedman(n_observations=100, n_dim=5, noise=0.0, random_seed=None,normalise=False):
    """ Generates the friedman regression problem described by Friedman [1] and Breiman [2].

    Inspired by :obj:`sklearn.datasets.make_friedman1`. The function has ``n_dim=5``, and choosing ``n_dim>5`` adds irrelevent input dimensions.
    Gaussian noise with standard deviation ``noise`` is added.

    Parameters
    ----------
    n_observations : int, optional
        The number of observations (samples).
    n_dim : int, optional
        The total number of dimensions. n_dim>=5, with n_dim>5 adding irrelevent input dimensions.
    noise : float, optional
        The standard deviation of the gaussian noise applied to the output.
    random_seed : int, optional
        Random number generator seed.
    normalise : bool, optional
        Normalise y to lie between -1 to 1.

    Returns
    -------
    tuple
        Tuple (X,y) containing two numpy.ndarray's; One with shape (n_observations,n_dim) containing the inputs,
        and one with shape (n_observations,1) containing the outputs/targets.

    References
    ----------
    1. J. Friedman, "Multivariate adaptive regression splines", The Annals of Statistics 19 (1), pages 1-67, 1991.
    2. L. Breiman, "Bagging predictors", Machine Learning 24, pages 123-140, 1996.
    """

    if n_dim < 5:
        raise ValueError("n_dim must be at least five.")

    if np.__version__ >= '1.17':
        generator = np.random.default_rng(random_seed)
    else:
        generator = np.random.RandomState(random_seed)
    X = generator.standard_normal((n_observations,n_dim))
    X = scaler_minmax().transform(X)

    y = 10 * np.sin(np.pi * X[:, 0] * X[:, 1]) + 20 * (X[:, 2] - 0.5) ** 2 \
        + 10 * X[:, 3] + 5 * X[:, 4]
    y+= noise*np.std(y)*generator.standard_normal(n_observations)
    if normalise:
        y = scaler_minmax().transform(y.reshape(-1,1))

    return X,y

def load_eq_dataset(dataset,data_dir=None):
    """
    Loads the requested dataset from the `equadratures dataset repository <https://github.com/Effective-Quadratures/data-sets>`__.

    Visit the aforementioned repo for a description of the available datasets.

    The requested dataset can either be downloaded directly upon request, or to minimise downloads the repo can be cloned
    once by the user, and the local repo directory can be given via ``data_dir`` (see examples).

    Parameters
    ----------
    dataset : str
        The dataset to download. Options are ```naca0012```, ```blade_envelopes```, ```probes```, ```3Dfan_blades```.
    data_dir : str, optional
        Directory name where a local clone of the data-sets repo is located. If given, the dataset will be loaded from here
        instead of downloading from the remote repo.

    Returns
    -------
    NpzFile
        NpzFile instance (see `numpy.lib.format <https://numpy.org/devdocs/reference/generated/numpy.lib.format.html#module-numpy.lib.format>`__)
        containing the dataset. Contents can be accessed in the usual way e.g. ``X = NpzFile['X']``.

    Examples
    --------
    Loading from remote repository
        >>> # Load the naca0012 aerofoil dataset
        >>> data = eq.datasets.load_eq_dataset('naca0012')
        >>> print(data.files)
        ['X', 'Cp', 'Cl', 'Cd']
        >>> X = data['X']
        >>> y = data['Cp']

    Loading from a locally cloned repository
        >>> git clone https://github.com/Effective-Quadratures/data-sets.git
        >>> data = eq.datasets.load_eq_dataset('naca0012', data_dir='/Users/user/Documents/data-sets')
    """
    # Check if valid dataset
    datasets = ['naca0012','blade_envelopes','probes', '3Dfan_blades']
    if dataset not in datasets:
        raise ValueError('dataset specified in load_eq_dataset not recognised, avaiable datasets: ', datasets)

    # Download from github
    if data_dir is None:
        print('Downloading the ' + dataset + ' dataset from github...')
        # .npz file
        git_url = os.path.join('https://github.com/Effective-Quadratures/data-sets/raw/main/',dataset,dataset+'.npz')
        try:
            r = requests.get(git_url,stream=True)
            r.raise_for_status()
            data = np.load(BytesIO(r.raw.read()))
        except requests.exceptions.RequestException as e:
            raise SystemExit(e)
        # .md file
        git_url = os.path.join('https://raw.githubusercontent.com/Effective-Quadratures/data-sets/main',dataset,'README.md')
        try:
            r = requests.get(git_url)
            r.raise_for_status()
            print('\n',r.text)
        except requests.exceptions.RequestException as e:
            raise SystemExit(e)

    # If the user has cloned the data-sets repo and provided its location in data_dir
    else:
        print('Loading the dataset from ', data_dir)
        data = np.load(os.path.join(data_dir,dataset,dataset+'.npz'))
        f = open(os.path.join(data_dir,dataset,'README.md'))
        print(f.read())

    return data

def train_test_split(X,y,train=0.7,random_seed=None,shuffle=True):
    """ Split arrays or matrices into random train and test subsets.

    Inspired by :obj:`sklearn.model_selection.train_test_split`.

    Parameters
    ----------
    X : numpy.ndarray
        Array with shape (n_observations,n_dim) containing the inputs.
    y : numpy.ndarray
        Array with shape (n_observations,1) containing the outputs/targets.
    train : float, optional
        Fraction between 0.0 and 1.0, representing the proportion of the dataset to include in the train split.
    random_seed : int, optional
        Seed for random number generator.
    shuffle : bool, optional
        Whether to shuffle the rows of data when spliting.

    Returns
    -------
    tuple
        Tuple (X_train, X_test, y_train, y_test) containing the split data, output as numpy.ndarray's.

    Example
    -------
    >>> X_train, X_test, y_train, y_test = eq.datasets.train_test_split(X, y,
    >>>                                    train=0.8, random_seed = 42)
    """
    if X.shape[0] == y.shape[0]:
        n_observations = X.shape[0]
    else:
        raise ValueError("X and y have different numbers of rows")
    if train > 0 or train < 1:
        n_train = int(train*n_observations)
        n_test  = n_observations - n_train
    else:
        raise ValueError("train should be between 0 and 1")
    if shuffle:
        if np.__version__ >= '1.17':
            generator = np.random.default_rng(random_seed)
        else:
            generator = np.random.RandomState(random_seed)
        idx = generator.permutation(n_observations)
    else:
        idx = np.arange(n_observations)
    idx_train, idx_test = idx[:n_train], idx[n_train:]
    return X[idx_train], X[idx_test], y[idx_train], y[idx_test]

def score(y_true,y_pred,metric='r2',X=None):
    """ Evaluates the accuracy/error score between predictions and the truth, according to the given accuracy metric.

    Parameters
    ----------
    y_true : numpy.ndarray
        Array with shape (number_of_observations, 1), containing predictions.
    y_pred : numpy.ndarray
        Array with shape (number_of_observations, 1) containing the true data.
    metric : str, optional
        The scoring metric to use. Avaliable options are: ```adjusted_r2```, ```r2```, ```mae```, ```rmse```, or ```normalised_mae```.
    X : numpy.ndarray
        The input data associated with **y_pred**. Required if ``metric=`adjusted_r2```.

    Returns
    -------
    float
        The accuracy or error score.
    """
    y_true = y_true.flatten()
    y_pred = y_pred.flatten()
    if metric == 'r2':
         score = st.linregress(y_true,y_pred)[2]**2
    elif metric == 'adjusted_r2':
        if X is None: raise ValueError('Must specify X in _score if adjusted_r2 metric used')
        N,d = X.shape
        r2 = st.linregress(y_true,y_pred)[2]**2
        score = 1.0 - (((1-r2)*(N-1))/(N-d-1))
    elif metric == 'mae':
        score = np.mean(np.abs(y_true-y_pred))
    elif metric == 'normalised_mae':
        score = np.mean(np.abs(y_true-y_pred))/np.std(y_true)
    elif metric == 'rmse':
        score = np.sqrt(np.mean((y_true-y_pred)**2))
    else:
        raise ValueError('Only r2, adjusted_r2, mae, normalised_mae, rmse scoring metrics currently supported')
    return score
<commit_msg>Corrected score docs<commit_after>"""
Utilities for downloading or generating datasets, splitting data, and computing accuracy metrics.
"""
import os
import numpy as np
import scipy.stats as st
import requests
from io import BytesIO
from equadratures.scalers import scaler_minmax

def gen_linear(n_observations=100, n_dim=5, n_relevent=5,bias=0.0, noise=0.0, random_seed=None):
    """ Generate a synthetic linear dataset for regression.

    Data is generated using a random linear regression model with ``n_relevent`` input dimensions.
    The remaining dimensions are "irrelevent" noise i.e. they do not affect the output.
    Gaussian noise with standard deviation ``noise`` is added.

    Parameters
    ----------
    n_observations : int, optional
        The number of observations (samples).
    n_dim : int, optional
        The total number of dimensions.
    n_relevent : int, optional
        The number of relevent input dimensions, i.e., the number of features used to build the linear model used to generate the output.
    bias : float, optional
        The bias term in the underlying linear model.
    noise : float, optional
        The standard deviation of the gaussian noise applied to the output.
    random_seed : int, optional
        Random number generator seed.

    Returns
    -------
    tuple
        Tuple (X,y) containing two numpy.ndarray's; One with shape (n_observations,n_dim) containing the inputs,
        and one with shape (n_observations,1) containing the outputs/targets.
    """
    # Generate input data
    n_relevent = min(n_dim, n_relevent)
    if np.__version__ >= '1.17':
        generator = np.random.default_rng(random_seed)
    else:
        generator = np.random.RandomState(random_seed)
    X = generator.standard_normal((n_observations,n_dim))
    X = scaler_minmax().transform(X)

    # Generate the truth model with n_relevent input dimensions
    truth_model = np.zeros((n_dim, 1))
#    truth_model[:n_relevent, :] = generator.standard_normal((n_relevent,1))
    truth_model[:n_relevent, :] = generator.uniform(-1,1,n_relevent).reshape(-1,1)
    y = scaler_minmax().transform(np.dot(X, truth_model)) + bias

    # Add noise
    if noise > 0.0:
        y += generator.normal(scale=noise, size=y.shape)

    return X, y

def gen_friedman(n_observations=100, n_dim=5, noise=0.0, random_seed=None,normalise=False):
    """ Generates the friedman regression problem described by Friedman [1] and Breiman [2].

    Inspired by :obj:`sklearn.datasets.make_friedman1`. The function has ``n_dim=5``, and choosing ``n_dim>5`` adds irrelevent input dimensions.
    Gaussian noise with standard deviation ``noise`` is added.

    Parameters
    ----------
    n_observations : int, optional
        The number of observations (samples).
    n_dim : int, optional
        The total number of dimensions. n_dim>=5, with n_dim>5 adding irrelevent input dimensions.
    noise : float, optional
        The standard deviation of the gaussian noise applied to the output.
    random_seed : int, optional
        Random number generator seed.
    normalise : bool, optional
        Normalise y to lie between -1 to 1.

    Returns
    -------
    tuple
        Tuple (X,y) containing two numpy.ndarray's; One with shape (n_observations,n_dim) containing the inputs,
        and one with shape (n_observations,1) containing the outputs/targets.

    References
    ----------
    1. J. Friedman, "Multivariate adaptive regression splines", The Annals of Statistics 19 (1), pages 1-67, 1991.
    2. L. Breiman, "Bagging predictors", Machine Learning 24, pages 123-140, 1996.
    """

    if n_dim < 5:
        raise ValueError("n_dim must be at least five.")

    if np.__version__ >= '1.17':
        generator = np.random.default_rng(random_seed)
    else:
        generator = np.random.RandomState(random_seed)
    X = generator.standard_normal((n_observations,n_dim))
    X = scaler_minmax().transform(X)

    y = 10 * np.sin(np.pi * X[:, 0] * X[:, 1]) + 20 * (X[:, 2] - 0.5) ** 2 \
        + 10 * X[:, 3] + 5 * X[:, 4]
    y+= noise*np.std(y)*generator.standard_normal(n_observations)
    if normalise:
        y = scaler_minmax().transform(y.reshape(-1,1))

    return X,y

def load_eq_dataset(dataset,data_dir=None):
    """
    Loads the requested dataset from the `equadratures dataset repository <https://github.com/Effective-Quadratures/data-sets>`__.

    Visit the aforementioned repo for a description of the available datasets.

    The requested dataset can either be downloaded directly upon request, or to minimise downloads the repo can be cloned
    once by the user, and the local repo directory can be given via ``data_dir`` (see examples).

    Parameters
    ----------
    dataset : str
        The dataset to download. Options are ```naca0012```, ```blade_envelopes```, ```probes```, ```3Dfan_blades```.
    data_dir : str, optional
        Directory name where a local clone of the data-sets repo is located. If given, the dataset will be loaded from here
        instead of downloading from the remote repo.

    Returns
    -------
    NpzFile
        NpzFile instance (see `numpy.lib.format <https://numpy.org/devdocs/reference/generated/numpy.lib.format.html#module-numpy.lib.format>`__)
        containing the dataset. Contents can be accessed in the usual way e.g. ``X = NpzFile['X']``.

    Examples
    --------
    Loading from remote repository
        >>> # Load the naca0012 aerofoil dataset
        >>> data = eq.datasets.load_eq_dataset('naca0012')
        >>> print(data.files)
        ['X', 'Cp', 'Cl', 'Cd']
        >>> X = data['X']
        >>> y = data['Cp']

    Loading from a locally cloned repository
        >>> git clone https://github.com/Effective-Quadratures/data-sets.git
        >>> data = eq.datasets.load_eq_dataset('naca0012', data_dir='/Users/user/Documents/data-sets')
    """
    # Check if valid dataset
    datasets = ['naca0012','blade_envelopes','probes', '3Dfan_blades']
    if dataset not in datasets:
        raise ValueError('dataset specified in load_eq_dataset not recognised, avaiable datasets: ', datasets)

    # Download from github
    if data_dir is None:
        print('Downloading the ' + dataset + ' dataset from github...')
        # .npz file
        git_url = os.path.join('https://github.com/Effective-Quadratures/data-sets/raw/main/',dataset,dataset+'.npz')
        try:
            r = requests.get(git_url,stream=True)
            r.raise_for_status()
            data = np.load(BytesIO(r.raw.read()))
        except requests.exceptions.RequestException as e:
            raise SystemExit(e)
        # .md file
        git_url = os.path.join('https://raw.githubusercontent.com/Effective-Quadratures/data-sets/main',dataset,'README.md')
        try:
            r = requests.get(git_url)
            r.raise_for_status()
            print('\n',r.text)
        except requests.exceptions.RequestException as e:
            raise SystemExit(e)

    # If the user has cloned the data-sets repo and provided its location in data_dir
    else:
        print('Loading the dataset from ', data_dir)
        data = np.load(os.path.join(data_dir,dataset,dataset+'.npz'))
        f = open(os.path.join(data_dir,dataset,'README.md'))
        print(f.read())

    return data

def train_test_split(X,y,train=0.7,random_seed=None,shuffle=True):
    """ Split arrays or matrices into random train and test subsets.

    Inspired by :obj:`sklearn.model_selection.train_test_split`.

    Parameters
    ----------
    X : numpy.ndarray
        Array with shape (n_observations,n_dim) containing the inputs.
    y : numpy.ndarray
        Array with shape (n_observations,1) containing the outputs/targets.
    train : float, optional
        Fraction between 0.0 and 1.0, representing the proportion of the dataset to include in the train split.
    random_seed : int, optional
        Seed for random number generator.
    shuffle : bool, optional
        Whether to shuffle the rows of data when spliting.

    Returns
    -------
    tuple
        Tuple (X_train, X_test, y_train, y_test) containing the split data, output as numpy.ndarray's.

    Example
    -------
    >>> X_train, X_test, y_train, y_test = eq.datasets.train_test_split(X, y,
    >>>                                    train=0.8, random_seed = 42)
    """
    if X.shape[0] == y.shape[0]:
        n_observations = X.shape[0]
    else:
        raise ValueError("X and y have different numbers of rows")
    if train > 0 or train < 1:
        n_train = int(train*n_observations)
        n_test  = n_observations - n_train
    else:
        raise ValueError("train should be between 0 and 1")
    if shuffle:
        if np.__version__ >= '1.17':
            generator = np.random.default_rng(random_seed)
        else:
            generator = np.random.RandomState(random_seed)
        idx = generator.permutation(n_observations)
    else:
        idx = np.arange(n_observations)
    idx_train, idx_test = idx[:n_train], idx[n_train:]
    return X[idx_train], X[idx_test], y[idx_train], y[idx_test]

def score(y_true,y_pred,metric='r2',X=None):
    """ Evaluates the accuracy/error score between predictions and the truth, according to the given accuracy metric.

    Parameters
    ----------
    y_pred : numpy.ndarray
        Array with shape (number_of_observations, 1), containing predictions.
    y_true : numpy.ndarray
        Array with shape (number_of_observations, 1) containing the true data.
    metric : str, optional
        The scoring metric to use. Avaliable options are: ```adjusted_r2```, ```r2```, ```mae```, ```rmse```, or ```normalised_mae```.
    X : numpy.ndarray
        The input data associated with **y_pred**. Required if ``metric=`adjusted_r2```.

    Returns
    -------
    float
        The accuracy or error score.
    """
    y_true = y_true.flatten()
    y_pred = y_pred.flatten()
    if metric == 'r2':
         score = st.linregress(y_true,y_pred)[2]**2
    elif metric == 'adjusted_r2':
        if X is None: raise ValueError('Must specify X in _score if adjusted_r2 metric used')
        N,d = X.shape
        r2 = st.linregress(y_true,y_pred)[2]**2
        score = 1.0 - (((1-r2)*(N-1))/(N-d-1))
    elif metric == 'mae':
        score = np.mean(np.abs(y_true-y_pred))
    elif metric == 'normalised_mae':
        score = np.mean(np.abs(y_true-y_pred))/np.std(y_true)
    elif metric == 'rmse':
        score = np.sqrt(np.mean((y_true-y_pred)**2))
    else:
        raise ValueError('Only r2, adjusted_r2, mae, normalised_mae, rmse scoring metrics currently supported')
    return score
<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys
from optparse import OptionParser
import logging

from django.conf import settings

logging.disable(logging.CRITICAL)

if not settings.configured:
    datacash_settings = {}
    try:
        from integration import *
    except ImportError:
        datacash_settings.update({
            'DATACASH_HOST': 'testserver.datacash.com',
            'DATACASH_CLIENT': '',
            'DATACASH_PASSWORD': '',
            'DATACASH_CURRENCY': 'GBP',
            'DATACASH_USE_CV2AVS': True,
        })
    else:
        for key, value in locals().items():
            if key.startswith('DATACASH'):
                datacash_settings[key] = value

    from oscar.defaults import *
    for key, value in locals().items():
        if key.startswith('OSCAR'):
            datacash_settings[key] = value
    datacash_settings['OSCAR_EAGER_ALERTS'] = False

    from oscar import get_core_apps

    settings.configure(
            DATABASES={
                'default': {
                    'ENGINE': 'django.db.backends.sqlite3',
                    }
                },
            INSTALLED_APPS=[
                'django.contrib.auth',
                'django.contrib.admin',
                'django.contrib.contenttypes',
                'django.contrib.sessions',
                'django.contrib.sites',
                'datacash',
                'south',
                ] + get_core_apps(),
            DEBUG=False,
            SITE_ID=1,
            ROOT_URLCONF='tests.urls',
            NOSE_ARGS=['-s'],
            HAYSTACK_CONNECTIONS={
                'default': {
                    'ENGINE': 'haystack.backends.simple_backend.SimpleEngine',
                },
            },
            **datacash_settings)

# Needs to be here to avoid missing SETTINGS env var
from django_nose import NoseTestSuiteRunner

def run_tests(*test_args):
    if 'south' in settings.INSTALLED_APPS:
        from south.management.commands import patch_for_test_db_setup
        patch_for_test_db_setup()

    if not test_args:
        test_args = ['tests']

    # Run tests
    test_runner = NoseTestSuiteRunner(verbosity=1)

    num_failures = test_runner.run_tests(test_args)

    if num_failures > 0:
        sys.exit(num_failures)

def generate_migration():
    from south.management.commands.schemamigration import Command
    com = Command()
    com.handle(app='datacash', initial=True)

if __name__ == '__main__':
    parser = OptionParser()
    (options, args) = parser.parse_args()
    run_tests(*args)
<commit_msg>Conditionally add South to INSTALLED_APPS<commit_after>#!/usr/bin/env python
import sys
from optparse import OptionParser
import logging

import django
from django.conf import settings

logging.disable(logging.CRITICAL)

if not settings.configured:
    datacash_settings = {}
    try:
        from integration import *
    except ImportError:
        datacash_settings.update({
            'DATACASH_HOST': 'testserver.datacash.com',
            'DATACASH_CLIENT': '',
            'DATACASH_PASSWORD': '',
            'DATACASH_CURRENCY': 'GBP',
            'DATACASH_USE_CV2AVS': True,
        })
    else:
        for key, value in locals().items():
            if key.startswith('DATACASH'):
                datacash_settings[key] = value

    from oscar.defaults import *
    for key, value in locals().items():
        if key.startswith('OSCAR'):
            datacash_settings[key] = value
    datacash_settings['OSCAR_EAGER_ALERTS'] = False

    from oscar import get_core_apps

    sandbox_settings = {
        'DATABASES': {
            'default': {
                'ENGINE': 'django.db.backends.sqlite3',
            }
        },
        'INSTALLED_APPS': [
            'django.contrib.auth',
            'django.contrib.admin',
            'django.contrib.contenttypes',
            'django.contrib.sessions',
            'django.contrib.sites',
            'datacash',
        ] + get_core_apps(),
        'DEBUG': False,
        'SITE_ID': 1,
        'ROOT_URLCONF': 'tests.urls',
        'NOSE_ARGS': ['-s'],
        'HAYSTACK_CONNECTIONS': {
            'default': {
                'ENGINE': 'haystack.backends.simple_backend.SimpleEngine',
            },
    }}
    if django.VERSION < (1,7):
        sandbox_settings['INSTALLED_APPS'] += ['south']
    sandbox_settings.update(datacash_settings)
    settings.configure(**sandbox_settings)

# Needs to be here to avoid missing SETTINGS env var
from django_nose import NoseTestSuiteRunner

def run_tests(*test_args):
    if 'south' in settings.INSTALLED_APPS:
        from south.management.commands import patch_for_test_db_setup
        patch_for_test_db_setup()

    if not test_args:
        test_args = ['tests']

    # Run tests
    test_runner = NoseTestSuiteRunner(verbosity=1)

    num_failures = test_runner.run_tests(test_args)

    if num_failures > 0:
        sys.exit(num_failures)

def generate_migration():
    from south.management.commands.schemamigration import Command
    com = Command()
    com.handle(app='datacash', initial=True)

if __name__ == '__main__':
    parser = OptionParser()
    (options, args) = parser.parse_args()
    run_tests(*args)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Sat Apr 08 21:03:32 2017

@author: Ramanuja
"""
from files.configurationFile import Configurations
from files.status import status
from pymongo import MongoClient
from tweetsInsert import getAllTweets

def mongoSearchIndex(Configurations):
    client = MongoClient(Configurations().mongoDBConfig())

    data = getAllTweets(Configurations)
    # to check whether connected to the database
    db = client.test_df# test_df --> database name
    count = 1
    for record in data:
        count = count + 1
        _id = record[0]
        tweet = (record[1].lower()).split()# sentence in to a list
        for word in tweet:
            if db.searchIndex.find_one({'_id':word}):
                temp = db.searchIndex.find_one({'_id':word})
                tweet_ids = temp['tweet_id']#t
                tweet_ids.append(_id)
                db.searchIndex.update_many({'_id':word},{'$set':{'tweet_id':tweet_ids}})
            else:
                db.searchIndex.insert_one({'_id':word,'tweet_id':[_id]})
        status(count,len(data))

mongoSearchIndex(Configurations)
<commit_msg>Update mongoIndex.py<commit_after>from files.configurationFile import Configurations
from files.status import status
from pymongo import MongoClient
from tweetsInsert import getAllTweets

def mongoSearchIndex(Configurations):
    client = MongoClient(Configurations().mongoDBConfig())

    data = getAllTweets(Configurations)
    # to check whether connected to the database
    db = client.test_df# test_df --> database name
    count = 1
    for record in data:
        count = count + 1
        _id = record[0]
        tweet = (record[1].lower()).split()# sentence in to a list
        for word in tweet:
            if db.searchIndex.find_one({'_id':word}):
                temp = db.searchIndex.find_one({'_id':word})
                tweet_ids = temp['tweet_id']#t
                tweet_ids.append(_id)
                db.searchIndex.update_many({'_id':word},{'$set':{'tweet_id':tweet_ids}})
            else:
                db.searchIndex.insert_one({'_id':word,'tweet_id':[_id]})
        status(count,len(data))

mongoSearchIndex(Configurations)
<|endoftext|>


<commit_before># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: GNU General Public License v3. See license.txt

from __future__ import unicode_literals

import frappe

default_mail_footer = """
Sent via
	ERPNext
"""

def after_install():
	frappe.get_doc({'doctype': "Role", "role_name": "Analytics"}).insert()
	set_single_defaults()
	frappe.db.set_default('desktop:home_page', 'setup-wizard');
	feature_setup()
	from erpnext.setup.page.setup_wizard.setup_wizard import add_all_roles_to
	add_all_roles_to("Administrator")
	# TODO remove the admin roles
	remove_admin_roles()
	add_web_forms()
	frappe.db.commit()

def feature_setup():
	"""save global defaults and features setup"""
	doc = frappe.get_doc("Features Setup", "Features Setup")
	doc.flags.ignore_permissions = True

	# store value as 1 for all these fields
	flds = ['fs_item_serial_nos', 'fs_item_batch_nos', 'fs_brands', 'fs_item_barcode',
		'fs_item_advanced', 'fs_packing_details', 'fs_item_group_in_details',
		'fs_exports', 'fs_imports', 'fs_discounts', 'fs_purchase_discounts',
		'fs_after_sales_installations', 'fs_projects', 'fs_sales_extras',
		'fs_recurring_invoice', 'fs_pos', 'fs_manufacturing', 'fs_quality',
		'fs_page_break', 'fs_more_info', 'fs_pos_view'
	]
	for f in flds:
		doc.set(f, 1)
	doc.save()

def set_single_defaults():
	for dt in frappe.db.sql_list("""select name from `tabDocType` where issingle=1"""):
		default_values = frappe.db.sql("""select fieldname, `default` from `tabDocField`
			where parent=%s""", dt)
		if default_values:
			try:
				b = frappe.get_doc(dt, dt)
				for fieldname, value in default_values:
					b.set(fieldname, value)
				b.save()
			except frappe.MandatoryError:
				pass

	frappe.db.set_default("date_format", "dd-mm-yyyy")

def add_web_forms():
	"""Import web forms for Issues and Addresses"""
	from frappe.modules.import_file import import_file_by_path

	import_file_by_path(frappe.get_app_path("erpnext", "setup/fixtures/web_form/issues.json"),
		data_import=True)
	import_file_by_path(frappe.get_app_path("erpnext", "setup/fixtures/web_form/addresses.json"),
		data_import=True)

def remove_admin_roles():

	query = """ DELETE FROM tabUserRole WHERE parent='Administrator' and
				role not in ('Administrator','%s')"""%("roles")
	pass<commit_msg>removed the restricted roles from admin user<commit_after># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: GNU General Public License v3. See license.txt

from __future__ import unicode_literals

import frappe

default_mail_footer = """
Sent via
	ERPNext
"""

def after_install():
	frappe.get_doc({'doctype': "Role", "role_name": "Analytics"}).insert()
	set_single_defaults()
	frappe.db.set_default('desktop:home_page', 'setup-wizard');
	feature_setup()
	from erpnext.setup.page.setup_wizard.setup_wizard import add_all_roles_to
	add_all_roles_to("Administrator")
	# TODO remove the admin roles
	remove_admin_roles()
	add_web_forms()
	frappe.db.commit()

def feature_setup():
	"""save global defaults and features setup"""
	doc = frappe.get_doc("Features Setup", "Features Setup")
	doc.flags.ignore_permissions = True

	# store value as 1 for all these fields
	flds = ['fs_item_serial_nos', 'fs_item_batch_nos', 'fs_brands', 'fs_item_barcode',
		'fs_item_advanced', 'fs_packing_details', 'fs_item_group_in_details',
		'fs_exports', 'fs_imports', 'fs_discounts', 'fs_purchase_discounts',
		'fs_after_sales_installations', 'fs_projects', 'fs_sales_extras',
		'fs_recurring_invoice', 'fs_pos', 'fs_manufacturing', 'fs_quality',
		'fs_page_break', 'fs_more_info', 'fs_pos_view'
	]
	for f in flds:
		doc.set(f, 1)
	doc.save()

def set_single_defaults():
	for dt in frappe.db.sql_list("""select name from `tabDocType` where issingle=1"""):
		default_values = frappe.db.sql("""select fieldname, `default` from `tabDocField`
			where parent=%s""", dt)
		if default_values:
			try:
				b = frappe.get_doc(dt, dt)
				for fieldname, value in default_values:
					b.set(fieldname, value)
				b.save()
			except frappe.MandatoryError:
				pass

	frappe.db.set_default("date_format", "dd-mm-yyyy")

def add_web_forms():
	"""Import web forms for Issues and Addresses"""
	from frappe.modules.import_file import import_file_by_path

	import_file_by_path(frappe.get_app_path("erpnext", "setup/fixtures/web_form/issues.json"),
		data_import=True)
	import_file_by_path(frappe.get_app_path("erpnext", "setup/fixtures/web_form/addresses.json"),
		data_import=True)

def remove_admin_roles():
	"""Remove all the Administrator roles except allowed roles"""
	from omnitechapp.omnitechapp.doctype.package_detail.package_detail import get_allowed_roles
	try:
		roles = ["'%s'"%(role) for role in get_allowed_roles()]

		if not roles:
			frappe.throw("Error occured during setup, Please contact Administrator")

		query = """ DELETE FROM tabUserRole WHERE parent='Administrator' AND
					role NOT IN ('Administrator',%s)"""%(",".join(roles))

		frappe.db.sql(query)
	except Exception, e:
		frappe.throw(e)<|endoftext|>