<commit_before>"""Implementation of restricted boltzmann machine

You need to be able to deal with different energy functions
This allows you to deal with real valued units.

TODO: monitor overfitting
"""
import numpy as np
from common import *

import theano
from theano import tensor as T
from theano.ifelse import ifelse as theanoifelse
from theano.tensor.shared_randomstreams import RandomStreams

theanoFloat  = theano.config.floatX

EXPENSIVE_CHECKS_ON = False

# I need a mini batch trainer for this

class RBMMiniBatchTrainer(object):

  # TODO: i need to see how I do it with the sampling, because
  # we do not sample all the time to make them binary
  def __init__(self, input, initialWeights, initialBiases,
             visibleDropout, hiddenDropout, cdSteps=1):

    self.visible = input
    self.theano_rng = RandomStreams(seed=np.random.randint(1, 1000))

    self.weights = theano.shared(value=np.asarray(initialWeights,
                                         dtype=theanoFloat),
                        name='W')
    self.biasVisible = theano.shared(value=np.asarray(initialBiases[0],
                                         dtype=theanoFloat),
                        name='bvis')
    self.biasHidden = theano.shared(value=np.asarray(initialBiases[1],
                                         dtype=theanoFloat),
                        name='bhid')

    oldDw = theano.shared(value=np.zeros(shape=initialWeights.shape,
                                           dtype=theanoFloat))
    oldDVis = theano.shared(value=np.zeros(shape=initialBiases[0].shape,
                                           dtype=theanoFloat))
    oldDHid = theano.shared(value=np.zeros(shape=initialBiases[1].shape,
                                           dtype=theanoFloat))

    self.oldDParams = [oldDw, oldDVis, oldDHid]

    # TODO: scan for a loop in theano
    # TODO: only sample from the hidden ones that are not the last one
    # Now do 1 step in CD, but later you will have to do more
    # I think that should happen with scan or something
    hiddenActivations = T.nnet.sigmoid(T.dot(input, self.weights) + self.biasHidden)
    self.hidden = self.theano_rng.binomial(size=hiddenActivations.shape,
                                          n=1, p=hiddenActivations,
                                          dtype=theanoFloat)

    visibleRec = T.nnet.sigmoid(T.dot(self.hidden, self.weights.T) + self.biasVisible)
    self.visibleReconstruction = self.theano_rng.binomial(size=visibleRec.shape,
                                          n=1, p=visibleRec,
                                          dtype=theanoFloat)
    hiddenRec = T.nnet.sigmoid(T.dot(self.visibleReconstruction, self.weights) + self.biasHidden)
    self.hiddenReconstruction = hiddenRec

# TODO: different learning rates for weights and biases
"""
 Represents a RBM
"""
class RBM(object):

  def __init__(self, nrVisible, nrHidden, trainingFunction, dropout,
                visibleDropout, activationFun=sigmoid):
    # dropout = 1 means no dropout, keep all the weights
    self.dropout = dropout
    # dropout = 1 means no dropout, keep all the weights
    self.visibleDropout = visibleDropout
    self.nrHidden = nrHidden
    self.nrVisible = nrVisible
    self.trainingFunction = trainingFunction
    self.activationFun = activationFun
    self.initialized = False

  def train(self, data, learningRate=0.01, miniBatchSize=10):
    if not self.initialized:
      self.weights = initializeWeights(self.nrVisible, self.nrHidden)
      self.biases = intializeBiases(data, self.nrHidden)
      self.initialized = True

    sharedData = theano.shared(np.asarray(data, dtype=theanoFloat))

    self.miniBatchSize = miniBatchSize
    # Now you have to build the training function
    # and the updates
    # The mini-batch data is a matrix
    x = T.matrix('x', dtype=theanoFloat)

    miniBatchIndex = T.lscalar()
    momentum = T.fscalar()

    batchLearningRate = learningRate / miniBatchSize

    batchTrainer = RBMMiniBatchTrainer(input=x,
                                       initialWeights=self.weights,
                                       initialBiases=self.biases,
                                       visibleDropout=0.8,
                                       hiddenDropout=0.5)

    updates = []
    # The theano people do not need this because they use gradient
    # I wonder how that works
    positiveDifference = T.dot(batchTrainer.visible.T, batchTrainer.hidden)
    negativeDifference = T.dot(batchTrainer.visibleReconstruction.T,
                               batchTrainer.hiddenReconstruction)
    wUpdate = momentum * batchTrainer.oldDParams[0] + batchLearningRate * (positiveDifference - negativeDifference)
    updates.append((batchTrainer.weights, batchTrainer.weights + wUpdate))
    updates.append((batchTrainer.oldDParams[0], wUpdate))

    visibleBiasDiff = T.sum(x - batchTrainer.visible, axis=0)
    biasVisUpdate = momentum * batchTrainer.oldDParams[1] + batchLearningRate * visibleBiasDiff
    updates.append((batchTrainer.biasVisible, batchTrainer.biasVisible + biasVisUpdate))
    updates.append((batchTrainer.oldDParams[1], biasVisUpdate))

    hiddenBiasDiff = T.sum(batchTrainer.hidden - batchTrainer.hiddenReconstruction, axis=0)
    biasHidUpdate = momentum * batchTrainer.oldDParams[2] + batchLearningRate * hiddenBiasDiff
    updates.append((batchTrainer.biasHidden, batchTrainer.biasHidden + biasHidUpdate))
    updates.append((batchTrainer.oldDParams[2], biasHidUpdate))

    train_function = theano.function(
      inputs=[miniBatchIndex, momentum],
      outputs=[], # TODO: output error
      updates=updates,
      givens={
        x: sharedData[miniBatchIndex * self.miniBatchSize:(miniBatchIndex + 1) * self.miniBatchSize]
        })

    nrMiniBatches = len(data) / miniBatchSize
    # The rbm trainign has only one step, you do multiple for the dbn,
    # so maybe not put it here
    epochs = 100
    for epoch in xrange(epochs):
      for miniBatchIndex in range(nrMiniBatches):
        if epoch < 10:
          momentum = 0.5
        else:
          momentum = 0.95

        train_function(miniBatchIndex, momentum)

    self.weights = batchTrainer.weights.get_value()
    self.biases = [batchTrainer.biasVisible.get_value(),
                   batchTrainer.biasHidden.get_value()]

    self.testWeights = self.weights

    print reconstructionError(self.biases, self.weights, data, self.activationFun)

    assert self.weights.shape == (self.nrVisible, self.nrHidden)
    assert self.biases[0].shape[0] == self.nrVisible
    assert self.biases[1].shape[0] == self.nrHidden

  def trainold(self, data):
    # If the network has not been initialized yet, do it now
    # Ie if this is the time it is traning batch of traning
    if not self.initialized:
      self.weights = self.initializeWeights(self.nrVisible, self.nrHidden)
      self.biases = self.intializeBiases(data, self.nrHidden)
      self.initialized = True

    for i in xrange(50):
      self.biases, self.weights = self.trainingFunction(data,
                                                      self.biases,
                                                      self.weights,
                                                      self.activationFun,
                                                      self.dropout,
                                                      self.visibleDropout)
    # TODO:you have to do this for the biases as well
    # TODO: check that I do this in the deep belief net
    self.testWeights = self.weights * self.dropout

    assert self.weights.shape == (self.nrVisible, self.nrHidden)
    assert self.biases[0].shape[0] == self.nrVisible
    assert self.biases[1].shape[0] == self.nrHidden

  """ Reconstructs the data given using this boltzmann machine."""
  def reconstruct(self, dataInstances):
    return reconstruct(self.biases, self.testWeights, dataInstances,
                       self.activationFun)

  def hiddenRepresentation(self, dataInstances):
    return updateLayer(Layer.HIDDEN, dataInstances, self.biases,
                       self.testWeights, self.activationFun, True)

def reconstruct(biases, weights, dataInstances, activationFun):
  hidden = updateLayer(Layer.HIDDEN, dataInstances, biases, weights,
                       activationFun, True)

  visibleReconstructions = updateLayer(Layer.VISIBLE, hidden,
                                      biases, weights, activationFun, False)
  return visibleReconstructions

def reconstructionError(biases, weights, data, activationFun):
    # Returns the rmse of the reconstruction of the data
    # Good to keep track of it, should decrease trough training
    # Initially faster, and then slower
    reconstructions = reconstruct(biases, weights, data, activationFun)
    return rmse(reconstructions, data)

""" Training functions."""

""" Full CD function.
Arguments:
  data: the data to use for traning.
    A numpy ndarray.
  biases:

Returns:

Defaults the mini batch size 1, so normal learning
"""
# Think of removing the step method all together and keep one to just
# optimize the code but also make it easier to change them
# rather than have a function  that you pass in for every batch
# if nice and easy refactoring can be seen then you can do that
def contrastiveDivergence(data, biases, weights, activationFun, dropout,
                          visibleDropout, miniBatchSize=10):
  N = len(data)
  epochs = N / miniBatchSize

  # sample the probabily distributions allow you to chose from the
  # visible units for dropout
  on = sample(visibleDropout, data.shape)
  dropoutData = data * on

  learningRate = 0.01
  decayFactor = 0.0002
  weightDecay = True
  reconstructionStep = 50

  oldDeltaWeights = np.zeros(weights.shape)
  oldDeltaVisible = np.zeros(biases[0].shape)
  oldDeltaHidden = np.zeros(biases[1].shape)

  batchLearningRate = learningRate / miniBatchSize
  print "batchLearningRate"
  print batchLearningRate

  for epoch in xrange(epochs):
    batchData = dropoutData[epoch * miniBatchSize: (epoch + 1) * miniBatchSize, :]
    if epoch < epochs / 100:
      momentum = 0.5
    else:
      momentum = 0.95

    if epoch < (N/7) * 10:
      cdSteps = 3
    elif epoch < (N/9) * 10:
      cdSteps = 5
    else:
      cdSteps = 10

    if EXPENSIVE_CHECKS_ON:
      if epoch % reconstructionStep == 0:
        print "reconstructionError"
        print reconstructionError(biases, weights, data, activationFun)

    weightsDiff, visibleBiasDiff, hiddenBiasDiff =\
            modelAndDataSampleDiffs(batchData, biases, weights,
            activationFun, dropout, cdSteps)
    # Update the weights
    # data - model
    # Positive phase - negative
    # Weight decay factor
    # TODO: RMSPROP here as well.
    deltaWeights = (batchLearningRate * weightsDiff
                    - epsilon * weightDecay * decayFactor * weights)

    deltaVisible = batchLearningRate * visibleBiasDiff
    deltaHidden  = batchLearningRate * hiddenBiasDiff

    deltaWeights += momentum * oldDeltaWeights
    deltaVisible += momentum * oldDeltaVisible
    deltaHidden += momentum * oldDeltaHidden

    oldDeltaWeights = deltaWeights
    oldDeltaVisible = deltaVisible
    oldDeltaHidden = deltaHidden

    # Update the weighths
    weights += deltaWeights
    # Update the visible biases
    biases[0] += deltaVisible

    # Update the hidden biases
    biases[1] += deltaHidden

  print reconstructionError(biases, weights, data, activationFun)
  return biases, weights

def modelAndDataSampleDiffs(batchData, biases, weights, activationFun,
                            dropout, cdSteps):
  # Reconstruct the hidden weigs from the data
  hidden = updateLayer(Layer.HIDDEN, batchData, biases, weights, activationFun,
                       binary=True)

  # Chose the units to be active at this point
  # different sets for each element in the mini batches
  on = sample(dropout, hidden.shape)
  dropoutHidden = on * hidden
  hiddenReconstruction = dropoutHidden

  for i in xrange(cdSteps - 1):
    visibleReconstruction = updateLayer(Layer.VISIBLE, hiddenReconstruction,
                                        biases, weights, activationFun,
                                        binary=False)
    hiddenReconstruction = updateLayer(Layer.HIDDEN, visibleReconstruction,
                                       biases, weights, activationFun,
                                       binary=True)
    # sample the hidden units active (for dropout)
    hiddenReconstruction = hiddenReconstruction * on

  # Do the last reconstruction from the probabilities in the last phase
  visibleReconstruction = updateLayer(Layer.VISIBLE, hiddenReconstruction,
                                      biases, weights, activationFun,
                                      binary=False)
  hiddenReconstruction = updateLayer(Layer.HIDDEN, visibleReconstruction,
                                     biases, weights, activationFun,
                                     binary=False)

  hiddenReconstruction = hiddenReconstruction * on
  # here it should be hidden * on - hiddenreconstruction
  # also below in the hidden bias
  weightsDiff = np.dot(batchData.T, dropoutHidden) -\
                np.dot(visibleReconstruction.T, hiddenReconstruction)
  assert weightsDiff.shape == weights.shape

  visibleBiasDiff = np.sum(batchData - visibleReconstruction, axis=0)
  assert visibleBiasDiff.shape == biases[0].shape

  hiddenBiasDiff = np.sum(dropoutHidden - hiddenReconstruction, axis=0)
  assert hiddenBiasDiff.shape == biases[1].shape

  return weightsDiff, visibleBiasDiff, hiddenBiasDiff

""" Updates an entire layer. This procedure can be used both in training
    and in testing.
    Can even take multiple values of the layer, each of them given as rows
    Uses matrix operations.
"""
def updateLayer(layer, otherLayerValues, biases, weights, activationFun,
                binary=False):

  bias = biases[layer]
  size = otherLayerValues.shape[0]

  if layer == Layer.VISIBLE:
    activation = np.dot(otherLayerValues, weights.T)
  else:
    activation = np.dot(otherLayerValues, weights)

  probs = activationFun(np.tile(bias, (size, 1)) + activation)

  if binary:
    # Sample from the distributions
    return sampleAll(probs)

  return probs

def initializeWeights(nrVisible, nrHidden):
  return np.random.normal(0, 0.01, (nrVisible, nrHidden))

def intializeBiases(data, nrHidden):
  # get the procentage of data points that have the i'th unit on
  # and set the visible vias to log (p/(1-p))
  percentages = data.mean(axis=0, dtype='float')
  vectorized = np.vectorize(safeLogFraction, otypes=[np.float])
  visibleBiases = vectorized(percentages)

  hiddenBiases = np.zeros(nrHidden)
  return np.array([visibleBiases, hiddenBiases])
<commit_msg>trying to use theano scan to have multiple cd steps<commit_after>"""Implementation of restricted boltzmann machine

You need to be able to deal with different energy functions
This allows you to deal with real valued units.

TODO: monitor overfitting
"""
import numpy as np
from common import *

import theano
from theano import tensor as T
from theano.ifelse import ifelse as theanoifelse
from theano.tensor.shared_randomstreams import RandomStreams

theanoFloat  = theano.config.floatX

EXPENSIVE_CHECKS_ON = False

# I need a mini batch trainer for this

class RBMMiniBatchTrainer(object):

  # TODO: i need to see how I do it with the sampling, because
  # we do not sample all the time to make them binary
  def __init__(self, input, initialWeights, initialBiases,
             visibleDropout, hiddenDropout, cdSteps):

    self.visible = input
    self.theano_rng = RandomStreams(seed=np.random.randint(1, 1000))

    self.weights = theano.shared(value=np.asarray(initialWeights,
                                         dtype=theanoFloat),
                        name='W')
    self.biasVisible = theano.shared(value=np.asarray(initialBiases[0],
                                         dtype=theanoFloat),
                        name='bvis')
    self.biasHidden = theano.shared(value=np.asarray(initialBiases[1],
                                         dtype=theanoFloat),
                        name='bhid')

    oldDw = theano.shared(value=np.zeros(shape=initialWeights.shape,
                                           dtype=theanoFloat))
    oldDVis = theano.shared(value=np.zeros(shape=initialBiases[0].shape,
                                           dtype=theanoFloat))
    oldDHid = theano.shared(value=np.zeros(shape=initialBiases[1].shape,
                                           dtype=theanoFloat))

    self.oldDParams = [oldDw, oldDVis, oldDHid]

    # This does not sample the visible layers, but samples
    # The hidden layers up to the last one, like Hinton suggests
    def OneSampleStep(visibleSample):
      hiddenActivations = T.nnet.sigmoid(T.dot(visibleSample, self.weights) + self.biasHidden)
      hidden = self.theano_rng.binomial(size=hiddenActivations.shape,
                                          n=1, p=hiddenActivations,
                                          dtype=theanoFloat)
      visibleRec = T.nnet.sigmoid(T.dot(self.hidden, self.weights.T) + self.biasVisible)
      return hidden, visibleRec

    # sample = theano.tensor.vector()
    results, updates = theano.scan(OneSampleStep,
                          outputs_info=self.visible,
                          n_steps=cdSteps)

    self.updates = updates
    # Create the gibbs sampling function
    # It is important to add the updates to the function in order to change
    # the theano random number generator
    # gibbsSampling = theano.function(inputs=[sample],
    #     outputs=[results[0], results[-2], results[-3]]
    #     , updates=updates)

    self.hidden = self.results[0][0]
    self.visibleReconstruction = self.results[-1][1]

    # Do not sample for the last one, in order to get less sampling noise
    hiddenRec = T.nnet.sigmoid(T.dot(self.visibleReconstruction, self.weights) + self.biasHidden)
    self.hiddenReconstruction = hiddenRec

# TODO: different learning rates for weights and biases
"""
 Represents a RBM
"""
class RBM(object):

  def __init__(self, nrVisible, nrHidden, trainingFunction, dropout,
                visibleDropout, activationFun=sigmoid):
    # dropout = 1 means no dropout, keep all the weights
    self.dropout = dropout
    # dropout = 1 means no dropout, keep all the weights
    self.visibleDropout = visibleDropout
    self.nrHidden = nrHidden
    self.nrVisible = nrVisible
    self.trainingFunction = trainingFunction
    self.activationFun = activationFun
    self.initialized = False

  def train(self, data, learningRate=0.01, miniBatchSize=10):
    if not self.initialized:
      self.weights = initializeWeights(self.nrVisible, self.nrHidden)
      self.biases = intializeBiases(data, self.nrHidden)
      self.initialized = True

    sharedData = theano.shared(np.asarray(data, dtype=theanoFloat))

    self.miniBatchSize = miniBatchSize
    # Now you have to build the training function
    # and the updates
    # The mini-batch data is a matrix
    x = T.matrix('x', dtype=theanoFloat)

    miniBatchIndex = T.lscalar()
    momentum = T.fscalar()

    batchLearningRate = learningRate / miniBatchSize

    batchTrainer = RBMMiniBatchTrainer(input=x,
                                       initialWeights=self.weights,
                                       initialBiases=self.biases,
                                       visibleDropout=0.8,
                                       hiddenDropout=0.5,
                                       cdSteps=1)

    updates = []
    # The theano people do not need this because they use gradient
    # I wonder how that works
    positiveDifference = T.dot(batchTrainer.visible.T, batchTrainer.hidden)
    negativeDifference = T.dot(batchTrainer.visibleReconstruction.T,
                               batchTrainer.hiddenReconstruction)
    wUpdate = momentum * batchTrainer.oldDParams[0] + batchLearningRate * (positiveDifference - negativeDifference)
    updates.append((batchTrainer.weights, batchTrainer.weights + wUpdate))
    updates.append((batchTrainer.oldDParams[0], wUpdate))

    visibleBiasDiff = T.sum(x - batchTrainer.visible, axis=0)
    biasVisUpdate = momentum * batchTrainer.oldDParams[1] + batchLearningRate * visibleBiasDiff
    updates.append((batchTrainer.biasVisible, batchTrainer.biasVisible + biasVisUpdate))
    updates.append((batchTrainer.oldDParams[1], biasVisUpdate))

    hiddenBiasDiff = T.sum(batchTrainer.hidden - batchTrainer.hiddenReconstruction, axis=0)
    biasHidUpdate = momentum * batchTrainer.oldDParams[2] + batchLearningRate * hiddenBiasDiff
    updates.append((batchTrainer.biasHidden, batchTrainer.biasHidden + biasHidUpdate))
    updates.append((batchTrainer.oldDParams[2], biasHidUpdate))

    # Add the updates required for the theano random generator
    updates.append(batchTrainer.updates)

    train_function = theano.function(
      inputs=[miniBatchIndex, momentum],
      outputs=[], # TODO: output error
      updates=updates,
      givens={
        x: sharedData[miniBatchIndex * self.miniBatchSize:(miniBatchIndex + 1) * self.miniBatchSize]
        })

    nrMiniBatches = len(data) / miniBatchSize
    # The rbm trainign has only one step, you do multiple for the dbn,
    # so maybe not put it here
    epochs = 100
    for epoch in xrange(epochs):
      for miniBatchIndex in range(nrMiniBatches):
        if epoch < 10:
          momentum = 0.5
        else:
          momentum = 0.95
          batchTrainer.cdSteps = 3

        train_function(miniBatchIndex, momentum)

    self.weights = batchTrainer.weights.get_value()
    self.biases = [batchTrainer.biasVisible.get_value(),
                   batchTrainer.biasHidden.get_value()]

    self.testWeights = self.weights

    print reconstructionError(self.biases, self.weights, data, self.activationFun)

    assert self.weights.shape == (self.nrVisible, self.nrHidden)
    assert self.biases[0].shape[0] == self.nrVisible
    assert self.biases[1].shape[0] == self.nrHidden

  def trainold(self, data):
    # If the network has not been initialized yet, do it now
    # Ie if this is the time it is traning batch of traning
    if not self.initialized:
      self.weights = self.initializeWeights(self.nrVisible, self.nrHidden)
      self.biases = self.intializeBiases(data, self.nrHidden)
      self.initialized = True

    for i in xrange(50):
      self.biases, self.weights = self.trainingFunction(data,
                                                      self.biases,
                                                      self.weights,
                                                      self.activationFun,
                                                      self.dropout,
                                                      self.visibleDropout)
    # TODO:you have to do this for the biases as well
    # TODO: check that I do this in the deep belief net
    self.testWeights = self.weights * self.dropout

    assert self.weights.shape == (self.nrVisible, self.nrHidden)
    assert self.biases[0].shape[0] == self.nrVisible
    assert self.biases[1].shape[0] == self.nrHidden

  """ Reconstructs the data given using this boltzmann machine."""
  def reconstruct(self, dataInstances):
    return reconstruct(self.biases, self.testWeights, dataInstances,
                       self.activationFun)

  def hiddenRepresentation(self, dataInstances):
    return updateLayer(Layer.HIDDEN, dataInstances, self.biases,
                       self.testWeights, self.activationFun, True)

def reconstruct(biases, weights, dataInstances, activationFun):
  hidden = updateLayer(Layer.HIDDEN, dataInstances, biases, weights,
                       activationFun, True)

  visibleReconstructions = updateLayer(Layer.VISIBLE, hidden,
                                      biases, weights, activationFun, False)
  return visibleReconstructions

def reconstructionError(biases, weights, data, activationFun):
    # Returns the rmse of the reconstruction of the data
    # Good to keep track of it, should decrease trough training
    # Initially faster, and then slower
    reconstructions = reconstruct(biases, weights, data, activationFun)
    return rmse(reconstructions, data)

""" Training functions."""

""" Full CD function.
Arguments:
  data: the data to use for traning.
    A numpy ndarray.
  biases:

Returns:

Defaults the mini batch size 1, so normal learning
"""
# Think of removing the step method all together and keep one to just
# optimize the code but also make it easier to change them
# rather than have a function  that you pass in for every batch
# if nice and easy refactoring can be seen then you can do that
def contrastiveDivergence(data, biases, weights, activationFun, dropout,
                          visibleDropout, miniBatchSize=10):
  N = len(data)
  epochs = N / miniBatchSize

  # sample the probabily distributions allow you to chose from the
  # visible units for dropout
  on = sample(visibleDropout, data.shape)
  dropoutData = data * on

  learningRate = 0.01
  decayFactor = 0.0002
  weightDecay = True
  reconstructionStep = 50

  oldDeltaWeights = np.zeros(weights.shape)
  oldDeltaVisible = np.zeros(biases[0].shape)
  oldDeltaHidden = np.zeros(biases[1].shape)

  batchLearningRate = learningRate / miniBatchSize
  print "batchLearningRate"
  print batchLearningRate

  for epoch in xrange(epochs):
    batchData = dropoutData[epoch * miniBatchSize: (epoch + 1) * miniBatchSize, :]
    if epoch < epochs / 100:
      momentum = 0.5
    else:
      momentum = 0.95

    if epoch < (N/7) * 10:
      cdSteps = 3
    elif epoch < (N/9) * 10:
      cdSteps = 5
    else:
      cdSteps = 10

    if EXPENSIVE_CHECKS_ON:
      if epoch % reconstructionStep == 0:
        print "reconstructionError"
        print reconstructionError(biases, weights, data, activationFun)

    weightsDiff, visibleBiasDiff, hiddenBiasDiff =\
            modelAndDataSampleDiffs(batchData, biases, weights,
            activationFun, dropout, cdSteps)
    # Update the weights
    # data - model
    # Positive phase - negative
    # Weight decay factor
    # TODO: RMSPROP here as well.
    deltaWeights = (batchLearningRate * weightsDiff
                    - epsilon * weightDecay * decayFactor * weights)

    deltaVisible = batchLearningRate * visibleBiasDiff
    deltaHidden  = batchLearningRate * hiddenBiasDiff

    deltaWeights += momentum * oldDeltaWeights
    deltaVisible += momentum * oldDeltaVisible
    deltaHidden += momentum * oldDeltaHidden

    oldDeltaWeights = deltaWeights
    oldDeltaVisible = deltaVisible
    oldDeltaHidden = deltaHidden

    # Update the weighths
    weights += deltaWeights
    # Update the visible biases
    biases[0] += deltaVisible

    # Update the hidden biases
    biases[1] += deltaHidden

  print reconstructionError(biases, weights, data, activationFun)
  return biases, weights

def modelAndDataSampleDiffs(batchData, biases, weights, activationFun,
                            dropout, cdSteps):
  # Reconstruct the hidden weigs from the data
  hidden = updateLayer(Layer.HIDDEN, batchData, biases, weights, activationFun,
                       binary=True)

  # Chose the units to be active at this point
  # different sets for each element in the mini batches
  on = sample(dropout, hidden.shape)
  dropoutHidden = on * hidden
  hiddenReconstruction = dropoutHidden

  for i in xrange(cdSteps - 1):
    visibleReconstruction = updateLayer(Layer.VISIBLE, hiddenReconstruction,
                                        biases, weights, activationFun,
                                        binary=False)
    hiddenReconstruction = updateLayer(Layer.HIDDEN, visibleReconstruction,
                                       biases, weights, activationFun,
                                       binary=True)
    # sample the hidden units active (for dropout)
    hiddenReconstruction = hiddenReconstruction * on

  # Do the last reconstruction from the probabilities in the last phase
  visibleReconstruction = updateLayer(Layer.VISIBLE, hiddenReconstruction,
                                      biases, weights, activationFun,
                                      binary=False)
  hiddenReconstruction = updateLayer(Layer.HIDDEN, visibleReconstruction,
                                     biases, weights, activationFun,
                                     binary=False)

  hiddenReconstruction = hiddenReconstruction * on
  # here it should be hidden * on - hiddenreconstruction
  # also below in the hidden bias
  weightsDiff = np.dot(batchData.T, dropoutHidden) -\
                np.dot(visibleReconstruction.T, hiddenReconstruction)
  assert weightsDiff.shape == weights.shape

  visibleBiasDiff = np.sum(batchData - visibleReconstruction, axis=0)
  assert visibleBiasDiff.shape == biases[0].shape

  hiddenBiasDiff = np.sum(dropoutHidden - hiddenReconstruction, axis=0)
  assert hiddenBiasDiff.shape == biases[1].shape

  return weightsDiff, visibleBiasDiff, hiddenBiasDiff

""" Updates an entire layer. This procedure can be used both in training
    and in testing.
    Can even take multiple values of the layer, each of them given as rows
    Uses matrix operations.
"""
def updateLayer(layer, otherLayerValues, biases, weights, activationFun,
                binary=False):

  bias = biases[layer]
  size = otherLayerValues.shape[0]

  if layer == Layer.VISIBLE:
    activation = np.dot(otherLayerValues, weights.T)
  else:
    activation = np.dot(otherLayerValues, weights)

  probs = activationFun(np.tile(bias, (size, 1)) + activation)

  if binary:
    # Sample from the distributions
    return sampleAll(probs)

  return probs

def initializeWeights(nrVisible, nrHidden):
  return np.random.normal(0, 0.01, (nrVisible, nrHidden))

def intializeBiases(data, nrHidden):
  # get the procentage of data points that have the i'th unit on
  # and set the visible vias to log (p/(1-p))
  percentages = data.mean(axis=0, dtype='float')
  vectorized = np.vectorize(safeLogFraction, otypes=[np.float])
  visibleBiases = vectorized(percentages)

  hiddenBiases = np.zeros(nrHidden)
  return np.array([visibleBiases, hiddenBiases])
<|endoftext|>


<commit_before>from django.test import TestCase
from freezegun import freeze_time
from mock import patch
from corehq.util.soft_assert.core import SoftAssert

from corehq.form_processor.backends.sql.dbaccessors import CaseAccessorSQL
from corehq.form_processor.backends.sql.processor import FormProcessorSQL
from corehq.form_processor.backends.sql.update_strategy import SqlCaseUpdateStrategy
from corehq.form_processor.interfaces.processor import ProcessedForms
from corehq.form_processor.models import (
    CommCareCaseSQL,
    CaseTransaction,
)
from corehq.form_processor.utils import TestFormMetadata
from corehq.form_processor.tests.utils import use_sql_backend, FormProcessorTestUtils
from corehq.util.test_utils import get_form_ready_to_save

import uuid
from datetime import datetime

@use_sql_backend
class SqlUpdateStrategyTest(TestCase):
    DOMAIN = 'update-strategy-test-' + uuid.uuid4().hex
    USER_ID = 'mr_wednesday_'

    @classmethod
    def setUpClass(cls):
        super(SqlUpdateStrategyTest, cls).setUpClass()
        FormProcessorTestUtils.delete_all_sql_forms()
        FormProcessorTestUtils.delete_all_sql_cases()

    @classmethod
    def tearDownClass(cls):
        FormProcessorTestUtils.delete_all_sql_forms()
        FormProcessorTestUtils.delete_all_sql_cases()
        super(SqlUpdateStrategyTest, cls).tearDownClass()

    @patch.object(SoftAssert, '_call')
    def test_reconcile_transactions(self, soft_assert_mock):
        with freeze_time("2018-10-10"):
            case = self._create_case()

        with freeze_time("2018-10-11"):
            new_old_xform = self._create_form()
        with freeze_time("2018-10-08"):
            new_old_trans = self._create_case_transaction(case, new_old_xform)
        with freeze_time("2018-10-11"):
            case.track_create(new_old_trans)
            FormProcessorSQL.save_processed_models(ProcessedForms(new_old_xform, []), [case])

        case = CaseAccessorSQL.get_case(case.case_id)
        update_strategy = SqlCaseUpdateStrategy(case)
        self.assertTrue(update_strategy.reconcile_transactions_if_necessary())
        for call in soft_assert_mock.call_args_list:
            self.assertNotIn('ReconciliationError', call[0][1])

        CaseAccessorSQL.save_case(case)

        case = CaseAccessorSQL.get_case(case.case_id)
        update_strategy = SqlCaseUpdateStrategy(case)
        self.assertFalse(update_strategy.reconcile_transactions_if_necessary())

    def _create_form(self, user_id=None, received_on=None):
        """
        Create the models directly so that these tests aren't dependent on any
        other apps.
        :return: XFormInstanceSQL
        """
        user_id = user_id or 'mr_wednesday'
        received_on = received_on or datetime.utcnow()

        metadata = TestFormMetadata(
            domain=self.DOMAIN,
            received_on=received_on,
            user_id=user_id
        )
        form = get_form_ready_to_save(metadata)

        return form

    def _create_case_transaction(self, case, form=None, submitted_on=None, action_types=None):
        form = form or self._create_form()
        submitted_on = submitted_on or datetime.utcnow()

        return CaseTransaction.form_transaction(case, form, submitted_on, action_types)

    def _create_case(self, case_type=None, user_id=None, case_id=None):
        case_id = case_id or uuid.uuid4().hex
        user_id = user_id or 'mr_wednesday'
        utcnow = datetime.utcnow()

        case = CommCareCaseSQL(
            case_id=case_id,
            domain=self.DOMAIN,
            type=case_type or '',
            owner_id=user_id,
            opened_on=utcnow,
            modified_on=utcnow,
            modified_by=utcnow,
            server_modified_on=utcnow
        )

        form = self._create_form(user_id, utcnow)
        case.track_create(self._create_case_transaction(case, form, utcnow, action_types=[128]))
        FormProcessorSQL.save_processed_models(ProcessedForms(form, []), [case])

        return CaseAccessorSQL.get_case(case_id)
<commit_msg>testing null case<commit_after>from django.test import TestCase
from freezegun import freeze_time
from mock import patch
from corehq.util.soft_assert.core import SoftAssert

from corehq.form_processor.backends.sql.dbaccessors import CaseAccessorSQL
from corehq.form_processor.backends.sql.processor import FormProcessorSQL
from corehq.form_processor.backends.sql.update_strategy import SqlCaseUpdateStrategy
from corehq.form_processor.interfaces.processor import ProcessedForms
from corehq.form_processor.models import (
    CommCareCaseSQL,
    CaseTransaction,
)
from corehq.form_processor.utils import TestFormMetadata
from corehq.form_processor.tests.utils import use_sql_backend, FormProcessorTestUtils
from corehq.util.test_utils import get_form_ready_to_save

import uuid
from datetime import datetime

@use_sql_backend
class SqlUpdateStrategyTest(TestCase):
    DOMAIN = 'update-strategy-test-' + uuid.uuid4().hex
    USER_ID = 'mr_wednesday_'

    @classmethod
    def setUpClass(cls):
        super(SqlUpdateStrategyTest, cls).setUpClass()
        FormProcessorTestUtils.delete_all_sql_forms()
        FormProcessorTestUtils.delete_all_sql_cases()

    @classmethod
    def tearDownClass(cls):
        FormProcessorTestUtils.delete_all_sql_forms()
        FormProcessorTestUtils.delete_all_sql_cases()
        super(SqlUpdateStrategyTest, cls).tearDownClass()

    @patch.object(SoftAssert, '_call')
    def test_reconcile_transactions(self, soft_assert_mock):
        with freeze_time("2018-10-10"):
            case = self._create_case()

        with freeze_time("2018-10-11"):
            new_old_xform = self._create_form()
        with freeze_time("2018-10-08"):
            new_old_trans = self._create_case_transaction(case, new_old_xform)
        with freeze_time("2018-10-11"):
            case.track_create(new_old_trans)
            FormProcessorSQL.save_processed_models(ProcessedForms(new_old_xform, []), [case])

        case = CaseAccessorSQL.get_case(case.case_id)
        update_strategy = SqlCaseUpdateStrategy(case)
        self.assertTrue(update_strategy.reconcile_transactions_if_necessary())
        for call in soft_assert_mock.call_args_list:
            self.assertNotIn('ReconciliationError', call[0][1])

        CaseAccessorSQL.save_case(case)

        case = CaseAccessorSQL.get_case(case.case_id)
        update_strategy = SqlCaseUpdateStrategy(case)
        self.assertFalse(update_strategy.reconcile_transactions_if_necessary())
        self._check_for_reconciliation_error_soft_assert(soft_assert_mock)

    def test_reconcile_not_necessary(self):
        with freeze_time("2018-10-10"):
            case = self._create_case()

        with freeze_time("2018-10-11"):
            new_old_xform = self._create_form()
            new_old_trans = self._create_case_transaction(case, new_old_xform)
            case.track_create(new_old_trans)
            FormProcessorSQL.save_processed_models(ProcessedForms(new_old_xform, []), [case])

        case = CaseAccessorSQL.get_case(case.case_id)
        update_strategy = SqlCaseUpdateStrategy(case)
        self.assertFalse(update_strategy.reconcile_transactions_if_necessary())

    def _create_form(self, user_id=None, received_on=None):
        """
        Create the models directly so that these tests aren't dependent on any
        other apps.
        :return: XFormInstanceSQL
        """
        user_id = user_id or 'mr_wednesday'
        received_on = received_on or datetime.utcnow()

        metadata = TestFormMetadata(
            domain=self.DOMAIN,
            received_on=received_on,
            user_id=user_id
        )
        form = get_form_ready_to_save(metadata)

        return form

    def _create_case_transaction(self, case, form=None, submitted_on=None, action_types=None):
        form = form or self._create_form()
        submitted_on = submitted_on or datetime.utcnow()

        return CaseTransaction.form_transaction(case, form, submitted_on, action_types)

    def _create_case(self, case_type=None, user_id=None, case_id=None):
        case_id = case_id or uuid.uuid4().hex
        user_id = user_id or 'mr_wednesday'
        utcnow = datetime.utcnow()

        case = CommCareCaseSQL(
            case_id=case_id,
            domain=self.DOMAIN,
            type=case_type or '',
            owner_id=user_id,
            opened_on=utcnow,
            modified_on=utcnow,
            modified_by=utcnow,
            server_modified_on=utcnow
        )

        form = self._create_form(user_id, utcnow)
        case.track_create(self._create_case_transaction(case, form, utcnow, action_types=[128]))
        FormProcessorSQL.save_processed_models(ProcessedForms(form, []), [case])

        return CaseAccessorSQL.get_case(case_id)

    def _check_for_reconciliation_error_soft_assert(self, soft_assert_mock):
        for call in soft_assert_mock.call_args_list:
            self.assertNotIn('ReconciliationError', call[0][1])
        soft_assert_mock.reset_mock()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# HTML source analyzer
# bl4de | bloorq@gmail.com | Twitter: @_bl4de
#
import sys

# TODO: Python doc in functions and for module
# TODO: args parser for: -c (comments) -s (links/src), -j (JavaScript)

class ConsoleOutputBeautifier:
    """This class defines properties and methods to manipulate console output"""
    colors = {
        "black": '\33[30m',
        "white": '\33[37m',
        "red": '\33[31m',
        "green": '\33[32m',
        "yellow": '\33[33m',
        "blue": '\33[34m',
        "magenta": '\33[35m',
        "cyan": '\33[36m'
    }

    characters = {
        "endline": '\33[0m'
    }

    def __init__(self):
        return None

    def getColor(self, color_name):
        """returns color identified by color_name or white as default value"""
        if color_name in ConsoleOutputBeautifier.colors:
            return ConsoleOutputBeautifier.colors[color_name]
        return ConsoleOutputBeautifier.colors["white"]

    def getSpecialChar(self, char_name):
        """returns special character identified by char_name"""
        if char_name in ConsoleOutputBeautifier.characters:
            return ConsoleOutputBeautifier.characters[char_name]
        return ""

co = ConsoleOutputBeautifier()

def main():
    _ident = ""
    _fw = ""

    try:
        _file = open(sys.argv[1], "r")
    except:
        pass

    i = 0
    print_header()

    for _line in _file:
        i += 1
        analyze_line(_line, i)
        if _ident == "":
            _ident = identify(_line)
        if _fw == "":
            _fw = detect_framework(_line)

    show_stats(_file, i, _ident, _fw)

# header
def print_header():
    print co.getColor("green"), "=" * 26, "HTML source code Analyzer", "=" * 26
    print " " + "-" * 10, "   https://github.com/bl4de | https://twitter.com/_bl4de | bloorq@gmail.com   ", \
        "-" * 10, "\n\n", co.getSpecialChar("endline")

# detects frontend framework used
def detect_framework(_line):
    _fw = ""
    if "ng-app" in _line:
        _fw = "Angular 1.*"
    return _fw

# using osftare recognition
def identify(_line):
    _ident = "unknown"
    if "Jommla" in _line:
        _ident = "Joomla CMS"
    if "wp-content" in _line:
        _ident = "WordPress CMS"
    return _ident

def show_stats(_file, i, _ident, _fw):
    print co.getColor("green"), "\n------ SUMMARY -------\n"
    print "total lines of code:     %d" % (i)
    print "identified CMS:          %s" % (_ident)
    print "identified framework:    %s" % (
        _fw), co.getSpecialChar("endline")
    # end of summary
    print "\n"

def print_output_line(i, col, msg, args):
    print co.getColor("white"), "line %d:" % (
        i), col, msg % (args), co.getSpecialChar("endline")

# find interesting string(s)
def analyze_line(_line, i):
    if _line.lstrip().startswith('<!--'):
        if "\"/" in _line:
            print_output_line(i, co.getColor("red"),
                              "COMMENTED PATH found at line %d:   %s",
                              (i, _line.lstrip().rstrip()))
        else:
            print_output_line(i, co.getColor("yellow"),
                              "COMMENT found at line %d:   %s",
                              (i, _line.lstrip().rstrip()))
    if "admin" in _line:
        print_output_line(i, co.getColor("red"),
                          "'admin' string found at line: %d", i)
    if "debug" in _line:
        print_output_line(i, co.getColor("red"),
                          "debug information found at line %d", i)
    if "src=" in _line:
        print_output_line(i, co.getColor("cyan"),
                          "PATH to external resource file (IMG, CSS, JS)"
                          " file found in %d:   %s",
                          (i, _line.lstrip().rstrip()[0:80]))
    if "<script>" in _line:
        print_output_line(i, co.getColor("green"),
                          "<SCRIPT> tag found at line %d", i)
    if "javascript:" in _line:
        print_output_line(i, co.getColor("cyan"),
                          "INLINE JavaScript found at line %d", i)

# main program
if __name__ == "__main__":
    if len(sys.argv) == 2:
        main()
    else:
        print "Enter HTML file name"
<commit_msg>detection rules improvements<commit_after>#!/usr/bin/env python
#
# HTML source analyzer
# bl4de | bloorq@gmail.com | Twitter: @_bl4de
#
import sys

# TODO: Python doc in functions and for module
# TODO: args parser for: -c (comments) -s (links/src), -j (JavaScript)

class ConsoleOutputBeautifier:
    """This class defines properties and methods to manipulate console output"""
    colors = {
        "black": '\33[30m',
        "white": '\33[37m',
        "red": '\33[31m',
        "green": '\33[32m',
        "yellow": '\33[33m',
        "blue": '\33[34m',
        "magenta": '\33[35m',
        "cyan": '\33[36m'
    }

    characters = {
        "endline": '\33[0m'
    }

    def __init__(self):
        return None

    @staticmethod
    def getColor(color_name):
        """returns color identified by color_name or white as default value"""
        if color_name in ConsoleOutputBeautifier.colors:
            return ConsoleOutputBeautifier.colors[color_name]
        return ConsoleOutputBeautifier.colors["white"]

    @staticmethod
    def getSpecialChar(char_name):
        """returns special character identified by char_name"""
        if char_name in ConsoleOutputBeautifier.characters:
            return ConsoleOutputBeautifier.characters[char_name]
        return ""

def main():
    _ident = ""
    _fw = ""

    try:
        _file = open(sys.argv[1], "r")
    except:
        pass

    i = 0
    print_header()

    for _line in _file:
        i += 1
        analyze_line(_line, i)
        if _ident == "":
            _ident = identify(_line)
        if _fw == "":
            _fw = detect_framework(_line)

    show_stats(_file, i, _ident, _fw)

# header
def print_header():
    print ConsoleOutputBeautifier.getColor(
        "green"), "=" * 26, "HTML source code Analyzer", "=" * 26
    print " " + "-" * 10, "   https://github.com/bl4de | https://twitter.com/_bl4de | bloorq@gmail.com   ", \
        "-" * 10, "\n\n", ConsoleOutputBeautifier.getSpecialChar("endline")

# detects frontend framework used
def detect_framework(_line):
    _fw = ""
    if "ng-app" in _line:
        _fw = "Angular 1.*"
    return _fw

# using osftare recognition
def identify(_line):
    _ident = "unknown"
    if "Jommla" in _line:
        _ident = "Joomla CMS"
    if "wp-content" in _line:
        _ident = "WordPress CMS"
    return _ident

def show_stats(_file, i, _ident, _fw):
    print ConsoleOutputBeautifier.getColor(
        "green"), "\n------ SUMMARY -------\n"
    print "total lines of code:     %d" % (i)
    print "identified CMS:          %s" % (_ident)
    print "identified framework:    %s" % (
        _fw), ConsoleOutputBeautifier.getSpecialChar("endline")
    # end of summary
    print "\n"

def print_output_line(i, col, msg, args):
    print ConsoleOutputBeautifier.getColor("white"), "line %d:" % (
        i), col, msg % (args), ConsoleOutputBeautifier.getSpecialChar("endline")

# find interesting string(s)
def analyze_line(_line, i):
    if '<!--' in _line.lstrip():
        if "\"/" in _line:
            print_output_line(i, ConsoleOutputBeautifier.getColor("red"),
                              "COMMENTED PATH found at line %d:   %s",
                              (i, _line.lstrip().rstrip()))
        else:
            print_output_line(i, ConsoleOutputBeautifier.getColor("yellow"),
                              "COMMENT found at line %d:   %s",
                              (i, _line.lstrip().rstrip()))
    if "admin" in _line.lower():
        print_output_line(i, ConsoleOutputBeautifier.getColor("red"),
                          "'admin' string found at line: %d", i)
    if "debug" in _line.lower():
        print_output_line(i, ConsoleOutputBeautifier.getColor("red"),
                          "DEBUG information found at line %d", i)
    if "src=" in _line.lower():
        print_output_line(i, ConsoleOutputBeautifier.getColor("cyan"),
                          "PATH to external resource file (IMG, CSS, JS)"
                          " file found in %d:   %s",
                          (i, _line.lstrip().rstrip()[0:80]))
    if "<script>" in _line.lower():
        print_output_line(i, ConsoleOutputBeautifier.getColor("green"),
                          "<SCRIPT> tag found at line %d", i)
    if "javascript:" in _line.lower():
        print_output_line(i, ConsoleOutputBeautifier.getColor("cyan"),
                          "INLINE JavaScript event handler found at line %d", i)

# main program
if __name__ == "__main__":
    if len(sys.argv) == 2:
        main()
    else:
        print "Enter HTML file name"
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    pygments.lexers.modeling
    ~~~~~~~~~~~~~~~~~~~~~~~~

    Lexers for modeling languages.

    :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import re

from pygments.lexer import RegexLexer, include, bygroups, using, default
from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
    Number, Punctuation, Whitespace

from pygments.lexers.html import HtmlLexer
from pygments.lexers import _stan_builtins

__all__ = ['ModelicaLexer', 'BugsLexer', 'JagsLexer', 'StanLexer']

class ModelicaLexer(RegexLexer):
    """
    For `Modelica <http://www.modelica.org/>`_ source code.

    .. versionadded:: 1.1
    """
    name = 'Modelica'
    aliases = ['modelica']
    filenames = ['*.mo']
    mimetypes = ['text/x-modelica']

    flags = re.DOTALL | re.MULTILINE

    _name = r"(?:'(?:[^\\']|\\.)+'|[a-zA-Z_]\w*)"

    tokens = {
        'whitespace': [
            (u'[\\s\ufeff]+', Text),
            (r'//[^\n]*\n?', Comment.Single),
            (r'/\*.*?\*/', Comment.Multiline)
        ],
        'root': [
            include('whitespace'),
            (r'"', String.Double, 'string'),
            (r'[()\[\]{},;]+', Punctuation),
            (r'\.?[*^/+-]|\.|<>|[<>:=]=?', Operator),
            (r'\d+(\.?\d*[eE][-+]?\d+|\.\d*)', Number.Float),
            (r'\d+', Number.Integer),
            (r'(abs|acos|actualStream|array|asin|assert|AssertionLevel|atan|'
             r'atan2|backSample|Boolean|cardinality|cat|ceil|change|Clock|'
             r'Connections|cos|cosh|cross|delay|diagonal|div|edge|exp|'
             r'ExternalObject|fill|floor|getInstanceName|hold|homotopy|'
             r'identity|inStream|integer|Integer|interval|inverse|isPresent|'
             r'linspace|log|log10|matrix|max|min|mod|ndims|noClock|noEvent|'
             r'ones|outerProduct|pre|previous|product|Real|reinit|rem|rooted|'
             r'sample|scalar|semiLinear|shiftSample|sign|sin|sinh|size|skew|'
             r'smooth|spatialDistribution|sqrt|StateSelect|String|subSample|'
             r'sum|superSample|symmetric|tan|tanh|terminal|terminate|time|'
             r'transpose|vector|zeros)\b', Name.Builtin),
            (r'(algorithm|annotation|break|connect|constant|constrainedby|der|'
             r'discrete|each|else|elseif|elsewhen|encapsulated|enumeration|'
             r'equation|exit|expandable|extends|external|final|flow|for|if|'
             r'import|impure|in|initial|inner|input|loop|nondiscrete|outer|'
             r'output|parameter|partial|protected|public|pure|redeclare|'
             r'replaceable|return|stream|then|when|while)\b',
             Keyword.Reserved),
            (r'(and|not|or)\b', Operator.Word),
            (r'(block|class|connector|end|function|model|operator|package|'
             r'record|type)\b', Keyword.Reserved, 'class'),
            (r'(false|true)\b', Keyword.Constant),
            (r'within\b', Keyword.Reserved, 'package-prefix'),
            (_name, Name)
        ],
        'class': [
            include('whitespace'),
            (r'(function|record)\b', Keyword.Reserved),
            (r'(if|for|when|while)\b', Keyword.Reserved, '#pop'),
            (_name, Name.Class, '#pop'),
            default('#pop')
        ],
        'package-prefix': [
            include('whitespace'),
            (_name, Name.Namespace, '#pop'),
            default('#pop')
        ],
        'string': [
            (r'"', String.Double, '#pop'),
            (r'\\[\'"?\\abfnrtv]', String.Escape),
            (r'(?i)<\s*html\s*>([^\\"]|\\.)+?(<\s*/\s*html\s*>|(?="))',
             using(HtmlLexer)),
            (r'<|\\?[^"\\<]+', String.Double)
        ]
    }

class BugsLexer(RegexLexer):
    """
    Pygments Lexer for `OpenBugs <http://www.openbugs.net/>`_ and WinBugs
    models.

    .. versionadded:: 1.6
    """

    name = 'BUGS'
    aliases = ['bugs', 'winbugs', 'openbugs']
    filenames = ['*.bug']

    _FUNCTIONS = (
        # Scalar functions
        'abs', 'arccos', 'arccosh', 'arcsin', 'arcsinh', 'arctan', 'arctanh',
        'cloglog', 'cos', 'cosh', 'cumulative', 'cut', 'density', 'deviance',
        'equals', 'expr', 'gammap', 'ilogit', 'icloglog', 'integral', 'log',
        'logfact', 'loggam', 'logit', 'max', 'min', 'phi', 'post.p.value',
        'pow', 'prior.p.value', 'probit', 'replicate.post', 'replicate.prior',
        'round', 'sin', 'sinh', 'solution', 'sqrt', 'step', 'tan', 'tanh',
        'trunc',
        # Vector functions
        'inprod', 'interp.lin', 'inverse', 'logdet', 'mean', 'eigen.vals',
        'ode', 'prod', 'p.valueM', 'rank', 'ranked', 'replicate.postM',
        'sd', 'sort', 'sum',
        # Special
        'D', 'I', 'F', 'T', 'C')
    """ OpenBUGS built-in functions

    From http://www.openbugs.info/Manuals/ModelSpecification.html#ContentsAII

    This also includes

    - T, C, I : Truncation and censoring.
      ``T`` and ``C`` are in OpenBUGS. ``I`` in WinBUGS.
    - D : ODE
    - F : Functional http://www.openbugs.info/Examples/Functionals.html

    """

    _DISTRIBUTIONS = ('dbern', 'dbin', 'dcat', 'dnegbin', 'dpois',
                      'dhyper', 'dbeta', 'dchisqr', 'ddexp', 'dexp',
                      'dflat', 'dgamma', 'dgev', 'df', 'dggamma', 'dgpar',
                      'dloglik', 'dlnorm', 'dlogis', 'dnorm', 'dpar',
                      'dt', 'dunif', 'dweib', 'dmulti', 'ddirch', 'dmnorm',
                      'dmt', 'dwish')
    """ OpenBUGS built-in distributions

    Functions from
    http://www.openbugs.info/Manuals/ModelSpecification.html#ContentsAI
    """

    tokens = {
        'whitespace': [
            (r"\s+", Text),
        ],
        'comments': [
            # Comments
            (r'#.*$', Comment.Single),
        ],
        'root': [
            # Comments
            include('comments'),
            include('whitespace'),
            # Block start
            (r'(model)(\s+)(\{)',
             bygroups(Keyword.Namespace, Text, Punctuation)),
            # Reserved Words
            (r'(for|in)(?![\w.])', Keyword.Reserved),
            # Built-in Functions
            (r'(%s)(?=\s*\()'
             % r'|'.join(_FUNCTIONS + _DISTRIBUTIONS),
             Name.Builtin),
            # Regular variable names
            (r'[A-Za-z][\w.]*', Name),
            # Number Literals
            (r'[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?', Number),
            # Punctuation
            (r'\[|\]|\(|\)|:|,|;', Punctuation),
            # Assignment operators
            # SLexer makes these tokens Operators.
            (r'<-|~', Operator),
            # Infix and prefix operators
            (r'\+|-|\*|/', Operator),
            # Block
            (r'[{}]', Punctuation),
        ]
    }

    def analyse_text(text):
        if re.search(r"^\s*model\s*{", text, re.M):
            return 0.7
        else:
            return 0.0

class JagsLexer(RegexLexer):
    """
    Pygments Lexer for JAGS.

    .. versionadded:: 1.6
    """

    name = 'JAGS'
    aliases = ['jags']
    filenames = ['*.jag', '*.bug']

    # JAGS
    _FUNCTIONS = (
        'abs', 'arccos', 'arccosh', 'arcsin', 'arcsinh', 'arctan', 'arctanh',
        'cos', 'cosh', 'cloglog',
        'equals', 'exp', 'icloglog', 'ifelse', 'ilogit', 'log', 'logfact',
        'loggam', 'logit', 'phi', 'pow', 'probit', 'round', 'sin', 'sinh',
        'sqrt', 'step', 'tan', 'tanh', 'trunc', 'inprod', 'interp.lin',
        'logdet', 'max', 'mean', 'min', 'prod', 'sum', 'sd', 'inverse',
        'rank', 'sort', 't', 'acos', 'acosh', 'asin', 'asinh', 'atan',
        # Truncation/Censoring (should I include)
        'T', 'I')
    # Distributions with density, probability and quartile functions
    _DISTRIBUTIONS = tuple('[dpq]%s' % x for x in
                           ('bern', 'beta', 'dchiqsqr', 'ddexp', 'dexp',
                            'df', 'gamma', 'gen.gamma', 'logis', 'lnorm',
                            'negbin', 'nchisqr', 'norm', 'par', 'pois', 'weib'))
    # Other distributions without density and probability
    _OTHER_DISTRIBUTIONS = (
        'dt', 'dunif', 'dbetabin', 'dbern', 'dbin', 'dcat', 'dhyper',
        'ddirch', 'dmnorm', 'dwish', 'dmt', 'dmulti', 'dbinom', 'dchisq',
        'dnbinom', 'dweibull', 'ddirich')

    tokens = {
        'whitespace': [
            (r"\s+", Text),
        ],
        'names': [
            # Regular variable names
            (r'[a-zA-Z][\w.]*\b', Name),
        ],
        'comments': [
            # do not use stateful comments
            (r'(?s)/\*.*?\*/', Comment.Multiline),
            # Comments
            (r'#.*$', Comment.Single),
        ],
        'root': [
            # Comments
            include('comments'),
            include('whitespace'),
            # Block start
            (r'(model|data)(\s+)(\{)',
             bygroups(Keyword.Namespace, Text, Punctuation)),
            (r'var(?![\w.])', Keyword.Declaration),
            # Reserved Words
            (r'(for|in)(?![\w.])', Keyword.Reserved),
            # Builtins
            # Need to use lookahead because . is a valid char
            (r'(%s)(?=\s*\()' % r'|'.join(_FUNCTIONS
                                          + _DISTRIBUTIONS
                                          + _OTHER_DISTRIBUTIONS),
             Name.Builtin),
            # Names
            include('names'),
            # Number Literals
            (r'[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?', Number),
            (r'\[|\]|\(|\)|:|,|;', Punctuation),
            # Assignment operators
            (r'<-|~', Operator),
            # # JAGS includes many more than OpenBUGS
            (r'\+|-|\*|\/|\|\|[&]{2}|[<>=]=?|\^|%.*?%', Operator),
            (r'[{}]', Punctuation),
        ]
    }

    def analyse_text(text):
        if re.search(r'^\s*model\s*\{', text, re.M):
            if re.search(r'^\s*data\s*\{', text, re.M):
                return 0.9
            elif re.search(r'^\s*var', text, re.M):
                return 0.9
            else:
                return 0.3
        else:
            return 0

class StanLexer(RegexLexer):
    """Pygments Lexer for Stan models.

    The Stan modeling language is specified in the *Stan Modeling Language
    User's Guide and Reference Manual, v2.17.0*,
    `pdf <https://github.com/stan-dev/stan/releases/download/v2.17.0/stan-reference-2.17.0.pdf>`__.

    .. versionadded:: 1.6
    """

    name = 'Stan'
    aliases = ['stan']
    filenames = ['*.stan']

    tokens = {
        'whitespace': [
            (r"\s+", Text),
        ],
        'comments': [
            (r'(?s)/\*.*?\*/', Comment.Multiline),
            # Comments
            (r'(//|#).*$', Comment.Single),
        ],
        'root': [
            # Stan is more restrictive on strings than this regex
            (r'"[^"]*"', String),
            # Comments
            include('comments'),
            # block start
            include('whitespace'),
            # Block start
            (r'(%s)(\s*)(\{)' %
             r'|'.join(('functions', 'data', r'transformed\s+?data',
                        'parameters', r'transformed\s+parameters',
                        'model', r'generated\s+quantities')),
             bygroups(Keyword.Namespace, Text, Punctuation)),
            # target keyword
            (r'target\s*\+=', Keyword),
            # Reserved Words
            (r'(%s)\b' % r'|'.join(_stan_builtins.KEYWORDS), Keyword),
            # Truncation
            (r'T(?=\s*\[)', Keyword),
            # Data types
            (r'(%s)\b' % r'|'.join(_stan_builtins.TYPES), Keyword.Type),
             # < should be punctuation, but elsewhere I can't tell if it is in
             # a range constraint
            (r'(<)(\s*)(upper|lower)(\s*)(=)',
             bygroups(Operator, Whitespace, Keyword, Whitespace, Punctuation)),
            (r'(,)(\s*)(upper)(\s*)(=)',
             bygroups(Punctuation, Whitespace, Keyword, Whitespace, Punctuation)),
            # Punctuation
            (r"[;,\[\]()]", Punctuation),
            # Builtin
            (r'(%s)(?=\s*\()' % '|'.join(_stan_builtins.FUNCTIONS), Name.Builtin),
            (r'(~)(\s*)(%s)(?=\s*\()' % '|'.join(_stan_builtins.DISTRIBUTIONS),
                bygroups(Operator, Whitespace, Name.Builtin)),
            # Special names ending in __, like lp__
            (r'[A-Za-z]\w*__\b', Name.Builtin.Pseudo),
            (r'(%s)\b' % r'|'.join(_stan_builtins.RESERVED), Keyword.Reserved),
            # user-defined functions
            (r'[A-Za-z]\w*(?=\s*\()]', Name.Function),
            # Regular variable names
            (r'[A-Za-z]\w*\b', Name),
            # Real Literals
            (r'[0-9]+(\.[0-9]*)?([eE][+-]?[0-9]+)?', Number.Float),
            (r'\.[0-9]+([eE][+-]?[0-9]+)?', Number.Float),
            # Integer Literals
            (r'[0-9]+', Number.Integer),
            # Assignment operators
            (r'<-|(?:\+|-|\.?/|\.?\*|=)?=|~', Operator),
            # Infix, prefix and postfix operators (and = )
            (r"\+|-|\.?\*|\.?/|\\|'|\^|!=?|<=?|>=?|\|\||&&|%|\?|:", Operator),
            # Block delimiters
            (r'[{}]', Punctuation),
            # Distribution |
            (r'\|', Punctuation)
        ]
    }

    def analyse_text(text):
        if re.search(r'^\s*parameters\s*\{', text, re.M):
            return 1.0
        else:
            return 0.0
<commit_msg>update ModelicaLexer added new Operators `interval` and `firstTick` from [ModelicaSpec34](https://www.modelica.org/documents/ModelicaSpec34.pdf), 16.10 Other Operators<commit_after># -*- coding: utf-8 -*-
"""
    pygments.lexers.modeling
    ~~~~~~~~~~~~~~~~~~~~~~~~

    Lexers for modeling languages.

    :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import re

from pygments.lexer import RegexLexer, include, bygroups, using, default
from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
    Number, Punctuation, Whitespace

from pygments.lexers.html import HtmlLexer
from pygments.lexers import _stan_builtins

__all__ = ['ModelicaLexer', 'BugsLexer', 'JagsLexer', 'StanLexer']

class ModelicaLexer(RegexLexer):
    """
    For `Modelica <http://www.modelica.org/>`_ source code.

    .. versionadded:: 1.1
    """
    name = 'Modelica'
    aliases = ['modelica']
    filenames = ['*.mo']
    mimetypes = ['text/x-modelica']

    flags = re.DOTALL | re.MULTILINE

    _name = r"(?:'(?:[^\\']|\\.)+'|[a-zA-Z_]\w*)"

    tokens = {
        'whitespace': [
            (u'[\\s\ufeff]+', Text),
            (r'//[^\n]*\n?', Comment.Single),
            (r'/\*.*?\*/', Comment.Multiline)
        ],
        'root': [
            include('whitespace'),
            (r'"', String.Double, 'string'),
            (r'[()\[\]{},;]+', Punctuation),
            (r'\.?[*^/+-]|\.|<>|[<>:=]=?', Operator),
            (r'\d+(\.?\d*[eE][-+]?\d+|\.\d*)', Number.Float),
            (r'\d+', Number.Integer),
            (r'(abs|acos|actualStream|array|asin|assert|AssertionLevel|atan|'
             r'atan2|backSample|Boolean|cardinality|cat|ceil|change|Clock|'
             r'Connections|cos|cosh|cross|delay|diagonal|div|edge|exp|'
             r'ExternalObject|fill|floor|getInstanceName|hold|homotopy|'
             r'identity|inStream|integer|Integer|interval|inverse|isPresent|'
             r'linspace|log|log10|matrix|max|min|mod|ndims|noClock|noEvent|'
             r'ones|outerProduct|pre|previous|product|Real|reinit|rem|rooted|'
             r'sample|scalar|semiLinear|shiftSample|sign|sin|sinh|size|skew|'
             r'smooth|spatialDistribution|sqrt|StateSelect|String|subSample|'
             r'sum|superSample|symmetric|tan|tanh|terminal|terminate|time|'
             r'transpose|vector|zeros)\b', Name.Builtin),
            (r'(algorithm|annotation|break|connect|constant|constrainedby|der|'
             r'discrete|each|else|elseif|elsewhen|encapsulated|enumeration|'
             r'equation|exit|expandable|extends|external|firstTick|final|flow|for|if|'
             r'import|impure|in|initial|inner|input|interval|loop|nondiscrete|outer|'
             r'output|parameter|partial|protected|public|pure|redeclare|'
             r'replaceable|return|stream|then|when|while)\b',
             Keyword.Reserved),
            (r'(and|not|or)\b', Operator.Word),
            (r'(block|class|connector|end|function|model|operator|package|'
             r'record|type)\b', Keyword.Reserved, 'class'),
            (r'(false|true)\b', Keyword.Constant),
            (r'within\b', Keyword.Reserved, 'package-prefix'),
            (_name, Name)
        ],
        'class': [
            include('whitespace'),
            (r'(function|record)\b', Keyword.Reserved),
            (r'(if|for|when|while)\b', Keyword.Reserved, '#pop'),
            (_name, Name.Class, '#pop'),
            default('#pop')
        ],
        'package-prefix': [
            include('whitespace'),
            (_name, Name.Namespace, '#pop'),
            default('#pop')
        ],
        'string': [
            (r'"', String.Double, '#pop'),
            (r'\\[\'"?\\abfnrtv]', String.Escape),
            (r'(?i)<\s*html\s*>([^\\"]|\\.)+?(<\s*/\s*html\s*>|(?="))',
             using(HtmlLexer)),
            (r'<|\\?[^"\\<]+', String.Double)
        ]
    }

class BugsLexer(RegexLexer):
    """
    Pygments Lexer for `OpenBugs <http://www.openbugs.net/>`_ and WinBugs
    models.

    .. versionadded:: 1.6
    """

    name = 'BUGS'
    aliases = ['bugs', 'winbugs', 'openbugs']
    filenames = ['*.bug']

    _FUNCTIONS = (
        # Scalar functions
        'abs', 'arccos', 'arccosh', 'arcsin', 'arcsinh', 'arctan', 'arctanh',
        'cloglog', 'cos', 'cosh', 'cumulative', 'cut', 'density', 'deviance',
        'equals', 'expr', 'gammap', 'ilogit', 'icloglog', 'integral', 'log',
        'logfact', 'loggam', 'logit', 'max', 'min', 'phi', 'post.p.value',
        'pow', 'prior.p.value', 'probit', 'replicate.post', 'replicate.prior',
        'round', 'sin', 'sinh', 'solution', 'sqrt', 'step', 'tan', 'tanh',
        'trunc',
        # Vector functions
        'inprod', 'interp.lin', 'inverse', 'logdet', 'mean', 'eigen.vals',
        'ode', 'prod', 'p.valueM', 'rank', 'ranked', 'replicate.postM',
        'sd', 'sort', 'sum',
        # Special
        'D', 'I', 'F', 'T', 'C')
    """ OpenBUGS built-in functions

    From http://www.openbugs.info/Manuals/ModelSpecification.html#ContentsAII

    This also includes

    - T, C, I : Truncation and censoring.
      ``T`` and ``C`` are in OpenBUGS. ``I`` in WinBUGS.
    - D : ODE
    - F : Functional http://www.openbugs.info/Examples/Functionals.html

    """

    _DISTRIBUTIONS = ('dbern', 'dbin', 'dcat', 'dnegbin', 'dpois',
                      'dhyper', 'dbeta', 'dchisqr', 'ddexp', 'dexp',
                      'dflat', 'dgamma', 'dgev', 'df', 'dggamma', 'dgpar',
                      'dloglik', 'dlnorm', 'dlogis', 'dnorm', 'dpar',
                      'dt', 'dunif', 'dweib', 'dmulti', 'ddirch', 'dmnorm',
                      'dmt', 'dwish')
    """ OpenBUGS built-in distributions

    Functions from
    http://www.openbugs.info/Manuals/ModelSpecification.html#ContentsAI
    """

    tokens = {
        'whitespace': [
            (r"\s+", Text),
        ],
        'comments': [
            # Comments
            (r'#.*$', Comment.Single),
        ],
        'root': [
            # Comments
            include('comments'),
            include('whitespace'),
            # Block start
            (r'(model)(\s+)(\{)',
             bygroups(Keyword.Namespace, Text, Punctuation)),
            # Reserved Words
            (r'(for|in)(?![\w.])', Keyword.Reserved),
            # Built-in Functions
            (r'(%s)(?=\s*\()'
             % r'|'.join(_FUNCTIONS + _DISTRIBUTIONS),
             Name.Builtin),
            # Regular variable names
            (r'[A-Za-z][\w.]*', Name),
            # Number Literals
            (r'[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?', Number),
            # Punctuation
            (r'\[|\]|\(|\)|:|,|;', Punctuation),
            # Assignment operators
            # SLexer makes these tokens Operators.
            (r'<-|~', Operator),
            # Infix and prefix operators
            (r'\+|-|\*|/', Operator),
            # Block
            (r'[{}]', Punctuation),
        ]
    }

    def analyse_text(text):
        if re.search(r"^\s*model\s*{", text, re.M):
            return 0.7
        else:
            return 0.0

class JagsLexer(RegexLexer):
    """
    Pygments Lexer for JAGS.

    .. versionadded:: 1.6
    """

    name = 'JAGS'
    aliases = ['jags']
    filenames = ['*.jag', '*.bug']

    # JAGS
    _FUNCTIONS = (
        'abs', 'arccos', 'arccosh', 'arcsin', 'arcsinh', 'arctan', 'arctanh',
        'cos', 'cosh', 'cloglog',
        'equals', 'exp', 'icloglog', 'ifelse', 'ilogit', 'log', 'logfact',
        'loggam', 'logit', 'phi', 'pow', 'probit', 'round', 'sin', 'sinh',
        'sqrt', 'step', 'tan', 'tanh', 'trunc', 'inprod', 'interp.lin',
        'logdet', 'max', 'mean', 'min', 'prod', 'sum', 'sd', 'inverse',
        'rank', 'sort', 't', 'acos', 'acosh', 'asin', 'asinh', 'atan',
        # Truncation/Censoring (should I include)
        'T', 'I')
    # Distributions with density, probability and quartile functions
    _DISTRIBUTIONS = tuple('[dpq]%s' % x for x in
                           ('bern', 'beta', 'dchiqsqr', 'ddexp', 'dexp',
                            'df', 'gamma', 'gen.gamma', 'logis', 'lnorm',
                            'negbin', 'nchisqr', 'norm', 'par', 'pois', 'weib'))
    # Other distributions without density and probability
    _OTHER_DISTRIBUTIONS = (
        'dt', 'dunif', 'dbetabin', 'dbern', 'dbin', 'dcat', 'dhyper',
        'ddirch', 'dmnorm', 'dwish', 'dmt', 'dmulti', 'dbinom', 'dchisq',
        'dnbinom', 'dweibull', 'ddirich')

    tokens = {
        'whitespace': [
            (r"\s+", Text),
        ],
        'names': [
            # Regular variable names
            (r'[a-zA-Z][\w.]*\b', Name),
        ],
        'comments': [
            # do not use stateful comments
            (r'(?s)/\*.*?\*/', Comment.Multiline),
            # Comments
            (r'#.*$', Comment.Single),
        ],
        'root': [
            # Comments
            include('comments'),
            include('whitespace'),
            # Block start
            (r'(model|data)(\s+)(\{)',
             bygroups(Keyword.Namespace, Text, Punctuation)),
            (r'var(?![\w.])', Keyword.Declaration),
            # Reserved Words
            (r'(for|in)(?![\w.])', Keyword.Reserved),
            # Builtins
            # Need to use lookahead because . is a valid char
            (r'(%s)(?=\s*\()' % r'|'.join(_FUNCTIONS
                                          + _DISTRIBUTIONS
                                          + _OTHER_DISTRIBUTIONS),
             Name.Builtin),
            # Names
            include('names'),
            # Number Literals
            (r'[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?', Number),
            (r'\[|\]|\(|\)|:|,|;', Punctuation),
            # Assignment operators
            (r'<-|~', Operator),
            # # JAGS includes many more than OpenBUGS
            (r'\+|-|\*|\/|\|\|[&]{2}|[<>=]=?|\^|%.*?%', Operator),
            (r'[{}]', Punctuation),
        ]
    }

    def analyse_text(text):
        if re.search(r'^\s*model\s*\{', text, re.M):
            if re.search(r'^\s*data\s*\{', text, re.M):
                return 0.9
            elif re.search(r'^\s*var', text, re.M):
                return 0.9
            else:
                return 0.3
        else:
            return 0

class StanLexer(RegexLexer):
    """Pygments Lexer for Stan models.

    The Stan modeling language is specified in the *Stan Modeling Language
    User's Guide and Reference Manual, v2.17.0*,
    `pdf <https://github.com/stan-dev/stan/releases/download/v2.17.0/stan-reference-2.17.0.pdf>`__.

    .. versionadded:: 1.6
    """

    name = 'Stan'
    aliases = ['stan']
    filenames = ['*.stan']

    tokens = {
        'whitespace': [
            (r"\s+", Text),
        ],
        'comments': [
            (r'(?s)/\*.*?\*/', Comment.Multiline),
            # Comments
            (r'(//|#).*$', Comment.Single),
        ],
        'root': [
            # Stan is more restrictive on strings than this regex
            (r'"[^"]*"', String),
            # Comments
            include('comments'),
            # block start
            include('whitespace'),
            # Block start
            (r'(%s)(\s*)(\{)' %
             r'|'.join(('functions', 'data', r'transformed\s+?data',
                        'parameters', r'transformed\s+parameters',
                        'model', r'generated\s+quantities')),
             bygroups(Keyword.Namespace, Text, Punctuation)),
            # target keyword
            (r'target\s*\+=', Keyword),
            # Reserved Words
            (r'(%s)\b' % r'|'.join(_stan_builtins.KEYWORDS), Keyword),
            # Truncation
            (r'T(?=\s*\[)', Keyword),
            # Data types
            (r'(%s)\b' % r'|'.join(_stan_builtins.TYPES), Keyword.Type),
             # < should be punctuation, but elsewhere I can't tell if it is in
             # a range constraint
            (r'(<)(\s*)(upper|lower)(\s*)(=)',
             bygroups(Operator, Whitespace, Keyword, Whitespace, Punctuation)),
            (r'(,)(\s*)(upper)(\s*)(=)',
             bygroups(Punctuation, Whitespace, Keyword, Whitespace, Punctuation)),
            # Punctuation
            (r"[;,\[\]()]", Punctuation),
            # Builtin
            (r'(%s)(?=\s*\()' % '|'.join(_stan_builtins.FUNCTIONS), Name.Builtin),
            (r'(~)(\s*)(%s)(?=\s*\()' % '|'.join(_stan_builtins.DISTRIBUTIONS),
                bygroups(Operator, Whitespace, Name.Builtin)),
            # Special names ending in __, like lp__
            (r'[A-Za-z]\w*__\b', Name.Builtin.Pseudo),
            (r'(%s)\b' % r'|'.join(_stan_builtins.RESERVED), Keyword.Reserved),
            # user-defined functions
            (r'[A-Za-z]\w*(?=\s*\()]', Name.Function),
            # Regular variable names
            (r'[A-Za-z]\w*\b', Name),
            # Real Literals
            (r'[0-9]+(\.[0-9]*)?([eE][+-]?[0-9]+)?', Number.Float),
            (r'\.[0-9]+([eE][+-]?[0-9]+)?', Number.Float),
            # Integer Literals
            (r'[0-9]+', Number.Integer),
            # Assignment operators
            (r'<-|(?:\+|-|\.?/|\.?\*|=)?=|~', Operator),
            # Infix, prefix and postfix operators (and = )
            (r"\+|-|\.?\*|\.?/|\\|'|\^|!=?|<=?|>=?|\|\||&&|%|\?|:", Operator),
            # Block delimiters
            (r'[{}]', Punctuation),
            # Distribution |
            (r'\|', Punctuation)
        ]
    }

    def analyse_text(text):
        if re.search(r'^\s*parameters\s*\{', text, re.M):
            return 1.0
        else:
            return 0.0
<|endoftext|>


<commit_before>import unittest

import numpy as np
import pytest

import coremltools
from coremltools._deps import HAS_KERAS_TF
from coremltools._deps import HAS_TF
from coremltools.models.utils import get_custom_layer_names, \
    replace_custom_layer_name, macos_version, is_macos
from coremltools.proto import Model_pb2

if HAS_KERAS_TF:
    from keras.models import Sequential
    from keras.layers import Dense, LSTM
    from coremltools.converters import keras as keras_converter

if HAS_TF :
    import tensorflow as tf
    from tensorflow.python.platform import gfile
    from tensorflow.python.tools import freeze_graph
    import tfcoreml as tf_converter
    tf.disable_eager_execution()

@unittest.skipIf(not HAS_KERAS_TF, 'Missing keras. Skipping tests.')
@pytest.mark.keras1
class KerasBasicNumericCorrectnessTest(unittest.TestCase):

    def test_classifier(self):
        np.random.seed(1988)

        print('running test classifier')

        input_dim = 5
        num_hidden = 12
        num_classes = 6
        input_length = 3

        model = Sequential()
        model.add(LSTM(num_hidden, input_dim=input_dim, input_length=input_length, return_sequences=False))
        model.add(Dense(num_classes, activation='softmax'))

        model.set_weights([np.random.rand(*w.shape) for w in model.get_weights()])

        input_names = ['input']
        output_names = ['zzzz']
        class_labels = ['a', 'b', 'c', 'd', 'e', 'f']
        predicted_feature_name = 'pf'
        coremlmodel = keras_converter.convert(model, input_names, output_names, class_labels=class_labels, predicted_feature_name=predicted_feature_name, predicted_probabilities_output=output_names[0])

        if is_macos() and macos_version() >= (10, 13):
            inputs = np.random.rand(input_dim)
            outputs = coremlmodel.predict({'input': inputs})
            # this checks that the dictionary got the right name and type
            self.assertEquals(type(outputs[output_names[0]]), type({'a': 0.5}))

    def test_classifier_no_name(self):
        np.random.seed(1988)

        input_dim = 5
        num_hidden = 12
        num_classes = 6
        input_length = 3

        model = Sequential()
        model.add(LSTM(num_hidden, input_dim=input_dim, input_length=input_length, return_sequences=False))
        model.add(Dense(num_classes, activation='softmax'))

        model.set_weights([np.random.rand(*w.shape) for w in model.get_weights()])

        input_names = ['input']
        output_names = ['zzzz']
        class_labels = ['a', 'b', 'c', 'd', 'e', 'f']
        predicted_feature_name = 'pf'
        coremlmodel = keras_converter.convert(model, input_names, output_names, class_labels=class_labels, predicted_feature_name=predicted_feature_name)

        if is_macos() and macos_version() >= (10, 13):
            inputs = np.random.rand(input_dim)
            outputs = coremlmodel.predict({'input': inputs})
            # this checks that the dictionary got the right name and type
            self.assertEquals(type(outputs[output_names[0]]), type({'a': 0.5}))

    def test_internal_layer(self):

        np.random.seed(1988)

        input_dim = 5
        num_channels1 = 10
        num_channels2 = 7
        num_channels3 = 5

        w1 = (np.random.rand(input_dim, num_channels1)-0.5)/5.0;
        w2 = (np.random.rand(num_channels1, num_channels2)-0.5)/5.0;
        w3 = (np.random.rand(num_channels2, num_channels3)-0.5)/5.0;

        b1 = (np.random.rand(num_channels1,)-0.5)/5.0;
        b2 = (np.random.rand(num_channels2,)-0.5)/5.0;
        b3 = (np.random.rand(num_channels3,)-0.5)/5.0;

        model = Sequential()
        model.add(Dense(num_channels1, input_dim = input_dim))
        model.add(Dense(num_channels2, name='middle_layer'))
        model.add(Dense(num_channels3))

        model.set_weights([w1, b1, w2, b2, w3, b3])

        input_names = ['input']
        output_names = ['output']
        coreml1 = keras_converter.convert(model, input_names, output_names)

        # adjust the output parameters of coreml1 to include the intermediate layer
        spec = coreml1.get_spec()
        coremlNewOutputs = spec.description.output.add()
        coremlNewOutputs.name = 'middle_layer_output'
        coremlNewParams = coremlNewOutputs.type.multiArrayType
        coremlNewParams.dataType = coremltools.proto.FeatureTypes_pb2.ArrayFeatureType.ArrayDataType.Value('DOUBLE')
        coremlNewParams.shape.extend([num_channels2])

        coremlfinal = coremltools.models.MLModel(spec)

        # generate a second model which
        model2 = Sequential()
        model2.add(Dense(num_channels1, input_dim = input_dim))
        model2.add(Dense(num_channels2))
        model2.set_weights([w1, b1, w2, b2])

        coreml2 = keras_converter.convert(model2, input_names, ['output2'])

        if is_macos() and macos_version() >= (10, 13):
            # generate input data
            inputs = np.random.rand(input_dim)

            fullOutputs = coremlfinal.predict({'input': inputs})

            partialOutput = coreml2.predict({'input': inputs})

            for i in range(0, num_channels2):
                self.assertAlmostEquals(fullOutputs['middle_layer_output'][i], partialOutput['output2'][i], 2)

@unittest.skipIf(not HAS_TF, 'Missing TF. Skipping tests.')
class TFBasicConversionTest(unittest.TestCase):

    frozen_graph_file = '/tmp/my_frozen_classifier_graph.pb'
    converted_coreml_file = '/tmp/my_frozen_classifier_graph.mlmodel'

    def my_conv_2d(self, input, weight_shape, num_filters, strides, name, activation_fn=tf.nn.relu, with_bias_add=True):
        my_weights = tf.get_variable(name=name+"weights", shape=weight_shape)
        if with_bias_add:
            my_bias = tf.get_variable(name=name+"bias", shape=num_filters)
        my_conv = tf.nn.conv2d(input, my_weights, strides=strides, padding='SAME', name=name)
        if with_bias_add:
            my_conv = tf.nn.bias_add(my_conv, my_bias)
        if (activation_fn != None):
            conv_layer_out = activation_fn(my_conv)
        else:
            conv_layer_out = my_conv
        return conv_layer_out

    def test_1(self):
        with open('/tmp/labels.txt', 'w+') as labels_file:
            for a in range(10):
                labels_file.write(str(a + 1)+"\n")

        image_size = 224

        with tf.Graph().as_default():
            batch_size, height, width, channels = 1, image_size, image_size, 3
            images = tf.random.uniform([batch_size, height, width, channels], maxval=1)

            # Create the model.
            i_placeholder = tf.placeholder(name='input', dtype=tf.float32, shape=[1, image_size, image_size, 3])
            net = self.my_conv_2d(i_placeholder, [1, 3, 3, 1], 1, 1, 'first')
            net = tf.nn.avg_pool2d(net, 224, strides=1, padding='VALID', name='AvgPool_1a')
            net = self.my_conv_2d(net,[1, 1, 1, 10], 10, 1, 'fc', activation_fn=None, with_bias_add=False)
            net = tf.squeeze(net, [1, 2], name='SpatialSqueeze')
            probabilities = tf.nn.softmax(net, name='Softmax')

            saver = tf.train.Saver()
            init_op = tf.global_variables_initializer()

            checkpoint_file='/tmp/my_classifier_model.ckpt'
            graph_file = '/tmp/my_classifier_graph.pb'

            with tf.Session() as sess:
                sess.run(init_op)
                probabilities = sess.run(probabilities, {i_placeholder: images.eval()})
                save_path = saver.save(sess, checkpoint_file)

                with gfile.GFile(graph_file, 'wb') as f:
                    f.write(sess.graph_def.SerializeToString())

                freeze_graph.freeze_graph(graph_file,
                                          '',
                                          True,
                                          checkpoint_file,
                                          'Softmax',
                                          '',
                                          '',
                                          self.frozen_graph_file,
                                          False,
                                          '')

    def test_cassifier_with_label_file(self):
        tf_converter.convert(self.frozen_graph_file,
                             mlmodel_path = self.converted_coreml_file,
                             input_name_shape_dict = {'input' : [1, 224, 224, 3]},
                             image_input_names = ['input'],
                             output_feature_names = ['Softmax'],
                             predicted_feature_name = 'classLabel',
                             class_labels = '/tmp/labels.txt',
                             minimum_ios_deployment_target='13')

    def test_cassifier_with_int_label_list(self):
        tf_converter.convert(self.frozen_graph_file,
                             mlmodel_path = self.converted_coreml_file,
                             input_name_shape_dict = {'input' : [1, 224, 224, 3]},
                             image_input_names = ['input'],
                             output_feature_names = ['Softmax'],
                             predicted_feature_name = 'classLabel',
                             class_labels = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
                             minimum_ios_deployment_target='13')

    def test_cassifier_with_string_label_list(self):
        tf_converter.convert(self.frozen_graph_file,
                             mlmodel_path = self.converted_coreml_file,
                             input_name_shape_dict = {'input' : [1, 224, 224, 3]},
                             image_input_names = ['input'],
                             output_feature_names = ['Softmax'],
                             predicted_feature_name = 'classLabel',
                             class_labels = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'],
                             minimum_ios_deployment_target='13')

class CustomLayerUtilsTest(unittest.TestCase):

    @classmethod
    def setUpClass(self):

        spec = Model_pb2.Model()
        spec.specificationVersion = coremltools.SPECIFICATION_VERSION

        features = ['feature_1', 'feature_2']
        output = 'output'
        for f in features:
            input_ = spec.description.input.add()
            input_.name = f
            input_.type.doubleType.MergeFromString(b'')

        output_ = spec.description.output.add()
        output_.name = output
        output_.type.doubleType.MergeFromString(b'')

        layer = spec.neuralNetwork.layers.add()
        layer.name = 'custom1'
        layer.input.append('input')
        layer.output.append('temp1')
        layer.custom.className = 'name1'

        layer2 = spec.neuralNetwork.layers.add()
        layer2.name = 'custom2'
        layer2.input.append('temp1')
        layer2.output.append('temp2')
        layer2.custom.className = 'name2'

        layer3 = spec.neuralNetwork.layers.add()
        layer3.name = 'custom3'
        layer3.input.append('temp2')
        layer3.output.append('output')
        layer3.custom.className = 'name1'

        self.spec = spec

    def test_get_custom_names(self):

        names = get_custom_layer_names(self.spec)
        self.assertEqual(names, {'name1', 'name2'})

    def test_change_custom_name(self):

        replace_custom_layer_name(self.spec, 'name1', 'notname1')
        names = get_custom_layer_names(self.spec)
        self.assertEqual(names, {'notname1', 'name2'})
        #set it back for future tests
        replace_custom_layer_name(self.spec, 'notname1', 'name1')
<commit_msg>Removing  data after test #579<commit_after>import unittest

import numpy as np
import tempfile
import pytest
import shutil
import os

import coremltools
from coremltools._deps import HAS_KERAS_TF
from coremltools._deps import HAS_TF
from coremltools.models.utils import get_custom_layer_names, \
    replace_custom_layer_name, macos_version, is_macos
from coremltools.proto import Model_pb2

if HAS_KERAS_TF:
    from keras.models import Sequential
    from keras.layers import Dense, LSTM
    from coremltools.converters import keras as keras_converter

if HAS_TF:
    import tensorflow as tf
    from tensorflow.python.platform import gfile
    from tensorflow.python.tools import freeze_graph
    import tfcoreml as tf_converter

    tf.compat.v1.disable_eager_execution()

@unittest.skipIf(not HAS_KERAS_TF, 'Missing keras. Skipping tests.')
@pytest.mark.keras1
class KerasBasicNumericCorrectnessTest(unittest.TestCase):

    def test_classifier(self):
        np.random.seed(1988)

        print('running test classifier')

        input_dim = 5
        num_hidden = 12
        num_classes = 6
        input_length = 3

        model = Sequential()
        model.add(LSTM(num_hidden, input_dim=input_dim, input_length=input_length, return_sequences=False))
        model.add(Dense(num_classes, activation='softmax'))

        model.set_weights([np.random.rand(*w.shape) for w in model.get_weights()])

        input_names = ['input']
        output_names = ['zzzz']
        class_labels = ['a', 'b', 'c', 'd', 'e', 'f']
        predicted_feature_name = 'pf'
        coremlmodel = keras_converter.convert(model, input_names, output_names, class_labels=class_labels, predicted_feature_name=predicted_feature_name, predicted_probabilities_output=output_names[0])

        if is_macos() and macos_version() >= (10, 13):
            inputs = np.random.rand(input_dim)
            outputs = coremlmodel.predict({'input': inputs})
            # this checks that the dictionary got the right name and type
            self.assertEquals(type(outputs[output_names[0]]), type({'a': 0.5}))

    def test_classifier_no_name(self):
        np.random.seed(1988)

        input_dim = 5
        num_hidden = 12
        num_classes = 6
        input_length = 3

        model = Sequential()
        model.add(LSTM(num_hidden, input_dim=input_dim, input_length=input_length, return_sequences=False))
        model.add(Dense(num_classes, activation='softmax'))

        model.set_weights([np.random.rand(*w.shape) for w in model.get_weights()])

        input_names = ['input']
        output_names = ['zzzz']
        class_labels = ['a', 'b', 'c', 'd', 'e', 'f']
        predicted_feature_name = 'pf'
        coremlmodel = keras_converter.convert(model, input_names, output_names, class_labels=class_labels, predicted_feature_name=predicted_feature_name)

        if is_macos() and macos_version() >= (10, 13):
            inputs = np.random.rand(input_dim)
            outputs = coremlmodel.predict({'input': inputs})
            # this checks that the dictionary got the right name and type
            self.assertEquals(type(outputs[output_names[0]]), type({'a': 0.5}))

    def test_internal_layer(self):

        np.random.seed(1988)

        input_dim = 5
        num_channels1 = 10
        num_channels2 = 7
        num_channels3 = 5

        w1 = (np.random.rand(input_dim, num_channels1) - 0.5) / 5.0;
        w2 = (np.random.rand(num_channels1, num_channels2) - 0.5) / 5.0;
        w3 = (np.random.rand(num_channels2, num_channels3) - 0.5) / 5.0;

        b1 = (np.random.rand(num_channels1, ) - 0.5) / 5.0;
        b2 = (np.random.rand(num_channels2, ) - 0.5) / 5.0;
        b3 = (np.random.rand(num_channels3, ) - 0.5) / 5.0;

        model = Sequential()
        model.add(Dense(num_channels1, input_dim=input_dim))
        model.add(Dense(num_channels2, name='middle_layer'))
        model.add(Dense(num_channels3))

        model.set_weights([w1, b1, w2, b2, w3, b3])

        input_names = ['input']
        output_names = ['output']
        coreml1 = keras_converter.convert(model, input_names, output_names)

        # adjust the output parameters of coreml1 to include the intermediate layer
        spec = coreml1.get_spec()
        coremlNewOutputs = spec.description.output.add()
        coremlNewOutputs.name = 'middle_layer_output'
        coremlNewParams = coremlNewOutputs.type.multiArrayType
        coremlNewParams.dataType = coremltools.proto.FeatureTypes_pb2.ArrayFeatureType.ArrayDataType.Value('DOUBLE')
        coremlNewParams.shape.extend([num_channels2])

        coremlfinal = coremltools.models.MLModel(spec)

        # generate a second model which
        model2 = Sequential()
        model2.add(Dense(num_channels1, input_dim=input_dim))
        model2.add(Dense(num_channels2))
        model2.set_weights([w1, b1, w2, b2])

        coreml2 = keras_converter.convert(model2, input_names, ['output2'])

        if is_macos() and macos_version() >= (10, 13):
            # generate input data
            inputs = np.random.rand(input_dim)

            fullOutputs = coremlfinal.predict({'input': inputs})

            partialOutput = coreml2.predict({'input': inputs})

            for i in range(0, num_channels2):
                self.assertAlmostEquals(fullOutputs['middle_layer_output'][i], partialOutput['output2'][i], 2)

@unittest.skipIf(not HAS_TF, 'Missing TF. Skipping tests.')
class TFBasicConversionTest(unittest.TestCase):

    @classmethod
    def setUpClass(self):
        self.tmp_dir = tempfile.mkdtemp()
        _, self.graph_file = tempfile.mkstemp(suffix='.pb', prefix=self.tmp_dir)
        _, self.checkpoint_file = tempfile.mkstemp(suffix='.ckpt', prefix=self.tmp_dir)
        _, self.class_label_file = tempfile.mkstemp(suffix='.txt', prefix=self.tmp_dir)
        _, self.frozen_graph_file = tempfile.mkstemp(suffix='.pb', prefix=self.tmp_dir)
        _, self.converted_coreml_file = tempfile.mkstemp(suffix='.mlmodel', prefix=self.tmp_dir)

    @classmethod
    def tearDownClass(self):
        if os.path.exists(self.tmp_dir):
            shutil.rmtree(self.tmp_dir)

    def my_conv_2d(self, input, weight_shape, num_filters, strides, name, activation_fn=tf.nn.relu, with_bias_add=True):
        my_weights = tf.get_variable(name=name + 'weights', shape=weight_shape)
        if with_bias_add:
            my_bias = tf.get_variable(name=name + 'bias', shape=num_filters)
        my_conv = tf.nn.conv2d(input, my_weights, strides=strides, padding='SAME', name=name)
        if with_bias_add:
            my_conv = tf.nn.bias_add(my_conv, my_bias)
        if (activation_fn != None):
            conv_layer_out = activation_fn(my_conv)
        else:
            conv_layer_out = my_conv
        return conv_layer_out

    def test_1(self):
        with open(self.class_label_file, 'w+') as labels_file:
            for a in range(10):
                labels_file.write(str(a + 1) + "\n")

        image_size = 224

        with tf.Graph().as_default():
            batch_size, height, width, channels = 1, image_size, image_size, 3
            images = tf.random.uniform([batch_size, height, width, channels], maxval=1)

            # Create the model.
            i_placeholder = tf.placeholder(name='input', dtype=tf.float32, shape=[1, image_size, image_size, 3])
            net = self.my_conv_2d(i_placeholder, [1, 3, 3, 1], 1, 1, 'first')
            net = tf.nn.avg_pool2d(net, 224, strides=1, padding='VALID', name='AvgPool_1a')
            net = self.my_conv_2d(net, [1, 1, 1, 10], 10, 1, 'fc', activation_fn=None, with_bias_add=False)
            net = tf.squeeze(net, [1, 2], name='SpatialSqueeze')
            probabilities = tf.nn.softmax(net, name='Softmax')

            saver = tf.train.Saver()
            init_op = tf.global_variables_initializer()

            with tf.Session() as sess:
                sess.run(init_op)
                probabilities = sess.run(probabilities, {i_placeholder: images.eval()})
                save_path = saver.save(sess, self.checkpoint_file)

                with gfile.GFile(self.graph_file, 'wb') as f:
                    f.write(sess.graph_def.SerializeToString())

                freeze_graph.freeze_graph(self.graph_file,
                                          '',
                                          True,
                                          self.checkpoint_file,
                                          'Softmax',
                                          '',
                                          '',
                                          self.frozen_graph_file,
                                          False,
                                          '')

    def test_cassifier_with_label_file(self):
        tf_converter.convert(self.frozen_graph_file,
                             mlmodel_path=self.converted_coreml_file,
                             input_name_shape_dict={'input': [1, 224, 224, 3]},
                             image_input_names=['input'],
                             output_feature_names=['Softmax'],
                             predicted_feature_name='classLabel',
                             class_labels=self.class_label_file,
                             minimum_ios_deployment_target='13')

    def test_cassifier_with_int_label_list(self):
        tf_converter.convert(self.frozen_graph_file,
                             mlmodel_path=self.converted_coreml_file,
                             input_name_shape_dict={'input': [1, 224, 224, 3]},
                             image_input_names=['input'],
                             output_feature_names=['Softmax'],
                             predicted_feature_name='classLabel',
                             class_labels=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
                             minimum_ios_deployment_target='13')

    def test_cassifier_with_string_label_list(self):
        tf_converter.convert(self.frozen_graph_file,
                             mlmodel_path=self.converted_coreml_file,
                             input_name_shape_dict={'input': [1, 224, 224, 3]},
                             image_input_names=['input'],
                             output_feature_names=['Softmax'],
                             predicted_feature_name='classLabel',
                             class_labels=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'],
                             minimum_ios_deployment_target='13')

class CustomLayerUtilsTest(unittest.TestCase):

    @classmethod
    def setUpClass(self):
        spec = Model_pb2.Model()
        spec.specificationVersion = coremltools.SPECIFICATION_VERSION

        features = ['feature_1', 'feature_2']
        output = 'output'
        for f in features:
            input_ = spec.description.input.add()
            input_.name = f
            input_.type.doubleType.MergeFromString(b'')

        output_ = spec.description.output.add()
        output_.name = output
        output_.type.doubleType.MergeFromString(b'')

        layer = spec.neuralNetwork.layers.add()
        layer.name = 'custom1'
        layer.input.append('input')
        layer.output.append('temp1')
        layer.custom.className = 'name1'

        layer2 = spec.neuralNetwork.layers.add()
        layer2.name = 'custom2'
        layer2.input.append('temp1')
        layer2.output.append('temp2')
        layer2.custom.className = 'name2'

        layer3 = spec.neuralNetwork.layers.add()
        layer3.name = 'custom3'
        layer3.input.append('temp2')
        layer3.output.append('output')
        layer3.custom.className = 'name1'

        self.spec = spec

    def test_get_custom_names(self):
        names = get_custom_layer_names(self.spec)
        self.assertEqual(names, {'name1', 'name2'})

    def test_change_custom_name(self):
        replace_custom_layer_name(self.spec, 'name1', 'notname1')
        names = get_custom_layer_names(self.spec)
        self.assertEqual(names, {'notname1', 'name2'})
        # set it back for future tests
        replace_custom_layer_name(self.spec, 'notname1', 'name1')
<|endoftext|>