<commit_before>"""
  propagate.py
  by David Weinman
  4/26/13, 9:10p
"""

import os
import datetime
import numpy as np
from neuralNet import *
from pylab import xlabel, ylabel, show, plot, title, grid, savefig
from decimal import Decimal, getcontext
from copy import deepcopy as deepcopy
from random import randint as randint

# constants for printing
mapTitle = "=================================\nNeural Net Map\n================================="
backPropTitle = "=================================\nBack Propagation\n================================="
propLoopTitle = "---------------------------------\nBack Propagation (Loop: %d)\n---------------------------------"

# these are arrays to log neural network error values to visualize a graph while training
errorArray = []
iterations = []

"""
backProp takes a neural network (inputNN), a set of input training values (input),
a number of maximum allowed iterations (max_iterations), a threshold for the
calculated error values (error_threshhold) in order to tell when the network has
been sufficiently trained, and a constant learning rate for weight and threshold updates
(learningRate). backProp returns nothing, back propagation is a supervised training algorithm
for training a neural network.
"""
def backProp(inputNN, trainingSet, targets, max_iterations, error_threshhold, learningRate):
	n_iterations = 0 # counter for the number of propagation loops
	netError = float(error_threshhold + 0.1)

## -------------- foward propagate inputs through net and populate a list of outputs for training
	outputs = [[]] * len(targets)
	for i in range(0, len(targets)):
		outputs[i] = inputNN.update(trainingSet[i])

## -------------- get the current time, make a directory labeled with it to put error graphs in and make it the cwd
	now = datetime.datetime.now()
	print('now: ' + str(now))
	originalDir = os.getcwd()
	os.mkdir('errorGraphs_' + str(now))
	os.chdir('errorGraphs_' + str(now))

	print(backPropTitle) # prints out prompt to label the beginning of training

	while ((n_iterations < max_iterations) and (netError > error_threshhold)):

		print(propLoopTitle % n_iterations) # see global constants. prints which loop is training
		countPatterns = 0 # counter for the number of patterns into input the loop is

		input = randLst(zip(trainingSet, targets)) # makes a randomized list of tuples with the training inputs and their respective targets
		for i in input: #for every pattern in the training set

## -------------- foward propagate input through net
			outputCurrentPattern = inputNN.update(i[0])

## -------------- error calculation -> setup arrays to propagate error backwards
			outputLayerError = [] # create empty array for the error of the nodes in output layer
			for j in range(0, inputNN.l_layers[-1].n_neurons): # for every node in the output layer
				#outputLayerError.append(errorGradientOutputLayer(outputCurrentPattern[j], input[countPatterns][1])) #calc the error in the output layer
				outputLayerError.append(errorGradientOutputLayer(outputCurrentPattern[j], input[countPatterns][1][j])) #calc the error in the output layer
			newWeights = [] # to collect new weights for updating the neurons
			inputsForWeightChangeLoop = i[0] # this is actually to collect outputs for computing the weight change in hidden layers, which are then used as inputs

## -------------- error calculation -> calculate and store for weight updates
			counter = 0 # used for a condition to compute the error value in the hidden layer above the output layer.
			layersFromOut = list(range(1, inputNN.n_hiddenLayers + 2)) # this is in order to get the reverse of a list to do a backwards propagation,  + 1 for output layer
			layersFromOut.reverse() # reverses the list
			error2DArray = [] # this collects error values for use in the change of the weights
			for j in layersFromOut: # for every layer, starting with the output
				while (len(error2DArray) < (j + 1)): # keep the array big enough to write like a backprop
					error2DArray.append([])
				if (j != layersFromOut[0]): # if we're not dealing with the output layer
					for k in range(0, inputNN.l_layers[j].n_neurons): # for every neuron in the layer
						if counter != 0: # if the neuron isn't in the hidden layer above the output
							error2DArray[j].append(errorGradientHiddenLayer(k, j, inputNN, error2DArray[j + 1]))  # compute the error gradient for the neuron
						else:
							error2DArray[j].append(errorGradientHiddenLayer(k, j, inputNN, outputLayerError)) # '' same but for the hidden layer above the output layer
					counter += 1
				else: # deal with the output layer
					error2DArray[j] = outputLayerError

## -------------- weight / threshold update
			for j in range(1, inputNN.n_hiddenLayers + 1): # for every hidden layer, + 1 in range for output layer. (the + 2 was weird, but it made it so the output layer was always updated)
				for k in range(0, inputNN.l_layers[j].n_neurons): # for every neuron in the layer
					newWeights = []
					for h in range(0, inputNN.l_layers[j].l_neurons[k].n_inputs): #for every weight in the neuron

								 # def deltaWeight(oldWeight, learningRate,  x, errorValue, derivAct):
						newWeights.append(deltaWeight(inputNN.l_layers[j].l_neurons[k].l_weights[h], learningRate, inputsForWeightChangeLoop[h], error2DArray[j][k], derivActivation(inputsForWeightChangeLoop, inputNN.l_layers[j].l_neurons[k])))
                                                # ^^^^ get the change in weight

					inputNN.l_layers[j].l_neurons[k].putWeights(newWeights) #update the weights

					inputNN.l_layers[j].l_neurons[k].l_weights[-1] = deltaThreshold(inputNN.l_layers[j].l_neurons[k], error2DArray[j][k], learningRate) # update the threshold

				oldInputsWeightChange = inputsForWeightChangeLoop # this var is used to calculate the new inputs for the change in weight
				inputsForWeightChangeLoop = [] # clear it to re-populate
				for k in range(0, inputNN.l_layers[j].n_neurons): # for every neuron in the layer
										    inputsForWeightChangeLoop.append(float(y(oldInputsWeightChange, inputNN.l_layers[j].l_neurons[k]))) # collect the outputs to use for input to the next layer
			outputs[n_iterations % len(input)] = inputNN.update(i[0])

## -------------- network total error calculation and visualization
		n_iterations += 1
		errorVal = float(0) # sum unit for the net error
		for j in range(0, len(targets)): # for every pattern in the trainingset
			for h in range(0, inputNN.l_layers[-1].n_neurons): # for every output to the net
				errorVal += errorSignal(targets[j][h], outputs[j][h])
		netError = .5 * errorVal #calc the error fn for the net?
		errorArray.append(netError)
		iterations.append(n_iterations)
		plot(iterations, errorArray, linewidth=1.0) # plot(xArray, yArray, ...) from pylab lib
		xlabel('iterations')
		ylabel('error')
		title('error while training')
		grid(True)
		savefig('errorGraph' + str(n_iterations) + '.png')
		#im = Image.open('errorGraph' + str(n_iterations) + '.png') #### it would be so cool to display the error (GUI) instead of just saving it to files
		#im.show()                                                  #### I need to learn to use Popen in subprocess

## -------------- print stuff
		print(mapTitle)
		inputNN.printNN()

	print('propagate finished with %d iterations and %f net error' % (n_iterations, netError))
	os.chdir('..') # go to parent dir for calling f'ns purposes
	return

"""
errorSignal takes a target value for some given neuron (target)
and the output value for that given neuron (activation) and
returns the error for some given neuron and input
"""
def errorSignal(target, activation):
	return float((target - activation)**2)

"""
y takes a set of patterns or inputs (p), and a neuron (n) and returns the
output for the specified node in the neural net. [keep in mind that the
input of some neuron is really in terms of the layer above it.]
"""
def y(p, n):
	if (len(p) != n.n_inputs): # if the node has a different number of inputs than specified in params, throw error.
		raise ValueError('wrong number of inputs: y(p, n) in propagate.')
	return sigmoid(activation(p, n))

"""
sigmoid takes an activation value (activation) and calculates the sigmoid
function on the activation value. [here I use the tanh function]
"""
def sigmoid(activation):
	#return np.longfloat(math.e**activation - math.e**((-1) * activation)/np.longfloat(math.e**activation + math.e**((-1) * activation)))
    return 1/float(1 + (math.e**((-activation) / 1.0))) # where curve shape or 'p' is set to 1.0

"""
sigmoid f'ns derivative. takes an activation value (activation) and returns the
derivative of the sigmoid function.
"""
def derivSigmoid(activation):
	return sigmoid(activation) * (1 - sigmoid(activation))

"""
errorGradientOutputLayer takes an output of some neuron, n (outputN) and a target
value for the same neuron, n (targetN) and produces the basic error gradient
f'n for some output neuron. [this f'n is specific to the output layer of neurons]
"""
def errorGradientOutputLayer(outputN, targetN):
#	print('errorGradientOutputLayer(outputN value: %s, targetN value: %s)' % (str(outputN), str(targetN)))
	return outputN * (1 - outputN) * (targetN - outputN)

"""
errorGradientHiddenLayer takes a neuron index for some hidden neuron, n (neuronIndex),
a layer index for this neuron(layerIndex), an error value for the prior layer
(errorValue), and a neuralNet to which the hiddenN belongs (neuralNet) and returns
the basic error gradient f'n for some hidden neuron.
"""
def errorGradientHiddenLayer(neuronIndex, layerIndex, neuralNet, errorValue):
	weights = neuralNet.l_layers[layerIndex + 1].getWeights()
	sumOut = 0
	for i in range(0, len(weights)): # for every neuron in the layer below
		sumOut += errorValue[i] * weights[i][neuronIndex]
	return sumOut

"""
activation takes a neuron (n) and a set of patterns or inputs (p) and returns
the activation value of the neuron on that input pattern.
"""
def activation(p, n):
	activationValue = 0
	for i in range(0, len(p)):
		activationValue += p[i] * n.l_weights[i]
	activationValue += (-1) * n.l_weights[-1]
	return activationValue

"""
activation f'ns derivative. takes a neuron (n) and a set of patterns or inputs (p)
and returns the derivative of the activation function.
"""
def derivActivation(p, n):
	return activation(p, n) * sigmoid(activation(p, n)) * (1 - sigmoid(activation(p, n)))

"""
deltaThreshhold takes a neuron (neuron), error value for that neuron (error),
an alpha constant or learning rate (learningRate) and returns the change in threshold
for that neuron
"""
def deltaThreshold(neuron, error, learningRate):
	return neuron.l_weights[-1] - (error * learningRate)

"""
deltaWeight takes the current weight value for the given input and neuron (oldWeight),
a learning rate constant for the neural net (learningRate), the input to that branch for
that neuron (x), the error value for the neuron (errorValue), and the derivative of the
activation for that neuron (derivAct) and returns the change in weight for the given oldWeight
"""
def deltaWeight(oldWeight, learningRate,  x, errorValue, derivAct):
	#print('deltaWeight(oldWeight: %s, learningRate: %s,  x: %s, errorValue: %s, derivAct: %s)' % (oldWeight, learningRate,  x, errorValue, derivAct))
	return oldWeight + (learningRate * errorValue * derivAct * x)
#def deltaWeight(targetP, outputP, inputPI):
#	return (targetP - outputP) * inputPI

"""
randLst takes a list (lst) and returns a new list with the same contents but randomly
re-ordered.
"""
def randLst(lst):
	output = []
	inputLst = deepcopy(lst)
	for i in range(0, len(lst)):
		output.append(inputLst[random.randint(0, len(inputLst) - 1)])
		inputLst.remove(output[-1])
	return output

"""
sum takes a list of numbers and returns the sum of a list of numbers.
"""
def sum(lst):
	output = 0
	for i in range(0, len(lst)):
		output += lst[i]
	return output

""" PSEUDO CODE

backprop algorithm:
while (number of iterations < max_iterations && error_fn > error_threshhold) {

        for every pattern in the training set {

                present the pattern to the network
                for each layer in the network {

                        for each node in the layer {

                                calculate the weight sum of the inputs to the node
                                calculate the activation for the node

                        }

                }
                for every node in the output layer {

                        calculate the error signal

                }
                for all hidden layers {

                        for every node in the layer {

                                calculate the node's signal error
                                update each node's weight in the network

                        }

                }
                calculate the error fn

        }

}

"""

""" COPYPASTA
#			print('inputs: %s' % i[0])
#			print('outputs: %s' % outputCurrentPattern)
#			for j in range(0, len(inputNN.l_layers)):
#				print('error for layer %d: %s' % (j, error2DArray[j]))

"""

<commit_msg>updated propagate for readbility<commit_after>"""
  propagate.py
  by David Weinman
  4/26/13, 9:10p
"""

import os
import datetime
import numpy as np
from neuralNet import *
from pylab import xlabel, ylabel, show, plot, title, grid, savefig
from decimal import Decimal, getcontext
from copy import deepcopy as deepcopy
from random import randint as randint

# constants for printing
mapTitle = "=================================\nNeural Net Map\n================================="
backPropTitle = "=================================\nBack Propagation\n================================="
propLoopTitle = "---------------------------------\nBack Propagation (Loop: %d)\n---------------------------------"

# these are arrays to log neural network error values to visualize a graph while training
errorArray = []
iterations = []

"""
backProp takes a neural network (inputNN), a set of input training values (input),
a number of maximum allowed iterations (max_iterations), a threshold for the
calculated error values (error_threshhold) in order to tell when the network has
been sufficiently trained, and a constant learning rate for weight and threshold updates
(learningRate). backProp returns nothing, back propagation is a supervised training algorithm
for training a neural network.
"""
def backProp(inputNN, trainingSet, targets, max_iterations, error_threshhold, learningRate):
	n_iterations = 0 # counter for the number of propagation loops
	netError = float(error_threshhold + 0.1)

## -------------- foward propagate inputs through net and populate a list of outputs for training
	outputs = [[]] * len(targets)
	for i in range(0, len(targets)):
		outputs[i] = inputNN.update(trainingSet[i])

## -------------- get the current time, make a directory labeled with it to put error graphs in and make it the cwd
	now = datetime.datetime.now()
	print('now: ' + str(now))
	originalDir = os.getcwd()
	os.mkdir('errorGraphs_' + str(now))
	os.chdir('errorGraphs_' + str(now))

	print(backPropTitle) # prints out prompt to label the beginning of training

	while ((n_iterations < max_iterations) and (netError > error_threshhold)):

		print(propLoopTitle % n_iterations) # see global constants. prints which loop is training
		countPatterns = 0 # counter for the number of patterns into input the loop is

		input = randLst(zip(trainingSet, targets)) # makes a randomized list of tuples with the training inputs and their respective targets
		for i in input: #for every pattern in the training set

## -------------- foward propagate input through net
			outputCurrentPattern = inputNN.update(i[0])

## -------------- error calculation -> setup arrays to propagate error backwards
			outputLayerError = [] # create empty array for the error of the nodes in output layer
			for j in range(0, inputNN.l_layers[-1].n_neurons): # for every node in the output layer
				#outputLayerError.append(errorGradientOutputLayer(outputCurrentPattern[j], input[countPatterns][1])) #calc the error in the output layer
				outputLayerError.append(errorGradientOutputLayer(outputCurrentPattern[j], input[countPatterns][1][j])) #calc the error in the output layer
			newWeights = [] # to collect new weights for updating the neurons
			inputsForWeightChangeLoop = i[0] # this is actually to collect outputs for computing the weight change in hidden layers, which are then used as inputs

## -------------- error calculation -> calculate and store for weight updates
			counter = 0 # used for a condition to compute the error value in the hidden layer above the output layer.
			layersFromOut = list(range(1, inputNN.n_hiddenLayers + 2)) # this is in order to get the reverse of a list to do a backwards propagation,  + 1 for output layer
			layersFromOut.reverse() # reverses the list
			error2DArray = [] # this collects error values for use in the change of the weights
			for j in layersFromOut: # for every layer, starting with the output
				while (len(error2DArray) < (j + 1)): # keep the array big enough to write like a backprop
					error2DArray.append([])
				if (j != layersFromOut[0]): # if we're not dealing with the output layer
					for k in range(0, inputNN.l_layers[j].n_neurons): # for every neuron in the layer
						if counter != 0: # if the neuron isn't in the hidden layer above the output
							error2DArray[j].append(errorGradientHiddenLayer(k, j, inputNN, error2DArray[j + 1]))  # compute the error gradient for the neuron
						else:
							error2DArray[j].append(errorGradientHiddenLayer(k, j, inputNN, outputLayerError)) # '' same but for the hidden layer above the output layer
					counter += 1
				else: # deal with the output layer
					error2DArray[j] = outputLayerError

## -------------- weight / threshold update
			for j in range(1, inputNN.n_hiddenLayers + 1): # for every hidden layer, + 1 in range for output layer. (the + 2 was weird, but it made it so the output layer was always updated)
				for k in range(0, inputNN.l_layers[j].n_neurons): # for every neuron in the layer
					newWeights = []
					for h in range(0, inputNN.l_layers[j].l_neurons[k].n_inputs): #for every weight in the neuron

								 # def deltaWeight(oldWeight, learningRate,  x, errorValue, derivAct):
						newWeights.append(deltaWeight(inputNN.l_layers[j].l_neurons[k].l_weights[h], learningRate, inputsForWeightChangeLoop[h], error2DArray[j][k], derivActivation(inputsForWeightChangeLoop, inputNN.l_layers[j].l_neurons[k])))
                                                # ^^^^ get the change in weight

					inputNN.l_layers[j].l_neurons[k].putWeights(newWeights) #update the weights

					inputNN.l_layers[j].l_neurons[k].l_weights[-1] = deltaThreshold(inputNN.l_layers[j].l_neurons[k], error2DArray[j][k], learningRate) # update the threshold

				oldInputsWeightChange = inputsForWeightChangeLoop # this var is used to calculate the new inputs for the change in weight
				inputsForWeightChangeLoop = [] # clear it to re-populate
				for k in range(0, inputNN.l_layers[j].n_neurons): # for every neuron in the layer
					inputsForWeightChangeLoop.append(float(y(oldInputsWeightChange, inputNN.l_layers[j].l_neurons[k]))) # collect the outputs to use for input to the next layer
			outputs[n_iterations % len(input)] = inputNN.update(i[0])

## -------------- network total error calculation and visualization
		n_iterations += 1
		errorVal = float(0) # sum unit for the net error
		for j in range(0, len(targets)): # for every pattern in the trainingset
			for h in range(0, inputNN.l_layers[-1].n_neurons): # for every output to the net
				errorVal += errorSignal(targets[j][h], outputs[j][h])
		netError = .5 * errorVal #calc the error fn for the net?
		errorArray.append(netError)
		iterations.append(n_iterations)
		plot(iterations, errorArray, linewidth=1.0) # plot(xArray, yArray, ...) from pylab lib
		xlabel('iterations')
		ylabel('error')
		title('error while training')
		grid(True)
		savefig('errorGraph' + str(n_iterations) + '.png')
		#im = Image.open('errorGraph' + str(n_iterations) + '.png') #### it would be so cool to display the error (GUI) instead of just saving it to files
		#im.show()                                                  #### I need to learn to use Popen in subprocess

## -------------- print stuff
		print(mapTitle)
		inputNN.printNN()

	print('propagate finished with %d iterations and %f net error' % (n_iterations, netError))
	os.chdir('..') # go to parent dir for calling f'ns purposes
	return

"""
errorSignal takes a target value for some given neuron (target)
and the output value for that given neuron (activation) and
returns the error for some given neuron and input
"""
def errorSignal(target, activation):
	return float((target - activation)**2)

"""
y takes a set of patterns or inputs (p), and a neuron (n) and returns the
output for the specified node in the neural net. [keep in mind that the
input of some neuron is really in terms of the layer above it.]
"""
def y(p, n):
	if (len(p) != n.n_inputs): # if the node has a different number of inputs than specified in params, throw error.
		raise ValueError('wrong number of inputs: y(p, n) in propagate.')
	return sigmoid(activation(p, n))

"""
sigmoid takes an activation value (activation) and calculates the sigmoid
function on the activation value. [here I use the tanh function]
"""
def sigmoid(activation):
	#return np.longfloat(math.e**activation - math.e**((-1) * activation)/np.longfloat(math.e**activation + math.e**((-1) * activation)))
    return 1/float(1 + (math.e**((-activation) / 1.0))) # where curve shape or 'p' is set to 1.0

"""
sigmoid f'ns derivative. takes an activation value (activation) and returns the
derivative of the sigmoid function.
"""
def derivSigmoid(activation):
	return sigmoid(activation) * (1 - sigmoid(activation))

"""
errorGradientOutputLayer takes an output of some neuron, n (outputN) and a target
value for the same neuron, n (targetN) and produces the basic error gradient
f'n for some output neuron. [this f'n is specific to the output layer of neurons]
"""
def errorGradientOutputLayer(outputN, targetN):
#	print('errorGradientOutputLayer(outputN value: %s, targetN value: %s)' % (str(outputN), str(targetN)))
	return outputN * (1 - outputN) * (targetN - outputN)

"""
errorGradientHiddenLayer takes a neuron index for some hidden neuron, n (neuronIndex),
a layer index for this neuron(layerIndex), an error value for the prior layer
(errorValue), and a neuralNet to which the hiddenN belongs (neuralNet) and returns
the basic error gradient f'n for some hidden neuron.
"""
def errorGradientHiddenLayer(neuronIndex, layerIndex, neuralNet, errorValue):
	weights = neuralNet.l_layers[layerIndex + 1].getWeights()
	sumOut = 0
	for i in range(0, len(weights)): # for every neuron in the layer below
		sumOut += errorValue[i] * weights[i][neuronIndex]
	return sumOut

"""
activation takes a neuron (n) and a set of patterns or inputs (p) and returns
the activation value of the neuron on that input pattern.
"""
def activation(p, n):
	activationValue = 0
	for i in range(0, len(p)):
		activationValue += p[i] * n.l_weights[i]
	activationValue += (-1) * n.l_weights[-1]
	return activationValue

"""
activation f'ns derivative. takes a neuron (n) and a set of patterns or inputs (p)
and returns the derivative of the activation function.
"""
def derivActivation(p, n):
	return activation(p, n) * sigmoid(activation(p, n)) * (1 - sigmoid(activation(p, n)))

"""
deltaThreshhold takes a neuron (neuron), error value for that neuron (error),
an alpha constant or learning rate (learningRate) and returns the change in threshold
for that neuron
"""
def deltaThreshold(neuron, error, learningRate):
	return neuron.l_weights[-1] - (error * learningRate)

"""
deltaWeight takes the current weight value for the given input and neuron (oldWeight),
a learning rate constant for the neural net (learningRate), the input to that branch for
that neuron (x), the error value for the neuron (errorValue), and the derivative of the
activation for that neuron (derivAct) and returns the change in weight for the given oldWeight
"""
def deltaWeight(oldWeight, learningRate,  x, errorValue, derivAct):
	#print('deltaWeight(oldWeight: %s, learningRate: %s,  x: %s, errorValue: %s, derivAct: %s)' % (oldWeight, learningRate,  x, errorValue, derivAct))
	return oldWeight + (learningRate * errorValue * derivAct * x)
#def deltaWeight(targetP, outputP, inputPI):
#	return (targetP - outputP) * inputPI

"""
randLst takes a list (lst) and returns a new list with the same contents but randomly
re-ordered.
"""
def randLst(lst):
	output = []
	inputLst = deepcopy(lst)
	for i in range(0, len(lst)):
		output.append(inputLst[random.randint(0, len(inputLst) - 1)])
		inputLst.remove(output[-1])
	return output

"""
sum takes a list of numbers and returns the sum of a list of numbers.
"""
def sum(lst):
	output = 0
	for i in range(0, len(lst)):
		output += lst[i]
	return output

""" PSEUDO CODE

backprop algorithm:
while (number of iterations < max_iterations && error_fn > error_threshhold) {

        for every pattern in the training set {

                present the pattern to the network
                for each layer in the network {

                        for each node in the layer {

                                calculate the weight sum of the inputs to the node
                                calculate the activation for the node

                        }

                }
                for every node in the output layer {

                        calculate the error signal

                }
                for all hidden layers {

                        for every node in the layer {

                                calculate the node's signal error
                                update each node's weight in the network

                        }

                }
                calculate the error fn

        }

}

"""

""" COPYPASTA
#			print('inputs: %s' % i[0])
#			print('outputs: %s' % outputCurrentPattern)
#			for j in range(0, len(inputNN.l_layers)):
#				print('error for layer %d: %s' % (j, error2DArray[j]))

"""

<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (c) 2005-2010 ActiveState Software Inc.

"""Utilities for determining application-specific dirs.

See <http://github.com/ActiveState/appdirs> for details.
"""
# Dev Notes:
# - MSDN on where to store app data files:
#   http://support.microsoft.com/default.aspx?scid=kb;en-us;310294#XSLTH3194121123120121120120
# - Mac OS X: http://developer.apple.com/documentation/MacOSX/Conceptual/BPFileSystem/index.html

__version_info__ = (1, 0, 1)
__version__ = '.'.join(map(str, __version_info__))

import sys
import os

class AppDirsError(Exception):
    pass

def user_data_dir(appname, appauthor=None, version=None):
    r"""Return full path to the user-specific data dir for this application.

        "appname" is the name of application.
        "appauthor" (only required and used on Windows) is the name of the
            appauthor or distributing body for this application. Typically
            it is the owning company name.
        "version" is an optional version path element to append to the
            path. You might want to use this if you want multiple versions
            of your app to be able to run independently. If used, this
            would typically be "<major>.<minor>".

    Typical user data directories are:
        Win XP:     C:\Documents and Settings\USER\Application Data\<appauthor>\<appname>
        Mac OS X:   ~/Library/Application Support/<appname>
        Unix:       ~/.<lowercased-appname>

    For Unix there is no *real* standard here. For example, Firefox uses:
    "~/.mozilla/firefox" which is a "~/.<appauthor>/<appname>"-type scheme.
    """
    if sys.platform.startswith("win"):
        if appauthor is None:
            raise AppDirsError("must specify 'appauthor' on Windows")
        path = os.path.join(_get_win_folder("CSIDL_APPDATA"),
                            appauthor, appname)
    elif sys.platform == 'darwin':
        if os.uname()[-1] == 'i386':
            #XXX Folder.FSFindFolder() fails with error -43 on x86. See 42669.
            basepath = os.path.expanduser('~/Library/Application Support')
        else:
            from Carbon import Folder, Folders
            path = Folder.FSFindFolder(Folders.kUserDomain,
                                       Folders.kApplicationSupportFolderType,
                                       Folders.kDontCreateFolder)
            basepath = path.FSRefMakePath()
        path = os.path.join(basepath, appname)
    else:
        path = os.path.expanduser("~/." + appname.lower())
    if version:
        path = os.path.join(path, version)
    return path

def site_data_dir(appname, appauthor=None, version=None):
    """Return full path to the user-shared data dir for this application.

        "appname" is the name of application.
        "appauthor" (only required and used on Windows) is the name of the
            appauthor or distributing body for this application. Typically
            it is the owning company name.
        "version" is an optional version path element to append to the
            path. You might want to use this if you want multiple versions
            of your app to be able to run independently. If used, this
            would typically be "<major>.<minor>".

    Typical user data directories are:
        Win XP:     C:\Documents and Settings\All Users\Application Data\<appauthor>\<appname>
        Mac OS X:   /Library/Application Support/<appname>
        Unix:       /etc/<lowercased-appname>
    """
    if sys.platform.startswith("win"):
        if appauthor is None:
            raise AppDirsError("must specify 'appauthor' on Windows")
        path = os.path.join(_get_win_folder("CSIDL_COMMON_APPDATA"),
                            appauthor, appname)
    elif sys.platform == 'darwin':
        if os.uname()[-1] == 'i386':
            #XXX Folder.FSFindFolder() fails with error -43 on x86. See 42669.
            basepath = os.path.expanduser('~/Library/Application Support')
        else:
            from Carbon import Folder, Folders
            path = Folder.FSFindFolder(Folders.kLocalDomain,
                                       Folders.kApplicationSupportFolderType,
                                       Folders.kDontCreateFolder)
            basepath = os.path.join(path.FSRefMakePath(), appname)
        path = os.path.join(basepath, appname)
    else:
        path = "/etc/"+appname.lower()
    if version:
        path = os.path.join(path, version)
    return path

def user_cache_dir(appname, appauthor=None, version=None):
    r"""Return full path to the user-specific cache dir for this application.

        "appname" is the name of application.
        "appauthor" (only required and used on Windows) is the name of the
            appauthor or distributing body for this application. Typically
            it is the owning company name.
        "version" is an optional version path element to append to the
            path. You might want to use this if you want multiple versions
            of your app to be able to run independently. If used, this
            would typically be "<major>.<minor>".

    Typical user cache directories are:
        Win XP:     C:\Documents and Settings\USER\Local Settings\Application Data\<appauthor>\<appname>
        Mac OS X:   ~/Library/Caches/<appname>
        Unix:       ~/.<lowercased-appname>/caches

    For Unix there is no *real* standard here. Note that we are returning
    the *same dir as the user_data_dir()* for Unix. Use accordingly.
    """
    if sys.platform.startswith("win"):
        if appauthor is None:
            raise AppDirsError("must specify 'appauthor' on Windows")
        path = os.path.join(_get_win_folder("CSIDL_LOCAL_APPDATA"),
                            appauthor, appname)
    elif sys.platform == 'darwin':
        if os.uname()[-1] == 'i386':
            #XXX Folder.FSFindFolder() fails with error -43 on x86. See 42669.
            basepath = os.path.expanduser('~/Library/Caches')
        else:
            from Carbon import Folder, Folders
            path = Folder.FSFindFolder(Folders.kUserDomain,
                                       Folders.kCachedDataFolderType,
                                       Folders.kDontCreateFolder)
            basepath = path.FSRefMakePath()
        path = os.path.join(basepath, appname)
    else:
        path = os.path.expanduser("~/.%s/caches" % appname.lower())
    if version:
        path = os.path.join(path, version)
    return path

#---- internal support stuff

def _get_win_folder_from_registry(csidl_name):
    """This is a fallback technique at best. I'm not sure if using the
    registry for this guarantees us the correct answer for all CSIDL_*
    names.
    """
    import _winreg

    shell_folder_name = {
        "CSIDL_APPDATA": "AppData",
        "CSIDL_COMMON_APPDATA": "Common AppData",
        "CSIDL_LOCAL_APPDATA": "Local AppData",
    }[csidl_name]

    key = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER,
        r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders")
    dir, type = _winreg.QueryValueEx(key, shell_folder_name)
    return dir

def _get_win_folder_with_pywin32(csidl_name):
    from win32com.shell import shellcon, shell
    dir = shell.SHGetFolderPath(0, getattr(shellcon, csidl_name), 0, 0)
    # Try to make this a unicode path because SHGetFolderPath does
    # not return unicode strings when there is unicode data in the
    # path.
    try:
        dir = unicode(dir)
    except UnicodeError:
        pass
    return dir

def _get_win_folder_with_ctypes(csidl_name):
    import ctypes

    csidl_const = {
        "CSIDL_APPDATA": 26,
        "CSIDL_COMMON_APPDATA": 35,
        "CSIDL_LOCAL_APPDATA": 28,
    }[csidl_name]

    buf = ctypes.create_unicode_buffer(1024)
    ctypes.windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf)
    return buf.value

if sys.platform == "win32":
    try:
        import win32com.shell
        _get_win_folder = _get_win_folder_with_pywin32
    except ImportError:
        try:
            import ctypes
            _get_win_folder = _get_win_folder_with_ctypes
        except ImportError:
            _get_win_folder = _get_win_folder_from_registry

#---- self test code

if __name__ == "__main__":
    print("user data dir: %s" % user_data_dir("Komodo", "ActiveState"))
    print("site data dir: %s" % site_data_dir("Komodo", "ActiveState"))
    print("user cache dir: %s" % user_cache_dir("Komodo", "ActiveState"))

<commit_msg>doc tweaks<commit_after>#!/usr/bin/env python
# Copyright (c) 2005-2010 ActiveState Software Inc.

"""Utilities for determining application-specific dirs.

See <http://github.com/ActiveState/appdirs> for details.
"""
# Dev Notes:
# - MSDN on where to store app data files:
#   http://support.microsoft.com/default.aspx?scid=kb;en-us;310294#XSLTH3194121123120121120120
# - Mac OS X: http://developer.apple.com/documentation/MacOSX/Conceptual/BPFileSystem/index.html

__version_info__ = (1, 0, 1)
__version__ = '.'.join(map(str, __version_info__))

import sys
import os

class AppDirsError(Exception):
    pass

def user_data_dir(appname, appauthor=None, version=None):
    r"""Return full path to the user-specific data dir for this application.

        "appname" is the name of application.
        "appauthor" (only required and used on Windows) is the name of the
            appauthor or distributing body for this application. Typically
            it is the owning company name.
        "version" is an optional version path element to append to the
            path. You might want to use this if you want multiple versions
            of your app to be able to run independently. If used, this
            would typically be "<major>.<minor>".

    Typical user data directories are:
        Win XP:     C:\Documents and Settings\USER\Application Data\<AppAuthor>\<AppName>
        Mac OS X:   ~/Library/Application Support/<AppName>
        Unix:       ~/.<appname>

    For Unix there is no *real* standard here. For example, Firefox uses:
    "~/.mozilla/firefox" which is a "~/.<appauthor>/<appname>"-type scheme.
    """
    if sys.platform.startswith("win"):
        if appauthor is None:
            raise AppDirsError("must specify 'appauthor' on Windows")
        path = os.path.join(_get_win_folder("CSIDL_APPDATA"),
                            appauthor, appname)
    elif sys.platform == 'darwin':
        if os.uname()[-1] == 'i386':
            # Folder.FSFindFolder() fails with error -43 on x86.
            # See Komodo bug 42669.
            basepath = os.path.expanduser('~/Library/Application Support')
        else:
            from Carbon import Folder, Folders
            path = Folder.FSFindFolder(Folders.kUserDomain,
                                       Folders.kApplicationSupportFolderType,
                                       Folders.kDontCreateFolder)
            basepath = path.FSRefMakePath()
        path = os.path.join(basepath, appname)
    else:
        path = os.path.expanduser("~/." + appname.lower())
    if version:
        path = os.path.join(path, version)
    return path

def site_data_dir(appname, appauthor=None, version=None):
    """Return full path to the user-shared data dir for this application.

        "appname" is the name of application.
        "appauthor" (only required and used on Windows) is the name of the
            appauthor or distributing body for this application. Typically
            it is the owning company name.
        "version" is an optional version path element to append to the
            path. You might want to use this if you want multiple versions
            of your app to be able to run independently. If used, this
            would typically be "<major>.<minor>".

    Typical user data directories are:
        Win XP:     C:\Documents and Settings\All Users\Application Data\<AppAuthor>\<AppName>
        Mac OS X:   /Library/Application Support/<AppName>
        Unix:       /etc/<appname>
    """
    if sys.platform.startswith("win"):
        if appauthor is None:
            raise AppDirsError("must specify 'appauthor' on Windows")
        path = os.path.join(_get_win_folder("CSIDL_COMMON_APPDATA"),
                            appauthor, appname)
    elif sys.platform == 'darwin':
        if os.uname()[-1] == 'i386':
            #XXX Folder.FSFindFolder() fails with error -43 on x86. See 42669.
            basepath = os.path.expanduser('~/Library/Application Support')
        else:
            from Carbon import Folder, Folders
            path = Folder.FSFindFolder(Folders.kLocalDomain,
                                       Folders.kApplicationSupportFolderType,
                                       Folders.kDontCreateFolder)
            basepath = os.path.join(path.FSRefMakePath(), appname)
        path = os.path.join(basepath, appname)
    else:
        path = "/etc/"+appname.lower()
    if version:
        path = os.path.join(path, version)
    return path

def user_cache_dir(appname, appauthor=None, version=None):
    r"""Return full path to the user-specific cache dir for this application.

        "appname" is the name of application.
        "appauthor" (only required and used on Windows) is the name of the
            appauthor or distributing body for this application. Typically
            it is the owning company name.
        "version" is an optional version path element to append to the
            path. You might want to use this if you want multiple versions
            of your app to be able to run independently. If used, this
            would typically be "<major>.<minor>".

    Typical user cache directories are:
        Win XP:     C:\Documents and Settings\USER\Local Settings\Application Data\<AppAuthor>\<AppName>
        Mac OS X:   ~/Library/Caches/<AppName>
        Unix:       ~/.<appname>/caches

    For Unix there is no *real* standard here. Note that we are returning
    the *same dir as the user_data_dir()* for Unix. Use accordingly.
    """
    if sys.platform.startswith("win"):
        if appauthor is None:
            raise AppDirsError("must specify 'appauthor' on Windows")
        path = os.path.join(_get_win_folder("CSIDL_LOCAL_APPDATA"),
                            appauthor, appname)
    elif sys.platform == 'darwin':
        if os.uname()[-1] == 'i386':
            # Folder.FSFindFolder() fails with error -43 on x86.
            # See Komodo bug 42669.
            basepath = os.path.expanduser('~/Library/Caches')
        else:
            from Carbon import Folder, Folders
            path = Folder.FSFindFolder(Folders.kUserDomain,
                                       Folders.kCachedDataFolderType,
                                       Folders.kDontCreateFolder)
            basepath = path.FSRefMakePath()
        path = os.path.join(basepath, appname)
    else:
        path = os.path.expanduser("~/.%s/caches" % appname.lower())
    if version:
        path = os.path.join(path, version)
    return path

#---- internal support stuff

def _get_win_folder_from_registry(csidl_name):
    """This is a fallback technique at best. I'm not sure if using the
    registry for this guarantees us the correct answer for all CSIDL_*
    names.
    """
    import _winreg

    shell_folder_name = {
        "CSIDL_APPDATA": "AppData",
        "CSIDL_COMMON_APPDATA": "Common AppData",
        "CSIDL_LOCAL_APPDATA": "Local AppData",
    }[csidl_name]

    key = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER,
        r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders")
    dir, type = _winreg.QueryValueEx(key, shell_folder_name)
    return dir

def _get_win_folder_with_pywin32(csidl_name):
    from win32com.shell import shellcon, shell
    dir = shell.SHGetFolderPath(0, getattr(shellcon, csidl_name), 0, 0)
    # Try to make this a unicode path because SHGetFolderPath does
    # not return unicode strings when there is unicode data in the
    # path.
    try:
        dir = unicode(dir)
    except UnicodeError:
        pass
    return dir

def _get_win_folder_with_ctypes(csidl_name):
    import ctypes

    csidl_const = {
        "CSIDL_APPDATA": 26,
        "CSIDL_COMMON_APPDATA": 35,
        "CSIDL_LOCAL_APPDATA": 28,
    }[csidl_name]

    buf = ctypes.create_unicode_buffer(1024)
    ctypes.windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf)
    return buf.value

if sys.platform == "win32":
    try:
        import win32com.shell
        _get_win_folder = _get_win_folder_with_pywin32
    except ImportError:
        try:
            import ctypes
            _get_win_folder = _get_win_folder_with_ctypes
        except ImportError:
            _get_win_folder = _get_win_folder_from_registry

#---- self test code

if __name__ == "__main__":
    print("user data dir: %s" % user_data_dir("Komodo", "ActiveState"))
    print("site data dir: %s" % site_data_dir("Komodo", "ActiveState"))
    print("user cache dir: %s" % user_cache_dir("Komodo", "ActiveState"))

<|endoftext|>


<commit_before>from flask import Blueprint, request, url_for
from sqlalchemy.sql.functions import count
from sqlalchemy.sql.expression import or_
from sqlalchemy.orm import aliased

from datawire.core import db
from datawire.auth import require
from datawire.model import Entity, Facet, Match, Frame

from datawire.views.util import jsonify, obj_or_404
from datawire.views.pager import query_pager

entities = Blueprint('entities', __name__)

@entities.route('/facets')
def facet_index():
    facets = Facet.all()
    return jsonify({
        'results': facets,
        'count': len(facets)
    })

@entities.route('/facets/<key>')
def facet_get(key):
    facet = obj_or_404(Facet.by_key(key))
    return jsonify(facet)

@entities.route('/users/<int:id>/entities')
def user_index(id):
    require.user_id(id)
    q = Entity.all().filter_by(user_id=id)
    q = q.add_column(count(Match.id))
    q = q.outerjoin(Entity.matches)
    q = q.group_by(Entity)
    if 'facet' in request.args:
        q = q.filter(Entity.facet==request.args.get('facet'))

    if 'entity' in request.args:
        q = q.outerjoin(Match.frame)
        other_match = aliased(Match)
        q = q.outerjoin(other_match, Frame.urn==other_match.urn)
        fq = or_(other_match.entity_id.in_(request.args.get('entity')),
                 other_match.entity_id==None)
        q = q.filter(fq)

    q = q.order_by(count(Match.id).desc())
    q = q.order_by(Entity.text.desc())

    def transform_result(result):
        entity_obj, count_ = result
        data = entity_obj.to_ref()
        data['count'] = count_
        return data

    return query_pager(q, 'entities.user_index', transform=transform_result, id=id)

@entities.route('/entities/<id>')
def get(id):
    require.logged_in()
    entity = obj_or_404(Entity.by_user_and_id(request.user, id))
    return jsonify(entity)

@entities.route('/entities', methods=['POST'])
def create():
    require.logged_in()
    entity = Entity.create(request.form, request.user)
    db.session.commit()
    return jsonify(entity)

@entities.route('/entities/<int:id>', methods=['POST'])
def update(id):
    require.logged_in()
    entity = obj_or_404(Entity.by_user_and_id(request.user, id))
    entity.update(request.form)
    db.session.commit()
    return jsonify(entity)

@entities.route('/entities/<int:id>', methods=['DELETE'])
def delete(id):
    require.logged_in()
    entity = obj_or_404(Entity.by_user_and_id(request.user, id))
    entity.delete()
    db.session.commit()
    return jsonify({'status': 'gone'}, status=410)
<commit_msg>Gratuitous use of distinct for greater benefit.<commit_after>from flask import Blueprint, request, url_for
from sqlalchemy.sql.functions import count
from sqlalchemy.sql.expression import or_, func
from sqlalchemy.orm import aliased

from datawire.core import db
from datawire.auth import require
from datawire.model import Entity, Facet, Match, Frame

from datawire.views.util import jsonify, obj_or_404
from datawire.views.pager import query_pager

entities = Blueprint('entities', __name__)

@entities.route('/facets')
def facet_index():
    facets = Facet.all()
    return jsonify({
        'results': facets,
        'count': len(facets)
    })

@entities.route('/facets/<key>')
def facet_get(key):
    facet = obj_or_404(Facet.by_key(key))
    return jsonify(facet)

@entities.route('/users/<int:id>/entities')
def user_index(id):
    require.user_id(id)
    q = Entity.all().filter_by(user_id=id)
    count_field = count(func.distinct(Match.id))
    q = q.add_column(count_field)
    q = q.outerjoin(Entity.matches)
    q = q.group_by(Entity)
    if 'facet' in request.args:
        q = q.filter(Entity.facet == request.args.get('facet'))

    if 'entity' in request.args:
        q = q.outerjoin(Match.frame)
        other_match = aliased(Match)
        q = q.outerjoin(other_match, Frame.urn == other_match.urn)
        fq = or_(other_match.entity_id.in_(request.args.get('entity')),
                 other_match.entity_id == None)
        q = q.filter(fq)

    q = q.order_by(count_field.desc())
    q = q.order_by(Entity.text.desc())

    def transform_result(result):
        entity_obj, count_ = result
        data = entity_obj.to_ref()
        data['count'] = count_
        return data

    return query_pager(q, 'entities.user_index', transform=transform_result, id=id)

@entities.route('/entities/<id>')
def get(id):
    require.logged_in()
    entity = obj_or_404(Entity.by_user_and_id(request.user, id))
    return jsonify(entity)

@entities.route('/entities', methods=['POST'])
def create():
    require.logged_in()
    entity = Entity.create(request.form, request.user)
    db.session.commit()
    return jsonify(entity)

@entities.route('/entities/<int:id>', methods=['POST'])
def update(id):
    require.logged_in()
    entity = obj_or_404(Entity.by_user_and_id(request.user, id))
    entity.update(request.form)
    db.session.commit()
    return jsonify(entity)

@entities.route('/entities/<int:id>', methods=['DELETE'])
def delete(id):
    require.logged_in()
    entity = obj_or_404(Entity.by_user_and_id(request.user, id))
    entity.delete()
    db.session.commit()
    return jsonify({'status': 'gone'}, status=410)
<|endoftext|>


<commit_before>'''
Created on May 8, 2013

@author: mandd
'''
import xml.etree.ElementTree as ET
import numpy as np
from BaseType import BaseType
from Csv_loader import CsvLoader as ld
import DataSets
import matplotlib.pyplot as plt
#from hdf5_manager import hdf5Manager as AAFManager
#import h5py as h5

class OutStream(BaseType):
  def __init__(self):
    BaseType.__init__(self)
    self.fileNameRoot   = None
    self.variables      = []
    self.toLoadFromList = []
    self.alreadyRead    = []
    self.histories      = {}
  def readMoreXML(self,xmlNode):
    var = xmlNode.find('variable').text
    var.replace(" ","")
    if var.lower() == 'all':
      self.variables = ['all']
    else:
      self.variables = var.split(',')

    try:
      var = xmlNode.find('fileNameRoot').text
      self.fileNameRoot = var.replace(" ","")
    except:
      pass

  def addInitParams(self,tempDict):
    for i in range(len(self.variables)):
      tempDict['Variables_'+str(i)] = self.variables[i]
    if self.fileNameRoot:
      tempDict['FileNameRoot'] = self.fileNameRoot
    return tempDict

  def finalize(self):
    pass
  def addOutput(self,toLoadFrom):
    # this function adds the file name/names to the
    # filename list
    print('toLoadFrom :')
    print(toLoadFrom)

    self.toLoadFromList.append(toLoadFrom)
    return
#  def getInpParametersValues(self):
#    return self.inpParametersValues
#
#  def getOutParametersValues(self):
#    return self.outParametersValues
  def retrieveHistories(self):
    try:
      if self.toLoadFromList[0].type == "HDF5":
        endGroupNames = self.toLoadFromList[0].getEndingGroupNames()

        for index in xrange(len(endGroupNames)):
          if not endGroupNames[index] in self.alreadyRead:
            self.histories[endGroupNames[index]] = self.toLoadFromList[0].returnHistory({'history':endGroupNames[index],'filter':'whole'})
            self.alreadyRead.append(endGroupNames[index])
    except:
      # loading from file (csv)
      for index in xrange(len(self.toLoadFromList)):
        groupname = self.toLoadFromList[index].split('~')[1]
        if not groupname in self.alreadyRead:
           # open file
          myFile = open (self.toLoadFromList[index],'rb')
          # read the field names
          all_field_names = myFile.readline().split(',')
          # load the table data (from the csv file) into a numpy nd array
          data = np.loadtxt(myFile,dtype='float',delimiter=',',ndmin=2)
          # close file
          myFile.close()
          self.histories[groupname] = (data,{'headers':all_field_names})
          self.alreadyRead.append(groupname)
    return

  def getParam(self,typeVar,keyword):
    pass

class ScreenPlot(OutStream):
  def __init__(self):
    OutStream.__init__(self)

  def finalize(self):
    try:
      self.retrieveHistories()
    except:
      OutStream.retrieveHistories(self)
    headers = self.histories[self.alreadyRead[0]][1]['headers']
    timeVar = ''
    for i in xrange(len(headers)):
      if headers[i].lower() == 'time':
        #timeVar = headers.pop(i)
        timeLoc = i
        break

    for index in xrange(len(headers)):
      if headers[index].lower() != 'time':
        if not self.variables[0]=='all':
          if headers[index] in self.variables:
            plot_it = True
          else:
            plot_it = False
        else:
          plot_it = True
        if plot_it:
          plt.figure()
          plt.xlabel(headers[timeLoc])
          plt.ylabel(headers[index])
          plt.title('Plot of histories')
          for key in self.histories:
            plt.plot(self.histories[key][0][:,timeLoc],self.histories[key][0][:,index])
          plt.show()

class PdfPlot(OutStream):
  def __init__(self):
    OutStream.__init__(self)

  def finalize(self):
    try:
      self.retrieveHistories()
    except:
      OutStream.retrieveHistories(self)
    headers = self.histories[self.alreadyRead[0]][1]['headers']
    timeVar = ''
    for i in xrange(len(headers)):
      if headers[i].lower() == 'time':
        #timeVar = headers.pop(i)
        timeLoc = i
        break

    for index in xrange(len(headers)):
      if headers[index].lower() != 'time':
        if not self.variables[0]=='all':
          if headers[index] in self.variables:
            plot_it = True
          else:
            plot_it = False
        else:
          plot_it = True
        if plot_it:
          plt.figure()
          plt.xlabel(headers[timeLoc])
          plt.ylabel(headers[index])
          plt.title('Plot of histories')
          for key in self.histories:
            plt.plot(self.histories[key][0][:,timeLoc],self.histories[key][0][:,index])
        fileName = self.fileNameRoot + '.pdf'
        plt.savefig(fileName, dpi=fig.dpi)
class PngPlot(OutStream):
  def __init__(self):
    OutStream.__init__(self)

  def finalize(self):
    try:
      self.retrieveHistories()
    except:
      OutStream.retrieveHistories(self)
    headers = self.histories[self.alreadyRead[0]][1]['headers']
    timeVar = ''
    for i in xrange(len(headers)):
      if headers[i].lower() == 'time':
        #timeVar = headers.pop(i)
        timeLoc = i
        break

    for index in xrange(len(headers)):
      if headers[index].lower() != 'time':
        if not self.variables[0]=='all':
          if headers[index] in self.variables:
            plot_it = True
          else:
            plot_it = False
        else:
          plot_it = True
        if plot_it:
          plt.figure()
          plt.xlabel(headers[timeLoc])
          plt.ylabel(headers[index])
          plt.title('Plot of histories')
          for key in self.histories:
            plt.plot(self.histories[key][0][:,timeLoc],self.histories[key][0][:,index])
        fileName = self.fileNameRoot + '.png'
        fig.savefig(fileName, dpi=fig.dpi)
class JpegPlot(OutStream):
  def __init__(self):
    OutStream.__init__(self)

  def finalize(self):
    try:
      self.retrieveHistories()
    except:
      OutStream.retrieveHistories(self)
    headers = self.histories[self.alreadyRead[0]][1]['headers']
    timeVar = ''
    for i in xrange(len(headers)):
      if headers[i].lower() == 'time':
        #timeVar = headers.pop(i)
        timeLoc = i
        break

    for index in xrange(len(headers)):
      if headers[index].lower() != 'time':
        if not self.variables[0]=='all':
          if headers[index] in self.variables:
            plot_it = True
          else:
            plot_it = False
        else:
          plot_it = True
        if plot_it:
          plt.figure()
          plt.xlabel(headers[timeLoc])
          plt.ylabel(headers[index])
          plt.title('Plot of histories')
          for key in self.histories:
            plt.plot(self.histories[key][0][:,timeLoc],self.histories[key][0][:,index])
        fileName = self.fileNameRoot + '.jpeg'
        fig.savefig(fileName, dpi=fig.dpi)
class EpsPlot(OutStream):
  def __init__(self):
    OutStream.__init__(self)

  def finalize(self):
    try:
      self.retrieveHistories()
    except:
      OutStream.retrieveHistories(self)
    headers = self.histories[self.alreadyRead[0]][1]['headers']
    timeVar = ''
    for i in xrange(len(headers)):
      if headers[i].lower() == 'time':
        #timeVar = headers.pop(i)
        timeLoc = i
        break

    for index in xrange(len(headers)):
      if headers[index].lower() != 'time':
        if not self.variables[0]=='all':
          if headers[index] in self.variables:
            plot_it = True
          else:
            plot_it = False
        else:
          plot_it = True
        if plot_it:
          plt.figure()
          plt.xlabel(headers[timeLoc])
          plt.ylabel(headers[index])
          plt.title('Plot of histories')
          for key in self.histories:
            plt.plot(self.histories[key][0][:,timeLoc],self.histories[key][0][:,index])
        fileName = self.fileNameRoot + '.eps'
        fig.savefig(fileName, dpi=fig.dpi)

def returnInstance(Type):
  base = 'OutStream'
  InterfaceDict = {}
  InterfaceDict['Screen'   ]    = ScreenPlot
  InterfaceDict['Pdf'      ]    = PdfPlot
  InterfaceDict['Png'      ]    = PngPlot
  InterfaceDict['Jpeg'     ]    = JpegPlot
  InterfaceDict['Eps'      ]    = EpsPlot
  try:
    if Type in InterfaceDict.keys():
      return InterfaceDict[Type]()
  except:
    raise NameError('not known '+base+' type'+Type)<commit_msg>small fixes<commit_after>'''
Created on May 8, 2013

@author: mandd
'''
import xml.etree.ElementTree as ET
import numpy as np
from BaseType import BaseType
from Csv_loader import CsvLoader as ld
import DataSets
import matplotlib.pyplot as plt
#from hdf5_manager import hdf5Manager as AAFManager
#import h5py as h5

class OutStream(BaseType):
  def __init__(self):
    BaseType.__init__(self)
    self.fileNameRoot   = None
    self.variables      = []
    self.toLoadFromList = []
    self.alreadyRead    = []
    self.histories      = {}
  def readMoreXML(self,xmlNode):
    var = xmlNode.find('variable').text
    var.replace(" ","")
    if var.lower() == 'all':
      self.variables = ['all']
    else:
      self.variables = var.split(',')

    try:
      var = xmlNode.find('fileNameRoot').text
      self.fileNameRoot = var.replace(" ","")
    except:
      pass

  def addInitParams(self,tempDict):
    for i in range(len(self.variables)):
      tempDict['Variables_'+str(i)] = self.variables[i]
    if self.fileNameRoot:
      tempDict['FileNameRoot'] = self.fileNameRoot
    return tempDict

  def finalize(self):
    pass
  def addOutput(self,toLoadFrom):
    # this function adds the file name/names to the
    # filename list
    print('toLoadFrom :')
    print(toLoadFrom)

    self.toLoadFromList.append(toLoadFrom)
    return
#  def getInpParametersValues(self):
#    return self.inpParametersValues
#
#  def getOutParametersValues(self):
#    return self.outParametersValues
  def retrieveHistories(self):
    try:
      if self.toLoadFromList[0].type == "HDF5":
        endGroupNames = self.toLoadFromList[0].getEndingGroupNames()

        for index in xrange(len(endGroupNames)):
          if not endGroupNames[index] in self.alreadyRead:
            self.histories[endGroupNames[index]] = self.toLoadFromList[0].returnHistory({'history':endGroupNames[index],'filter':'whole'})
            self.alreadyRead.append(endGroupNames[index])
    except:
      # loading from file (csv)
      for index in xrange(len(self.toLoadFromList)):
        groupname = self.toLoadFromList[index].split('~')[1]
        if not groupname in self.alreadyRead:
           # open file
          myFile = open (self.toLoadFromList[index],'rb')
          # read the field names
          all_field_names = myFile.readline().split(',')
          # load the table data (from the csv file) into a numpy nd array
          data = np.loadtxt(myFile,dtype='float',delimiter=',',ndmin=2)
          # close file
          myFile.close()
          self.histories[groupname] = (data,{'headers':all_field_names})
          self.alreadyRead.append(groupname)
    return

  def getParam(self,typeVar,keyword):
    pass

class ScreenPlot(OutStream):
  def __init__(self):
    OutStream.__init__(self)

  def finalize(self):
    try:
      self.retrieveHistories()
    except:
      OutStream.retrieveHistories(self)
    headers = self.histories[self.alreadyRead[0]][1]['headers']
    timeVar = ''
    for i in xrange(len(headers)):
      if headers[i].lower() == 'time':
        #timeVar = headers.pop(i)
        timeLoc = i
        break

    for index in xrange(len(headers)):
      if headers[index].lower() != 'time':
        if not self.variables[0]=='all':
          if headers[index] in self.variables:
            plot_it = True
          else:
            plot_it = False
        else:
          plot_it = True
        if plot_it:
          plt.figure(index)
          plt.xlabel(headers[timeLoc])
          plt.ylabel(headers[index])
          plt.title('Plot of histories')
          for key in self.histories:
            plt.plot(self.histories[key][0][:,timeLoc],self.histories[key][0][:,index])
    plt.show()

class PdfPlot(OutStream):
  def __init__(self):
    OutStream.__init__(self)

  def finalize(self):
    try:
      self.retrieveHistories()
    except:
      OutStream.retrieveHistories(self)
    headers = self.histories[self.alreadyRead[0]][1]['headers']
    timeVar = ''
    for i in xrange(len(headers)):
      if headers[i].lower() == 'time':
        #timeVar = headers.pop(i)
        timeLoc = i
        break

    for index in xrange(len(headers)):
      if headers[index].lower() != 'time':
        if not self.variables[0]=='all':
          if headers[index] in self.variables:
            plot_it = True
          else:
            plot_it = False
        else:
          plot_it = True
        if plot_it:
          plt.figure()
          plt.xlabel(headers[timeLoc])
          plt.ylabel(headers[index])
          plt.title('Plot of histories')
          for key in self.histories:
            plt.plot(self.histories[key][0][:,timeLoc],self.histories[key][0][:,index])
        fileName = self.fileNameRoot + '.pdf'
        plt.savefig(fileName, dpi=fig.dpi)
class PngPlot(OutStream):
  def __init__(self):
    OutStream.__init__(self)

  def finalize(self):
    try:
      self.retrieveHistories()
    except:
      OutStream.retrieveHistories(self)
    headers = self.histories[self.alreadyRead[0]][1]['headers']
    timeVar = ''
    for i in xrange(len(headers)):
      if headers[i].lower() == 'time':
        #timeVar = headers.pop(i)
        timeLoc = i
        break

    for index in xrange(len(headers)):
      if headers[index].lower() != 'time':
        if not self.variables[0]=='all':
          if headers[index] in self.variables:
            plot_it = True
          else:
            plot_it = False
        else:
          plot_it = True
        if plot_it:
          plt.figure()
          plt.xlabel(headers[timeLoc])
          plt.ylabel(headers[index])
          plt.title('Plot of histories')
          for key in self.histories:
            plt.plot(self.histories[key][0][:,timeLoc],self.histories[key][0][:,index])
        fileName = self.fileNameRoot + '.png'
        fig.savefig(fileName, dpi=fig.dpi)
class JpegPlot(OutStream):
  def __init__(self):
    OutStream.__init__(self)

  def finalize(self):
    try:
      self.retrieveHistories()
    except:
      OutStream.retrieveHistories(self)
    headers = self.histories[self.alreadyRead[0]][1]['headers']
    timeVar = ''
    for i in xrange(len(headers)):
      if headers[i].lower() == 'time':
        #timeVar = headers.pop(i)
        timeLoc = i
        break

    for index in xrange(len(headers)):
      if headers[index].lower() != 'time':
        if not self.variables[0]=='all':
          if headers[index] in self.variables:
            plot_it = True
          else:
            plot_it = False
        else:
          plot_it = True
        if plot_it:
          plt.figure()
          plt.xlabel(headers[timeLoc])
          plt.ylabel(headers[index])
          plt.title('Plot of histories')
          for key in self.histories:
            plt.plot(self.histories[key][0][:,timeLoc],self.histories[key][0][:,index])
        fileName = self.fileNameRoot + '.jpeg'
        fig.savefig(fileName, dpi=fig.dpi)
class EpsPlot(OutStream):
  def __init__(self):
    OutStream.__init__(self)

  def finalize(self):
    try:
      self.retrieveHistories()
    except:
      OutStream.retrieveHistories(self)
    headers = self.histories[self.alreadyRead[0]][1]['headers']
    timeVar = ''
    for i in xrange(len(headers)):
      if headers[i].lower() == 'time':
        #timeVar = headers.pop(i)
        timeLoc = i
        break

    for index in xrange(len(headers)):
      if headers[index].lower() != 'time':
        if not self.variables[0]=='all':
          if headers[index] in self.variables:
            plot_it = True
          else:
            plot_it = False
        else:
          plot_it = True
        if plot_it:
          plt.figure()
          plt.xlabel(headers[timeLoc])
          plt.ylabel(headers[index])
          plt.title('Plot of histories')
          for key in self.histories:
            plt.plot(self.histories[key][0][:,timeLoc],self.histories[key][0][:,index])
        fileName = self.fileNameRoot + '.eps'
        fig.savefig(fileName, dpi=fig.dpi)

def returnInstance(Type):
  base = 'OutStream'
  InterfaceDict = {}
  InterfaceDict['Screen'   ]    = ScreenPlot
  InterfaceDict['Pdf'      ]    = PdfPlot
  InterfaceDict['Png'      ]    = PngPlot
  InterfaceDict['Jpeg'     ]    = JpegPlot
  InterfaceDict['Eps'      ]    = EpsPlot
  try:
    if Type in InterfaceDict.keys():
      return InterfaceDict[Type]()
  except:
    raise NameError('not known '+base+' type'+Type)<|endoftext|>


<commit_before>from datetime import datetime
from dbaas.celery import app
import models
import logging
from notification.models import TaskHistory
from util import exec_remote_command_host, get_worker_name, \
    build_context_script, get_dict_lines
from registered_functions.functools import get_function
from workflow.steps.util.dns import ChangeTTLTo5Minutes, ChangeTTLTo3Hours
from workflow.steps.util.db_monitor import DisableMonitoring, EnableMonitoring
from workflow.steps.util.zabbix import DisableAlarms, EnableAlarms
from workflow.workflow import steps_for_instances

LOG = logging.getLogger(__name__)

@app.task(bind=True)
def execute_scheduled_maintenance(self, maintenance_id):
    LOG.debug("Maintenance id: {}".format(maintenance_id))
    maintenance = models.Maintenance.objects.get(id=maintenance_id)
    models.Maintenance.objects.filter(id=maintenance_id).update(
        status=maintenance.RUNNING, started_at=datetime.now()
    )
    LOG.info("Maintenance {} is RUNNING".format(maintenance,))

    worker_name = get_worker_name()
    task_history = TaskHistory.register(
        request=self.request, worker_name=worker_name
    )
    task_history.relevance = TaskHistory.RELEVANCE_CRITICAL
    LOG.info("id: {} | task: {} | kwargs: {} | args: {}".format(
        self.request.id, self.request.task,
        self.request.kwargs, str(self.request.args)
    ))
    task_history.update_details(
        persist=True, details="Executing Maintenance: {}".format(maintenance)
    )
    for hm in models.HostMaintenance.objects.filter(maintenance=maintenance):
        main_output = {}
        hm.status = hm.RUNNING
        hm.started_at = datetime.now()
        hm.save()
        if hm.host is None:
            hm.status = hm.UNAVAILABLEHOST
            hm.finished_at = datetime.now()
            hm.save()
            continue

        host = hm.host
        update_task = "\nRunning Maintenance on {}".format(host)

        if maintenance.disable_alarms:
            disable_alarms(hm.host)

        param_dict = {}
        params = models.MaintenanceParameters.objects.filter(
            maintenance=maintenance
        )
        for param in params:
            param_function = get_function(param.function_name)
            param_dict[param.parameter_name] = param_function(host.id)

        main_script = build_context_script(param_dict, maintenance.main_script)
        exit_status = exec_remote_command_host(host, main_script, main_output)

        if exit_status == 0:
            hm.status = hm.SUCCESS
        else:

            if maintenance.rollback_script:
                rollback_output = {}
                hm.status = hm.ROLLBACK
                hm.save()

                rollback_script = build_context_script(
                    param_dict, maintenance.rollback_script
                )
                exit_status = exec_remote_command_host(
                    host, rollback_script, rollback_output
                )

                if exit_status == 0:
                    hm.status = hm.ROLLBACK_SUCCESS
                else:
                    hm.status = hm.ROLLBACK_ERROR

                hm.rollback_log = get_dict_lines(rollback_output)

            else:
                hm.status = hm.ERROR

        if maintenance.disable_alarms:
            enable_alarms(hm.host)

        update_task += "...status: {}".format(hm.status)

        task_history.update_details(persist=True, details=update_task)

        hm.main_log = get_dict_lines(main_output)
        hm.finished_at = datetime.now()
        hm.save()

    models.Maintenance.objects.filter(id=maintenance_id).update(
        status=maintenance.FINISHED, finished_at=datetime.now()
    )
    task_history.update_status_for(
        TaskHistory.STATUS_SUCCESS, details='Maintenance executed succesfully'
    )
    LOG.info("Maintenance: {} has FINISHED".format(maintenance))

def disable_alarms(host):
    for instance in host.instances.all():
        DisableMonitoring(instance).do()
        DisableAlarms(instance).do()

def enable_alarms(host):
    for instance in host.instances.all():
        EnableMonitoring(instance).do()
        EnableAlarms(instance).do()

def region_migration_prepare(infra):
    instance = infra.instances.first()
    ChangeTTLTo5Minutes(instance).do()

def region_migration_finish(infra):
    instance = infra.instances.first()
    ChangeTTLTo3Hours(instance).do()

@app.task(bind=True)
def region_migration_start(self, infra, instances, since_step=None):
    steps = [{
        'Disable monitoring and alarms': (
            'workflow.steps.util.zabbix.DestroyAlarms',
            'workflow.steps.util.db_monitor.DisableMonitoring',
        )}, {
        'Stopping infra': (
            'workflow.steps.util.database.Stop',
            'workflow.steps.util.database.CheckIsDown',
        )}, {
        'Creating new virtual machine': (
            'workflow.steps.util.vm.MigrationCreateNewVM',
        )}, {
        'Creating new infra': (
            'workflow.steps.util.vm.MigrationWaitingBeReady',
            'workflow.steps.util.infra.MigrationCreateInstance',
            'workflow.steps.util.disk.MigrationCreateExport',
        )}, {
        'Configuring new infra': (
            'workflow.steps.util.volume_provider.MountDataVolume',
            'workflow.steps.util.plan.InitializationMigration',
            'workflow.steps.util.plan.ConfigureMigration',
            'workflow.steps.util.metric_collector.ConfigureTelegraf',
        )}, {
        'Preparing new environment': (
            'workflow.steps.util.disk.AddDiskPermissionsOldest',
            'workflow.steps.util.disk.MountOldestExportMigration',
            'workflow.steps.util.disk.CopyDataBetweenExportsMigration',
            'workflow.steps.util.disk.FilePermissionsMigration',
            'workflow.steps.util.disk.UnmountNewerExportMigration',
            'workflow.steps.util.vm.ChangeInstanceHost',
            'workflow.steps.util.vm.UpdateOSDescription',
            'workflow.steps.util.infra.OfferingMigration',
            'workflow.steps.util.infra.UpdateMigrateEnvironment',
            'workflow.steps.util.infra.UpdateMigratePlan',
        )}, {
        'Starting new infra': (
            'workflow.steps.util.database.Start',
            'workflow.steps.util.database.CheckIsUp',
            'workflow.steps.util.metric_collector.RestartTelegraf',
        )}, {
        'Enabling access': (
            'workflow.steps.util.dns.ChangeEndpoint',
            'workflow.steps.util.acl.ReplicateAclsMigration',
        )}, {
        'Destroying old infra': (
            'workflow.steps.util.disk.DisableOldestExportMigration',
            'workflow.steps.util.disk.DiskUpdateHost',
            'workflow.steps.util.vm.RemoveHostMigration',
        )}, {
        'Enabling monitoring and alarms': (
            'workflow.steps.util.db_monitor.EnableMonitoring',
            'workflow.steps.util.zabbix.CreateAlarms',
        )}, {
        'Restart replication': (
            'workflow.steps.util.database.SetSlavesMigration',
        )
    }]

    task = TaskHistory()
    task.task_id = self.request.id
    task.task_name = "migrating_zone"
    task.task_status = TaskHistory.STATUS_RUNNING
    task.context = {'infra': infra, 'instances': instances}
    task.arguments = {'infra': infra, 'instances': instances}
    task.user = 'admin'
    task.save()

    if steps_for_instances(steps, instances, task, since_step=since_step):
        task.set_status_success('Region migrated with success')
    else:
        task.set_status_error('Could not migrate region')

    database = infra.databases.first()
    database.environment = infra.environment
    database.save()

@app.task(bind=True)
def create_database(
    self, name, plan, environment, team, project, description, task,
    subscribe_to_email_events=True, is_protected=False, user=None,
    retry_from=None
):
    task = TaskHistory.register(
        request=self.request, task_history=task, user=user,
        worker_name=get_worker_name()
    )

    from tasks_create_database import create_database
    create_database(
        name, plan, environment, team, project, description, task,
        subscribe_to_email_events, is_protected, user, retry_from
    )

@app.task(bind=True)
def restore_database(self, database, task, snapshot, user, retry_from=None):
    task = TaskHistory.register(
        request=self.request, task_history=task, user=user,
        worker_name=get_worker_name()
    )

    from backup.models import Snapshot
    snapshot = Snapshot.objects.get(id=snapshot)

    from tasks_restore_backup import restore_snapshot
    restore_snapshot(database, snapshot.group, task, retry_from)

def _create_database_rollback(self, rollback_from, task, user):
    task = TaskHistory.register(
        request=self.request, task_history=task, user=user,
        worker_name=get_worker_name()
    )

    from tasks_create_database import rollback_create
    rollback_create(rollback_from, task, user)

@app.task(bind=True)
def create_database_rollback(self, rollback_from, task, user):
    _create_database_rollback(self, rollback_from, task, user)

@app.task(bind=True)
def node_zone_migrate(
    self, host, zone, new_environment, task, since_step=None, step_manager=None
):
    task = TaskHistory.register(
        request=self.request, task_history=task, user=task.user,
        worker_name=get_worker_name()
    )
    from tasks_migrate import node_zone_migrate
    node_zone_migrate(host, zone, new_environment, task, since_step, step_manager=step_manager)

@app.task(bind=True)
def node_zone_migrate_rollback(self, migrate, task):
    task = TaskHistory.register(
        request=self.request, task_history=task, user=task.user,
        worker_name=get_worker_name()
    )
    from tasks_migrate import rollback_node_zone_migrate
    rollback_node_zone_migrate(migrate, task)

@app.task(bind=True)
def database_environment_migrate(
    self, database, new_environment, new_offering, task, hosts_zones,
    since_step=None
):
    task = TaskHistory.register(
        request=self.request, task_history=task, user=task.user,
        worker_name=get_worker_name()
    )
    from tasks_database_migrate import database_environment_migrate
    database_environment_migrate(
        database, new_environment, new_offering, task, hosts_zones, since_step
    )

@app.task(bind=True)
def database_environment_migrate_rollback(self, migrate, task):
    task = TaskHistory.register(
        request=self.request, task_history=task, user=task.user,
        worker_name=get_worker_name()
    )
    from tasks_database_migrate import rollback_database_environment_migrate
    rollback_database_environment_migrate(migrate, task)
<commit_msg>Add step manager<commit_after>from datetime import datetime
from dbaas.celery import app
import models
import logging
from notification.models import TaskHistory
from util import exec_remote_command_host, get_worker_name, \
    build_context_script, get_dict_lines
from registered_functions.functools import get_function
from workflow.steps.util.dns import ChangeTTLTo5Minutes, ChangeTTLTo3Hours
from workflow.steps.util.db_monitor import DisableMonitoring, EnableMonitoring
from workflow.steps.util.zabbix import DisableAlarms, EnableAlarms
from workflow.workflow import steps_for_instances

LOG = logging.getLogger(__name__)

@app.task(bind=True)
def execute_scheduled_maintenance(self, maintenance_id):
    LOG.debug("Maintenance id: {}".format(maintenance_id))
    maintenance = models.Maintenance.objects.get(id=maintenance_id)
    models.Maintenance.objects.filter(id=maintenance_id).update(
        status=maintenance.RUNNING, started_at=datetime.now()
    )
    LOG.info("Maintenance {} is RUNNING".format(maintenance,))

    worker_name = get_worker_name()
    task_history = TaskHistory.register(
        request=self.request, worker_name=worker_name
    )
    task_history.relevance = TaskHistory.RELEVANCE_CRITICAL
    LOG.info("id: {} | task: {} | kwargs: {} | args: {}".format(
        self.request.id, self.request.task,
        self.request.kwargs, str(self.request.args)
    ))
    task_history.update_details(
        persist=True, details="Executing Maintenance: {}".format(maintenance)
    )
    for hm in models.HostMaintenance.objects.filter(maintenance=maintenance):
        main_output = {}
        hm.status = hm.RUNNING
        hm.started_at = datetime.now()
        hm.save()
        if hm.host is None:
            hm.status = hm.UNAVAILABLEHOST
            hm.finished_at = datetime.now()
            hm.save()
            continue

        host = hm.host
        update_task = "\nRunning Maintenance on {}".format(host)

        if maintenance.disable_alarms:
            disable_alarms(hm.host)

        param_dict = {}
        params = models.MaintenanceParameters.objects.filter(
            maintenance=maintenance
        )
        for param in params:
            param_function = get_function(param.function_name)
            param_dict[param.parameter_name] = param_function(host.id)

        main_script = build_context_script(param_dict, maintenance.main_script)
        exit_status = exec_remote_command_host(host, main_script, main_output)

        if exit_status == 0:
            hm.status = hm.SUCCESS
        else:

            if maintenance.rollback_script:
                rollback_output = {}
                hm.status = hm.ROLLBACK
                hm.save()

                rollback_script = build_context_script(
                    param_dict, maintenance.rollback_script
                )
                exit_status = exec_remote_command_host(
                    host, rollback_script, rollback_output
                )

                if exit_status == 0:
                    hm.status = hm.ROLLBACK_SUCCESS
                else:
                    hm.status = hm.ROLLBACK_ERROR

                hm.rollback_log = get_dict_lines(rollback_output)

            else:
                hm.status = hm.ERROR

        if maintenance.disable_alarms:
            enable_alarms(hm.host)

        update_task += "...status: {}".format(hm.status)

        task_history.update_details(persist=True, details=update_task)

        hm.main_log = get_dict_lines(main_output)
        hm.finished_at = datetime.now()
        hm.save()

    models.Maintenance.objects.filter(id=maintenance_id).update(
        status=maintenance.FINISHED, finished_at=datetime.now()
    )
    task_history.update_status_for(
        TaskHistory.STATUS_SUCCESS, details='Maintenance executed succesfully'
    )
    LOG.info("Maintenance: {} has FINISHED".format(maintenance))

def disable_alarms(host):
    for instance in host.instances.all():
        DisableMonitoring(instance).do()
        DisableAlarms(instance).do()

def enable_alarms(host):
    for instance in host.instances.all():
        EnableMonitoring(instance).do()
        EnableAlarms(instance).do()

def region_migration_prepare(infra):
    instance = infra.instances.first()
    ChangeTTLTo5Minutes(instance).do()

def region_migration_finish(infra):
    instance = infra.instances.first()
    ChangeTTLTo3Hours(instance).do()

@app.task(bind=True)
def region_migration_start(self, infra, instances, since_step=None):
    steps = [{
        'Disable monitoring and alarms': (
            'workflow.steps.util.zabbix.DestroyAlarms',
            'workflow.steps.util.db_monitor.DisableMonitoring',
        )}, {
        'Stopping infra': (
            'workflow.steps.util.database.Stop',
            'workflow.steps.util.database.CheckIsDown',
        )}, {
        'Creating new virtual machine': (
            'workflow.steps.util.vm.MigrationCreateNewVM',
        )}, {
        'Creating new infra': (
            'workflow.steps.util.vm.MigrationWaitingBeReady',
            'workflow.steps.util.infra.MigrationCreateInstance',
            'workflow.steps.util.disk.MigrationCreateExport',
        )}, {
        'Configuring new infra': (
            'workflow.steps.util.volume_provider.MountDataVolume',
            'workflow.steps.util.plan.InitializationMigration',
            'workflow.steps.util.plan.ConfigureMigration',
            'workflow.steps.util.metric_collector.ConfigureTelegraf',
        )}, {
        'Preparing new environment': (
            'workflow.steps.util.disk.AddDiskPermissionsOldest',
            'workflow.steps.util.disk.MountOldestExportMigration',
            'workflow.steps.util.disk.CopyDataBetweenExportsMigration',
            'workflow.steps.util.disk.FilePermissionsMigration',
            'workflow.steps.util.disk.UnmountNewerExportMigration',
            'workflow.steps.util.vm.ChangeInstanceHost',
            'workflow.steps.util.vm.UpdateOSDescription',
            'workflow.steps.util.infra.OfferingMigration',
            'workflow.steps.util.infra.UpdateMigrateEnvironment',
            'workflow.steps.util.infra.UpdateMigratePlan',
        )}, {
        'Starting new infra': (
            'workflow.steps.util.database.Start',
            'workflow.steps.util.database.CheckIsUp',
            'workflow.steps.util.metric_collector.RestartTelegraf',
        )}, {
        'Enabling access': (
            'workflow.steps.util.dns.ChangeEndpoint',
            'workflow.steps.util.acl.ReplicateAclsMigration',
        )}, {
        'Destroying old infra': (
            'workflow.steps.util.disk.DisableOldestExportMigration',
            'workflow.steps.util.disk.DiskUpdateHost',
            'workflow.steps.util.vm.RemoveHostMigration',
        )}, {
        'Enabling monitoring and alarms': (
            'workflow.steps.util.db_monitor.EnableMonitoring',
            'workflow.steps.util.zabbix.CreateAlarms',
        )}, {
        'Restart replication': (
            'workflow.steps.util.database.SetSlavesMigration',
        )
    }]

    task = TaskHistory()
    task.task_id = self.request.id
    task.task_name = "migrating_zone"
    task.task_status = TaskHistory.STATUS_RUNNING
    task.context = {'infra': infra, 'instances': instances}
    task.arguments = {'infra': infra, 'instances': instances}
    task.user = 'admin'
    task.save()

    if steps_for_instances(steps, instances, task, since_step=since_step):
        task.set_status_success('Region migrated with success')
    else:
        task.set_status_error('Could not migrate region')

    database = infra.databases.first()
    database.environment = infra.environment
    database.save()

@app.task(bind=True)
def create_database(
    self, name, plan, environment, team, project, description, task,
    subscribe_to_email_events=True, is_protected=False, user=None,
    retry_from=None
):
    task = TaskHistory.register(
        request=self.request, task_history=task, user=user,
        worker_name=get_worker_name()
    )

    from tasks_create_database import create_database
    create_database(
        name, plan, environment, team, project, description, task,
        subscribe_to_email_events, is_protected, user, retry_from
    )

@app.task(bind=True)
def restore_database(self, database, task, snapshot, user, retry_from=None):
    task = TaskHistory.register(
        request=self.request, task_history=task, user=user,
        worker_name=get_worker_name()
    )

    from backup.models import Snapshot
    snapshot = Snapshot.objects.get(id=snapshot)

    from tasks_restore_backup import restore_snapshot
    restore_snapshot(database, snapshot.group, task, retry_from)

def _create_database_rollback(self, rollback_from, task, user):
    task = TaskHistory.register(
        request=self.request, task_history=task, user=user,
        worker_name=get_worker_name()
    )

    from tasks_create_database import rollback_create
    rollback_create(rollback_from, task, user)

@app.task(bind=True)
def create_database_rollback(self, rollback_from, task, user):
    _create_database_rollback(self, rollback_from, task, user)

@app.task(bind=True)
def node_zone_migrate(
    self, host, zone, new_environment, task, since_step=None, step_manager=None
):
    task = TaskHistory.register(
        request=self.request, task_history=task, user=task.user,
        worker_name=get_worker_name()
    )
    from tasks_migrate import node_zone_migrate
    node_zone_migrate(host, zone, new_environment, task, since_step, step_manager=step_manager)

@app.task(bind=True)
def node_zone_migrate_rollback(self, migrate, task):
    task = TaskHistory.register(
        request=self.request, task_history=task, user=task.user,
        worker_name=get_worker_name()
    )
    from tasks_migrate import rollback_node_zone_migrate
    rollback_node_zone_migrate(migrate, task)

@app.task(bind=True)
def database_environment_migrate(
    self, database, new_environment, new_offering, task, hosts_zones,
    since_step=None, step_manager=None
):
    task = TaskHistory.register(
        request=self.request, task_history=task, user=task.user,
        worker_name=get_worker_name()
    )
    from tasks_database_migrate import database_environment_migrate
    database_environment_migrate(
        database, new_environment, new_offering, task, hosts_zones, since_step,
        step_manager=step_manager
    )

@app.task(bind=True)
def database_environment_migrate_rollback(self, migrate, task):
    task = TaskHistory.register(
        request=self.request, task_history=task, user=task.user,
        worker_name=get_worker_name()
    )
    from tasks_database_migrate import rollback_database_environment_migrate
    rollback_database_environment_migrate(migrate, task)
<|endoftext|>