<commit_before># coding=utf-8

"""This module, server_scripts.py, manages the server scripts directory for the QuasarSource project."""

from code_api.discrete_projects.predefined_code.shell_code.safety_checks import *
from code_api.discrete_projects.predefined_code.shell_code.variable_setters import *
from code_api.source_file_abstraction.code_files.shell_file import *
from code_api.source_file_abstraction.code_directories.shell_directory import ShellDirectory
from code_api.source_file_abstraction.code_files.shell_file import ShellFile

from universal_code.useful_file_operations import get_ini_section_dictionary as config
from universal_code import path_manager as pm

def ini(section):
	"""Utility function."""
	return config(pm.PATH_TO_CONFIG_FILE, section)

VALS = ini('quasar')
QUASAR_SOURCE      = VALS['path_to_quasar_source']
IS_PROGRAM_RUNNING = VALS['path_server_is_program_running']

# Django server.
DJANGO_MANAGE = VALS['path_server_django_manage']
TERMINATE_SCRIPT_PATH_QUASAR = VALS['path_server_quasar_terminate']
RUN_IN_BACKGROUND_SCRIPT_PATH_QUASAR = VALS['path_server_quasar_run_in_background']
_RUN_QUASAR = ["sudo -E python3 ${APPLICATION_PATH} migrate", "\tsudo -E nohup python3 ${APPLICATION_PATH} runserver 0:80 &"]
_RUN_QUASAR_LIVE = ["sudo -E python3 ${APPLICATION_PATH} migrate", "\tsudo -E python3 ${APPLICATION_PATH} runserver 0:80"]

# Entity server.
ENTITY_SERVER = VALS['entity_server']
TERMINATE_SCRIPT_PATH_ENTITY = VALS['path_server_entity_terminate']
RUN_IN_BACKGROUND_SCRIPT_PATH_ENTITY = VALS['path_server_entity_run_in_background']
_RUN_ENTITY = "nohup python3 ${APPLICATION_PATH} -r &"
_RUN_ENTITY_LIVE = "python3 ${APPLICATION_PATH} -r"

def load_server_scripts():
	"""Generates the server scripts."""
	directory_server = CodeDirectory('/Users/utarsuno/git_repos/quasar_source/configuration_files/server_scripts')
	directory_server_quasar = directory_server.add_new_child_code_directory_from_current_path('quasar')
	directory_server_entity = directory_server.add_new_child_code_directory_from_current_path('entity')

	load_specific_server_application_scripts(directory_server_quasar, 'quasar', DJANGO_MANAGE, TERMINATE_SCRIPT_PATH_QUASAR, RUN_IN_BACKGROUND_SCRIPT_PATH_QUASAR, _RUN_QUASAR, _RUN_QUASAR_LIVE)
	load_specific_server_application_scripts(directory_server_entity, 'entity', ENTITY_SERVER, TERMINATE_SCRIPT_PATH_ENTITY, RUN_IN_BACKGROUND_SCRIPT_PATH_ENTITY, _RUN_ENTITY, _RUN_ENTITY_LIVE)

	directory_server.add_code_file(add_update_code_script())
	directory_server.add_code_file(add_health_check_script())

	return directory_server

def add_health_check_script():
	"""Adds a script to ensure Pymongo is running."""
	s = ShellFile('health_check')
	s.set_to_simple_shell_file()
	code = '''
export PYTHONPATH=${PATH_TO_QUASAR_SOURCE}

PYMONGO_PID=`python3 ${PATH_TO_IS_PROGRAM_RUNNING} mongod`

if [ ${PYMONGO_PID} -eq -1 ]; then
	sudo service mongod start
fi
'''
	code = code.replace('${PATH_TO_QUASAR_SOURCE}', QUASAR_SOURCE)
	code = code.replace('${PATH_TO_IS_PROGRAM_RUNNING}', IS_PROGRAM_RUNNING)
	s.set_main_code(CodeChunk(code))
	return s

def add_update_code_script():
	"""Adds the update code for the server script."""
	s = ShellFile('update_code')
	s.set_to_simple_shell_file()
	s.set_main_code(CodeChunk('''
# Go to the projects base directory.
cd ${PATH_TO_QUASAR_SOURCE};
git fetch --all;
reslog=$(git log HEAD..origin/master --oneline)
if [[ "${reslog}" != "" ]] ; then
    # This resets to master.
    git reset --hard origin/master;
fi
'''.replace('${PATH_TO_QUASAR_SOURCE}', QUASAR_SOURCE)))
	return s

def load_specific_server_application_scripts(directory, application_name, application_path, terminate_script_path, run_in_background_script_path, run_code, run_code_live):
	"""Generates a set of scripts to manage a specific server application."""
	terminate         = get_terminate_script(application_path)
	run_in_background = get_run_in_background_script(application_name, application_path, run_code)
	restart           = get_restart_script(terminate_script_path, run_in_background_script_path)
	live_run          = get_live_run_script(application_name, application_path, run_code_live)

	directory.add_code_file(terminate)
	directory.add_code_file(run_in_background)
	directory.add_code_file(restart)
	directory.add_code_file(live_run)

def get_live_run_script(application_name, application_path, run_code_live):
	"""Returns the server application live run script."""
	s = ShellFile('live_run')
	s.set_to_simple_shell_file()
	code = '''
export PYTHONPATH=${PATH_TO_QUASAR_SOURCE}

APPLICATION_PID=`python3 ${PATH_TO_IS_PROGRAM_RUNNING} ${APPLICATION_PATH}`

if [ ${APPLICATION_PID} -gt -1 ]; then
	echo '${APPLICATION} server is already running!'
else
	${RUN_CODE_LIVE}
fi
'''
	code = code.replace('${PATH_TO_QUASAR_SOURCE}', QUASAR_SOURCE)
	code = code.replace('${APPLICATION}', application_name)
	code = code.replace('${PATH_TO_IS_PROGRAM_RUNNING}', IS_PROGRAM_RUNNING)
	if type(run_code_live) == list:
		code = code.replace('${RUN_CODE_LIVE}', run_code_live[0] + '\n' + run_code_live[1] + '\n')
	else:
		code = code.replace('${RUN_CODE_LIVE}', run_code_live + '\n')
	code = code.replace('${APPLICATION_PATH}', application_path)
	s.set_main_code(CodeChunk(code))
	return s

def get_restart_script(terminate_script_path, run_in_background_script_path):
	"""Returns the server application restart script."""
	s = ShellFile('restart')
	s.set_to_simple_shell_file()
	s.set_main_code(CodeChunk('''
sudo nohup ${TERMINATE_SCRIPT_PATH} &
sudo nohup ${RUN_IN_BACKGROUND_SCRIPT_PATH} &
'''.replace('${TERMINATE_SCRIPT_PATH}', terminate_script_path).replace('${RUN_IN_BACKGROUND_SCRIPT_PATH}', run_in_background_script_path)))
	return s

def get_run_in_background_script(application_name, application_path, run_code):
	"""Returns the server application run in background script."""
	s = ShellFile('run_in_background')
	s.set_to_simple_shell_file()
	code = '''
export PYTHONPATH=${PATH_TO_QUASAR_SOURCE}

APPLICATION_PID=`python3 ${PATH_TO_IS_PROGRAM_RUNNING} ${APPLICATION_PATH}`

if [ ${APPLICATION_PID} -gt -1 ]; then
	echo '${APPLICATION} server is already running!'
else
	${RUN_CODE}
fi
'''
	code = code.replace('${PATH_TO_QUASAR_SOURCE}', QUASAR_SOURCE)
	code = code.replace('${APPLICATION}', application_name)
	code = code.replace('${PATH_TO_IS_PROGRAM_RUNNING}', IS_PROGRAM_RUNNING)
	if type(run_code) == list:
		code = code.replace('${RUN_CODE}', run_code[0] + '\n' + run_code[1] + '\n')
	else:
		code = code.replace('${RUN_CODE}', run_code + '\n')
	code = code.replace('${APPLICATION_PATH}', application_path)
	s.set_main_code(CodeChunk(code))
	return s

def get_terminate_script(application_path):
	"""Returns the server application terminate script."""
	s = ShellFile('terminate')
	s.set_to_simple_shell_file()
	s.set_main_code(CodeChunk('''
APPLICATION_PID=`python3 ${PATH_TO_IS_PROGRAM_RUNNING} ${APPLICATION_PATH}`

if [ ${APPLICATION_PID} -gt -1 ]; then
	sudo kill ${APPLICATION_PID}
fi
'''.replace('${APPLICATION_PATH}', application_path).replace('${PATH_TO_IS_PROGRAM_RUNNING}', IS_PROGRAM_RUNNING)))
	return s
<commit_msg>fixing scripts 20<commit_after># coding=utf-8

"""This module, server_scripts.py, manages the server scripts directory for the QuasarSource project."""

from code_api.discrete_projects.predefined_code.shell_code.safety_checks import *
from code_api.discrete_projects.predefined_code.shell_code.variable_setters import *
from code_api.source_file_abstraction.code_files.shell_file import *
from code_api.source_file_abstraction.code_directories.shell_directory import ShellDirectory
from code_api.source_file_abstraction.code_files.shell_file import ShellFile

from universal_code.useful_file_operations import get_ini_section_dictionary as config
from universal_code import path_manager as pm

def ini(section):
	"""Utility function."""
	return config(pm.PATH_TO_CONFIG_FILE, section)

VALS = ini('quasar')
QUASAR_SOURCE      = VALS['path_to_quasar_source']
IS_PROGRAM_RUNNING = VALS['path_server_is_program_running']

# Django server.
DJANGO_MANAGE = VALS['path_server_django_manage']
TERMINATE_SCRIPT_PATH_QUASAR = VALS['path_server_quasar_terminate']
RUN_IN_BACKGROUND_SCRIPT_PATH_QUASAR = VALS['path_server_quasar_run_in_background']
_RUN_QUASAR = ["sudo -E python3 ${APPLICATION_PATH} migrate", "\tsudo -E nohup python3 ${APPLICATION_PATH} runserver 0:80 &"]
_RUN_QUASAR_LIVE = ["sudo -E python3 ${APPLICATION_PATH} migrate", "\tsudo -E python3 ${APPLICATION_PATH} runserver 0:80"]

# Entity server.
ENTITY_SERVER = VALS['entity_server']
TERMINATE_SCRIPT_PATH_ENTITY = VALS['path_server_entity_terminate']
RUN_IN_BACKGROUND_SCRIPT_PATH_ENTITY = VALS['path_server_entity_run_in_background']
_RUN_ENTITY = "nohup python3 ${APPLICATION_PATH} -r &"
_RUN_ENTITY_LIVE = "python3 ${APPLICATION_PATH} -r"

def load_server_scripts():
	"""Generates the server scripts."""
	directory_server = CodeDirectory('/Users/utarsuno/git_repos/quasar_source/configuration_files/server_scripts')
	directory_server_quasar = directory_server.add_new_child_code_directory_from_current_path('quasar')
	directory_server_entity = directory_server.add_new_child_code_directory_from_current_path('entity')

	load_specific_server_application_scripts(directory_server_quasar, 'quasar', DJANGO_MANAGE, TERMINATE_SCRIPT_PATH_QUASAR, RUN_IN_BACKGROUND_SCRIPT_PATH_QUASAR, _RUN_QUASAR, _RUN_QUASAR_LIVE)
	load_specific_server_application_scripts(directory_server_entity, 'entity', ENTITY_SERVER, TERMINATE_SCRIPT_PATH_ENTITY, RUN_IN_BACKGROUND_SCRIPT_PATH_ENTITY, _RUN_ENTITY, _RUN_ENTITY_LIVE)

	directory_server.add_code_file(add_update_code_script())
	directory_server.add_code_file(add_health_check_script())

	return directory_server

def add_health_check_script():
	"""Adds a script to ensure Pymongo is running."""
	s = ShellFile('health_check')
	s.set_to_simple_shell_file()
	code = '''
export PYTHONPATH=${PATH_TO_QUASAR_SOURCE}

PYMONGO_PID=`python3 ${PATH_TO_IS_PROGRAM_RUNNING} mongod`

if [ ${PYMONGO_PID} -eq -1 ]; then
	sudo service mongod start
fi
'''
	code = code.replace('${PATH_TO_QUASAR_SOURCE}', QUASAR_SOURCE)
	code = code.replace('${PATH_TO_IS_PROGRAM_RUNNING}', IS_PROGRAM_RUNNING)
	s.set_main_code(CodeChunk(code))
	return s

def add_update_code_script():
	"""Adds the update code for the server script."""
	s = ShellFile('update_code')
	s.set_to_simple_shell_file()
	s.set_main_code(CodeChunk('''
# Go to the projects base directory.
cd ${PATH_TO_QUASAR_SOURCE};
git fetch --all;
reslog=$(git log HEAD..origin/master --oneline)
if [[ "${reslog}" != "" ]] ; then
    # This resets to master.
    git reset --hard origin/master;
fi
'''.replace('${PATH_TO_QUASAR_SOURCE}', QUASAR_SOURCE)))
	return s

def load_specific_server_application_scripts(directory, application_name, application_path, terminate_script_path, run_in_background_script_path, run_code, run_code_live):
	"""Generates a set of scripts to manage a specific server application."""
	terminate         = get_terminate_script(application_path)
	run_in_background = get_run_in_background_script(application_name, application_path, run_code)
	restart           = get_restart_script(terminate_script_path, run_in_background_script_path)
	live_run          = get_live_run_script(application_name, application_path, run_code_live)

	directory.add_code_file(terminate)
	directory.add_code_file(run_in_background)
	directory.add_code_file(restart)
	directory.add_code_file(live_run)

def get_live_run_script(application_name, application_path, run_code_live):
	"""Returns the server application live run script."""
	s = ShellFile('live_run')
	s.set_to_simple_shell_file()
	code = '''
export PYTHONPATH=${PATH_TO_QUASAR_SOURCE}

APPLICATION_PID=`python3 ${PATH_TO_IS_PROGRAM_RUNNING} ${APPLICATION_PATH}`

if [ ${APPLICATION_PID} -gt -1 ]; then
	echo '${APPLICATION} server is already running!'
else
	${RUN_CODE_LIVE}
fi
'''
	code = code.replace('${PATH_TO_QUASAR_SOURCE}', QUASAR_SOURCE)
	code = code.replace('${APPLICATION}', application_name)
	code = code.replace('${PATH_TO_IS_PROGRAM_RUNNING}', IS_PROGRAM_RUNNING)
	if type(run_code_live) == list:
		code = code.replace('${RUN_CODE_LIVE}', run_code_live[0] + '\n' + run_code_live[1] + '\n')
	else:
		code = code.replace('${RUN_CODE_LIVE}', run_code_live + '\n')
	code = code.replace('${APPLICATION_PATH}', application_path)
	s.set_main_code(CodeChunk(code))
	return s

def get_restart_script(terminate_script_path, run_in_background_script_path):
	"""Returns the server application restart script."""
	s = ShellFile('restart')
	s.set_to_simple_shell_file()
	s.set_main_code(CodeChunk('''
sudo nohup ${TERMINATE_SCRIPT_PATH} &
sudo nohup ${RUN_IN_BACKGROUND_SCRIPT_PATH} &
'''.replace('${TERMINATE_SCRIPT_PATH}', terminate_script_path).replace('${RUN_IN_BACKGROUND_SCRIPT_PATH}', run_in_background_script_path)))
	return s

def get_run_in_background_script(application_name, application_path, run_code):
	"""Returns the server application run in background script."""
	s = ShellFile('run_in_background')
	s.set_to_simple_shell_file()
	code = '''
export PYTHONPATH=${PATH_TO_QUASAR_SOURCE}

APPLICATION_PID=`python3 ${PATH_TO_IS_PROGRAM_RUNNING} ${APPLICATION_PATH}`

if [ ${APPLICATION_PID} -gt -1 ]; then
	echo '${APPLICATION} server is already running!'
else
	export PYTHONPATH=${PATH_TO_QUASAR_SOURCE}
	${RUN_CODE}
fi
'''
	code = code.replace('${PATH_TO_QUASAR_SOURCE}', QUASAR_SOURCE)
	code = code.replace('${APPLICATION}', application_name)
	code = code.replace('${PATH_TO_IS_PROGRAM_RUNNING}', IS_PROGRAM_RUNNING)
	if type(run_code) == list:
		code = code.replace('${RUN_CODE}', run_code[0] + '\n' + run_code[1] + '\n')
	else:
		code = code.replace('${RUN_CODE}', run_code + '\n')
	code = code.replace('${APPLICATION_PATH}', application_path)
	s.set_main_code(CodeChunk(code))
	return s

def get_terminate_script(application_path):
	"""Returns the server application terminate script."""
	s = ShellFile('terminate')
	s.set_to_simple_shell_file()
	s.set_main_code(CodeChunk('''
APPLICATION_PID=`python3 ${PATH_TO_IS_PROGRAM_RUNNING} ${APPLICATION_PATH}`

if [ ${APPLICATION_PID} -gt -1 ]; then
	sudo kill ${APPLICATION_PID}
fi
'''.replace('${APPLICATION_PATH}', application_path).replace('${PATH_TO_IS_PROGRAM_RUNNING}', IS_PROGRAM_RUNNING)))
	return s
<|endoftext|>


<commit_before>"""Implementation of restricted boltzmann machine

You need to be able to deal with different energy functions

This allows you to deal with real valued unit

do updates in parallel using multiprocessing.pool

TODO: monitor overfitting
TODO: weight decay (to control overfitting and other things)
TODO: mean filed and dumped mean field

"""
import numpy as np
import math
# TODO: work out if you can use this somehow
import multiprocessing
import logging

# Do not print unless needed
logging.basicConfig(level=logging.CRITICAL)
# Global multiprocessing pool, used for all updates in the networks
pool = multiprocessing.Pool()

"""
 Represents a RBM
"""
class RBM(object):
  # 2 layers
  # weights
  # biases for each layer.
  # you can have a type of RBM that just extends the kind of functions
  # you use to update, depending on what kind of energy you use, which also
  # decides what kind of units you have
  # data is a numpy array:
  def __init__(self, data, nrHidden, trainingFunction):
    # Initialize weights to random
    assert len(data) !=0
    self.nrHidden = nrHidden
    self.nrVisible = len(data[0])
    self.data = data
    self.trainingFunction = trainingFunction
    self.weights = self.initializeWeights(self.nrVisible, self.nrHidden)
    self.biases = self.intializeBiases(data, self.nrHidden)

  # you need to take a training algorithm as a parameter (CD, PCD)
  def train(self):
    self.biases, self.weights = self.trainingFunction(self.data, self.biases, self.weights)

  def reconstruct(self, dataInstance):
    hidden = updateLayer(Layer.HIDDEN, dataInstance, self.biases, self.weights, True)
    visibleReconstruction = updateLayer(Layer.VISIBLE, hidden, self.biases, self.weights, False)
    return visibleReconstruction

  @classmethod
  def initializeWeights(cls, nrVisible, nrHidden):
    return np.random.normal(0, 0.01, (nrVisible, nrHidden))

  @classmethod
  def intializeBiases(cls, data, nrHidden):
    # get the procentage of data points that have the i'th unit on
    # and set the visible vias to log (p/(1-p))
    percentages = data.mean(axis=0, dtype='float') / len(data)
    vectorized = np.vectorize(safeLogFraction)
    visibleBiases = vectorized(percentages)

    # TODO: if sparse hiddeen weights, use that information
    hiddenBiases = np.zeros(nrHidden)
    return np.array([visibleBiases, hiddenBiases])

def safeLogFraction(p):
  assert p >=0 and p <= 1
  # TODO: think about this a bit better
  # you should not set them to be equal, on the contrary, they should be opposites
  if p * (1 - p) == 0:
    return 0
  return math.log(p / (1 -p))

# TODO: add momentum to learning
# TODO: different learning rates for weights and biases

""" Training functions."""

""" Full CD function.
Arguments:
  data: the data to use for traning.
    A numpy ndarray.
  biases:

Returns:
"""
def contrastiveDivergence(data, biases, weights, miniBatch=False):

  N = len(data)
  # Train the first 70 percent of the data with CD1
  endCD1 = math.floor(N / 10 * 7)
  cd1Data = data[0:endCD1]
  biases, weights = contrastiveDivergenceStep(cd1Data, biases, weights, cdSteps=1)

  # Train the next 20 percent with CD3
  endCD3 = math.floor(N / 10 * 2) + endCD1
  cd3Data = data[endCD1:endCD3]
  biases, weights = contrastiveDivergenceStep(cd3Data, biases, weights, cdSteps=3)

  # Train the next 10 percent of data with CD10
  cd5Data = data[endCD3:N]
  biases, weights = contrastiveDivergenceStep(cd5Data, biases, weights, cdSteps=5)

# Makes a step in the contrastiveDivergence algorithm
# online or with mini-bathces?
# you have multiple choices about how to implement this
# It is importaant that the hidden values from the data are binary,
# not probabilities
def contrastiveDivergenceStep(data, biases, weights, cdSteps=1):
  # TODO: do something smarter with the learning
  epsilon = 0.0001
  assert cdSteps >=1
  # Check that it does rows in loops
  for visible in data:
    # TODO: do CDn after some point
    # you can do it by calling the same function with the remaining data
    hidden = updateLayer(Layer.HIDDEN, visible, biases, weights, True)
    visibleReconstruction = updateLayer(Layer.VISIBLE, hidden, biases, weights, False)
    # TODO: consider this
    for i in xrange(cdSteps - 1):
      hiddenReconstruction = updateLayer(Layer.HIDDEN, visibleReconstruction, biases, weights, True)
      weights += epsilon * (np.outer(visible, hidden)
           - np.outer(visibleReconstruction, hiddenReconstruction))

    # Do the last reconstruction from the probabilities
    hiddenReconstruction = updateLayer(Layer.HIDDEN, visibleReconstruction, biases, weights, False)
    weights += epsilon * (np.outer(visible, hidden)
                - np.outer(visibleReconstruction, hiddenReconstruction))

    # Update the visible biases
    biases[0] += epsilon * (visible - visibleReconstruction)
    # Update the hidden biases
    biases[1] += epsilon * (hidden - hiddenReconstruction)
  return biases, weights

""" Updates an entire layer. This procedure can be used both in training
    and in testing.
"""
def updateLayer(layer, otherLayerValues, biases, weightMatrix, binary=False):
  bias = biases[layer]

  # logging.debug("updating layer " + str(layer))
  # logging.debug("with bias" + str(bias))

  # logging.debug("weights" + str(weightMatrix.shape))
  def activation(x):
    if layer == Layer.VISIBLE:
      w =  weightMatrix[x, :]
    else:
      w =  weightMatrix[:, x]

    return sigmoid(bias[x] + np.dot(w, otherLayerValues))
    # w = weightVectorForNeuron(layer, weightMatrix, x)
    # return activationProbability(activationSum(w, bias[x], otherLayerValues))

  probs = map(activation, xrange(weightMatrix.shape[layer]))
  probs = np.array(probs)

  if binary:
    # Sample from the distributions
    return sampleAll(probs)

  return probs

def weightVectorForNeuron(layer, weightMatrix, neuronNumber):
  if layer == Layer.VISIBLE:
    return weightMatrix[neuronNumber, :]
  # else layer == Layer.HIDDEN
  return weightMatrix[:, neuronNumber]

# TODO: check if you do it faster with matrix multiplication stuff
# but hinton was adamant about the paralell thing
def activationSum(weights, bias, otherLayerValues):
  # logging.debug("in activationSum")
  # logging.debug(otherLayerValues)
  # logging.debug(weights)

  return bias + np.dot(weights, otherLayerValues)

""" Gets the activation sums for all the units in one layer.
    Assumesthat the dimensions of the weihgt matrix and biases
    are given correctly. It will throw an exception otherwise.
"""

def activationProbability(activationSum):
  return sigmoid(activationSum)

# Another training algorithm. Slower than Contrastive divergence, but
# gives better results. Not used in practice as it is too slow.
def PCD():
  pass

""" general unitily functions"""

def sigmoid(x):
  return 1 / (1 + np.exp(-x));

def sample(p):
  return int(np.random.uniform() < p)

def sampleAll(probs):
  return np.random.uniform(size=probs.shape) < probs

def enum(**enums):
  return type('Enum', (), enums)

# Create an enum for visible and hidden, for
Layer = enum(VISIBLE=0, HIDDEN=1)

"""Main. """
def main():
  X = np.array([[1,1,1,0,0,0], # Training subset
                [1,0,1,0,0,0],
                [1,1,1,0,0,0],
                [0,0,1,1,1,0],
                [0,0,1,1,0,0],
                [0,0,1,1,1,0],

                [0,0,1,1,1,1], # Validation subset
                [1,1,1,0,0,0]])
  rbm = RBM(X, 3, contrastiveDivergence)
  rbm.train()

def rmse(prediction, actual):
  return np.linalg.norm(prediction - actual) / np.sqrt(len(prediction))

if __name__ == '__main__':
  main()<commit_msg>Fixed bug in which new contrastive divergence is not returning anything<commit_after>"""Implementation of restricted boltzmann machine

You need to be able to deal with different energy functions

This allows you to deal with real valued unit

do updates in parallel using multiprocessing.pool

TODO: monitor overfitting
TODO: weight decay (to control overfitting and other things)
TODO: mean filed and dumped mean field

"""
import numpy as np
import math
# TODO: work out if you can use this somehow
import multiprocessing
import logging

# Do not print unless needed
logging.basicConfig(level=logging.CRITICAL)
# Global multiprocessing pool, used for all updates in the networks
pool = multiprocessing.Pool()

"""
 Represents a RBM
"""
class RBM(object):
  # 2 layers
  # weights
  # biases for each layer.
  # you can have a type of RBM that just extends the kind of functions
  # you use to update, depending on what kind of energy you use, which also
  # decides what kind of units you have
  # data is a numpy array:
  def __init__(self, data, nrHidden, trainingFunction):
    # Initialize weights to random
    assert len(data) !=0
    self.nrHidden = nrHidden
    self.nrVisible = len(data[0])
    self.data = data
    self.trainingFunction = trainingFunction
    self.weights = self.initializeWeights(self.nrVisible, self.nrHidden)
    self.biases = self.intializeBiases(data, self.nrHidden)

  # you need to take a training algorithm as a parameter (CD, PCD)
  def train(self):
    self.biases, self.weights = self.trainingFunction(self.data, self.biases, self.weights)

  def reconstruct(self, dataInstance):
    hidden = updateLayer(Layer.HIDDEN, dataInstance, self.biases, self.weights, True)
    visibleReconstruction = updateLayer(Layer.VISIBLE, hidden, self.biases, self.weights, False)
    return visibleReconstruction

  @classmethod
  def initializeWeights(cls, nrVisible, nrHidden):
    return np.random.normal(0, 0.01, (nrVisible, nrHidden))

  @classmethod
  def intializeBiases(cls, data, nrHidden):
    # get the procentage of data points that have the i'th unit on
    # and set the visible vias to log (p/(1-p))
    percentages = data.mean(axis=0, dtype='float') / len(data)
    vectorized = np.vectorize(safeLogFraction)
    visibleBiases = vectorized(percentages)

    # TODO: if sparse hiddeen weights, use that information
    hiddenBiases = np.zeros(nrHidden)
    return np.array([visibleBiases, hiddenBiases])

def safeLogFraction(p):
  assert p >=0 and p <= 1
  # TODO: think about this a bit better
  # you should not set them to be equal, on the contrary, they should be opposites
  if p * (1 - p) == 0:
    return 0
  return math.log(p / (1 -p))

# TODO: add momentum to learning
# TODO: different learning rates for weights and biases

""" Training functions."""

""" Full CD function.
Arguments:
  data: the data to use for traning.
    A numpy ndarray.
  biases:

Returns:
"""
def contrastiveDivergence(data, biases, weights, miniBatch=False):

  N = len(data)
  # Train the first 70 percent of the data with CD1
  endCD1 = math.floor(N / 10 * 7)
  cd1Data = data[0:endCD1]
  biases, weights = contrastiveDivergenceStep(cd1Data, biases, weights, cdSteps=1)

  # Train the next 20 percent with CD3
  endCD3 = math.floor(N / 10 * 2) + endCD1
  cd3Data = data[endCD1:endCD3]
  biases, weights = contrastiveDivergenceStep(cd3Data, biases, weights, cdSteps=3)

  # Train the next 10 percent of data with CD10
  cd5Data = data[endCD3:N]
  biases, weights = contrastiveDivergenceStep(cd5Data, biases, weights, cdSteps=5)

  return biases, weights

# Makes a step in the contrastiveDivergence algorithm
# online or with mini-bathces?
# you have multiple choices about how to implement this
# It is importaant that the hidden values from the data are binary,
# not probabilities
def contrastiveDivergenceStep(data, biases, weights, cdSteps=1):
  # TODO: do something smarter with the learning
  epsilon = 0.0001
  assert cdSteps >=1
  # Check that it does rows in loops
  for visible in data:
    # TODO: do CDn after some point
    # you can do it by calling the same function with the remaining data
    hidden = updateLayer(Layer.HIDDEN, visible, biases, weights, True)
    visibleReconstruction = updateLayer(Layer.VISIBLE, hidden, biases, weights, False)
    # TODO: consider this
    for i in xrange(cdSteps - 1):
      hiddenReconstruction = updateLayer(Layer.HIDDEN, visibleReconstruction, biases, weights, True)
      weights += epsilon * (np.outer(visible, hidden)
           - np.outer(visibleReconstruction, hiddenReconstruction))

    # Do the last reconstruction from the probabilities
    hiddenReconstruction = updateLayer(Layer.HIDDEN, visibleReconstruction, biases, weights, False)
    weights += epsilon * (np.outer(visible, hidden)
                - np.outer(visibleReconstruction, hiddenReconstruction))

    # Update the visible biases
    biases[0] += epsilon * (visible - visibleReconstruction)
    # Update the hidden biases
    biases[1] += epsilon * (hidden - hiddenReconstruction)
  return biases, weights

""" Updates an entire layer. This procedure can be used both in training
    and in testing.
"""
def updateLayer(layer, otherLayerValues, biases, weightMatrix, binary=False):
  bias = biases[layer]

  # logging.debug("updating layer " + str(layer))
  # logging.debug("with bias" + str(bias))

  # logging.debug("weights" + str(weightMatrix.shape))
  def activation(x):
    if layer == Layer.VISIBLE:
      w =  weightMatrix[x, :]
    else:
      w =  weightMatrix[:, x]

    return sigmoid(bias[x] + np.dot(w, otherLayerValues))
    # w = weightVectorForNeuron(layer, weightMatrix, x)
    # return activationProbability(activationSum(w, bias[x], otherLayerValues))

  probs = map(activation, xrange(weightMatrix.shape[layer]))
  probs = np.array(probs)

  if binary:
    # Sample from the distributions
    return sampleAll(probs)

  return probs

def weightVectorForNeuron(layer, weightMatrix, neuronNumber):
  if layer == Layer.VISIBLE:
    return weightMatrix[neuronNumber, :]
  # else layer == Layer.HIDDEN
  return weightMatrix[:, neuronNumber]

# TODO: check if you do it faster with matrix multiplication stuff
# but hinton was adamant about the paralell thing
def activationSum(weights, bias, otherLayerValues):
  # logging.debug("in activationSum")
  # logging.debug(otherLayerValues)
  # logging.debug(weights)

  return bias + np.dot(weights, otherLayerValues)

""" Gets the activation sums for all the units in one layer.
    Assumesthat the dimensions of the weihgt matrix and biases
    are given correctly. It will throw an exception otherwise.
"""

def activationProbability(activationSum):
  return sigmoid(activationSum)

# Another training algorithm. Slower than Contrastive divergence, but
# gives better results. Not used in practice as it is too slow.
def PCD():
  pass

""" general unitily functions"""

def sigmoid(x):
  return 1 / (1 + np.exp(-x));

def sample(p):
  return int(np.random.uniform() < p)

def sampleAll(probs):
  return np.random.uniform(size=probs.shape) < probs

def enum(**enums):
  return type('Enum', (), enums)

# Create an enum for visible and hidden, for
Layer = enum(VISIBLE=0, HIDDEN=1)

"""Main. """
def main():
  X = np.array([[1,1,1,0,0,0], # Training subset
                [1,0,1,0,0,0],
                [1,1,1,0,0,0],
                [0,0,1,1,1,0],
                [0,0,1,1,0,0],
                [0,0,1,1,1,0],

                [0,0,1,1,1,1], # Validation subset
                [1,1,1,0,0,0]])
  rbm = RBM(X, 3, contrastiveDivergence)
  rbm.train()

def rmse(prediction, actual):
  return np.linalg.norm(prediction - actual) / np.sqrt(len(prediction))

if __name__ == '__main__':
  main()<|endoftext|>


<commit_before>
##############################################################################
class attribute (property):
    _attvals_name = "__attribute_values__"
    makeclass = False

    #-------------------------------------------------------------------------
    class __metaclass__ (type):
        def __new__(metaclass, name, bases, dictionary, makeclass=False):
            # return the attribute class
            if bases == (property,) or dictionary.get("makeclass") or makeclass:
                return type.__new__(metaclass, name, bases, dictionary)
            # return a attribute instance
            return metaclass(name, bases, dictionary, makeclass=True)(
              name, default=dictionary.get('default'),
              doc=dictionary.get('__doc__'))

    #-------------------------------------------------------------------------
    def __init__(self, name, default=None, doc=None):
        self.name = name
        self.default = default
        property.__init__(self,
          fget=self.get, fset=self.set, fdel=self.delete, doc=doc)

    #-------------------------------------------------------------------------
    def super(self): return super(self.__class__.__bases__[0], self)

    #-------------------------------------------------------------------------
    def isinitialized(self, host):
        return self._get_attvals(host).has_key(self.name)

    #-------------------------------------------------------------------------
    def _get_attvals(self, host):
        if not hasattr(host, self._attvals_name):
            setattr(host, self._attvals_name, {})
        return getattr(host, self._attvals_name)

    #-------------------------------------------------------------------------
    def get(self, host):
        attvals = self._get_attvals(host)
        if not attvals.has_key(self.name):
            if self.default is not None: attvals[self.name] = self.default
            else: raise AttributeError(self.name)
        return attvals[self.name]

    #-------------------------------------------------------------------------
    def set(self, host, value): self._get_attvals(host)[self.name] = value

    #-------------------------------------------------------------------------
    def delete(self, host):
        attvals = self._get_attvals(host)
        if attvals.has_key(self.name): del attvals[self.name]

class readonly (attribute):
    "A attribute which is read-only."
    makeclass = True
    def set(self, host, value):
        raise AttributeError("attribute %s is read only"%self.name)

class setonce (attribute):
    makeclass = True
    def set(self, host, value):
        if self.isinitialized(host):
            raise AttributeError(
              "setonce attribute %s has already been set"%self.name)
        else: self.super().set(host, value)

def _test():
    class Foo (object):
        class x (attribute):
            "test attribute x"; default=11
            def get(self, host):
                print "Customised getter: getting",self.name,"from",host
                return attribute.get(self, host)

        class y (setonce): "test attribute y";

    f = Foo()
    print f.x
    f.x = 10
    print f.x
    f.y = "fnorb"
    print f.y
    f.y = "fnarb"
    print f.y

if __name__ == "__main__": _test()

<commit_msg>added simple type and protocol checking<commit_after>import sets

class InterfaceOmission (Exception): pass

##############################################################################
class attribute (property):
    _attvals_name = "__attribute_values__"
    makeclass = False
    valtype = None
    implements = ()

    #-------------------------------------------------------------------------
    class __metaclass__ (type):
        def __new__(metaclass, classname, bases, classdict, makeclass=False):
            # return the attribute class
            if bases == (property,) or classdict.get("makeclass") or makeclass:
                return type.__new__(metaclass, classname, bases, classdict)
            # return a attribute instance
            return metaclass(classname, bases, classdict, makeclass=True)(
              classname,
              valtype=classdict.get("valtype"),
              default=classdict.get("default"),
              doc=classdict.get("__doc__"))

    #-------------------------------------------------------------------------
    def __init__(self, name, valtype=None, default=None, doc=None):
        self.name = name
        self.valtype = valtype
        self.default = default
        if self.valtype is None and default is not None:
            self.valtype = type(default)
        property.__init__(self,
          fget=self.get, fset=self.set, fdel=self.delete, doc=doc)

    #-------------------------------------------------------------------------
    def super(self, base=None):
        return super(base or self.__class__.__bases__[0], self)

    #-------------------------------------------------------------------------
    def isinitialized(self, host):
        return self._get_attvals(host).has_key(self.name)

    #-------------------------------------------------------------------------
    def _get_attvals(self, host):
        if not hasattr(host, self._attvals_name):
            setattr(host, self._attvals_name, {})
        return getattr(host, self._attvals_name)

    #-------------------------------------------------------------------------
    def validate(self, host, value):
        # type check
        if self.valtype is not None and \
          not issubclass(type(value), self.valtype):
            raise TypeError,\
              "attribute %s value %s must have type %s"% \
              (self.name,`value`,self.valtype)

        # protocol check
        defined = sets.Set(dir(value))
        for protocol in self.implements:
            required = sets.Set(dir(protocol))
            if not required.issubset(defined):
                raise InterfaceOmission, list(required - defined)

    #-------------------------------------------------------------------------
    def isvalid(self, host, value):
        try:
            self.validate(host, value)
            return True
        except TypeError, InterfaceOmission:
            return False

    #-------------------------------------------------------------------------
    def get(self, host):
        attvals = self._get_attvals(host)
        if not attvals.has_key(self.name):
            if self.default is not None:
                self.set(host, self.default)
            else: raise AttributeError(self.name)
        return attvals[self.name]

    #-------------------------------------------------------------------------
    def set(self, host, value):
        self.validate(host, value)
        self._get_attvals(host)[self.name] = value

    #-------------------------------------------------------------------------
    def delete(self, host):
        attvals = self._get_attvals(host)
        if attvals.has_key(self.name): del attvals[self.name]

class readonly (attribute):
    "A attribute which is read-only."
    makeclass = True
    def set(self, host, value):
        raise AttributeError("attribute %s is read only"%self.name)

class setonce (attribute):
    makeclass = True
    def set(self, host, value):
        if self.isinitialized(host):
            raise AttributeError(
              "setonce attribute %s has already been set"%self.name)
        else: self.super().set(host, value)

def _test():
    class Foo (object):
        class x (attribute):
            "test attribute x"; valtype=str; default=11
            def get(self, host):
                print "Customised getter: getting",self.name,"from",host
                return attribute.get(self, host)

        class y (setonce): "test attribute y";

        class z (readonly): default=10

    f = Foo()
    print f.x
    f.x = "borp"
    print f.x
    f.y = "fnorb"
    print f.y
    f.y = "fnarb"
    print f.y

if __name__ == "__main__": _test()

<|endoftext|>


<commit_before># Tweepy
# Copyright 2009-2021 Joshua Roesslein
# See LICENSE for details.

import asyncio
import json
import logging
from math import inf
from platform import python_version

import aiohttp
from oauthlib.oauth1 import Client as OAuthClient
from yarl import URL

import tweepy
from tweepy.errors import TweepyException
from tweepy.models import Status

log = logging.getLogger(__name__)

class AsyncStream:
    """Stream realtime Tweets asynchronously

    Parameters
    ----------
    consumer_key: str
        Twitter API Consumer Key
    consumer_secret: str
        Twitter API Consumer Secret
    access_token: str
        Twitter API Access Token
    access_token_secret: str
        Twitter API Access Token Secret
    max_retries: Optional[int]
        Number of times to attempt to (re)connect the stream.
    proxy: Optional[str]
        Proxy URL

    Attributes
    ----------
    session : Optional[aiohttp.ClientSession]
        Aiohttp client session used to connect to the API
    task : Optional[asyncio.Task]
        The task running the stream
    user_agent : str
        User agent used when connecting to the API
    """

    def __init__(self, consumer_key, consumer_secret, access_token,
                 access_token_secret, max_retries=inf, proxy=None):
        self.consumer_key = consumer_key
        self.consumer_secret = consumer_secret
        self.access_token = access_token
        self.access_token_secret = access_token_secret
        self.max_retries = max_retries
        self.proxy = proxy

        self.session = None
        self.task = None
        self.user_agent = (
            f"Python/{python_version()} "
            f"aiohttp/{aiohttp.__version__} "
            f"Tweepy/{tweepy.__version__}"
        )

    async def _connect(self, method, endpoint, params={}, headers=None,
                       body=None):
        error_count = 0
        # https://developer.twitter.com/en/docs/twitter-api/v1/tweets/filter-realtime/guides/connecting
        stall_timeout = 90
        network_error_wait = network_error_wait_step = 0.25
        network_error_wait_max = 16
        http_error_wait = http_error_wait_start = 5
        http_error_wait_max = 320
        http_420_error_wait_start = 60

        oauth_client = OAuthClient(self.consumer_key, self.consumer_secret,
                                   self.access_token, self.access_token_secret)

        if self.session is None or self.session.closed:
            self.session = aiohttp.ClientSession(
                headers={"User-Agent": self.user_agent},
                timeout=aiohttp.ClientTimeout(sock_read=stall_timeout)
            )

        url = f"https://stream.twitter.com/1.1/{endpoint}.json"
        url = str(URL(url).with_query(sorted(params.items())))

        try:
            while error_count <= self.max_retries:
                request_url, request_headers, request_body = oauth_client.sign(
                    url, method, body, headers
                )
                try:
                    async with self.session.request(
                        method, request_url, headers=request_headers,
                        data=request_body, proxy=self.proxy
                    ) as resp:
                        if resp.status == 200:
                            error_count = 0
                            http_error_wait = http_error_wait_start
                            network_error_wait = network_error_wait_step

                            await self.on_connect()

                            async for line in resp.content:
                                line = line.strip()
                                if line:
                                    await self.on_data(line)
                                else:
                                    await self.on_keep_alive()

                            await self.on_closed(resp)
                        else:
                            await self.on_request_error(resp.status)

                            error_count += 1

                            if resp.status == 420:
                                if http_error_wait < http_420_error_wait_start:
                                    http_error_wait = http_420_error_wait_start

                            await asyncio.sleep(http_error_wait)

                            http_error_wait *= 2
                            if resp.status != 420:
                                if http_error_wait > http_error_wait_max:
                                    http_error_wait = http_error_wait_max
                except (aiohttp.ClientConnectionError,
                        aiohttp.ClientPayloadError) as e:
                    await self.on_connection_error()

                    await asyncio.sleep(network_error_wait)

                    network_error_wait += network_error_wait_step
                    if network_error_wait > network_error_wait_max:
                        network_error_wait = network_error_wait_max
        except asyncio.CancelledError:
            return
        except Exception as e:
            await self.on_exception(e)
        finally:
            await self.session.close()
            await self.on_disconnect()

    async def filter(self, follow=None, track=None, locations=None,
                     stall_warnings=False):
        """|coroutine|

        Filter realtime Tweets

        Parameters
        ----------
        follow: Optional[List[Union[int, str]]]
            A list of user IDs, indicating the users to return statuses for in
            the stream. See https://developer.twitter.com/en/docs/twitter-api/v1/tweets/filter-realtime/guides/basic-stream-parameters
            for more information.
        track: Optional[List[str]]
            Keywords to track. Phrases of keywords are specified by a list. See
            https://developer.twitter.com/en/docs/tweets/filter-realtime/guides/basic-stream-parameters
            for more information.
        locations: Optional[List[float]]
            Specifies a set of bounding boxes to track. See
            https://developer.twitter.com/en/docs/tweets/filter-realtime/guides/basic-stream-parameters
            for more information.
        stall_warnings: Optional[bool]
            Specifies whether stall warnings should be delivered. See
            https://developer.twitter.com/en/docs/tweets/filter-realtime/guides/basic-stream-parameters
            for more information.

        Returns
        -------
        asyncio.Task
            The task running the stream

        References
        ----------
        https://developer.twitter.com/en/docs/twitter-api/v1/tweets/filter-realtime/api-reference/post-statuses-filter
        """
        if self.task is not None and not self.task.done():
            raise TweepyException("Stream is already connected")

        endpoint = "statuses/filter"
        headers = {"Content-Type": "application/x-www-form-urlencoded"}

        body = {}
        if follow is not None:
            body["follow"] = ','.join(map(str, follow))
        if track is not None:
            body["track"] = ','.join(map(str, track))
        if locations is not None:
            if len(locations) % 4:
                raise TweepyException(
                    "Number of location coordinates should be a multiple of 4"
                )
            body["locations"] = ','.join(
                f"{location:.4f}" for location in locations
            )
        if stall_warnings:
            body["stall_warnings"] = "true"

        self.task = asyncio.create_task(
            self._connect("POST", endpoint, headers=headers, body=body or None)
        )
        return self.task

    async def sample(self, stall_warnings=False):
        """|coroutine|

        Sample realtime Tweets

        Parameters
        ----------
        stall_warnings: Optional[bool]
            Specifies whether stall warnings should be delivered. See
            https://developer.twitter.com/en/docs/tweets/filter-realtime/guides/basic-stream-parameters
            for more information.

        Returns
        -------
        asyncio.Task
            The task running the stream

        References
        ----------
        https://developer.twitter.com/en/docs/twitter-api/v1/tweets/sample-realtime/api-reference/get-statuses-sample
        """
        if self.task is not None and not self.task.done():
            raise TweepyException("Stream is already connected")

        endpoint = "statuses/sample"

        params = {}
        if stall_warnings:
            params["stall_warnings"] = "true"

        self.task = asyncio.create_task(
            self._connect("GET", endpoint, params=params)
        )
        return self.task

    def disconnect(self):
        """Disconnect the stream"""
        if self.task is not None:
            self.task.cancel()

    async def on_closed(self, resp):
        """|coroutine|

        This is called when the stream has been closed by Twitter.
        """
        log.error("Stream connection closed by Twitter")

    async def on_connect(self):
        """|coroutine|

        This is called after successfully connecting to the streaming API.
        """
        log.info("Stream connected")

    async def on_connection_error(self):
        """|coroutine|

        This is called when the stream connection errors or times out.
        """
        log.error("Stream connection has errored or timed out")

    async def on_disconnect(self):
        """|coroutine|

        This is called when the stream has disconnected.
        """
        log.info("Stream disconnected")

    async def on_exception(self, exception):
        """|coroutine|

        This is called when an unhandled exception occurs.
        """
        log.exception("Stream encountered an exception")

    async def on_keep_alive(self):
        """|coroutine|

        This is called when a keep-alive signal is received.
        """
        log.debug("Received keep-alive signal")

    async def on_request_error(self, status_code):
        """|coroutine|

        This is called when a non-200 HTTP status code is encountered.
        """
        log.error("Stream encountered HTTP Error: %d", status_code)

    async def on_data(self, raw_data):
        """|coroutine|

        This is called when raw data is received from the stream.
        This method handles sending the data to other methods, depending on the
        message type.

        References
        ----------
        https://developer.twitter.com/en/docs/twitter-api/v1/tweets/filter-realtime/guides/streaming-message-types
        """
        data = json.loads(raw_data)

        if "in_reply_to_status_id" in data:
            status = Status.parse(None, data)
            return await self.on_status(status)
        if "delete" in data:
            delete = data["delete"]["status"]
            return await self.on_delete(delete["id"], delete["user_id"])
        if "disconnect" in data:
            return await self.on_disconnect_message(data["disconnect"])
        if "limit" in data:
            return await self.on_limit(data["limit"]["track"])
        if "scrub_geo" in data:
            return await self.on_scrub_geo(data["scrub_geo"])
        if "status_withheld" in data:
            return await self.on_status_withheld(data["status_withheld"])
        if "user_withheld" in data:
            return await self.on_user_withheld(data["user_withheld"])
        if "warning" in data:
            return await self.on_warning(data["warning"])

        log.warning("Received unknown message type: %s", raw_data)

    async def on_status(self, status):
        """|coroutine|

        This is called when a status is received.
        """
        log.debug("Received status: %d", status.id)

    async def on_delete(self, status_id, user_id):
        """|coroutine|

        This is called when a status deletion notice is received.
        """
        log.debug("Received status deletion notice: %d", status_id)

    async def on_disconnect_message(self, message):
        """|coroutine|

        This is called when a disconnect message is received.
        """
        log.warning("Received disconnect message: %s", message)

    async def on_limit(self, track):
        """|coroutine|

        This is called when a limit notice is received.
        """
        log.debug("Received limit notice: %d", track)

    async def on_scrub_geo(self, notice):
        """|coroutine|

        This is called when a location deletion notice is received.
        """
        log.debug("Received location deletion notice: %s", notice)

    async def on_status_withheld(self, notice):
        """|coroutine|

        This is called when a status withheld content notice is received.
        """
        log.debug("Received status withheld content notice: %s", notice)

    async def on_user_withheld(self, notice):
        """|coroutine|

        This is called when a user withheld content notice is received.
        """
        log.debug("Received user withheld content notice: %s", notice)

    async def on_warning(self, notice):
        """|coroutine|

        This is called when a stall warning message is received.
        """
        log.warning("Received stall warning: %s", notice)
<commit_msg>Add Parameters and Raises to documentation for AsyncStream<commit_after># Tweepy
# Copyright 2009-2021 Joshua Roesslein
# See LICENSE for details.

import asyncio
import json
import logging
from math import inf
from platform import python_version

import aiohttp
from oauthlib.oauth1 import Client as OAuthClient
from yarl import URL

import tweepy
from tweepy.errors import TweepyException
from tweepy.models import Status

log = logging.getLogger(__name__)

class AsyncStream:
    """Stream realtime Tweets asynchronously

    Parameters
    ----------
    consumer_key: str
        Twitter API Consumer Key
    consumer_secret: str
        Twitter API Consumer Secret
    access_token: str
        Twitter API Access Token
    access_token_secret: str
        Twitter API Access Token Secret
    max_retries: Optional[int]
        Number of times to attempt to (re)connect the stream.
    proxy: Optional[str]
        Proxy URL

    Attributes
    ----------
    session : Optional[aiohttp.ClientSession]
        Aiohttp client session used to connect to the API
    task : Optional[asyncio.Task]
        The task running the stream
    user_agent : str
        User agent used when connecting to the API
    """

    def __init__(self, consumer_key, consumer_secret, access_token,
                 access_token_secret, max_retries=inf, proxy=None):
        self.consumer_key = consumer_key
        self.consumer_secret = consumer_secret
        self.access_token = access_token
        self.access_token_secret = access_token_secret
        self.max_retries = max_retries
        self.proxy = proxy

        self.session = None
        self.task = None
        self.user_agent = (
            f"Python/{python_version()} "
            f"aiohttp/{aiohttp.__version__} "
            f"Tweepy/{tweepy.__version__}"
        )

    async def _connect(self, method, endpoint, params={}, headers=None,
                       body=None):
        error_count = 0
        # https://developer.twitter.com/en/docs/twitter-api/v1/tweets/filter-realtime/guides/connecting
        stall_timeout = 90
        network_error_wait = network_error_wait_step = 0.25
        network_error_wait_max = 16
        http_error_wait = http_error_wait_start = 5
        http_error_wait_max = 320
        http_420_error_wait_start = 60

        oauth_client = OAuthClient(self.consumer_key, self.consumer_secret,
                                   self.access_token, self.access_token_secret)

        if self.session is None or self.session.closed:
            self.session = aiohttp.ClientSession(
                headers={"User-Agent": self.user_agent},
                timeout=aiohttp.ClientTimeout(sock_read=stall_timeout)
            )

        url = f"https://stream.twitter.com/1.1/{endpoint}.json"
        url = str(URL(url).with_query(sorted(params.items())))

        try:
            while error_count <= self.max_retries:
                request_url, request_headers, request_body = oauth_client.sign(
                    url, method, body, headers
                )
                try:
                    async with self.session.request(
                        method, request_url, headers=request_headers,
                        data=request_body, proxy=self.proxy
                    ) as resp:
                        if resp.status == 200:
                            error_count = 0
                            http_error_wait = http_error_wait_start
                            network_error_wait = network_error_wait_step

                            await self.on_connect()

                            async for line in resp.content:
                                line = line.strip()
                                if line:
                                    await self.on_data(line)
                                else:
                                    await self.on_keep_alive()

                            await self.on_closed(resp)
                        else:
                            await self.on_request_error(resp.status)

                            error_count += 1

                            if resp.status == 420:
                                if http_error_wait < http_420_error_wait_start:
                                    http_error_wait = http_420_error_wait_start

                            await asyncio.sleep(http_error_wait)

                            http_error_wait *= 2
                            if resp.status != 420:
                                if http_error_wait > http_error_wait_max:
                                    http_error_wait = http_error_wait_max
                except (aiohttp.ClientConnectionError,
                        aiohttp.ClientPayloadError) as e:
                    await self.on_connection_error()

                    await asyncio.sleep(network_error_wait)

                    network_error_wait += network_error_wait_step
                    if network_error_wait > network_error_wait_max:
                        network_error_wait = network_error_wait_max
        except asyncio.CancelledError:
            return
        except Exception as e:
            await self.on_exception(e)
        finally:
            await self.session.close()
            await self.on_disconnect()

    async def filter(self, follow=None, track=None, locations=None,
                     stall_warnings=False):
        """|coroutine|

        Filter realtime Tweets

        Parameters
        ----------
        follow: Optional[List[Union[int, str]]]
            A list of user IDs, indicating the users to return statuses for in
            the stream. See https://developer.twitter.com/en/docs/twitter-api/v1/tweets/filter-realtime/guides/basic-stream-parameters
            for more information.
        track: Optional[List[str]]
            Keywords to track. Phrases of keywords are specified by a list. See
            https://developer.twitter.com/en/docs/tweets/filter-realtime/guides/basic-stream-parameters
            for more information.
        locations: Optional[List[float]]
            Specifies a set of bounding boxes to track. See
            https://developer.twitter.com/en/docs/tweets/filter-realtime/guides/basic-stream-parameters
            for more information.
        stall_warnings: Optional[bool]
            Specifies whether stall warnings should be delivered. See
            https://developer.twitter.com/en/docs/tweets/filter-realtime/guides/basic-stream-parameters
            for more information.

        Raises
        ------
        TweepyException
            When the stream is already connected or when the number of location
            coordinates is not a multiple of 4

        Returns
        -------
        asyncio.Task
            The task running the stream

        References
        ----------
        https://developer.twitter.com/en/docs/twitter-api/v1/tweets/filter-realtime/api-reference/post-statuses-filter
        """
        if self.task is not None and not self.task.done():
            raise TweepyException("Stream is already connected")

        endpoint = "statuses/filter"
        headers = {"Content-Type": "application/x-www-form-urlencoded"}

        body = {}
        if follow is not None:
            body["follow"] = ','.join(map(str, follow))
        if track is not None:
            body["track"] = ','.join(map(str, track))
        if locations is not None:
            if len(locations) % 4:
                raise TweepyException(
                    "Number of location coordinates should be a multiple of 4"
                )
            body["locations"] = ','.join(
                f"{location:.4f}" for location in locations
            )
        if stall_warnings:
            body["stall_warnings"] = "true"

        self.task = asyncio.create_task(
            self._connect("POST", endpoint, headers=headers, body=body or None)
        )
        return self.task

    async def sample(self, stall_warnings=False):
        """|coroutine|

        Sample realtime Tweets

        Parameters
        ----------
        stall_warnings: Optional[bool]
            Specifies whether stall warnings should be delivered. See
            https://developer.twitter.com/en/docs/tweets/filter-realtime/guides/basic-stream-parameters
            for more information.

        Raises
        ------
        TweepyException
            When the stream is already connected

        Returns
        -------
        asyncio.Task
            The task running the stream

        References
        ----------
        https://developer.twitter.com/en/docs/twitter-api/v1/tweets/sample-realtime/api-reference/get-statuses-sample
        """
        if self.task is not None and not self.task.done():
            raise TweepyException("Stream is already connected")

        endpoint = "statuses/sample"

        params = {}
        if stall_warnings:
            params["stall_warnings"] = "true"

        self.task = asyncio.create_task(
            self._connect("GET", endpoint, params=params)
        )
        return self.task

    def disconnect(self):
        """Disconnect the stream"""
        if self.task is not None:
            self.task.cancel()

    async def on_closed(self, resp):
        """|coroutine|

        This is called when the stream has been closed by Twitter.

        Parameters
        ----------
        response : aiohttp.ClientResponse
            The response from Twitter
        """
        log.error("Stream connection closed by Twitter")

    async def on_connect(self):
        """|coroutine|

        This is called after successfully connecting to the streaming API.
        """
        log.info("Stream connected")

    async def on_connection_error(self):
        """|coroutine|

        This is called when the stream connection errors or times out.
        """
        log.error("Stream connection has errored or timed out")

    async def on_disconnect(self):
        """|coroutine|

        This is called when the stream has disconnected.
        """
        log.info("Stream disconnected")

    async def on_exception(self, exception):
        """|coroutine|

        This is called when an unhandled exception occurs.

        Parameters
        ----------
        exception : Exception
            The unhandled exception
        """
        log.exception("Stream encountered an exception")

    async def on_keep_alive(self):
        """|coroutine|

        This is called when a keep-alive signal is received.
        """
        log.debug("Received keep-alive signal")

    async def on_request_error(self, status_code):
        """|coroutine|

        This is called when a non-200 HTTP status code is encountered.

        Parameters
        ----------
        status_code : int
            The HTTP status code encountered
        """
        log.error("Stream encountered HTTP Error: %d", status_code)

    async def on_data(self, raw_data):
        """|coroutine|

        This is called when raw data is received from the stream.
        This method handles sending the data to other methods, depending on the
        message type.

        Parameters
        ----------
        raw_data : JSON
            The raw data from the stream

        References
        ----------
        https://developer.twitter.com/en/docs/twitter-api/v1/tweets/filter-realtime/guides/streaming-message-types
        """
        data = json.loads(raw_data)

        if "in_reply_to_status_id" in data:
            status = Status.parse(None, data)
            return await self.on_status(status)
        if "delete" in data:
            delete = data["delete"]["status"]
            return await self.on_delete(delete["id"], delete["user_id"])
        if "disconnect" in data:
            return await self.on_disconnect_message(data["disconnect"])
        if "limit" in data:
            return await self.on_limit(data["limit"]["track"])
        if "scrub_geo" in data:
            return await self.on_scrub_geo(data["scrub_geo"])
        if "status_withheld" in data:
            return await self.on_status_withheld(data["status_withheld"])
        if "user_withheld" in data:
            return await self.on_user_withheld(data["user_withheld"])
        if "warning" in data:
            return await self.on_warning(data["warning"])

        log.warning("Received unknown message type: %s", raw_data)

    async def on_status(self, status):
        """|coroutine|

        This is called when a status is received.

        Parameters
        ----------
        status : Status
            The Status received
        """
        log.debug("Received status: %d", status.id)

    async def on_delete(self, status_id, user_id):
        """|coroutine|

        This is called when a status deletion notice is received.

        Parameters
        ----------
        status_id : int
            The ID of the deleted Tweet
        user_id : int
            The ID of the author of the Tweet
        """
        log.debug("Received status deletion notice: %d", status_id)

    async def on_disconnect_message(self, message):
        """|coroutine|

        This is called when a disconnect message is received.

        Parameters
        ----------
        message : JSON
            The disconnect message
        """
        log.warning("Received disconnect message: %s", message)

    async def on_limit(self, track):
        """|coroutine|

        This is called when a limit notice is received.

        Parameters
        ----------
        track : int
            Total count of the number of undelivered Tweets since the
            connection was opened
        """
        log.debug("Received limit notice: %d", track)

    async def on_scrub_geo(self, notice):
        """|coroutine|

        This is called when a location deletion notice is received.

        Parameters
        ----------
        notice : JSON
            The location deletion notice
        """
        log.debug("Received location deletion notice: %s", notice)

    async def on_status_withheld(self, notice):
        """|coroutine|

        This is called when a status withheld content notice is received.

        Parameters
        ----------
        notice : JSON
            The status withheld content notice
        """
        log.debug("Received status withheld content notice: %s", notice)

    async def on_user_withheld(self, notice):
        """|coroutine|

        This is called when a user withheld content notice is received.

        Parameters
        ----------
        notice : JSON
            The user withheld content notice
        """
        log.debug("Received user withheld content notice: %s", notice)

    async def on_warning(self, notice):
        """|coroutine|

        This is called when a stall warning message is received.

        Parameters
        ----------
        warning : JSON
            The stall warning
        """
        log.warning("Received stall warning: %s", notice)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Production Configurations

- Use djangosecure
- Use Amazon's S3 for storing static files and uploaded media
- Use mailgun to send emails
- Use Redis on Heroku

- Use sentry for error logging

'''
from __future__ import absolute_import, unicode_literals

from boto.s3.connection import OrdinaryCallingFormat
from django.utils import six

import logging

from .common import *  # noqa

# SECRET CONFIGURATION
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#secret-key
# Raises ImproperlyConfigured exception if DJANGO_SECRET_KEY not in os.environ
SECRET_KEY = env("DJANGO_SECRET_KEY")

# This ensures that Django will be able to detect a secure connection
# properly on Heroku.
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')

# django-secure
# ------------------------------------------------------------------------------
INSTALLED_APPS += ("djangosecure", )
# raven sentry client
# See https://docs.getsentry.com/hosted/clients/python/integrations/django/
INSTALLED_APPS += ('raven.contrib.django.raven_compat', )
SECURITY_MIDDLEWARE = (
    'djangosecure.middleware.SecurityMiddleware',
)
RAVEN_MIDDLEWARE = ('raven.contrib.django.raven_compat.middleware.Sentry404CatchMiddleware',
                    'raven.contrib.django.raven_compat.middleware.SentryResponseErrorIdMiddleware',)
MIDDLEWARE_CLASSES = SECURITY_MIDDLEWARE + \
    RAVEN_MIDDLEWARE + MIDDLEWARE_CLASSES

# set this to 60 seconds and then to 518400 when you can prove it works
SECURE_HSTS_SECONDS = 60
SECURE_HSTS_INCLUDE_SUBDOMAINS = env.bool(
    "DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS", default=True)
SECURE_FRAME_DENY = env.bool("DJANGO_SECURE_FRAME_DENY", default=True)
SECURE_CONTENT_TYPE_NOSNIFF = env.bool(
    "DJANGO_SECURE_CONTENT_TYPE_NOSNIFF", default=True)
SECURE_BROWSER_XSS_FILTER = True
SESSION_COOKIE_SECURE = False
SESSION_COOKIE_HTTPONLY = True
SECURE_SSL_REDIRECT = env.bool("DJANGO_SECURE_SSL_REDIRECT", default=True)

# SITE CONFIGURATION
# ------------------------------------------------------------------------------
# Hosts/domain names that are valid for this site
# See https://docs.djangoproject.com/en/1.6/ref/settings/#allowed-hosts
ALLOWED_HOSTS = env.list('DJANGO_ALLOWED_HOSTS', default=['.eraldo.org'])
# END SITE CONFIGURATION

INSTALLED_APPS += ("gunicorn", )

# STORAGE CONFIGURATION
# ------------------------------------------------------------------------------
# Uploaded Media Files
# ------------------------
# See: http://django-storages.readthedocs.org/en/latest/index.html
INSTALLED_APPS += (
    'storages',
)
DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage'

# Fix for aws s3 europe locations: https://github.com/jschneier/django-storages/issues/28
# os.environ['S3_USE_SIGV4'] = 'True'
AWS_S3_HOST = 's3.eu-central-1.amazonaws.com'

AWS_ACCESS_KEY_ID = env('DJANGO_AWS_ACCESS_KEY_ID')
AWS_SECRET_ACCESS_KEY = env('DJANGO_AWS_SECRET_ACCESS_KEY')
AWS_STORAGE_BUCKET_NAME = env('DJANGO_AWS_STORAGE_BUCKET_NAME')
# AWS_AUTO_CREATE_BUCKET = True
AWS_QUERYSTRING_AUTH = False
AWS_S3_CALLING_FORMAT = OrdinaryCallingFormat()

# AWS cache settings, don't change unless you know what you're doing:
AWS_EXPIRY = 60 * 60 * 24 * 7

# TODO See: https://github.com/jschneier/django-storages/issues/47
# Revert the following and use str after the above-mentioned bug is fixed in
# either django-storage-redux or boto
AWS_HEADERS = {
    'Cache-Control': six.b('max-age=%d, s-maxage=%d, must-revalidate' % (
        AWS_EXPIRY, AWS_EXPIRY))
}

# URL that handles the media served from MEDIA_ROOT, used for managing
# stored files.
MEDIA_URL = 'https://s3.amazonaws.com/%s/' % AWS_STORAGE_BUCKET_NAME

# Static Assets
# ------------------------
STATICFILES_STORAGE = 'whitenoise.django.GzipManifestStaticFilesStorage'

# EMAIL
# ------------------------------------------------------------------------------
DEFAULT_FROM_EMAIL = env('DJANGO_DEFAULT_FROM_EMAIL',
                         default='Eraldo Energy <noreply@eraldo.org>')
EMAIL_BACKEND = 'django_mailgun.MailgunBackend'
MAILGUN_ACCESS_KEY = env('DJANGO_MAILGUN_API_KEY')
MAILGUN_SERVER_NAME = env('DJANGO_MAILGUN_SERVER_NAME')
EMAIL_SUBJECT_PREFIX = env("DJANGO_EMAIL_SUBJECT_PREFIX", default='[Eraldo Energy] ')
SERVER_EMAIL = env('DJANGO_SERVER_EMAIL', default=DEFAULT_FROM_EMAIL)

# TEMPLATE CONFIGURATION
# ------------------------------------------------------------------------------
# See:
# https://docs.djangoproject.com/en/dev/ref/templates/api/#django.template.loaders.cached.Loader
TEMPLATES[0]['OPTIONS']['loaders'] = [
    ('django.template.loaders.cached.Loader', [
        'django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader', ]),
]

# DATABASE CONFIGURATION
# ------------------------------------------------------------------------------
# Raises ImproperlyConfigured exception if DATABASE_URL not in os.environ
DATABASES['default'] = env.db("DATABASE_URL")

# CACHING
# ------------------------------------------------------------------------------
# Heroku URL does not pass the DB number, so we parse it in
CACHES = {
    "default": {
        "BACKEND": "django_redis.cache.RedisCache",
        "LOCATION": "{0}/{1}".format(env.cache_url('REDIS_URL', default="redis://127.0.0.1:6379"), 0),
        "OPTIONS": {
            "CLIENT_CLASS": "django_redis.client.DefaultClient",
            "IGNORE_EXCEPTIONS": True,  # mimics memcache behavior.
                                        # http://niwinz.github.io/django-redis/latest/#_memcached_exceptions_behavior
        }
    }
}

# Sentry Configuration
SENTRY_DSN = env('DJANGO_SENTRY_DSN')
SENTRY_CLIENT = env('DJANGO_SENTRY_CLIENT', default='raven.contrib.django.raven_compat.DjangoClient')
LOGGING = {
    'version': 1,
    'disable_existing_loggers': True,
    'root': {
        'level': 'WARNING',
        'handlers': ['sentry'],
    },
    'formatters': {
        'verbose': {
            'format': '%(levelname)s %(asctime)s %(module)s '
                      '%(process)d %(thread)d %(message)s'
        },
    },
    'handlers': {
        'sentry': {
            'level': 'ERROR',
            'class': 'raven.contrib.django.raven_compat.handlers.SentryHandler',
        },
        'console': {
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
            'formatter': 'verbose'
        }
    },
    'loggers': {
        'django.db.backends': {
            'level': 'ERROR',
            'handlers': ['console'],
            'propagate': False,
        },
        'raven': {
            'level': 'DEBUG',
            'handlers': ['console'],
            'propagate': False,
        },
        'sentry.errors': {
            'level': 'DEBUG',
            'handlers': ['console'],
            'propagate': False,
        },
        'django.security.DisallowedHost': {
            'level': 'ERROR',
            'handlers': ['console', 'sentry'],
            'propagate': False,
        },
    },
}
SENTRY_CELERY_LOGLEVEL = env.int('DJANGO_SENTRY_LOG_LEVEL', logging.INFO)
RAVEN_CONFIG = {
    'CELERY_LOGLEVEL': env.int('DJANGO_SENTRY_LOG_LEVEL', logging.INFO),
    'DSN': SENTRY_DSN
}

# Your production stuff: Below this line define 3rd party library settings
<commit_msg>Testing s3 settings.<commit_after># -*- coding: utf-8 -*-
'''
Production Configurations

- Use djangosecure
- Use Amazon's S3 for storing static files and uploaded media
- Use mailgun to send emails
- Use Redis on Heroku

- Use sentry for error logging

'''
from __future__ import absolute_import, unicode_literals

from boto.s3.connection import OrdinaryCallingFormat
from django.utils import six

import logging

from .common import *  # noqa

# SECRET CONFIGURATION
# ------------------------------------------------------------------------------
# See: https://docs.djangoproject.com/en/dev/ref/settings/#secret-key
# Raises ImproperlyConfigured exception if DJANGO_SECRET_KEY not in os.environ
SECRET_KEY = env("DJANGO_SECRET_KEY")

# This ensures that Django will be able to detect a secure connection
# properly on Heroku.
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')

# django-secure
# ------------------------------------------------------------------------------
INSTALLED_APPS += ("djangosecure", )
# raven sentry client
# See https://docs.getsentry.com/hosted/clients/python/integrations/django/
INSTALLED_APPS += ('raven.contrib.django.raven_compat', )
SECURITY_MIDDLEWARE = (
    'djangosecure.middleware.SecurityMiddleware',
)
RAVEN_MIDDLEWARE = ('raven.contrib.django.raven_compat.middleware.Sentry404CatchMiddleware',
                    'raven.contrib.django.raven_compat.middleware.SentryResponseErrorIdMiddleware',)
MIDDLEWARE_CLASSES = SECURITY_MIDDLEWARE + \
    RAVEN_MIDDLEWARE + MIDDLEWARE_CLASSES

# set this to 60 seconds and then to 518400 when you can prove it works
SECURE_HSTS_SECONDS = 60
SECURE_HSTS_INCLUDE_SUBDOMAINS = env.bool(
    "DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS", default=True)
SECURE_FRAME_DENY = env.bool("DJANGO_SECURE_FRAME_DENY", default=True)
SECURE_CONTENT_TYPE_NOSNIFF = env.bool(
    "DJANGO_SECURE_CONTENT_TYPE_NOSNIFF", default=True)
SECURE_BROWSER_XSS_FILTER = True
SESSION_COOKIE_SECURE = False
SESSION_COOKIE_HTTPONLY = True
SECURE_SSL_REDIRECT = env.bool("DJANGO_SECURE_SSL_REDIRECT", default=True)

# SITE CONFIGURATION
# ------------------------------------------------------------------------------
# Hosts/domain names that are valid for this site
# See https://docs.djangoproject.com/en/1.6/ref/settings/#allowed-hosts
ALLOWED_HOSTS = env.list('DJANGO_ALLOWED_HOSTS', default=['.eraldo.org'])
# END SITE CONFIGURATION

INSTALLED_APPS += ("gunicorn", )

# STORAGE CONFIGURATION
# ------------------------------------------------------------------------------
# Uploaded Media Files
# ------------------------
# See: http://django-storages.readthedocs.org/en/latest/index.html
INSTALLED_APPS += (
    'storages',
)
DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage'

# Fix for aws s3 europe locations: https://github.com/jschneier/django-storages/issues/28
# os.environ['S3_USE_SIGV4'] = 'True'
# AWS_S3_HOST = 's3.eu-central-1.amazonaws.com'

AWS_ACCESS_KEY_ID = env('DJANGO_AWS_ACCESS_KEY_ID')
AWS_SECRET_ACCESS_KEY = env('DJANGO_AWS_SECRET_ACCESS_KEY')
AWS_STORAGE_BUCKET_NAME = env('DJANGO_AWS_STORAGE_BUCKET_NAME')
# AWS_AUTO_CREATE_BUCKET = True
AWS_QUERYSTRING_AUTH = False
AWS_S3_CALLING_FORMAT = OrdinaryCallingFormat()

# AWS cache settings, don't change unless you know what you're doing:
AWS_EXPIRY = 60 * 60 * 24 * 7

# TODO See: https://github.com/jschneier/django-storages/issues/47
# Revert the following and use str after the above-mentioned bug is fixed in
# either django-storage-redux or boto
AWS_HEADERS = {
    'Cache-Control': six.b('max-age=%d, s-maxage=%d, must-revalidate' % (
        AWS_EXPIRY, AWS_EXPIRY))
}

# URL that handles the media served from MEDIA_ROOT, used for managing
# stored files.
MEDIA_URL = 'https://s3.amazonaws.com/%s/' % AWS_STORAGE_BUCKET_NAME

# Static Assets
# ------------------------
STATICFILES_STORAGE = 'whitenoise.django.GzipManifestStaticFilesStorage'

# EMAIL
# ------------------------------------------------------------------------------
DEFAULT_FROM_EMAIL = env('DJANGO_DEFAULT_FROM_EMAIL',
                         default='Eraldo Energy <noreply@eraldo.org>')
EMAIL_BACKEND = 'django_mailgun.MailgunBackend'
MAILGUN_ACCESS_KEY = env('DJANGO_MAILGUN_API_KEY')
MAILGUN_SERVER_NAME = env('DJANGO_MAILGUN_SERVER_NAME')
EMAIL_SUBJECT_PREFIX = env("DJANGO_EMAIL_SUBJECT_PREFIX", default='[Eraldo Energy] ')
SERVER_EMAIL = env('DJANGO_SERVER_EMAIL', default=DEFAULT_FROM_EMAIL)

# TEMPLATE CONFIGURATION
# ------------------------------------------------------------------------------
# See:
# https://docs.djangoproject.com/en/dev/ref/templates/api/#django.template.loaders.cached.Loader
TEMPLATES[0]['OPTIONS']['loaders'] = [
    ('django.template.loaders.cached.Loader', [
        'django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader', ]),
]

# DATABASE CONFIGURATION
# ------------------------------------------------------------------------------
# Raises ImproperlyConfigured exception if DATABASE_URL not in os.environ
DATABASES['default'] = env.db("DATABASE_URL")

# CACHING
# ------------------------------------------------------------------------------
# Heroku URL does not pass the DB number, so we parse it in
CACHES = {
    "default": {
        "BACKEND": "django_redis.cache.RedisCache",
        "LOCATION": "{0}/{1}".format(env.cache_url('REDIS_URL', default="redis://127.0.0.1:6379"), 0),
        "OPTIONS": {
            "CLIENT_CLASS": "django_redis.client.DefaultClient",
            "IGNORE_EXCEPTIONS": True,  # mimics memcache behavior.
                                        # http://niwinz.github.io/django-redis/latest/#_memcached_exceptions_behavior
        }
    }
}

# Sentry Configuration
SENTRY_DSN = env('DJANGO_SENTRY_DSN')
SENTRY_CLIENT = env('DJANGO_SENTRY_CLIENT', default='raven.contrib.django.raven_compat.DjangoClient')
LOGGING = {
    'version': 1,
    'disable_existing_loggers': True,
    'root': {
        'level': 'WARNING',
        'handlers': ['sentry'],
    },
    'formatters': {
        'verbose': {
            'format': '%(levelname)s %(asctime)s %(module)s '
                      '%(process)d %(thread)d %(message)s'
        },
    },
    'handlers': {
        'sentry': {
            'level': 'ERROR',
            'class': 'raven.contrib.django.raven_compat.handlers.SentryHandler',
        },
        'console': {
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
            'formatter': 'verbose'
        }
    },
    'loggers': {
        'django.db.backends': {
            'level': 'ERROR',
            'handlers': ['console'],
            'propagate': False,
        },
        'raven': {
            'level': 'DEBUG',
            'handlers': ['console'],
            'propagate': False,
        },
        'sentry.errors': {
            'level': 'DEBUG',
            'handlers': ['console'],
            'propagate': False,
        },
        'django.security.DisallowedHost': {
            'level': 'ERROR',
            'handlers': ['console', 'sentry'],
            'propagate': False,
        },
    },
}
SENTRY_CELERY_LOGLEVEL = env.int('DJANGO_SENTRY_LOG_LEVEL', logging.INFO)
RAVEN_CONFIG = {
    'CELERY_LOGLEVEL': env.int('DJANGO_SENTRY_LOG_LEVEL', logging.INFO),
    'DSN': SENTRY_DSN
}

# Your production stuff: Below this line define 3rd party library settings
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
PURPOSE:

To return the theoretical Limb Darkening coefficients and/or model
for observations taken with the MOST Satellite given the input
parameters.

This code makes use of the Claret et al. (2013) A&A 552A, 16C
paper, which should be cited if you use this code:

http://adsabs.harvard.edu/abs/2013A%26A...552A..16C

Created on 2015-05-08T12:51:52
"""

from __future__ import division, print_function
import sys
import argparse

try:
    import numpy as np
except ImportError:
    print('You need numpy installed')
    sys.exit(1)

import pandas as pd
import limbDarkening
ld_dir = '/'.join(limbDarkening.__file__.split('/')[:-1])+'/'

__author__ = "Matt Giguere (github: @mattgiguere)"
__license__ = "MIT"
__version__ = '0.0.1'
__maintainer__ = "Matt Giguere"
__email__ = "matthew.giguere@yale.edu"
__status__ = " Development NOT(Prototype or Production)"

class LimbDarkening:
    """
    The main class that creates a limb darkening object.

    :param teff:
        The effective temperature in Kelvin

    :param logg:
        The log of the surface gravity

    :param passband:
        The passband of interest. Acceptable values are
        ['Kp', 'C', 'S1', 'S2', 'S3', 'S4',
         'u', 'v', 'b', 'y',
         'U', 'B', 'V', 'R', 'I',
         'J', 'H', 'K',
         "u'", "g'", "r'", "i'", "z'",
         'J2', 'H2', 'Ks']
    """

    def __init__(self, teff, logg, passband):
        self.fn = ld_dir+'data/Claret2013_PHOENIX_Nonlinear.tsv'

        self.teff = teff
        self.logg = logg
        self.pb = passband
        self.round_values()

    def round_values(self):
        """
        PURPOSE:
        To round the input parameters for lookup in the Claret table.
        """
        self.rteff = np.round(self.teff/250) * 250
        self.rlogg = np.round(self.logg/0.5) * 0.5

    def get_data(self):
        """
        PURPOSE:
        Restore the data from file and return them as a pandas DataFrame.
        """
        return pd.read_csv(self.fn, sep='\s*\s*', engine='python', header=47)[2:]

    def get_ldcs(self):
        """
        PURPOSE:
        This function will extract and return the limb darkening coefficients
        from the Claret table.
        """
        ldcs = np.zeros(4)
        coeffNames = ['a1', 'a2', 'a3', 'a4']
        idx = 0
        lddf = self.get_data()
        for cNm in coeffNames:
            ldcs[idx] = lddf[((lddf['Teff'] == '{:n}'.format(self.rteff)) &
                             (lddf['logg'] == '{:.2f}'.format(self.rlogg)) &
                             (lddf['Filt'] == self.pb))][cNm].values[0]
            idx += 1
        return ldcs

    def claret_model(self, ldcs):

        def limb_darkening_function(theta):
            mu = np.cos(theta)
            limbdark = 1. - \
                ldcs[0] * (1. - mu**0.5) - \
                ldcs[1] * (1. - mu**1.0) - \
                ldcs[2] * (1. - mu**1.5) - \
                ldcs[3] * (1. - mu**2.0)
            return limbdark
        return limb_darkening_function

def claret13_limb_darkening_coeffs(teff, logg, passband):
    """
    PURPOSE:
    To round the input values in order to extract the limb darkening
    coefficients from the Claret et al. (2013) table.

    :param teff:
    The stellar effective temperature in Kelvin

    :param monh:
    The stellar metalicity

    :param logg:
    The log of the stellar surface gravity

    :param turbvel:
    The stellar turbulent velocity

    See the Claret et al. (2013) paper for more details.
    """

    # create an array of acceptable passband values:
    pbs = ['Kp', 'C', 'S1', 'S2', 'S3', 'S4',
           'u', 'v', 'b', 'y',
           'U', 'B', 'V', 'R', 'I',
           'J', 'H', 'K',
           "u'", "g'", "r'", "i'", "z'",
           'J2', 'H2', 'Ks']

    assert teff >= 5000, 'teff should be greater than 3000: {}'.format(teff)
    assert teff <= 10000, 'teff should be less than 50000: {}'.format(teff)
    assert logg >= 3.0, 'logg should be greater than 0.0: {}'.format(logg)
    assert logg <= 5.5, 'logg should be less than 5.0: {}'.format(logg)
    assert passband in pbs, 'that is not an acceptable passband value'

    ldo = LimbDarkening(teff, logg, passband)
    print(ldo.get_ldcs())

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='argparse object.')
    parser.add_argument(
        'teff',
        help='The stellar effective temperature.')
    parser.add_argument(
        'logg',
        help='The log of the stellar surface gravity')
    parser.add_argument(
        'passband',
        help='The filter/passband')

    args = parser.parse_args()

    claret13_limb_darkening_coeffs(int(args.teff),
                                   float(args.logg),
                                   args.passband)
 <commit_msg>add special eps eri case<commit_after>#!/usr/bin/env python

"""
PURPOSE:

To return the theoretical Limb Darkening coefficients and/or model
for observations taken with the MOST Satellite given the input
parameters.

This code makes use of the Claret et al. (2013) A&A 552A, 16C
paper, which should be cited if you use this code:

http://adsabs.harvard.edu/abs/2013A%26A...552A..16C

Created on 2015-05-08T12:51:52
"""

from __future__ import division, print_function
import sys
import argparse

try:
    import numpy as np
except ImportError:
    print('You need numpy installed')
    sys.exit(1)

import pandas as pd
import limbDarkening
ld_dir = '/'.join(limbDarkening.__file__.split('/')[:-1])+'/'

__author__ = "Matt Giguere (github: @mattgiguere)"
__license__ = "MIT"
__version__ = '0.0.1'
__maintainer__ = "Matt Giguere"
__email__ = "matthew.giguere@yale.edu"
__status__ = " Development NOT(Prototype or Production)"

class LimbDarkening:
    """
    The main class that creates a limb darkening object.

    :param teff:
        The effective temperature in Kelvin

    :param logg:
        The log of the surface gravity

    :param passband:
        The passband of interest. Acceptable values are
        ['Kp', 'C', 'S1', 'S2', 'S3', 'S4',
         'u', 'v', 'b', 'y',
         'U', 'B', 'V', 'R', 'I',
         'J', 'H', 'K',
         "u'", "g'", "r'", "i'", "z'",
         'J2', 'H2', 'Ks']
    """

    def __init__(self, teff, logg, passband):
        self.fn = ld_dir+'data/Claret2013_PHOENIX_Nonlinear.tsv'

        self.teff = teff
        self.logg = logg
        self.pb = passband
        self.round_values()

    def round_values(self):
        """
        PURPOSE:
        To round the input parameters for lookup in the Claret table.
        """
        self.rteff = np.round(self.teff/250) * 250
        self.rlogg = np.round(self.logg/0.5) * 0.5

    def get_data(self):
        """
        PURPOSE:
        Restore the data from file and return them as a pandas DataFrame.
        """
        return pd.read_csv(self.fn, sep='\s*\s*', engine='python', header=47)[2:]

    def get_ldcs(self):
        """
        PURPOSE:
        This function will extract and return the limb darkening coefficients
        from the Claret table.
        """
        ldcs = np.zeros(4)
        coeffNames = ['a1', 'a2', 'a3', 'a4']
        idx = 0
        lddf = self.get_data()
        for cNm in coeffNames:
            ldcs[idx] = lddf[((lddf['Teff'] == '{:n}'.format(self.rteff)) &
                             (lddf['logg'] == '{:.2f}'.format(self.rlogg)) &
                             (lddf['Filt'] == self.pb))][cNm].values[0]
            idx += 1
        return ldcs

    def claret_model(self, ldcs):

        def limb_darkening_function(theta):
            mu = np.cos(theta)
            limbdark = 1. - \
                ldcs[0] * (1. - mu**0.5) - \
                ldcs[1] * (1. - mu**1.0) - \
                ldcs[2] * (1. - mu**1.5) - \
                ldcs[3] * (1. - mu**2.0)
            return limbdark
        return limb_darkening_function

    def get_eps_eri_ldcs(self):
        """
        PURPOSE:
        This function returns the 4-parameter limb darkening coefficients for
        Epsilon Eri as observed with MOST. This method was added because of
        problems reading the limb darkening coefficients from file when using
        MPI.
        """

        ldcs = np.array([ 0.78552, -0.88512,  1.60631, -0.60365])
        return ldcs

def claret13_limb_darkening_coeffs(teff, logg, passband):
    """
    PURPOSE:
    To round the input values in order to extract the limb darkening
    coefficients from the Claret et al. (2013) table.

    :param teff:
    The stellar effective temperature in Kelvin

    :param monh:
    The stellar metalicity

    :param logg:
    The log of the stellar surface gravity

    :param turbvel:
    The stellar turbulent velocity

    See the Claret et al. (2013) paper for more details.
    """

    # create an array of acceptable passband values:
    pbs = ['Kp', 'C', 'S1', 'S2', 'S3', 'S4',
           'u', 'v', 'b', 'y',
           'U', 'B', 'V', 'R', 'I',
           'J', 'H', 'K',
           "u'", "g'", "r'", "i'", "z'",
           'J2', 'H2', 'Ks']

    assert teff >= 5000, 'teff should be greater than 3000: {}'.format(teff)
    assert teff <= 10000, 'teff should be less than 50000: {}'.format(teff)
    assert logg >= 3.0, 'logg should be greater than 0.0: {}'.format(logg)
    assert logg <= 5.5, 'logg should be less than 5.0: {}'.format(logg)
    assert passband in pbs, 'that is not an acceptable passband value'

    ldo = LimbDarkening(teff, logg, passband)
    print(ldo.get_ldcs())

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='argparse object.')
    parser.add_argument(
        'teff',
        help='The stellar effective temperature.')
    parser.add_argument(
        'logg',
        help='The log of the stellar surface gravity')
    parser.add_argument(
        'passband',
        help='The filter/passband')

    args = parser.parse_args()

    claret13_limb_darkening_coeffs(int(args.teff),
                                   float(args.logg),
                                   args.passband)
 <|endoftext|>


<commit_before>"""Create certificates table

Revision ID: af4ba256efde
Revises: 0ab46b2f6d8c
Create Date: 2017-05-19 19:36:12.171390

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'af4ba256efde'
down_revision = '0ab46b2f6d8c'
branch_labels = None
depends_on = None

def upgrade():
    op.create_table('certificates',
                    sa.Column('id', sa.Integer(), nullable=False),
                    sa.Column('pem_data', sa.Text(), nullable=False),
                    sa.Column('rsa_private_key_id', sa.Integer(), nullable=True),
                    sa.Column('x509_cn', sa.String(length=64), nullable=True),
                    sa.Column('x509_ou', sa.String(length=32), nullable=True),
                    sa.Column('x509_o', sa.String(length=64), nullable=True),
                    sa.Column('x509_c', sa.String(length=2), nullable=True),
                    sa.Column('x509_st', sa.String(length=128), nullable=True),
                    sa.Column('not_before', sa.DateTime(), nullable=False),
                    sa.Column('not_after', sa.DateTime(), nullable=False),
                    sa.Column('fingerprint', sa.String(length=64), nullable=False),
                    sa.Column('push_topic', sa.String(), nullable=True),
                    sa.Column('discriminator', sa.String(length=20), nullable=True),
                    sa.ForeignKeyConstraint(['rsa_private_key_id'], ['rsa_private_keys.id'], ),
                    sa.PrimaryKeyConstraint('id')
                    )
    op.create_index(op.f('ix_certificates_fingerprint'), 'certificates', ['fingerprint'], unique=True)

def downgrade():
    op.drop_index(op.f('ix_certificates_fingerprint'), table_name='certificates')
    op.drop_table('certificates')
<commit_msg>Temp remove unique constraint on cert fingerprint because CSR's have an empty value<commit_after>"""Create certificates table

Revision ID: af4ba256efde
Revises: 0ab46b2f6d8c
Create Date: 2017-05-19 19:36:12.171390

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'af4ba256efde'
down_revision = '0ab46b2f6d8c'
branch_labels = None
depends_on = None

def upgrade():
    op.create_table('certificates',
                    sa.Column('id', sa.Integer(), nullable=False),
                    sa.Column('pem_data', sa.Text(), nullable=False),
                    sa.Column('rsa_private_key_id', sa.Integer(), nullable=True),
                    sa.Column('x509_cn', sa.String(length=64), nullable=True),
                    sa.Column('x509_ou', sa.String(length=32), nullable=True),
                    sa.Column('x509_o', sa.String(length=64), nullable=True),
                    sa.Column('x509_c', sa.String(length=2), nullable=True),
                    sa.Column('x509_st', sa.String(length=128), nullable=True),
                    sa.Column('not_before', sa.DateTime(), nullable=False),
                    sa.Column('not_after', sa.DateTime(), nullable=False),
                    sa.Column('fingerprint', sa.String(length=64), nullable=False),
                    sa.Column('push_topic', sa.String(), nullable=True),
                    sa.Column('discriminator', sa.String(length=20), nullable=True),
                    sa.ForeignKeyConstraint(['rsa_private_key_id'], ['rsa_private_keys.id'], ),
                    sa.PrimaryKeyConstraint('id')
                    )
    op.create_index(op.f('ix_certificates_fingerprint'), 'certificates', ['fingerprint'], unique=False)

def downgrade():
    op.drop_index(op.f('ix_certificates_fingerprint'), table_name='certificates')
    op.drop_table('certificates')
<|endoftext|>